Skip to content

Commit

Permalink
miniogw: add Object-Expires alias for Object TTL
Browse files Browse the repository at this point in the history
Channel partners who white label Storj have concerns of having the word
"Storj" in the custom metadata key, so we add a generic alias.

Change-Id: I6dd4677ad0513bd568556043db6a60b9f7664fc2
  • Loading branch information
kaloyan-raev committed Feb 7, 2023
1 parent 35b4585 commit 8e8db65
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 19 deletions.
12 changes: 6 additions & 6 deletions docs/s3-compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,8 @@ $ aws --profile storj --endpoint-url https://gateway.storjshare.io s3 cp 6TiB_fi

It's possible to specify TTL for the object by sending the

- `X-Amz-Meta-Storj-Expires` or
- `X-Minio-Meta-Storj-Expires`
- `X-Amz-Meta-Object-Expires` or
- `X-Minio-Meta-Object-Expires`

header (note: S3-compatible clients usually add the `X-Amz-Meta-` /
`X-Minio-Meta-` prefix themselves) with one of the following values:
Expand All @@ -231,19 +231,19 @@ It's also possible to specify `none` for no expiration (or not send the header).
#### AWS CLI example

```console
$ aws s3 --endpoint-url https://gateway.storjshare.io cp file s3://bucket/object --metadata Storj-Expires=+2h
$ aws s3 --endpoint-url https://gateway.storjshare.io cp file s3://bucket/object --metadata Object-Expires=+2h
upload: ./file to s3://bucket/object

# or

$ aws s3 --endpoint-url https://gateway.storjshare.io cp file s3://bucket/object --metadata Storj-Expires=2022-05-19T00:10:55Z
$ aws s3 --endpoint-url https://gateway.storjshare.io cp file s3://bucket/object --metadata Object-Expires=2022-05-19T00:10:55Z
upload: ./file to s3://bucket/object
```

#### Caveats

The value under `X-Amz-Meta-Storj-Expires` has priority over the value under
`X-Minio-Meta-Storj-Expires`.
The value under `X-Amz-Meta-Object-Expires` has priority over the value under
`X-Minio-Meta-Object-Expires`.

### ListBucketsWithAttribution (Gateway-MT only)

Expand Down
37 changes: 24 additions & 13 deletions miniogw/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,21 @@ var (
}

// ErrInvalidTTL indicates that the value under
// X-Amz-Meta-Storj-Expires/X-Minio-Meta-Storj-Expires couldn't be parsed.
// X-Amz-Meta-Object-Expires/X-Minio-Meta-Object-Expires couldn't be parsed.
ErrInvalidTTL = miniogo.ErrorResponse{
Code: "XStorjInvalidTTL",
Message: "The TTL you have specified is invalid.",
StatusCode: http.StatusBadRequest,
}

// objectTTLKeyAliases is the list of all supported object TTL keys in custom metadata.
objectTTLKeyAliases = []string{
"X-Amz-Meta-Object-Expires",
"X-Minio-Meta-Object-Expires",
"X-Amz-Meta-Storj-Expires",
"X-Minio-Meta-Storj-Expires",
}

// ErrSlowDown is a custom error for when a user is exceeding Satellite
// request rate limits. We don't use the built-in `minio.SlowDown{}` error
// because minio doesn't expect this error would be returned by the object
Expand Down Expand Up @@ -1228,22 +1236,25 @@ func minioObjectInfo(bucket, etag string, object *uplink.Object) minio.ObjectInf
}

func parseTTL(userDefined map[string]string) (time.Time, error) {
date, ok := userDefined["X-Amz-Meta-Storj-Expires"]
if !ok {
if date, ok = userDefined["X-Minio-Meta-Storj-Expires"]; !ok {
for _, alias := range objectTTLKeyAliases {
date, ok := userDefined[alias]
if !ok {
continue
}
switch {
case date == "none":
return time.Time{}, nil
case date == "":
return time.Time{}, nil
case strings.HasPrefix(date, "+"):
d, err := time.ParseDuration(date)
return time.Now().Add(d), err
default:
return time.Parse(time.RFC3339, date)
}
}

if date == "none" {
return time.Time{}, nil
} else if date == "" {
return time.Time{}, nil
} else if strings.HasPrefix(date, "+") {
d, err := time.ParseDuration(date)
return time.Now().Add(d), err
}
return time.Parse(time.RFC3339, date)
return time.Time{}, nil
}

// limitResults returns limit restricted to configuredLimit, aligned with paging
Expand Down
4 changes: 4 additions & 0 deletions testsuite/miniogw/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3783,8 +3783,12 @@ func TestObjectTTL(t *testing.T) {
expectedErr: miniogw.ErrInvalidTTL,
},
} {
testObjectTTL(ctx, t, layer, project, bucket, "X-Amz-Meta-Object-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testObjectTTL(ctx, t, layer, project, bucket, "X-Minio-Meta-Object-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testObjectTTL(ctx, t, layer, project, bucket, "X-Amz-Meta-Storj-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testObjectTTL(ctx, t, layer, project, bucket, "X-Minio-Meta-Storj-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testMultipartObjectTTL(ctx, t, layer, project, bucket, "X-Amz-Meta-Object-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testMultipartObjectTTL(ctx, t, layer, project, bucket, "X-Minio-Meta-Object-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testMultipartObjectTTL(ctx, t, layer, project, bucket, "X-Amz-Meta-Storj-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
testMultipartObjectTTL(ctx, t, layer, project, bucket, "X-Minio-Meta-Storj-Expires", tt.ttl, tt.expectedTTL, tt.expectedErr)
}
Expand Down

0 comments on commit 8e8db65

Please sign in to comment.