-
Notifications
You must be signed in to change notification settings - Fork 0
/
storage.go
146 lines (125 loc) · 4.13 KB
/
storage.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
package main
import (
"bytes"
"context"
"fmt"
"log"
"net/http"
"os"
"path"
"sort"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/teris-io/shortid"
)
// StorageClient handles Cloudflare R2 storage operations
type StorageClient struct {
client *s3.Client
}
func NewStorageClient() *StorageClient {
r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: fmt.Sprintf("https://%s.r2.cloudflarestorage.com", r2AccountId),
}, nil
})
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithEndpointResolverWithOptions(r2Resolver),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(r2AccessKeyId, r2AccessKeySecret, "")),
)
if err != nil {
log.Fatalf("failed to configure s3 client: %v", err)
}
return &StorageClient{client: s3.NewFromConfig(cfg)}
}
func (sc *StorageClient) UploadFile(filepath string) (string, error) {
objectKey := fmt.Sprintf("%s/%s%s", "shots", shortid.MustGenerate(), path.Ext(filepath))
bytes, err := os.ReadFile(filepath)
if err != nil {
return "", fmt.Errorf("failed to read file for upload: %v", err)
}
uploadLargeObject(sc.client, r2BucketName, objectKey, bytes)
if err != nil {
return "", fmt.Errorf("failed to upload file to R2: %v", err)
}
return fmt.Sprintf("%s/%s", r2BucketDomain, objectKey), nil
}
func (u *StorageClient) DeleteObject(bucketName string, objectKey string) error {
_, err := u.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
log.Printf("Couldn't delete object to %v:%v: %v\n",
bucketName, objectKey, err)
}
log.Println("deleted", objectKey, "from", bucketName)
return err
}
func (u *StorageClient) ListObjects(bucketName string, prefix string) ([]string, error) {
// Get the list of items
resp, err := u.client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
Bucket: aws.String(bucketName),
Prefix: aws.String(prefix),
})
if err != nil {
log.Printf("Unable to list items in bucket %q, %v", bucketName, err)
}
log.Printf("Listed %d items from '%s/%s'\n", len(resp.Contents), bucketName, prefix)
items := []struct {
key string
lastModified *time.Time
}{}
for _, item := range resp.Contents {
items = append(items, struct {
key string
lastModified *time.Time
}{
key: *item.Key,
lastModified: item.LastModified,
})
}
sort.Slice(items, func(i, j int) bool {
return items[i].lastModified.After(*items[j].lastModified)
})
keys := []string{}
for _, v := range items {
keys = append(keys, v.key)
}
return keys, nil
}
// UploadLargeObject uses an upload manager to upload data to an object in a bucket.
// The upload manager breaks large data into parts and uploads the parts concurrently.
func uploadLargeObject(s3Client *s3.Client, bucketName string, objectKey string, largeObject []byte) error {
largeBuffer := bytes.NewReader(largeObject)
contentType := http.DetectContentType(largeObject)
log.Printf("Detected content type for '%s' is '%s'", objectKey, contentType)
// fallback to checking file ext in case we failed to detect content-type
if contentType == "application/octet-stream" {
if strings.HasSuffix(objectKey, "mp4") {
contentType = "video/mp4"
} else if strings.HasSuffix(objectKey, "png") {
contentType = "image/png"
}
log.Printf("Using fallback content type '%s' for file '%s'", contentType, objectKey)
}
var partMiBs int64 = 10
uploader := manager.NewUploader(s3Client, func(u *manager.Uploader) {
u.PartSize = partMiBs * 1024 * 1024
})
_, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: largeBuffer,
ContentType: &contentType,
})
if err != nil {
log.Printf("Couldn't upload large object to %v:%v: %v\n",
bucketName, objectKey, err)
}
return err
}