/
cloudflare.go
145 lines (114 loc) · 3.89 KB
/
cloudflare.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
package utils
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
r2 "github.com/aws/aws-sdk-go-v2/service/s3"
)
var (
// Read the creds from the JSON file
data = ParseCredsFromJSON("./credentials/creds.json")
// Create a new R2 client
r2Client = CreateR2Client(data.AccessKeyID, data.AccessKeySecret, data.AccountID)
ctx = context.TODO()
)
// R2Obj is an object struct for R2 bucket objects
type R2Obj struct {
ChecksumAlgorithm string `json:"checksumAlgorithm"`
Etag string `json:"Etag"`
Key string `json:"Key"`
LastModified string `json:"LastModified"`
Size int64 `json:"Size"`
StorageClass string `json:"StorageClass"`
Metadata string
}
// R2UploadObject upload an object to R2
func R2UploadObject(fileName string, uploadIdentifier string, fileData io.Reader) {
// Upload an object to R2
_, err := r2Client.PutObject(ctx, &r2.PutObjectInput{
Bucket: &data.BucketName,
Key: aws.String(fileName),
Body: fileData,
Metadata: map[string]string{
"uploadedby": uploadIdentifier,
},
})
ErrorHandler(err)
log.Printf("File: %s uploaded", fileName)
// Remove the file from the local filesystem
err = os.Remove(fileName)
ErrorHandler(err)
}
// R2ListObjects List objects in R2 and return a string slice of the file names
func R2ListObjects() []R2Obj {
// List objects in R2
listObjectsOutput, err := r2Client.ListObjectsV2(ctx, &r2.ListObjectsV2Input{
Bucket: &data.BucketName,
FetchOwner: aws.Bool(true),
})
ErrorHandler(err)
// String slice to hold the r2 object information
files := []R2Obj{}
// The logic below maps the JSON to the R2Obj struct and then appends the struct to the slice of the same type
// _ required to ignore the error
for _, object := range listObjectsOutput.Contents {
// Marshal the object to JSON in a pretty format
obj, err := json.MarshalIndent(object, "", "\t")
ErrorHandler(err)
// Create a new R2 object
r2Obj := R2Obj{}
// Unmarshal the JSON into the R2Obj struct
err = json.Unmarshal([]byte(obj), &r2Obj)
ErrorHandler(err)
// Append the object to the files slice
files = append(files, r2Obj)
}
return files
}
// CreateR2Client creates a new R2 client
func CreateR2Client(accessKeyID string, accessKeySecret string, accountID string) *r2.Client {
// Logic from the documentation
r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service string, region string, options ...interface{}) (aws.Endpoint, error) {
// Logic from the documentation
return aws.Endpoint{
URL: fmt.Sprintf("https://%s.r2.cloudflarestorage.com", accountID),
}, nil
})
// Load the default configuration
cfg, err := config.LoadDefaultConfig(ctx,
config.WithEndpointResolverWithOptions(r2Resolver),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret, "")),
config.WithRegion("auto"),
)
ErrorHandler(err)
client := r2.NewFromConfig(cfg)
return client
}
// R2EnrichMetaData enriches the R2 object in the list with the metadata
func R2EnrichMetaData(r2ObjectList []R2Obj) []R2Obj {
// A slice of map of string key-value pairs to hold the metadata
metaDataMap := []map[string]string{}
// Loop through the the R2 object list and get the metadata for each object
for _, r2Obj := range r2ObjectList {
// Call HeadObject to retrieve metadata for the object
metaResp, err := r2Client.HeadObject(ctx, &r2.HeadObjectInput{
Bucket: &data.BucketName,
Key: aws.String(r2Obj.Key),
})
ErrorHandler(err)
// Append the metadata to the metaDataMap slice
metaDataMap = append(metaDataMap, metaResp.Metadata)
}
// Enrich the R2 object list with the metadata
for k, v := range metaDataMap {
r2ObjectList[k].Metadata = v["uploadedby"]
}
sorted := sortStructByTime(r2ObjectList)
return sorted
}