-
Notifications
You must be signed in to change notification settings - Fork 124
/
generated_json_s3.go
94 lines (80 loc) · 2.81 KB
/
generated_json_s3.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package task
import (
"bytes"
"context"
"fmt"
"io"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/evergreen-ci/evergreen"
"github.com/evergreen-ci/pail"
"github.com/pkg/errors"
)
// generatedJSONS3Storage implements the GeneratedJSONFileStorage interface to
// access generated JSON files stored in S3.
type generatedJSONS3Storage struct {
bucket pail.Bucket
}
// newGeneratedJSONS3Storage sets up access to generated JSON files stored in
// S3. If this returns a non-nil GeneratedJSONFileStorage, callers are expected
// to call Close when they are finished with it.
func newGeneratedJSONS3Storage(ppConf evergreen.ParserProjectS3Config) (*generatedJSONS3Storage, error) {
b, err := pail.NewS3MultiPartBucket(pail.S3Options{
Name: ppConf.Bucket,
Prefix: ppConf.GeneratedJSONPrefix,
Region: endpoints.UsEast1RegionID,
})
if err != nil {
return nil, errors.Wrap(err, "setting up S3 multipart bucket")
}
return &generatedJSONS3Storage{bucket: b}, nil
}
// Find finds the generated JSON files from S3 for the given task.
func (s *generatedJSONS3Storage) Find(ctx context.Context, t *Task) (GeneratedJSONFiles, error) {
it, err := s.bucket.List(ctx, t.Id)
if err != nil {
return nil, errors.Wrapf(err, "getting generated JSON files for task '%s'", t.Id)
}
var files GeneratedJSONFiles
for it.Next(ctx) {
item := it.Item()
file, err := s.downloadFile(ctx, item)
if err != nil {
return nil, errors.Wrapf(err, "downloading file for task '%s'", t.Id)
}
files = append(files, file)
}
if err := it.Err(); err != nil {
return nil, errors.Wrapf(err, "downloading generated JSON files from S3")
}
return files, nil
}
func (s *generatedJSONS3Storage) downloadFile(ctx context.Context, item pail.BucketItem) (string, error) {
r, err := item.Get(ctx)
if err != nil {
return "", errors.Wrapf(err, "downloading generated JSON file '%s'", item.Name())
}
defer r.Close()
b, err := io.ReadAll(r)
if err != nil {
return "", errors.Wrapf(err, "reading generated JSON file '%s'", item.Name())
}
return string(b), nil
}
// Insert inserts all the generated JSON files for the given task and sets the
// task's generated JSON storage method to S3. If the files are already
// persisted, this will no-op.
func (s *generatedJSONS3Storage) Insert(ctx context.Context, t *Task, files GeneratedJSONFiles) error {
if t.GeneratedJSONStorageMethod != "" {
return nil
}
for idx, file := range files {
r := bytes.NewBufferString(file)
if err := s.bucket.Put(ctx, s.bucket.Join(t.Id, fmt.Sprint(idx)), r); err != nil {
return errors.Wrapf(err, "inserting generated JSON file #%d for task '%s'", idx, t.Id)
}
}
if err := t.SetGeneratedJSONStorageMethod(evergreen.ProjectStorageMethodS3); err != nil {
return errors.Wrapf(err, "settings generated JSON storage method to S3 for task '%s'", t.Id)
}
return nil
}