/
s3_uploader.go
152 lines (121 loc) · 3.83 KB
/
s3_uploader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
package agent
import (
"context"
"fmt"
"net/url"
"os"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/buildkite/agent/v3/api"
"github.com/buildkite/agent/v3/logger"
)
type S3UploaderConfig struct {
// The destination which includes the S3 bucket name and the path.
// For example, s3://my-bucket-name/foo/bar
Destination string
// Whether or not HTTP calls should be debugged
DebugHTTP bool
}
type S3Uploader struct {
// The s3 bucket path set from the destination
BucketPath string
// The s3 bucket name set from the destination
BucketName string
// The s3 client to use
client *s3.S3
// The configuration
conf S3UploaderConfig
// The logger instance to use
logger logger.Logger
}
func NewS3Uploader(l logger.Logger, c S3UploaderConfig) (*S3Uploader, error) {
bucketName, bucketPath := ParseS3Destination(c.Destination)
// Initialize the s3 client, and authenticate it
s3Client, err := NewS3Client(l, bucketName)
if err != nil {
return nil, err
}
return &S3Uploader{
logger: l,
conf: c,
client: s3Client,
BucketName: bucketName,
BucketPath: bucketPath,
}, nil
}
func ParseS3Destination(destination string) (string, string) {
destinationWithNoTrailingSlash := strings.TrimSuffix(destination, "/")
destinationWithNoProtocol := strings.TrimPrefix(destinationWithNoTrailingSlash, "s3://")
parts := strings.Split(destinationWithNoProtocol, "/")
path := strings.Join(parts[1:], "/")
bucket := parts[0]
return bucket, path
}
func (u *S3Uploader) URL(artifact *api.Artifact) string {
baseUrl := "https://" + u.BucketName + ".s3.amazonaws.com"
if os.Getenv("BUILDKITE_S3_ACCESS_URL") != "" {
baseUrl = os.Getenv("BUILDKITE_S3_ACCESS_URL")
}
url, _ := url.Parse(baseUrl)
url.Path += u.artifactPath(artifact)
return url.String()
}
func (u *S3Uploader) Upload(_ context.Context, artifact *api.Artifact) error {
permission, err := u.resolvePermission()
if err != nil {
return err
}
// Create an uploader with the session and default options
uploader := s3manager.NewUploaderWithClient(u.client)
// Open file from filesystem
u.logger.Debug("Reading file \"%s\"", artifact.AbsolutePath)
f, err := os.Open(artifact.AbsolutePath)
if err != nil {
return fmt.Errorf("failed to open file %q (%w)", artifact.AbsolutePath, err)
}
// Upload the file to S3.
u.logger.Debug("Uploading \"%s\" to bucket with permission `%s`", u.artifactPath(artifact), permission)
params := &s3manager.UploadInput{
Bucket: aws.String(u.BucketName),
Key: aws.String(u.artifactPath(artifact)),
ContentType: aws.String(artifact.ContentType),
ACL: aws.String(permission),
Body: f,
}
// if enabled we assign the sse configuration
if u.serverSideEncryptionEnabled() {
params.ServerSideEncryption = aws.String("AES256")
}
_, err = uploader.Upload(params)
return err
}
func (u *S3Uploader) artifactPath(artifact *api.Artifact) string {
parts := []string{u.BucketPath, artifact.Path}
return strings.Join(parts, "/")
}
func (u *S3Uploader) resolvePermission() (string, error) {
permission := "public-read"
if os.Getenv("BUILDKITE_S3_ACL") != "" {
permission = os.Getenv("BUILDKITE_S3_ACL")
} else if os.Getenv("AWS_S3_ACL") != "" {
permission = os.Getenv("AWS_S3_ACL")
}
switch permission {
case "private", "public-read", "public-read-write", "authenticated-read", "bucket-owner-read", "bucket-owner-full-control":
return permission, nil
default:
return "", fmt.Errorf("Invalid S3 ACL value: `%s`", permission)
}
}
// is encryption at rest enabled for artifacts to satisfy basic security requirements
func (u *S3Uploader) serverSideEncryptionEnabled() bool {
sse := os.Getenv("BUILDKITE_S3_SSE_ENABLED")
switch {
case strings.ToLower(sse) == "true":
return true
default:
return false
}
}