forked from graymeta/stow
/
config.go
183 lines (147 loc) · 4.51 KB
/
config.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
package s3
import (
"net/http"
"net/url"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/flyteorg/stow"
"github.com/pkg/errors"
)
// Kind represents the name of the location/storage type.
const Kind = "s3"
var (
authTypeAccessKey = "accesskey"
authTypeIAM = "iam"
)
const (
// ConfigAuthType is an optional argument that defines whether to use an IAM role or access key based auth
ConfigAuthType = "auth_type"
// ConfigAccessKeyID is one key of a pair of AWS credentials.
ConfigAccessKeyID = "access_key_id"
// ConfigSecretKey is one key of a pair of AWS credentials.
ConfigSecretKey = "secret_key"
// ConfigToken is an optional argument which is required when providing
// credentials with temporary access.
ConfigToken = "token"
// ConfigRegion represents the region/availability zone of the session.
ConfigRegion = "region"
// ConfigEndpoint is optional config value for changing s3 endpoint
// used for e.g. minio.io
ConfigEndpoint = "endpoint"
// ConfigDisableSSL is optional config value for disabling SSL support on custom endpoints
// Its default value is "false", to disable SSL set it to "true".
ConfigDisableSSL = "disable_ssl"
// ConfigV2Signing is an optional config value for signing requests with the v2 signature.
// Its default value is "false", to enable set to "true".
// This feature is useful for s3-compatible blob stores -- ie minio.
ConfigV2Signing = "v2_signing"
)
func init() {
validatefn := func(config stow.Config) error {
authType, ok := config.Config(ConfigAuthType)
if !ok || authType == "" {
authType = authTypeAccessKey
}
if !(authType == authTypeAccessKey || authType == authTypeIAM) {
return errors.New("invalid auth_type")
}
if authType == authTypeAccessKey {
_, ok := config.Config(ConfigAccessKeyID)
if !ok {
return errors.New("missing Access Key ID")
}
_, ok = config.Config(ConfigSecretKey)
if !ok {
return errors.New("missing Secret Key")
}
}
return nil
}
makefn := func(config stow.Config) (stow.Location, error) {
authType, ok := config.Config(ConfigAuthType)
if !ok || authType == "" {
authType = authTypeAccessKey
}
if !(authType == authTypeAccessKey || authType == authTypeIAM) {
return nil, errors.New("invalid auth_type")
}
if authType == authTypeAccessKey {
_, ok := config.Config(ConfigAccessKeyID)
if !ok {
return nil, errors.New("missing Access Key ID")
}
_, ok = config.Config(ConfigSecretKey)
if !ok {
return nil, errors.New("missing Secret Key")
}
}
// Create a new client (s3 session)
client, endpoint, err := newS3Client(config, "")
if err != nil {
return nil, err
}
// Create a location with given config and client (s3 session).
loc := &location{
config: config,
client: client,
customEndpoint: endpoint,
}
return loc, nil
}
kindfn := func(u *url.URL) bool {
return u.Scheme == Kind
}
stow.Register(Kind, makefn, kindfn, validatefn)
}
// Attempts to create a session based on the information given.
func newS3Client(config stow.Config, region string) (client *s3.S3, endpoint string, err error) {
authType, _ := config.Config(ConfigAuthType)
accessKeyID, _ := config.Config(ConfigAccessKeyID)
secretKey, _ := config.Config(ConfigSecretKey)
token, _ := config.Config(ConfigToken)
if authType == "" {
authType = authTypeAccessKey
}
awsConfig := aws.NewConfig().
WithHTTPClient(http.DefaultClient).
WithMaxRetries(aws.UseServiceDefaultRetries).
WithLogger(aws.NewDefaultLogger()).
WithLogLevel(aws.LogOff).
WithSleepDelay(time.Sleep)
if region == "" {
region, _ = config.Config(ConfigRegion)
}
if region != "" {
awsConfig.WithRegion(region)
} else {
awsConfig.WithRegion("us-east-1")
}
if authType == authTypeAccessKey {
awsConfig.WithCredentials(credentials.NewStaticCredentials(accessKeyID, secretKey, token))
}
endpoint, ok := config.Config(ConfigEndpoint)
if ok {
awsConfig.WithEndpoint(endpoint).
WithS3ForcePathStyle(true)
}
disableSSL, ok := config.Config(ConfigDisableSSL)
if ok && disableSSL == "true" {
awsConfig.WithDisableSSL(true)
}
sess, err := session.NewSession(awsConfig)
if err != nil {
return nil, "", err
}
if sess == nil {
return nil, "", errors.New("creating the S3 session")
}
s3Client := s3.New(sess)
usev2, ok := config.Config(ConfigV2Signing)
if ok && usev2 == "true" {
setv2Handlers(s3Client)
}
return s3Client, endpoint, nil
}