forked from graymeta/stow
/
location.go
192 lines (153 loc) · 5.77 KB
/
location.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
package s3
import (
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aeppert/stow"
"github.com/pkg/errors"
)
// A location contains a client + the configurations used to create the client.
type location struct {
config stow.Config
client *s3.S3
}
// CreateContainer creates a new container, in this case an S3 bucket.
// The bare minimum needed is a container name, but there are many other
// options that can be provided.
func (l *location) CreateContainer(containerName string) (stow.Container, error) {
createBucketParams := &s3.CreateBucketInput{
Bucket: aws.String(containerName), // required
}
_, err := l.client.CreateBucket(createBucketParams)
if err != nil {
return nil, errors.Wrap(err, "CreateContainer, creating the bucket")
}
region, _ := l.config.Config("region")
newContainer := &container{
name: containerName,
client: l.client,
region: region,
}
return newContainer, nil
}
// Containers returns a slice of the Container interface, a cursor, and an error.
// This doesn't seem to exist yet in the API without doing a ton of manual work.
// Get the list of buckets, query every single one to retrieve region info, and finally
// return the list of containers that have a matching region against the client. It's not
// possible to manipulate a container within a region that doesn't match the clients'.
// This is because AWS user credentials can be tied to regions. One solution would be
// to start a new client for every single container where the region matches, this would
// also check the credentials on every new instance... Tabled for later.
func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) {
var params *s3.ListBucketsInput
var containers []stow.Container
// Response returns exported Owner(*s3.Owner) and Bucket(*s3.[]Bucket)
bucketList, err := l.client.ListBuckets(params)
if err != nil {
return nil, "", errors.Wrap(err, "Containers, listing the buckets")
}
// Iterate through the slice of pointers to buckets
for _, bucket := range bucketList.Buckets {
// Retrieve region information.
bucketLocParams := &s3.GetBucketLocationInput{
Bucket: aws.String(*bucket.Name),
}
// Buckets with region 'US Standard' return nothing.
bucketLocResponse, err := l.client.GetBucketLocation(bucketLocParams)
if err != nil {
return nil, "", errors.Wrap(err, "Containers, getting the bucket location")
}
clientRegion, _ := l.config.Config("region")
containerRegion := bucketLocResponse.LocationConstraint
// If containerRegion (* string) is nil, the region is US Standard, which is "us-east-1"
// by default.
if containerRegion == nil {
usStandardRegion := "us-east-1"
containerRegion = &usStandardRegion
}
// Add buckets with 'US Standard' region. The containerRegion, a pointer, will return nil.
// Also add buckets that have the same region as the client, otherwise continue on.
// The second condition ensures that the bucket contains the given prefix.
if *containerRegion != clientRegion || !strings.HasPrefix(*bucket.Name, prefix) {
continue
}
newContainer := &container{
name: *(bucket.Name),
client: l.client,
region: clientRegion,
}
containers = append(containers, newContainer)
}
return containers, "", nil
}
// Close simply satisfies the Location interface. There's nothing that
// needs to be done in order to satisfy the interface.
func (l *location) Close() error {
return nil // nothing to close
}
// Container retrieves a stow.Container based on its name which must be
// exact.
func (l *location) Container(id string) (stow.Container, error) {
params := &s3.GetBucketLocationInput{
Bucket: aws.String(id), // Required
}
_, err := l.client.GetBucketLocation(params)
if err != nil {
// stow needs ErrNotFound to pass the test but amazon returns an opaque error
if strings.Contains(err.Error(), "NoSuchBucket") {
return nil, stow.ErrNotFound
}
return nil, errors.Wrap(err, "Container, getting the bucket location")
}
region, _ := l.config.Config("region")
c := &container{
name: id,
client: l.client,
region: region,
}
return c, nil
}
// RemoveContainer removes a container simply by name.
func (l *location) RemoveContainer(id string) error {
params := &s3.DeleteBucketInput{
Bucket: aws.String(id),
}
_, err := l.client.DeleteBucket(params)
if err != nil {
return errors.Wrap(err, "RemoveContainer, deleting the bucket")
}
return nil
}
// ItemByURL retrieves a stow.Item by parsing the URL, in this
// case an item is an object.
func (l *location) ItemByURL(url *url.URL) (stow.Item, error) {
genericURL := []string{"https://s3-", ".amazonaws.com/"}
// Remove genericURL[0] from URL:
// url = <genericURL[0]><region><genericURL[1]><bucket name><object path>
firstCut := strings.Replace(url.Path, genericURL[0], "", 1)
// find first dot so that we could extract region.
dotIndex := strings.Index(firstCut, ".")
// region of the s3 bucket.
region := firstCut[0:dotIndex]
// Remove <region><genericURL[1]> from
// <region><genericURL[1]><bucket name><object path>
secondCut := strings.Replace(firstCut, region+genericURL[1], "", 1)
// Get the index of the first slash to get the end of the bucket name.
firstSlash := strings.Index(secondCut, "/")
// Grab bucket name
bucketName := secondCut[:firstSlash]
// Everything afterwards pertains to object.
objectPath := secondCut[firstSlash+1:]
// Get the container by bucket name.
cont, err := l.Container(bucketName)
if err != nil {
return nil, errors.Wrapf(err, "ItemByURL, getting container by the bucketname %v", bucketName)
}
// Get the item by object name.
it, err := cont.Item(objectPath)
if err != nil {
return nil, errors.Wrapf(err, "ItemByURL, getting item by object name %v", objectPath)
}
return it, err
}