forked from gruntwork-io/terratest
/
storage.go
219 lines (174 loc) · 6.03 KB
/
storage.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
package gcp
import (
"context"
"fmt"
"io"
"testing"
"cloud.google.com/go/storage"
"github.com/gruntwork-io/terratest/modules/logger"
"google.golang.org/api/iterator"
)
// CreateStorageBucket creates a Google Cloud bucket with the given BucketAttrs. Note that Google Storage bucket names must be globally unique.
func CreateStorageBucket(t *testing.T, projectID string, name string, attr *storage.BucketAttrs) {
err := CreateStorageBucketE(t, projectID, name, attr)
if err != nil {
t.Fatal(err)
}
}
// CreateStorageBucketE creates a Google Cloud bucket with the given BucketAttrs. Note that Google Storage bucket names must be globally unique.
func CreateStorageBucketE(t *testing.T, projectID string, name string, attr *storage.BucketAttrs) error {
logger.Logf(t, "Creating bucket %s", name)
ctx := context.Background()
// Creates a client.
client, err := storage.NewClient(ctx)
if err != nil {
return err
}
// Creates a Bucket instance.
bucket := client.Bucket(name)
// Creates the new bucket.
return bucket.Create(ctx, projectID, attr)
}
// DeleteStorageBucket destroys the Google Storage bucket.
func DeleteStorageBucket(t *testing.T, name string) {
err := DeleteStorageBucketE(t, name)
if err != nil {
t.Fatal(err)
}
}
// DeleteStorageBucketE destroys the S3 bucket in the given region with the given name.
func DeleteStorageBucketE(t *testing.T, name string) error {
logger.Logf(t, "Deleting bucket %s", name)
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return err
}
return client.Bucket(name).Delete(ctx)
}
// ReadBucketObject reads an object from the given Storage Bucket and returns its contents.
func ReadBucketObject(t *testing.T, bucketName string, filePath string) io.Reader {
out, err := ReadBucketObjectE(t, bucketName, filePath)
if err != nil {
t.Fatal(err)
}
return out
}
// ReadBucketObjectE reads an object from the given Storage Bucket and returns its contents.
func ReadBucketObjectE(t *testing.T, bucketName string, filePath string) (io.Reader, error) {
logger.Logf(t, "Reading object from bucket %s using path %s", bucketName, filePath)
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
bucket := client.Bucket(bucketName)
r, err := bucket.Object(filePath).NewReader(ctx)
if err != nil {
return nil, err
}
return r, nil
}
// WriteBucketObject writes an object to the given Storage Bucket and returns its URL.
func WriteBucketObject(t *testing.T, bucketName string, filePath string, body io.Reader, contentType string) string {
out, err := WriteBucketObjectE(t, bucketName, filePath, body, contentType)
if err != nil {
t.Fatal(err)
}
return out
}
// WriteBucketObjectE writes an object to the given Storage Bucket and returns its URL.
func WriteBucketObjectE(t *testing.T, bucketName string, filePath string, body io.Reader, contentType string) (string, error) {
// set a default content type
if contentType == "" {
contentType = "application/octet-stream"
}
logger.Logf(t, "Writing object to bucket %s using path %s and content type %s", bucketName, filePath, contentType)
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return "", err
}
w := client.Bucket(bucketName).Object(filePath).NewWriter(ctx)
w.ContentType = contentType
// Don't set any ACL or cache control properties for now
//w.ACL = []storage.ACLRule{{Entity: storage.AllAuthenticatedUsers, Role: storage.RoleReader}}
// set a default cache control (1 day)
//w.CacheControl = "public, max-age=86400"
if _, err := io.Copy(w, body); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
const publicURL = "https://storage.googleapis.com/%s/%s"
return fmt.Sprintf(publicURL, bucketName, filePath), nil
}
// EmptyStorageBucket removes the contents of a storage bucket with the given name.
func EmptyStorageBucket(t *testing.T, name string) {
err := EmptyStorageBucketE(t, name)
if err != nil {
t.Fatal(err)
}
}
// EmptyStorageBucketE removes the contents of a storage bucket with the given name.
func EmptyStorageBucketE(t *testing.T, name string) error {
logger.Logf(t, "Emptying storage bucket %s", name)
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return err
}
// List all objects in the bucket
//
// TODO - we should really do a bulk delete call here, but I couldn't find
// anything in the SDK.
bucket := client.Bucket(name)
it := bucket.Objects(ctx, nil)
for {
objectAttrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
return err
}
// purge the object
logger.Logf(t, "Deleting storage bucket object %s", objectAttrs.Name)
bucket.Object(objectAttrs.Name).Delete(ctx)
}
return nil
}
// AssertStorageBucketExists checks if the given storage bucket exists and fails the test if it does not.
func AssertStorageBucketExists(t *testing.T, name string) {
err := AssertStorageBucketExistsE(t, name)
if err != nil {
t.Fatal(err)
}
}
// AssertStorageBucketExistsE checks if the given storage bucket exists and returns an error if it does not.
func AssertStorageBucketExistsE(t *testing.T, name string) error {
logger.Logf(t, "Finding bucket %s", name)
ctx := context.Background()
// Creates a client.
client, err := storage.NewClient(ctx)
if err != nil {
return err
}
// Creates a Bucket instance.
bucket := client.Bucket(name)
// TODO - the code below attempts to determine whether the storage bucket
// exists by making a making a number of API calls, then attemping to
// list the contents of the bucket. It was adapted from Google's own integration
// tests and should be improved once the appropriate API call is added.
// For more info see: https://github.com/GoogleCloudPlatform/google-cloud-go/blob/de879f7be552d57556875b8aaa383bce9396cc8c/storage/integration_test.go#L1231
if _, err := bucket.Attrs(ctx); err != nil {
// ErrBucketNotExist
return err
}
it := bucket.Objects(ctx, nil)
if _, err := it.Next(); err == storage.ErrBucketNotExist {
return err
}
return nil
}