/
s3.go
131 lines (108 loc) · 3.5 KB
/
s3.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
package aws
import (
"bytes"
"fmt"
"log"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
const velerosubstr = "managed-velero"
// ReadFromS3Session reads a key from S3 using given AWS context.
func ReadFromS3Session(session *session.Session, inputKey string) ([]byte, error) {
bucket, key, err := ParseS3URL(inputKey)
if err != nil {
return nil, fmt.Errorf("error trying to parse S3 URL: %v", err)
}
downloader := s3manager.NewDownloader(session)
buffer := aws.NewWriteAtBuffer([]byte{})
_, err = downloader.Download(buffer, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return nil, err
}
return buffer.Bytes(), nil
}
// WriteToS3Session writes the given byte array to S3.
func WriteToS3Session(session *session.Session, outputKey string, data []byte) {
bucket, key, err := ParseS3URL(outputKey)
if err != nil {
log.Printf("error trying to parse S3 URL %s: %v", outputKey, err)
return
}
uploader := s3manager.NewUploader(session)
reader := bytes.NewReader(data)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: reader,
})
if err != nil {
log.Printf("Failed to upload to s3 %s", err.Error())
return
}
log.Printf("Uploaded to %s", outputKey)
return
}
// CreateS3URL creates an S3 URL from a bucket and a key string.
func CreateS3URL(bucket string, keys ...string) string {
strippedBucket := strings.Trim(bucket, "/")
strippedKeys := make([]string, len(keys))
for i, key := range keys {
strippedKeys[i] = strings.Trim(key, "/")
}
s3JoinArray := []string{"s3:/", strippedBucket}
s3JoinArray = append(s3JoinArray, strippedKeys...)
return strings.Join(s3JoinArray, "/")
}
// ParseS3URL parses an S3 url into a bucket and key.
func ParseS3URL(s3URL string) (string, string, error) {
parsedURL, err := url.Parse(s3URL)
if err != nil {
return "", "", err
}
return parsedURL.Host, parsedURL.Path, nil
}
// CleanupS3Buckets finds buckets with substring "osde2e-" or "managed-velero",
// older than given duration, then deletes bucket objects and then buckets
func (CcsAwsSession *ccsAwsSession) CleanupS3Buckets(olderthan time.Duration, dryrun bool) error {
err := CcsAwsSession.GetAWSSessions()
if err != nil {
return err
}
result, err := CcsAwsSession.s3.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return err
}
// Setup BatchDeleteIterator to iterate through a list of objects.
batchDeleteClient := s3manager.NewBatchDeleteWithClient(CcsAwsSession.s3)
for _, bucket := range result.Buckets {
if (strings.Contains(*bucket.Name, rolesubstr) || strings.Contains(*bucket.Name, velerosubstr)) && time.Since(*bucket.CreationDate) > olderthan {
fmt.Printf("Bucket will be deleted: %s\n", bucket)
if !dryrun {
iter := s3manager.NewDeleteListIterator(CcsAwsSession.s3, &s3.ListObjectsInput{
Bucket: bucket.Name,
})
if err := batchDeleteClient.Delete(aws.BackgroundContext(), iter); err != nil {
fmt.Printf("error deleting objects from bucket %s, skipping: %s", *bucket.Name, err)
continue
}
fmt.Println("Deleted object(s) from bucket")
if _, err := CcsAwsSession.s3.DeleteBucket(&s3.DeleteBucketInput{
Bucket: bucket.Name,
}); err != nil {
fmt.Printf("error deleting bucket: %s: %s", *bucket.Name, err)
continue
}
fmt.Println("Deleted bucket")
}
}
}
return nil
}