-
Notifications
You must be signed in to change notification settings - Fork 1
/
s3.go
129 lines (111 loc) · 4.13 KB
/
s3.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
package gbdx
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"unicode/utf8"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
// s3Info stores temporary credentials, bucket and customer prefix for
// data stored in AWS S3 by GBDX. The values should be obtained via
// the GBDX s3creds API.
type s3Info struct {
S3SecretKey string `json:"S3_secret_key"`
Prefix string `json:"prefix"`
Bucket string `json:"bucket"`
S3AccessKey string `json:"S3_access_key"`
S3SessionToken string `json:"S3_session_token"`
}
// getS3Info returns an s3Info struct that defines temporary S3
// credentials, customer bucket & customer prefix via the GBDX s3Creds
// api. https://gbdxdocs.digitalglobe.com/docs/s3-storage-service-course
func getS3Info(client *http.Client) (tmpCreds s3Info, err error) {
url := "https://geobigdata.io/s3creds/v1/prefix"
response, err := client.Get(url)
if err != nil {
return tmpCreds, fmt.Errorf("HTTP GET %s: %v", url, err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
var byteSlice []byte
response.Request.Body.Read(byteSlice)
return tmpCreds, fmt.Errorf("HTTP POST %s; returned status %s, expected status %d; request body %q",
url, response.Status, http.StatusOK, byteSlice)
}
if err = json.NewDecoder(response.Body).Decode(&tmpCreds); err != nil {
return tmpCreds, fmt.Errorf("Decoding search response %q: %v", response.Body, err)
}
return tmpCreds, nil
}
// getAwsS3Client returns an S3 client with the credentials stored in s3Info
func getAwsS3Client(tmpCreds s3Info, awsRegion string) *s3.S3 {
creds := credentials.NewStaticCredentials(tmpCreds.S3AccessKey, tmpCreds.S3SecretKey, tmpCreds.S3SessionToken)
// Return an AWS S3 client with the provided credentials
awsSession := session.New(&aws.Config{
Region: &awsRegion,
Credentials: creds,
})
s3Client := s3.New(awsSession)
return s3Client
}
// ListBucket writes a listing of contents of the GBDX customer bucket
// to w.
func (a *Api) ListBucket(requestedPrefix string, recursive bool, w io.Writer) (err error) {
tmpCreds, err := getS3Info(a.client)
if err != nil {
return fmt.Errorf("getS3Info(%v): %v", a.client, err)
}
s3Client := getAwsS3Client(tmpCreds, "us-east-1")
var rootPrefix string
// Note we dont use path.Join as it will gobble up any
// trailing "/"
if utf8.RuneCountInString(requestedPrefix) == 0 {
rootPrefix = fmt.Sprintf("%s/", tmpCreds.Prefix)
} else if strings.Contains(requestedPrefix, tmpCreds.Prefix) {
rootPrefix = requestedPrefix
} else if requestedPrefix[len(requestedPrefix)-1] == '/' {
rootPrefix = fmt.Sprintf("%s/%s", tmpCreds.Prefix, requestedPrefix)
} else {
rootPrefix = fmt.Sprintf("%s/%s/", tmpCreds.Prefix, requestedPrefix)
}
// Delimiter tells s3.ListObjects where to stop fetching. If
// it's "", you get recursive output.
delimiter := ""
if !recursive {
delimiter = "/"
}
inputParams := &s3.ListObjectsInput{
Bucket: &tmpCreds.Bucket,
Delimiter: &delimiter,
Prefix: &rootPrefix,
}
// ListObjects returns a ListObjectsOutput struct
listObjectsOutput, err := s3Client.ListObjects(inputParams)
if err != nil {
return fmt.Errorf("s3Client.ListObjects(%q)b: %v", inputParams, err)
}
//_, err = io.WriteString(w, fmt.Sprintf("%10v %30v %v\n", "size", "LastModified", "Key"))
//if err != nil {
// return fmt.Errorf("Writing header: %v", err)
//}
// CommonPrefixes is a slice of "directories" found in the bucket.
for _, obj := range listObjectsOutput.CommonPrefixes {
_, err = io.WriteString(w, fmt.Sprintf("%-20v %10v PRE %v\n", "", "", (*obj.Prefix)[len(rootPrefix):]))
if err != nil {
return fmt.Errorf("Writing listObjectsOutput.CommonPrefixes: %v", err)
}
}
// Contents is a slice of files found in the bucket.
for _, obj := range listObjectsOutput.Contents {
_, err = io.WriteString(w, fmt.Sprintf("%-20v %10v %v\n", (*obj.LastModified).Format("2006-01-02 15:04:05"), *obj.Size, (*obj.Key)[len(rootPrefix):]))
if err != nil {
return fmt.Errorf("Writing listObjectsOutput.Contents: %v", err)
}
}
return nil
}