forked from pachyderm/pachyderm
-
Notifications
You must be signed in to change notification settings - Fork 0
/
amazon_client.go
113 lines (100 loc) · 2.55 KB
/
amazon_client.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package obj
import (
"fmt"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3Manager"
)
type amazonClient struct {
bucket string
s3 *s3.S3
uploader *s3manager.Uploader
}
func newAmazonClient(bucket string, id string, secret string, token string, region string) (*amazonClient, error) {
session := session.New(&aws.Config{
Credentials: credentials.NewStaticCredentials(id, secret, token),
Region: aws.String(region),
})
return &amazonClient{
bucket: bucket,
s3: s3.New(session),
uploader: s3manager.NewUploader(session),
}, nil
}
func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
return newWriter(c, name), nil
}
func (c *amazonClient) Walk(name string, fn func(name string) error) error {
var fnErr error
if err := c.s3.ListObjectsPages(
&s3.ListObjectsInput{
Bucket: aws.String(c.bucket),
Prefix: aws.String(name),
},
func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
for _, object := range listObjectsOutput.Contents {
if err := fn(*object.Key); err != nil {
fnErr = err
return false
}
}
return true
},
); err != nil {
return err
}
return fnErr
}
func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
byteRange := ""
if size == 0 {
byteRange = fmt.Sprintf("bytes=%d-", offset)
} else {
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+size)
}
getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(name),
Range: aws.String(byteRange),
})
if err != nil {
return nil, err
}
return getObjectOutput.Body, nil
}
func (c *amazonClient) Delete(name string) error {
return nil
}
type amazonWriter struct {
errChan chan error
pipe *io.PipeWriter
}
func newWriter(client *amazonClient, name string) *amazonWriter {
reader, writer := io.Pipe()
w := &amazonWriter{
errChan: make(chan error),
pipe: writer,
}
go func() {
_, err := client.uploader.Upload(&s3manager.UploadInput{
Body: reader,
Bucket: aws.String(client.bucket),
Key: aws.String(name),
ContentEncoding: aws.String("application/octet-stream"),
})
w.errChan <- err
}()
return w
}
func (w *amazonWriter) Write(p []byte) (int, error) {
return w.pipe.Write(p)
}
func (w *amazonWriter) Close() error {
if err := w.pipe.Close(); err != nil {
return err
}
return <-w.errChan
}