/
s3.go
68 lines (61 loc) · 1.76 KB
/
s3.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
package benchclient
import (
"bytes"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
infinistore "github.com/ds2-lab/infinistore/client"
)
var (
// The session the S3 Downloader will use
AWSSession = session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: aws.Config{Region: aws.String("us-east-1")},
}))
downloadBufferProvider = s3manager.NewPooledBufferedWriterReadFromProvider(1024)
uploadBufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(1024)
)
type S3 struct {
*defaultClient
bucket string
uploader *s3manager.Uploader
downloader *s3manager.Downloader
}
func NewS3(bk string) *S3 {
client := &S3{
defaultClient: newDefaultClient("S3: "),
bucket: bk,
uploader: s3manager.NewUploader(AWSSession, func(u *s3manager.Uploader) {
u.BufferProvider = uploadBufferProvider
}),
downloader: s3manager.NewDownloader(AWSSession, func(d *s3manager.Downloader) {
d.BufferProvider = downloadBufferProvider
}),
}
client.setter = client.set
client.getter = client.get
client.abbr = "s3"
return client
}
func (c *S3) set(key string, val []byte) error {
// Upload the file to S3.
_, err := c.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
Body: bytes.NewReader(val),
})
return err
}
func (c *S3) get(key string) (infinistore.ReadAllCloser, error) {
buff := new(aws.WriteAtBuffer)
_, err := c.downloader.Download(buff, &s3.GetObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
})
if err != nil {
return nil, err
} else {
return NewByteReader(buff.Bytes()), nil
}
}