/
s3_archive.go
159 lines (141 loc) · 3.58 KB
/
s3_archive.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
// Copyright 2016 DiamNet Development Foundation and contributors. Licensed
// under the Apache License, Version 2.0. See the COPYING file at the root
// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
package historyarchive
import (
"bytes"
"io"
"net/http"
"path"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/diamnet/go/support/errors"
)
type S3ArchiveBackend struct {
svc *s3.S3
bucket string
prefix string
unsignedRequests bool
}
func (b *S3ArchiveBackend) GetFile(pth string) (io.ReadCloser, error) {
params := &s3.GetObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(path.Join(b.prefix, pth)),
}
req, resp := b.svc.GetObjectRequest(params)
if b.unsignedRequests {
req.Handlers.Sign.Clear() // makes this request unsigned
}
err := req.Send()
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (b *S3ArchiveBackend) Exists(pth string) (bool, error) {
params := &s3.HeadObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(path.Join(b.prefix, pth)),
}
req, _ := b.svc.HeadObjectRequest(params)
if b.unsignedRequests {
req.Handlers.Sign.Clear() // makes this request unsigned
}
err := req.Send()
if err != nil {
return false, err
}
if req.HTTPResponse.StatusCode >= 200 && req.HTTPResponse.StatusCode < 400 {
return true, nil
} else if req.HTTPResponse.StatusCode == http.StatusNotFound {
return false, nil
} else {
return false, errors.Errorf("Unkown status code=%d", req.HTTPResponse.StatusCode)
}
}
func (b *S3ArchiveBackend) PutFile(pth string, in io.ReadCloser) error {
var buf bytes.Buffer
_, err := buf.ReadFrom(in)
in.Close()
if err != nil {
return err
}
params := &s3.PutObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(path.Join(b.prefix, pth)),
ACL: aws.String(s3.ObjectCannedACLPublicRead),
Body: bytes.NewReader(buf.Bytes()),
}
req, _ := b.svc.PutObjectRequest(params)
if b.unsignedRequests {
req.Handlers.Sign.Clear() // makes this request unsigned
}
err = req.Send()
in.Close()
return err
}
func (b *S3ArchiveBackend) ListFiles(pth string) (chan string, chan error) {
prefix := path.Join(b.prefix, pth)
ch := make(chan string)
errs := make(chan error)
params := &s3.ListObjectsInput{
Bucket: aws.String(b.bucket),
MaxKeys: aws.Int64(1000),
Prefix: aws.String(prefix),
}
req, resp := b.svc.ListObjectsRequest(params)
if b.unsignedRequests {
req.Handlers.Sign.Clear() // makes this request unsigned
}
err := req.Send()
if err != nil {
errs <- err
close(ch)
close(errs)
return ch, errs
}
go func() {
for {
for _, c := range resp.Contents {
params.Marker = c.Key
ch <- *c.Key
}
if *resp.IsTruncated {
req, resp = b.svc.ListObjectsRequest(params)
if b.unsignedRequests {
req.Handlers.Sign.Clear() // makes this request unsigned
}
err := req.Send()
if err != nil {
errs <- err
}
} else {
break
}
}
close(ch)
close(errs)
}()
return ch, errs
}
func (b *S3ArchiveBackend) CanListFiles() bool {
return true
}
func makeS3Backend(bucket string, prefix string, opts ConnectOptions) (ArchiveBackend, error) {
cfg := aws.Config{
Region: aws.String(opts.S3Region),
Endpoint: aws.String(opts.S3Endpoint),
}
sess, err := session.NewSession(&cfg)
if err != nil {
return nil, err
}
backend := S3ArchiveBackend{
svc: s3.New(sess),
bucket: bucket,
prefix: prefix,
unsignedRequests: opts.UnsignedRequests,
}
return &backend, nil
}