/
main.go
138 lines (97 loc) · 3.21 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
// copy-uri is a command-line tool for cloning the body of a URL (on the web) to a file
// in a blob.Bucket endpoint.
package main
// ./bin/copy-uri -source-uri https://static.sfomuseum.org/media/189/736/720/3/1897367203_lt3HuJ5ALbY4SDoxTd4oi7abOF7gQZKM_c.jpg -target-uri cwd:// -filename test.jpg
import (
"context"
"fmt"
"log"
"path/filepath"
_ "github.com/aaronland/gocloud-blob-s3"
"github.com/aaronland/gocloud-blob/bucket"
"github.com/aaronland/gocloud-blob/copy"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/sfomuseum/go-flags/flagset"
"gocloud.dev/blob"
_ "gocloud.dev/blob/fileblob"
_ "gocloud.dev/blob/memblob"
)
func main() {
var source_uri string
var filename string
var target_uri string
var show_progress bool
var acl string
var part_size int64
var mode string
fs := flagset.NewFlagSet("copy")
fs.StringVar(&source_uri, "source-uri", "", "The URI of the file to copy.")
fs.StringVar(&target_uri, "target-uri", "", "A valid gocloud.dev/blob.Bucket URI.")
fs.StringVar(&filename, "filename", "", "The final filename of the file to copy. If empty the basename of the `-source-uri` flag value will be used.")
fs.StringVar(&acl, "acl", "", "An optional AWS S3 ACL to assign to the file being copied.")
fs.Int64Var(&part_size, "part-size", 0, "The buffer size (in bytes) to use when buffering data into chunks and sending them as parts to S3. If 0 the default value for the `aws/aws-sdk-go/service/s3/s3manager` package will be used.")
fs.BoolVar(&show_progress, "show-progress", false, "Show copy progress.")
fs.StringVar(&mode, "mode", "cli", "Valid options are: cli, lambda.")
flagset.Parse(fs)
err := flagset.SetFlagsFromEnvVars(fs, "BLOB")
if err != nil {
log.Fatalf("Failed to set flags from environment, %v", err)
}
ctx := context.Background()
bucket, err := bucket.OpenBucket(ctx, target_uri)
if err != nil {
log.Fatalf("Failed to open bucket '%s', %v", target_uri, err)
}
defer bucket.Close()
opts := ©.CopyURLOptions{
Bucket: bucket,
Filename: filename,
ShowProgress: show_progress,
}
if acl != "" || part_size != 0 {
before := func(asFunc func(interface{}) bool) error {
if acl != "" {
input := &s3manager.UploadInput{}
ok := asFunc(&input)
if !ok {
return fmt.Errorf("Not an S3 type")
}
input.ACL = aws.String(acl)
}
if part_size != 0 {
uploader := &s3manager.Uploader{}
ok := asFunc(&uploader)
if !ok {
return fmt.Errorf("Not an S3 type")
}
uploader.PartSize = part_size
}
return nil
}
opts.WriterOptions = &blob.WriterOptions{
BeforeWrite: before,
}
}
run := func(ctx context.Context, opts *copy.CopyURLOptions, source string) error {
if opts.Filename == "" {
opts.Filename = filepath.Base(source)
}
return copy.CopyURLStringToBucket(ctx, opts, source)
}
switch mode {
case "cli":
err := run(ctx, opts, source_uri)
if err != nil {
log.Fatalf("Failed to copy URL to bucket, %v", err)
}
case "lambda":
handler := func(ctx context.Context, source string) error {
return run(ctx, opts, source)
}
lambda.Start(handler)
default:
log.Fatalf("Invalid mode")
}
}