forked from johannesboyne/gofakes3
/
main.go
271 lines (225 loc) · 8.06 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
package main
import (
"expvar"
"flag"
"fmt"
"log"
"net"
"net/http"
httppprof "net/http/pprof"
"os"
"runtime/pprof"
"time"
"github.com/ideal/gofakes3"
"github.com/ideal/gofakes3/backend/s3afero"
"github.com/ideal/gofakes3/backend/s3bolt"
"github.com/ideal/gofakes3/backend/s3mem"
"github.com/spf13/afero"
)
const usage = `
`
func main() {
if err := run(); err != nil {
log.Fatal(err)
}
}
type fakeS3Flags struct {
host string
backendKind string
initialBucket string
fixedTimeStr string
noIntegrity bool
hostBucket bool
autoBucket bool
quiet bool
boltDb string
directFsPath string
directFsMeta string
directFsBucket string
fsPath string
fsMeta string
debugCPU string
debugHost string
}
func (f *fakeS3Flags) attach(flagSet *flag.FlagSet) {
flagSet.StringVar(&f.host, "host", ":9000", "Host to run the service")
flagSet.StringVar(&f.fixedTimeStr, "time", "", "RFC3339 format. If passed, the server's clock will always see this time; does not affect existing stored dates.")
flagSet.StringVar(&f.initialBucket, "initialbucket", "", "If passed, this bucket will be created on startup if it does not already exist.")
flagSet.BoolVar(&f.noIntegrity, "no-integrity", false, "Pass this flag to disable Content-MD5 validation when uploading.")
flagSet.BoolVar(&f.hostBucket, "hostbucket", false, "If passed, the bucket name will be extracted from the first segment of the hostname, rather than the first part of the URL path.")
flagSet.BoolVar(&f.autoBucket, "autobucket", false, "If passed, nonexistent buckets will be created on first use instead of raising an error")
// Logging
flagSet.BoolVar(&f.quiet, "quiet", false, "If passed, log messages are not printed to stderr")
// Backend specific:
flagSet.StringVar(&f.backendKind, "backend", "", "Backend to use to store data (memory, bolt, directfs, fs)")
flagSet.StringVar(&f.boltDb, "bolt.db", "locals3.db", "Database path / name when using bolt backend")
flagSet.StringVar(&f.directFsPath, "directfs.path", "", "File path to serve using S3. You should not modify the contents of this path outside gofakes3 while it is running as it can cause inconsistencies.")
flagSet.StringVar(&f.directFsMeta, "directfs.meta", "", "Optional path for storing S3 metadata for your bucket. If not passed, metadata will not persist between restarts of gofakes3.")
flagSet.StringVar(&f.directFsBucket, "directfs.bucket", "mybucket", "Name of the bucket for your file path; this will be the only supported bucket by the 'directfs' backend for the duration of your run.")
flagSet.StringVar(&f.fsPath, "fs.path", "", "Path to your S3 buckets. Buckets are stored under the '/buckets' subpath.")
flagSet.StringVar(&f.fsMeta, "fs.meta", "", "Optional path for storing S3 metadata for your buckets. Defaults to the '/metadata' subfolder of -fs.path if not passed.")
// Debugging:
flagSet.StringVar(&f.debugHost, "debug.host", "", "Run the debug server on this host")
flagSet.StringVar(&f.debugCPU, "debug.cpu", "", "Create CPU profile in this file")
// Deprecated:
flagSet.StringVar(&f.boltDb, "db", "locals3.db", "Deprecated; use -bolt.db")
flagSet.StringVar(&f.initialBucket, "bucket", "", `Deprecated; use -initialbucket`)
}
func (f *fakeS3Flags) timeOptions() (source gofakes3.TimeSource, skewLimit time.Duration, err error) {
skewLimit = gofakes3.DefaultSkewLimit
if f.fixedTimeStr != "" {
fixedTime, err := time.Parse(time.RFC3339Nano, f.fixedTimeStr)
if err != nil {
return nil, 0, err
}
source = gofakes3.FixedTimeSource(fixedTime)
skewLimit = 0
}
return source, skewLimit, nil
}
func debugServer(host string) {
mux := http.NewServeMux()
mux.Handle("/debug/vars", expvar.Handler())
mux.HandleFunc("/debug/pprof/", httppprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
srv := &http.Server{Addr: host}
srv.Handler = mux
if err := srv.ListenAndServe(); err != nil {
panic(err)
}
}
func run() error {
var values fakeS3Flags
flagSet := flag.NewFlagSet("", 0)
values.attach(flagSet)
if err := flagSet.Parse(os.Args[1:]); err != nil {
return err
}
stopper, err := profile(values)
if err != nil {
return err
}
defer stopper()
if values.debugHost != "" {
log.Println("starting debug server at", fmt.Sprintf("http://%s/debug/pprof", values.debugHost))
go debugServer(values.debugHost)
}
var backend gofakes3.Backend
timeSource, timeSkewLimit, err := values.timeOptions()
if err != nil {
return err
}
switch values.backendKind {
case "":
flag.PrintDefaults()
fmt.Println()
return fmt.Errorf("-backend is required")
case "bolt":
var err error
backend, err = s3bolt.NewFile(values.boltDb, s3bolt.WithTimeSource(timeSource))
if err != nil {
return err
}
log.Println("using bolt backend with file", values.boltDb)
case "mem", "memory":
if values.initialBucket == "" {
log.Println("no buckets available; consider passing -initialbucket")
}
backend = s3mem.New(s3mem.WithTimeSource(timeSource))
log.Println("using memory backend")
case "fs":
if timeSource != nil {
log.Println("warning: time source not supported by this backend")
}
baseFs, err := s3afero.FsPath(values.fsPath)
if err != nil {
return fmt.Errorf("gofakes3: could not create -fs.path: %v", err)
}
var options []s3afero.MultiOption
if values.fsMeta != "" {
metaFs, err := s3afero.FsPath(values.fsMeta)
if err != nil {
return fmt.Errorf("gofakes3: could not create -fs.meta: %v", err)
}
options = append(options, s3afero.MultiWithMetaFs(metaFs))
}
backend, err = s3afero.MultiBucket(baseFs, options...)
if err != nil {
return err
}
case "directfs":
if values.initialBucket != "" {
return fmt.Errorf("gofakes3: -initialbucket not supported by directfs")
}
if values.autoBucket {
return fmt.Errorf("gofakes3: -autobucket not supported by directfs")
}
if timeSource != nil {
log.Println("warning: time source not supported by this backend")
}
baseFs, err := s3afero.FsPath(values.directFsPath)
if err != nil {
return fmt.Errorf("gofakes3: could not create -directfs.path: %v", err)
}
var metaFs afero.Fs
if values.directFsMeta != "" {
metaFs, err = s3afero.FsPath(values.directFsMeta)
if err != nil {
return fmt.Errorf("gofakes3: could not create -directfs.meta: %v", err)
}
} else {
log.Println("using ephemeral memory backend for metadata; this will not persist. See -directfs.metapath flag if you need persistence.")
}
backend, err = s3afero.SingleBucket(values.directFsBucket, baseFs, metaFs)
if err != nil {
return err
}
default:
return fmt.Errorf("unknown backend %q", values.backendKind)
}
if values.initialBucket != "" {
if err := backend.CreateBucket(values.initialBucket); err != nil && !gofakes3.IsAlreadyExists(err) {
return fmt.Errorf("gofakes3: could not create initial bucket %q: %v", values.initialBucket, err)
}
log.Println("created -initialbucket", values.initialBucket)
}
logger := gofakes3.GlobalLog()
if values.quiet {
logger = gofakes3.DiscardLog()
}
faker := gofakes3.New(backend,
gofakes3.WithIntegrityCheck(!values.noIntegrity),
gofakes3.WithTimeSkewLimit(timeSkewLimit),
gofakes3.WithTimeSource(timeSource),
gofakes3.WithLogger(logger),
gofakes3.WithHostBucket(values.hostBucket),
gofakes3.WithAutoBucket(values.autoBucket),
)
return listenAndServe(values.host, faker.Server())
}
func listenAndServe(addr string, handler http.Handler) error {
listener, err := net.Listen("tcp", addr)
if err != nil {
return err
}
log.Println("using port:", listener.Addr().(*net.TCPAddr).Port)
server := &http.Server{Addr: addr, Handler: handler}
return server.Serve(listener)
}
func profile(values fakeS3Flags) (func(), error) {
fn := func() {}
if values.debugCPU != "" {
f, err := os.Create(values.debugCPU)
if err != nil {
return fn, err
}
if err := pprof.StartCPUProfile(f); err != nil {
return fn, err
}
return pprof.StopCPUProfile, nil
}
return fn, nil
}