-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
87 lines (71 loc) · 1.81 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
package main
import (
"context"
"fmt"
"os"
"os/signal"
"strings"
"sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/kelseyhightower/envconfig"
"github.com/sirupsen/logrus"
)
var config struct {
Region string `default:"us-west-1"`
Bucket string `required:"true"`
FilePrefix string `split_words:"true" default:"flood"`
FileCount int `split_words:"true" default:"1100"`
Concurrency int `default:"10"`
}
func main() {
envconfig.MustProcess("", &config)
cctx, cancel := context.WithCancel(context.Background())
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion(config.Region)))
uploader := s3manager.NewUploader(sess)
var wg sync.WaitGroup
fileRequests := make(chan int)
for i := 0; i < config.Concurrency; i++ {
wg.Add(1)
go produceFiles(cctx, uploader, fileRequests, &wg)
}
go func() {
for i := 0; i < config.FileCount; i++ {
fileRequests <- i
}
close(fileRequests)
}()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt)
go func() {
s := <-sigChan
logrus.Error("Received signal: ", s)
cancel()
}()
wg.Wait()
}
func produceFiles(ctx context.Context, uploader *s3manager.Uploader, requests <-chan int, wg *sync.WaitGroup) {
for {
select {
case <-ctx.Done():
wg.Done()
return
case r, ok := <-requests:
if !ok {
wg.Done()
return
}
fileName := fmt.Sprintf("%s-%d", config.FilePrefix, r)
logrus.Infof("Uploading file %s", fileName)
_, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(config.Bucket),
Key: aws.String(fileName),
Body: strings.NewReader(fileName),
})
if err != nil {
logrus.Warnf("Upload of file '%s' failed: %v", fileName, err)
}
}
}
}