-
Notifications
You must be signed in to change notification settings - Fork 4
/
file_uploader.go
92 lines (76 loc) · 1.89 KB
/
file_uploader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package mindreader
import (
"context"
"fmt"
"sync"
"time"
"github.com/abourget/llerrgroup"
"github.com/streamingfast/dstore"
"github.com/streamingfast/shutter"
"go.uber.org/zap"
)
type FileUploader struct {
*shutter.Shutter
mutex sync.Mutex
localStore dstore.Store
destinationStore dstore.Store
logger *zap.Logger
complete chan struct{}
}
func NewFileUploader(localStore dstore.Store, destinationStore dstore.Store, logger *zap.Logger) *FileUploader {
return &FileUploader{
Shutter: shutter.New(),
complete: make(chan struct{}),
localStore: localStore,
destinationStore: destinationStore,
logger: logger,
}
}
func (fu *FileUploader) Start(ctx context.Context) {
defer close(fu.complete)
fu.OnTerminating(func(_ error) {
<-fu.complete
})
if fu.IsTerminating() {
return
}
var terminating bool
for {
err := fu.uploadFiles(ctx)
if err != nil {
fu.logger.Warn("failed to upload file", zap.Error(err))
}
if terminating {
return
}
select {
case <-fu.Terminating():
fu.logger.Info("terminating upload loop on next pass")
terminating = true
case <-time.After(500 * time.Millisecond):
}
}
}
func (fu *FileUploader) uploadFiles(ctx context.Context) error {
fu.mutex.Lock()
defer fu.mutex.Unlock()
eg := llerrgroup.New(200)
_ = fu.localStore.Walk(ctx, "", func(filename string) (err error) {
if eg.Stop() {
return nil
}
eg.Go(func() error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
if traceEnabled {
fu.logger.Debug("uploading file to storage", zap.String("local_file", filename))
}
if err = fu.destinationStore.PushLocalFile(ctx, fu.localStore.ObjectPath(filename), filename); err != nil {
return fmt.Errorf("moving file %q to storage: %w", filename, err)
}
return nil
})
return nil
})
return eg.Wait()
}