/
directory_uploader.go
80 lines (65 loc) · 2.48 KB
/
directory_uploader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package internal
import (
"path/filepath"
"sync/atomic"
"github.com/apecloud/dataprotection-wal-g/internal/crypto"
"github.com/apecloud/dataprotection-wal-g/utility"
"github.com/wal-g/tracelog"
)
type DirectoryUploader interface {
Upload(path string) TarFileSets
}
type CommonDirectoryUploader struct {
crypter crypto.Crypter
tarBallFilePacker TarBallFilePacker
tarBallComposerMaker TarBallComposerMaker
tarSizeThreshold int64
excludedFiles map[string]utility.Empty
backupName string
uploader Uploader
}
func NewCommonDirectoryUploader(
crypter crypto.Crypter, packer TarBallFilePacker,
tarBallComposerMaker TarBallComposerMaker, tarSizeThreshold int64,
excludedFiles map[string]utility.Empty, backupName string,
uploader Uploader) *CommonDirectoryUploader {
return &CommonDirectoryUploader{
crypter: crypter,
tarBallFilePacker: packer,
tarBallComposerMaker: tarBallComposerMaker,
tarSizeThreshold: tarSizeThreshold,
excludedFiles: excludedFiles,
backupName: backupName,
uploader: uploader,
}
}
func (u *CommonDirectoryUploader) Upload(path string) TarFileSets {
bundle := NewBundle(path, u.crypter, u.tarSizeThreshold, u.excludedFiles)
// Start a new tar bundle, walk the pgDataDirectory and upload everything there.
tracelog.InfoLogger.Println("Starting a new tar bundle")
err := bundle.StartQueue(NewStorageTarBallMaker(u.backupName, u.uploader))
tracelog.ErrorLogger.FatalOnError(err)
err = bundle.SetupComposer(u.tarBallComposerMaker)
tracelog.ErrorLogger.FatalOnError(err)
tracelog.InfoLogger.Println("Walking ...")
err = filepath.Walk(path, bundle.AddToBundle)
tracelog.ErrorLogger.FatalOnError(err)
tracelog.InfoLogger.Println("Packing ...")
tarFileSets, err := bundle.FinishComposing()
tracelog.ErrorLogger.FatalOnError(err)
tracelog.DebugLogger.Println("Finishing queue ...")
err = bundle.FinishQueue()
tracelog.ErrorLogger.FatalOnError(err)
uncompressedSize := atomic.LoadInt64(bundle.TarBallQueue.AllTarballsSize)
compressedSize, err := u.uploader.UploadedDataSize()
tracelog.ErrorLogger.FatalOnError(err)
tracelog.DebugLogger.Printf("Uncompressed size: %d", uncompressedSize)
tracelog.DebugLogger.Printf("Compressed size: %d", compressedSize)
// Wait for all uploads to finish.
tracelog.DebugLogger.Println("Waiting for all uploads to finish")
u.uploader.Finish()
if u.uploader.Failed() {
tracelog.ErrorLogger.Fatalf("Uploading failed during '%s' backup.\n", path)
}
return tarFileSets
}