forked from SegundamanoMX/backup-my-bucket
/
common.go
152 lines (133 loc) · 3.86 KB
/
common.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
/*
Copyright 2015 ASM Clasificados de Mexico, SA de CV
This file is part of backup-my-bucket.
backup-my-bucket is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License Version 2 as published by
the Free Software Foundation.
backup-my-bucket is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with backup-my-bucket. If not, see <http://www.gnu.org/licenses/>.
*/
package common
import (
"compress/gzip"
"encoding/json"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/SegundamanoMX/backup-my-bucket/log"
"io/ioutil"
"os"
"path/filepath"
"time"
)
type BackupSet struct {
SnapshotsDir string
CompressSnapshots bool
MinimumRedundancy int
RetentionPolicy int
MasterBucket string
MasterRegion string
SlaveBucket string
SlaveRegion string
AccessKey string
SecretKey string
}
type AppConfig struct {
LogLevel int
AwsLogLevel aws.LogLevelType
Syslog bool
BackupSet BackupSet
}
type Version struct {
Key string
LastModified time.Time
Size int64
VersionId string
}
type Snapshot struct {
File string
Timestamp time.Time `json:"Timestamp"`
Contents []Version `json:"Contents"`
}
const (
SnapshotWorkerCount = 128
SnapshotBatchSize = 100000
RestoreWorkerCount = 1024
MaxRetries = 10
GcBatchSize = 1
)
var (
Cfg AppConfig
)
func LoadSnapshots() (snapshots []Snapshot) {
log.Info("Loading snapshots")
files, _ := filepath.Glob(Cfg.BackupSet.SnapshotsDir + "/*")
for _, file := range files {
snapshots = append(snapshots, LoadSnapshot(file))
}
return
}
func LoadSnapshot(file string) (snapshot Snapshot) {
log.Info("Loading snapshot file '%s'.", file)
var bytes []byte
var readErr error
if filepath.Ext(file) == ".Z" {
f, openErr := os.OpenFile(file, os.O_RDONLY, 0000)
if openErr != nil {
log.Fatal("Could not open file %s: %s", file, openErr)
}
defer f.Close()
r, nrErr := gzip.NewReader(f)
if nrErr != nil {
log.Fatal("Could not initialize gzip decompressor: %s", nrErr)
}
defer r.Close()
bytes, readErr = ioutil.ReadAll(r)
if readErr != nil {
log.Fatal("Could not read compressed snapshot file %s: %s", file, readErr)
}
} else {
bytes, readErr = ioutil.ReadFile(file)
if readErr != nil {
log.Fatal("Could not read snapshot file '%s': %s", file, readErr)
}
}
err := json.Unmarshal(bytes, &snapshot)
if err != nil {
log.Fatal("Could not parse snapshot file '%s': %s", file, err)
}
snapshot.File = file
if Cfg.LogLevel > 0 {
pretty, _ := json.MarshalIndent(snapshot, "", " ")
log.Debug("Snapshot '%s':\n%s", file, pretty)
}
return
}
func ConfigureAws(region string) {
logLevelType := func(logLevel uint) aws.LogLevelType {
var awsLogLevel aws.LogLevelType
switch logLevel {
case 0: awsLogLevel = aws.LogOff
case 1: awsLogLevel = aws.LogDebug
case 2: awsLogLevel = aws.LogDebugWithSigning
case 3: awsLogLevel = aws.LogDebugWithHTTPBody
case 4: awsLogLevel = aws.LogDebugWithRequestRetries
case 5: awsLogLevel = aws.LogDebugWithRequestErrors
}
return awsLogLevel
}
defaults.DefaultConfig.Credentials = credentials.NewStaticCredentials(Cfg.BackupSet.AccessKey, Cfg.BackupSet.SecretKey, "")
defaults.DefaultConfig.Region = ®ion
defaults.DefaultConfig.LogLevel = aws.LogLevel(logLevelType(uint(Cfg.AwsLogLevel)))
}
func Min(x int, y int) int {
if (x < y) {
return x
} else {
return y
}
}