-
Notifications
You must be signed in to change notification settings - Fork 2
/
app.go
210 lines (177 loc) · 7.92 KB
/
app.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
package main
import (
"github.com/alexflint/go-arg"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/daniel-cole/GoS3GFSBackup/download"
"github.com/daniel-cole/GoS3GFSBackup/log"
"github.com/daniel-cole/GoS3GFSBackup/rotate"
"github.com/daniel-cole/GoS3GFSBackup/rpolicy"
"github.com/daniel-cole/GoS3GFSBackup/s3client"
"github.com/daniel-cole/GoS3GFSBackup/upload"
"github.com/daniel-cole/GoS3GFSBackup/util"
"os"
"strconv"
"time"
)
type args struct {
Action string `arg:"help:The intended action for the tool to run [backup|upload|download|rotate]"`
Region string `arg:"required,help:The AWS region to upload the specified file to"`
Bucket string `arg:"required,help:The S3 bucket to upload the specified file to"`
CredFile string `arg:"help:The full path to the AWS CLI credential file if environment variables are not being used to provide the access id and key"`
Profile string `arg:"help:The profile to use for the AWS CLI credential file"`
PathToFile string `arg:"help:The full path to the file to upload to the specified S3 bucket. Must be specified unless --rotateonly=true"`
S3FileName string `arg:"help:The name of the file as it should appear in the S3 bucket. Must be specified unless --rotateonly=true"`
BucketDir string `arg:"help:The directory chain in the bucket in which to upload the S3 object to. Must include the trailing slash"`
Timeout int `arg:"help:The timeout to upload the specified file (seconds)"`
DryRun bool `arg:"help:If enabled then no upload or rotation actions will be executed [default: false]"`
ConcurrentWorkers int `arg:"help:The number of threads to use when uploading the file to S3"`
PartSize int `arg:"help:The part size to use when performing a multipart upload or download (MB)"`
EnforceRetentionPeriod bool `arg:"help:If enabled then objects in the S3 bucket will only be rotated if they are older then the retention period"`
DailyRetentionCount int `arg:"help:The number of daily objects to keep in S3"`
DailyRetentionPeriod int `arg:"help:The retention period (hours) that a daily object should be kept in S3"`
WeeklyRetentionCount int `arg:"help:The number of weekly objects to keep in S3"`
WeeklyRetentionPeriod int `arg:"help:The retention period (hours) that a weekly object should be kept in S3"`
}
func init() {
log.Init(os.Stdout, os.Stdout, os.Stderr)
}
func main() {
// Set default args
args := args{}
args.Timeout = 3600 // Default timeout to 1 hour for file upload
args.CredFile = ""
args.Profile = "default"
args.BucketDir = ""
args.EnforceRetentionPeriod = true
args.DryRun = false
args.ConcurrentWorkers = 5
args.PartSize = 50
args.DailyRetentionCount = 6
args.DailyRetentionPeriod = 168
args.WeeklyRetentionCount = 4
args.WeeklyRetentionPeriod = 672
// Parse args from command line
arg.MustParse(&args)
logArgs(args)
log.Info.Println(`
######################################
# GoS3GFSBackup Started #
######################################
`)
svc, err := s3client.CreateS3Client(args.CredFile, args.Profile, args.Region)
if err != nil {
log.Error.Println(err)
os.Exit(1)
}
runAction(svc, args)
log.Info.Println("Finished GoS3GFSBackup!")
log.Info.Println(`
######################################
# GoS3GFSBackup Finished #
######################################
`)
}
func runAction(svc *s3.S3, args args) {
switch args.Action {
case "backup":
runBackupAction(svc, args)
case "upload":
runUploadAction(svc, args)
case "download":
runDownloadAction(svc, args)
case "rotate":
runRotateAction(svc, args)
default:
log.Error.Println("unexpected action specified: " + args.Action)
}
}
func runBackupAction(svc *s3.S3, arguments args) {
log.Info.Println("Backup action specified, backing up file")
rotationPolicy := getRotationPolicy(arguments)
log.Info.Println("Starting standard GFS upload and rotation")
prefix := util.GetKeyType(rotationPolicy, time.Now())
_, err := upload.UploadFile(svc, getUploadObject(arguments, true), prefix, arguments.DryRun)
if err != nil {
log.Error.Printf("Failed to upload file. Aborting backup. Reason: %v\n", err)
os.Exit(1)
}
rotate.StartRotation(svc, arguments.Bucket, rotationPolicy, arguments.DryRun)
log.Info.Println("Upload and Rotation Complete!")
}
func runUploadAction(svc *s3.S3, arguments args) {
log.Info.Println("Upload action specified, uploading file")
_, err := upload.UploadFile(svc, getUploadObject(arguments, false), "", arguments.DryRun)
if err != nil {
log.Error.Printf("Failed to upload file. Reason: %v\n", err)
os.Exit(1)
}
}
func runRotateAction(svc *s3.S3, arguments args) {
log.Info.Println("Rotate action specified, proceeding with rotation only")
rotate.StartRotation(svc, arguments.Bucket, getRotationPolicy(arguments), arguments.DryRun)
}
func runDownloadAction(svc *s3.S3, arguments args) {
log.Info.Println("Download action specified, downloading file")
downloadObject := download.DownloadObject{
DownloadLocation: arguments.PathToFile,
S3FileKey: arguments.S3FileName,
BucketDir: arguments.BucketDir,
Bucket: arguments.Bucket,
NumWorkers: arguments.ConcurrentWorkers,
PartSize: arguments.PartSize,
}
err := download.DownloadFile(svc, downloadObject)
if err != nil {
log.Error.Printf("Failed to download file. Aborting. Reason: %v\n", err)
os.Exit(1)
}
}
func getUploadObject(arguments args, manipulate bool) upload.UploadObject {
return upload.UploadObject{
PathToFile: arguments.PathToFile,
S3FileName: arguments.S3FileName,
BucketDir: arguments.BucketDir,
Bucket: arguments.Bucket,
Timeout: time.Second * time.Duration(arguments.Timeout),
NumWorkers: arguments.ConcurrentWorkers,
PartSize: arguments.PartSize,
Manipulate: manipulate,
}
}
func getRotationPolicy(arguments args) rpolicy.RotationPolicy {
if !arguments.EnforceRetentionPeriod {
log.Warn.Println("GoS3GFSBackup is running with enforce retention period disabled. " +
"This may result in objects being deleted that which have not exceeded the retention period")
}
// Standard GFS rotation policy
return rpolicy.RotationPolicy{
DailyRetentionPeriod: time.Hour * time.Duration(arguments.DailyRetentionPeriod),
DailyRetentionCount: arguments.DailyRetentionCount,
DailyPrefix: "daily_",
WeeklyRetentionPeriod: time.Hour * time.Duration(arguments.WeeklyRetentionPeriod),
WeeklyRetentionCount: arguments.WeeklyRetentionCount,
WeeklyPrefix: "weekly_",
MonthlyPrefix: "monthly_",
EnforceRetentionPeriod: arguments.EnforceRetentionPeriod,
}
}
func logArgs(arguments args) {
log.Info.Println("Loaded GoS3GFSBackup with arguments: ")
log.Info.Println("--credfile=" + arguments.CredFile)
log.Info.Println("--region=" + arguments.Region)
log.Info.Println("--bucket=" + arguments.Bucket)
log.Info.Println("--bucketdir=" + arguments.BucketDir)
log.Info.Println("--profile=" + arguments.Profile)
log.Info.Println("--action=" + arguments.Action)
log.Info.Println("--pathtofile=" + arguments.PathToFile)
log.Info.Println("--s3filename=" + arguments.S3FileName)
log.Info.Println("--dryrun=" + strconv.FormatBool(arguments.DryRun))
log.Info.Println("--timeout=" + strconv.Itoa(arguments.Timeout))
log.Info.Println("--enforceretentionperiod=" + strconv.FormatBool(arguments.EnforceRetentionPeriod))
log.Info.Println("--concurrentworkers=" + strconv.Itoa(arguments.ConcurrentWorkers))
log.Info.Println("--partsize=" + strconv.Itoa(arguments.PartSize))
log.Info.Println("--dailyretentioncount=" + strconv.Itoa(arguments.DailyRetentionCount))
log.Info.Println("--dailyretentionperiod=" + strconv.Itoa(arguments.DailyRetentionPeriod))
log.Info.Println("--weeklyretentioncount=" + strconv.Itoa(arguments.WeeklyRetentionCount))
log.Info.Println("--weeklyretentionperiod=" + strconv.Itoa(arguments.WeeklyRetentionPeriod))
}