-
Notifications
You must be signed in to change notification settings - Fork 211
/
utils.go
130 lines (125 loc) · 5.09 KB
/
utils.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
package storage
import (
"fmt"
"github.com/apex/log"
"github.com/klauspost/compress/zstd"
"github.com/mholt/archiver/v4"
"sort"
"strings"
"time"
)
func GetBackupsToDeleteRemote(backups []Backup, keep int) []Backup {
if len(backups) > keep {
// sort backup ascending
sort.SliceStable(backups, func(i, j int) bool {
return backups[i].UploadDate.After(backups[j].UploadDate)
})
// KeepRemoteBackups should respect incremental backups sequences and don't deleteKey required backups
// fix https://github.com/Altinity/clickhouse-backup/issues/111
// fix https://github.com/Altinity/clickhouse-backup/issues/385
// fix https://github.com/Altinity/clickhouse-backup/issues/525
deletedBackups := make([]Backup, len(backups)-keep)
copied := copy(deletedBackups, backups[keep:])
if copied != len(backups)-keep {
log.Warnf("copied wrong items from backup list expected=%d, actual=%d", len(backups)-keep, copied)
}
keepBackups := make([]Backup, keep)
copied = copy(keepBackups, backups[:keep])
if copied != keep {
log.Warnf("copied wrong items from backup list expected=%d, actual=%d", keep, copied)
}
var findRequiredBackup func(b Backup)
findRequiredBackup = func(b Backup) {
if b.RequiredBackup != "" {
for i, deletedBackup := range deletedBackups {
if b.RequiredBackup == deletedBackup.BackupName {
deletedBackups = append(deletedBackups[:i], deletedBackups[i+1:]...)
findRequiredBackup(deletedBackup)
break
}
}
}
}
for _, b := range keepBackups {
findRequiredBackup(b)
}
// remove from old backup list backup with UploadDate `0001-01-01 00:00:00`, to avoid race condition for multiple shards copy
// fix https://github.com/Altinity/clickhouse-backup/issues/409
i := 0
for _, b := range deletedBackups {
if b.UploadDate != time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) {
deletedBackups[i] = b
i++
}
}
deletedBackups = deletedBackups[:i]
return deletedBackups
}
return []Backup{}
}
func getArchiveWriter(format string, level int) (*archiver.CompressedArchive, error) {
switch format {
case "tar":
return &archiver.CompressedArchive{Archival: archiver.Tar{}}, nil
case "lz4":
return &archiver.CompressedArchive{Compression: archiver.Lz4{CompressionLevel: level}, Archival: archiver.Tar{}}, nil
case "bzip2", "bz2":
return &archiver.CompressedArchive{Compression: archiver.Bz2{CompressionLevel: level}, Archival: archiver.Tar{}}, nil
case "gzip", "gz":
return &archiver.CompressedArchive{Compression: archiver.Gz{CompressionLevel: level, Multithreaded: true}, Archival: archiver.Tar{}}, nil
case "sz":
return &archiver.CompressedArchive{Compression: archiver.Sz{}, Archival: archiver.Tar{}}, nil
case "xz":
return &archiver.CompressedArchive{Compression: archiver.Xz{}, Archival: archiver.Tar{}}, nil
case "br", "brotli":
return &archiver.CompressedArchive{Compression: archiver.Brotli{Quality: level}, Archival: archiver.Tar{}}, nil
case "zstd":
return &archiver.CompressedArchive{Compression: archiver.Zstd{EncoderOptions: []zstd.EOption{zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))}}, Archival: archiver.Tar{}}, nil
}
return nil, fmt.Errorf("wrong compression_format: %s, supported: 'tar', 'lz4', 'bzip2', 'bz2', 'gzip', 'gz', 'sz', 'xz', 'br', 'brotli', 'zstd'", format)
}
func getArchiveReader(format string) (*archiver.CompressedArchive, error) {
switch format {
case "tar":
return &archiver.CompressedArchive{Archival: archiver.Tar{}}, nil
case "lz4":
return &archiver.CompressedArchive{Compression: archiver.Lz4{}, Archival: archiver.Tar{}}, nil
case "bzip2", "bz2":
return &archiver.CompressedArchive{Compression: archiver.Bz2{}, Archival: archiver.Tar{}}, nil
case "gzip", "gz":
return &archiver.CompressedArchive{Compression: archiver.Gz{Multithreaded: true}, Archival: archiver.Tar{}}, nil
case "sz":
return &archiver.CompressedArchive{Compression: archiver.Sz{}, Archival: archiver.Tar{}}, nil
case "xz":
return &archiver.CompressedArchive{Compression: archiver.Xz{}, Archival: archiver.Tar{}}, nil
case "br", "brotli":
return &archiver.CompressedArchive{Compression: archiver.Brotli{}, Archival: archiver.Tar{}}, nil
case "zstd":
return &archiver.CompressedArchive{Compression: archiver.Zstd{}, Archival: archiver.Tar{}}, nil
}
return nil, fmt.Errorf("wrong compression_format: %s, supported: 'tar', 'lz4', 'bzip2', 'bz2', 'gzip', 'gz', 'sz', 'xz', 'br', 'brotli', 'zstd'", format)
}
func checkArchiveExtension(ext, format string) bool {
if (format == "gz" || format == "gzip") && ext != ".gz" && ext != ".gzip" {
return false
}
if (format == "bz2" || format == "bzip2") && ext != ".bz2" && ext != ".bzip2" {
return false
}
if (format == "br" || format == "brotli") && ext != ".br" && ext != ".brotli" {
return false
}
if strings.HasSuffix(ext, format) {
return true
}
if (format == "gz" || format == "gzip") && (ext == ".gz" || ext == ".gzip") {
return true
}
if (format == "bz2" || format == "bzip2") && (ext == ".bz2" || ext == ".bzip2") {
return true
}
if (format == "br" || format == "brotli") && (ext == ".br" || ext == ".brotli") {
return true
}
return false
}