-
Notifications
You must be signed in to change notification settings - Fork 36
/
delta.go
249 lines (212 loc) · 7.16 KB
/
delta.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
package swupd
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"sync"
"syscall"
"github.com/clearlinux/mixer-tools/helpers"
"github.com/pkg/errors"
)
const (
// From swupd-server's include/swupd.h:
//
// Approximately the smallest size of a pair of input files which differ by a
// single bit that bsdiff can produce a more compact deltafile. Files smaller
// than this are always marked as different. See the magic 200 value in the
// bsdiff/src/diff.c code.
//
minimumSizeToMakeDeltaInBytes = 200
)
// Delta represents a delta file between two other files. If Error is present, it
// indicates that the delta couldn't be created.
type Delta struct {
Path string
Error error
from *File
to *File
}
var bsdiffLog *log.Logger
// CreateDeltasForManifest creates all delta files between the previous and current version of the
// supplied manifest. Returns a list of deltas (which contains information about
// individual delta errors). Returns error (and no deltas) if it can't assemble the delta
// list. If number of workers is zero or less, 1 worker is used.
func CreateDeltasForManifest(manifest, statedir string, from, to uint32, numWorkers int) ([]Delta, error) {
var c config
c, err := getConfig(statedir)
if err != nil {
return nil, err
}
var oldManifest *Manifest
var newManifest *Manifest
if oldManifest, err = ParseManifestFile(filepath.Join(c.outputDir, fmt.Sprintf("%d", from), manifest)); err != nil {
return nil, err
}
if newManifest, err = ParseManifestFile(filepath.Join(c.outputDir, fmt.Sprintf("%d", to), manifest)); err != nil {
return nil, err
}
return createDeltasFromManifests(&c, oldManifest, newManifest, numWorkers)
}
func createDeltasFromManifests(c *config, oldManifest, newManifest *Manifest, numWorkers int) ([]Delta, error) {
logFile, err := os.OpenFile(filepath.Join(c.stateDir, "bsdiff_errors.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return nil, errors.Wrap(err, "Cannot create log file for delta creation")
}
defer func() {
_ = logFile.Close()
}()
bsdiffLog = log.New(logFile, "DELTA: ", log.Lshortfile)
deltas, err := findDeltas(c, oldManifest, newManifest)
if err != nil {
return nil, errors.Wrapf(err, "Failed to create deltas list %s", newManifest.Name)
}
if len(deltas) == 0 {
return []Delta{}, nil
}
if numWorkers < 1 {
numWorkers = 1
}
var deltaQueue = make(chan *Delta)
var wg sync.WaitGroup
wg.Add(numWorkers)
// Delta creation takes a lot of memory, so create a limited amount of goroutines.
for i := 0; i < numWorkers; i++ {
go func() {
defer wg.Done()
for delta := range deltaQueue {
delta.Error = createDelta(c, delta)
}
}()
}
// Send jobs to the queue for delta goroutines to pick up.
for i := range deltas {
deltaQueue <- &deltas[i]
}
// Send message that no more jobs are being sent
close(deltaQueue)
wg.Wait()
return deltas, nil
}
// deltaTooLarge returns true if the delta file is larger than or equal in size
// to the compressed fullfile. This is not a critical check so any failures in
// the process just cause a false return.
func deltaTooLarge(c *config, delta *Delta, newPath string) bool {
dInfo, err := os.Stat(delta.Path)
if err != nil {
return false
}
deltaSize := dInfo.Size()
fHash, err := GetHashForFile(newPath)
if err != nil {
return false
}
fCompressed := filepath.Join(c.outputDir,
fmt.Sprint(delta.to.Version),
"files",
fHash+".tar")
fcInfo, err := os.Stat(fCompressed)
if err != nil {
return false
}
fcSize := fcInfo.Size()
return deltaSize >= fcSize
}
func createDelta(c *config, delta *Delta) error {
if _, err := os.Stat(delta.Path); err == nil {
// Skip existing deltas. Not verifying since client is resilient about that.
return nil
}
oldPath := filepath.Join(c.imageBase, fmt.Sprint(delta.from.Version), "full", delta.from.Name)
newPath := filepath.Join(c.imageBase, fmt.Sprint(delta.to.Version), "full", delta.to.Name)
// Set timeout to 1 minute (60 seconds) for bsdiff
// The majority of all delta creations take significantly less than 1 minute
// to run, which signifies that bsdiff is working on a delta that may be very
// large, or very difficult to diff. In all the cases where bsdiff took
// multiple minutes to finish, the delta ended up not being used because it
// was larger than the compressed fullfile. This attempts to skip those cases.
if err := helpers.RunCommandTimeout(60, "bsdiff", oldPath, newPath, delta.Path); err != nil {
_ = os.Remove(delta.Path)
if exitErr, ok := errors.Cause(err).(*exec.ExitError); ok {
// bsdiff returns 1 that stands for "FULLDL", i.e. it decided that
// a delta is not worth. Give a better error message for that case.
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
if status.ExitStatus() == 1 {
return fmt.Errorf("bsdiff returned FULLDL, not using delta")
}
}
}
errStr := fmt.Sprintf("Failed to create delta for %s (%d) -> %s (%d)", delta.from.Name, delta.from.Version, delta.to.Name, delta.to.Version)
bsdiffLog.SetPrefix("BSDIFF: ")
bsdiffLog.Println(errStr)
return errors.Wrap(err, errStr)
}
// Check that delta is smaller than compressed full file
if deltaTooLarge(c, delta, newPath) {
_ = os.Remove(delta.Path)
errStr := fmt.Sprintf("Delta file %s (%d) larger than compressed full file %s", delta.to.Name, delta.to.Version, newPath)
bsdiffLog.SetPrefix("LARGER-DELTA: ")
bsdiffLog.Println(errStr)
return errors.New(errStr)
}
// Check that the delta actually applies correctly.
testPath := delta.Path + ".testnewfile"
if err := helpers.RunCommandSilent("bspatch", oldPath, testPath, delta.Path); err != nil {
errStr := fmt.Sprintf("Failed to apply delta %s", delta.Path)
bsdiffLog.SetPrefix("BSPATCH: ")
bsdiffLog.Println(errStr)
return errors.Wrapf(err, errStr)
}
defer func() {
_ = os.Remove(testPath)
}()
testHash, err := Hashcalc(testPath)
if err != nil {
_ = os.Remove(delta.Path)
return errors.Wrap(err, "Failed to calculate hash for test file created applying delta")
}
if testHash != delta.to.Hash {
_ = os.Remove(delta.Path)
return errors.Wrapf(err, "Delta mismatch: %s -> %s via delta: %s", oldPath, newPath, delta.Path)
}
return nil
}
func findDeltas(c *config, oldManifest, newManifest *Manifest) ([]Delta, error) {
oldManifest.sortFilesName()
newManifest.sortFilesName()
err := linkDeltaPeersForPack(c, oldManifest, newManifest)
if err != nil {
return nil, err
}
deltaCount := 0
for _, nf := range newManifest.Files {
if nf.DeltaPeer != nil {
deltaCount++
}
}
deltas := make([]Delta, 0, deltaCount)
// Use set to remove completely equal delta entries. These happen when two files that look
// the same, change content in next version (but still look the same).
seen := make(map[string]bool)
for _, nf := range newManifest.Files {
if nf.DeltaPeer == nil {
continue
}
from := nf.DeltaPeer
to := nf
dir := filepath.Join(c.outputDir, fmt.Sprint(to.Version), "delta")
name := fmt.Sprintf("%d-%d-%s-%s", from.Version, to.Version, from.Hash, to.Hash)
path := filepath.Join(dir, name)
if seen[path] {
continue
}
seen[path] = true
deltas = append(deltas, Delta{
Path: path,
from: from,
to: to,
})
}
return deltas, nil
}