-
Notifications
You must be signed in to change notification settings - Fork 2
/
local.go
373 lines (309 loc) · 8.81 KB
/
local.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
package local
import (
"context"
"hash"
"io"
"io/ioutil"
"os"
"path/filepath"
"syscall"
"github.com/rubiojr/rapi/internal/errors"
"github.com/rubiojr/rapi/restic"
"github.com/rubiojr/rapi/backend"
"github.com/rubiojr/rapi/internal/debug"
"github.com/rubiojr/rapi/internal/fs"
"github.com/cenkalti/backoff/v4"
)
// Local is a backend in a local directory.
type Local struct {
Config
backend.Layout
}
// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}
const defaultLayout = "default"
// Open opens the local backend as specified by config.
func Open(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
return &Local{Config: cfg, Layout: l}, nil
}
// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
be := &Local{
Config: cfg,
Layout: l,
}
// test if config file already exists
_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
// create paths for data and refs
for _, d := range be.Paths() {
err := fs.MkdirAll(d, backend.Modes.Dir)
if err != nil {
return nil, errors.WithStack(err)
}
}
return be, nil
}
// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
return b.Path
}
// Hasher may return a hash function for calculating a content hash for the backend
func (b *Local) Hasher() hash.Hash {
return nil
}
// IsNotExist returns true if the error is caused by a non existing file.
func (b *Local) IsNotExist(err error) bool {
return errors.Is(err, os.ErrNotExist)
}
// Save stores data in the backend at the handle.
func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) {
debug.Log("Save %v", h)
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
finalname := b.Filename(h)
dir := filepath.Dir(finalname)
defer func() {
// Mark non-retriable errors as such
if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) {
err = backoff.Permanent(err)
}
}()
// Create new file with a temporary name.
tmpname := filepath.Base(finalname) + "-tmp-"
f, err := tempFile(dir, tmpname)
if b.IsNotExist(err) {
debug.Log("error %v: creating dir", err)
// error is caused by a missing directory, try to create it
mkdirErr := fs.MkdirAll(dir, backend.Modes.Dir)
if mkdirErr != nil {
debug.Log("error creating dir %v: %v", dir, mkdirErr)
} else {
// try again
f, err = tempFile(dir, tmpname)
}
}
if err != nil {
return errors.WithStack(err)
}
defer func(f *os.File) {
if err != nil {
_ = f.Close() // Double Close is harmless.
// Remove after Rename is harmless: we embed the final name in the
// temporary's name and no other goroutine will get the same data to
// Save, so the temporary name should never be reused by another
// goroutine.
_ = fs.Remove(f.Name())
}
}(f)
// save data, then sync
wbytes, err := io.Copy(f, rd)
if err != nil {
return errors.WithStack(err)
}
// sanity check
if wbytes != rd.Length() {
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
}
// Ignore error if filesystem does not support fsync.
err = f.Sync()
syncNotSup := errors.Is(err, syscall.ENOTSUP)
if err != nil && !syncNotSup {
return errors.WithStack(err)
}
// Close, then rename. Windows doesn't like the reverse order.
if err = f.Close(); err != nil {
return errors.WithStack(err)
}
if err = os.Rename(f.Name(), finalname); err != nil {
return errors.WithStack(err)
}
// Now sync the directory to commit the Rename.
if !syncNotSup {
err = fsyncDir(dir)
if err != nil {
return errors.WithStack(err)
}
}
// try to mark file as read-only to avoid accidential modifications
// ignore if the operation fails as some filesystems don't allow the chmod call
// e.g. exfat and network file systems with certain mount options
err = setFileReadonly(finalname, backend.Modes.File)
if err != nil && !os.IsPermission(err) {
return errors.WithStack(err)
}
return nil
}
var tempFile = ioutil.TempFile // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}
func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, backoff.Permanent(err)
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
f, err := fs.Open(b.Filename(h))
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(offset, 0)
if err != nil {
_ = f.Close()
return nil, err
}
}
if length > 0 {
return backend.LimitReadCloser(f, int64(length)), nil
}
return f, nil
}
// Stat returns information about a blob.
func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
debug.Log("Stat %v", h)
if err := h.Valid(); err != nil {
return restic.FileInfo{}, backoff.Permanent(err)
}
fi, err := fs.Stat(b.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
debug.Log("Test %v", h)
_, err := fs.Stat(b.Filename(h))
if err != nil {
if b.IsNotExist(err) {
return false, nil
}
return false, errors.WithStack(err)
}
return true, nil
}
// Remove removes the blob with the given name and type.
func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove %v", h)
fn := b.Filename(h)
// reset read-only flag
err := fs.Chmod(fn, 0666)
if err != nil && !os.IsPermission(err) {
return errors.WithStack(err)
}
return fs.Remove(fn)
}
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) {
debug.Log("List %v", t)
basedir, subdirs := b.Basedir(t)
if subdirs {
err = visitDirs(ctx, basedir, fn)
} else {
err = visitFiles(ctx, basedir, fn, false)
}
if b.IsNotExist(err) {
debug.Log("ignoring non-existing directory")
return nil
}
return err
}
// The following two functions are like filepath.Walk, but visit only one or
// two levels of directory structure (including dir itself as the first level).
// Also, visitDirs assumes it sees a directory full of directories, while
// visitFiles wants a directory full or regular files.
func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error {
d, err := fs.Open(dir)
if err != nil {
return err
}
sub, err := d.Readdirnames(-1)
if err != nil {
// ignore subsequent errors
_ = d.Close()
return err
}
err = d.Close()
if err != nil {
return err
}
for _, f := range sub {
err = visitFiles(ctx, filepath.Join(dir, f), fn, true)
if err != nil {
return err
}
}
return ctx.Err()
}
func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error {
d, err := fs.Open(dir)
if err != nil {
return err
}
if ignoreNotADirectory {
fi, err := d.Stat()
if err != nil || !fi.IsDir() {
// ignore subsequent errors
_ = d.Close()
return err
}
}
sub, err := d.Readdir(-1)
if err != nil {
// ignore subsequent errors
_ = d.Close()
return err
}
err = d.Close()
if err != nil {
return err
}
for _, fi := range sub {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
err := fn(restic.FileInfo{
Name: fi.Name(),
Size: fi.Size(),
})
if err != nil {
return err
}
}
return nil
}
// Delete removes the repository and all files.
func (b *Local) Delete(ctx context.Context) error {
debug.Log("Delete()")
return fs.RemoveAll(b.Path)
}
// Close closes all open files.
func (b *Local) Close() error {
debug.Log("Close()")
// this does not need to do anything, all open files are closed within the
// same function.
return nil
}