forked from restic/restic
-
Notifications
You must be signed in to change notification settings - Fork 2
/
backend_retry.go
144 lines (122 loc) · 3.99 KB
/
backend_retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package backend
import (
"context"
"fmt"
"io"
"time"
"github.com/cenkalti/backoff"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// RetryBackend retries operations on the backend in case of an error with a
// backoff.
type RetryBackend struct {
restic.Backend
MaxTries int
Report func(string, error, time.Duration)
}
// statically ensure that RetryBackend implements restic.Backend.
var _ restic.Backend = &RetryBackend{}
// NewRetryBackend wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred.
func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error, time.Duration)) *RetryBackend {
return &RetryBackend{
Backend: be,
MaxTries: maxTries,
Report: report,
}
}
func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error {
err := backoff.RetryNotify(f,
backoff.WithContext(backoff.WithMaxTries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), ctx),
func(err error, d time.Duration) {
if be.Report != nil {
be.Report(msg, err, d)
}
},
)
return err
}
// Save stores the data in the backend under the given handle.
func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error {
seeker, ok := rd.(io.Seeker)
if !ok {
return errors.Errorf("reader %T is not a seeker", rd)
}
pos, err := seeker.Seek(0, io.SeekCurrent)
if err != nil {
return errors.Wrap(err, "Seek")
}
if pos != 0 {
return errors.Errorf("reader is not at the beginning (pos %v)", pos)
}
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
_, err := seeker.Seek(0, io.SeekStart)
if err != nil {
return err
}
err = be.Backend.Save(ctx, h, rd)
if err == nil {
return nil
}
debug.Log("Save(%v) failed with error, removing file: %v", h, err)
rerr := be.Backend.Remove(ctx, h)
if rerr != nil {
debug.Log("Remove(%v) returned error: %v", h, err)
}
// return original error
return err
})
}
// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil.
func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (rd io.ReadCloser, err error) {
err = be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error {
var innerError error
rd, innerError = be.Backend.Load(ctx, h, length, offset)
return innerError
})
return rd, err
}
// Stat returns information about the File identified by h.
func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error {
var innerError error
fi, innerError = be.Backend.Stat(ctx, h)
return innerError
})
return fi, err
}
// Remove removes a File with type t and name.
func (be *RetryBackend) Remove(ctx context.Context, h restic.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h)
})
}
// Test a boolean value whether a File with the name and type exists.
func (be *RetryBackend) Test(ctx context.Context, h restic.Handle) (exists bool, err error) {
err = be.retry(ctx, fmt.Sprintf("Test(%v)", h), func() error {
var innerError error
exists, innerError = be.Backend.Test(ctx, h)
return innerError
})
return exists, err
}
// List runs fn for each file in the backend which has the type t.
func (be *RetryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
listed := make(map[string]struct{})
return be.retry(ctx, fmt.Sprintf("List(%v)", t), func() error {
return be.Backend.List(ctx, t, func(fi restic.FileInfo) error {
if _, ok := listed[fi.Name]; ok {
return nil
}
listed[fi.Name] = struct{}{}
return fn(fi)
})
})
}