-
Notifications
You must be signed in to change notification settings - Fork 42
/
expbackoff.go
237 lines (209 loc) · 6.02 KB
/
expbackoff.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
// Package expbackoff implements exponential backoff.
//
// It was copied from the GPL version of Obol which was
// originally copied from google.golang.org/grpc.
//
// See:
// - https://github.com/grpc/grpc-go/tree/master/backoff
// - https://github.com/ObolNetwork/charon/tree/v0.14.0/app/expbackoff
//
//nolint:gochecknoglobals // Default config and test-alias globals are ok.
package expbackoff
import (
"context"
"math/rand"
"testing"
"time"
)
// Config defines the configuration options for backoff.
type Config struct {
// BaseDelay is the amount of time to backoff after the first failure.
BaseDelay time.Duration
// Multiplier is the factor with which to multiply backoffs after a
// failed retry. Should ideally be greater than 1.
Multiplier float64
// Jitter is the factor with which backoffs are randomized.
Jitter float64
// MaxDelay is the upper bound of backoff delay.
MaxDelay time.Duration
}
// DefaultConfig is a backoff configuration with the default values specified
// at https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
//
// This should be useful for callers who want to configure backoff with
// non-default values only for a subset of the options.
//
// Copied from google.golang.org/grpc@v1.48.0/backoff/backoff.go.
var DefaultConfig = Config{
BaseDelay: 1.0 * time.Second,
Multiplier: 1.6,
Jitter: 0.2,
MaxDelay: 120 * time.Second,
}
// FastConfig is a common configuration for fast backoff.
var FastConfig = Config{
BaseDelay: 100 * time.Millisecond,
Multiplier: 1.6,
Jitter: 0.2,
MaxDelay: 5 * time.Second,
}
// WithPeriodicConfig configures the backoff with periodic backoff.
func WithPeriodicConfig(period time.Duration) func(*Config) {
return func(config *Config) {
config.BaseDelay = period
config.Multiplier = 1
}
}
// WithFastConfig configures the backoff with FastConfig.
func WithFastConfig() func(*Config) {
return func(config *Config) {
*config = FastConfig
}
}
// With configures the backoff with the provided config.
func With(c Config) func(*Config) {
return func(config *Config) {
*config = c
}
}
// New returns a backoff function configured via functional options applied to DefaultConfig.
// The backoff function will exponentially sleep longer each time it is called.
// The backoff function returns immediately after the context is canceled.
//
// Usage:
//
// backoff := expbackoff.New(ctx)
// for ctx.Err() == nil {
// resp, err := doThing(ctx)
// if err != nil {
// backoff()
// continue
// } else {
// return resp
// }
// }
//
//nolint:nonamedreturns // Named returns used for clear code.
func New(ctx context.Context, opts ...func(*Config)) (backoff func()) {
backoff, _ = NewWithReset(ctx, opts...)
return backoff
}
// NewWithReset returns a backoff and a reset function configured via functional options applied to DefaultConfig.
// The backoff function will exponentially sleep longer each time it is called.
// Calling the reset function will reset the backoff sleep duration to Config.BaseDelay.
// The backoff function returns immediately after the context is canceled.
//
// Usage:
//
// backoff, reset := expbackoff.NewWithReset(ctx)
// for ctx.Err() == nil {
// resp, err := doThing(ctx)
// if err != nil {
// backoff()
// continue
// } else {
// reset()
// // Do something with the response.
// }
// }
//
//nolint:nonamedreturns // Named returns used for clear code.
func NewWithReset(ctx context.Context, opts ...func(*Config)) (backoff func(), reset func()) {
conf := DefaultConfig
for _, opt := range opts {
opt(&conf)
}
var retries int
backoff = func() {
if ctx.Err() != nil {
return
}
select {
case <-ctx.Done():
case <-after(Backoff(conf, retries)):
}
retries++
}
reset = func() {
retries = 0
}
return backoff, reset
}
// NewWithAutoReset returns a backoff function configured via functional options applied to DefaultConfig.
// The backoff function will exponentially sleep longer each time it is called.
// The backoff function is automatically reset if sufficient time has passed since the last backoff.
//
// This "sufficient delay" is the next backoff duration, so if the next backoff duration is 1s,
// the backoff will reset to initial duration after 1s of not being called.
//
//nolint:nonamedreturns // Named returns used for clear code.
func NewWithAutoReset(ctx context.Context, opts ...func(*Config)) (backoff func()) {
conf := DefaultConfig
for _, opt := range opts {
opt(&conf)
}
var retries int
lastBackoff := time.Now()
backoff = func() {
if ctx.Err() != nil {
return
}
autoResetAt := lastBackoff.Add(Backoff(conf, retries))
if time.Now().After(autoResetAt) {
retries = 0
}
select {
case <-ctx.Done():
case <-after(Backoff(conf, retries)):
}
retries++
lastBackoff = time.Now()
}
return backoff
}
// Backoff returns the amount of time to wait before the next retry given the
// number of retries.
// Copied from google.golang.org/grpc@v1.48.0/internal/backoff/backoff.go.
func Backoff(config Config, retries int) time.Duration {
if retries == 0 {
return config.BaseDelay
}
backoff := float64(config.BaseDelay)
max := float64(config.MaxDelay)
for backoff < max && retries > 0 {
backoff *= config.Multiplier
retries--
}
if backoff > max {
backoff = max
}
// Randomize backoff delays so that if a cluster of requests start at
// the same time, they won't operate in lockstep.
backoff *= 1 + config.Jitter*(randFloat()*2-1)
if backoff < 0 {
return 0
}
return time.Duration(backoff)
}
// after is aliased for testing.
var after = time.After
// SetAfterForT sets the after internal function for testing.
func SetAfterForT(t *testing.T, fn func(d time.Duration) <-chan time.Time) {
t.Helper()
cached := after
after = fn
t.Cleanup(func() {
after = cached
})
}
// randFloat is aliased for testing.
var randFloat = rand.Float64
// SetRandFloatForT sets the random float internal function for testing.
func SetRandFloatForT(t *testing.T, fn func() float64) {
t.Helper()
cached := randFloat
randFloat = fn
t.Cleanup(func() {
randFloat = cached
})
}