/
retry.go
123 lines (103 loc) · 3.01 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
package decorator
import (
"github.com/Clymene-project/Clymene/cmd/ingester/app/processor"
"io"
"math/rand"
"time"
"github.com/uber/jaeger-lib/metrics"
)
type retryDecorator struct {
processor processor.MetricProcessor
retryAttempts metrics.Counter
exhausted metrics.Counter
options retryOptions
io.Closer
}
// RetryOption allows setting options for exponential backoff retried
type RetryOption func(*retryOptions)
type retryOptions struct {
minInterval, maxInterval time.Duration
maxAttempts uint
propagateError bool
rand randInt63
}
type randInt63 interface {
Int63n(int64) int64
}
var defaultOpts = retryOptions{
minInterval: time.Second,
maxInterval: 1 * time.Minute,
maxAttempts: 1,
propagateError: false,
rand: rand.New(rand.NewSource(time.Now().UnixNano())),
}
// MinBackoffInterval sets the minimum backoff interval
func MinBackoffInterval(t time.Duration) RetryOption {
return func(opt *retryOptions) {
opt.minInterval = t
}
}
// MaxAttempts sets the maximum number of attempts to retry
func MaxAttempts(attempts uint) RetryOption {
return func(opt *retryOptions) {
opt.maxAttempts = attempts
}
}
// MaxBackoffInterval sets the maximum backoff interval
func MaxBackoffInterval(t time.Duration) RetryOption {
return func(opt *retryOptions) {
opt.maxInterval = t
}
}
// Rand sets a random number generator
func Rand(r randInt63) RetryOption {
return func(opt *retryOptions) {
opt.rand = r
}
}
// PropagateError sets whether to propagate errors when retries are exhausted
func PropagateError(b bool) RetryOption {
return func(opt *retryOptions) {
opt.propagateError = b
}
}
// NewRetryingProcessor returns a processor that retries failures using an exponential backoff
// with jitter.
func NewRetryingProcessor(f metrics.Factory, processor processor.MetricProcessor, opts ...RetryOption) processor.MetricProcessor {
options := defaultOpts
for _, opt := range opts {
opt(&options)
}
m := f.Namespace(metrics.NSOptions{Name: "metric-processor", Tags: nil})
return &retryDecorator{
retryAttempts: m.Counter(metrics.Options{Name: "retry-attempts", Tags: nil}),
exhausted: m.Counter(metrics.Options{Name: "retry-exhausted", Tags: nil}),
processor: processor,
options: options,
}
}
func (d *retryDecorator) Process(message processor.Message) error {
err := d.processor.Process(message)
if err == nil {
return nil
}
for attempts := uint(0); err != nil && d.options.maxAttempts > attempts; attempts++ {
time.Sleep(d.computeInterval(attempts))
err = d.processor.Process(message)
d.retryAttempts.Inc(1)
}
if err != nil {
d.exhausted.Inc(1)
if d.options.propagateError {
return err
}
}
return nil
}
func (d *retryDecorator) computeInterval(attempts uint) time.Duration {
dur := (1 << attempts) * d.options.minInterval.Nanoseconds()
if dur <= 0 || dur > d.options.maxInterval.Nanoseconds() {
dur = d.options.maxInterval.Nanoseconds()
}
return time.Duration(d.options.rand.Int63n(dur))
}