/
retry.go
90 lines (71 loc) · 2.18 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
package interceptors
import (
"context"
"strings"
"time"
"github.com/rs/zerolog/log"
"google.golang.org/grpc"
)
type ctxLastErrorMarker struct{}
var ctxLastErrorKey = &ctxLastErrorMarker{}
// RetryInterceptor will retry a failed request according to its checker decision and time budget.
type RetryInterceptor struct {
shouldRetry RetryChecker
retryBudget time.Duration
}
func NewRetryInterceptor(retryBudget time.Duration, checker RetryChecker) *RetryInterceptor {
return &RetryInterceptor{
retryBudget: retryBudget,
shouldRetry: checker,
}
}
// RetryChecker determines if a request should be retried.
// It receives the returned error and elapsed time since first attempt.
type RetryChecker = func(error) bool
func (ri *RetryInterceptor) UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
// Passthrough for ignored methods
for _, m := range IgnoredMethods {
if strings.Contains(info.FullMethod, m) {
return handler(ctx, req)
}
}
attempt := 1
start := time.Now()
retryCtx := ctx
for {
attemptStart := time.Now()
res, err := handler(retryCtx, req)
attemptDuration := time.Since(attemptStart)
retryCtx = WithLastError(retryCtx, err)
logger := log.Ctx(ctx).With().Str("method", info.FullMethod).Dur("attempt_duration", attemptDuration).Int("attempt", attempt).Logger()
if err == nil {
return res, nil
}
if !ri.budgetAllowRetry(start) {
logger.Error().Msg("retry budget exceeded")
return nil, err
}
if ri.shouldRetry(err) {
attempt++
logger.Info().Msg("retrying failed transaction")
continue
}
logger.Debug().Err(err).Msg("should not retry on this error")
return nil, err
}
}
func (ri *RetryInterceptor) budgetAllowRetry(start time.Time) bool {
return time.Since(start) <= ri.retryBudget
}
// WithLastError adds last error to the context
func WithLastError(ctx context.Context, err error) context.Context {
return context.WithValue(ctx, ctxLastErrorKey, err)
}
// GetLastError returns the last error in a retry context
func GetLastError(ctx context.Context) error {
err, ok := ctx.Value(ctxLastErrorKey).(error)
if !ok {
return nil
}
return err
}