-
Notifications
You must be signed in to change notification settings - Fork 5k
/
failureretrywrapper.go
63 lines (56 loc) · 1.88 KB
/
failureretrywrapper.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
package kube
import (
"net/http"
"time"
log "github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/client-go/rest"
)
type failureRetryRoundTripper struct {
roundTripper http.RoundTripper
failureRetryCount int
failureRetryPeriodMilliSeconds int
}
// nolint:unparam
func shouldRetry(counter int, r *http.Request, response *http.Response, err error) bool {
if counter <= 0 {
return false
}
if err != nil {
if apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) ||
apierrors.IsTooManyRequests(err) || utilnet.IsProbableEOF(err) || utilnet.IsConnectionReset(err) {
return true
}
}
if response != nil && (response.StatusCode == 504 || response.StatusCode == 503) {
return true
}
return false
}
func (frt *failureRetryRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
resp, roundTimeErr := frt.roundTripper.RoundTrip(r)
counter := frt.failureRetryCount
for shouldRetry(counter, r, resp, roundTimeErr) {
log.Debug("failureRetryRoundTripper: ", r.URL.Path, " ", r.Method)
time.Sleep(time.Duration(frt.failureRetryPeriodMilliSeconds) * time.Millisecond)
resp, roundTimeErr = frt.roundTripper.RoundTrip(r)
counter--
}
return resp, roundTimeErr
}
// AddFailureRetryWrapper adds a transport wrapper which wraps a function call around each kubernetes request
func AddFailureRetryWrapper(config *rest.Config, failureRetryCount int, failureRetryPeriodSeconds int) *rest.Config {
wrap := config.WrapTransport
config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
if wrap != nil {
rt = wrap(rt)
}
return &failureRetryRoundTripper{
roundTripper: rt,
failureRetryCount: failureRetryCount,
failureRetryPeriodMilliSeconds: failureRetryPeriodSeconds,
}
}
return config
}