-
Notifications
You must be signed in to change notification settings - Fork 22
/
request_retrier.go
176 lines (160 loc) · 5.57 KB
/
request_retrier.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
// Copyright (c) 2020 Palantir Technologies. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"net/http"
"net/url"
"strings"
"github.com/palantir/pkg/retry"
)
const (
meshSchemePrefix = "mesh-"
)
// RequestRetrier manages URIs for an HTTP client, providing an API which determines whether requests should be retries
// and supplying the correct URL for the client to retry.
// In the case of servers in a service-mesh, requests will never be retried and the mesh URI will only be returned on the
// first call to GetNextURI
type RequestRetrier struct {
currentURI string
retrier retry.Retrier
uris []string
offset int
relocatedURIs map[string]struct{}
failedURIs map[string]struct{}
maxAttempts int
attemptCount int
}
// NewRequestRetrier creates a new request retrier.
// Regardless of maxAttempts, mesh URIs will never be retried.
func NewRequestRetrier(uris []string, retrier retry.Retrier, maxAttempts int) *RequestRetrier {
offset := 0
return &RequestRetrier{
currentURI: uris[offset],
retrier: retrier,
uris: uris,
offset: offset,
relocatedURIs: map[string]struct{}{},
failedURIs: map[string]struct{}{},
maxAttempts: maxAttempts,
attemptCount: 0,
}
}
func (r *RequestRetrier) attemptsRemaining() bool {
// maxAttempts of 0 indicates no limit
if r.maxAttempts == 0 {
return true
}
return r.attemptCount < r.maxAttempts
}
// GetNextURI returns the next URI a client should use, or empty string if no suitable URI remaining to retry.
// isRelocated is true when the URI comes from a redirect's Location header. In this case, it already includes the request path.
func (r *RequestRetrier) GetNextURI(resp *http.Response, respErr error) (uri string, isRelocated bool) {
defer func() {
r.attemptCount++
}()
if r.attemptCount == 0 {
// First attempt is always successful. Trigger the first retry so later calls have backoff
// but ignore the returned value to ensure that the client can instrument the request even
// if the context is done.
r.retrier.Next()
return r.removeMeshSchemeIfPresent(r.currentURI), false
}
if !r.attemptsRemaining() {
// Retries exhausted
return "", false
}
if r.isMeshURI(r.currentURI) {
// Mesh uris don't get retried
return "", false
}
retryFn := r.getRetryFn(resp, respErr)
if retryFn == nil {
// The previous response was not retryable
return "", false
}
// Updates currentURI
if !retryFn() {
return "", false
}
return r.currentURI, r.isRelocatedURI(r.currentURI)
}
func (r *RequestRetrier) getRetryFn(resp *http.Response, respErr error) func() bool {
errCode, _ := StatusCodeFromError(respErr)
if retryOther, _ := isThrottleResponse(resp, errCode); retryOther {
// 429: throttle
// Immediately backoff and select the next URI.
// TODO(whickman): use the retry-after header once #81 is resolved
return r.nextURIAndBackoff
} else if isUnavailableResponse(resp, errCode) {
// 503: go to next node
return r.nextURIOrBackoff
} else if shouldTryOther, otherURI := isRetryOtherResponse(resp, respErr, errCode); shouldTryOther {
// 307 or 308: go to next node, or particular node if provided.
if otherURI != nil {
return func() bool {
r.setURIAndResetBackoff(otherURI)
return true
}
}
return r.nextURIOrBackoff
} else if errCode >= http.StatusBadRequest && errCode < http.StatusInternalServerError {
return nil
} else if resp == nil {
// if we get a nil response, we can assume there is a problem with host and can move on to the next.
return r.nextURIOrBackoff
}
return nil
}
func (r *RequestRetrier) setURIAndResetBackoff(otherURI *url.URL) {
nextURI := otherURI.String()
r.relocatedURIs[otherURI.String()] = struct{}{}
r.retrier.Reset()
r.currentURI = nextURI
}
// If lastURI was already marked failed, we perform a backoff as determined by the retrier before returning the next URI and its offset.
// Otherwise, we add lastURI to failedURIs and return the next URI and its offset immediately.
func (r *RequestRetrier) nextURIOrBackoff() bool {
_, performBackoff := r.failedURIs[r.currentURI]
r.markFailedAndMoveToNextURI()
// If the URI has failed before, perform a backoff
if performBackoff || len(r.uris) == 1 {
return r.retrier.Next()
}
return true
}
// Marks the current URI as failed, gets the next URI, and performs a backoff as determined by the retrier.
func (r *RequestRetrier) nextURIAndBackoff() bool {
r.markFailedAndMoveToNextURI()
return r.retrier.Next()
}
func (r *RequestRetrier) markFailedAndMoveToNextURI() {
r.failedURIs[r.currentURI] = struct{}{}
nextURIOffset := (r.offset + 1) % len(r.uris)
nextURI := r.uris[nextURIOffset]
r.currentURI = nextURI
r.offset = nextURIOffset
}
func (r *RequestRetrier) removeMeshSchemeIfPresent(uri string) string {
if r.isMeshURI(uri) {
return strings.Replace(uri, meshSchemePrefix, "", 1)
}
return uri
}
func (r *RequestRetrier) isMeshURI(uri string) bool {
return strings.HasPrefix(uri, meshSchemePrefix)
}
func (r *RequestRetrier) isRelocatedURI(uri string) bool {
_, relocatedURI := r.relocatedURIs[uri]
return relocatedURI
}