-
Notifications
You must be signed in to change notification settings - Fork 15
/
retry.go
366 lines (335 loc) · 11.6 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
/*
* Copyright 2018-2023, CS Systemes d'Information, http://csgroup.eu
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net
import (
"errors"
"net"
"net/url"
"os"
"reflect"
"runtime"
"strings"
"syscall"
"time"
"github.com/CS-SI/SafeScale/v22/lib/backend/iaas/objectstorage"
"github.com/sirupsen/logrus"
"github.com/CS-SI/SafeScale/v22/lib/utils/debug"
"github.com/CS-SI/SafeScale/v22/lib/utils/fail"
"github.com/CS-SI/SafeScale/v22/lib/utils/retry"
"github.com/CS-SI/SafeScale/v22/lib/utils/retry/enums/verdict"
"github.com/CS-SI/SafeScale/v22/lib/utils/temporal"
)
// WhileUnsuccessfulButRetryable executes callback inside a retry loop with tolerance for communication errors (relative to net package),
// or some fail.Error that are considered retryable: asking "waiter" to wait between each try, with a duration limit of 'timeout'.
func WhileUnsuccessfulButRetryable(callback func() error, waiter *retry.Officer, timeout time.Duration, options ...retry.Option) fail.Error {
if waiter == nil {
return fail.InvalidParameterCannotBeNilError("waiter")
}
var arbiter retry.Arbiter
if timeout <= 0 {
arbiter = retry.Unsuccessful()
} else {
arbiter = retry.PrevailDone(retry.Unsuccessful(), retry.Timeout(timeout))
}
preparedAction := retry.NewAction(
waiter,
arbiter,
nil,
func(t retry.Try, v verdict.Enum) {
switch v {
case verdict.Retry:
logrus.Warnf("communication failed (%s), retrying", t.Err.Error())
default:
}
},
0,
)
for _, opt := range options { // this should change preparedAction.Other if needed
err := opt(preparedAction)
if err != nil {
return fail.ConvertError(err)
}
}
strict := false
if val, ok := preparedAction.Other["strict"]; ok {
if itis, ok := val.(bool); ok {
strict = itis
}
}
preparedAction.Run = func() (nested error) {
defer fail.OnPanic(&nested)
callbackErr := callback()
actionErr := normalizeErrorAndCheckIfRetriable(strict, callbackErr)
return actionErr
}
xerr := retry.Action(
preparedAction.Run,
preparedAction.Arbiter,
preparedAction.Officer,
nil,
nil,
preparedAction.Notify,
)
if xerr != nil {
switch realErr := xerr.(type) {
case *retry.ErrStopRetry:
return fail.Wrap(fail.Cause(realErr), "stopping retries")
case *retry.ErrTimeout:
return fail.Wrap(fail.Cause(realErr), "timeout")
default:
return xerr
}
}
return nil
}
// WhileCommunicationUnsuccessfulDelay1Second executes callback inside a retry loop with tolerance for communication errors (relative to net package),
// waiting 1 second between each try, with a limit of 'timeout'
func WhileCommunicationUnsuccessfulDelay1Second(callback func() error, timeout time.Duration) fail.Error {
return WhileUnsuccessfulButRetryable(callback, retry.Constant(temporal.MinDelay()), timeout)
}
func isRaw(in error) bool {
switch intype := in.(type) {
case *url.Error:
if reflect.TypeOf(intype.Err).String() == reflect.TypeOf(errors.New("")).String() {
return true
}
return false
default:
return false
}
}
// normalizeErrorAndCheckIfRetriable analyzes the error passed as parameter and rewrite it to be more explicit
// If the error is not a communication error, we return a *retry.ErrAborted error
// containing the causing error in it
func normalizeErrorAndCheckIfRetriable(strict bool, in error) (err error) { // nolint
defer func() {
if err != nil {
switch err.(type) {
case fail.ErrInvalidRequest, *fail.ErrInvalidRequest:
logrus.Warnf(err.Error())
default:
debug.IgnoreError(err)
}
}
}()
if in != nil {
if strings.Contains(in.Error(), "context canceled") {
return retry.StopRetryError(in)
}
switch realErr := in.(type) {
case *url.Error:
if realErr.Temporary() {
return realErr
}
return normalizeURLError(realErr)
case *net.OpError: // reset-by-peer errors will be captured here
if IsConnectionReset(realErr) { // give a chance to reset-by-peer errors
return realErr
}
if realErr.Temporary() {
return realErr
}
return retry.StopRetryError(realErr)
case net.Error:
if realErr.Temporary() || realErr.Timeout() { // nolint
return realErr
}
return retry.StopRetryError(realErr)
case fail.Error, fail.ErrNotAvailable, fail.ErrOverflow, fail.ErrOverload: // a fail.Error may contain a cause of type net.Error, being *url.Error a special subcase.
// net.Error, and by extension url.Error have methods to check if the error is temporary -Temporary()-, or it's a timeout -Timeout()-, we should use the information to make decisions
// In this case, handle those net.Error accordingly
cause := fail.Cause(realErr)
switch thecause := cause.(type) {
case *url.Error:
// errors here (*url.Error), are very low-level, its .err should be at most an errors.New() kind of error
// if we have something more complex, we are talking about hand-crafted errors that won't happen in real world
if thecause.Temporary() && isRaw(thecause) {
return realErr
}
if strict {
return normalizeURLError(thecause)
}
return realErr
case net.Error:
if thecause.Temporary() || thecause.Timeout() { // nolint
return realErr
}
if strict {
return retry.StopRetryError(realErr)
}
return realErr
case *fail.ErrNotAvailable, fail.ErrNotAvailable, *fail.ErrOverflow, fail.ErrOverflow, *fail.ErrOverload, fail.ErrOverload:
return realErr
default:
return retry.StopRetryError(realErr)
}
default:
// doing something based on error's Error() method is always dangerous, so a little log here might help to find problems later
logrus.Tracef("trying to normalize based on Error() string of: (%s): %v", reflect.TypeOf(in).String(), in)
str := in.Error()
switch str {
case objectstorage.NotFound: // stow may return that error message if it does not find something
return fail.NotFoundError(objectstorage.NotFound)
default: // stow may return an error containing "dial tcp:" for some HTTP errors
if strings.Contains(str, "dial tcp:") { // FIXME: This should be a constant
return fail.NotAvailableError(str)
}
if strings.Contains(str, "EOF") { // stow may return that error message if comm fails // FIXME: Also a constant
return fail.NotAvailableError("encountered end-of-file")
}
// In any other case, the error should explain the retry has to stop
return retry.StopRetryError(in)
}
}
}
return nil
}
// normalizeErrorAndCheckIfRetriable analyzes the error passed as parameter and rewrite it to be more explicit
// If the error is not a communication error, we return a *retry.ErrAborted error
// containing the causing error in it
func oldNormalizeErrorAndCheckIfRetriable(in error) (err error) {
defer func() {
if err != nil {
switch err.(type) {
case fail.ErrInvalidRequest, *fail.ErrInvalidRequest:
logrus.Warnf(err.Error())
default:
debug.IgnoreError(err)
}
}
}()
if in != nil {
switch realErr := in.(type) {
case *url.Error:
return normalizeURLError(realErr)
case fail.Error: // a fail.Error may contain a cause of type net.Error, being *url.Error a special subcase.
// net.Error, and by extension url.Error have methods to check if the error is temporary -Temporary()-, or it's a timeout -Timeout()-, we should use the information to make decisions
// In this case, handle those net.Error accordingly
if realErr.Cause() != nil {
switch cause := realErr.Cause().(type) { // nolint
case *url.Error:
return normalizeURLError(cause)
case net.Error:
return realErr
// If error is *fail.ErrNotAvailable, *fail.ErrOverflow or *fail.ErrOverload, leave a chance to retry
case *fail.ErrNotAvailable, fail.ErrNotAvailable, *fail.ErrOverflow, fail.ErrOverflow, *fail.ErrOverload, fail.ErrOverload:
return realErr
}
} else {
switch realErr.(type) { // nolint
// If error is *fail.ErrNotAvailable, *fail.ErrOverflow or *fail.ErrOverload, leave a chance to retry
case *fail.ErrNotAvailable, *fail.ErrOverflow, *fail.ErrOverload:
return realErr
case net.Error: // this also covers *url.Error, if the URL needs a specific error treatment, add a case BEFORE this line
return realErr
}
}
// If error is *fail.ErrNotAvailable, *fail.ErrOverflow or *fail.ErrOverload, leave a chance to retry
switch realErr.(type) {
case *fail.ErrNotAvailable, *fail.ErrOverflow, *fail.ErrOverload:
return realErr
default:
return retry.StopRetryError(realErr)
}
default:
// doing something based on error's Error() method is always dangerous, so a little log here might help to find problems later
logrus.Tracef("trying to normalize based on Error() string of: (%s): %v", reflect.TypeOf(in).String(), in)
str := in.Error()
switch str {
case objectstorage.NotFound: // stow may return that error message if it does not find something
return fail.NotFoundError(objectstorage.NotFound)
default: // stow may return an error containing "dial tcp:" for some HTTP errors
if strings.Contains(str, "dial tcp:") {
logrus.Tracef("encountered 'dial tcp' error")
return fail.NotAvailableError(str)
}
if strings.Contains(str, "EOF") { // stow may return that error message if comm fails
logrus.Tracef("encountered end-of-file")
return fail.NotAvailableError("encountered end-of-file")
}
// In any other case, the error should explain the retry has to stop
return retry.StopRetryError(in)
}
}
}
return nil
}
func normalizeURLError(err *url.Error) fail.Error {
if err != nil {
isTemporary := err.Temporary()
if err.Err != nil {
switch commErr := err.Err.(type) {
case *net.DNSError:
if isTemporary {
return fail.InvalidRequestError("failed to resolve by DNS: %v", commErr)
}
return retry.StopRetryError(commErr, "failed to resolve by DNS")
default:
if isTemporary {
if commErr != nil {
return fail.InvalidRequestError("failed to communicate (error type: %s): %v", reflect.TypeOf(commErr).String(), commErr)
}
return fail.InvalidRequestError("failed to communicate: %v", commErr)
}
return retry.StopRetryError(err)
}
}
return retry.StopRetryError(err)
}
return nil
}
func erz(v error) uintptr {
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Uintptr {
return uintptr(rv.Uint())
}
return 0
}
// IsConnectionReset returns true if given err is a "reset by peer" error
func IsConnectionReset(err error) bool {
if runtime.GOOS == "windows" {
const cWSAECONNABORTED = 10053
const cWSAECONNRESET = 10054
if oe, ok := err.(*net.OpError); ok {
if oe.Op == "read" {
if se, ok := oe.Err.(*os.SyscallError); ok {
if se.Syscall == "wsarecv" {
if n := erz(se.Err); n == cWSAECONNRESET || n == cWSAECONNABORTED {
return true
}
}
}
}
}
var errno syscall.Errno
if errors.As(err, &errno) {
return errno == syscall.ECONNRESET
}
return false
}
var errno syscall.Errno
if errors.As(err, &errno) {
return errno == syscall.ECONNRESET
}
return false
}
// IsConnectionRefused returns true if given err is a "connection refused" error
func IsConnectionRefused(err error) bool {
var errno syscall.Errno
if errors.As(err, &errno) {
return errno == syscall.ECONNREFUSED
}
return false
}