/
retry.go
238 lines (207 loc) · 8.98 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
// Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
package provider
import (
"math/rand"
"strings"
"time"
oci_common "github.com/oracle/oci-go-sdk/common"
"github.com/terraform-providers/terraform-provider-oci/httpreplay"
)
const (
quadraticBackoffCap = 12 // This corresponds to a 2*12*12=288 second cap on retry wait times (~5 minutes)
minRetryBackoff = 1 * time.Second // Must wait for at least 1 second before retrying
databaseService = "database"
identityService = "identity"
coreService = "core"
waasService = "waas"
kmsService = "kms"
objectstorageService = "object_storage"
deleteResource = "delete"
updateResource = "update"
createResource = "create"
getResource = "get"
)
type expectedRetryDurationFn func(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, optionals ...interface{}) time.Duration
type serviceExpectedRetryDurationFunc func(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, optionals ...interface{}) time.Duration
type getRetryPolicyFunc func(disableNotFoundRetries bool, service string, optionals ...interface{}) *oci_common.RetryPolicy
var serviceExpectedRetryDurationMap = map[string]serviceExpectedRetryDurationFunc{
coreService: getCoreExpectedRetryDuration,
databaseService: getDatabaseExpectedRetryDuration,
identityService: getIdentityExpectedRetryDuration,
objectstorageService: getObjectstorageServiceExpectedRetryDuration,
waasService: getWaasExpectedRetryDuration,
}
var serviceRetryPolicyFnMap = map[string]getRetryPolicyFunc{
kmsService: kmsGetRetryPolicy,
}
var shortRetryTime = 2 * time.Minute
var longRetryTime = 10 * time.Minute
var configuredRetryDuration *time.Duration
func init() {
rand.Seed(time.Now().UnixNano())
}
func getRetryBackoffDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, startTime time.Time, optionals ...interface{}) time.Duration {
return getRetryBackoffDurationWithExpectedRetryDurationFn(response, disableNotFoundRetries, service, startTime, getExpectedRetryDuration, optionals...)
}
func getRetryBackoffDurationWithExpectedRetryDurationFn(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, startTime time.Time, expectedRetryDurationFn expectedRetryDurationFn, optionals ...interface{}) time.Duration {
if httpreplay.ShouldRetryImmediately() {
return 0
}
// Avoid having a very large retry backoff
attempt := response.AttemptNumber
if attempt > quadraticBackoffCap {
attempt = quadraticBackoffCap
}
retryBackoffRange := time.Duration(2*attempt*attempt)*time.Second - minRetryBackoff
// Jitter the backoff time. The actual backoff time might be anywhere within the minimum and quadratic backoff time to avoid clustering.
backoffDuration := time.Duration(rand.Int63n(int64(retryBackoffRange+1))) + minRetryBackoff
// If we are about to exceed the retry duration; then reduce the backoff so that next attempt happens roughly when
// the entire retry duration is supposed to expire. Jitter is necessary again to avoid clustering.
expectedRetryDuration := expectedRetryDurationFn(response, disableNotFoundRetries, service, optionals...)
timeWaited := getElapsedRetryDuration(startTime)
if timeWaited < expectedRetryDuration && timeWaited+backoffDuration > expectedRetryDuration {
extraJitterRange := int64(float64(expectedRetryDuration) * 0.05)
finalBackoffDuration := expectedRetryDuration - timeWaited + time.Duration(rand.Int63n(extraJitterRange+1)) + minRetryBackoff
if finalBackoffDuration < backoffDuration {
backoffDuration = finalBackoffDuration
}
}
return backoffDuration
}
func getElapsedRetryDuration(firstAttemptTime time.Time) time.Duration {
return time.Now().Sub(firstAttemptTime)
}
func getExpectedRetryDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, optionals ...interface{}) time.Duration {
if retryDurationFn, ok := serviceExpectedRetryDurationMap[service]; ok {
return retryDurationFn(response, disableNotFoundRetries, optionals...)
}
return getDefaultExpectedRetryDuration(response, disableNotFoundRetries)
}
func getDefaultExpectedRetryDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool) time.Duration {
defaultRetryTime := shortRetryTime
if response.Response == nil || response.Response.HTTPResponse() == nil {
return 0
}
statusCode := response.Response.HTTPResponse().StatusCode
e := response.Error
if statusCode >= 200 && statusCode < 300 {
return 0
}
switch statusCode {
case 400, 401, 403:
return 0
case 404:
if disableNotFoundRetries {
return 0
}
case 409:
if e != nil && strings.Contains(e.Error(), "InvalidatedRetryToken") {
return 0
}
case 412:
return 0
case 429:
if configuredRetryDuration != nil {
return *configuredRetryDuration
}
defaultRetryTime = longRetryTime
case 500:
if configuredRetryDuration != nil {
return *configuredRetryDuration
}
}
return defaultRetryTime
}
func getIdentityExpectedRetryDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, optionals ...interface{}) time.Duration {
defaultRetryTime := getDefaultExpectedRetryDuration(response, disableNotFoundRetries)
if response.Response == nil || response.Response.HTTPResponse() == nil {
return defaultRetryTime
}
switch statusCode := response.Response.HTTPResponse().StatusCode; statusCode {
case 404:
if disableNotFoundRetries {
defaultRetryTime = 0
} else {
defaultRetryTime = longRetryTime
}
case 409:
if e := response.Error; e != nil {
if strings.Contains(e.Error(), "CompartmentAlreadyExists") || strings.Contains(e.Error(), "TagDefinitionAlreadyExists") ||
strings.Contains(e.Error(), "TenantCapacityExceeded") || strings.Contains(e.Error(), "TagNamespaceAlreadyExists") ||
strings.Contains(e.Error(), "InvalidatedRetryToken") {
defaultRetryTime = 0
} else if strings.Contains(e.Error(), "NotAuthorizedOrResourceAlreadyExists") {
defaultRetryTime = longRetryTime
}
}
}
return defaultRetryTime
}
func getDatabaseExpectedRetryDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, optionals ...interface{}) time.Duration {
defaultRetryTime := getDefaultExpectedRetryDuration(response, disableNotFoundRetries)
if response.Response == nil || response.Response.HTTPResponse() == nil {
return defaultRetryTime
}
switch statusCode := response.Response.HTTPResponse().StatusCode; statusCode {
case 409:
if e := response.Error; e != nil {
if strings.Contains(e.Error(), "InvalidatedRetryToken") {
defaultRetryTime = 0
} else {
defaultRetryTime = longRetryTime
}
}
}
return defaultRetryTime
}
func getObjectstorageServiceExpectedRetryDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, optionals ...interface{}) time.Duration {
defaultRetryTime := getDefaultExpectedRetryDuration(response, disableNotFoundRetries)
if response.Response == nil || response.Response.HTTPResponse() == nil {
return defaultRetryTime
}
switch statusCode := response.Response.HTTPResponse().StatusCode; statusCode {
case 404:
if disableNotFoundRetries {
defaultRetryTime = 0
} else {
defaultRetryTime = longRetryTime
}
case 409:
if e := response.Error; e != nil {
if strings.Contains(e.Error(), "NotAuthorizedOrResourceAlreadyExists") {
defaultRetryTime = longRetryTime
}
}
case 500:
if configuredRetryDuration != nil {
defaultRetryTime = *configuredRetryDuration
} else {
defaultRetryTime = longRetryTime
}
}
return defaultRetryTime
}
func shouldRetry(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, startTime time.Time, optionals ...interface{}) bool {
return getElapsedRetryDuration(startTime) < getExpectedRetryDuration(response, disableNotFoundRetries, service, optionals...)
}
// Because this function notes the start time for making should retry decisions, it's advised
// for this function call to be made immediately before the client API call.
func getRetryPolicy(disableNotFoundRetries bool, service string, optionals ...interface{}) *oci_common.RetryPolicy {
if serviceRetryPolicyFn, ok := serviceRetryPolicyFnMap[service]; ok {
return serviceRetryPolicyFn(disableNotFoundRetries, service, optionals...)
}
return getDefaultRetryPolicy(disableNotFoundRetries, service, optionals...)
}
func getDefaultRetryPolicy(disableNotFoundRetries bool, service string, optionals ...interface{}) *oci_common.RetryPolicy {
startTime := time.Now()
retryPolicy := &oci_common.RetryPolicy{
MaximumNumberAttempts: 0,
ShouldRetryOperation: func(response oci_common.OCIOperationResponse) bool {
return shouldRetry(response, disableNotFoundRetries, service, startTime, optionals...)
},
NextDuration: func(response oci_common.OCIOperationResponse) time.Duration {
return getRetryBackoffDuration(response, disableNotFoundRetries, service, startTime, optionals...)
},
}
return retryPolicy
}