Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 268 lines (212 sloc) 9.955 kb
28e56fa @rholder add even moar Python 3.x compatibility hacks from the six project, updat...
authored
1 ## Copyright 2013-2014 Ray Holder
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
2 ##
3 ## Licensed under the Apache License, Version 2.0 (the "License");
4 ## you may not use this file except in compliance with the License.
5 ## You may obtain a copy of the License at
6 ##
7 ## http://www.apache.org/licenses/LICENSE-2.0
8 ##
9 ## Unless required by applicable law or agreed to in writing, software
10 ## distributed under the License is distributed on an "AS IS" BASIS,
11 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 ## See the License for the specific language governing permissions and
13 ## limitations under the License.
14
15 import random
062c5ba @hguemar Replaced bundled code with using six
hguemar authored
16 import six
8035a32 @rholder fixed a bug where classes not extending from the Python exception hierar...
authored
17 import sys
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
18 import time
f405d7e @rholder propagate complete tracebacks such that they show up in nested stack fra...
authored
19 import traceback
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
20
28e56fa @rholder add even moar Python 3.x compatibility hacks from the six project, updat...
authored
21
e9a0bf4 @rholder Python 3.2 compatibility fix, since we don't have sys.maxint anymore...
authored
22 # sys.maxint / 2, since Python 3.2 doesn't have a sys.maxint...
23 MAX_WAIT = 1073741823
24
8ca5b41 @rholder add comments from rdooley, add comment around stop behavior
authored
25
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
26 def retry(*dargs, **dkw):
27 """
8ca5b41 @rholder add comments from rdooley, add comment around stop behavior
authored
28 Decorator function that instantiates the Retrying object
29 @param *dargs: positional arguments passed to Retrying object
30 @param **dkw: keyword arguments passed to the Retrying object
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
31 """
32 # support both @retry and @retry() as valid syntax
33 if len(dargs) == 1 and callable(dargs[0]):
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
34 def wrap_simple(f):
ee2ae2a @harlowja Ensure we wrap the decorated functions
harlowja authored
35
36 @six.wraps(f)
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
37 def wrapped_f(*args, **kw):
38 return Retrying().call(f, *args, **kw)
39
40 return wrapped_f
41
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
42 return wrap_simple(dargs[0])
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
43
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
44 else:
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
45 def wrap(f):
ee2ae2a @harlowja Ensure we wrap the decorated functions
harlowja authored
46
47 @six.wraps(f)
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
48 def wrapped_f(*args, **kw):
49 return Retrying(*dargs, **dkw).call(f, *args, **kw)
50
51 return wrapped_f
52
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
53 return wrap
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
54
55
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
56 class Retrying(object):
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
57
58 def __init__(self,
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
59 stop=None, wait=None,
60 stop_max_attempt_number=None,
61 stop_max_delay=None,
62 wait_fixed=None,
63 wait_random_min=None, wait_random_max=None,
64 wait_incrementing_start=None, wait_incrementing_increment=None,
65 wait_exponential_multiplier=None, wait_exponential_max=None,
946c3ff @rholder clean up some TODO's and comments
authored
66 retry_on_exception=None,
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
67 retry_on_result=None,
8b73aff @dnephin Support custom wait and stop functions.
dnephin authored
68 wrap_exception=False,
69 stop_func=None,
01a2173 @harlowja Allow a jitter value to be passed in
harlowja authored
70 wait_func=None,
71 wait_jitter_max=None):
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
72
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
73 self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number
74 self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay
75 self._wait_fixed = 1000 if wait_fixed is None else wait_fixed
76 self._wait_random_min = 0 if wait_random_min is None else wait_random_min
77 self._wait_random_max = 1000 if wait_random_max is None else wait_random_max
78 self._wait_incrementing_start = 0 if wait_incrementing_start is None else wait_incrementing_start
79 self._wait_incrementing_increment = 100 if wait_incrementing_increment is None else wait_incrementing_increment
80 self._wait_exponential_multiplier = 1 if wait_exponential_multiplier is None else wait_exponential_multiplier
81 self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max
01a2173 @harlowja Allow a jitter value to be passed in
harlowja authored
82 self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
83
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
84 # TODO add chaining of stop behaviors
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
85 # stop behavior
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
86 stop_funcs = []
87 if stop_max_attempt_number is not None:
88 stop_funcs.append(self.stop_after_attempt)
89
90 if stop_max_delay is not None:
91 stop_funcs.append(self.stop_after_delay)
92
8b73aff @dnephin Support custom wait and stop functions.
dnephin authored
93 if stop_func is not None:
94 self.stop = stop_func
95
96 elif stop is None:
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
97 self.stop = lambda attempts, delay: any(f(attempts, delay) for f in stop_funcs)
98
99 else:
100 self.stop = getattr(self, stop)
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
101
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
102 # TODO add chaining of wait behaviors
946c3ff @rholder clean up some TODO's and comments
authored
103 # wait behavior
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
104 wait_funcs = [lambda *args, **kwargs: 0]
105 if wait_fixed is not None:
106 wait_funcs.append(self.fixed_sleep)
107
108 if wait_random_min is not None or wait_random_max is not None:
109 wait_funcs.append(self.random_sleep)
110
111 if wait_incrementing_start is not None or wait_incrementing_increment is not None:
112 wait_funcs.append(self.incrementing_sleep)
113
114 if wait_exponential_multiplier is not None or wait_exponential_max is not None:
115 wait_funcs.append(self.exponential_sleep)
116
8b73aff @dnephin Support custom wait and stop functions.
dnephin authored
117 if wait_func is not None:
118 self.wait = wait_func
119
120 elif wait is None:
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
121 self.wait = lambda attempts, delay: max(f(attempts, delay) for f in wait_funcs)
122
123 else:
ff817d9 @alexkuang Fix AttributeError when explicitly specifying wait type, and add appropr...
alexkuang authored
124 self.wait = getattr(self, wait)
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
125
946c3ff @rholder clean up some TODO's and comments
authored
126 # retry on exception filter
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
127 if retry_on_exception is None:
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
128 self._retry_on_exception = self.always_reject
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
129 else:
130 self._retry_on_exception = retry_on_exception
131
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
132 # TODO simplify retrying by Exception types
946c3ff @rholder clean up some TODO's and comments
authored
133 # retry on result filter
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
134 if retry_on_result is None:
135 self._retry_on_result = self.never_reject
136 else:
137 self._retry_on_result = retry_on_result
138
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
139 self._wrap_exception = wrap_exception
140
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
141 def stop_after_attempt(self, previous_attempt_number, delay_since_first_attempt_ms):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
142 """Stop after the previous attempt >= stop_max_attempt_number."""
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
143 return previous_attempt_number >= self._stop_max_attempt_number
144
145 def stop_after_delay(self, previous_attempt_number, delay_since_first_attempt_ms):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
146 """Stop after the time from the first attempt >= stop_max_delay."""
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
147 return delay_since_first_attempt_ms >= self._stop_max_delay
148
149 def no_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
150 """Don't sleep at all before retrying."""
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
151 return 0
152
153 def fixed_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
154 """Sleep a fixed amount of time between each retry."""
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
155 return self._wait_fixed
156
157 def random_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
158 """Sleep a random amount of time between wait_random_min and wait_random_max"""
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
159 return random.randint(self._wait_random_min, self._wait_random_max)
160
161 def incrementing_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
162 """
163 Sleep an incremental amount of time after each attempt, starting at
164 wait_incrementing_start and incrementing by wait_incrementing_increment
165 """
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
166 result = self._wait_incrementing_start + (self._wait_incrementing_increment * (previous_attempt_number - 1))
167 if result < 0:
168 result = 0
169 return result
170
171 def exponential_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
172 exp = 2 ** previous_attempt_number
173 result = self._wait_exponential_multiplier * exp
174 if result > self._wait_exponential_max:
175 result = self._wait_exponential_max
176 if result < 0:
177 result = 0
178 return result
179
180 def never_reject(self, result):
181 return False
182
fd4d0e8 @rholder default to retry on any exception, handle @retry and @retry() behavior
authored
183 def always_reject(self, result):
184 return True
185
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
186 def should_reject(self, attempt):
187 reject = False
188 if attempt.has_exception:
f405d7e @rholder propagate complete tracebacks such that they show up in nested stack fra...
authored
189 reject |= self._retry_on_exception(attempt.value[1])
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
190 else:
191 reject |= self._retry_on_result(attempt.value)
192
193 return reject
194
195 def call(self, fn, *args, **kwargs):
196 start_time = int(round(time.time() * 1000))
197 attempt_number = 1
198 while True:
199 try:
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
200 attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
8035a32 @rholder fixed a bug where classes not extending from the Python exception hierar...
authored
201 except:
f405d7e @rholder propagate complete tracebacks such that they show up in nested stack fra...
authored
202 tb = sys.exc_info()
203 attempt = Attempt(tb, attempt_number, True)
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
204
205 if not self.should_reject(attempt):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
206 return attempt.get(self._wrap_exception)
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
207
208 delay_since_first_attempt_ms = int(round(time.time() * 1000)) - start_time
209 if self.stop(attempt_number, delay_since_first_attempt_ms):
017dfa7 @simondolle If possible, raise the last exception when stopping
simondolle authored
210 if not self._wrap_exception and attempt.has_exception:
8ca5b41 @rholder add comments from rdooley, add comment around stop behavior
authored
211 # get() on an attempt with an exception should cause it to be raised, but raise just in case
017dfa7 @simondolle If possible, raise the last exception when stopping
simondolle authored
212 raise attempt.get()
213 else:
214 raise RetryError(attempt)
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
215 else:
216 sleep = self.wait(attempt_number, delay_since_first_attempt_ms)
01a2173 @harlowja Allow a jitter value to be passed in
harlowja authored
217 if self._wait_jitter_max:
218 jitter = random.random() * self._wait_jitter_max
219 sleep = sleep + max(0, jitter)
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
220 time.sleep(sleep / 1000.0)
221
222 attempt_number += 1
223
8ca5b41 @rholder add comments from rdooley, add comment around stop behavior
authored
224
5271e77 remove need for specification of stop/wait type
Derek Wilson authored
225 class Attempt(object):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
226 """
227 An Attempt encapsulates a call to a target function that may end as a
228 normal return value from the function or an Exception depending on what
229 occurred during the execution.
230 """
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
231
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
232 def __init__(self, value, attempt_number, has_exception):
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
233 self.value = value
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
234 self.attempt_number = attempt_number
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
235 self.has_exception = has_exception
236
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
237 def get(self, wrap_exception=False):
238 """
239 Return the return value of this Attempt instance or raise an Exception.
240 If wrap_exception is true, this Attempt is wrapped inside of a
241 RetryError before being raised.
242 """
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
243 if self.has_exception:
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
244 if wrap_exception:
245 raise RetryError(self)
246 else:
062c5ba @hguemar Replaced bundled code with using six
hguemar authored
247 six.reraise(self.value[0], self.value[1], self.value[2])
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
248 else:
249 return self.value
250
f405d7e @rholder propagate complete tracebacks such that they show up in nested stack fra...
authored
251 def __repr__(self):
252 if self.has_exception:
253 return "Attempts: {0}, Error:\n{1}".format(self.attempt_number, "".join(traceback.format_tb(self.value[2])))
254 else:
255 return "Attempts: {0}, Value: {1}".format(self.attempt_number, self.value)
256
8ca5b41 @rholder add comments from rdooley, add comment around stop behavior
authored
257
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
258 class RetryError(Exception):
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
259 """
260 A RetryError encapsulates the last Attempt instance right before giving up.
261 """
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
262
c4f99b5 @rholder add configurable support for wrapping any raised Exception in a RetryErr...
authored
263 def __init__(self, last_attempt):
bf2248c @rholder adding first stable implementation, reasonable passing test coverage
authored
264 self.last_attempt = last_attempt
265
266 def __str__(self):
f405d7e @rholder propagate complete tracebacks such that they show up in nested stack fra...
authored
267 return "RetryError[{0}]".format(self.last_attempt)
Something went wrong with that request. Please try again.