From ce9d114ecc14ee7d9c2506c8f1a8c913ea5c6cc6 Mon Sep 17 00:00:00 2001 From: Bob Green Date: Wed, 3 Aug 2016 10:57:00 -0400 Subject: [PATCH] Support runtime configuration with optionally callable kwargs This makes the max_tries kwarg on on_exception and on_predicate optionally accept a callable to be evaluated at runtime rather than a fixed value. Similarly, wait_gen_kwargs also optionally accept callables to be evaluated before passing into the wait_gen. This means that all current kwargs on backoff.expo, backoff.fibo, backoff.constant now accept callables, and also that all future and user-space wait generators will as well. This addresses https://github.com/litl/backoff/issues/10 --- README.rst | 31 +++++++++++++++++++++++++++++++ backoff.py | 39 +++++++++++++++++++++++++++++---------- backoff_tests.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 29bc9a2..2bdcae5 100644 --- a/README.rst +++ b/README.rst @@ -128,6 +128,37 @@ backoff behavior for different cases:: def poll_for_message(queue): return queue.get() +Runtime Configuration +--------------------- + +The decorator functions ``on_exception`` and ``on_predicate`` are +generally evaluated at import time. This is fine when the keyword args +are passed as constant values, but suppose we want to consult a +dictionary with configuration options that only become available at +runtime. The relevant values are not available at import time. Instead, +decorator functions can be passed callables which are evaluated at +runtime to obtain the value:: + + def lookup_max_tries(): + # pretend we have a global reference to 'app' here + # and that it has a dictionary-like 'config' property + return app.config["BACKOFF_MAX_TRIES"] + + @backoff.on_exception(backoff.expo, + ValueError, + max_tries=lookup_max_tries) + +More cleverly, you might define a function which returns a lookup +function for a specified variable:: + + def config(app, name): + return functools.partial(app.config.get, name) + + @backoff.on_exception(backoff.expo, + ValueError, + max_value=config(app, "BACKOFF_MAX_VALUE") + max_tries=config(app, "BACKOFF_MAX_TRIES")) + Event handlers -------------- diff --git a/backoff.py b/backoff.py index 8822e21..c46137d 100644 --- a/backoff.py +++ b/backoff.py @@ -148,7 +148,9 @@ def on_predicate(wait_gen, is exceeded. The parameter is a dict containing details about the invocation. **wait_gen_kwargs: Any additional keyword args specified will be - passed to wait_gen when it is initialized. + passed to wait_gen when it is initialized. Any callable + args will first be evaluated and their return values passed. + This is useful for runtime configuration. """ success_hdlrs = _handlers(on_success) backoff_hdlrs = _handlers(on_backoff, _log_backoff) @@ -158,14 +160,19 @@ def decorate(target): @functools.wraps(target) def retry(*args, **kwargs): - tries = 0 + # change names because python 2.x doesn't have nonlocal + max_tries_ = _maybe_call(max_tries) + + # there are no dictionary comprehensions in python 2.6 + wait = wait_gen(**dict((k, _maybe_call(v)) + for k, v in wait_gen_kwargs.items())) - wait = wait_gen(**wait_gen_kwargs) + tries = 0 while True: tries += 1 ret = target(*args, **kwargs) if predicate(ret): - if tries == max_tries: + if tries == max_tries_: for hdlr in giveup_hdlrs: hdlr({'target': target, 'args': args, @@ -231,7 +238,8 @@ def on_exception(wait_gen, max_tries: The maximum number of attempts to make before giving up. Once exhausted, the exception will be allowed to escape. The default value of None means their is no limit to the - number of tries. + number of tries. If a callable is passed, it will be + evaluated at runtime and its return value used. jitter: A function of the value yielded by wait_gen returning the actual time to wait. This distributes wait times stochastically in order to avoid timing collisions across @@ -252,8 +260,9 @@ def on_exception(wait_gen, is exceeded. The parameter is a dict containing details about the invocation. **wait_gen_kwargs: Any additional keyword args specified will be - passed to wait_gen when it is initialized. - + passed to wait_gen when it is initialized. Any callable + args will first be evaluated and their return values passed. + This is useful for runtime configuration. """ success_hdlrs = _handlers(on_success) backoff_hdlrs = _handlers(on_backoff, _log_backoff) @@ -263,15 +272,20 @@ def decorate(target): @functools.wraps(target) def retry(*args, **kwargs): + # change names because python 2.x doesn't have nonlocal + max_tries_ = _maybe_call(max_tries) + + # there are no dictionary comprehensions in python 2.6 + wait = wait_gen(**dict((k, _maybe_call(v)) + for k, v in wait_gen_kwargs.items())) + tries = 0 - wait = wait_gen(**wait_gen_kwargs) while True: try: tries += 1 ret = target(*args, **kwargs) except exception as e: - if giveup(e) or tries == max_tries: - + if giveup(e) or tries == max_tries_: for hdlr in giveup_hdlrs: hdlr({'target': target, 'args': args, @@ -326,6 +340,11 @@ def _handlers(hdlr, default=None): return defaults + [hdlr] +# Evaluate arg that can be either a fixed value or a callable. +def _maybe_call(f, *args, **kwargs): + return f(*args, **kwargs) if callable(f) else f + + # Formats a function invocation as a unicode string for logging. def _invoc_repr(details): f, args, kwargs = details['target'], details['args'], details['kwargs'] diff --git a/backoff_tests.py b/backoff_tests.py index 040ede3..aac213c 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -516,3 +516,48 @@ def success(*args, **kwargs): 'target': success._target, 'tries': 3, 'value': True} + + +def test_on_exception_callable_max_tries(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + def lookup_max_tries(): + return 3 + + log = [] + + @backoff.on_exception(backoff.constant, + ValueError, + max_tries=lookup_max_tries) + def exceptor(): + log.append(True) + raise ValueError() + + with pytest.raises(ValueError): + exceptor() + + assert len(log) == 3 + + +def test_on_exception_callable_gen_kwargs(): + + def lookup_foo(): + return "foo" + + def wait_gen(foo=None, bar=None): + assert foo == "foo" + assert bar == "bar" + + while True: + yield 0 + + @backoff.on_exception(wait_gen, + ValueError, + max_tries=2, + foo=lookup_foo, + bar="bar") + def exceptor(): + raise ValueError("aah") + + with pytest.raises(ValueError): + exceptor()