From 0832e278ec5f91b54eb9d01d2dc6eff1025f010c Mon Sep 17 00:00:00 2001 From: Jude Date: Sun, 17 Nov 2024 09:04:49 +0000 Subject: [PATCH 1/4] Add functools.retry --- Lib/functools.py | 49 ++++++++++++++++++++++++++++- Lib/test/test_functools.py | 64 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index eff6540c7f606e..030ad601a9ec82 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -12,7 +12,7 @@ __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property', 'Placeholder'] + 'cached_property', 'Placeholder', 'retry'] from abc import get_cache_token from collections import namedtuple @@ -21,6 +21,7 @@ from reprlib import recursive_repr from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock +from time import sleep ################################################################################ ### update_wrapper() and wraps() decorator @@ -1121,3 +1122,49 @@ def __get__(self, instance, owner=None): return val __class_getitem__ = classmethod(GenericAlias) + + +################################################################################ +### retry() - simple retry decorator +################################################################################ + +def retry(_kwargs=None, *, retry_attempts=3, interval_seconds=.1, backoff_type='linear'): + """ + This function is intended to be used as a decorator and will retry + the function that it decorates if an excpetion is raised in that + function. Several aspects of the retries can be configured with + keyword arguments. Also, no keyword arguments can be used to retry + with the default values. + + NOTE: if using backoff_type='exponential', ensure that + interval_seconds > 1, otherise the subsequent retries will be + shorter. + """ + + def _retry(user_function): + @wraps(user_function) + def _retry_wrapper_user_function(*args, **kwargs): + for attempt_number in range(retry_attempts+1): + try: + return_value = user_function(*args, **kwargs) + break + except Exception as e: + if attempt_number < retry_attempts: + if backoff_type == 'exponential': + # If user inputs interval_seconds < 1 with exponential backoff, retries will get shorter + sleep(interval_seconds * (attempt_number + 1)**2) + elif backoff_type == 'linear': + sleep(interval_seconds) + else: + # Retry attempts reached, raise the last exception + raise e + return return_value + return _retry_wrapper_user_function + + if backoff_type != 'linear' and backoff_type != 'exponential': + raise TypeError("Keyword argument backoff_type must be 'exponential' or 'linear'.") + + if _kwargs is None: + return _retry + else: + return _retry(_kwargs) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 6d60f6941c4c5d..0059073ee8ac51 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -3381,5 +3381,69 @@ def prop(self): self.assertEqual(t.prop, 1) +class TestRetry(unittest.TestCase): + + def test_function_fail(self): + + @functools.retry + def fail_function1(): + raise ValueError + + with self.assertRaises(ValueError): + fail_function1() + + @functools.retry(interval_seconds=.1, retry_attempts=1, backoff_type='exponential') + def fail_function2(): + raise ValueError + + with self.assertRaises(ValueError): + fail_function2() + + def test_function_success(self): + + @functools.retry + def success_function(a, b, c='test_value'): + return (a, b, c) + + value = success_function('a', 'b') + + self.assertEqual(value, ('a', 'b', 'test_value')) + + class TestObject: + def __init__(self, call_count): + self.call_count = call_count + test_object = TestObject(0) + + @functools.retry(interval_seconds=.01, retry_attempts=3, backoff_type='exponential') + def success_function_after_3_failures(test_object): + test_object.call_count += 1 + if test_object.call_count > 3: + return True + raise ValueError('Some error message!') + + value = success_function_after_3_failures(test_object) + self.assertTrue(value) + + def test_backoff_type(self): + @functools.retry(backoff_type='exponential') + def user_function1(): + return True + value1 = user_function1() + self.assertTrue(value1) + + @functools.retry(backoff_type='linear') + def user_function2(): + return True + value2 = user_function2() + self.assertTrue(value2) + + with self.assertRaises(TypeError): + @functools.retry(backoff_type='incorrect_value') + def user_function3(): + return True + + user_function3() + + if __name__ == '__main__': unittest.main() From a7f347338ad136c49a135910e3a98619523fd387 Mon Sep 17 00:00:00 2001 From: Jude Date: Mon, 18 Nov 2024 03:18:23 +0000 Subject: [PATCH 2/4] Run linter on functools.retry and test --- Lib/functools.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 030ad601a9ec82..59a56c75ce503e 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -1124,11 +1124,9 @@ def __get__(self, instance, owner=None): __class_getitem__ = classmethod(GenericAlias) -################################################################################ -### retry() - simple retry decorator -################################################################################ - -def retry(_kwargs=None, *, retry_attempts=3, interval_seconds=.1, backoff_type='linear'): +def retry( + _kwargs=None, *, retry_attempts=3, interval_seconds=0.1, backoff_type="linear" +): """ This function is intended to be used as a decorator and will retry the function that it decorates if an excpetion is raised in that @@ -1136,34 +1134,37 @@ def retry(_kwargs=None, *, retry_attempts=3, interval_seconds=.1, backoff_type=' keyword arguments. Also, no keyword arguments can be used to retry with the default values. - NOTE: if using backoff_type='exponential', ensure that - interval_seconds > 1, otherise the subsequent retries will be + NOTE: if using backoff_type='exponential', ensure that + interval_seconds > 1, otherise the subsequent retries will be shorter. """ def _retry(user_function): @wraps(user_function) def _retry_wrapper_user_function(*args, **kwargs): - for attempt_number in range(retry_attempts+1): + for attempt_number in range(retry_attempts + 1): try: return_value = user_function(*args, **kwargs) break except Exception as e: if attempt_number < retry_attempts: - if backoff_type == 'exponential': + if backoff_type == "exponential": # If user inputs interval_seconds < 1 with exponential backoff, retries will get shorter - sleep(interval_seconds * (attempt_number + 1)**2) - elif backoff_type == 'linear': + sleep(interval_seconds * (attempt_number + 1) ** 2) + elif backoff_type == "linear": sleep(interval_seconds) else: # Retry attempts reached, raise the last exception - raise e + raise e return return_value + return _retry_wrapper_user_function - - if backoff_type != 'linear' and backoff_type != 'exponential': - raise TypeError("Keyword argument backoff_type must be 'exponential' or 'linear'.") - + + if backoff_type != "linear" and backoff_type != "exponential": + raise TypeError( + "Keyword argument backoff_type must be 'exponential' or 'linear'." + ) + if _kwargs is None: return _retry else: From 890f990902b9917e0ee65d7451d92437721cc6ac Mon Sep 17 00:00:00 2001 From: Jude Date: Mon, 18 Nov 2024 03:33:49 +0000 Subject: [PATCH 3/4] Add news and whatsnew --- Doc/whatsnew/3.14.rst | 3 +++ .../Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 958efbe73c1c27..27d50b7406daf8 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -324,6 +324,9 @@ functools as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) +* Add function decorator :func:`functools.retry` that allows configurable + retries for functions that raise an exception. It can be used with or + without keyword arguments. getopt ------ diff --git a/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst b/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst new file mode 100644 index 00000000000000..d1725b9a7e65ba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst @@ -0,0 +1,4 @@ +Add :func:`functools.retry` function to be used as a decorator. The +decorator gives retries to a function that raises exeptions. It can be +used with default options or have options specified with keyword +arguments. From 6eb9423078c09250e69306ed65c56b543b2fbec0 Mon Sep 17 00:00:00 2001 From: Jude Date: Mon, 18 Nov 2024 03:44:57 +0000 Subject: [PATCH 4/4] Further linting --- Doc/whatsnew/3.14.rst | 2 +- Lib/test/test_functools.py | 10 +++++----- .../2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 27d50b7406daf8..68b49a1e8a35f7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -324,7 +324,7 @@ functools as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) -* Add function decorator :func:`functools.retry` that allows configurable +* Add function decorator ``functools.retry`` that allows configurable retries for functions that raise an exception. It can be used with or without keyword arguments. diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 0059073ee8ac51..fe06751316e270 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -3388,14 +3388,14 @@ def test_function_fail(self): @functools.retry def fail_function1(): raise ValueError - + with self.assertRaises(ValueError): fail_function1() @functools.retry(interval_seconds=.1, retry_attempts=1, backoff_type='exponential') def fail_function2(): raise ValueError - + with self.assertRaises(ValueError): fail_function2() @@ -3404,7 +3404,7 @@ def test_function_success(self): @functools.retry def success_function(a, b, c='test_value'): return (a, b, c) - + value = success_function('a', 'b') self.assertEqual(value, ('a', 'b', 'test_value')) @@ -3420,7 +3420,7 @@ def success_function_after_3_failures(test_object): if test_object.call_count > 3: return True raise ValueError('Some error message!') - + value = success_function_after_3_failures(test_object) self.assertTrue(value) @@ -3441,7 +3441,7 @@ def user_function2(): @functools.retry(backoff_type='incorrect_value') def user_function3(): return True - + user_function3() diff --git a/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst b/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst index d1725b9a7e65ba..4bc210a558eb96 100644 --- a/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst +++ b/Misc/NEWS.d/next/Library/2024-11-18-03-26-52.gh-issue-126927.K-gKnU.rst @@ -1,4 +1,4 @@ -Add :func:`functools.retry` function to be used as a decorator. The +Add ``functools.retry`` function to be used as a decorator. The decorator gives retries to a function that raises exeptions. It can be used with default options or have options specified with keyword arguments.