Issue441 automatic backoff retry on http429

openeo/rest/connection.py

@@ -46,6 +46,7 @@
     OidcResourceOwnerPasswordAuthenticator,
 )
 from openeo.rest.datacube import DataCube, InputDate
+from openeo.rest.http import requests_with_retry
 from openeo.rest.job import BatchJob, RESTJob
 from openeo.rest.mlmodel import MlModel
 from openeo.rest.rest_capabilities import RESTCapabilities
@@ -85,7 +86,8 @@ def __init__(
     ):
         self._root_url = root_url
         self.auth = auth or NullAuth()
-        self.session = session or requests.Session()
+        # TODO: #441 [WIP] Add requests_with_retry here to the session?
+        self.session = session or requests_with_retry()
         self.default_timeout = default_timeout or DEFAULT_TIMEOUT
         self.default_headers = {
             "User-Agent": "openeo-python-client/{cv} {py}/{pv} {pl}".format(

openeo/rest/http.py

@@ -0,0 +1,39 @@
+from typing import Set
+
+import requests
+import requests.adapters
+
+MAX_RETRIES = 3
+
+
+def requests_with_retry(
+    total: int = MAX_RETRIES,
+    read: int = MAX_RETRIES,
+    other: int = MAX_RETRIES,
+    status: int = MAX_RETRIES,
+    backoff_factor: float = 1,
+    status_forcelist: Set[int] = frozenset([429, 500, 502, 503, 504]),
+    **kwargs,
+) -> requests.Session:
+    """
+    Create a `requests.Session` with automatic retrying
+
+    Inspiration and references:
+    - https://requests.readthedocs.io/en/latest/api/#requests.adapters.HTTPAdapter
+    - https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry
+    - https://findwork.dev/blog/advanced-usage-python-requests-timeouts-retries-hooks/#retry-on-failure
+    """
+    session = requests.Session()
+    retry = requests.adapters.Retry(
+        total=total,
+        read=read,
+        other=other,
+        status=status,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+        **kwargs,
+    )
+    adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session

setup.py

@@ -28,6 +28,7 @@
     "flake8>=5.0.0",
     "time_machine",
     "pyproj>=3.2.0",  # Pyproj is an optional, best-effort runtime dependency
+    "re-assert",
 ]
 
 docs_require = [

tests/rest/test_http.py

@@ -0,0 +1,40 @@
+import logging
+
+import pytest
+import requests.exceptions
+from re_assert import Matches
+
+from openeo.rest.http import requests_with_retry
+
+
+def test_requests_with_retry(caplog):
+    """Simple test for retrying using an invalid domain."""
+    caplog.set_level(logging.DEBUG)
+
+    session = requests_with_retry(total=2, backoff_factor=0.1)
+    with pytest.raises(requests.exceptions.ConnectionError, match="Max retries exceeded"):
+        _ = session.get("https://example.test")
+
+    assert caplog.messages == [
+        "Starting new HTTPS connection (1): example.test:443",
+        Matches("Incremented Retry.*Retry\(total=1"),
+        # Matches("Retrying.*total=1.*Failed to establish a new connection"),
+        Matches("Retrying.*total=1.*Failed to resolve 'example.test'"),
+        "Starting new HTTPS connection (2): example.test:443",
+        Matches("Incremented Retry.*Retry\(total=0"),
+        Matches("Retrying.*total=0.*Failed to resolve 'example.test'"),
+        "Starting new HTTPS connection (3): example.test:443",
+    ]
+
+
+def test_requests_with_retry_zero(caplog):
+    """Simple test for retrying using an invalid domain."""
+    caplog.set_level(logging.DEBUG)
+
+    session = requests_with_retry(total=0)
+    with pytest.raises(requests.exceptions.ConnectionError, match="Max retries exceeded"):
+        _ = session.get("https://example.test")
+
+    assert caplog.messages == [
+        "Starting new HTTPS connection (1): example.test:443",
+    ]