Retry HTTP calls to the backends with backoff

Fixes: #1064
Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

changelog.d/1064.added

@@ -0,0 +1 @@
+Retry HTTP calls to the backends with backoff

fmn/backends/base.py

@@ -3,35 +3,68 @@
 # SPDX-License-Identifier: MIT
 
 import logging
+import sys
+import traceback
 from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator
 from copy import deepcopy
 from functools import cached_property as ft_cached_property
 from functools import wraps
 from typing import Any
 
+import backoff
 from httpx import AsyncClient, HTTPStatusError
 
 log = logging.getLogger(__name__)
 
-NextPageParams = tuple[str, dict] | tuple[None, None]
-
 
 def handle_http_error(default_factory):
+    def backoff_hdlr(details):
+        log.warning(
+            "Request failed (try %s). Retrying in %ss. %s",
+            details["tries"],
+            "{:0.1f}".format(details["wait"]),
+            traceback.format_tb(sys.exc_info()[2]),
+        )
+
+    def giveup_hdlr(details):
+        log.warning(
+            "Request failed after %s tries. Giving up. %s",
+            details["tries"],
+            traceback.format_tb(sys.exc_info()[2]),
+        )
+
+    def is_fatal(e):
+        return e.response.status_code < 500
+
     def exception_handler(f):
         @wraps(f)
         async def wrapper(*args, **kw):
-            try:
+            @backoff.on_exception(
+                backoff.expo,
+                HTTPStatusError,
+                max_tries=3,
+                giveup=is_fatal,
+                on_backoff=backoff_hdlr,
+                on_giveup=giveup_hdlr,
+                logger=None,
+            )
+            async def _retrying_wrapper(*args, **kw):
                 return await f(*args, **kw)
-            except HTTPStatusError as e:
-                log.warning("Request failed: %s", e)
+
+            try:
+                return await _retrying_wrapper(*args, **kw)
+            except HTTPStatusError:
                 return default_factory()
 
         return wrapper
 
     return exception_handler
 
 
+NextPageParams = tuple[str, dict] | tuple[None, None]
+
+
 class PaginationRecursionError(RuntimeError):
     pass
 

tests/backends/test_base.py

@@ -203,3 +203,18 @@ async def fn_to_be_decorated():
         assert result == ["item"]
     else:
         assert result == []
+
+
+async def test_handle_http_error_retries(mocker, caplog):
+    fn_to_be_decorated = mocker.AsyncMock()
+    fn_to_be_decorated.side_effect = httpx.HTTPStatusError(
+        "Boo.", request=None, response=httpx.Response(status_code=500)
+    )
+    decorated_fn = base.handle_http_error(list)(fn_to_be_decorated)
+    result = await decorated_fn()
+    assert result == []
+    assert fn_to_be_decorated.call_count == 3
+    assert len(caplog.messages) == 3
+    assert caplog.messages[0].startswith("Request failed (try 1).")
+    assert caplog.messages[1].startswith("Request failed (try 2).")
+    assert caplog.messages[2].startswith("Request failed after 3 tries. Giving up.")

tests/backends/test_pagure.py

@@ -199,7 +199,7 @@ async def test_get_user_projects(self, respx_mocker, proxy_unmocked_client):
     async def test_get_projects_failure(self, respx_mocker, proxy_unmocked_client):
         route = respx_mocker.get(
             f"{self.expected_api_url}/projects", params={"fork": False, "short": True}
-        ).mock(side_effect=[httpx.Response(fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR)])
+        ).mock(side_effect=httpx.Response(fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR))
 
         response = await proxy_unmocked_client.get_projects()
         assert route.called