chore: handle timeout errors in API requests (DEV-2513) (#457)

src/dsp_tools/fast_xmlupload/process_files.py

@@ -20,6 +20,8 @@
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.logging import get_logger
 
+from dsp_tools.utils.shared import http_call_with_retry
+
 logger = get_logger(__name__, filesize_mb=100, backupcount=36)
 sipi_container: Optional[Container] = None
 export_moving_image_frames_script: Optional[Path] = None
@@ -33,10 +35,11 @@ def _get_export_moving_image_frames_script() -> None:
     user_folder.mkdir(parents=True, exist_ok=True)
     global export_moving_image_frames_script
     export_moving_image_frames_script = user_folder / "export-moving-image-frames.sh"
-    script_text = requests.get(
-        "https://github.com/dasch-swiss/dsp-api/raw/main/sipi/scripts/export-moving-image-frames.sh",
-        timeout=10,
-    ).text
+    script_text_response = http_call_with_retry(
+        action=requests.get,
+        url="https://github.com/dasch-swiss/dsp-api/raw/main/sipi/scripts/export-moving-image-frames.sh",
+    )
+    script_text = script_text_response.text
     with open(export_moving_image_frames_script, "w", encoding="utf-8") as f:
         f.write(script_text)
 

src/dsp_tools/fast_xmlupload/upload_files.py

@@ -11,7 +11,7 @@
 from dsp_tools.models.connection import Connection
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.logging import get_logger
-from dsp_tools.utils.shared import login
+from dsp_tools.utils.shared import http_call_with_retry, login
 
 logger = get_logger(__name__)
 
@@ -170,11 +170,12 @@ def _upload_without_processing(
     try:
         with open(file, "rb") as bitstream:
             try:
-                response_upload = requests.post(
+                response_upload = http_call_with_retry(
+                    action=requests.post,
+                    initial_timeout=8 * 60,
                     url=f"{regex.sub(r'/$', '', sipi_url)}/upload_without_processing",
                     headers={"Authorization": f"Bearer {con.get_token()}"},
                     files={"file": bitstream},
-                    timeout=8 * 60,
                 )
             except Exception:  # pylint: disable=broad-exception-caught
                 err_msg = f"An exception was raised while calling the /upload_without_processing route for {file}"

src/dsp_tools/models/connection.py

@@ -68,7 +68,7 @@ def login(self, email: str, password: str) -> None:
             self._server + "/v2/authentication",
             headers={"Content-Type": "application/json; charset=UTF-8"},
             data=jsondata,
-            timeout=10,
+            timeout=20,
         )
         check_for_api_error(response)
         result = response.json()
@@ -105,7 +105,7 @@ def logout(self) -> None:
             response = requests.delete(
                 self._server + "/v2/authentication",
                 headers={"Authorization": "Bearer " + self._token},
-                timeout=10,
+                timeout=20,
             )
             check_for_api_error(response)
             self._token = None
@@ -192,19 +192,19 @@ def get(self, path: str, headers: Optional[dict[str, str]] = None) -> dict[str,
             path = "/" + path
         if not self._token:
             if not headers:
-                response = requests.get(self._server + path, timeout=10)
+                response = requests.get(self._server + path, timeout=20)
             else:
-                response = requests.get(self._server + path, headers, timeout=10)
+                response = requests.get(self._server + path, headers, timeout=20)
         else:
             if not headers:
                 response = requests.get(
                     self._server + path,
                     headers={"Authorization": "Bearer " + self._token},
-                    timeout=10,
+                    timeout=20,
                 )
             else:
                 headers["Authorization"] = "Bearer " + self._token
-                response = requests.get(self._server + path, headers, timeout=10)
+                response = requests.get(self._server + path, headers, timeout=20)
 
         check_for_api_error(response)
         json_response = response.json()
@@ -225,14 +225,14 @@ def put(self, path: str, jsondata: Optional[str] = None, content_type: str = "ap
             response = requests.put(
                 self._server + path,
                 headers={"Authorization": "Bearer " + self._token},
-                timeout=10,
+                timeout=20,
             )
         else:
             response = requests.put(
                 self._server + path,
                 headers={"Content-Type": content_type + "; charset=UTF-8", "Authorization": "Bearer " + self._token},
                 data=jsondata,
-                timeout=10,
+                timeout=20,
             )
         check_for_api_error(response)
         result = response.json()
@@ -252,14 +252,14 @@ def delete(self, path: str, params: Optional[any] = None):
                 self._server + path,
                 headers={"Authorization": "Bearer " + self._token},
                 params=params,
-                timeout=10,
+                timeout=20,
             )
 
         else:
             response = requests.delete(
                 self._server + path,
                 headers={"Authorization": "Bearer " + self._token},
-                timeout=10,
+                timeout=20,
             )
         check_for_api_error(response)
         result = response.json()

src/dsp_tools/models/sipi.py

@@ -7,6 +7,8 @@
 
 from dsp_tools.models.connection import check_for_api_error
 
+from dsp_tools.utils.shared import http_call_with_retry
+
 
 class Sipi:
     """Represents the Sipi instance"""
@@ -29,11 +31,12 @@ def upload_bitstream(self, filepath: str) -> dict[Any, Any]:
             files = {
                 "file": (os.path.basename(filepath), bitstream_file),
             }
-            response = requests.post(
-                self.sipi_server + "/upload",
+            response = http_call_with_retry(
+                action=requests.post,
+                initial_timeout=5 * 60,
+                url=self.sipi_server + "/upload",
                 headers={"Authorization": "Bearer " + self.token},
                 files=files,
-                timeout=5 * 60,
             )
         check_for_api_error(response)
         res: dict[Any, Any] = response.json()

src/dsp_tools/utils/shared.py

@@ -7,12 +7,14 @@
 import unicodedata
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable, Optional, Union, cast
 
 import pandas as pd
 import regex
 from lxml import etree
 from requests import ReadTimeout, RequestException
+import requests
+from urllib3.exceptions import ReadTimeoutError
 
 from dsp_tools.models.connection import Connection
 from dsp_tools.models.exceptions import BaseError, UserError
@@ -52,14 +54,66 @@ def login(
     return con
 
 
+def http_call_with_retry(
+    action: Callable[..., Any],
+    *args: Any,
+    initial_timeout: int = 10,
+    **kwargs: Any,
+) -> requests.Response:
+    """
+    Function that tries 7 times to execute an HTTP request.
+    Timeouts (and only timeouts) are catched, and the request is retried after a waiting time.
+    The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds.
+    Every time, the previous timeout is increased by 10 seconds.
+    Use this only for actions that can be retried without side effects.
+
+    Args:
+        action: one of requests.get(), requests.post(), requests.put(), requests.delete()
+        initial_timeout: Timeout to start with. Defaults to 10 seconds.
+
+    Raises:
+        errors from the requests library that are not timeouts
+
+    Returns:
+        response of the HTTP request
+    """
+    if action not in (requests.get, requests.post, requests.put, requests.delete):
+        raise BaseError(
+            "This function can only be used with the methods get, post, put, and delete of the Python requests library."
+        )
+    action_as_str = f"action='{action}', args='{args}', kwargs='{kwargs}'"
+    timeout = initial_timeout
+    for i in range(7):
+        try:
+            if args and not kwargs:
+                result = action(*args, timeout=timeout)
+            elif kwargs and not args:
+                result = action(**kwargs, timeout=timeout)
+            elif args and kwargs:
+                result = action(*args, **kwargs, timeout=timeout)
+            else:
+                result = action(timeout=timeout)
+            return cast(requests.Response, result)
+        except (TimeoutError, ReadTimeout, ReadTimeoutError):
+            timeout += 10
+            msg = f"Timeout Error: Retry request with timeout {timeout} in {2 ** i} seconds..."
+            print(f"{datetime.now().isoformat()}: {msg}")
+            logger.error(f"{msg} {action_as_str} (retry-counter i={i})", exc_info=True)
+            time.sleep(2**i)
+            continue
+
+    logger.error("Permanently unable to execute the API call. See logs for more details.")
+    raise BaseError("Permanently unable to execute the API call. See logs for more details.")
+
+
 def try_network_action(
     action: Callable[..., Any],
     *args: Any,
     **kwargs: Any,
 ) -> Any:
     """
     Helper method that tries 7 times to execute an action.
-    If a ConnectionError, a requests.exceptions.RequestException, or a non-permanent BaseError occors,
+    If a timeout error, a ConnectionError, a requests.exceptions.RequestException, or a non-permanent BaseError occors,
     it waits and retries.
     The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds.
     If another exception occurs, it escalates.

src/dsp_tools/utils/stack_handling.py

@@ -13,6 +13,8 @@
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.logging import get_logger
 
+from dsp_tools.utils.shared import http_call_with_retry
+
 logger = get_logger(__name__)
 
 
@@ -121,7 +123,11 @@ def _get_sipi_docker_config_lua(self) -> None:
         Raises:
             UserError: if max_file_size is set but cannot be injected into sipi.docker-config.lua
         """
-        docker_config_lua_text = requests.get(f"{self.__url_prefix}sipi/config/sipi.docker-config.lua", timeout=10).text
+        docker_config_lua_response = http_call_with_retry(
+            action=requests.get,
+            url=f"{self.__url_prefix}sipi/config/sipi.docker-config.lua",
+        )
+        docker_config_lua_text = docker_config_lua_response.text
         if self.__stack_configuration.max_file_size:
             max_post_size_regex = r"max_post_size ?= ?[\'\"]\d+M[\'\"]"
             if not re.search(max_post_size_regex, docker_config_lua_text):
@@ -159,7 +165,11 @@ def _wait_for_fuseki(self) -> None:
         """
         for _ in range(6 * 60):
             try:
-                response = requests.get(url="http://0.0.0.0:3030/$/server", auth=("admin", "test"), timeout=10)
+                response = http_call_with_retry(
+                    action=requests.get,
+                    url="http://0.0.0.0:3030/$/server",
+                    auth=("admin", "test"),
+                )
                 if response.ok:
                     break
             except Exception:  # pylint: disable=broad-exception-caught
@@ -174,16 +184,18 @@ def _create_knora_test_repo(self) -> None:
         Raises:
             UserError: in case of failure
         """
-        repo_template = requests.get(
-            f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template",
-            timeout=10,
-        ).text
+        repo_template_response = http_call_with_retry(
+            action=requests.get,
+            url=f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template",
+        )
+        repo_template = repo_template_response.text
         repo_template = repo_template.replace("@REPOSITORY@", "knora-test")
-        response = requests.post(
+        response = http_call_with_retry(
+            action=requests.post,
+            initial_timeout=10,
             url="http://0.0.0.0:3030/$/datasets",
             files={"file": ("file.ttl", repo_template, "text/turtle; charset=utf8")},
             auth=("admin", "test"),
-            timeout=10,
         )
         if not response.ok:
             msg = (
@@ -215,17 +227,18 @@ def _load_data_into_repo(self) -> None:
             ("test_data/project_data/anything-data.ttl", "http://www.knora.org/data/0001/anything"),
         ]
         for ttl_file, graph in ttl_files:
-            ttl_text_response = requests.get(self.__url_prefix + ttl_file, timeout=10)
-            if not ttl_text_response.ok:
+            ttl_response = http_call_with_retry(action=requests.get, url=self.__url_prefix + ttl_file)
+            if not ttl_response.ok:
                 msg = f"Cannot start DSP-API: Error when retrieving '{self.__url_prefix + ttl_file}'"
-                logger.error(f"{msg}'. response = {vars(ttl_text_response)}")
+                logger.error(f"{msg}'. response = {vars(ttl_response)}")
                 raise UserError(msg)
-            ttl_text = ttl_text_response.text
-            response = requests.post(
+            ttl_text = ttl_response.text
+            response = http_call_with_retry(
+                action=requests.post,
+                initial_timeout=10,
                 url=graph_prefix + graph,
                 files={"file": ("file.ttl", ttl_text, "text/turtle; charset: utf-8")},
                 auth=("admin", "test"),
-                timeout=10,
             )
             if not response.ok:
                 logger.error(f"Cannot start DSP-API: Error when creating graph '{graph}'. response = {vars(response)}")