From 70f680843efd9c9398d869b83f46d0e0eea0fb83 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum <39048939+jnussbaum@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:38:03 +0100 Subject: [PATCH] chore: get rid of http_call_with_retry (#694) --- .../commands/fast_xmlupload/process_files.py | 8 +-- src/dsp_tools/commands/start_stack.py | 34 ++++------- src/dsp_tools/utils/shared.py | 61 +------------------ 3 files changed, 17 insertions(+), 86 deletions(-) diff --git a/src/dsp_tools/commands/fast_xmlupload/process_files.py b/src/dsp_tools/commands/fast_xmlupload/process_files.py index 00b4c39ad..bb7eef927 100644 --- a/src/dsp_tools/commands/fast_xmlupload/process_files.py +++ b/src/dsp_tools/commands/fast_xmlupload/process_files.py @@ -19,7 +19,7 @@ from dsp_tools.models.exceptions import UserError from dsp_tools.utils.create_logger import get_logger -from dsp_tools.utils.shared import http_call_with_retry, make_chunks +from dsp_tools.utils.shared import make_chunks logger = get_logger(__name__) sipi_container: Optional[Container] = None @@ -34,9 +34,9 @@ def _get_export_moving_image_frames_script() -> None: user_folder.mkdir(parents=True, exist_ok=True) global export_moving_image_frames_script export_moving_image_frames_script = user_folder / "export-moving-image-frames.sh" - script_text_response = http_call_with_retry( - action=requests.get, - url="https://github.com/dasch-swiss/dsp-api/raw/main/sipi/scripts/export-moving-image-frames.sh", + script_text_response = requests.get( + "https://github.com/dasch-swiss/dsp-api/raw/main/sipi/scripts/export-moving-image-frames.sh", + timeout=30, ) script_text = script_text_response.text with open(export_moving_image_frames_script, "w", encoding="utf-8") as f: diff --git a/src/dsp_tools/commands/start_stack.py b/src/dsp_tools/commands/start_stack.py index c105b17d7..7c241ef0a 100644 --- a/src/dsp_tools/commands/start_stack.py +++ b/src/dsp_tools/commands/start_stack.py @@ -12,7 +12,6 @@ from dsp_tools.models.exceptions import UserError from dsp_tools.utils.create_logger import get_logger -from dsp_tools.utils.shared import http_call_with_retry logger = get_logger(__name__) @@ -131,10 +130,7 @@ def _get_sipi_docker_config_lua(self) -> None: Raises: UserError: if max_file_size is set but cannot be injected into sipi.docker-config.lua """ - docker_config_lua_response = http_call_with_retry( - action=requests.get, - url=f"{self.__url_prefix}sipi/config/sipi.docker-config.lua", - ) + docker_config_lua_response = requests.get(f"{self.__url_prefix}sipi/config/sipi.docker-config.lua", timeout=30) docker_config_lua_text = docker_config_lua_response.text if self.__stack_configuration.max_file_size: max_post_size_regex = r"max_post_size ?= ?[\'\"]?\d+[MG][\'\"]?" @@ -173,11 +169,7 @@ def _wait_for_fuseki(self) -> None: """ for _ in range(6 * 60): try: - response = http_call_with_retry( - action=requests.get, - url="http://0.0.0.0:3030/$/server", - auth=("admin", "test"), - ) + response = requests.get("http://0.0.0.0:3030/$/server", auth=("admin", "test"), timeout=10) if response.ok: break except Exception: # noqa: BLE001 (blind-except) @@ -192,18 +184,17 @@ def _create_knora_test_repo(self) -> None: Raises: UserError: in case of failure """ - repo_template_response = http_call_with_retry( - action=requests.get, - url=f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template", + repo_template_response = requests.get( + f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template", + timeout=30, ) repo_template = repo_template_response.text repo_template = repo_template.replace("@REPOSITORY@", "knora-test") - response = http_call_with_retry( - action=requests.post, - initial_timeout=10, - url="http://0.0.0.0:3030/$/datasets", + response = requests.post( + "http://0.0.0.0:3030/$/datasets", files={"file": ("file.ttl", repo_template, "text/turtle; charset=utf8")}, auth=("admin", "test"), + timeout=30, ) if not response.ok: msg = ( @@ -235,18 +226,17 @@ def _load_data_into_repo(self) -> None: ("test_data/project_data/anything-data.ttl", "http://www.knora.org/data/0001/anything"), ] for ttl_file, graph in ttl_files: - ttl_response = http_call_with_retry(action=requests.get, url=self.__url_prefix + ttl_file) + ttl_response = requests.get(self.__url_prefix + ttl_file, timeout=30) if not ttl_response.ok: msg = f"Cannot start DSP-API: Error when retrieving '{self.__url_prefix + ttl_file}'" logger.error(f"{msg}'. response = {vars(ttl_response)}") raise UserError(msg) ttl_text = ttl_response.text - response = http_call_with_retry( - action=requests.post, - initial_timeout=10, - url=graph_prefix + graph, + response = requests.post( + graph_prefix + graph, files={"file": ("file.ttl", ttl_text, "text/turtle; charset: utf-8")}, auth=("admin", "test"), + timeout=30, ) if not response.ok: logger.error(f"Cannot start DSP-API: Error when creating graph '{graph}'. response = {vars(response)}") diff --git a/src/dsp_tools/utils/shared.py b/src/dsp_tools/utils/shared.py index ed3f6f4d1..80097af53 100644 --- a/src/dsp_tools/utils/shared.py +++ b/src/dsp_tools/utils/shared.py @@ -4,18 +4,14 @@ import glob import importlib.resources import json -import time import unicodedata from datetime import datetime from pathlib import Path -from typing import Any, Callable, Iterable, Optional, TypeGuard, TypeVar, Union, cast +from typing import Any, Iterable, Optional, TypeGuard, TypeVar, Union import pandas as pd import regex -import requests from lxml import etree -from requests import ReadTimeout -from urllib3.exceptions import ReadTimeoutError from dsp_tools.commands.excel2xml.propertyelement import PropertyElement from dsp_tools.models.exceptions import BaseError, UserError @@ -79,61 +75,6 @@ def login( return con -def http_call_with_retry( - action: Callable[..., Any], - *args: Any, - initial_timeout: int = 10, - **kwargs: Any, -) -> requests.Response: - """ - Function that tries 7 times to execute an HTTP request. - Timeouts (and only timeouts) are catched, and the request is retried after a waiting time. - The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds. - Every time, the previous timeout is increased by 10 seconds. - Use this only for actions that can be retried without side effects. - - Args: - action: one of requests.get(), requests.post(), requests.put(), requests.delete() - initial_timeout: Timeout to start with. Defaults to 10 seconds. - *args: positional arguments for the action - **kwargs: keyword arguments for the action - - Raises: - BaseError: if the action is not one of one of requests.get(), requests.post(), requests.put(), requests.delete() - Other Errors: errors from the requests library that are not timeouts - - Returns: - response of the HTTP request - """ - if action not in (requests.get, requests.post, requests.put, requests.delete): - raise BaseError( - "This function can only be used with the methods get, post, put, and delete of the Python requests library." - ) - action_as_str = f"{action=}, {args=}, {kwargs=}" - timeout = initial_timeout - for i in range(7): - try: - if args and not kwargs: - result = action(*args, timeout=timeout) - elif kwargs and not args: - result = action(**kwargs, timeout=timeout) - elif args and kwargs: - result = action(*args, **kwargs, timeout=timeout) - else: - result = action(timeout=timeout) - return cast(requests.Response, result) - except (TimeoutError, ReadTimeout, ReadTimeoutError): - timeout += 10 - msg = f"Timeout Error: Retry request with timeout {timeout} in {2 ** i} seconds..." - print(f"{datetime.now()}: {msg}") - logger.error(f"{msg} {action_as_str} (retry-counter {i=:})", exc_info=True) - time.sleep(2**i) - continue - - logger.error("Permanently unable to execute the API call. See logs for more details.") - raise BaseError("Permanently unable to execute the API call. See logs for more details.") - - def validate_xml_against_schema(input_file: Union[str, Path, etree._ElementTree[Any]]) -> bool: """ Validates an XML file against the DSP XSD schema.