Skip to content

Commit

Permalink
feat: add timeout error handling to different plugins
Browse files Browse the repository at this point in the history
  • Loading branch information
jlahovnik committed Jan 18, 2024
1 parent de954eb commit 9dbaabf
Show file tree
Hide file tree
Showing 8 changed files with 48 additions and 18 deletions.
4 changes: 3 additions & 1 deletion eodag/plugins/authentication/openid_connect.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

from eodag.plugins.authentication import Authentication
from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, parse_qs, repeatfunc, urlparse
from eodag.utils.exceptions import AuthenticationError, MisconfiguredError
from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, TimeOutError

if TYPE_CHECKING:
from requests import PreparedRequest, Response
Expand Down Expand Up @@ -154,6 +154,8 @@ def authenticate(self) -> Union[AuthBase, Dict[str, str]]:
exchange_url = user_consent_response.url
try:
token = self.exchange_code_for_token(exchange_url, state)
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except Exception:
import traceback as tb

Expand Down
4 changes: 3 additions & 1 deletion eodag/plugins/authentication/qsauth.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

from eodag.plugins.authentication import Authentication
from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT
from eodag.utils.exceptions import AuthenticationError
from eodag.utils.exceptions import AuthenticationError, TimeOutError

if TYPE_CHECKING:
from requests import PreparedRequest
Expand Down Expand Up @@ -76,6 +76,8 @@ def authenticate(self) -> Union[AuthBase, Dict[str, str]]:
auth=auth,
)
response.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except RequestException as e:
raise AuthenticationError(f"Could no authenticate: {str(e)}")

Expand Down
4 changes: 3 additions & 1 deletion eodag/plugins/authentication/sas_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

from eodag.plugins.authentication.base import Authentication
from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, deepcopy, format_dict_items
from eodag.utils.exceptions import AuthenticationError
from eodag.utils.exceptions import AuthenticationError, TimeOutError

if TYPE_CHECKING:
from requests import PreparedRequest
Expand Down Expand Up @@ -64,6 +64,8 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest:
)
response.raise_for_status()
signed_url = response.json().get(self.signed_url_key)
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except (requests.RequestException, JSONDecodeError, KeyError) as e:
raise AuthenticationError(f"Could no get signed url: {str(e)}")
else:
Expand Down
4 changes: 3 additions & 1 deletion eodag/plugins/authentication/token.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

from eodag.plugins.authentication.base import Authentication
from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT
from eodag.utils.exceptions import AuthenticationError, MisconfiguredError
from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, TimeOutError

if TYPE_CHECKING:
from requests import PreparedRequest
Expand Down Expand Up @@ -98,6 +98,8 @@ def authenticate(self) -> Union[AuthBase, Dict[str, str]]:
**req_kwargs,
)
response.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except RequestException as e:
response_text = getattr(e.response, "text", "").strip()
raise AuthenticationError(
Expand Down
14 changes: 12 additions & 2 deletions eodag/plugins/download/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,12 @@
path_to_uri,
rename_subfolder,
)
from eodag.utils.exceptions import AuthenticationError, DownloadError, NotAvailableError
from eodag.utils.exceptions import (
AuthenticationError,
DownloadError,
NotAvailableError,
TimeOutError,
)

if TYPE_CHECKING:
from boto3.resources.collection import ResourceCollection
Expand Down Expand Up @@ -294,7 +299,12 @@ def download(
**product.properties
)
logger.info("Fetching extra metadata from %s" % fetch_url)
resp = requests.get(fetch_url, headers=USER_AGENT, timeout=HTTP_REQ_TIMEOUT)
try:
resp = requests.get(
fetch_url, headers=USER_AGENT, timeout=HTTP_REQ_TIMEOUT
)
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
update_metadata = mtd_cfg_as_conversion_and_querypath(update_metadata)
if fetch_format == "json":
json_resp = resp.json()
Expand Down
22 changes: 11 additions & 11 deletions eodag/plugins/download/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
DownloadError,
MisconfiguredError,
NotAvailableError,
TimeOutError,
)

if TYPE_CHECKING:
Expand Down Expand Up @@ -167,6 +168,8 @@ def orderDownload(
ordered_message = response.text
logger.debug(ordered_message)
logger.info("%s was ordered", product.properties["title"])
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except RequestException as e:
if e.response and hasattr(e.response, "content"):
error_message = f"{e.response.content.decode('utf-8')} - {e}"
Expand Down Expand Up @@ -359,6 +362,8 @@ def orderDownloadStatus(
f"after order success. Please search and download {product} again"
)

except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except RequestException as e:
logger.warning(
"%s order status could not be checked, request returned %s",
Expand Down Expand Up @@ -426,9 +431,7 @@ def download_request(
timeout: int,
**kwargs: Dict[str, Any],
) -> None:
chunks = self._stream_download(
product, auth, progress_callback, wait, timeout, **kwargs
)
chunks = self._stream_download(product, auth, progress_callback, **kwargs)

with open(fs_path, "wb") as fhandle:
for chunk in chunks:
Expand Down Expand Up @@ -557,9 +560,7 @@ def _stream_download_dict(
else:
pass

chunks = self._stream_download(
product, auth, progress_callback, wait, timeout, **kwargs
)
chunks = self._stream_download(product, auth, progress_callback, **kwargs)
# start reading chunks to set product.headers
first_chunk = next(chunks)

Expand Down Expand Up @@ -613,8 +614,6 @@ def _stream_download(
product: EOProduct,
auth: Optional[PluginConfig] = None,
progress_callback: Optional[ProgressCallback] = None,
wait: int = DEFAULT_DOWNLOAD_WAIT,
timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
**kwargs: Dict[str, Any],
) -> Iterator[Any]:
"""
Expand All @@ -630,9 +629,6 @@ def _stream_download(
creation and update to give the user a
feedback on the download progress
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
:param ordered_message: message to be used in case of error because
the product is unavailable
:type ordered_message: str
:param kwargs: additional arguments
:type kwargs: dict
"""
Expand Down Expand Up @@ -688,6 +684,8 @@ def _stream_download(
try:
self.stream.raise_for_status()

except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except RequestException as e:
self._process_exception(e, product, ordered_message)
else:
Expand Down Expand Up @@ -799,6 +797,8 @@ def get_chunks(stream: Response) -> Any:
) as stream:
try:
stream.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except RequestException as e:
raise_errors = True if len(assets_values) == 1 else False
self._handle_asset_exception(e, asset, raise_errors=raise_errors)
Expand Down
10 changes: 9 additions & 1 deletion eodag/plugins/search/data_request_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
deepcopy,
string_to_jsonpath,
)
from eodag.utils.exceptions import NotAvailableError, RequestError
from eodag.utils.exceptions import NotAvailableError, RequestError, TimeOutError

if TYPE_CHECKING:
from eodag.config import PluginConfig
Expand Down Expand Up @@ -258,6 +258,8 @@ def _create_data_request(
url, json=request_body, headers=headers, timeout=HTTP_REQ_TIMEOUT
)
request_job.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except requests.RequestException as e:
raise RequestError(
f"search job for product_type {product_type} could not be created: {str(e)}, {request_job.text}"
Expand All @@ -274,6 +276,8 @@ def _cancel_request(self, data_request_id: str) -> None:
delete_url, headers=self.auth.headers, timeout=HTTP_REQ_TIMEOUT
)
delete_resp.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except requests.RequestException as e:
raise RequestError(f"_cancel_request failed: {str(e)}")

Expand All @@ -285,6 +289,8 @@ def _check_request_status(self, data_request_id: str) -> bool:
status_url, headers=self.auth.headers, timeout=HTTP_REQ_TIMEOUT
)
status_resp.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except requests.RequestException as e:
raise RequestError(f"_check_request_status failed: {str(e)}")
else:
Expand Down Expand Up @@ -313,6 +319,8 @@ def _get_result_data(
return requests.get(
url, headers=self.auth.headers, timeout=HTTP_REQ_TIMEOUT
).json()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except requests.RequestException:
logger.error(f"Result could not be retrieved for {url}")
return {}
Expand Down
4 changes: 4 additions & 0 deletions eodag/plugins/search/qssearch.py
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,8 @@ def do_search(self, *args: Any, **kwargs: Any) -> List[Any]:
metadata_url, headers=USER_AGENT, timeout=HTTP_REQ_TIMEOUT
)
response.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except requests.RequestException:
logger.exception(
"Skipping error while searching for %s %s instance:",
Expand Down Expand Up @@ -1219,6 +1221,8 @@ def _request(
**kwargs,
)
response.raise_for_status()
except requests.exceptions.Timeout as exc:
raise TimeOutError(str(exc))
except (requests.RequestException, URLError) as err:
# check if error is identified as auth_error in provider conf
auth_errors = getattr(self.config, "auth_error_code", [None])
Expand Down

0 comments on commit 9dbaabf

Please sign in to comment.