From e590d8554249d834aed829c51cd84d8d4ea7aaaa Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Tue, 13 Dec 2022 15:23:21 +0800 Subject: [PATCH] cgit_atom_feed: Better exception handling In case of HTTP error on requesting cgit atom feed, you will see an exception like this: ``` python38/root/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default raise HTTPError(req.full_url, code, msg, hdrs, fp) urllib.error.HTTPError: HTTP Error 404: Not found ``` This is not good for troubleshooting. This PR changes to use `requests` and `tenacity` library to handle HTTP requests and retries. You will see the following exception in case of HTTP error: ``` Traceback (most recent call last): File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 407, in __call__ result = fn(*args, **kwargs) File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 319, in _make_request resp.raise_for_status() File "/Users/yux/Library/Python/3.10/lib/python/site-packages/requests/models.py", line 941, in raise_for_status raise HTTPError(http_error_msg, response=self) requests.exceptions.HTTPError: 404 Client Error: Not found for url: https://pkgs.devel.redhat.com/cgit/rpms/openshift-clients/atom/?h=rhaos-4.8-rhel-7 The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/runtime.py", line 101, in wrapper return func(*args, **kwargs) File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/runtime.py", line 113, in wrapper return func(*args) File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/runtime.py", line 1584, in lambda meta, _: (meta, meta.needs_rebuild()), File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 624, in needs_rebuild hint = self._target_needs_rebuild(el_target=target) File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 663, in _target_needs_rebuild atom_entries = self.cgit_atom_feed(commit_hash=distgit_commitish, branch=self.branch()) File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 322, in cgit_atom_feed content = retry( File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 324, in wrapped_f return self(f, *args, **kw) File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 404, in __call__ do = self.iter(retry_state=retry_state) File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 361, in iter raise retry_exc from fut.exception() tenacity.RetryError: RetryError[] ``` --- doozerlib/metadata.py | 54 ++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/doozerlib/metadata.py b/doozerlib/metadata.py index ec98456c6..6c5478e0a 100644 --- a/doozerlib/metadata.py +++ b/doozerlib/metadata.py @@ -1,30 +1,31 @@ -from typing import Dict, Optional, List, Tuple, Any, Union -import urllib.parse -import yaml -from collections import OrderedDict -import pathlib -import io import datetime -import time -import sys +import io +import pathlib import re +import sys +import time +import urllib.parse +from collections import OrderedDict from enum import Enum - +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union from xml.etree import ElementTree -from typing import NamedTuple -import dateutil.parser +import dateutil.parser +import requests +import yaml from dockerfile_parse import DockerfileParser -import doozerlib +from tenacity import (retry, retry_if_exception_type, stop_after_attempt, + wait_fixed) -from doozerlib.pushd import Dir -from doozerlib.distgit import ImageDistGitRepo, RPMDistGitRepo +import doozerlib from doozerlib import exectools, logutil +from doozerlib.assembly import assembly_basis_event, assembly_metadata_config from doozerlib.brew import BuildStates -from doozerlib.util import isolate_el_version_in_brew_tag, isolate_git_commit_in_release - -from doozerlib.model import Model, Missing -from doozerlib.assembly import assembly_metadata_config, assembly_basis_event +from doozerlib.distgit import ImageDistGitRepo, RPMDistGitRepo +from doozerlib.model import Missing, Model +from doozerlib.pushd import Dir +from doozerlib.util import (isolate_el_version_in_brew_tag, + isolate_git_commit_in_release) class CgitAtomFeedEntry(NamedTuple): @@ -311,14 +312,19 @@ def cgit_atom_feed(self, commit_hash: Optional[str] = None, branch: Optional[str branch = self.branch() if not commit_hash and branch: params["h"] = branch - if params: - url += "?" + urllib.parse.urlencode(params) - req = exectools.retry( - 3, lambda: urllib.request.urlopen(url), - check_f=lambda req: req.code == 200) + def _make_request(): + self.logger.info("Getting cgit atom feed %s ...", url) + resp = requests.get(url, params=params) + resp.raise_for_status() + return resp.text + + content = retry( + stop=stop_after_attempt(3), # wait for 10 seconds * 3 = 30 seconds + wait=wait_fixed(10), # wait for 10 seconds between retries + retry=retry_if_exception_type(), + )(_make_request)() - content = req.read() et = ElementTree.fromstring(content) entry_list = list()