Skip to content
This repository has been archived by the owner on Oct 13, 2023. It is now read-only.

Commit

Permalink
cgit_atom_feed: Better exception handling
Browse files Browse the repository at this point in the history
In case of HTTP error on requesting cgit atom feed, you will see an
exception like this:

```
python38/root/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not found
```

This is not good for troubleshooting.

This PR changes to use `requests` and `tenacity` library to handle HTTP
requests and retries. You will see the following exception in case of
HTTP error:

```
Traceback (most recent call last):
  File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 407, in __call__
    result = fn(*args, **kwargs)
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 319, in _make_request
    resp.raise_for_status()
  File "/Users/yux/Library/Python/3.10/lib/python/site-packages/requests/models.py", line 941, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not found for url: https://pkgs.devel.redhat.com/cgit/rpms/openshift-clients/atom/?h=rhaos-4.8-rhel-7

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/runtime.py", line 101, in wrapper
    return func(*args, **kwargs)
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/runtime.py", line 113, in wrapper
    return func(*args)
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/runtime.py", line 1584, in <lambda>
    lambda meta, _: (meta, meta.needs_rebuild()),
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 624, in needs_rebuild
    hint = self._target_needs_rebuild(el_target=target)
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 663, in _target_needs_rebuild
    atom_entries = self.cgit_atom_feed(commit_hash=distgit_commitish, branch=self.branch())
  File "/Volumes/Workspaces/projects/openshift/doozer/./doozerlib/metadata.py", line 322, in cgit_atom_feed
    content = retry(
  File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 324, in wrapped_f
    return self(f, *args, **kw)
  File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 404, in __call__
    do = self.iter(retry_state=retry_state)
  File "/Users/yux/Library/Python/3.10/lib/python/site-packages/tenacity/__init__.py", line 361, in iter
    raise retry_exc from fut.exception()
tenacity.RetryError: RetryError[<Future at 0x11160beb0 state=finished raised HTTPError>]
```
  • Loading branch information
vfreex committed Dec 13, 2022
1 parent 0464ac6 commit e590d85
Showing 1 changed file with 30 additions and 24 deletions.
54 changes: 30 additions & 24 deletions doozerlib/metadata.py
@@ -1,30 +1,31 @@
from typing import Dict, Optional, List, Tuple, Any, Union
import urllib.parse
import yaml
from collections import OrderedDict
import pathlib
import io
import datetime
import time
import sys
import io
import pathlib
import re
import sys
import time
import urllib.parse
from collections import OrderedDict
from enum import Enum

from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union
from xml.etree import ElementTree
from typing import NamedTuple
import dateutil.parser

import dateutil.parser
import requests
import yaml
from dockerfile_parse import DockerfileParser
import doozerlib
from tenacity import (retry, retry_if_exception_type, stop_after_attempt,
wait_fixed)

from doozerlib.pushd import Dir
from doozerlib.distgit import ImageDistGitRepo, RPMDistGitRepo
import doozerlib
from doozerlib import exectools, logutil
from doozerlib.assembly import assembly_basis_event, assembly_metadata_config
from doozerlib.brew import BuildStates
from doozerlib.util import isolate_el_version_in_brew_tag, isolate_git_commit_in_release

from doozerlib.model import Model, Missing
from doozerlib.assembly import assembly_metadata_config, assembly_basis_event
from doozerlib.distgit import ImageDistGitRepo, RPMDistGitRepo
from doozerlib.model import Missing, Model
from doozerlib.pushd import Dir
from doozerlib.util import (isolate_el_version_in_brew_tag,
isolate_git_commit_in_release)


class CgitAtomFeedEntry(NamedTuple):
Expand Down Expand Up @@ -311,14 +312,19 @@ def cgit_atom_feed(self, commit_hash: Optional[str] = None, branch: Optional[str
branch = self.branch()
if not commit_hash and branch:
params["h"] = branch
if params:
url += "?" + urllib.parse.urlencode(params)

req = exectools.retry(
3, lambda: urllib.request.urlopen(url),
check_f=lambda req: req.code == 200)
def _make_request():
self.logger.info("Getting cgit atom feed %s ...", url)
resp = requests.get(url, params=params)
resp.raise_for_status()
return resp.text

content = retry(
stop=stop_after_attempt(3), # wait for 10 seconds * 3 = 30 seconds
wait=wait_fixed(10), # wait for 10 seconds between retries
retry=retry_if_exception_type(),
)(_make_request)()

content = req.read()
et = ElementTree.fromstring(content)

entry_list = list()
Expand Down

0 comments on commit e590d85

Please sign in to comment.