Initial implementation

Small refactors

Remove rpmdiff code + refactors

Add rhcos file metadata

Lint fix

elliottlib/cli/find_builds_cli.py

@@ -10,15 +10,16 @@
 from errata_tool import ErrataException
 
 import elliottlib
-from elliottlib import Runtime, brew, constants, errata, logutil
-from elliottlib import exectools
+from elliottlib import Runtime, brew, constants, logutil, errata
+from elliottlib import exectools, util
 from elliottlib.assembly import assembly_metadata_config, assembly_rhcos_config
 from elliottlib.build_finder import BuildFinder
 from elliottlib.cli.common import (cli, find_default_advisory,
                                    use_default_advisory_option, click_coroutine)
 from elliottlib.exceptions import ElliottFatalError
 from elliottlib.imagecfg import ImageMetadata
-from elliottlib.util import (ensure_erratatool_auth, exit_unauthenticated,
+from elliottlib.cli.rhcos_cli import get_build_id_from_rhcos_pullspec
+from elliottlib.util import (ensure_erratatool_auth,
                              get_release_version, green_prefix, green_print,
                              isolate_el_version_in_brew_tag,
                              parallel_results_with_progress, pbar_header, progress_func,
@@ -36,7 +37,7 @@
 
 @cli.command('find-builds', short_help='Find or attach builds to ADVISORY')
 @click.option(
-    '--attach', '-a', 'advisory',
+    '--attach', '-a', 'advisory_id',
     type=int, metavar='ADVISORY',
     help='Attach the builds to ADVISORY (by default only a list of builds are displayed)')
 @use_default_advisory_option
@@ -46,8 +47,8 @@
     help='Add build NVR_OR_ID to ADVISORY [MULTIPLE]')
 @click.option(
     '--kind', '-k', metavar='KIND', required=True,
-    type=click.Choice(['rpm', 'image']),
-    help='Find builds of the given KIND [rpm, image]')
+    type=click.Choice(['rpm', 'image', 'rhcos']),
+    help='Find builds of the given KIND [rpm, image, rhcos]')
 @click.option(
     '--from-diff', '--between',
     required=False,
@@ -79,7 +80,7 @@
     help='(For rpms) Only sweep member rpms')
 @click_coroutine
 @pass_runtime
-async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, builds, kind, from_diff, as_json,
+async def find_builds_cli(runtime: Runtime, advisory_id, default_advisory_type, builds, kind, from_diff, as_json,
                           remove, clean, no_cdn_repos, payload, non_payload, include_shipped, member_only: bool):
     '''Automatically or manually find or attach/remove viable rpm or image builds
 to ADVISORY. Default behavior searches Brew for viable builds in the
@@ -133,7 +134,7 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
         raise click.BadParameter('Option --remove only support removing specific build with -b.')
     if from_diff and kind != "image":
         raise click.BadParameter('Option --from-diff/--between should be used with --kind/-k image.')
-    if advisory and default_advisory_type:
+    if advisory_id and default_advisory_type:
         raise click.BadParameter('Use only one of --use-default-advisory or --attach')
     if payload and non_payload:
         raise click.BadParameter('Use only one of --payload or --non-payload.')
@@ -144,7 +145,7 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
     tag_pv_map = et_data.get('brew_tag_product_version_mapping')
 
     if default_advisory_type is not None:
-        advisory = find_default_advisory(runtime, default_advisory_type)
+        advisory_id = find_default_advisory(runtime, default_advisory_type)
 
     ensure_erratatool_auth()  # before we waste time looking up builds we can't process
 
@@ -158,7 +159,7 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
         green_prefix('Fetching builds...')
         unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(builds, tag_pv_map, ignore_product_version=remove, brew_session=brew_session)
     elif clean:
-        unshipped_builds = errata.get_brew_builds(advisory)
+        unshipped_builds = errata.get_brew_builds(advisory_id)
     elif from_diff:
         unshipped_nvrps = _fetch_builds_from_diff(from_diff[0], from_diff[1], tag_pv_map)
     else:
@@ -167,13 +168,34 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
                                                                 non_payload, include_shipped)
         elif kind == 'rpm':
             unshipped_nvrps = await _fetch_builds_by_kind_rpm(runtime, tag_pv_map, brew_session, include_shipped, member_only)
+        elif kind == 'rhcos':
+            rhcos_config = assembly_rhcos_config(runtime.get_releases_config(), runtime.assembly)
+            build_ids_by_arch = dict()
+            nvrs = []
+            for _, tag_config in rhcos_config.items():
+                for arch, pullspec in tag_config['images'].items():
+                    build_id = get_build_id_from_rhcos_pullspec(pullspec, runtime.logger)
+                    if arch not in build_ids_by_arch:
+                        build_ids_by_arch[arch] = set()
+                    build_ids_by_arch[arch].add(build_id)
+
+            for arch, builds in build_ids_by_arch.items():
+                for build_id in builds:
+                    nvr = f'rhcos-{arch}-{build_id}'
+                    if brew_session.getBuild(nvr):
+                        runtime.logger.info(f'Found rhcos nvr: {nvr}')
+                        nvrs.append(nvr)
+                    else:
+                        runtime.logger.info(f'rhcos nvr not found: {nvr}')
+
+            unshipped_nvrps = _fetch_nvrps_by_nvr_or_id(nvrs, tag_pv_map, brew_session=brew_session)
 
     pbar_header(
         'Fetching builds from Errata: ',
         'Hold on a moment, fetching buildinfos from Errata Tool...',
         unshipped_builds if clean else unshipped_nvrps)
 
-    if not clean and not remove:
+    if not (clean or remove):
         # if is --clean then batch fetch from Erratum no need to fetch them individually
         # if is not for --clean fetch individually using nvrp tuples then get specific
         # elliottlib.brew.Build Objects by get_brew_build()
@@ -182,10 +204,11 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
         # Build(atomic-openshift-descheduler-container-v4.3.23-202005250821).
         unshipped_builds = parallel_results_with_progress(
             unshipped_nvrps,
-            lambda nvrp: elliottlib.errata.get_brew_build('{}-{}-{}'.format(nvrp[0], nvrp[1], nvrp[2]), nvrp[3], session=requests.Session())
+            lambda nvrp: errata.get_brew_build(f'{nvrp[0]}-{nvrp[1]}-{nvrp[2]}',
+                                               nvrp[3], session=requests.Session())
         )
         previous = len(unshipped_builds)
-        unshipped_builds = _filter_out_inviable_builds(kind, unshipped_builds, elliottlib.errata)
+        unshipped_builds = _filter_out_inviable_builds(unshipped_builds)
         if len(unshipped_builds) != previous:
             click.echo(f'Filtered out {previous - len(unshipped_builds)} inviable build(s)')
 
@@ -195,7 +218,7 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
             green_print('No builds needed to be attached.')
             return
 
-    if not advisory:
+    if not advisory_id:
         click.echo('The following {n} builds '.format(n=len(unshipped_builds)), nl=False)
         if not (remove or clean):
             click.secho('may be attached', bold=True, nl=False)
@@ -212,7 +235,7 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
         return
 
     try:
-        erratum = elliottlib.errata.Advisory(errata_id=advisory)
+        erratum = errata.Advisory(errata_id=advisory_id)
         erratum.ensure_state('NEW_FILES')
         if remove:
             to_remove = [f"{nvrp[0]}-{nvrp[1]}-{nvrp[2]}" for nvrp in unshipped_nvrps]
@@ -222,12 +245,31 @@ async def find_builds_cli(runtime: Runtime, advisory, default_advisory_type, bui
         if to_remove:
             erratum.remove_builds(to_remove)
         else:  # attach
-            erratum.attach_builds(unshipped_builds, kind)
+            erratum.attach_builds(unshipped_builds, 'image' if kind == 'rhcos' else kind)
             cdn_repos = et_data.get('cdn_repos')
-            if cdn_repos and not no_cdn_repos and kind == "image":
+            if kind == 'image' and cdn_repos and not no_cdn_repos:
                 erratum.set_cdn_repos(cdn_repos)
+
+            if kind == 'rhcos':
+                file_meta = errata.get_file_meta(advisory_id)
+                runtime.logger.info('Setting rhcos file metadata..')
+                rhcos_file_meta = []
+                for f in file_meta:
+                    # path is something like `/mnt/redhat/brewroot/packages/rhcos-x86_64/413.92.202307260246/0/images
+                    # /coreos-assembler-git.tar.gz`
+                    if 'rhcos' in f['path']:
+                        arch = None
+                        for a in util.brew_arches:
+                            if a in f['path']:
+                                arch = a
+                                break
+                        title = f'RHCOS Image metadata ({arch})'
+                        rhcos_file_meta.append({'file': f['id'], 'title': title})
+                if rhcos_file_meta:
+                    errata.put_file_meta(advisory_id, rhcos_file_meta)
+
     except ErrataException as e:
-        red_print(f'Cannot change advisory {advisory}: {e}')
+        red_print(f'Cannot change advisory {advisory_id}: {e}')
         exit(1)
 
 
@@ -431,10 +473,10 @@ async def _fetch_builds_by_kind_rpm(runtime: Runtime, tag_pv_map: Dict[str, str]
     return nvrps
 
 
-def _filter_out_inviable_builds(kind, results, errata):
+def _filter_out_inviable_builds(build_objects):
     unshipped_builds = []
     errata_version_cache = {}  # avoid reloading the same errata for multiple builds
-    for b in results:
+    for b in build_objects:
         # check if build is attached to any existing advisory for this version
         in_same_version = False
         for eid in [e['id'] for e in b.all_errata]:

elliottlib/errata.py

@@ -639,44 +639,6 @@ def add_jira_bugs_with_retry(advisory: Erratum, bugids: List[str], noop: bool =
                 raise e
 
 
-def get_rpmdiff_runs(advisory_id, status=None, session=None):
-    """ Get RPMDiff runs for a given advisory.
-    :param advisory_id: advisory number
-    :param status: If set, only returns RPMDiff runs in the status.
-    :param session: requests.Session object.
-    """
-    params = {
-        "filter[active]": "true",
-        "filter[test_type]": "rpmdiff",
-        "filter[errata_id]": advisory_id,
-    }
-    if status:
-        if status not in constants.ET_EXTERNAL_TEST_STATUSES:
-            raise ValueError("{} is not a valid RPMDiff run status.".format(status))
-        params["filter[status]"] = status
-    url = constants.errata_url + "/api/v1/external_tests"
-    if not session:
-        session = requests.Session()
-
-    # This is a paginated API. We need to increment page[number] until an empty array is returned.
-    # https://errata.devel.redhat.com/developer-guide/api-http-api.html#api-pagination
-    page_number = 1
-    while True:
-        params["page[number]"] = page_number
-        resp = session.get(
-            url,
-            params=params,
-            auth=HTTPSPNEGOAuth(),
-        )
-        resp.raise_for_status()
-        data = resp.json()["data"]
-        if not data:
-            break
-        for item in data:
-            yield item
-        page_number += 1
-
-
 def get_image_cdns(advisory_id):
     return errata_xmlrpc.get_advisory_cdn_docker_file_list(advisory_id)
 
@@ -855,3 +817,12 @@ def remove_dependent_advisories(advisory_id):
         if response.status_code != requests.codes.created:
             raise IOError(f'Failed to remove dependent {dependent} from {advisory_id}'
                           f'with code {response.status_code} and error: {response.text}')
+
+
+def get_file_meta(advisory_id):
+    return ErrataConnector()._get(f'/api/v1/erratum/{advisory_id}/filemeta')
+
+
+def put_file_meta(advisory_id, file_meta):
+    return ErrataConnector()._put(f'/api/v1/erratum/{advisory_id}/filemeta',
+                                  data=file_meta)

tests/test_errata.py

@@ -141,36 +141,6 @@ def test_parse_product_version(self):
 
         self.assertEqual(product_version_map, {'rhaos-4.1-rhel-8-candidate': 'OSE-4.1-RHEL-8'})
 
-    def test_get_rpmdiff_runs(self):
-        advisory_id = 12345
-        responses = [
-            {
-                "data": [
-                    {"id": 1},
-                    {"id": 2},
-                ]
-            },
-            {
-                "data": [
-                    {"id": 3},
-                ]
-            },
-            {
-                "data": []
-            },
-        ]
-        session = mock.MagicMock()
-
-        def mock_response(*args, **kwargs):
-            page_number = kwargs["params"]["page[number]"]
-            resp = mock.MagicMock()
-            resp.json.return_value = responses[page_number - 1]
-            return resp
-
-        session.get.side_effect = mock_response
-        actual = errata.get_rpmdiff_runs(advisory_id, None, session)
-        self.assertEqual(len(list(actual)), 3)
-
 
 class TestAdvisoryImages(unittest.TestCase):
 

tests/test_find_builds_cli.py

@@ -1,7 +1,7 @@
 import unittest
 from elliottlib.cli.find_builds_cli import _filter_out_inviable_builds, _find_shipped_builds
 from elliottlib.brew import Build
-import elliottlib
+from elliottlib import errata as erratalib
 from flexmock import flexmock
 import json
 from unittest import mock
@@ -12,43 +12,23 @@ class TestFindBuildsCli(unittest.TestCase):
     Test elliott find-builds command and internal functions
     """
 
-    def test_attached_errata_failed(self):
-        """
-        Test the internal wrapper function _attached_to_open_erratum_with_correct_product_version
-        attached_to_open_erratum = True, product_version is also the same:
-            _attached_to_open_erratum_with_correct_product_version() should return []
-        """
-        metadata_json_list = []
+    def test_filter_out_inviable_builds_inviable(self):
         metadata = json.loads('''{"release": "4.1", "kind": "rpm", "impetus": "cve"}''')
-        metadata_json_list.append(metadata)
-
-        fake_errata = flexmock(model=elliottlib.errata, get_metadata_comments_json=lambda: 1234)
-        fake_errata.should_receive("get_metadata_comments_json").and_return(metadata_json_list)
+        flexmock(erratalib).should_receive("get_metadata_comments_json").and_return([metadata])
 
         builds = flexmock(Build(nvr="test-1.1.1", product_version="RHEL-7-OSE-4.1"))
         builds.should_receive("all_errata").and_return([{"id": 12345}])
 
-        # expect return empty list []
-        self.assertEqual([], _filter_out_inviable_builds("image", [builds], fake_errata))
+        self.assertEqual([], _filter_out_inviable_builds([builds]))
 
-    def test_attached_errata_succeed(self):
-        """
-        Test the internal wrapper function _attached_to_open_erratum_with_correct_product_version
-        attached_to_open_erratum = True but product_version is not same:
-            _attached_to_open_erratum_with_correct_product_version() should return [Build("test-1.1.1")]
-        """
-        metadata_json_list = []
+    def test_filter_out_inviable_builds_viable(self):
         metadata = json.loads('''{"release": "4.1", "kind": "rpm", "impetus": "cve"}''')
-        metadata_json_list.append(metadata)
-
-        fake_errata = flexmock(model=elliottlib.errata, get_metadata_comments_json=lambda: 1234)
-        fake_errata.should_receive("get_metadata_comments_json").and_return(metadata_json_list)
+        flexmock(erratalib).should_receive("get_metadata_comments_json").and_return([metadata])
 
         builds = flexmock(Build(nvr="test-1.1.1", product_version="RHEL-7-OSE-4.5"))
         builds.should_receive("all_errata").and_return([{"id": 12345}])
 
-        # expect return list with one build
-        self.assertEqual([Build("test-1.1.1")], _filter_out_inviable_builds("image", [builds], fake_errata))
+        self.assertEqual([Build("test-1.1.1")], _filter_out_inviable_builds([builds]))
 
     @mock.patch("elliottlib.brew.get_builds_tags")
     def test_find_shipped_builds(self, get_builds_tags: mock.MagicMock):