From 289b2ae943958752b5dcdbeda4483d50cf5e0665 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Thu, 17 Jun 2021 12:55:01 +1000 Subject: [PATCH] Add preliminary support for container-related push items This commit introduces support for two new push item types which can now be located by errata or koji source: - ContainerImagePushItem: a container image - OperatorManifestPushItem: operator manifests bundle typically found alongside container images in koji This is referred to as "preliminary support" because these classes haven't defined any attributes yet (i.e. they only have the inherited attributes from the base PushItem class). This is done to avoid the change becoming too large and complicated. In this commit, we focus only on how we locate these push items when given an advisory; filling out the model with all needed attributes will come later, and it's expected to be rather complicated to figure out what's needed there. --- CHANGELOG.md | 3 + docs/index.rst | 1 + docs/model/containers.rst | 8 + src/pushsource/__init__.py | 2 + .../backend/errata_source/errata_client.py | 5 + .../backend/errata_source/errata_source.py | 105 ++++- src/pushsource/_impl/backend/koji_source.py | 140 ++++++- src/pushsource/_impl/helpers.py | 28 ++ src/pushsource/_impl/model/__init__.py | 1 + src/pushsource/_impl/model/container.py | 22 + .../errata/data/RHBA-2020:2807-no-repos.yaml | 250 ++++++++++++ tests/errata/data/RHBA-2020:2807.yaml | 264 ++++++++++++ tests/errata/data/dump | 2 + tests/errata/fake_errata_tool.py | 3 + tests/errata/test_errata_containers.py | 378 ++++++++++++++++++ tests/helpers/test_argument_helpers.py | 30 +- ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ ...tainer-v4.3.28.202006290519.p0.prod-1.yaml | 94 +++++ tests/koji/data/dump | 99 +++++ tests/koji/fake_koji.py | 27 ++ 27 files changed, 2209 insertions(+), 5 deletions(-) create mode 100644 docs/model/containers.rst create mode 100644 src/pushsource/_impl/model/container.py create mode 100644 tests/errata/data/RHBA-2020:2807-no-repos.yaml create mode 100644 tests/errata/data/RHBA-2020:2807.yaml create mode 100644 tests/errata/test_errata_containers.py create mode 100644 tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100644 tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml create mode 100755 tests/koji/data/dump diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e800213..b164d6d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 by architecture. - Added `ModuleMdSourcePushItem` class. Source modulemd documents are now represented by this class rather than `ModuleMdPushItem`. +- Added minimal `ContainerImagePushItem` and `OperatorManifestPushItem` classes for + container images. These classes currently are of limited use as they do not yet + carry relevant metadata. ### Changed diff --git a/docs/index.rst b/docs/index.rst index 38df4440..338e6f10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ A library for accessing push items from various sources. model/rpm model/modulemd model/errata + model/containers model/ami model/productid model/comps diff --git a/docs/model/containers.rst b/docs/model/containers.rst new file mode 100644 index 00000000..68e32bf6 --- /dev/null +++ b/docs/model/containers.rst @@ -0,0 +1,8 @@ +Push items: containers +====================== + +.. autoclass:: pushsource.ContainerImagePushItem() + :members: + +.. autoclass:: pushsource.OperatorManifestPushItem() + :members: diff --git a/src/pushsource/__init__.py b/src/pushsource/__init__.py index c63a3013..d79774e5 100644 --- a/src/pushsource/__init__.py +++ b/src/pushsource/__init__.py @@ -7,6 +7,8 @@ ModuleMdSourcePushItem, ProductIdPushItem, RpmPushItem, + ContainerImagePushItem, + OperatorManifestPushItem, AmiPushItem, AmiRelease, AmiBillingCodes, diff --git a/src/pushsource/_impl/backend/errata_source/errata_client.py b/src/pushsource/_impl/backend/errata_source/errata_client.py index fc333dd1..9086637c 100644 --- a/src/pushsource/_impl/backend/errata_source/errata_client.py +++ b/src/pushsource/_impl/backend/errata_source/errata_client.py @@ -16,6 +16,7 @@ class ErrataRaw(object): # Helper to collect raw responses of all ET APIs for a single advisory advisory_cdn_metadata = attr.ib(type=dict) advisory_cdn_file_list = attr.ib(type=dict) + advisory_cdn_docker_file_list = attr.ib(type=dict) ftp_paths = attr.ib(type=dict) @@ -31,6 +32,9 @@ def __init__(self, threads, url): self._get_advisory_cdn_file_list = partial( self._call_et, "get_advisory_cdn_file_list" ) + self._get_advisory_cdn_docker_file_list = partial( + self._call_et, "get_advisory_cdn_docker_file_list" + ) self._get_ftp_paths = partial(self._call_et, "get_ftp_paths") @property @@ -47,6 +51,7 @@ def get_raw_f(self, advisory_id): all_responses = f_zip( self._executor.submit(self._get_advisory_cdn_metadata, advisory_id), self._executor.submit(self._get_advisory_cdn_file_list, advisory_id), + self._executor.submit(self._get_advisory_cdn_docker_file_list, advisory_id), self._executor.submit(self._get_ftp_paths, advisory_id), ) return f_map(all_responses, lambda tup: ErrataRaw(*tup)) diff --git a/src/pushsource/_impl/backend/errata_source/errata_source.py b/src/pushsource/_impl/backend/errata_source/errata_source.py index abfd8763..a536022a 100644 --- a/src/pushsource/_impl/backend/errata_source/errata_source.py +++ b/src/pushsource/_impl/backend/errata_source/errata_source.py @@ -10,8 +10,15 @@ from ... import compat_attr as attr from ...source import Source -from ...model import ErratumPushItem, RpmPushItem, ModuleMdSourcePushItem, conv -from ...helpers import list_argument +from ...model import ( + ErratumPushItem, + ContainerImagePushItem, + ModuleMdSourcePushItem, + RpmPushItem, + OperatorManifestPushItem, + conv, +) +from ...helpers import list_argument, try_bool LOG = logging.getLogger("pushsource") @@ -25,6 +32,7 @@ def __init__( errata, koji_source=None, rpm_filter_arch=None, + legacy_container_repos=False, threads=4, timeout=60 * 60 * 4, ): @@ -48,6 +56,14 @@ def __init__( If provided, only RPMs for these given arch(es) will be produced; e.g. "x86_64", "src" and "noarch". + legacy_container_repos (bool) + If ``True``, any container push items generated by this source will + use a legacy format for repository IDs. + + This is intended to better support certain legacy code and will be + removed when no longer needed. Only use this if you know that you + need it. + threads (int) Number of threads used for concurrent queries to Errata Tool and koji. @@ -71,6 +87,8 @@ def __init__( self._koji_executor = Executors.thread_pool(max_workers=threads).with_retry() self._koji_cache = {} self._koji_source_url = koji_source + + self._legacy_container_repos = try_bool(legacy_container_repos) self._timeout = timeout @property @@ -119,8 +137,91 @@ def _push_items_from_raw(self, raw): # Adjust push item destinations according to FTP paths from ET, if any. items = self._add_ftp_paths(items, erratum, raw.ftp_paths) + items = items + self._push_items_from_container_manifests( + erratum, raw.advisory_cdn_docker_file_list + ) + return [erratum] + items + def _push_items_from_container_manifests(self, erratum, docker_file_list): + if not docker_file_list: + return [] + + # Example of container list for one item to one repo: + # + # { + # "dotnet-21-container-2.1-77.1621419388": { + # "docker": { + # "target": { + # "external_repos": { + # "rhel8/dotnet-21": { + # "container_full_sig_key": "199e2f91fd431d51", + # "container_sig_key": "fd431d51", + # "tags": ["2.1", "2.1-77.1621419388", "latest"], + # }, + # }, + # "repos": /* like external_repos but uses pulp repo IDs */, + # } + # } + # } + # } + # + + # We'll be getting container metadata from these builds. + koji_source = self._koji_source(container_build=list(docker_file_list.keys())) + + out = [] + + for item in koji_source: + if isinstance(item, ContainerImagePushItem): + item = self._enrich_container_push_item(erratum, docker_file_list, item) + elif isinstance(item, OperatorManifestPushItem): + # Accept this item but nothing special to do + pass + else: + # If build contained anything else, ignore it + LOG.debug( + "Erratum %s: ignored unexpected item from koji source: %s", + erratum.name, + item, + ) + continue + + item = attr.evolve(item, origin=erratum.name) + out.append(item) + + return out + + def _enrich_container_push_item(self, erratum, docker_file_list, item): + # metadata from koji doesn't contain info about where the image should be + # pushed and a few other things - enrich it now + errata_meta = docker_file_list.get(item.build) or {} + target = (errata_meta.get("docker") or {}).get("target") or {} + + repos = target.get("external_repos") or {} + + if self._legacy_container_repos: + repos = target.get("repos") or {} + + dest = [] + for repo_id, repo_data in repos.items(): + for tag in repo_data.get("tags") or []: + dest.append("%s:%s" % (repo_id, tag)) + + dest = sorted(set(dest)) + + # If ET is not requesting to push this to any repos or tags at all, + # it's considered an error. + if not dest: + raise ValueError( + "Erratum %s requests container build %s but provides no repositories" + % (erratum.name, item.build) + ) + + # koji source provided basic info on container image, ET provides policy on + # where/how it should be pushed, combine them both to get final push item + return attr.evolve(item, dest=dest) + def _push_items_from_rpms(self, erratum, rpm_list): out = [] diff --git a/src/pushsource/_impl/backend/koji_source.py b/src/pushsource/_impl/backend/koji_source.py index 0afc5b99..1152b6f6 100644 --- a/src/pushsource/_impl/backend/koji_source.py +++ b/src/pushsource/_impl/backend/koji_source.py @@ -12,13 +12,21 @@ from more_executors.futures import f_map from ..source import Source -from ..model import RpmPushItem, ModuleMdPushItem, ModuleMdSourcePushItem +from ..model import ( + RpmPushItem, + ModuleMdPushItem, + ModuleMdSourcePushItem, + OperatorManifestPushItem, + ContainerImagePushItem, +) from ..helpers import list_argument, try_int from .modulemd import Module LOG = logging.getLogger("pushsource") CACHE_LOCK = threading.RLock() +MIME_TYPE_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" + class ListArchivesCommand(object): def __init__(self, build): @@ -104,6 +112,7 @@ def __init__( rpm=None, module_build=None, module_filter_filename=None, + container_build=None, signing_key=None, basedir=None, threads=4, @@ -143,6 +152,12 @@ def __init__( Has no effect if ``module_build`` is not provided. + container_build (list[str, int]) + Build identifier(s). Can be koji IDs (integers) or build NVRs. + The source will yield all container images produced by these + build(s), along with associated artifacts such as operator manifest + bundles. + signing_key (list[str]) GPG signing key ID(s). If provided, content must be signed using one of the provided keys. Include ``None`` if unsigned @@ -179,6 +194,7 @@ def __init__( self._module_filter_filename = list_argument( module_filter_filename, retain_none=True ) + self._container_build = [try_int(x) for x in list_argument(container_build)] self._signing_key = list_argument(signing_key) self._dest = list_argument(dest) self._timeout = timeout @@ -342,6 +358,107 @@ def _push_items_from_module_build(self, nvr, meta): ) return out + def _push_items_from_container_build(self, nvr, meta): + LOG.debug("Looking for container images on %s, %s", nvr, meta) + + if not meta: + message = "Container image build not found in koji: %s" % nvr + LOG.error(message) + raise ValueError(message) + + # The metadata we are interested in is documented here: + # https://github.com/containerbuildsystem/atomic-reactor/blob/061d92e63cf27ae030e8ceed388ec34f51afb17b/docs/koji.md#type-specific-metadata + extra = meta.get("extra") or {} + typeinfo = extra.get("typeinfo") or {} + # Per above doc, it is preferred to use the metadata under 'typeinfo' if present + image = typeinfo.get("image") or extra.get("image") + + if not image: + message = "Build %s not recognized as a container image build" % nvr + LOG.error(message) + raise ValueError(message) + + build_id = meta["id"] + + archives = self._get_archives(build_id) + + image_archives = [ + elem + for elem in archives + if elem.get("btype") == "image" and elem.get("type_name") == "tar" + ] + + media_types = image.get("media_types") or [] + if MIME_TYPE_MANIFEST_LIST in media_types: + # in manifest list case it is OK to have multiple archives (different arches). + pass + elif len(image_archives) != 1: + # If we don't have a manifest list, we expect to have an image for one arch. + # If there's not exactly one, it's unclear what's going on and we'd better halt. + message = ( + "Could not find (exactly) one container image archive on koji build %s" + % nvr + ) + LOG.error(message) + raise ValueError(message) + + out = [] + + for archive in image_archives: + path = self._pathinfo.typedir(meta, archive["btype"]) + item_src = os.path.join(path, archive["filename"]) + + out.append( + ContainerImagePushItem( + # TODO: name might be changed once we start parsing + # the metadata from atomic-reactor. + name=archive["filename"], + dest=self._dest, + src=item_src, + build=nvr, + ) + ) + + # If this build had any operator-manifests archive, add that too. + operator = self._get_operator_item(nvr, meta, archives) + if operator: + out.append(operator) + + return out + + def _get_operator_item(self, nvr, meta, archives): + extra = meta.get("extra") or {} + typeinfo = extra.get("typeinfo") or {} + operator_manifests = typeinfo.get("operator-manifests") or {} + archive_name = operator_manifests.get("archive") + + if not archive_name: + # Try legacy form + archive_name = extra.get("operator_manifests_archive") + + if not archive_name: + # No operator manifests on this build + return + + operator_archive = [a for a in archives if a["filename"] == archive_name] + if len(operator_archive) != 1: + message = ( + "koji build %s metadata refers to missing operator-manifests " + 'archive "%s"' + ) % (nvr, archive_name) + raise ValueError(message) + + archive = operator_archive[0] + path = self._pathinfo.typedir(meta, archive["btype"]) + item_src = os.path.join(path, archive["filename"]) + + return OperatorManifestPushItem( + name=os.path.join(nvr, archive_name), + dest=self._dest, + src=item_src, + build=nvr, + ) + def _rpm_futures(self): # Get info from each requested RPM. rpm_fs = [(self._executor.submit(self._get_rpm, rpm), rpm) for rpm in self._rpm] @@ -364,6 +481,19 @@ def _modulemd_futures(self): for f, build in module_build_fs ] + def _container_futures(self): + # Get info from each requested container build. + build_fs = [ + (self._executor.submit(self._get_build, build), build) + for build in self._container_build + ] + + # Convert them to lists of push items + return [ + f_map(f, partial(self._push_items_from_container_build, build)) + for f, build in build_fs + ] + def _do_fetch(self, koji_queue, exceptions): try: done = False @@ -421,6 +551,10 @@ def __iter__(self): for build_id in self._module_build: koji_queue.put(GetBuildCommand(ident=build_id, list_archives=True)) + # We'll need to obtain all container image builds + for build_id in self._container_build: + koji_queue.put(GetBuildCommand(ident=build_id, list_archives=True)) + # Put some threads to work on the queue. fetch_exceptions = [] fetch_threads = [ @@ -446,7 +580,9 @@ def __iter__(self): # The queue must be empty now assert koji_queue.empty() - push_items_fs = self._modulemd_futures() + self._rpm_futures() + push_items_fs = ( + self._modulemd_futures() + self._rpm_futures() + self._container_futures() + ) completed_fs = futures.as_completed(push_items_fs, timeout=self._timeout) for f in completed_fs: diff --git a/src/pushsource/_impl/helpers.py b/src/pushsource/_impl/helpers.py index a1614d04..2216d578 100644 --- a/src/pushsource/_impl/helpers.py +++ b/src/pushsource/_impl/helpers.py @@ -57,3 +57,31 @@ def try_int(value): return int(value) except ValueError: return value + + +def try_bool(value): + """Convert values into booleans where possible. + + This function is intended for use in :class:`~pushsource.Source` constructors + to ease the processing of arguments passed via URL (which always arrive as a string). + + Parameters: + value + Any value. + + Returns: + bool + ``value`` converted to a boolean, if it was a string representation + of a boolean. + other + ``value`` unmodified, in any other case. + """ + if not isinstance(value, six.string_types): + return value + + value = value.lower() + if value in ["1", "true", "yes"]: + return True + elif value in ["", "0", "false", "no"]: + return False + raise ValueError("Expected a boolean, got %s" % repr(value)) diff --git a/src/pushsource/_impl/model/__init__.py b/src/pushsource/_impl/model/__init__.py index 692e0154..5f7ecd75 100644 --- a/src/pushsource/_impl/model/__init__.py +++ b/src/pushsource/_impl/model/__init__.py @@ -8,6 +8,7 @@ ) from .rpm import RpmPushItem from .file import FilePushItem +from .container import ContainerImagePushItem, OperatorManifestPushItem from .modulemd import ModuleMdPushItem, ModuleMdSourcePushItem from .comps import CompsXmlPushItem from .productid import ProductIdPushItem diff --git a/src/pushsource/_impl/model/container.py b/src/pushsource/_impl/model/container.py new file mode 100644 index 00000000..97bc1238 --- /dev/null +++ b/src/pushsource/_impl/model/container.py @@ -0,0 +1,22 @@ +from .base import PushItem +from .. import compat_attr as attr + + +@attr.s() +class ContainerImagePushItem(PushItem): + """A :class:`~pushsource.PushItem` representing a container image. + + This item represents any kind of image which can be pulled + from a container image registry (i.e. an object identified by a + ``NAME[:TAG|@DIGEST]`` string). + + In the case of a multi-arch image, one ContainerImagePushItem represents + an image for a single architecture only. + """ + + +@attr.s() +class OperatorManifestPushItem(PushItem): + """A :class:`~pushsource.PushItem` representing an operator manifests archive + (typically named ``operator_manifests.zip``). + """ diff --git a/tests/errata/data/RHBA-2020:2807-no-repos.yaml b/tests/errata/data/RHBA-2020:2807-no-repos.yaml new file mode 100644 index 00000000..53029ec9 --- /dev/null +++ b/tests/errata/data/RHBA-2020:2807-no-repos.yaml @@ -0,0 +1,250 @@ +# A copy of RHBA-2020:2807 which has removed all repos from one build +# (expected to trigger an error) +advisory_id: RHBA-2020:2807-no-repos +cdn_docker_file_list: + cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + # no repos here + external_repos: {} + repos: {} + cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 +cdn_file_list: {} +cdn_metadata: + cdn_repo: + - rhel-7-for-power-le-ose-4_DOT_3-rpms__ppc64le + - rhel-7-for-system-z-ose-4_DOT_3-rpms__s390x + - rhel-7-server-ose-4_DOT_3-rpms__x86_64 + - rhocp-4_DOT_3-for-rhel-8-ppc64le-rpms + - rhocp-4_DOT_3-for-rhel-8-s390x-rpms + - rhocp-4_DOT_3-for-rhel-8-x86_64-rpms + description: 'Red Hat OpenShift Container Platform is Red Hat''s cloud computing + + Kubernetes application platform solution designed for on-premise or private + + cloud deployments. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers. + + + All OpenShift Container Platform 4.3 users are advised to upgrade to these + + updated packages and images when they are available in the appropriate + + release channel. To check for available updates, use the OpenShift Console + + or the CLI oc command. Instructions for upgrading a cluster are available + + at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-between-minor.html#understanding-upgrade-channels_updating-cluster-between-minor.' + from: release-engineering@redhat.com + id: RHBA-2020:2807 + issued: 2020-07-08 22:16:09 UTC + pkglist: + - name: RHBA-2020:2807 + packages: [] + short: '' + pulp_user_metadata: + content_types: + - docker + pushcount: '3' + reboot_suggested: false + references: + - href: https://access.redhat.com/errata/RHBA-2020:2807 + id: RHBA-2020:2807 + title: RHBA-2020:2807 + type: self + - href: https://bugzilla.redhat.com/show_bug.cgi?id=1850434 + id: '1850434' + title: Placeholder bug for OCP 4.3.z metadata release + type: bugzilla + release: '0' + rights: Copyright 2020 Red Hat Inc + severity: None + solution: 'For OpenShift Container Platform 4.3 see the following documentation, + which + + will be updated shortly for release 4.3.28, for important instructions on + + how to upgrade your cluster and fully apply this asynchronous errata + + update: + + + https://docs.openshift.com/container-platform/4.3/release_notes/ocp-4-3-release-notes.html + + + Details on how to access this content are available at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-cli.html.' + status: final + summary: 'Red Hat OpenShift Container Platform release 4.3.28 is now available with + + updates to packages and images that fix several bugs. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers.' + title: OpenShift Container Platform 4.3.28 OLM Operators metadata update + type: bugfix + updated: 2020-07-08 22:16:09 UTC + version: '3' +ftp_paths: {} diff --git a/tests/errata/data/RHBA-2020:2807.yaml b/tests/errata/data/RHBA-2020:2807.yaml new file mode 100644 index 00000000..41664321 --- /dev/null +++ b/tests/errata/data/RHBA-2020:2807.yaml @@ -0,0 +1,264 @@ +# Content in this advisory includes: +# - multi-arch container images +# - operator manifests +advisory_id: RHBA-2020:2807 +cdn_docker_file_list: + cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-cluster-logging-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-cluster-logging-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 +cdn_file_list: {} +cdn_metadata: + cdn_repo: + - rhel-7-for-power-le-ose-4_DOT_3-rpms__ppc64le + - rhel-7-for-system-z-ose-4_DOT_3-rpms__s390x + - rhel-7-server-ose-4_DOT_3-rpms__x86_64 + - rhocp-4_DOT_3-for-rhel-8-ppc64le-rpms + - rhocp-4_DOT_3-for-rhel-8-s390x-rpms + - rhocp-4_DOT_3-for-rhel-8-x86_64-rpms + description: 'Red Hat OpenShift Container Platform is Red Hat''s cloud computing + + Kubernetes application platform solution designed for on-premise or private + + cloud deployments. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers. + + + All OpenShift Container Platform 4.3 users are advised to upgrade to these + + updated packages and images when they are available in the appropriate + + release channel. To check for available updates, use the OpenShift Console + + or the CLI oc command. Instructions for upgrading a cluster are available + + at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-between-minor.html#understanding-upgrade-channels_updating-cluster-between-minor.' + from: release-engineering@redhat.com + id: RHBA-2020:2807 + issued: 2020-07-08 22:16:09 UTC + pkglist: + - name: RHBA-2020:2807 + packages: [] + short: '' + pulp_user_metadata: + content_types: + - docker + pushcount: '3' + reboot_suggested: false + references: + - href: https://access.redhat.com/errata/RHBA-2020:2807 + id: RHBA-2020:2807 + title: RHBA-2020:2807 + type: self + - href: https://bugzilla.redhat.com/show_bug.cgi?id=1850434 + id: '1850434' + title: Placeholder bug for OCP 4.3.z metadata release + type: bugzilla + release: '0' + rights: Copyright 2020 Red Hat Inc + severity: None + solution: 'For OpenShift Container Platform 4.3 see the following documentation, + which + + will be updated shortly for release 4.3.28, for important instructions on + + how to upgrade your cluster and fully apply this asynchronous errata + + update: + + + https://docs.openshift.com/container-platform/4.3/release_notes/ocp-4-3-release-notes.html + + + Details on how to access this content are available at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-cli.html.' + status: final + summary: 'Red Hat OpenShift Container Platform release 4.3.28 is now available with + + updates to packages and images that fix several bugs. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers.' + title: OpenShift Container Platform 4.3.28 OLM Operators metadata update + type: bugfix + updated: 2020-07-08 22:16:09 UTC + version: '3' +ftp_paths: {} diff --git a/tests/errata/data/dump b/tests/errata/data/dump index 9457ed0d..eb9d1ad2 100755 --- a/tests/errata/data/dump +++ b/tests/errata/data/dump @@ -15,6 +15,8 @@ def run(server_url, advisory_id): "advisory_id": advisory_id, "cdn_metadata": server.get_advisory_cdn_metadata(advisory_id), "cdn_file_list": server.get_advisory_cdn_file_list(advisory_id), + "cdn_docker_file_list": server.get_advisory_cdn_docker_file_list(advisory_id), + "ftp_paths": server.get_ftp_paths(advisory_id), } data_file = os.path.join(DATADIR, "%s.yaml" % advisory_id) diff --git a/tests/errata/fake_errata_tool.py b/tests/errata/fake_errata_tool.py index c0136029..ea3dcc34 100644 --- a/tests/errata/fake_errata_tool.py +++ b/tests/errata/fake_errata_tool.py @@ -50,5 +50,8 @@ def get_advisory_cdn_file_list(self, advisory_id): def get_advisory_cdn_metadata(self, advisory_id): return self._get_data(advisory_id, "cdn_metadata") + def get_advisory_cdn_docker_file_list(self, advisory_id): + return self._get_data(advisory_id, "cdn_docker_file_list") + def get_ftp_paths(self, advisory_id): return self._get_data(advisory_id, "ftp_paths") diff --git a/tests/errata/test_errata_containers.py b/tests/errata/test_errata_containers.py new file mode 100644 index 00000000..14211ef5 --- /dev/null +++ b/tests/errata/test_errata_containers.py @@ -0,0 +1,378 @@ +import os + +import pytest + +from pushsource import Source, ContainerImagePushItem, OperatorManifestPushItem + + +@pytest.fixture +def source_factory(fake_errata_tool, fake_koji, koji_dir): + ctor = Source.get_partial( + "errata:https://errata.example.com", + koji_source="koji:https://koji.example.com?basedir=%s" % koji_dir, + ) + + for nvr in [ + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + ]: + fake_koji.load_build(nvr) + + # Tweak this one build to have legacy reference to operator_manifests_archive + # to prove that this still works. + build = fake_koji.build_data[ + "cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + extra = build["extra"] + extra["operator_manifests_archive"] = extra["typeinfo"]["operator-manifests"].pop( + "archive" + ) + + # And tweak this one to not have any operator archive at all to prove this works too. + build = fake_koji.build_data[ + "sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + extra = build["extra"] + del extra["typeinfo"]["operator-manifests"] + + yield ctor + + +def test_errata_containers(source_factory, koji_dir): + """Errata source can be used to retrieve container-related push items, + typical scenario""" + + source = source_factory(errata="RHBA-2020:2807") + + items = list(source) + items = sorted(items, key=lambda item: item.name) + + image_items = [i for i in items if isinstance(i, ContainerImagePushItem)] + operator_items = [i for i in items if isinstance(i, OperatorManifestPushItem)] + + # Helper to make 'src' attributes below a bit less cumbersome + def koji_path(*args): + return os.path.join(koji_dir, *args) + + # This is quite a large set of very similar images. We'll only do a full check on the first few + # to avoid the test becoming huge. + assert image_items[0:4] == [ + ContainerImagePushItem( + name="docker-image-sha256:05db9dc3ed76f6cf993f0dd35aa51892a38ba51afdf8ed564a236d53d8375a47.x86_64.tar.gz", + state="PENDING", + src=koji_path( + "packages/sriov-network-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:05db9dc3ed76f6cf993f0dd35aa51892a38ba51afdf8ed564a236d53d8375a47.x86_64.tar.gz", + ), + dest=[ + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ContainerImagePushItem( + name="docker-image-sha256:088c14ddcc989815bb38871fe9e0d3387a762c07a2ca23c9fd375ce42e3e53d6.ppc64le.tar.gz", + state="PENDING", + src=koji_path( + "packages/elasticsearch-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:088c14ddcc989815bb38871fe9e0d3387a762c07a2ca23c9fd375ce42e3e53d6.ppc64le.tar.gz", + ), + dest=[ + "openshift4/ose-elasticsearch-rhel7-operator-metadata:v4.3", + "openshift4/ose-elasticsearch-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-elasticsearch-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ContainerImagePushItem( + name="docker-image-sha256:115387bd7efba2bdb7ee9f0d4fff88f03c1579859c6230cc67bb5c0b31263ba8.ppc64le.tar.gz", + state="PENDING", + src=koji_path( + "packages/sriov-network-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:115387bd7efba2bdb7ee9f0d4fff88f03c1579859c6230cc67bb5c0b31263ba8.ppc64le.tar.gz", + ), + dest=[ + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ContainerImagePushItem( + name="docker-image-sha256:176277a797bafa67823741efa319c2844758b2d5a4cb796b75a8b67580b54d56.aarch64.tar.gz", + state="PENDING", + src=koji_path( + "packages/ose-metering-ansible-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:176277a797bafa67823741efa319c2844758b2d5a4cb796b75a8b67580b54d56.aarch64.tar.gz", + ), + dest=[ + "openshift4/ose-metering-ansible-rhel7-operator-metadata:v4.3", + "openshift4/ose-metering-ansible-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-metering-ansible-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ] + + # Similarly for operators. + assert operator_items[0:4] == [ + OperatorManifestPushItem( + name="cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/cluster-logging-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + # Note that operators don't have any 'dest' assigned. + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + OperatorManifestPushItem( + name="cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/cluster-nfd-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + OperatorManifestPushItem( + name="elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/elasticsearch-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + OperatorManifestPushItem( + name="local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/local-storage-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ] + + +def test_errata_legacy_repos(source_factory): + """Errata source can make use of legacy repo IDs as needed""" + source1 = source_factory(errata="RHBA-2020:2807") + source2 = source_factory(errata="RHBA-2020:2807", legacy_container_repos=True) + + items1 = sorted( + [i for i in source1 if isinstance(i, ContainerImagePushItem)], + key=lambda i: i.name, + ) + items2 = sorted( + [i for i in source2 if isinstance(i, ContainerImagePushItem)], + key=lambda i: i.name, + ) + + # OK, we've got two lists of push items generated identically except that one requested + # legacy repos and the other didn't. Let's sample the first push item to see how they differ. + + assert items1[0].dest == [ + # This is the preferred format. + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ] + + assert items2[0].dest == [ + # This is the weird legacy stuff. + "redhat-openshift4-ose-sriov-network-rhel7-operator-metadata:v4.3", + "redhat-openshift4-ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "redhat-openshift4-ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ] + + +def test_errata_containers_no_repos(source_factory, koji_dir): + """Errata source fails to retrieve container-related push items if ET requests no repos""" + + source = source_factory(errata="RHBA-2020:2807-no-repos") + + # It should fail + with pytest.raises(ValueError) as exc_info: + list(source) + + # And tell us why + assert ( + "Erratum RHBA-2020:2807 requests container build " + "cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 " + "but provides no repositories" + ) in str(exc_info) + + +def test_errata_containers_missing_build(source_factory, fake_koji): + """Errata source gives an error if a requested container build is missing""" + + fake_koji.remove_build( + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ) + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "Container image build not found in koji: " + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ) in str(exc_info) + + +def test_errata_ignores_unknown_koji_types(source_factory, koji_dir): + """Errata source, when requesting containers, will skip unknown push item types + yielded by koji source.""" + + # This is a very niche case, but to get that 100% coverage... + # It's possible that koji_source might produce something other than ContainerImagePushItem + # or OperatorManifestPushItem, and we want to be forwards-compatible with that. + + # This is our hacked source which returns whatever koji returns, but also some + # arbitrary objects + class WeirdKoji(object): + def __init__(self, **kwargs): + # Get a normal koji source... + self.koji = Source.get( + "koji:https://koji.example.com?basedir=%s" % koji_dir, **kwargs + ) + + def __iter__(self): + # We'll yield whatever koji yields but surround it with + # unexpected junk + yield object() + for item in self.koji: + yield item + yield object() + + Source.register_backend("weird-koji", WeirdKoji) + + source = source_factory(errata="RHBA-2020:2807", koji_source="weird-koji:") + + # It should still work as normal + items = list(source) + + # Sanity check we got the right number of items + assert len(items) == 45 + + +def test_errata_containers_missing_image_meta(source_factory, fake_koji): + """Errata source gives an error if a requested container build exists but + image data is missing from build. + """ + + build = fake_koji.build_data[ + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + del build["extra"] + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "Build elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 " + "not recognized as a container image build" + ) in str(exc_info) + + +def test_errata_containers_broken_multiarch(source_factory, fake_koji): + """Errata source gives an error if a requested container build exists, has multiple + image archives and misses the manifest list mimetype in metadata. + """ + + build = fake_koji.build_data[ + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + del build["extra"]["typeinfo"]["image"]["media_types"] + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "Could not find (exactly) one container image archive on koji build " + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ) in str(exc_info) + + +def test_errata_containers_broken_operator_reference(source_factory, fake_koji): + """Errata source gives an error if a requested container build exists and points + at an operator archive, but the archive is missing from koji. + """ + + fake_koji.remove_archive( + "operator_manifests.zip", + "cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + ) + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "koji build cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 " + 'metadata refers to missing operator-manifests archive "operator_manifests.zip"' + ) in str(exc_info) diff --git a/tests/helpers/test_argument_helpers.py b/tests/helpers/test_argument_helpers.py index 19ba8153..0822abbe 100644 --- a/tests/helpers/test_argument_helpers.py +++ b/tests/helpers/test_argument_helpers.py @@ -1,4 +1,6 @@ -from pushsource._impl.helpers import list_argument +import pytest + +from pushsource._impl.helpers import list_argument, try_bool def test_list_argument(): @@ -7,3 +9,29 @@ def test_list_argument(): assert list_argument(123) == [123] assert list_argument("123") == ["123"] assert list_argument("123,456,789") == ["123", "456", "789"] + + +@pytest.mark.parametrize( + "input,expected_output", + [ + (True, True), + (False, False), + (12345, 12345), + ("1", True), + ("true", True), + ("tRUe", True), + ("yes", True), + ("0", False), + ("false", False), + ("FaLsE", False), + ("no", False), + ("", False), + ], +) +def test_try_bool(input, expected_output): + assert try_bool(input) == expected_output + + +def test_try_bool_invalid(): + with pytest.raises(ValueError): + try_bool("this ain't no boolean") diff --git a/tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..e2e4f3fd --- /dev/null +++ b/tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:da1d6900c5f541a18295f66312cb8d0999aab6f425a6bbccdd2fd8bcc72300f1.aarch64.tar.gz + id: 4038509 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038510 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:e3f74d3b725d2f6f32d652e4c13854e789e283f8673514b0749f796f5367b46f.x86_64.tar.gz + id: 4038511 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:2c6e2f845291914c4e9ecf2eff31be45379ba3ad8b42fc133df83219d5c6039d.s390x.tar.gz + id: 4038512 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:e859f5623bde399f46690b91c5be657aaa1fe98943930e6f6a2848199b6a7b03.ppc64le.tar.gz + id: 4038513 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata@sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-56520-20200707074906 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata@sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-56520-20200707074906 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248993 +name: cluster-logging-operator-metadata-container +nvr: cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..ae72c268 --- /dev/null +++ b/tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:f49167b645594fec8171bcffa2ba4c652d2fc6a8a39c8d04c8ac9891833429f6.aarch64.tar.gz + id: 4038525 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038526 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:bc6da6c39d2778a070d09c619927d0fc3890b71080b00c89412e2b55859a040a.x86_64.tar.gz + id: 4038527 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:e395580a86026948eeed45d1ac406a5b3ce355aa0f48de23877ddfb61ca2cf62.s390x.tar.gz + id: 4038528 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:d9ca1606333cc9badec48c4fe4895ae19b5ea5b29bb9601e17c7fb79cdd7ea19.ppc64le.tar.gz + id: 4038529 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata@sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-65305-20200707075731 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata@sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-65305-20200707075731 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248998 +name: cluster-nfd-operator-metadata-container +nvr: cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..81a7321a --- /dev/null +++ b/tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:21d47ff01a369d38611df7d513636bd1eaf2787b4424620fa2297093cb7efd87.aarch64.tar.gz + id: 4038514 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038515 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:6f1e0eb900873d800ec3cfc2c07405067679a935b4986e4dddb53037b78276ed.x86_64.tar.gz + id: 4038516 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:30c942b23500e7d289671ebfa3f5a840a264620bae2c470fcf6f8b2928f066a0.s390x.tar.gz + id: 4038517 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:088c14ddcc989815bb38871fe9e0d3387a762c07a2ca23c9fd375ce42e3e53d6.ppc64le.tar.gz + id: 4038518 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata@sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-57368-20200707075038 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata@sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-57368-20200707075038 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248994 +name: elasticsearch-operator-metadata-container +nvr: elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..d2cc02db --- /dev/null +++ b/tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:cc201958cbe1eb7f5282c1df2a1d1baf8f7c0ab0d2783a4cb1d4e1aa88ba6543.aarch64.tar.gz + id: 4038472 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038473 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:497acb6ab8035229e662283a4fed020835ff92831ea0300864ae428874181df8.x86_64.tar.gz + id: 4038474 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:f01dbc59e3d0c08d9df77852cefcd8f8bc4bd66587cd4afbb399853f38b1085c.s390x.tar.gz + id: 4038475 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:e86f4ab4555a4acf8dcb6f2349b0a8fd8ed16ac9d3df039c50014b36a7a8b181.ppc64le.tar.gz + id: 4038476 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata@sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-33656-20200707072400 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata@sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-33656-20200707072400 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248963 +name: local-storage-operator-metadata-container +nvr: local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..762f3c1d --- /dev/null +++ b/tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:24d0f51bd83a274c584a2aef98e6e504e71399c1cd536c3891a70d3ae3627278.aarch64.tar.gz + id: 4038574 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038575 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:e507f8aae9b8b13573e39ffded36375daeed686c0953bc1fd6af601713e0d191.x86_64.tar.gz + id: 4038576 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:f5b69a6fd18b7c6189824dff60f834ee4405b338579f5e37b7e6316f70460ac8.s390x.tar.gz + id: 4038577 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:ac3482bc646895faea206cae1af0ab7fbe5a5e8e323f7fdcd01bda47e33d569f.ppc64le.tar.gz + id: 4038578 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata@sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-52606-20200707080252 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata@sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-52606-20200707080252 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1249006 +name: openshift-enterprise-ansible-service-broker-operator-metadata-container +nvr: openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..ca286a9b --- /dev/null +++ b/tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:8aff8f23622971d84fea7892062731af8c8611b887cbad8a3ef488055fa4ed04.aarch64.tar.gz + id: 4038520 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038521 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:39fa57800aaa583cce939c5d1f6b8b9cc47fd903b1f2e8776d5b1d7cbf9dee4b.x86_64.tar.gz + id: 4038522 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:f40245eeba63f6f1cd0b1b59a683a6ca6a5051194f55386e26e5fdd234433bb4.s390x.tar.gz + id: 4038523 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:9661ac2d9c97b5cffdf6bc41f1bc98020890a63918416d32e3fdead286229342.ppc64le.tar.gz + id: 4038524 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata@sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-98159-20200707075510 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata@sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-98159-20200707075510 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248997 +name: openshift-enterprise-template-service-broker-operator-metadata-container +nvr: openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..f5cbfecd --- /dev/null +++ b/tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:176277a797bafa67823741efa319c2844758b2d5a4cb796b75a8b67580b54d56.aarch64.tar.gz + id: 4038500 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038501 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:b248b0d65ac239f8cacbe71d892ea7d829db0171abb922f1caf6e888fb8c1bcc.x86_64.tar.gz + id: 4038502 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:1dd3df4e8e532526b10155839caddef35cc731e586de169c60af9d72234e6640.s390x.tar.gz + id: 4038503 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:e437e702740efe93906c9b2e9e2a6e53863ccaa2aa49dfc6fa4a9047d37f0d2b.ppc64le.tar.gz + id: 4038504 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata@sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-80281-20200707073911 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata@sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-80281-20200707073911 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248981 +name: ose-metering-ansible-operator-metadata-container +nvr: ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..f0841531 --- /dev/null +++ b/tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:fe0511050968b1355e91c143eea7937e6b96e50e41f1c07d09af3c344c53df65.aarch64.tar.gz + id: 4038464 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038465 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:d55b1d209c064dc8fd17952f2b0fdaccedf3905bce5701910302a2cd08b3a2a7.x86_64.tar.gz + id: 4038466 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:28b65974f88b0bc2c39b1e8f4aba6b5554f72b1650f69e63aeb5449bb48f58ee.s390x.tar.gz + id: 4038467 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:96b0fc687f7eac12b7773dfc568cab1f39a005378108a057408cfb184c7b25c5.ppc64le.tar.gz + id: 4038468 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata@sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-15503-20200707071931 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata@sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-15503-20200707071931 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248956 +name: ose-ptp-operator-metadata-container +nvr: ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..8f9fe54c --- /dev/null +++ b/tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:be2056f3f63c8391fe60f5a6043c4d1a718bf52fb49fba39c18628b930661baf.aarch64.tar.gz + id: 4038495 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038496 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:05db9dc3ed76f6cf993f0dd35aa51892a38ba51afdf8ed564a236d53d8375a47.x86_64.tar.gz + id: 4038497 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:bbb14d32c62f4303e16256fd8d3f5123e789da1b2aee81e82b316908fcf9e28b.s390x.tar.gz + id: 4038498 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:115387bd7efba2bdb7ee9f0d4fff88f03c1579859c6230cc67bb5c0b31263ba8.ppc64le.tar.gz + id: 4038499 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata@sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-88311-20200707073051 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata@sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-88311-20200707073051 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248969 +name: sriov-network-operator-metadata-container +nvr: sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/dump b/tests/koji/data/dump new file mode 100755 index 00000000..1bf2d8b1 --- /dev/null +++ b/tests/koji/data/dump @@ -0,0 +1,99 @@ +#!/usr/bin/python3 +# +# A helper script to dump build and archive data from a koji server into a YAML file +# from which it can be loaded by fake_koji fixture. +# +# Usage: +# +# tests/koji/data/dump https://koji.example.com/hub/ my-build-nvr +# +import os +import koji +import sys +import yaml + +BUILDS_DIR = os.path.join(os.path.dirname(__file__), "builds") + +# We have a few functions here to sanitize data before saving. +# Instead of saving all the raw data, we opt-in to specific fields. +# This helps keep the data cleaner and helps control which fields +# we rely on. + + +def sanitized_build(build): + out = {} + + out["id"] = build["id"] + out["name"] = build["name"] + out["nvr"] = build["nvr"] + out["release"] = build["release"] + out["version"] = build["version"] + out["extra"] = {} + + if (build.get("extra") or {}).get("image"): + out["extra"]["image"] = build["extra"]["image"] + + if (build.get("extra") or {}).get("typeinfo"): + out["extra"]["typeinfo"] = build["extra"]["typeinfo"] + + return out + + +def sanitized_archive_extra(extra): + out = {} + + if not extra.get("docker"): + return out + + if "image" in extra: + out["image"] = extra["image"] + + # Will likely need 'docker' here soon + + return out + + +def sanitized_archive(archive): + out = {} + + out["id"] = archive["id"] + out["btype"] = archive["btype"] + out["type_name"] = archive["type_name"] + out["filename"] = archive["filename"] + out["extra"] = sanitized_archive_extra(archive.get("extra") or {}) + + return out + + +def sanitized_archives(archives): + out = [] + for archive in sorted(archives, key=lambda a: a["id"]): + out.append(sanitized_archive(archive)) + return out + + +def run(server_url, nvr): + server = koji.ClientSession(server_url) + + build = server.getBuild(nvr) + + # In case user passed an ID + nvr = build["nvr"] + + build = sanitized_build(build) + + archives = server.listArchives(build["id"]) + archives = sanitized_archives(archives) + build["archives"] = archives + + data_file = os.path.join(BUILDS_DIR, "%s.yaml" % nvr) + with open(data_file, "w") as fh: + fh.write("############### GENERATED TEST DATA ########################\n") + fh.write("# This file was created by tests/koji/data/dump script.\n") + fh.write("#\n") + yaml.dump(build, fh) + print("Wrote to", data_file) + + +if __name__ == "__main__": + run(*sys.argv[1:]) diff --git a/tests/koji/fake_koji.py b/tests/koji/fake_koji.py index fa90ccc6..35b3de07 100644 --- a/tests/koji/fake_koji.py +++ b/tests/koji/fake_koji.py @@ -1,5 +1,9 @@ import os +import yaml + +BUILDS_DIR = os.path.join(os.path.dirname(__file__), "data", "builds") + class FakeKojiController(object): def __init__(self): @@ -20,6 +24,21 @@ def session(self, url): self.last_url = url return FakeKojiSession(self) + def load_build(self, nvr): + build_file = os.path.join(BUILDS_DIR, nvr + ".yaml") + + with open(build_file, "rt") as f: + build = yaml.load(f, Loader=yaml.BaseLoader) + + archives = build.pop("archives", []) + + self.build_data[build["id"]] = build + self.build_data[build["nvr"]] = build + self.archive_data[build["id"]] = archives + self.archive_data[build["nvr"]] = archives + + return build + def ensure_build(self, build_nvr): if build_nvr in self.build_data: return self.build_data[build_nvr] @@ -108,6 +127,14 @@ def remove_archive(self, filename, build_nvr): return archives.pop(idx) + def remove_build(self, nvr): + build = self.build_data[nvr] + + del self.archive_data[build["id"]] + del self.archive_data[build["nvr"]] + del self.build_data[build["id"]] + del self.build_data[build["nvr"]] + def insert_modules(self, filenames, build_nvr): archives = [ {"btype": "module", "filename": filename, "nvr": build_nvr}