diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e800213..b164d6d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 by architecture. - Added `ModuleMdSourcePushItem` class. Source modulemd documents are now represented by this class rather than `ModuleMdPushItem`. +- Added minimal `ContainerImagePushItem` and `OperatorManifestPushItem` classes for + container images. These classes currently are of limited use as they do not yet + carry relevant metadata. ### Changed diff --git a/docs/index.rst b/docs/index.rst index 38df4440..338e6f10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ A library for accessing push items from various sources. model/rpm model/modulemd model/errata + model/containers model/ami model/productid model/comps diff --git a/docs/model/containers.rst b/docs/model/containers.rst new file mode 100644 index 00000000..68e32bf6 --- /dev/null +++ b/docs/model/containers.rst @@ -0,0 +1,8 @@ +Push items: containers +====================== + +.. autoclass:: pushsource.ContainerImagePushItem() + :members: + +.. autoclass:: pushsource.OperatorManifestPushItem() + :members: diff --git a/src/pushsource/__init__.py b/src/pushsource/__init__.py index c63a3013..d79774e5 100644 --- a/src/pushsource/__init__.py +++ b/src/pushsource/__init__.py @@ -7,6 +7,8 @@ ModuleMdSourcePushItem, ProductIdPushItem, RpmPushItem, + ContainerImagePushItem, + OperatorManifestPushItem, AmiPushItem, AmiRelease, AmiBillingCodes, diff --git a/src/pushsource/_impl/backend/errata_source/errata_client.py b/src/pushsource/_impl/backend/errata_source/errata_client.py index fc333dd1..9086637c 100644 --- a/src/pushsource/_impl/backend/errata_source/errata_client.py +++ b/src/pushsource/_impl/backend/errata_source/errata_client.py @@ -16,6 +16,7 @@ class ErrataRaw(object): # Helper to collect raw responses of all ET APIs for a single advisory advisory_cdn_metadata = attr.ib(type=dict) advisory_cdn_file_list = attr.ib(type=dict) + advisory_cdn_docker_file_list = attr.ib(type=dict) ftp_paths = attr.ib(type=dict) @@ -31,6 +32,9 @@ def __init__(self, threads, url): self._get_advisory_cdn_file_list = partial( self._call_et, "get_advisory_cdn_file_list" ) + self._get_advisory_cdn_docker_file_list = partial( + self._call_et, "get_advisory_cdn_docker_file_list" + ) self._get_ftp_paths = partial(self._call_et, "get_ftp_paths") @property @@ -47,6 +51,7 @@ def get_raw_f(self, advisory_id): all_responses = f_zip( self._executor.submit(self._get_advisory_cdn_metadata, advisory_id), self._executor.submit(self._get_advisory_cdn_file_list, advisory_id), + self._executor.submit(self._get_advisory_cdn_docker_file_list, advisory_id), self._executor.submit(self._get_ftp_paths, advisory_id), ) return f_map(all_responses, lambda tup: ErrataRaw(*tup)) diff --git a/src/pushsource/_impl/backend/errata_source/errata_source.py b/src/pushsource/_impl/backend/errata_source/errata_source.py index abfd8763..a536022a 100644 --- a/src/pushsource/_impl/backend/errata_source/errata_source.py +++ b/src/pushsource/_impl/backend/errata_source/errata_source.py @@ -10,8 +10,15 @@ from ... import compat_attr as attr from ...source import Source -from ...model import ErratumPushItem, RpmPushItem, ModuleMdSourcePushItem, conv -from ...helpers import list_argument +from ...model import ( + ErratumPushItem, + ContainerImagePushItem, + ModuleMdSourcePushItem, + RpmPushItem, + OperatorManifestPushItem, + conv, +) +from ...helpers import list_argument, try_bool LOG = logging.getLogger("pushsource") @@ -25,6 +32,7 @@ def __init__( errata, koji_source=None, rpm_filter_arch=None, + legacy_container_repos=False, threads=4, timeout=60 * 60 * 4, ): @@ -48,6 +56,14 @@ def __init__( If provided, only RPMs for these given arch(es) will be produced; e.g. "x86_64", "src" and "noarch". + legacy_container_repos (bool) + If ``True``, any container push items generated by this source will + use a legacy format for repository IDs. + + This is intended to better support certain legacy code and will be + removed when no longer needed. Only use this if you know that you + need it. + threads (int) Number of threads used for concurrent queries to Errata Tool and koji. @@ -71,6 +87,8 @@ def __init__( self._koji_executor = Executors.thread_pool(max_workers=threads).with_retry() self._koji_cache = {} self._koji_source_url = koji_source + + self._legacy_container_repos = try_bool(legacy_container_repos) self._timeout = timeout @property @@ -119,8 +137,91 @@ def _push_items_from_raw(self, raw): # Adjust push item destinations according to FTP paths from ET, if any. items = self._add_ftp_paths(items, erratum, raw.ftp_paths) + items = items + self._push_items_from_container_manifests( + erratum, raw.advisory_cdn_docker_file_list + ) + return [erratum] + items + def _push_items_from_container_manifests(self, erratum, docker_file_list): + if not docker_file_list: + return [] + + # Example of container list for one item to one repo: + # + # { + # "dotnet-21-container-2.1-77.1621419388": { + # "docker": { + # "target": { + # "external_repos": { + # "rhel8/dotnet-21": { + # "container_full_sig_key": "199e2f91fd431d51", + # "container_sig_key": "fd431d51", + # "tags": ["2.1", "2.1-77.1621419388", "latest"], + # }, + # }, + # "repos": /* like external_repos but uses pulp repo IDs */, + # } + # } + # } + # } + # + + # We'll be getting container metadata from these builds. + koji_source = self._koji_source(container_build=list(docker_file_list.keys())) + + out = [] + + for item in koji_source: + if isinstance(item, ContainerImagePushItem): + item = self._enrich_container_push_item(erratum, docker_file_list, item) + elif isinstance(item, OperatorManifestPushItem): + # Accept this item but nothing special to do + pass + else: + # If build contained anything else, ignore it + LOG.debug( + "Erratum %s: ignored unexpected item from koji source: %s", + erratum.name, + item, + ) + continue + + item = attr.evolve(item, origin=erratum.name) + out.append(item) + + return out + + def _enrich_container_push_item(self, erratum, docker_file_list, item): + # metadata from koji doesn't contain info about where the image should be + # pushed and a few other things - enrich it now + errata_meta = docker_file_list.get(item.build) or {} + target = (errata_meta.get("docker") or {}).get("target") or {} + + repos = target.get("external_repos") or {} + + if self._legacy_container_repos: + repos = target.get("repos") or {} + + dest = [] + for repo_id, repo_data in repos.items(): + for tag in repo_data.get("tags") or []: + dest.append("%s:%s" % (repo_id, tag)) + + dest = sorted(set(dest)) + + # If ET is not requesting to push this to any repos or tags at all, + # it's considered an error. + if not dest: + raise ValueError( + "Erratum %s requests container build %s but provides no repositories" + % (erratum.name, item.build) + ) + + # koji source provided basic info on container image, ET provides policy on + # where/how it should be pushed, combine them both to get final push item + return attr.evolve(item, dest=dest) + def _push_items_from_rpms(self, erratum, rpm_list): out = [] diff --git a/src/pushsource/_impl/backend/koji_source.py b/src/pushsource/_impl/backend/koji_source.py index 0afc5b99..1152b6f6 100644 --- a/src/pushsource/_impl/backend/koji_source.py +++ b/src/pushsource/_impl/backend/koji_source.py @@ -12,13 +12,21 @@ from more_executors.futures import f_map from ..source import Source -from ..model import RpmPushItem, ModuleMdPushItem, ModuleMdSourcePushItem +from ..model import ( + RpmPushItem, + ModuleMdPushItem, + ModuleMdSourcePushItem, + OperatorManifestPushItem, + ContainerImagePushItem, +) from ..helpers import list_argument, try_int from .modulemd import Module LOG = logging.getLogger("pushsource") CACHE_LOCK = threading.RLock() +MIME_TYPE_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" + class ListArchivesCommand(object): def __init__(self, build): @@ -104,6 +112,7 @@ def __init__( rpm=None, module_build=None, module_filter_filename=None, + container_build=None, signing_key=None, basedir=None, threads=4, @@ -143,6 +152,12 @@ def __init__( Has no effect if ``module_build`` is not provided. + container_build (list[str, int]) + Build identifier(s). Can be koji IDs (integers) or build NVRs. + The source will yield all container images produced by these + build(s), along with associated artifacts such as operator manifest + bundles. + signing_key (list[str]) GPG signing key ID(s). If provided, content must be signed using one of the provided keys. Include ``None`` if unsigned @@ -179,6 +194,7 @@ def __init__( self._module_filter_filename = list_argument( module_filter_filename, retain_none=True ) + self._container_build = [try_int(x) for x in list_argument(container_build)] self._signing_key = list_argument(signing_key) self._dest = list_argument(dest) self._timeout = timeout @@ -342,6 +358,107 @@ def _push_items_from_module_build(self, nvr, meta): ) return out + def _push_items_from_container_build(self, nvr, meta): + LOG.debug("Looking for container images on %s, %s", nvr, meta) + + if not meta: + message = "Container image build not found in koji: %s" % nvr + LOG.error(message) + raise ValueError(message) + + # The metadata we are interested in is documented here: + # https://github.com/containerbuildsystem/atomic-reactor/blob/061d92e63cf27ae030e8ceed388ec34f51afb17b/docs/koji.md#type-specific-metadata + extra = meta.get("extra") or {} + typeinfo = extra.get("typeinfo") or {} + # Per above doc, it is preferred to use the metadata under 'typeinfo' if present + image = typeinfo.get("image") or extra.get("image") + + if not image: + message = "Build %s not recognized as a container image build" % nvr + LOG.error(message) + raise ValueError(message) + + build_id = meta["id"] + + archives = self._get_archives(build_id) + + image_archives = [ + elem + for elem in archives + if elem.get("btype") == "image" and elem.get("type_name") == "tar" + ] + + media_types = image.get("media_types") or [] + if MIME_TYPE_MANIFEST_LIST in media_types: + # in manifest list case it is OK to have multiple archives (different arches). + pass + elif len(image_archives) != 1: + # If we don't have a manifest list, we expect to have an image for one arch. + # If there's not exactly one, it's unclear what's going on and we'd better halt. + message = ( + "Could not find (exactly) one container image archive on koji build %s" + % nvr + ) + LOG.error(message) + raise ValueError(message) + + out = [] + + for archive in image_archives: + path = self._pathinfo.typedir(meta, archive["btype"]) + item_src = os.path.join(path, archive["filename"]) + + out.append( + ContainerImagePushItem( + # TODO: name might be changed once we start parsing + # the metadata from atomic-reactor. + name=archive["filename"], + dest=self._dest, + src=item_src, + build=nvr, + ) + ) + + # If this build had any operator-manifests archive, add that too. + operator = self._get_operator_item(nvr, meta, archives) + if operator: + out.append(operator) + + return out + + def _get_operator_item(self, nvr, meta, archives): + extra = meta.get("extra") or {} + typeinfo = extra.get("typeinfo") or {} + operator_manifests = typeinfo.get("operator-manifests") or {} + archive_name = operator_manifests.get("archive") + + if not archive_name: + # Try legacy form + archive_name = extra.get("operator_manifests_archive") + + if not archive_name: + # No operator manifests on this build + return + + operator_archive = [a for a in archives if a["filename"] == archive_name] + if len(operator_archive) != 1: + message = ( + "koji build %s metadata refers to missing operator-manifests " + 'archive "%s"' + ) % (nvr, archive_name) + raise ValueError(message) + + archive = operator_archive[0] + path = self._pathinfo.typedir(meta, archive["btype"]) + item_src = os.path.join(path, archive["filename"]) + + return OperatorManifestPushItem( + name=os.path.join(nvr, archive_name), + dest=self._dest, + src=item_src, + build=nvr, + ) + def _rpm_futures(self): # Get info from each requested RPM. rpm_fs = [(self._executor.submit(self._get_rpm, rpm), rpm) for rpm in self._rpm] @@ -364,6 +481,19 @@ def _modulemd_futures(self): for f, build in module_build_fs ] + def _container_futures(self): + # Get info from each requested container build. + build_fs = [ + (self._executor.submit(self._get_build, build), build) + for build in self._container_build + ] + + # Convert them to lists of push items + return [ + f_map(f, partial(self._push_items_from_container_build, build)) + for f, build in build_fs + ] + def _do_fetch(self, koji_queue, exceptions): try: done = False @@ -421,6 +551,10 @@ def __iter__(self): for build_id in self._module_build: koji_queue.put(GetBuildCommand(ident=build_id, list_archives=True)) + # We'll need to obtain all container image builds + for build_id in self._container_build: + koji_queue.put(GetBuildCommand(ident=build_id, list_archives=True)) + # Put some threads to work on the queue. fetch_exceptions = [] fetch_threads = [ @@ -446,7 +580,9 @@ def __iter__(self): # The queue must be empty now assert koji_queue.empty() - push_items_fs = self._modulemd_futures() + self._rpm_futures() + push_items_fs = ( + self._modulemd_futures() + self._rpm_futures() + self._container_futures() + ) completed_fs = futures.as_completed(push_items_fs, timeout=self._timeout) for f in completed_fs: diff --git a/src/pushsource/_impl/helpers.py b/src/pushsource/_impl/helpers.py index a1614d04..2216d578 100644 --- a/src/pushsource/_impl/helpers.py +++ b/src/pushsource/_impl/helpers.py @@ -57,3 +57,31 @@ def try_int(value): return int(value) except ValueError: return value + + +def try_bool(value): + """Convert values into booleans where possible. + + This function is intended for use in :class:`~pushsource.Source` constructors + to ease the processing of arguments passed via URL (which always arrive as a string). + + Parameters: + value + Any value. + + Returns: + bool + ``value`` converted to a boolean, if it was a string representation + of a boolean. + other + ``value`` unmodified, in any other case. + """ + if not isinstance(value, six.string_types): + return value + + value = value.lower() + if value in ["1", "true", "yes"]: + return True + elif value in ["", "0", "false", "no"]: + return False + raise ValueError("Expected a boolean, got %s" % repr(value)) diff --git a/src/pushsource/_impl/model/__init__.py b/src/pushsource/_impl/model/__init__.py index 692e0154..5f7ecd75 100644 --- a/src/pushsource/_impl/model/__init__.py +++ b/src/pushsource/_impl/model/__init__.py @@ -8,6 +8,7 @@ ) from .rpm import RpmPushItem from .file import FilePushItem +from .container import ContainerImagePushItem, OperatorManifestPushItem from .modulemd import ModuleMdPushItem, ModuleMdSourcePushItem from .comps import CompsXmlPushItem from .productid import ProductIdPushItem diff --git a/src/pushsource/_impl/model/container.py b/src/pushsource/_impl/model/container.py new file mode 100644 index 00000000..97bc1238 --- /dev/null +++ b/src/pushsource/_impl/model/container.py @@ -0,0 +1,22 @@ +from .base import PushItem +from .. import compat_attr as attr + + +@attr.s() +class ContainerImagePushItem(PushItem): + """A :class:`~pushsource.PushItem` representing a container image. + + This item represents any kind of image which can be pulled + from a container image registry (i.e. an object identified by a + ``NAME[:TAG|@DIGEST]`` string). + + In the case of a multi-arch image, one ContainerImagePushItem represents + an image for a single architecture only. + """ + + +@attr.s() +class OperatorManifestPushItem(PushItem): + """A :class:`~pushsource.PushItem` representing an operator manifests archive + (typically named ``operator_manifests.zip``). + """ diff --git a/tests/errata/data/RHBA-2020:2807-no-repos.yaml b/tests/errata/data/RHBA-2020:2807-no-repos.yaml new file mode 100644 index 00000000..53029ec9 --- /dev/null +++ b/tests/errata/data/RHBA-2020:2807-no-repos.yaml @@ -0,0 +1,250 @@ +# A copy of RHBA-2020:2807 which has removed all repos from one build +# (expected to trigger an error) +advisory_id: RHBA-2020:2807-no-repos +cdn_docker_file_list: + cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + # no repos here + external_repos: {} + repos: {} + cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 +cdn_file_list: {} +cdn_metadata: + cdn_repo: + - rhel-7-for-power-le-ose-4_DOT_3-rpms__ppc64le + - rhel-7-for-system-z-ose-4_DOT_3-rpms__s390x + - rhel-7-server-ose-4_DOT_3-rpms__x86_64 + - rhocp-4_DOT_3-for-rhel-8-ppc64le-rpms + - rhocp-4_DOT_3-for-rhel-8-s390x-rpms + - rhocp-4_DOT_3-for-rhel-8-x86_64-rpms + description: 'Red Hat OpenShift Container Platform is Red Hat''s cloud computing + + Kubernetes application platform solution designed for on-premise or private + + cloud deployments. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers. + + + All OpenShift Container Platform 4.3 users are advised to upgrade to these + + updated packages and images when they are available in the appropriate + + release channel. To check for available updates, use the OpenShift Console + + or the CLI oc command. Instructions for upgrading a cluster are available + + at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-between-minor.html#understanding-upgrade-channels_updating-cluster-between-minor.' + from: release-engineering@redhat.com + id: RHBA-2020:2807 + issued: 2020-07-08 22:16:09 UTC + pkglist: + - name: RHBA-2020:2807 + packages: [] + short: '' + pulp_user_metadata: + content_types: + - docker + pushcount: '3' + reboot_suggested: false + references: + - href: https://access.redhat.com/errata/RHBA-2020:2807 + id: RHBA-2020:2807 + title: RHBA-2020:2807 + type: self + - href: https://bugzilla.redhat.com/show_bug.cgi?id=1850434 + id: '1850434' + title: Placeholder bug for OCP 4.3.z metadata release + type: bugzilla + release: '0' + rights: Copyright 2020 Red Hat Inc + severity: None + solution: 'For OpenShift Container Platform 4.3 see the following documentation, + which + + will be updated shortly for release 4.3.28, for important instructions on + + how to upgrade your cluster and fully apply this asynchronous errata + + update: + + + https://docs.openshift.com/container-platform/4.3/release_notes/ocp-4-3-release-notes.html + + + Details on how to access this content are available at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-cli.html.' + status: final + summary: 'Red Hat OpenShift Container Platform release 4.3.28 is now available with + + updates to packages and images that fix several bugs. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers.' + title: OpenShift Container Platform 4.3.28 OLM Operators metadata update + type: bugfix + updated: 2020-07-08 22:16:09 UTC + version: '3' +ftp_paths: {} diff --git a/tests/errata/data/RHBA-2020:2807.yaml b/tests/errata/data/RHBA-2020:2807.yaml new file mode 100644 index 00000000..41664321 --- /dev/null +++ b/tests/errata/data/RHBA-2020:2807.yaml @@ -0,0 +1,264 @@ +# Content in this advisory includes: +# - multi-arch container images +# - operator manifests +advisory_id: RHBA-2020:2807 +cdn_docker_file_list: + cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-cluster-logging-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-cluster-logging-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-cluster-nfd-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-elasticsearch-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-local-storage-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ansible-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - latest + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-template-service-broker-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-metering-ansible-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-ptp-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1: + docker: + target: + external_repos: + openshift4/ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 + repos: + redhat-openshift4-ose-sriov-network-rhel7-operator-metadata: + container_full_sig_key: 199e2f91fd431d51 + container_sig_key: fd431d51 + tags: + - v4.3 + - v4.3.28.202006290519.p0.prod + - v4.3.28.202006290519.p0.prod-1 +cdn_file_list: {} +cdn_metadata: + cdn_repo: + - rhel-7-for-power-le-ose-4_DOT_3-rpms__ppc64le + - rhel-7-for-system-z-ose-4_DOT_3-rpms__s390x + - rhel-7-server-ose-4_DOT_3-rpms__x86_64 + - rhocp-4_DOT_3-for-rhel-8-ppc64le-rpms + - rhocp-4_DOT_3-for-rhel-8-s390x-rpms + - rhocp-4_DOT_3-for-rhel-8-x86_64-rpms + description: 'Red Hat OpenShift Container Platform is Red Hat''s cloud computing + + Kubernetes application platform solution designed for on-premise or private + + cloud deployments. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers. + + + All OpenShift Container Platform 4.3 users are advised to upgrade to these + + updated packages and images when they are available in the appropriate + + release channel. To check for available updates, use the OpenShift Console + + or the CLI oc command. Instructions for upgrading a cluster are available + + at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-between-minor.html#understanding-upgrade-channels_updating-cluster-between-minor.' + from: release-engineering@redhat.com + id: RHBA-2020:2807 + issued: 2020-07-08 22:16:09 UTC + pkglist: + - name: RHBA-2020:2807 + packages: [] + short: '' + pulp_user_metadata: + content_types: + - docker + pushcount: '3' + reboot_suggested: false + references: + - href: https://access.redhat.com/errata/RHBA-2020:2807 + id: RHBA-2020:2807 + title: RHBA-2020:2807 + type: self + - href: https://bugzilla.redhat.com/show_bug.cgi?id=1850434 + id: '1850434' + title: Placeholder bug for OCP 4.3.z metadata release + type: bugzilla + release: '0' + rights: Copyright 2020 Red Hat Inc + severity: None + solution: 'For OpenShift Container Platform 4.3 see the following documentation, + which + + will be updated shortly for release 4.3.28, for important instructions on + + how to upgrade your cluster and fully apply this asynchronous errata + + update: + + + https://docs.openshift.com/container-platform/4.3/release_notes/ocp-4-3-release-notes.html + + + Details on how to access this content are available at + + https://docs.openshift.com/container-platform/4.3/updating/updating-cluster-cli.html.' + status: final + summary: 'Red Hat OpenShift Container Platform release 4.3.28 is now available with + + updates to packages and images that fix several bugs. + + + This advisory will be used to release the corresponding operator manifests + + via new operator metadata containers.' + title: OpenShift Container Platform 4.3.28 OLM Operators metadata update + type: bugfix + updated: 2020-07-08 22:16:09 UTC + version: '3' +ftp_paths: {} diff --git a/tests/errata/data/dump b/tests/errata/data/dump index 9457ed0d..eb9d1ad2 100755 --- a/tests/errata/data/dump +++ b/tests/errata/data/dump @@ -15,6 +15,8 @@ def run(server_url, advisory_id): "advisory_id": advisory_id, "cdn_metadata": server.get_advisory_cdn_metadata(advisory_id), "cdn_file_list": server.get_advisory_cdn_file_list(advisory_id), + "cdn_docker_file_list": server.get_advisory_cdn_docker_file_list(advisory_id), + "ftp_paths": server.get_ftp_paths(advisory_id), } data_file = os.path.join(DATADIR, "%s.yaml" % advisory_id) diff --git a/tests/errata/fake_errata_tool.py b/tests/errata/fake_errata_tool.py index c0136029..ea3dcc34 100644 --- a/tests/errata/fake_errata_tool.py +++ b/tests/errata/fake_errata_tool.py @@ -50,5 +50,8 @@ def get_advisory_cdn_file_list(self, advisory_id): def get_advisory_cdn_metadata(self, advisory_id): return self._get_data(advisory_id, "cdn_metadata") + def get_advisory_cdn_docker_file_list(self, advisory_id): + return self._get_data(advisory_id, "cdn_docker_file_list") + def get_ftp_paths(self, advisory_id): return self._get_data(advisory_id, "ftp_paths") diff --git a/tests/errata/test_errata_containers.py b/tests/errata/test_errata_containers.py new file mode 100644 index 00000000..14211ef5 --- /dev/null +++ b/tests/errata/test_errata_containers.py @@ -0,0 +1,378 @@ +import os + +import pytest + +from pushsource import Source, ContainerImagePushItem, OperatorManifestPushItem + + +@pytest.fixture +def source_factory(fake_errata_tool, fake_koji, koji_dir): + ctor = Source.get_partial( + "errata:https://errata.example.com", + koji_source="koji:https://koji.example.com?basedir=%s" % koji_dir, + ) + + for nvr in [ + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + "sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + ]: + fake_koji.load_build(nvr) + + # Tweak this one build to have legacy reference to operator_manifests_archive + # to prove that this still works. + build = fake_koji.build_data[ + "cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + extra = build["extra"] + extra["operator_manifests_archive"] = extra["typeinfo"]["operator-manifests"].pop( + "archive" + ) + + # And tweak this one to not have any operator archive at all to prove this works too. + build = fake_koji.build_data[ + "sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + extra = build["extra"] + del extra["typeinfo"]["operator-manifests"] + + yield ctor + + +def test_errata_containers(source_factory, koji_dir): + """Errata source can be used to retrieve container-related push items, + typical scenario""" + + source = source_factory(errata="RHBA-2020:2807") + + items = list(source) + items = sorted(items, key=lambda item: item.name) + + image_items = [i for i in items if isinstance(i, ContainerImagePushItem)] + operator_items = [i for i in items if isinstance(i, OperatorManifestPushItem)] + + # Helper to make 'src' attributes below a bit less cumbersome + def koji_path(*args): + return os.path.join(koji_dir, *args) + + # This is quite a large set of very similar images. We'll only do a full check on the first few + # to avoid the test becoming huge. + assert image_items[0:4] == [ + ContainerImagePushItem( + name="docker-image-sha256:05db9dc3ed76f6cf993f0dd35aa51892a38ba51afdf8ed564a236d53d8375a47.x86_64.tar.gz", + state="PENDING", + src=koji_path( + "packages/sriov-network-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:05db9dc3ed76f6cf993f0dd35aa51892a38ba51afdf8ed564a236d53d8375a47.x86_64.tar.gz", + ), + dest=[ + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ContainerImagePushItem( + name="docker-image-sha256:088c14ddcc989815bb38871fe9e0d3387a762c07a2ca23c9fd375ce42e3e53d6.ppc64le.tar.gz", + state="PENDING", + src=koji_path( + "packages/elasticsearch-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:088c14ddcc989815bb38871fe9e0d3387a762c07a2ca23c9fd375ce42e3e53d6.ppc64le.tar.gz", + ), + dest=[ + "openshift4/ose-elasticsearch-rhel7-operator-metadata:v4.3", + "openshift4/ose-elasticsearch-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-elasticsearch-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ContainerImagePushItem( + name="docker-image-sha256:115387bd7efba2bdb7ee9f0d4fff88f03c1579859c6230cc67bb5c0b31263ba8.ppc64le.tar.gz", + state="PENDING", + src=koji_path( + "packages/sriov-network-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:115387bd7efba2bdb7ee9f0d4fff88f03c1579859c6230cc67bb5c0b31263ba8.ppc64le.tar.gz", + ), + dest=[ + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ContainerImagePushItem( + name="docker-image-sha256:176277a797bafa67823741efa319c2844758b2d5a4cb796b75a8b67580b54d56.aarch64.tar.gz", + state="PENDING", + src=koji_path( + "packages/ose-metering-ansible-operator-metadata-container/v4.3.28.202006290519.p0.prod/1/images", + "docker-image-sha256:176277a797bafa67823741efa319c2844758b2d5a4cb796b75a8b67580b54d56.aarch64.tar.gz", + ), + dest=[ + "openshift4/ose-metering-ansible-rhel7-operator-metadata:v4.3", + "openshift4/ose-metering-ansible-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-metering-ansible-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ] + + # Similarly for operators. + assert operator_items[0:4] == [ + OperatorManifestPushItem( + name="cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/cluster-logging-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + # Note that operators don't have any 'dest' assigned. + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + OperatorManifestPushItem( + name="cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/cluster-nfd-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + OperatorManifestPushItem( + name="elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/elasticsearch-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + OperatorManifestPushItem( + name="local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1/operator_manifests.zip", + state="PENDING", + src=koji_path( + "packages/local-storage-operator-metadata-container", + "v4.3.28.202006290519.p0.prod/1/files", + "operator-manifests/operator_manifests.zip", + ), + dest=[], + md5sum=None, + sha256sum=None, + origin="RHBA-2020:2807", + build="local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + signing_key=None, + ), + ] + + +def test_errata_legacy_repos(source_factory): + """Errata source can make use of legacy repo IDs as needed""" + source1 = source_factory(errata="RHBA-2020:2807") + source2 = source_factory(errata="RHBA-2020:2807", legacy_container_repos=True) + + items1 = sorted( + [i for i in source1 if isinstance(i, ContainerImagePushItem)], + key=lambda i: i.name, + ) + items2 = sorted( + [i for i in source2 if isinstance(i, ContainerImagePushItem)], + key=lambda i: i.name, + ) + + # OK, we've got two lists of push items generated identically except that one requested + # legacy repos and the other didn't. Let's sample the first push item to see how they differ. + + assert items1[0].dest == [ + # This is the preferred format. + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "openshift4/ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ] + + assert items2[0].dest == [ + # This is the weird legacy stuff. + "redhat-openshift4-ose-sriov-network-rhel7-operator-metadata:v4.3", + "redhat-openshift4-ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod", + "redhat-openshift4-ose-sriov-network-rhel7-operator-metadata:v4.3.28.202006290519.p0.prod-1", + ] + + +def test_errata_containers_no_repos(source_factory, koji_dir): + """Errata source fails to retrieve container-related push items if ET requests no repos""" + + source = source_factory(errata="RHBA-2020:2807-no-repos") + + # It should fail + with pytest.raises(ValueError) as exc_info: + list(source) + + # And tell us why + assert ( + "Erratum RHBA-2020:2807 requests container build " + "cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 " + "but provides no repositories" + ) in str(exc_info) + + +def test_errata_containers_missing_build(source_factory, fake_koji): + """Errata source gives an error if a requested container build is missing""" + + fake_koji.remove_build( + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ) + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "Container image build not found in koji: " + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ) in str(exc_info) + + +def test_errata_ignores_unknown_koji_types(source_factory, koji_dir): + """Errata source, when requesting containers, will skip unknown push item types + yielded by koji source.""" + + # This is a very niche case, but to get that 100% coverage... + # It's possible that koji_source might produce something other than ContainerImagePushItem + # or OperatorManifestPushItem, and we want to be forwards-compatible with that. + + # This is our hacked source which returns whatever koji returns, but also some + # arbitrary objects + class WeirdKoji(object): + def __init__(self, **kwargs): + # Get a normal koji source... + self.koji = Source.get( + "koji:https://koji.example.com?basedir=%s" % koji_dir, **kwargs + ) + + def __iter__(self): + # We'll yield whatever koji yields but surround it with + # unexpected junk + yield object() + for item in self.koji: + yield item + yield object() + + Source.register_backend("weird-koji", WeirdKoji) + + source = source_factory(errata="RHBA-2020:2807", koji_source="weird-koji:") + + # It should still work as normal + items = list(source) + + # Sanity check we got the right number of items + assert len(items) == 45 + + +def test_errata_containers_missing_image_meta(source_factory, fake_koji): + """Errata source gives an error if a requested container build exists but + image data is missing from build. + """ + + build = fake_koji.build_data[ + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + del build["extra"] + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "Build elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 " + "not recognized as a container image build" + ) in str(exc_info) + + +def test_errata_containers_broken_multiarch(source_factory, fake_koji): + """Errata source gives an error if a requested container build exists, has multiple + image archives and misses the manifest list mimetype in metadata. + """ + + build = fake_koji.build_data[ + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ] + del build["extra"]["typeinfo"]["image"]["media_types"] + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "Could not find (exactly) one container image archive on koji build " + "elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1" + ) in str(exc_info) + + +def test_errata_containers_broken_operator_reference(source_factory, fake_koji): + """Errata source gives an error if a requested container build exists and points + at an operator archive, but the archive is missing from koji. + """ + + fake_koji.remove_archive( + "operator_manifests.zip", + "cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1", + ) + + source = source_factory(errata="RHBA-2020:2807") + + # It should raise + with pytest.raises(ValueError) as exc_info: + list(source) + + # It should tell us why + assert ( + "koji build cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 " + 'metadata refers to missing operator-manifests archive "operator_manifests.zip"' + ) in str(exc_info) diff --git a/tests/helpers/test_argument_helpers.py b/tests/helpers/test_argument_helpers.py index 19ba8153..0822abbe 100644 --- a/tests/helpers/test_argument_helpers.py +++ b/tests/helpers/test_argument_helpers.py @@ -1,4 +1,6 @@ -from pushsource._impl.helpers import list_argument +import pytest + +from pushsource._impl.helpers import list_argument, try_bool def test_list_argument(): @@ -7,3 +9,29 @@ def test_list_argument(): assert list_argument(123) == [123] assert list_argument("123") == ["123"] assert list_argument("123,456,789") == ["123", "456", "789"] + + +@pytest.mark.parametrize( + "input,expected_output", + [ + (True, True), + (False, False), + (12345, 12345), + ("1", True), + ("true", True), + ("tRUe", True), + ("yes", True), + ("0", False), + ("false", False), + ("FaLsE", False), + ("no", False), + ("", False), + ], +) +def test_try_bool(input, expected_output): + assert try_bool(input) == expected_output + + +def test_try_bool_invalid(): + with pytest.raises(ValueError): + try_bool("this ain't no boolean") diff --git a/tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..e2e4f3fd --- /dev/null +++ b/tests/koji/data/builds/cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:da1d6900c5f541a18295f66312cb8d0999aab6f425a6bbccdd2fd8bcc72300f1.aarch64.tar.gz + id: 4038509 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038510 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:e3f74d3b725d2f6f32d652e4c13854e789e283f8673514b0749f796f5367b46f.x86_64.tar.gz + id: 4038511 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:2c6e2f845291914c4e9ecf2eff31be45379ba3ad8b42fc133df83219d5c6039d.s390x.tar.gz + id: 4038512 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:e859f5623bde399f46690b91c5be657aaa1fe98943930e6f6a2848199b6a7b03.ppc64le.tar.gz + id: 4038513 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata@sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-56520-20200707074906 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata@sha256:e38da0971a67c6b3d4261edd9bd679de1be15d0716d208cddf54b0478a73f0a2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-logging-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-56520-20200707074906 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248993 +name: cluster-logging-operator-metadata-container +nvr: cluster-logging-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..ae72c268 --- /dev/null +++ b/tests/koji/data/builds/cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:f49167b645594fec8171bcffa2ba4c652d2fc6a8a39c8d04c8ac9891833429f6.aarch64.tar.gz + id: 4038525 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038526 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:bc6da6c39d2778a070d09c619927d0fc3890b71080b00c89412e2b55859a040a.x86_64.tar.gz + id: 4038527 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:e395580a86026948eeed45d1ac406a5b3ce355aa0f48de23877ddfb61ca2cf62.s390x.tar.gz + id: 4038528 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:d9ca1606333cc9badec48c4fe4895ae19b5ea5b29bb9601e17c7fb79cdd7ea19.ppc64le.tar.gz + id: 4038529 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata@sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-65305-20200707075731 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata@sha256:625ed0e36cff8819aabaadfc67e983ba4486c3ab2c73286fe7bc1726ae08e2c4 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-cluster-nfd-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-65305-20200707075731 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248998 +name: cluster-nfd-operator-metadata-container +nvr: cluster-nfd-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..81a7321a --- /dev/null +++ b/tests/koji/data/builds/elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:21d47ff01a369d38611df7d513636bd1eaf2787b4424620fa2297093cb7efd87.aarch64.tar.gz + id: 4038514 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038515 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:6f1e0eb900873d800ec3cfc2c07405067679a935b4986e4dddb53037b78276ed.x86_64.tar.gz + id: 4038516 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:30c942b23500e7d289671ebfa3f5a840a264620bae2c470fcf6f8b2928f066a0.s390x.tar.gz + id: 4038517 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:088c14ddcc989815bb38871fe9e0d3387a762c07a2ca23c9fd375ce42e3e53d6.ppc64le.tar.gz + id: 4038518 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata@sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-57368-20200707075038 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata@sha256:a595bff056e37d4d61cc9225504bf7f969adcdd53f8f5180176b612c17ceaedc + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-elasticsearch-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-57368-20200707075038 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248994 +name: elasticsearch-operator-metadata-container +nvr: elasticsearch-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..d2cc02db --- /dev/null +++ b/tests/koji/data/builds/local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:cc201958cbe1eb7f5282c1df2a1d1baf8f7c0ab0d2783a4cb1d4e1aa88ba6543.aarch64.tar.gz + id: 4038472 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038473 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:497acb6ab8035229e662283a4fed020835ff92831ea0300864ae428874181df8.x86_64.tar.gz + id: 4038474 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:f01dbc59e3d0c08d9df77852cefcd8f8bc4bd66587cd4afbb399853f38b1085c.s390x.tar.gz + id: 4038475 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:e86f4ab4555a4acf8dcb6f2349b0a8fd8ed16ac9d3df039c50014b36a7a8b181.ppc64le.tar.gz + id: 4038476 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata@sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-33656-20200707072400 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata@sha256:440f56c49b3a3b7490716e5370eacf684529f950f79527016bb265073ffd5332 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-local-storage-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-33656-20200707072400 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248963 +name: local-storage-operator-metadata-container +nvr: local-storage-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..762f3c1d --- /dev/null +++ b/tests/koji/data/builds/openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:24d0f51bd83a274c584a2aef98e6e504e71399c1cd536c3891a70d3ae3627278.aarch64.tar.gz + id: 4038574 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038575 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:e507f8aae9b8b13573e39ffded36375daeed686c0953bc1fd6af601713e0d191.x86_64.tar.gz + id: 4038576 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:f5b69a6fd18b7c6189824dff60f834ee4405b338579f5e37b7e6316f70460ac8.s390x.tar.gz + id: 4038577 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:ac3482bc646895faea206cae1af0ab7fbe5a5e8e323f7fdcd01bda47e33d569f.ppc64le.tar.gz + id: 4038578 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata@sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-52606-20200707080252 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata@sha256:c87692a65005472766fc95effc7e4e0f049e3cd6c244f8e07b8da0604c6c7540 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-ansible-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-52606-20200707080252 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1249006 +name: openshift-enterprise-ansible-service-broker-operator-metadata-container +nvr: openshift-enterprise-ansible-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..ca286a9b --- /dev/null +++ b/tests/koji/data/builds/openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:8aff8f23622971d84fea7892062731af8c8611b887cbad8a3ef488055fa4ed04.aarch64.tar.gz + id: 4038520 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038521 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:39fa57800aaa583cce939c5d1f6b8b9cc47fd903b1f2e8776d5b1d7cbf9dee4b.x86_64.tar.gz + id: 4038522 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:f40245eeba63f6f1cd0b1b59a683a6ca6a5051194f55386e26e5fdd234433bb4.s390x.tar.gz + id: 4038523 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:9661ac2d9c97b5cffdf6bc41f1bc98020890a63918416d32e3fdead286229342.ppc64le.tar.gz + id: 4038524 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata@sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-98159-20200707075510 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata@sha256:2e3b45e6ca30acddd1058a026b1a3fd47fbb02120e7e48b207954307dac53535 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-openshift-enterprise-template-service-broker-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-98159-20200707075510 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248997 +name: openshift-enterprise-template-service-broker-operator-metadata-container +nvr: openshift-enterprise-template-service-broker-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..f5cbfecd --- /dev/null +++ b/tests/koji/data/builds/ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:176277a797bafa67823741efa319c2844758b2d5a4cb796b75a8b67580b54d56.aarch64.tar.gz + id: 4038500 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038501 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:b248b0d65ac239f8cacbe71d892ea7d829db0171abb922f1caf6e888fb8c1bcc.x86_64.tar.gz + id: 4038502 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:1dd3df4e8e532526b10155839caddef35cc731e586de169c60af9d72234e6640.s390x.tar.gz + id: 4038503 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:e437e702740efe93906c9b2e9e2a6e53863ccaa2aa49dfc6fa4a9047d37f0d2b.ppc64le.tar.gz + id: 4038504 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata@sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-80281-20200707073911 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata@sha256:5c27155eaec4c610bc8389f7068c4713fcf6148097479f8e1fa8a12d5f3cad75 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ose-metering-ansible-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-80281-20200707073911 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248981 +name: ose-metering-ansible-operator-metadata-container +nvr: ose-metering-ansible-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..f0841531 --- /dev/null +++ b/tests/koji/data/builds/ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:fe0511050968b1355e91c143eea7937e6b96e50e41f1c07d09af3c344c53df65.aarch64.tar.gz + id: 4038464 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038465 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:d55b1d209c064dc8fd17952f2b0fdaccedf3905bce5701910302a2cd08b3a2a7.x86_64.tar.gz + id: 4038466 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:28b65974f88b0bc2c39b1e8f4aba6b5554f72b1650f69e63aeb5449bb48f58ee.s390x.tar.gz + id: 4038467 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:96b0fc687f7eac12b7773dfc568cab1f39a005378108a057408cfb184c7b25c5.ppc64le.tar.gz + id: 4038468 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata@sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-15503-20200707071931 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata@sha256:9e7f37ad9893d218c9998e53bdb847505c103999200a36afa9fd0875facd7643 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-ptp-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-15503-20200707071931 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248956 +name: ose-ptp-operator-metadata-container +nvr: ose-ptp-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml b/tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml new file mode 100644 index 00000000..8f9fe54c --- /dev/null +++ b/tests/koji/data/builds/sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1.yaml @@ -0,0 +1,94 @@ +############### GENERATED TEST DATA ######################## +# This file was created by tests/koji/data/dump script. +# +archives: +- btype: image + extra: + image: + arch: aarch64 + filename: docker-image-sha256:be2056f3f63c8391fe60f5a6043c4d1a718bf52fb49fba39c18628b930661baf.aarch64.tar.gz + id: 4038495 + type_name: tar +- btype: operator-manifests + extra: {} + filename: operator_manifests.zip + id: 4038496 + type_name: zip +- btype: image + extra: + image: + arch: x86_64 + filename: docker-image-sha256:05db9dc3ed76f6cf993f0dd35aa51892a38ba51afdf8ed564a236d53d8375a47.x86_64.tar.gz + id: 4038497 + type_name: tar +- btype: image + extra: + image: + arch: s390x + filename: docker-image-sha256:bbb14d32c62f4303e16256fd8d3f5123e789da1b2aee81e82b316908fcf9e28b.s390x.tar.gz + id: 4038498 + type_name: tar +- btype: image + extra: + image: + arch: ppc64le + filename: docker-image-sha256:115387bd7efba2bdb7ee9f0d4fff88f03c1579859c6230cc67bb5c0b31263ba8.ppc64le.tar.gz + id: 4038499 + type_name: tar +extra: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata@sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-88311-20200707073051 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + typeinfo: + image: + autorebuild: false + help: null + index: + digests: + application/vnd.docker.distribution.manifest.list.v2+json: sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + floating_tags: + - latest + - v4.3.28.202006290519.p0.prod + pull: + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata@sha256:16a04fde191ebd8858fa7f97605a9783be570e37d329eec599cfecf03b5cd8f2 + - registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-sriov-network-operator-metadata:v4.3.28.202006290519.p0.prod-1 + tags: + - v4.3.28.202006290519.p0.prod-1 + unique_tags: + - rhaos-4.3-rhel-7-candidate-88311-20200707073051 + isolated: false + media_types: + - application/vnd.docker.distribution.manifest.list.v2+json + - application/vnd.docker.distribution.manifest.v1+json + - application/vnd.docker.distribution.manifest.v2+json + parent_image_builds: {} + parent_images: + - scratch + operator-manifests: + archive: operator_manifests.zip +id: 1248969 +name: sriov-network-operator-metadata-container +nvr: sriov-network-operator-metadata-container-v4.3.28.202006290519.p0.prod-1 +release: '1' +version: v4.3.28.202006290519.p0.prod diff --git a/tests/koji/data/dump b/tests/koji/data/dump new file mode 100755 index 00000000..1bf2d8b1 --- /dev/null +++ b/tests/koji/data/dump @@ -0,0 +1,99 @@ +#!/usr/bin/python3 +# +# A helper script to dump build and archive data from a koji server into a YAML file +# from which it can be loaded by fake_koji fixture. +# +# Usage: +# +# tests/koji/data/dump https://koji.example.com/hub/ my-build-nvr +# +import os +import koji +import sys +import yaml + +BUILDS_DIR = os.path.join(os.path.dirname(__file__), "builds") + +# We have a few functions here to sanitize data before saving. +# Instead of saving all the raw data, we opt-in to specific fields. +# This helps keep the data cleaner and helps control which fields +# we rely on. + + +def sanitized_build(build): + out = {} + + out["id"] = build["id"] + out["name"] = build["name"] + out["nvr"] = build["nvr"] + out["release"] = build["release"] + out["version"] = build["version"] + out["extra"] = {} + + if (build.get("extra") or {}).get("image"): + out["extra"]["image"] = build["extra"]["image"] + + if (build.get("extra") or {}).get("typeinfo"): + out["extra"]["typeinfo"] = build["extra"]["typeinfo"] + + return out + + +def sanitized_archive_extra(extra): + out = {} + + if not extra.get("docker"): + return out + + if "image" in extra: + out["image"] = extra["image"] + + # Will likely need 'docker' here soon + + return out + + +def sanitized_archive(archive): + out = {} + + out["id"] = archive["id"] + out["btype"] = archive["btype"] + out["type_name"] = archive["type_name"] + out["filename"] = archive["filename"] + out["extra"] = sanitized_archive_extra(archive.get("extra") or {}) + + return out + + +def sanitized_archives(archives): + out = [] + for archive in sorted(archives, key=lambda a: a["id"]): + out.append(sanitized_archive(archive)) + return out + + +def run(server_url, nvr): + server = koji.ClientSession(server_url) + + build = server.getBuild(nvr) + + # In case user passed an ID + nvr = build["nvr"] + + build = sanitized_build(build) + + archives = server.listArchives(build["id"]) + archives = sanitized_archives(archives) + build["archives"] = archives + + data_file = os.path.join(BUILDS_DIR, "%s.yaml" % nvr) + with open(data_file, "w") as fh: + fh.write("############### GENERATED TEST DATA ########################\n") + fh.write("# This file was created by tests/koji/data/dump script.\n") + fh.write("#\n") + yaml.dump(build, fh) + print("Wrote to", data_file) + + +if __name__ == "__main__": + run(*sys.argv[1:]) diff --git a/tests/koji/fake_koji.py b/tests/koji/fake_koji.py index fa90ccc6..35b3de07 100644 --- a/tests/koji/fake_koji.py +++ b/tests/koji/fake_koji.py @@ -1,5 +1,9 @@ import os +import yaml + +BUILDS_DIR = os.path.join(os.path.dirname(__file__), "data", "builds") + class FakeKojiController(object): def __init__(self): @@ -20,6 +24,21 @@ def session(self, url): self.last_url = url return FakeKojiSession(self) + def load_build(self, nvr): + build_file = os.path.join(BUILDS_DIR, nvr + ".yaml") + + with open(build_file, "rt") as f: + build = yaml.load(f, Loader=yaml.BaseLoader) + + archives = build.pop("archives", []) + + self.build_data[build["id"]] = build + self.build_data[build["nvr"]] = build + self.archive_data[build["id"]] = archives + self.archive_data[build["nvr"]] = archives + + return build + def ensure_build(self, build_nvr): if build_nvr in self.build_data: return self.build_data[build_nvr] @@ -108,6 +127,14 @@ def remove_archive(self, filename, build_nvr): return archives.pop(idx) + def remove_build(self, nvr): + build = self.build_data[nvr] + + del self.archive_data[build["id"]] + del self.archive_data[build["nvr"]] + del self.build_data[build["id"]] + del self.build_data[build["nvr"]] + def insert_modules(self, filenames, build_nvr): archives = [ {"btype": "module", "filename": filename, "nvr": build_nvr}