Add preliminary support for container-related push items

CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   by architecture.
 - Added `ModuleMdSourcePushItem` class. Source modulemd documents are now represented
   by this class rather than `ModuleMdPushItem`.
+- Added minimal `ContainerImagePushItem` and `OperatorManifestPushItem` classes for
+  container images. These classes currently are of limited use as they do not yet
+  carry relevant metadata.
 
 ### Changed
 

docs/index.rst

@@ -17,6 +17,7 @@ A library for accessing push items from various sources.
    model/rpm
    model/modulemd
    model/errata
+   model/containers
    model/ami
    model/productid
    model/comps

docs/model/containers.rst

@@ -0,0 +1,8 @@
+Push items: containers
+======================
+
+.. autoclass:: pushsource.ContainerImagePushItem()
+   :members:
+
+.. autoclass:: pushsource.OperatorManifestPushItem()
+   :members:

src/pushsource/__init__.py

@@ -7,6 +7,8 @@
     ModuleMdSourcePushItem,
     ProductIdPushItem,
     RpmPushItem,
+    ContainerImagePushItem,
+    OperatorManifestPushItem,
     AmiPushItem,
     AmiRelease,
     AmiBillingCodes,

src/pushsource/_impl/backend/errata_source/errata_client.py

@@ -16,6 +16,7 @@ class ErrataRaw(object):
     # Helper to collect raw responses of all ET APIs for a single advisory
     advisory_cdn_metadata = attr.ib(type=dict)
     advisory_cdn_file_list = attr.ib(type=dict)
+    advisory_cdn_docker_file_list = attr.ib(type=dict)
     ftp_paths = attr.ib(type=dict)
 
 
@@ -31,6 +32,9 @@ def __init__(self, threads, url):
         self._get_advisory_cdn_file_list = partial(
             self._call_et, "get_advisory_cdn_file_list"
         )
+        self._get_advisory_cdn_docker_file_list = partial(
+            self._call_et, "get_advisory_cdn_docker_file_list"
+        )
         self._get_ftp_paths = partial(self._call_et, "get_ftp_paths")
 
     @property
@@ -47,6 +51,7 @@ def get_raw_f(self, advisory_id):
         all_responses = f_zip(
             self._executor.submit(self._get_advisory_cdn_metadata, advisory_id),
             self._executor.submit(self._get_advisory_cdn_file_list, advisory_id),
+            self._executor.submit(self._get_advisory_cdn_docker_file_list, advisory_id),
             self._executor.submit(self._get_ftp_paths, advisory_id),
         )
         return f_map(all_responses, lambda tup: ErrataRaw(*tup))

src/pushsource/_impl/backend/errata_source/errata_source.py

@@ -10,8 +10,15 @@
 
 from ... import compat_attr as attr
 from ...source import Source
-from ...model import ErratumPushItem, RpmPushItem, ModuleMdSourcePushItem, conv
-from ...helpers import list_argument
+from ...model import (
+    ErratumPushItem,
+    ContainerImagePushItem,
+    ModuleMdSourcePushItem,
+    RpmPushItem,
+    OperatorManifestPushItem,
+    conv,
+)
+from ...helpers import list_argument, try_bool
 
 LOG = logging.getLogger("pushsource")
 
@@ -25,6 +32,7 @@ def __init__(
         errata,
         koji_source=None,
         rpm_filter_arch=None,
+        legacy_container_repos=False,
         threads=4,
         timeout=60 * 60 * 4,
     ):
@@ -48,6 +56,14 @@ def __init__(
                 If provided, only RPMs for these given arch(es) will be produced;
                 e.g. "x86_64", "src" and "noarch".
 
+            legacy_container_repos (bool)
+                If ``True``, any container push items generated by this source will
+                use a legacy format for repository IDs.
+
+                This is intended to better support certain legacy code and will be
+                removed when no longer needed. Only use this if you know that you
+                need it.
+
             threads (int)
                 Number of threads used for concurrent queries to Errata Tool
                 and koji.
@@ -71,6 +87,8 @@ def __init__(
         self._koji_executor = Executors.thread_pool(max_workers=threads).with_retry()
         self._koji_cache = {}
         self._koji_source_url = koji_source
+
+        self._legacy_container_repos = try_bool(legacy_container_repos)
         self._timeout = timeout
 
     @property
@@ -119,8 +137,91 @@ def _push_items_from_raw(self, raw):
         # Adjust push item destinations according to FTP paths from ET, if any.
         items = self._add_ftp_paths(items, erratum, raw.ftp_paths)
 
+        items = items + self._push_items_from_container_manifests(
+            erratum, raw.advisory_cdn_docker_file_list
+        )
+
         return [erratum] + items
 
+    def _push_items_from_container_manifests(self, erratum, docker_file_list):
+        if not docker_file_list:
+            return []
+
+        # Example of container list for one item to one repo:
+        #
+        # {
+        #     "dotnet-21-container-2.1-77.1621419388": {
+        #         "docker": {
+        #             "target": {
+        #                 "external_repos": {
+        #                     "rhel8/dotnet-21": {
+        #                         "container_full_sig_key": "199e2f91fd431d51",
+        #                         "container_sig_key": "fd431d51",
+        #                         "tags": ["2.1", "2.1-77.1621419388", "latest"],
+        #                     },
+        #                 },
+        #                 "repos": /* like external_repos but uses pulp repo IDs */,
+        #             }
+        #         }
+        #     }
+        # }
+        #
+
+        # We'll be getting container metadata from these builds.
+        koji_source = self._koji_source(container_build=list(docker_file_list.keys()))
+
+        out = []
+
+        for item in koji_source:
+            if isinstance(item, ContainerImagePushItem):
+                item = self._enrich_container_push_item(erratum, docker_file_list, item)
+            elif isinstance(item, OperatorManifestPushItem):
+                # Accept this item but nothing special to do
+                pass
+            else:
+                # If build contained anything else, ignore it
+                LOG.debug(
+                    "Erratum %s: ignored unexpected item from koji source: %s",
+                    erratum.name,
+                    item,
+                )
+                continue
+
+            item = attr.evolve(item, origin=erratum.name)
+            out.append(item)
+
+        return out
+
+    def _enrich_container_push_item(self, erratum, docker_file_list, item):
+        # metadata from koji doesn't contain info about where the image should be
+        # pushed and a few other things - enrich it now
+        errata_meta = docker_file_list.get(item.build) or {}
+        target = (errata_meta.get("docker") or {}).get("target") or {}
+
+        repos = target.get("external_repos") or {}
+
+        if self._legacy_container_repos:
+            repos = target.get("repos") or {}
+
+        dest = []
+        for repo_id, repo_data in repos.items():
+            for tag in repo_data.get("tags") or []:
+                dest.append("%s:%s" % (repo_id, tag))
+
+        dest = sorted(set(dest))
+
+        # If ET is not requesting to push this to any repos or tags at all,
+        # it's considered an error.
+        if not dest:
+            raise ValueError(
+                "Erratum %s requests container build %s but provides no repositories"
+                % (erratum.name, item.build)
+            )
+
+        # koji source provided basic info on container image, ET provides policy on
+        # where/how it should be pushed, combine them both to get final push item
+        return attr.evolve(item, dest=dest)
+
     def _push_items_from_rpms(self, erratum, rpm_list):
         out = []