ART-3050: add plashet from-tags --rhcos

This PR adds `--rhcos` option to Plashet from-tags subverb.
When "--rhcos" argument is specified, the final list of package NVRs should include any dependencies specified in the assembly's assembly.rhcos.dependencies field.

doozerlib/assembly.py

@@ -1,7 +1,7 @@
 import typing
 import copy
 
-from doozerlib.model import Model
+from doozerlib.model import Missing, Model
 
 
 def merger(a, b):
@@ -121,6 +121,25 @@ def assembly_metadata_config(releases_config: Model, assembly: str, meta_type: s
     return Model(dict_to_model=config_dict)
 
 
+def assembly_rhcos_config(releases_config: Model, assembly: str) -> Model:
+    """
+    :param releases_config: The content of releases.yml in Model form.
+    :param assembly: The name of the assembly to assess
+    Returns the a computed rhcos config model for a given assembly.
+    """
+    if not assembly or not isinstance(releases_config, Model):
+        return Missing
+
+    _check_recursion(releases_config, assembly)
+    target_assembly = releases_config.releases[assembly].assembly
+    rhcos_config_dict = target_assembly.get("rhcos", {})
+    if target_assembly.basis.assembly:  # Does this assembly inherit from another?
+        # Recursive apply ancestor assemblies
+        basis_rhcos_config = assembly_rhcos_config(releases_config, target_assembly.basis.assembly)
+        rhcos_config_dict = merger(rhcos_config_dict, basis_rhcos_config.primitive())
+    return Model(dict_to_model=rhcos_config_dict)
+
+
 def assembly_basis_event(releases_config: Model, assembly: str) -> typing.Optional[int]:
     """
     :param releases_config: The content of releases.yml in Model form.

doozerlib/cli/config_plashet.py

@@ -731,6 +731,9 @@ def for_assembly(config: SimpleNamespace, image: Optional[str], rhcos: bool, bre
     Creates a directory containing arch specific yum repository subdirectories based on a complete set of RPMs required for image builds.
     In other words, when doozer runs brew builds for an assembly image, it should only need to pass in a single repository created by for-assembly in order for the image to build successfully.
     """
+    if image and rhcos:
+        raise click.BadParameter("Cannot use --image and --rhcos at the same time.")
+
     runtime: Runtime = config.runtime
     runtime.initialize(mode="both", clone_source=False, clone_distgits=False, prevent_cloning=True)
     if not runtime.assembly_basis_event:
@@ -775,7 +778,7 @@ def for_assembly(config: SimpleNamespace, image: Optional[str], rhcos: bool, bre
             # Builds pinned by "is" should take precedence over every build from tag
             for component, pinned_build in pinned_by_is.items():
                 if component in component_builds and pinned_build["id"] != component_builds[component]["id"]:
-                    logger.warning("Swapping stream nvr %s for pinned nvr %s...", component_builds[component]["nvr"], pinned_build["nvr"])
+                    logger.warning("Swapping tagged nvr %s for pinned nvr %s...", component_builds[component]["nvr"], pinned_build["nvr"])
             component_builds.update(pinned_by_is)  # pinned rpms take precedence over those from tags
             signable_components |= pinned_by_is.keys()  # ART-managed rpms are always signable
 
@@ -784,24 +787,33 @@ def for_assembly(config: SimpleNamespace, image: Optional[str], rhcos: bool, bre
             # Group dependencies should take precedence over anything previously determined except those pinned by "is".
             for component, dep_build in group_deps.items():
                 if component in component_builds and dep_build["id"] != component_builds[component]["id"]:
-                    logger.warning("Swapping stream nvr %s for group dependency nvr %s...", component_builds[component]["nvr"], dep_build["nvr"])
+                    logger.warning("Swapping tagged nvr %s for group dependency nvr %s...", component_builds[component]["nvr"], dep_build["nvr"])
             component_builds.update(group_deps)
 
-        # If "--image" is specified, the final list of package NVRs should include any dependencies specified in the component overrides for the assembly.
-        if image:
-            image_meta = runtime.image_map.get(image)
-            if not image_meta:
-                raise IOError(f"Distgit key '{image}' specified by '--image' is not found or excluded from build data.")
-            image_el_version = isolate_el_version_in_brew_tag(image_meta.branch())
-            if image_el_version is None:  # This should never happen, but be safe
-                raise ValueError(f"Distgit repo {image} uses a distgit branch {image_meta.branch()} that is irrelevant to any RHEL version.")
-            if image_el_version == tag_el_version:
-                image_deps = builder.from_image_member_deps(image_el_version, runtime.assembly, runtime.get_releases_config(), image_meta, runtime.rpm_map)  # the return value doesn't include any ART managed rpms
-                # image member dependencies should take precedence over anything previously determined except those pinned by "is".
-                for component, dep_build in image_deps.items():
+            # If "--image" is specified, the final list of package NVRs should include any dependencies specified in the component overrides for the assembly.
+            if image:
+                image_meta = runtime.image_map.get(image)
+                if not image_meta:
+                    raise IOError(f"Distgit key '{image}' specified by '--image' is not found or excluded from build data.")
+                image_el_version = isolate_el_version_in_brew_tag(image_meta.branch())
+                if image_el_version is None:  # This should never happen, but be safe
+                    raise ValueError(f"Distgit repo {image} uses a distgit branch {image_meta.branch()} that is irrelevant to any RHEL version.")
+                if image_el_version == tag_el_version:
+                    image_deps = builder.from_image_member_deps(image_el_version, runtime.assembly, runtime.get_releases_config(), image_meta, runtime.rpm_map)  # the return value doesn't include any ART managed rpms
+                    # image member dependencies should take precedence over anything previously determined except those pinned by "is".
+                    for component, dep_build in image_deps.items():
+                        if component in component_builds and dep_build["id"] != component_builds[component]["id"]:
+                            logger.warning("Swapping tagged nvr %s for image member dependency nvr %s...", component_builds[component]["nvr"], dep_build["nvr"])
+                    component_builds.update(image_deps)
+
+            # If "--rhcos" argument is specified, the final list of package NVRs should include any dependencies specified in the assembly's assembly.rhcos.dependencies field.
+            elif rhcos:
+                rhcos_deps = builder.from_rhcos_deps(tag_el_version, runtime.assembly, runtime.get_releases_config(), runtime.rpm_map)   # the return value doesn't include any ART managed rpms
+                # RHCOS dependencies should take precedence over anything previously determined except those pinned by "is".
+                for component, dep_build in rhcos_deps.items():
                     if component in component_builds and dep_build["id"] != component_builds[component]["id"]:
-                        logger.warning("Swapping stream nvr %s for group dependency nvr %s...", component_builds[component]["nvr"], dep_build["nvr"])
-                component_builds.update(image_deps)
+                        logger.warning("Swapping tagged nvr %s for RHCOS dependency nvr %s...", component_builds[component]["nvr"], dep_build["nvr"])
+                component_builds.update(rhcos_deps)
 
         for component, build in component_builds.items():
             nvre = to_nvre(build)

doozerlib/plashet.py

@@ -6,7 +6,7 @@
 from kobo.rpmlib import parse_nvr
 from koji import ClientSession
 
-from doozerlib.assembly import assembly_metadata_config
+from doozerlib.assembly import assembly_metadata_config, assembly_rhcos_config
 from doozerlib.brew import get_build_objects, list_archives_by_builds
 from doozerlib.image import ImageMetadata
 from doozerlib.model import Model
@@ -170,11 +170,39 @@ def from_image_member_deps(self, el_version: int, assembly: str, releases_config
             dep_builds = self._get_builds(dep_nvr_list)
             missing_nvrs = [nvr for nvr, build in zip(dep_nvr_list, dep_builds) if not build]
             if missing_nvrs:
-                raise IOError(f"The following group dependency NVRs don't exist: {missing_nvrs}")
+                raise IOError(f"The following image member dependency NVRs don't exist: {missing_nvrs}")
             # Make sure image member dependencies have no ART managed rpms.
             art_rpms_in_deps = {dep_build["name"] for dep_build in dep_builds} & {meta.rpm_name for meta in rpm_map.values()}
             if art_rpms_in_deps:
                 raise ValueError(f"Unable to build plashet. Image member dependencies cannot have ART managed RPMs: {art_rpms_in_deps}")
             for dep_build in dep_builds:
                 component_builds[dep_build["name"]] = dep_build
         return component_builds
+
+    def from_rhcos_deps(self, el_version: int, assembly: str, releases_config: Model, rpm_map: Dict[str, RPMMetadata]):
+        """ Returns RPM builds defined in RHCOS config dependencies
+        :param el_version: RHEL version
+        :param assembly: Assembly name to query. If None, this method will return true latest builds.
+        :param releases_config: a Model for releases.yaml
+         :param rpm_map: Map of rpm_distgit_key -> RPMMetadata
+        :return: a dict; keys are component names, values are Brew build dicts
+        """
+        component_builds: Dict[str, Dict] = {}  # keys are rpm component names, values are brew build dicts
+        rhcos_config = assembly_rhcos_config(releases_config, assembly)
+        # honor RHCOS dependencies
+        # rpms for this rhel version listed in RHCOS dependencies; keys are rpm component names, values are nvrs
+        dep_nvrs = {parse_nvr(dep[f"el{el_version}"])["name"]: dep[f"el{el_version}"] for dep in rhcos_config.dependencies.rpms if dep[f"el{el_version}"]}
+        if dep_nvrs:
+            dep_nvr_list = list(dep_nvrs.values())
+            self._logger.info("Found %s NVRs defined in RHCOS dependencies. Fetching build infos from Brew...", len(dep_nvr_list))
+            dep_builds = self._get_builds(dep_nvr_list)
+            missing_nvrs = [nvr for nvr, build in zip(dep_nvr_list, dep_builds) if not build]
+            if missing_nvrs:
+                raise IOError(f"The following RHCOS dependency NVRs don't exist: {missing_nvrs}")
+            # Make sure RHCOS dependencies have no ART managed rpms.
+            art_rpms_in_rhcos_deps = {dep_build["name"] for dep_build in dep_builds} & {meta.rpm_name for meta in rpm_map.values()}
+            if art_rpms_in_rhcos_deps:
+                raise ValueError(f"Unable to build plashet. Group dependencies cannot have ART managed RPMs: {art_rpms_in_rhcos_deps}")
+            for dep_build in dep_builds:
+                component_builds[dep_build["name"]] = dep_build
+        return component_builds

tests/test_assembly.py

@@ -2,7 +2,7 @@
 
 from unittest import TestCase
 
-from doozerlib.assembly import merger, assembly_group_config, assembly_metadata_config, assembly_basis_event
+from doozerlib.assembly import assembly_rhcos_config, merger, assembly_group_config, assembly_metadata_config, assembly_basis_event
 from doozerlib.model import Model, Missing
 
 
@@ -117,6 +117,16 @@ def setUp(self) -> None:
           rpms:
             - el7: some-nvr-3
               non_gc_tag: some-tag-3
+      rhcos:
+        machine-os-content:
+          images:
+            x86_64: registry.example.com/rhcos-x86_64:test
+        dependencies:
+          rpms:
+            - el7: some-nvr-4
+              non_gc_tag: some-tag-4
+            - el8: some-nvr-5
+              non_gc_tag: some-tag-4
 
   ART_8:
     assembly:
@@ -136,6 +146,13 @@ def setUp(self) -> None:
           rpms:
             - el7: some-nvr-4
               non_gc_tag: some-tag-4
+      rhcos:
+        machine-os-content:
+          images: {}
+        dependencies:
+          rpms:
+            - el8: some-nvr-6
+              non_gc_tag: some-tag-6
 
   ART_INFINITE:
     assembly:
@@ -333,3 +350,7 @@ def test_asembly_metadata_config(self):
             pass
         except Exception as e:
             self.fail(f'Expected ValueError on assembly infinite recursion but got: {type(e)}: {e}')
+
+    def test_assembly_rhcos_config(self):
+        rhcos_config = assembly_rhcos_config(self.releases_config, "ART_8")
+        self.assertEqual(len(rhcos_config.dependencies.rpms), 3)

tests/test_plashet.py

@@ -181,3 +181,28 @@ def test_from_image_member_deps(self, assembly_metadata_config: Mock):
         self.assertEqual([b["nvr"] for b in actual.values()], ["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8", "fake3-1.2.3-1.el8"])
         builder._get_builds.assert_called_once_with(["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8", "fake3-1.2.3-1.el8"])
         assembly_metadata_config.assert_called_once()
+
+    @patch("doozerlib.plashet.assembly_rhcos_config")
+    def test_from_rhcos_deps(self, assembly_rhcos_config: Mock):
+        builder = PlashetBuilder(MagicMock())
+
+        builder._get_builds = MagicMock(return_value=[
+            {"id": 1, "build_id": 1, "name": "fake1", "nvr": "fake1-1.2.3-1.el8"},
+            {"id": 2, "build_id": 2, "name": "fake2", "nvr": "fake2-1.2.3-1.el8"},
+            {"id": 3, "build_id": 3, "name": "fake3", "nvr": "fake3-1.2.3-1.el8"},
+        ])
+        assembly_rhcos_config.return_value = Model({
+            "dependencies": {
+                "rpms": [
+                    {"el8": "fake1-1.2.3-1.el8"},
+                    {"el8": "fake2-1.2.3-1.el8"},
+                    {"el8": "fake3-1.2.3-1.el8"},
+                    {"el7": "fake2-1.2.3-1.el7"},
+                    {"el7": "fake2-1.2.3-1.el7"},
+                ]
+            }
+        })
+        actual = builder.from_rhcos_deps(8, "art1", Model(), {})
+        self.assertEqual([b["nvr"] for b in actual.values()], ["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8", "fake3-1.2.3-1.el8"])
+        builder._get_builds.assert_called_once_with(["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8", "fake3-1.2.3-1.el8"])
+        assembly_rhcos_config.assert_called_once()