[ART-3883] enable multiple rhcos containers (#618)

https://issues.redhat.com/browse/ART-3883

We will need to configurably inject multiple containers from an RHCOS build into the payload. Eventually machine-os-content may not be the reference container and may not be present at all. https://github.com/openshift/ocp-build-data/pull/1773/files shows how we might expect config to look for the first additional container. openshift-eng/ocp-build-data-validator#78 enables `rhel-coreos-8` in assembly definitions.

doozerlib/assembly.py

@@ -21,6 +21,7 @@ class AssemblyIssueCode(Enum):
     OUTDATED_RPMS_IN_STREAM_BUILD = 4
     INCONSISTENT_RHCOS_RPMS = 5
     MISSING_INHERITED_DEPENDENCY = 6
+    MISSING_RHCOS_CONTAINER = 7
 
 
 class AssemblyIssue:

doozerlib/assembly_inspector.py

@@ -8,7 +8,7 @@
 from doozerlib.rpmcfg import RPMMetadata
 from doozerlib.assembly import assembly_rhcos_config, AssemblyTypes, assembly_permits, AssemblyIssue, \
     AssemblyIssueCode, assembly_type
-from doozerlib.rhcos import RHCOSBuildInspector, RHCOSBuildFinder
+from doozerlib.rhcos import RHCOSBuildInspector, RHCOSBuildFinder, get_container_configs, RhcosMissingContainerException
 
 
 class AssemblyInspector:
@@ -309,17 +309,34 @@ def get_rhcos_build(self, arch: str, private: bool = False, custom: bool = False
         brew_arch = util.brew_arch_for_go_arch(arch)
         runtime.logger.info(f"Getting latest RHCOS source for {brew_arch}...")
 
-        # See if this assembly has assembly.rhcos.machine-os-content.images populated for this architecture.
-        assembly_rhcos_arch_pullspec = self.assembly_rhcos_config['machine-os-content'].images[brew_arch]
+        # See if this assembly has assembly.rhcos.*.images populated for this architecture.
+        pullspec_for_tag = dict()
+        for container_conf in get_container_configs(runtime):
+            # first we look at the assembly definition as the source of truth for RHCOS containers
+            assembly_rhcos_arch_pullspec = self.assembly_rhcos_config[container_conf.name].images[brew_arch]
+            if assembly_rhcos_arch_pullspec:
+                pullspec_for_tag[container_conf.name] = assembly_rhcos_arch_pullspec
+                continue
 
-        if self.runtime.assembly_type != AssemblyTypes.STREAM and not assembly_rhcos_arch_pullspec:
-            raise Exception(f'Assembly {runtime.assembly} has is not a STREAM but no assembly.rhcos MOSC image data for {brew_arch}; all MOSC image data must be populated for this assembly to be valid')
+            # for non-stream assemblies we expect explicit config for RHCOS
+            if self.runtime.assembly_type != AssemblyTypes.STREAM:
+                if container_conf.primary:
+                    raise Exception(f'Assembly {runtime.assembly} is not type STREAM but no assembly.rhcos.{container_conf.name} image data for {brew_arch}; all RHCOS image data must be populated for this assembly to be valid')
+                # require the primary container at least to be specified, but
+                # allow the edge case where we add an RHCOS container type and
+                # previous assemblies don't specify it
+                continue
 
-        version = self.runtime.get_minor_version()
-        if assembly_rhcos_arch_pullspec:
-            return RHCOSBuildInspector(runtime, assembly_rhcos_arch_pullspec, brew_arch)
-        else:
-            _, pullspec = RHCOSBuildFinder(runtime, version, brew_arch, private, custom=custom).latest_machine_os_content()
-            if not pullspec:
-                raise IOError(f"No RHCOS latest found for {version} / {brew_arch}")
-            return RHCOSBuildInspector(runtime, pullspec, brew_arch)
+            try:
+                version = self.runtime.get_minor_version()
+                _, pullspec = RHCOSBuildFinder(runtime, version, brew_arch, private, custom=custom).latest_container(container_conf)
+                if not pullspec:
+                    raise IOError(f"No RHCOS latest found for {version} / {brew_arch}")
+                pullspec_for_tag[container_conf.name] = pullspec
+            except RhcosMissingContainerException:
+                if container_conf.primary:
+                    # accommodate RHCOS build metadata not specifying all expected containers, but require primary.
+                    # their absence will be noted when generating payloads anyway.
+                    raise
+
+        return RHCOSBuildInspector(runtime, pullspec_for_tag, brew_arch)

doozerlib/cli/detect_embargo.py

@@ -7,7 +7,7 @@
 import click
 import yaml
 
-from doozerlib import Runtime, brew, exectools
+from doozerlib import Runtime, brew, exectools, rhcos
 from doozerlib import build_status_detector as bs_detector
 from doozerlib.cli import cli, pass_runtime
 from doozerlib.exceptions import DoozerFatalError
@@ -204,8 +204,12 @@ def detect_embargoes_in_releases(runtime: Runtime, pullspecs: List[str]):
     :return: list of Brew build dicts that have embargoed fixes
     """
     runtime.logger.info(f"Fetching component pullspecs from {len(pullspecs)} release payloads...")
-    jobs = runtime.parallel_exec(lambda pullspec, _: get_image_pullspecs_from_release_payload(pullspec), pullspecs,
-                                 min(len(pullspecs), multiprocessing.cpu_count() * 4, 32))
+    ignore_rhcos_tags = rhcos.get_container_names(runtime)
+    jobs = runtime.parallel_exec(
+        lambda pullspec, _: get_image_pullspecs_from_release_payload(pullspec, ignore_rhcos_tags),
+        pullspecs,
+        min(len(pullspecs), multiprocessing.cpu_count() * 4, 32)
+    )
     pullspec_lists = jobs.get()
     embargoed_releases = []
     embargoed_pullspecs = []
@@ -272,7 +276,7 @@ def get_nvr_by_pullspec(pullspec: str) -> Tuple[str, str, str]:
     return (labels.get("com.redhat.component"), labels.get("version"), labels.get("release"))
 
 
-def get_image_pullspecs_from_release_payload(payload_pullspec: str, ignore={"machine-os-content"}) -> Iterable[str]:
+def get_image_pullspecs_from_release_payload(payload_pullspec: str, ignore=set()) -> Iterable[str]:
     """ Retrieves pullspecs of images in a release payload.
     :param payload_pullspec: release payload pullspec
     :param ignore: a set of image names that we want to exclude from the return value (e.g. machine-os-content)

doozerlib/cli/get_nightlies.py

@@ -4,10 +4,7 @@
 from urllib import request
 from doozerlib.cli import cli
 from doozerlib import constants, util, exectools
-
-# See https://github.com/openshift/machine-config-operator/blob/master/docs/OSUpgrades.md
-# But in the future this will be replaced, see https://github.com/coreos/enhancements/blob/main/os/coreos-layering.md
-OLD_FORMAT_COREOS_TAG = 'machine-os-content'
+from doozerlib.rhcos import get_primary_container_name
 
 
 @cli.command("get-nightlies", short_help="Get sets of Accepted nightlies. A set contains nightly for each arch, "
@@ -40,6 +37,8 @@ def ignore_arch(arch):
             phase = ''
         nightlies_with_phase[arch] = all_nightlies_in_phase(major, minor, arch, phase)
 
+    primary_coreos_tag = get_primary_container_name(runtime)
+
     i = 0
     for x64_nightly, x64_phase in nightlies_with_phase["amd64"]:
         if i >= limit:
@@ -56,7 +55,7 @@ def ignore_arch(arch):
             nightly_str = f'{nightly} {phase}'
             if rhcos:
                 if phase != 'Pending':
-                    rhcos = get_coreos_build_from_payload(get_nightly_pullspec(nightly, arch))
+                    rhcos = get_coreos_build_from_payload(get_nightly_pullspec(nightly, arch), primary_coreos_tag)
                     nightly_str += f' {rhcos}'
             print(nightly_str)
         print(",".join(nightly_set))
@@ -68,9 +67,9 @@ def get_nightly_pullspec(release, arch):
     return f'registry.ci.openshift.org/ocp{suffix}/release{suffix}:{release}'
 
 
-def get_coreos_build_from_payload(payload_pullspec):
-    """Retrive the build version of machine-os-content (e.g. 411.86.202206131434-0)"""
-    out, err = exectools.cmd_assert(["oc", "adm", "release", "info", "--image-for", OLD_FORMAT_COREOS_TAG, "--", payload_pullspec])
+def get_coreos_build_from_payload(payload_pullspec, primary_coreos_tag):
+    """Retrieve the build version of RHCOS (e.g. 411.86.202206131434-0)"""
+    out, err = exectools.cmd_assert(["oc", "adm", "release", "info", "--image-for", primary_coreos_tag, "--", payload_pullspec])
     if err:
         raise Exception(f"Error running oc adm: {err}")
     rhcos_pullspec = out.split('\n')[0]

doozerlib/cli/release_gen_assembly.py

@@ -13,6 +13,7 @@
 from doozerlib import exectools
 from doozerlib.model import Model
 from doozerlib import brew
+from doozerlib import rhcos
 from doozerlib.rpmcfg import RPMMetadata
 from doozerlib.image import BrewBuildImageInspector
 from doozerlib.runtime import Runtime
@@ -103,7 +104,7 @@ def exit_with_error(msg):
     final_previous_list: List[VersionInfo] = sorted(final_previous_list)
 
     reference_releases_by_arch: Dict[str, str] = dict()  # Maps brew arch name to nightly name
-    mosc_by_arch: Dict[str, str] = dict()  # Maps brew arch name to machine-os-content pullspec from nightly
+    rhcos_by_tag: Dict[str, Dict[str, str]] = dict()  # Maps RHCOS container name(s) to brew arch name to pullspec(s) from nightly
     component_image_builds: Dict[str, BrewBuildImageInspector] = dict()  # Maps component package_name to brew build dict found for nightly
     component_rpm_builds: Dict[str, Dict[int, Dict]] = dict()  # Dict[ package_name ] -> Dict[ el? ] -> brew build dict
     basis_event_ts: float = 0.0
@@ -130,6 +131,7 @@ def exit_with_error(msg):
             raise ValueError(f'Cannot process {standard_release_name} since {release_pullspecs[brew_cpu_arch]} is already included')
         release_pullspecs[brew_cpu_arch] = standard_pullspec
 
+    rhcos_tag_names = rhcos.get_container_names(runtime)
     for brew_cpu_arch, pullspec in release_pullspecs.items():
         runtime.logger.info(f'Processing release: {pullspec}')
 
@@ -143,8 +145,8 @@ def exit_with_error(msg):
             payload_tag_name = component_tag.name  # e.g. "aws-ebs-csi-driver"
             payload_tag_pullspec = component_tag['from'].name  # quay pullspec
 
-            if payload_tag_name == 'machine-os-content':
-                mosc_by_arch[brew_cpu_arch] = payload_tag_pullspec
+            if payload_tag_name in rhcos_tag_names:
+                rhcos_by_tag.setdefault(payload_tag_name, {})[brew_cpu_arch] = payload_tag_pullspec
                 continue
 
             # The brew_build_inspector will take this archive image and find the actual
@@ -184,7 +186,7 @@ def exit_with_error(msg):
             basis_event_ts = max(basis_event_ts, completion_ts + (60.0 * 5))
 
     # basis_event_ts should now be greater than the build completion / target tagging operation
-    # for any (non machine-os-content) image in the nightlies. Because images are built after RPMs,
+    # for any (non RHCOS) image in the nightlies. Because images are built after RPMs,
     # it must also hold that the basis_event_ts is also greater than build completion & tagging
     # of any member RPM.
 
@@ -253,15 +255,16 @@ def exit_with_error(msg):
         # image NVR does not need to be pinned. Yeah!
         pass
 
-    # We should have found a machine-os-content for each architecture in the group for a standard assembly
+    # We should have found an RHCOS container for each architecture in the group for a standard assembly
+    primary_rhcos_tag = rhcos.get_primary_container_name(runtime)
     for arch in runtime.arches:
-        if arch not in mosc_by_arch:
+        if arch not in rhcos_by_tag[primary_rhcos_tag]:
             if custom:
                 # This is permitted for custom assemblies which do not need to be assembled for every
                 # architecture. The customer may just need x86_64.
-                logger.info(f'Did not find machine-os-content image for active group architecture: {arch}; ignoring since this is custom.')
+                logger.info(f'Did not find RHCOS "{primary_rhcos_tag}" image for active group architecture: {arch}; ignoring for custom assembly type.')
             else:
-                exit_with_error(f'Did not find machine-os-content image for active group architecture: {arch}')
+                exit_with_error(f'Did not find RHCOS "{primary_rhcos_tag}" image for active group architecture: {arch}')
 
     # We now have a list of image builds that should be selected by the assembly basis event
     # and those that will need to be forced with 'is'. We now need to perform a similar step
@@ -355,7 +358,7 @@ def exit_with_error(msg):
         # If the user has specified fewer nightlies than is required by this
         # group, then we need to override the group arches.
         group_info = {
-            'arches!': list(mosc_by_arch.keys())
+            'arches!': list(rhcos_by_tag[primary_rhcos_tag].keys())
         }
     if assembly_type not in [AssemblyTypes.CUSTOM, AssemblyTypes.PREVIEW]:
         # Add placeholder advisory numbers and JIRA key.
@@ -382,9 +385,8 @@ def exit_with_error(msg):
                     },
                     'group': group_info,
                     'rhcos': {
-                        'machine-os-content': {
-                            "images": mosc_by_arch,
-                        }
+                        tag: dict(images={arch: pullspec for arch, pullspec in specs_by_arch.items()})
+                        for tag, specs_by_arch in rhcos_by_tag.items()
                     },
                     'members': {
                         'rpms': rpm_member_overrides,