enable PayloadGenerator to validate RHCOS<->member consistency requir…

…ements

doozerlib/assembly.py

@@ -56,6 +56,11 @@ class AssemblyIssueCode(Enum):
     # configured in the RHCOS pipeline for all arches.
     MISSING_RHCOS_CONTAINER = 7
 
+    # A consistency requirement for packages between members (currently only
+    # specifiable in group.yml rhcos config) failed -- the members did not have
+    # the same version installed.
+    FAILED_CONSISTENCY_REQUIREMENT = 8
+
 
 class AssemblyIssue:
     """

doozerlib/assembly_inspector.py

@@ -13,24 +13,28 @@
 
 class AssemblyInspector:
     """ It inspects an assembly """
-    def __init__(self, runtime: Runtime, brew_session: ClientSession = None, lite: bool = False):
+    def __init__(self, runtime: Runtime, brew_session: ClientSession = None, lookup_mode: str = "both"):
         """
         :param runtime: Doozer runtime
         :param brew_session: Brew session object to use for communicating with Brew
-        :param lite: Create a lite version without the ability to inspect Images; can be used to check AssemblyIssues,
-        fetch rhcos_builds and other defined methods
+        :param lookup_mode:
+            None: Create a lite version without the ability to inspect Images; can be used to check
+                  AssemblyIssues, fetch rhcos_builds and other defined methods
+            "images": Do the lookups to enable image inspection, but expect code touching group RPMs
+                      to fail (limited use case)
+            "both": Do the lookups for a full inspection
         """
         self.runtime = runtime
         self.brew_session = brew_session
-        if not lite and runtime.mode != 'both':
-            raise ValueError('Runtime must be initialized with "both"')
+        if lookup_mode and runtime.mode != lookup_mode:
+            raise ValueError(f'Runtime must be initialized with "{lookup_mode}"')
 
         self.assembly_rhcos_config = assembly_rhcos_config(self.runtime.releases_config, self.runtime.assembly)
         self.assembly_type: AssemblyTypes = assembly_type(self.runtime.releases_config, self.runtime.assembly)
         self._rpm_build_cache: Dict[int, Dict[str, Optional[Dict]]] = {}  # Dict[rhel_ver] -> Dict[distgit_key] -> Optional[BuildDict]
         self._permits = assembly_permits(self.runtime.releases_config, self.runtime.assembly)
 
-        if lite:
+        if not lookup_mode:  # do no lookups
             return
         # If an image component has a latest build, an ImageInspector associated with the image.
         self._release_image_inspectors: Dict[str, Optional[BrewBuildImageInspector]] = dict()
@@ -287,7 +291,7 @@ def get_group_rpm_build_dicts(self, el_ver: int) -> Dict[str, Optional[Dict]]:
                 if el_ver in rpm_meta.determine_rhel_targets():
                     latest_build = rpm_meta.get_latest_build(default=None, el_target=el_ver)
                     if not latest_build:
-                        raise IOError(f'RPM {rpm_meta.distgit_key} claims to have a rhel-{el_ver} build target, but not build was detected')
+                        raise IOError(f'RPM {rpm_meta.distgit_key} claims to have a rhel-{el_ver} build target, but no build was detected')
                     assembly_rpm_dicts[rpm_meta.distgit_key] = latest_build
                 else:
                     # The RPM does not claim to build for this rhel version, so return None as a value.

doozerlib/cli/inspect_stream.py

@@ -17,11 +17,11 @@ def inspect_stream(runtime, code, strict):
     code = AssemblyIssueCode[code]
     if runtime.assembly != 'stream':
         print(f'Disregarding non-stream assembly: {runtime.assembly}. This command is only intended for stream')
-    runtime.assembly = 'stream'
+        runtime.assembly = 'stream'
     runtime.initialize(clone_distgits=False)
-    assembly_inspector = AssemblyInspector(runtime, lite=True)
 
     if code == AssemblyIssueCode.INCONSISTENT_RHCOS_RPMS:
+        assembly_inspector = AssemblyInspector(runtime, lookup_mode=None)
         rhcos_builds, rhcos_inconsistencies = _check_inconsistent_rhcos_rpms(runtime, assembly_inspector)
         if rhcos_inconsistencies:
             msg = f'Found RHCOS inconsistencies in builds {rhcos_builds}'
@@ -35,18 +35,46 @@ def inspect_stream(runtime, code, strict):
                 print('Running in strict mode')
             exit(1)
         print(f'RHCOS builds consistent {rhcos_builds}')
-        exit(0)
+    elif code == AssemblyIssueCode.FAILED_CONSISTENCY_REQUIREMENT:
+        requirements = runtime.group_config.rhcos.require_consistency
+        if not requirements:
+            print("No cross-payload consistency requirements defined in group.yml")
+            exit(0)
+
+        runtime.logger.info("Checking cross-payload consistency requirements defined in group.yml")
+        assembly_inspector = AssemblyInspector(runtime, lookup_mode="images")
+        issues = _check_cross_payload_consistency_requirements(runtime, assembly_inspector, requirements)
+        if issues:
+            print('Payload contents consistency requirements not satisfied')
+            pprint(issues)
+            not_permitted = [issue for issue in issues if not assembly_inspector.does_permit(issue)]
+            if not_permitted:
+                print(f'Assembly does not permit: {not_permitted}')
+                exit(1)
+            elif strict:
+                print('Running in strict mode; saw: {issues}')
+                exit(1)
+        print('Payload contents consistency requirements satisfied')
     else:
         print(f'AssemblyIssueCode {code} not supported at this time :(')
         exit(1)
 
 
 def _check_inconsistent_rhcos_rpms(runtime, assembly_inspector):
-    logger = runtime.logger
     rhcos_builds = []
     for arch in runtime.group_config.arches:
         build_inspector = assembly_inspector.get_rhcos_build(arch)
         rhcos_builds.append(build_inspector)
-    logger.info(f"Checking following builds for inconsistency: {rhcos_builds}")
+    runtime.logger.info(f"Checking following builds for inconsistency: {rhcos_builds}")
     rhcos_inconsistencies = PayloadGenerator.find_rhcos_build_rpm_inconsistencies(rhcos_builds)
     return rhcos_builds, rhcos_inconsistencies
+
+
+def _check_cross_payload_consistency_requirements(runtime, assembly_inspector, requirements):
+    issues = []
+    for arch in runtime.group_config.arches:
+        issues.extend(PayloadGenerator.find_rhcos_payload_rpm_inconsistencies(
+            assembly_inspector.get_rhcos_build(arch),
+            assembly_inspector.get_group_release_images(),
+            requirements))
+    return issues

doozerlib/cli/release_gen_payload.py

@@ -538,6 +538,10 @@ def detect_extend_payload_entry_issues(self, assembly_inspector: AssemblyInspect
         the single-arch payloads.
         """
 
+        primary_container_name = rhcos.get_primary_container_name(self.runtime)
+        cross_payload_requirements = self.runtime.group_config.rhcos.require_consistency
+        if not cross_payload_requirements:
+            self.runtime.logger.debug("No cross-payload consistency requirements defined in group.yml")
         # Structure to record rhcos builds we use so that they can be analyzed for inconsistencies
         targeted_rhcos_builds: Dict[bool, List[RHCOSBuildInspector]] = \
             {False: [], True: []}  # privacy mode: list of BuildInspector
@@ -550,10 +554,22 @@ def detect_extend_payload_entry_issues(self, assembly_inspector: AssemblyInspect
                         if ai.component == payload_entry.image_meta.distgit_key
                     )
                 elif payload_entry.rhcos_build:
+                    if tag != primary_container_name:
+                        continue  # RHCOS is one build, only analyze once (for primary container)
+
                     self.detect_rhcos_issues(payload_entry, assembly_inspector)
                     # Record the build to enable later consistency checks between all RHCOS builds.
                     # There are presently no private RHCOS builds, so add only to private_mode=False.
                     targeted_rhcos_builds[False].append(payload_entry.rhcos_build)
+
+                    if cross_payload_requirements:
+                        self.assembly_issues.extend(
+                            PayloadGenerator.find_rhcos_payload_rpm_inconsistencies(
+                                payload_entry.rhcos_build,
+                                assembly_inspector.get_group_release_images(),
+                                cross_payload_requirements,
+                            )
+                        )
                 else:
                     raise DoozerFatalError(f"Unsupported PayloadEntry: {payload_entry}")
 
@@ -1263,6 +1279,86 @@ def find_rhcos_build_rpm_inconsistencies(rhcos_builds: List[RHCOSBuildInspector]
         # Report back rpm name keys which were associated with more than one NVR in the set of RHCOS builds.
         return {rpm_name: nvr_dict for rpm_name, nvr_dict in rpm_uses.items() if len(nvr_dict) > 1}
 
+    @staticmethod
+    def find_rhcos_payload_rpm_inconsistencies(
+            primary_rhcos_build: RHCOSBuildInspector,
+            payload_bbii: Dict[str, BrewBuildImageInspector],
+            # payload tag -> [pkg_name1, ...]
+            payload_consistency_config: Dict[str, List[str]]) -> List[AssemblyIssue]:
+        """
+        Compares designated brew packages installed in designated payload members with the RPMs
+        in an RHCOS build, ensuring that both have the same version installed.
+
+        This is a little more tricky with RHCOS than other payload content because RHCOS metadata
+        supplies only the installed RPMs, not the package NVRs that brew metadata supplies for
+        containers, and the containers may install different subsets of the RPMs a package build
+        provides. So we have to do a little more work to find all RPMs for the NVRs installed in the
+        member and compare.
+
+        :return: Returns a list of AssemblyIssue objects describing any inconsistencies found.
+        """
+        issues: List[AssemblyIssue] = []
+
+        # index by name the RPMs installed in the RHCOS build
+        rhcos_rpm_vrs: Dict[str, str] = {}  # name -> version-release
+        for rpm in primary_rhcos_build.get_os_metadata_rpm_list():
+            name, _, version, release, _ = rpm
+            rhcos_rpm_vrs[name] = f"{version}-{release}"
+
+        # check that each member consistency condition is met
+        primary_rhcos_build.runtime.logger.debug(f"Running payload consistency checks against {primary_rhcos_build}")
+        for payload_tag, consistent_pkgs in payload_consistency_config.items():
+            bbii = payload_bbii.get(payload_tag)
+            if not bbii:
+                issues.append(AssemblyIssue(
+                    f"RHCOS consistency configuration specifies a payload tag '{payload_tag}'"
+                    " that does not exist", payload_tag, AssemblyIssueCode.IMPERMISSIBLE))
+                continue
+
+            # check that each specified package in the member is consistent with the RHCOS build
+            for pkg in consistent_pkgs:
+                issues.append(PayloadGenerator.validate_pkg_consistency_req(payload_tag, pkg, bbii, rhcos_rpm_vrs))
+
+        return [issue for issue in issues if issue]
+
+    @staticmethod
+    def validate_pkg_consistency_req(
+            payload_tag: str, pkg: str,
+            bbii: BrewBuildImageInspector,
+            rhcos_rpm_vrs: Dict[str, str]) -> Optional[AssemblyIssue]:
+        """check that the specified package in the member is consistent with the RHCOS build"""
+        logger = bbii.runtime.logger
+        logger.debug(f"Checking consistency of {pkg} for {payload_tag} against RHCOS")
+        member_nvrs: Dict[str, Dict] = bbii.get_all_installed_package_build_dicts()  # by name
+        try:
+            build = member_nvrs[pkg]
+        except KeyError:
+            return AssemblyIssue(
+                f"RHCOS consistency configuration specifies that payload tag '{payload_tag}' "
+                f"should install package '{pkg}', but it does not",
+                payload_tag, AssemblyIssueCode.FAILED_CONSISTENCY_REQUIREMENT)
+
+        # get names of all the actual RPMs included in this package build, because that's what we
+        # have for comparison in the RHCOS metadata (not the package name).
+        rpm_names = set(rpm["name"] for rpm in bbii.get_rpms_in_pkg_build(build["build_id"]))
+
+        # find package RPM names in the RHCOS build and check that they have the same version
+        required_vr = "-".join([build["version"], build["release"]])
+        logger.debug(f"{payload_tag} {pkg} has RPMs {rpm_names} at version {required_vr}")
+        for name in rpm_names:
+            vr = rhcos_rpm_vrs.get(name)
+            if vr:
+                logger.debug(f"RHCOS RPM {name} version is {vr}")
+            if vr and vr != required_vr:
+                return AssemblyIssue(
+                    f"RHCOS and '{payload_tag}' should both use the same build of "
+                    f"package '{pkg}', but RHCOS has installed {name}-{vr} and "
+                    f"'{payload_tag}' has installed from {build['nvr']}",
+                    payload_tag, AssemblyIssueCode.FAILED_CONSISTENCY_REQUIREMENT)
+                # no need to check other RPMs from this package build, one is enough
+
+        return None
+
     @staticmethod
     def get_mirroring_destination(sha256: str, dest_repo: str) -> str:
         """

doozerlib/image.py

@@ -918,6 +918,13 @@ def get_image_archive_inspectors(self) -> List[ArchiveImageInspector]:
 
         return self._cache[cn]
 
+    def get_rpms_in_pkg_build(self, build_id: int) -> List[Dict]:
+        """
+        :return: Returns a list of brew RPM records from a single package build.
+        """
+        with self.runtime.pooled_koji_client_session() as koji_api:
+            return koji_api.listRPMs(buildID=build_id)
+
     def get_all_installed_rpm_dicts(self) -> List[Dict]:
         """
         :return: Returns an aggregate set of all brew rpm definitions
@@ -936,10 +943,10 @@ def get_all_installed_rpm_dicts(self) -> List[Dict]:
 
     def get_all_installed_package_build_dicts(self) -> Dict[str, Dict]:
         """
-        :return: Returns an aggregate set of all brew build dicts for
-        packages installed on ALL architectures of this image build. This
-        code assumes that all image archives install the same package NVR
-        if they install it.
+        :return: Returns a Dict[package name -> brew build dict] for all
+        packages installed on ANY architecture of this image build.
+        (OSBS enforces that all image archives install the same package NVR
+        if they install it at all.)
         """
         cn = 'get_all_installed_package_build_dicts'
         if cn not in self._cache:

doozerlib/runtime.py

@@ -1394,6 +1394,7 @@ def _get_remote_branch_ref(self, git_url, branch):
                 hash, the hash will be returned without modification.
         """
         self.logger.info('Checking if target branch {} exists in {}'.format(branch, git_url))
+
         try:
             out, _ = exectools.cmd_assert('git ls-remote --heads {} {}'.format(git_url, branch), retries=3)
         except Exception as err:

tests/cli/test_gen_payload.py

@@ -208,13 +208,14 @@ def test_generate_payload_entries(self, pg_fpe_mock):
         self.assertEqual(gpcli.assembly_issues, ["issues"])
 
     def test_detect_extend_payload_entry_issues(self):
-        gpcli = flexmock(rgp_cli.GenPayloadCli())
+        runtime = MagicMock(group_config=Model())
+        gpcli = flexmock(rgp_cli.GenPayloadCli(runtime))
         spamEntry = rgp_cli.PayloadEntry(
             image_meta=Mock(distgit_key="spam"),
             issues=[], dest_pullspec="dummy",
         )
         rhcosEntry = rgp_cli.PayloadEntry(rhcos_build="rbi", dest_pullspec="dummy", issues=[])
-        gpcli.payload_entries_for_arch = dict(ppc64le=dict(spam=spamEntry, rhcos=rhcosEntry))
+        gpcli.payload_entries_for_arch = dict(ppc64le={"spam": spamEntry, "machine-os-content": rhcosEntry})
         gpcli.assembly_issues = [Mock(component="spam")]  # should associate with spamEntry
 
         gpcli.should_receive("detect_rhcos_issues").with_args(rhcosEntry, None).once()