gen-payload: Detect RHCOS kernel version inconsistencies

Produces `FAILED_CROSS_RPM_VERSIONS_REQUIREMENT` error if kernel-rt and
kernel have different versions in RHCOS.

doozerlib/assembly.py

@@ -61,6 +61,11 @@ class AssemblyIssueCode(Enum):
     # the same version installed.
     FAILED_CONSISTENCY_REQUIREMENT = 8
 
+    # A consistency requirement for different packages failed
+    # Currently this failure only occurs if kernel and kernel-rt rpms
+    # in x86_64 RHCOS have different versions
+    FAILED_CROSS_RPM_VERSIONS_REQUIREMENT = 9
+
 
 class AssemblyIssue:
     """

doozerlib/cli/release_gen_payload.py

@@ -573,6 +573,7 @@ def detect_extend_payload_entry_issues(self, assembly_inspector: AssemblyInspect
                 else:
                     raise DoozerFatalError(f"Unsupported PayloadEntry: {payload_entry}")
 
+        self.detect_rhcos_kernel_inconsistencies(targeted_rhcos_builds)
         self.detect_rhcos_inconsistent_rpms(targeted_rhcos_builds)  # across all arches
 
     def detect_rhcos_issues(self, payload_entry, assembly_inspector: AssemblyInspector):
@@ -612,6 +613,18 @@ def detect_rhcos_inconsistent_rpms(self, targeted_rhcos_builds: Dict[bool, List[
                     component="rhcos", code=AssemblyIssueCode.INCONSISTENT_RHCOS_RPMS
                 ))
 
+    def detect_rhcos_kernel_inconsistencies(self, targeted_rhcos_builds: Dict[bool, List[RHCOSBuildInspector]]):
+        for privacy_mode in self.privacy_modes:  # only for relevant modes
+            rhcos_builds = targeted_rhcos_builds[privacy_mode]
+            for rhcos_build in rhcos_builds:
+                inconsistencies = PayloadGenerator.find_rhcos_build_kernel_inconsistencies(rhcos_build)
+                if inconsistencies:
+                    self.assembly_issues.append(AssemblyIssue(
+                        f"Found kernel inconsistencies in RHCOS build {rhcos_build} "
+                        f"(private={privacy_mode}): {inconsistencies}",
+                        component="rhcos", code=AssemblyIssueCode.FAILED_CROSS_RPM_VERSIONS_REQUIREMENT
+                    ))
+
     def summarize_issue_permits(self, assembly_inspector: AssemblyInspector) -> (bool, Dict[str, Dict]):
         """
         Check whether the issues found are permitted,
@@ -1279,6 +1292,48 @@ def find_rhcos_build_rpm_inconsistencies(rhcos_builds: List[RHCOSBuildInspector]
         # Report back rpm name keys which were associated with more than one NVR in the set of RHCOS builds.
         return {rpm_name: nvr_dict for rpm_name, nvr_dict in rpm_uses.items() if len(nvr_dict) > 1}
 
+    @staticmethod
+    def find_rhcos_build_kernel_inconsistencies(rhcos_build: RHCOSBuildInspector) -> List[Dict[str, str]]:
+        """
+        Looks through a RHCOS build and finds if any of those builds contains a kernel-rt version that
+        is inconsistent with kernel.
+
+        e.g. kernel-4.18.0-372.43.1.el8_6 and kernel-rt-4.18.0-372.43.1.rt7.200.el8_6 are consistent,
+        while kernel-4.18.0-372.43.1.el8_6 and kernel-rt-4.18.0-372.41.1.rt7.198.el8_6 are not.
+        :return: Returns List[Dict[inconsistent_rpm_name, rpm_nvra]] The List will be empty
+                 if there are no inconsistencies detected.
+        """
+        inconsistencies = []
+        # rpm_list will be a list of rpms.
+        # Each entry is another list in the format of [name, epoch, version, release, arch].
+        rpm_list = rhcos_build.get_os_metadata_rpm_list()
+        rpms_dict = {entry[0]: entry for entry in rpm_list}
+
+        def _to_nvr(nevra):
+            return f'{nevra[0]}-{nevra[2]}-{nevra[3]}'
+
+        if {"kernel-core", "kernel-rt-core"} <= rpms_dict.keys():
+            # ["kernel-core", 0, "4.18.0", "372.43.1.el8_6", "x86_64"] => ["4.18.0", "372.43.1.el8_6"]
+            kernel_v, kernel_r = rpms_dict["kernel-core"][2:4]
+            # ["kernel-rt-core", 0, "4.18.0", "4.18.0-372.41.1.rt7.198.el8_6", "x86_64"] => ["4.18.0", "4.18.0-372.41.1.rt7.198.el8_6"]
+            kernel_rt_v, kernel_rt_r = rpms_dict["kernel-rt-core"][2:4]
+            inconsistency = False
+            if kernel_v != kernel_rt_v:
+                inconsistency = True
+            elif kernel_r != kernel_rt_r:
+                # 372.43.1.el8_6 => 372.43.1
+                kernel_r = kernel_r.rsplit('.', 1)[0]
+                # 372.41.1.rt7.198.el8_6 => 372.41.1
+                kernel_rt_r = kernel_rt_r.rsplit('.', 3)[0]
+                if kernel_r != kernel_rt_r:
+                    inconsistency = True
+            if inconsistency:
+                inconsistencies.append({
+                    "kernel-core": _to_nvr(rpms_dict["kernel-core"]),
+                    "kernel-rt-core": _to_nvr(rpms_dict["kernel-rt-core"]),
+                })
+        return inconsistencies
+
     @staticmethod
     def find_rhcos_payload_rpm_inconsistencies(
             primary_rhcos_build: RHCOSBuildInspector,

tests/cli/test_gen_payload.py

@@ -221,6 +221,8 @@ def test_detect_extend_payload_entry_issues(self):
         gpcli.should_receive("detect_rhcos_issues").with_args(rhcosEntry, None).once()
         gpcli.should_receive("detect_rhcos_inconsistent_rpms").once().with_args(
             {False: ["rbi"], True: []})
+        gpcli.should_receive("detect_rhcos_kernel_inconsistencies").once().with_args(
+            {False: ["rbi"], True: []})
 
         gpcli.detect_extend_payload_entry_issues(None)
         self.assertEqual(gpcli.assembly_issues, spamEntry.issues)