diff --git a/doozerlib/assembly.py b/doozerlib/assembly.py index f9c63d8e9..18d0445b5 100644 --- a/doozerlib/assembly.py +++ b/doozerlib/assembly.py @@ -56,6 +56,11 @@ class AssemblyIssueCode(Enum): # configured in the RHCOS pipeline for all arches. MISSING_RHCOS_CONTAINER = 7 + # A consistency requirement for packages between members (currently only + # specifiable in group.yml rhcos config) failed -- the members did not have + # the same version installed. + FAILED_CONSISTENCY_REQUIREMENT = 8 + class AssemblyIssue: """ diff --git a/doozerlib/assembly_inspector.py b/doozerlib/assembly_inspector.py index 70b743e77..adb6ad92d 100644 --- a/doozerlib/assembly_inspector.py +++ b/doozerlib/assembly_inspector.py @@ -13,24 +13,28 @@ class AssemblyInspector: """ It inspects an assembly """ - def __init__(self, runtime: Runtime, brew_session: ClientSession = None, lite: bool = False): + def __init__(self, runtime: Runtime, brew_session: ClientSession = None, lookup_mode: str = "both"): """ :param runtime: Doozer runtime :param brew_session: Brew session object to use for communicating with Brew - :param lite: Create a lite version without the ability to inspect Images; can be used to check AssemblyIssues, - fetch rhcos_builds and other defined methods + :param lookup_mode: + None: Create a lite version without the ability to inspect Images; can be used to check + AssemblyIssues, fetch rhcos_builds and other defined methods + "images": Do the lookups to enable image inspection, but expect code touching group RPMs + to fail (limited use case) + "both": Do the lookups for a full inspection """ self.runtime = runtime self.brew_session = brew_session - if not lite and runtime.mode != 'both': - raise ValueError('Runtime must be initialized with "both"') + if lookup_mode and runtime.mode != lookup_mode: + raise ValueError(f'Runtime must be initialized with "{lookup_mode}"') self.assembly_rhcos_config = assembly_rhcos_config(self.runtime.releases_config, self.runtime.assembly) self.assembly_type: AssemblyTypes = assembly_type(self.runtime.releases_config, self.runtime.assembly) self._rpm_build_cache: Dict[int, Dict[str, Optional[Dict]]] = {} # Dict[rhel_ver] -> Dict[distgit_key] -> Optional[BuildDict] self._permits = assembly_permits(self.runtime.releases_config, self.runtime.assembly) - if lite: + if not lookup_mode: # do no lookups return # If an image component has a latest build, an ImageInspector associated with the image. self._release_image_inspectors: Dict[str, Optional[BrewBuildImageInspector]] = dict() @@ -287,7 +291,7 @@ def get_group_rpm_build_dicts(self, el_ver: int) -> Dict[str, Optional[Dict]]: if el_ver in rpm_meta.determine_rhel_targets(): latest_build = rpm_meta.get_latest_build(default=None, el_target=el_ver) if not latest_build: - raise IOError(f'RPM {rpm_meta.distgit_key} claims to have a rhel-{el_ver} build target, but not build was detected') + raise IOError(f'RPM {rpm_meta.distgit_key} claims to have a rhel-{el_ver} build target, but no build was detected') assembly_rpm_dicts[rpm_meta.distgit_key] = latest_build else: # The RPM does not claim to build for this rhel version, so return None as a value. diff --git a/doozerlib/cli/inspect_stream.py b/doozerlib/cli/inspect_stream.py index f94370014..494978483 100644 --- a/doozerlib/cli/inspect_stream.py +++ b/doozerlib/cli/inspect_stream.py @@ -17,11 +17,11 @@ def inspect_stream(runtime, code, strict): code = AssemblyIssueCode[code] if runtime.assembly != 'stream': print(f'Disregarding non-stream assembly: {runtime.assembly}. This command is only intended for stream') - runtime.assembly = 'stream' + runtime.assembly = 'stream' runtime.initialize(clone_distgits=False) - assembly_inspector = AssemblyInspector(runtime, lite=True) if code == AssemblyIssueCode.INCONSISTENT_RHCOS_RPMS: + assembly_inspector = AssemblyInspector(runtime, lookup_mode=None) rhcos_builds, rhcos_inconsistencies = _check_inconsistent_rhcos_rpms(runtime, assembly_inspector) if rhcos_inconsistencies: msg = f'Found RHCOS inconsistencies in builds {rhcos_builds}' @@ -35,18 +35,46 @@ def inspect_stream(runtime, code, strict): print('Running in strict mode') exit(1) print(f'RHCOS builds consistent {rhcos_builds}') - exit(0) + elif code == AssemblyIssueCode.FAILED_CONSISTENCY_REQUIREMENT: + requirements = runtime.group_config.rhcos.require_consistency + if not requirements: + print("No cross-payload consistency requirements defined in group.yml") + exit(0) + + runtime.logger.info("Checking cross-payload consistency requirements defined in group.yml") + assembly_inspector = AssemblyInspector(runtime, lookup_mode="images") + issues = _check_cross_payload_consistency_requirements(runtime, assembly_inspector, requirements) + if issues: + print('Payload contents consistency requirements not satisfied') + pprint(issues) + not_permitted = [issue for issue in issues if not assembly_inspector.does_permit(issue)] + if not_permitted: + print(f'Assembly does not permit: {not_permitted}') + exit(1) + elif strict: + print('Running in strict mode; saw: {issues}') + exit(1) + print('Payload contents consistency requirements satisfied') else: print(f'AssemblyIssueCode {code} not supported at this time :(') exit(1) def _check_inconsistent_rhcos_rpms(runtime, assembly_inspector): - logger = runtime.logger rhcos_builds = [] for arch in runtime.group_config.arches: build_inspector = assembly_inspector.get_rhcos_build(arch) rhcos_builds.append(build_inspector) - logger.info(f"Checking following builds for inconsistency: {rhcos_builds}") + runtime.logger.info(f"Checking following builds for inconsistency: {rhcos_builds}") rhcos_inconsistencies = PayloadGenerator.find_rhcos_build_rpm_inconsistencies(rhcos_builds) return rhcos_builds, rhcos_inconsistencies + + +def _check_cross_payload_consistency_requirements(runtime, assembly_inspector, requirements): + issues = [] + for arch in runtime.group_config.arches: + issues.extend(PayloadGenerator.find_rhcos_payload_rpm_inconsistencies( + assembly_inspector.get_rhcos_build(arch), + assembly_inspector.get_group_release_images(), + requirements)) + return issues diff --git a/doozerlib/cli/release_gen_payload.py b/doozerlib/cli/release_gen_payload.py index 512756150..f3a23e2dd 100644 --- a/doozerlib/cli/release_gen_payload.py +++ b/doozerlib/cli/release_gen_payload.py @@ -538,6 +538,10 @@ def detect_extend_payload_entry_issues(self, assembly_inspector: AssemblyInspect the single-arch payloads. """ + primary_container_name = rhcos.get_primary_container_name(self.runtime) + cross_payload_requirements = self.runtime.group_config.rhcos.require_consistency + if not cross_payload_requirements: + self.runtime.logger.debug("No cross-payload consistency requirements defined in group.yml") # Structure to record rhcos builds we use so that they can be analyzed for inconsistencies targeted_rhcos_builds: Dict[bool, List[RHCOSBuildInspector]] = \ {False: [], True: []} # privacy mode: list of BuildInspector @@ -550,10 +554,22 @@ def detect_extend_payload_entry_issues(self, assembly_inspector: AssemblyInspect if ai.component == payload_entry.image_meta.distgit_key ) elif payload_entry.rhcos_build: + if tag != primary_container_name: + continue # RHCOS is one build, only analyze once (for primary container) + self.detect_rhcos_issues(payload_entry, assembly_inspector) # Record the build to enable later consistency checks between all RHCOS builds. # There are presently no private RHCOS builds, so add only to private_mode=False. targeted_rhcos_builds[False].append(payload_entry.rhcos_build) + + if cross_payload_requirements: + self.assembly_issues.extend( + PayloadGenerator.find_rhcos_payload_rpm_inconsistencies( + payload_entry.rhcos_build, + assembly_inspector.get_group_release_images(), + cross_payload_requirements, + ) + ) else: raise DoozerFatalError(f"Unsupported PayloadEntry: {payload_entry}") @@ -1263,6 +1279,86 @@ def find_rhcos_build_rpm_inconsistencies(rhcos_builds: List[RHCOSBuildInspector] # Report back rpm name keys which were associated with more than one NVR in the set of RHCOS builds. return {rpm_name: nvr_dict for rpm_name, nvr_dict in rpm_uses.items() if len(nvr_dict) > 1} + @staticmethod + def find_rhcos_payload_rpm_inconsistencies( + primary_rhcos_build: RHCOSBuildInspector, + payload_bbii: Dict[str, BrewBuildImageInspector], + # payload tag -> [pkg_name1, ...] + payload_consistency_config: Dict[str, List[str]]) -> List[AssemblyIssue]: + """ + Compares designated brew packages installed in designated payload members with the RPMs + in an RHCOS build, ensuring that both have the same version installed. + + This is a little more tricky with RHCOS than other payload content because RHCOS metadata + supplies only the installed RPMs, not the package NVRs that brew metadata supplies for + containers, and the containers may install different subsets of the RPMs a package build + provides. So we have to do a little more work to find all RPMs for the NVRs installed in the + member and compare. + + :return: Returns a list of AssemblyIssue objects describing any inconsistencies found. + """ + issues: List[AssemblyIssue] = [] + + # index by name the RPMs installed in the RHCOS build + rhcos_rpm_vrs: Dict[str, str] = {} # name -> version-release + for rpm in primary_rhcos_build.get_os_metadata_rpm_list(): + name, _, version, release, _ = rpm + rhcos_rpm_vrs[name] = f"{version}-{release}" + + # check that each member consistency condition is met + primary_rhcos_build.runtime.logger.debug(f"Running payload consistency checks against {primary_rhcos_build}") + for payload_tag, consistent_pkgs in payload_consistency_config.items(): + bbii = payload_bbii.get(payload_tag) + if not bbii: + issues.append(AssemblyIssue( + f"RHCOS consistency configuration specifies a payload tag '{payload_tag}'" + " that does not exist", payload_tag, AssemblyIssueCode.IMPERMISSIBLE)) + continue + + # check that each specified package in the member is consistent with the RHCOS build + for pkg in consistent_pkgs: + issues.append(PayloadGenerator.validate_pkg_consistency_req(payload_tag, pkg, bbii, rhcos_rpm_vrs)) + + return [issue for issue in issues if issue] + + @staticmethod + def validate_pkg_consistency_req( + payload_tag: str, pkg: str, + bbii: BrewBuildImageInspector, + rhcos_rpm_vrs: Dict[str, str]) -> Optional[AssemblyIssue]: + """check that the specified package in the member is consistent with the RHCOS build""" + logger = bbii.runtime.logger + logger.debug(f"Checking consistency of {pkg} for {payload_tag} against RHCOS") + member_nvrs: Dict[str, Dict] = bbii.get_all_installed_package_build_dicts() # by name + try: + build = member_nvrs[pkg] + except KeyError: + return AssemblyIssue( + f"RHCOS consistency configuration specifies that payload tag '{payload_tag}' " + f"should install package '{pkg}', but it does not", + payload_tag, AssemblyIssueCode.FAILED_CONSISTENCY_REQUIREMENT) + + # get names of all the actual RPMs included in this package build, because that's what we + # have for comparison in the RHCOS metadata (not the package name). + rpm_names = set(rpm["name"] for rpm in bbii.get_rpms_in_pkg_build(build["build_id"])) + + # find package RPM names in the RHCOS build and check that they have the same version + required_vr = "-".join([build["version"], build["release"]]) + logger.debug(f"{payload_tag} {pkg} has RPMs {rpm_names} at version {required_vr}") + for name in rpm_names: + vr = rhcos_rpm_vrs.get(name) + if vr: + logger.debug(f"RHCOS RPM {name} version is {vr}") + if vr and vr != required_vr: + return AssemblyIssue( + f"RHCOS and '{payload_tag}' should both use the same build of " + f"package '{pkg}', but RHCOS has installed {name}-{vr} and " + f"'{payload_tag}' has installed from {build['nvr']}", + payload_tag, AssemblyIssueCode.FAILED_CONSISTENCY_REQUIREMENT) + # no need to check other RPMs from this package build, one is enough + + return None + @staticmethod def get_mirroring_destination(sha256: str, dest_repo: str) -> str: """ diff --git a/doozerlib/image.py b/doozerlib/image.py index 33107bc17..0e2ded3f9 100644 --- a/doozerlib/image.py +++ b/doozerlib/image.py @@ -918,6 +918,13 @@ def get_image_archive_inspectors(self) -> List[ArchiveImageInspector]: return self._cache[cn] + def get_rpms_in_pkg_build(self, build_id: int) -> List[Dict]: + """ + :return: Returns a list of brew RPM records from a single package build. + """ + with self.runtime.pooled_koji_client_session() as koji_api: + return koji_api.listRPMs(buildID=build_id) + def get_all_installed_rpm_dicts(self) -> List[Dict]: """ :return: Returns an aggregate set of all brew rpm definitions @@ -936,10 +943,10 @@ def get_all_installed_rpm_dicts(self) -> List[Dict]: def get_all_installed_package_build_dicts(self) -> Dict[str, Dict]: """ - :return: Returns an aggregate set of all brew build dicts for - packages installed on ALL architectures of this image build. This - code assumes that all image archives install the same package NVR - if they install it. + :return: Returns a Dict[package name -> brew build dict] for all + packages installed on ANY architecture of this image build. + (OSBS enforces that all image archives install the same package NVR + if they install it at all.) """ cn = 'get_all_installed_package_build_dicts' if cn not in self._cache: diff --git a/doozerlib/runtime.py b/doozerlib/runtime.py index de78ccfce..389e26cd9 100644 --- a/doozerlib/runtime.py +++ b/doozerlib/runtime.py @@ -1394,6 +1394,7 @@ def _get_remote_branch_ref(self, git_url, branch): hash, the hash will be returned without modification. """ self.logger.info('Checking if target branch {} exists in {}'.format(branch, git_url)) + try: out, _ = exectools.cmd_assert('git ls-remote --heads {} {}'.format(git_url, branch), retries=3) except Exception as err: diff --git a/tests/cli/test_gen_payload.py b/tests/cli/test_gen_payload.py index 7d0f688e1..41b4b01c2 100644 --- a/tests/cli/test_gen_payload.py +++ b/tests/cli/test_gen_payload.py @@ -208,13 +208,14 @@ def test_generate_payload_entries(self, pg_fpe_mock): self.assertEqual(gpcli.assembly_issues, ["issues"]) def test_detect_extend_payload_entry_issues(self): - gpcli = flexmock(rgp_cli.GenPayloadCli()) + runtime = MagicMock(group_config=Model()) + gpcli = flexmock(rgp_cli.GenPayloadCli(runtime)) spamEntry = rgp_cli.PayloadEntry( image_meta=Mock(distgit_key="spam"), issues=[], dest_pullspec="dummy", ) rhcosEntry = rgp_cli.PayloadEntry(rhcos_build="rbi", dest_pullspec="dummy", issues=[]) - gpcli.payload_entries_for_arch = dict(ppc64le=dict(spam=spamEntry, rhcos=rhcosEntry)) + gpcli.payload_entries_for_arch = dict(ppc64le={"spam": spamEntry, "machine-os-content": rhcosEntry}) gpcli.assembly_issues = [Mock(component="spam")] # should associate with spamEntry gpcli.should_receive("detect_rhcos_issues").with_args(rhcosEntry, None).once()