From 5bb97c70aa8b3f5d103eca77d8066048bff6a35a Mon Sep 17 00:00:00 2001 From: Justin Pierce Date: Wed, 19 Oct 2022 15:58:04 +0200 Subject: [PATCH] Add pipeline metadata to art imagestreams --- doozerlib/cli/release_gen_payload.py | 63 ++++++++++++++++++++++------ tests/cli/test_gen_payload.py | 25 +++++++---- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/doozerlib/cli/release_gen_payload.py b/doozerlib/cli/release_gen_payload.py index 883edd213..dbce3d042 100644 --- a/doozerlib/cli/release_gen_payload.py +++ b/doozerlib/cli/release_gen_payload.py @@ -2,6 +2,7 @@ import hashlib import traceback import sys +import os import json from pathlib import Path from typing import List, Optional, Tuple, Dict, NamedTuple, Iterable, Set, Any, Callable @@ -671,6 +672,7 @@ def write_imagestream_artifact_file(self, imagestream_namespace: str, imagestrea filename = f"updated-tags-for.{imagestream_namespace}.{imagestream_name}{'-partial' if incomplete_payload_update else ''}.yaml" with self.output_path.joinpath(filename).open("w+", encoding="utf-8") as out_file: istream_spec = PayloadGenerator.build_payload_imagestream( + self.runtime, imagestream_name, imagestream_namespace, istags, self.assembly_issues ) @@ -732,7 +734,7 @@ def update_single_arch_istags(apiobj: oc.APIObject): new_annotations = apiobj.model.metadata.annotations._primitive() new_annotations.pop("release.openshift.io/inconsistency", None) # Remove old inconsistency information if it exists - new_annotations.update(PayloadGenerator.build_inconsistency_annotation(self.assembly_issues)) + new_annotations.update(PayloadGenerator.build_imagestream_annotations(self.runtime, self.assembly_issues)) apiobj.model.metadata["annotations"] = new_annotations @@ -813,6 +815,7 @@ def sync_heterogeneous_payloads(self, multi_specs: Dict[bool, Dict[str, Dict[str # run oc adm release new on this set of tags -- once for each arch - to create the arch # specific release payloads. multi_release_is = PayloadGenerator.build_payload_imagestream( + self.runtime, imagestream_name, imagestream_namespace, multi_istags, assembly_wide_inconsistencies=self.assembly_issues) @@ -991,11 +994,22 @@ def create_multi_release_manifest_list( multi_release_dest, find_manifest_list_sha(multi_release_dest)) def apply_multi_imagestream_update(self, final_multi_pullspec: str, imagestream_name: str, multi_release_istag: str): - """Update the multi-release imagestream, pruning old nightlies and adding the new one.""" - # [lmeyer] Q: but what if this is NOT a nightly? then we should not prune. but it's probably - # ok because we won't have more than one tag in non-nightly imagestreams? + """ + If running with assembly==stream, updates release imagestream with a new imagestream tag for the nightly. Older + nightlies are pruned from the release imagestream. + When running for non-stream, creates 4.x-art-assembly-$name with the pullspec to contain the newly created + pullspec. + """ multi_art_latest_is = self.ensure_imagestream_apiobj(imagestream_name) + # For nightlies, these will to set as annotations on the release imagestream tag. + # For non-nightlies, the new 4.x-art-assembly-$name the annotations will also be + # applied at the top level annotations for the imagestream. + pipeline_metadata_annotations = { + 'release.openshift.io/build-url': os.getenv('BUILD_URL', ''), + 'release.openshift.io/runtime-brew-event': str(self.runtime.brew_event), + } + def add_multi_nightly_release(obj: oc.APIObject): obj_model = obj.model if obj_model.spec.tags is oc.Missing: @@ -1018,6 +1032,7 @@ def add_multi_nightly_release(obj: oc.APIObject): else: # For non-stream 4.x-art-assembly-$name, old imagestreamtags should be removed. release_tags.clear() + obj_model.metadata.annotations = pipeline_metadata_annotations # Now append a tag for our new nightly. release_tags.append({ @@ -1029,10 +1044,10 @@ def add_multi_nightly_release(obj: oc.APIObject): "type": "Source" }, "name": multi_release_istag, - "annotations": { - "release.openshift.io/rewrite": "false", # Prevents the release controller from trying to create a local registry release payload with oc adm release new. - # "release.openshift.io/name": f"{self.runtime.get_minor_version()}.0-0.nightly", - } + "annotations": dict(**{ + # Prevents the release controller from trying to create a local registry release payload with oc adm release new. + "release.openshift.io/rewrite": "false", + }, **pipeline_metadata_annotations) }) return True @@ -1214,10 +1229,10 @@ def build_payload_istag(payload_tag_name: str, payload_entry: PayloadEntry) -> D """ :param payload_tag_name: The name of the payload tag for which to create an istag. :param payload_entry: The payload entry to serialize into an imagestreamtag. - :return: Returns a imagestreamtag dict for a release payload imagestream. + :return: Returns an imagestreamtag dict for a release payload imagestream. """ return { - "annotations": PayloadGenerator.build_inconsistency_annotation(payload_entry.issues), + "annotations": PayloadGenerator.build_inconsistency_annotations(payload_entry.issues), "name": payload_tag_name, "from": { "kind": "DockerImage", @@ -1226,9 +1241,10 @@ def build_payload_istag(payload_tag_name: str, payload_entry: PayloadEntry) -> D } @staticmethod - def build_payload_imagestream(imagestream_name: str, imagestream_namespace: str, payload_istags: Iterable[Dict], assembly_wide_inconsistencies: Iterable[AssemblyIssue]) -> Dict: + def build_payload_imagestream(runtime, imagestream_name: str, imagestream_namespace: str, payload_istags: Iterable[Dict], assembly_wide_inconsistencies: Iterable[AssemblyIssue]) -> Dict: """ Builds a definition for a release payload imagestream from a set of payload istags. + :param runtime: The doozer Runtime. :param imagestream_name: The name of the imagstream to generate. :param imagestream_namespace: The nemspace in which the imagestream should be created. :param payload_istags: A list of istags generated by build_payload_istag. @@ -1242,7 +1258,7 @@ def build_payload_imagestream(imagestream_name: str, imagestream_namespace: str, "metadata": { "name": imagestream_name, "namespace": imagestream_namespace, - "annotations": PayloadGenerator.build_inconsistency_annotation(assembly_wide_inconsistencies) + "annotations": PayloadGenerator.build_imagestream_annotations(runtime, assembly_wide_inconsistencies) }, "spec": { "tags": list(payload_istags), @@ -1252,7 +1268,21 @@ def build_payload_imagestream(imagestream_name: str, imagestream_namespace: str, return istream_obj @staticmethod - def build_inconsistency_annotation(inconsistencies: Iterable[AssemblyIssue]): + def build_pipeline_metadata_annotations(runtime) -> Dict: + """ + :return: If the metadata is available, include information like the Jenkins job URL + in a nightly annotation. + Returns a dict of pipeline metadata annotations. + """ + pipeline_metadata = { + 'release.openshift.io/build-url': os.getenv('BUILD_URL', ''), + 'release.openshift.io/runtime-brew-event': str(runtime.brew_event), + } + + return pipeline_metadata + + @staticmethod + def build_inconsistency_annotations(inconsistencies: Iterable[AssemblyIssue]) -> Dict: """ :param inconsistencies: A list of strings to report as inconsistencies within an annotation. :return: Returns a dict containing an inconsistency annotation out of the specified issues list. @@ -1268,6 +1298,13 @@ def build_inconsistency_annotation(inconsistencies: Iterable[AssemblyIssue]): msgs[5:] = ["(...and more)"] return {"release.openshift.io/inconsistency": json.dumps(msgs)} + @staticmethod + def build_imagestream_annotations(runtime, inconsistencies: Iterable[AssemblyIssue]) -> Dict: + annotations = {} + annotations.update(PayloadGenerator.build_inconsistency_annotations(inconsistencies)) + annotations.update(PayloadGenerator.build_pipeline_metadata_annotations(runtime)) + return annotations + @staticmethod def get_group_payload_tag_mapping(assembly_inspector: AssemblyInspector, arch: str) -> Dict[str, Optional[ArchiveImageInspector]]: """ diff --git a/tests/cli/test_gen_payload.py b/tests/cli/test_gen_payload.py index a25033594..1e1edeb63 100644 --- a/tests/cli/test_gen_payload.py +++ b/tests/cli/test_gen_payload.py @@ -1,3 +1,4 @@ +import os from unittest import TestCase, skip from unittest.mock import MagicMock, Mock, patch from flexmock import flexmock @@ -389,18 +390,23 @@ def test_generate_specific_payload_imagestreams(self, build_mock): @patch("pathlib.Path.open") def test_write_imagestream_artifact_file(self, open_mock): - gpcli = rgp_cli.GenPayloadCli(output_dir="/tmp") + gpcli = rgp_cli.GenPayloadCli(output_dir="/tmp", runtime=Mock( + brew_event="999999", + assembly_type=AssemblyTypes.STREAM, + )) buffer = io.StringIO() cmgr = MagicMock(__enter__=lambda _: buffer) # make Path.open() return the buffer open_mock.return_value = cmgr gpcli.write_imagestream_artifact_file("ocp-s390x", "release-s390x", ["rhcos", "eggs"], True) - self.assertEqual(buffer.getvalue().strip(), """ + self.assertEqual(buffer.getvalue().strip(), f""" apiVersion: image.openshift.io/v1 kind: ImageStream metadata: - annotations: {} + annotations: + release.openshift.io/build-url: {os.getenv('BUILD_URL', "''")} + release.openshift.io/runtime-brew-event: '999999' name: release-s390x namespace: ocp-s390x spec: @@ -425,10 +431,13 @@ def test_ensure_imagestream_apiobj(self, oc_mock): oc_mock.selector.return_value = Mock(object=lambda **_: False) # other branch gpcli.ensure_imagestream_apiobj("release-s390x") - @patch("doozerlib.cli.release_gen_payload.PayloadGenerator.build_inconsistency_annotation") + @patch("doozerlib.cli.release_gen_payload.PayloadGenerator.build_inconsistency_annotations") @patch("doozerlib.cli.release_gen_payload.modify_and_replace_api_object") def test_apply_imagestream_update(self, mar_mock, binc_mock): - gpcli = rgp_cli.GenPayloadCli(output_dir="/tmp") + gpcli = rgp_cli.GenPayloadCli(output_dir="/tmp", runtime=Mock( + brew_event="999999", + assembly_type=AssemblyTypes.STREAM, + )) # make method do basically what it would, without writing all the files mar_mock.side_effect = lambda apiobj, func, *_: func(apiobj) @@ -615,5 +624,7 @@ def test_apply_multi_imagestream_update(self, mar_mock): gpcli.apply_multi_imagestream_update("final_pullspec", "is_name", "multi_release_name") self.assertNotIn(dict(name="spam1"), istream_apiobj.model.spec.tags, "should have been pruned") self.assertIn(dict(name="spam2"), istream_apiobj.model.spec.tags, "not pruned") - self.assertEqual({"release.openshift.io/rewrite": "false"}, - istream_apiobj.model.spec.tags[-1]["annotations"], "added") + new_tag_annotations = istream_apiobj.model.spec.tags[-1]['annotations'] + self.assertEqual('false', new_tag_annotations['release.openshift.io/rewrite']) + self.assertEqual(os.getenv('BUILD_URL', ''), new_tag_annotations['release.openshift.io/build-url']) + self.assertIn('release.openshift.io/runtime-brew-event', new_tag_annotations)