From 32e494c6df6588839600aecc168dd6a006ee6ce7 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 3 Mar 2023 18:19:00 +0100 Subject: [PATCH 1/5] Refactor image building and add preflight checks. --- .github/workflows/product_images_dispatch.yml | 12 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 1 + README.adoc | 17 +- build_product_images.py | 283 ------------------ image_tools/__init__.py | 1 + image_tools/args.py | 71 +++++ image_tools/bake.py | 154 ++++++++++ conf.py => image_tools/conf.py | 0 image_tools/lib.py | 26 ++ image_tools/preflight.py | 72 +++++ 11 files changed, 346 insertions(+), 293 deletions(-) delete mode 100755 build_product_images.py create mode 100644 image_tools/__init__.py create mode 100644 image_tools/args.py create mode 100644 image_tools/bake.py rename conf.py => image_tools/conf.py (100%) create mode 100644 image_tools/lib.py create mode 100644 image_tools/preflight.py diff --git a/.github/workflows/product_images_dispatch.yml b/.github/workflows/product_images_dispatch.yml index c87077e9a..93466bd83 100644 --- a/.github/workflows/product_images_dispatch.yml +++ b/.github/workflows/product_images_dispatch.yml @@ -52,4 +52,14 @@ jobs: - name: Build and push images on dispatch shell: bash - run: python build_product_images.py -p ${{ github.event.inputs.product }} -i ${{ github.event.inputs.image-version }} -o ${{ github.event.inputs.organization }} -a ${{ github.event.inputs.architecture }} -u + run: python -m image_tools.bake -p ${{ github.event.inputs.product }} -i ${{ github.event.inputs.image-version }} -o ${{ github.event.inputs.organization }} -a ${{ github.event.inputs.architecture }} -u + - + name: Install preflight + run: | + curl -fLo preflight https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/latest/download/preflight-linux-amd64 + chmod +x preflight + + - + name: Run preflight + shell: bash + run: python -m image_tools.preflight -p ${{ github.event.inputs.product }} -i ${{ github.event.inputs.image-version }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 352ad72c8..286ee3e30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,4 +52,4 @@ jobs: username: github password: ${{ secrets.NEXUS_PASSWORD }} - name: Build and push images (single arch) - run: python build_product_images.py --product "${{ matrix.product }}" --image-version "$GITHUB_REF_NAME" --organization stackable --architecture linux/amd64 --push + run: python -m image_tools.bake --product "${{ matrix.product }}" --image-version "$GITHUB_REF_NAME" --organization stackable --architecture linux/amd64 --push diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ec1a7561..09f0f6e0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file. - Prometheus image, NodeExporter image, Antora ([#95]). - Retired Java 1.8.0 support ([#248]). - Tools image ([#325]). +- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([xxx]) [#95]: https://github.com/stackabletech/docker-images/pull/95 [#248]: https://github.com/stackabletech/docker-images/pull/248 diff --git a/README.adoc b/README.adoc index 4c9b6ad85..a6861436e 100644 --- a/README.adoc +++ b/README.adoc @@ -33,19 +33,20 @@ USER 1000:1000 ENTRYPOINT ["/stackable-zookeeper-operator"] ---- -== Product images +== Build Product Images -Product images are used by stackable operators to set up service clusters. For example, the Apache ZooKeeper product image contains Apache ZooKeeper and some additional things required to monitor and set up containers corectly. - -Product images are tagged with `-` tags. The `product-version` is the version of the product installed in the image and `image-version` is the version of the image as referenced by the stackable plattform. - -Product images are published to the `docker.stackable.tech` registry. +Product images are published to the `docker.stackable.tech` registry under the `stackable` organization. To build and push product images to the default repository, use the `build_product_images.py` like this: - build_product_images.py -p zookeeper -i 0.1 -u + python -m image_tools.bake --product zookeeper --image 23.1.0-dev --push -This will build images for Apache ZooKeeper versions as defined in the `conf.py` and tag them with `image-version` 0.1. +This will build images for Apache ZooKeeper versions as defined in the `image_tools/conf.py` and tag them with the `image-version` 23.1.0-dev The GitHub action called `Product images` can be triggered manually to do the same but not on the local machine. +== Verify Product Images + +To verify if Apache Zookeeper validate against OpenShift preflight, run: + + python -m image_tools.preflight --product zookeeper --image 23.1.0-dev diff --git a/build_product_images.py b/build_product_images.py deleted file mode 100755 index 1713ef90d..000000000 --- a/build_product_images.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python -""" -Build and possibly publish product images. - -Run doc tests with: - - python -m doctest -v build_product_images.py - -Requirements: - -- Python 3 -- Docker with buildx. Installation details here: https://github.com/docker/buildx - -Usage: build_product_images.py --help - -Example: - - build_product_images.py --product zookeeper --image-version 23.1.1 --architecture linux/amd64 - -This will build all images parsed from conf.py (e.g. `docker.stackable.tech/stackable/zookeeper:3.8.0-stackable23.1.1`) for the linux/amd64 architecture. -To also push the image to a remote registry, add the the `--push` argument. - -NOTE: Pushing images to a remote registry assumes you have performed a `docker login` beforehand. - -Some images build on top of others. These images are used as base images and will be built first: - 1. java-base - 2. ubi8-rust-builder - 3. tools - -If a key in products[_].versions matches another product definition then it is assumed that it is a dependency, -and that product will be built first. It can then be used as a base layer using the format `stackable/image/{product}`. -""" -from os.path import isdir -from typing import List, Dict -from argparse import Namespace, ArgumentParser -import subprocess -import conf -import re -import json - -# This is the stackable release version -DEFAULT_IMAGE_VERSION_FROMATS = [re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+"), re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+-rc[1-9]\d?")] - - -def parse_args() -> Namespace: - parser = ArgumentParser( - description="Build and publish product images. Requires docker and buildx (https://github.com/docker/buildx)." - ) - parser.add_argument("-i", "--image-version", help="Image version", required=True, type=check_image_version_format) - parser.add_argument("-p", "--product", help="Product to build images for") - parser.add_argument("-u", "--push", help="Push images", action="store_true") - parser.add_argument("-d", "--dry", help="Dry run.", action="store_true") - parser.add_argument( - "-a", - "--architecture", - help="Target platform for image. Default: linux/amd64.", - nargs="+", - default=["linux/amd64"], - type=check_architecture_input, - ) - parser.add_argument( - "-o", - "--organization", - help="Organization name within the given registry. Default: stackable", - default="stackable", - ) - parser.add_argument( - "-r", - "--registry", - help="Image registry to publish to. Default: docker.stackable.tech", - default="docker.stackable.tech", - ) - return parser.parse_args() - - -def check_image_version_format(image_version) -> str: - """ - Check image version against allowed formats. - - >>> check_image_version_format("23.4.0") - '23.4.0' - >>> check_image_version_format("23.4.0-rc1") - '23.4.0-rc1' - >>> check_image_version_format("23.04.0") - Traceback (most recent call last): - ... - ValueError: Invalid image version: 23.04.0 - >>> check_image_version_format("23.4.0.prerelease") - Traceback (most recent call last): - ... - ValueError: Invalid image version: 23.4.0.prerelease - """ - for p in DEFAULT_IMAGE_VERSION_FROMATS: - if p.fullmatch(image_version): - return image_version - raise ValueError(f"Invalid image version: {image_version}") - - -def build_image_args(version, release_version): - """ - Returns a list of --build-arg command line arguments that are used by the - docker build command. - - Arguments: - - version: Can be a str, in which case it's considered the PRODUCT - or a dict. - """ - result = {} - - if isinstance(version, dict): - for k, v in version.items(): - result[k.upper()] = v - result["RELEASE"] = release_version - elif isinstance(version, str) and isinstance(release_version, str): - { - "PRODUCT": version, - "RELEASE": release_version, - } - else: - raise ValueError(f"Unsupported version object: {version}") - - return result - - -def build_image_tags(image_name: str, image_version: str, product_version: str) -> List[str]: - """ - Returns the --tag command line arguments that are used by the docker build command. - """ - return [ - f"{image_name}:{product_version}-stackable{image_version}", - ] - - -def generate_bakefile(args: Namespace): - """ - Generates a Bakefile (https://docs.docker.com/build/bake/file-definition/) describing how to build the whole image graph. - - build_and_publish_images() ensures that only the desired images are actually built. - """ - targets = {} - groups = {} - product_names = [product["name"] for product in conf.products] - for product in conf.products: - product_name = product["name"] - product_targets = {} - for version_dict in product.get("versions"): - product_targets.update(bakefile_product_version_targets(args, product_name, version_dict, product_names)) - groups[product_name] = { - "targets": list(product_targets.keys()), - } - targets.update(product_targets) - groups["default"] = { - "targets": list(groups.keys()), - } - return { - "target": targets, - "group": groups, - } - - -def bakefile_target_name_for_product_version(product_name: str, version: str) -> str: - """ - Creates a normalized Bakefile target name for a given (product, version) combination. - """ - return f"{ product_name }-{ version.replace('.', '_') }" - - -def bakefile_product_version_targets(args: Namespace, product_name: str, versions: Dict[str, str], product_names: List[str]): - """ - Creates Bakefile targets defining how to build a given product version. - - A product is assumed to depend on another if it defines a `versions` field with the same name as the other product. - """ - image_name = f'{args.registry}/{args.organization}/{product_name}' - tags = build_image_tags( - image_name, args.image_version, versions["product"] - ) - build_args = build_image_args(versions, args.image_version) - - return { - bakefile_target_name_for_product_version(product_name, versions['product']): { - "dockerfile": f"{ product_name }/Dockerfile", - "tags": tags, - "args": build_args, - "platforms": args.architecture, - "context": ".", - "contexts": {f"stackable/image/{dep_name}": f"target:{bakefile_target_name_for_product_version(dep_name, dep_version)}" for dep_name, dep_version in versions.items() if dep_name in product_names}, - }, - } - - -def build_and_publish_image(args: Namespace, product_name: str, bakefile): - """ - Returns a list of commands that need to be run in order to build and - publish product images. - - For local building, builder instances are supported. - """ - - if args.push: - target_mode = ["--push"] - elif len(args.architecture) == 1: - target_mode = ["--load"] - else: - target_mode = [] - - command = { - "args": [ - "docker", - "buildx", - "bake", - "--file", - "-", - *([] if product_name is None else [product_name]), - *target_mode, - ], - "stdin": json.dumps(bakefile), - } - return [command] - - -def run_commands(dry, commands): - """ - Runs the commands to build and publish images. In dry-run mode it only - lists the command on stdout. - """ - for cmd in commands: - if isinstance(cmd, dict): - args = cmd['args'] - stdin = cmd.get('stdin') - else: - args = cmd - stdin = None - - if dry: - if stdin: - print(f"{' '.join(args)} << 1: - create_virtual_environment(args) - - try: - commands = build_and_publish_image(args, args.product, bakefile) - run_commands(args.dry, commands) - finally: - if len(args.architecture) > 1: - remove_virtual_environment(args) - - -if __name__ == "__main__": - main() diff --git a/image_tools/__init__.py b/image_tools/__init__.py new file mode 100644 index 000000000..80d60700e --- /dev/null +++ b/image_tools/__init__.py @@ -0,0 +1 @@ +"""Tools for image management.""" diff --git a/image_tools/args.py b/image_tools/args.py new file mode 100644 index 000000000..4520097ff --- /dev/null +++ b/image_tools/args.py @@ -0,0 +1,71 @@ +# This is the stackable release version +from argparse import Namespace, ArgumentParser +from typing import List, Dict, Optional +import re + +DEFAULT_IMAGE_VERSION_FORMATS = [re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+"), re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+-rc[1-9]\d?")] + + +def parse() -> Namespace: + parser = ArgumentParser( + description="Build and publish product images. Requires docker and buildx (https://github.com/docker/buildx)." + ) + parser.add_argument("-i", "--image-version", help="Image version", required=True, type=check_image_version_format) + parser.add_argument("-p", "--product", help="Product to build images for") + parser.add_argument("-u", "--push", help="Push images", action="store_true") + parser.add_argument("-d", "--dry", help="Dry run.", action="store_true") + parser.add_argument( + "-a", + "--architecture", + help="Target platform for image. Default: linux/amd64.", + nargs="+", + default=["linux/amd64"], + type=check_architecture_input, + ) + parser.add_argument( + "-o", + "--organization", + help="Organization name within the given registry. Default: stackable", + default="stackable", + ) + parser.add_argument( + "-r", + "--registry", + help="Image registry to publish to. Default: docker.stackable.tech", + default="docker.stackable.tech", + ) + return parser.parse_args() + + +def check_image_version_format(image_version) -> str: + """ + Check image version against allowed formats. + + >>> check_image_version_format("23.4.0") + '23.4.0' + >>> check_image_version_format("23.4.0-rc1") + '23.4.0-rc1' + >>> check_image_version_format("23.04.0") + Traceback (most recent call last): + ... + ValueError: Invalid image version: 23.04.0 + >>> check_image_version_format("23.4.0.prerelease") + Traceback (most recent call last): + ... + ValueError: Invalid image version: 23.4.0.prerelease + """ + for p in DEFAULT_IMAGE_VERSION_FORMATS: + if p.fullmatch(image_version): + return image_version + raise ValueError(f"Invalid image version: {image_version}") + + +def check_architecture_input(architecture) -> List[str]: + supported_arch = ["linux/amd64", "linux/arm64"] + + if architecture not in supported_arch: + raise ValueError( + f"Architecture {architecture} not supported. Supported: {supported_arch}" + ) + + return architecture diff --git a/image_tools/bake.py b/image_tools/bake.py new file mode 100644 index 000000000..d280a89c7 --- /dev/null +++ b/image_tools/bake.py @@ -0,0 +1,154 @@ +"""Image builder + +Requirements: docker and buildx. + +Usage: + + python -m image_tools.bake -p opa -i 22.12.0 +""" +from image_tools.lib import Command +import image_tools.conf as conf +from image_tools.args import parse +from typing import List, Dict, Any +from argparse import Namespace +from subprocess import run, CompletedProcess +import json + + +def build_image_args(version, release_version): + """ + Returns a list of --build-arg command line arguments that are used by the + docker build command. + + Arguments: + - version: Can be a str, in which case it's considered the PRODUCT + or a dict. + """ + result = {} + + if isinstance(version, dict): + for k, v in version.items(): + result[k.upper()] = v + result["RELEASE"] = release_version + elif isinstance(version, str) and isinstance(release_version, str): + { + "PRODUCT": version, + "RELEASE": release_version, + } + else: + raise ValueError(f"Unsupported version object: {version}") + + return result + + +def build_image_tags(image_name: str, image_version: str, product_version: str) -> List[str]: + """ + Returns the --tag command line arguments that are used by the docker build command. + """ + return [ + f"{image_name}:{product_version}-stackable{image_version}", + ] + + +def generate_bakefile(args: Namespace) -> Dict[str, Any]: + """ + Generates a Bakefile (https://docs.docker.com/build/bake/file-definition/) describing how to build the whole image graph. + + build_and_publish_images() ensures that only the desired images are actually built. + """ + targets = {} + groups = {} + product_names = [product["name"] for product in conf.products] + for product in conf.products: + product_name = product["name"] + product_targets = {} + for version_dict in product.get("versions"): + product_targets.update(bakefile_product_version_targets(args, product_name, version_dict, product_names)) + groups[product_name] = { + "targets": list(product_targets.keys()), + } + targets.update(product_targets) + groups["default"] = { + "targets": list(groups.keys()), + } + return { + "target": targets, + "group": groups, + } + + +def bakefile_target_name_for_product_version(product_name: str, version: str) -> str: + """ + Creates a normalized Bakefile target name for a given (product, version) combination. + """ + return f"{ product_name }-{ version.replace('.', '_') }" + + +def bakefile_product_version_targets(args: Namespace, product_name: str, versions: Dict[str, str], product_names: List[str]): + """ + Creates Bakefile targets defining how to build a given product version. + + A product is assumed to depend on another if it defines a `versions` field with the same name as the other product. + """ + image_name = f'{args.registry}/{args.organization}/{product_name}' + tags = build_image_tags( + image_name, args.image_version, versions["product"] + ) + build_args = build_image_args(versions, args.image_version) + + return { + bakefile_target_name_for_product_version(product_name, versions['product']): { + "dockerfile": f"{ product_name }/Dockerfile", + "tags": tags, + "args": build_args, + "platforms": args.architecture, + "context": ".", + "contexts": {f"stackable/image/{dep_name}": f"target:{bakefile_target_name_for_product_version(dep_name, dep_version)}" for dep_name, dep_version in versions.items() if dep_name in product_names}, + }, + } + + +def bake_command(args: Namespace, product_name: str, bakefile) -> Command: + """ + Returns a list of commands that need to be run in order to build and + publish product images. + + For local building, builder instances are supported. + """ + + if args.push: + target_mode = ["--push"] + elif len(args.architecture) == 1: + target_mode = ["--load"] + else: + target_mode = [] + + return Command( + args=[ + "docker", + "buildx", + "bake", + "--file", + "-", + *([] if product_name is None else [product_name]), + *target_mode, + ], + stdin=json.dumps(bakefile), + ) + + +def main(): + """Generate a Docker bake file from conf.py and build the given args.product images.""" + args = parse() + bakefile = generate_bakefile(args) + + cmd = bake_command(args, args.product, bakefile) + + if args.dry: + print(cmd) + else: + run(cmd.args, input=cmd.input, check=True) + + +if __name__ == '__main__': + main() diff --git a/conf.py b/image_tools/conf.py similarity index 100% rename from conf.py rename to image_tools/conf.py diff --git a/image_tools/lib.py b/image_tools/lib.py new file mode 100644 index 000000000..6ddb80caf --- /dev/null +++ b/image_tools/lib.py @@ -0,0 +1,26 @@ +"""Library code for image tools.""" +from dataclasses import dataclass, field +from typing import List, Optional +from collections.abc import Sequence +from itertools import chain + + +@dataclass(frozen=True) +class Command: + """Command line program including arguments and optional standard input.""" + args: List[str] = field(default_factory=list) + stdin: Optional[str] = field(default=None) + + @property + def input(self) -> Optional[bytes]: + """stdin input as UTF8 bytes.""" + if self.stdin: + return self.stdin.encode("utf-8") + else: + return None + + def __str__(self) -> str: + if self.stdin: + return f"{' '.join(self.args)} << List[str]: + tags = [] + + for target in bakefile.get("group", {}).get(product, {}).get("targets", []): + tags.extend(get_images_for_target(target, bakefile)) + + tags.extend(bakefile["target"].get(product, {}).get("tags", [])) + + return tags + + +def main(): + """Generate a Docker bake file from conf.py and build the given args.product images.""" + logging.basicConfig(encoding="utf-8", level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') + + args = parse() + bakefile = generate_bakefile(args) + + images = get_images_for_target(args.product, bakefile) + if not images: + logging.error("No images found for product [%s]", args.product) + return 1 + + preflight_cmds = {image: Command(args=["preflight", "check", "container", image]) for image in images} + + failures = {} + for image, cmd in preflight_cmds.items(): + if args.dry: + print(str(cmd)) + else: + try: + preflight_result = run(cmd.args, input=cmd.input, check=True, capture_output=True) + preflight_json = json.loads(preflight_result.stdout) + failures[image] = preflight_json.get("results", {}).get("failed", []) + except CalledProcessError as error: + failures[image] = [error.stderr.decode('utf-8')] + + for image, ifails in failures.items(): + if len(ifails) == 0: + logging.info("Image [%s] preflight check successful.", image) + else: + logging.error("Image [%s] preflight check failures:\n%s", image, "\n".join(ifails)) + + fail_count = sum(map(lambda f: len(f), failures.values())) + return fail_count + + +if __name__ == '__main__': + sys.exit(main()) From 6e2dcb3a54a49e76f379efb4ebb6d693e049fd18 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 3 Mar 2023 18:21:22 +0100 Subject: [PATCH 2/5] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09f0f6e0c..ef27f5096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ All notable changes to this project will be documented in this file. - Prometheus image, NodeExporter image, Antora ([#95]). - Retired Java 1.8.0 support ([#248]). - Tools image ([#325]). -- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([xxx]) +- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([339]) [#95]: https://github.com/stackabletech/docker-images/pull/95 [#248]: https://github.com/stackabletech/docker-images/pull/248 @@ -31,3 +31,4 @@ All notable changes to this project will be documented in this file. [#321]: https://github.com/stackabletech/docker-images/pull/321 [#325]: https://github.com/stackabletech/docker-images/pull/325 [#326]: https://github.com/stackabletech/docker-images/pull/326 +[#339]: https://github.com/stackabletech/docker-images/pull/339 From 6c99b9b53e8010c1000f0e33c9cb9241faf3898d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 3 Mar 2023 18:31:31 +0100 Subject: [PATCH 3/5] Fix lint warnings. --- .github/workflows/product_images_dispatch.yml | 5 +++-- CHANGELOG.md | 2 +- image_tools/args.py | 2 +- image_tools/bake.py | 2 +- image_tools/lib.py | 2 -- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/product_images_dispatch.yml b/.github/workflows/product_images_dispatch.yml index 93466bd83..d4031f8f3 100644 --- a/.github/workflows/product_images_dispatch.yml +++ b/.github/workflows/product_images_dispatch.yml @@ -53,7 +53,8 @@ jobs: name: Build and push images on dispatch shell: bash run: python -m image_tools.bake -p ${{ github.event.inputs.product }} -i ${{ github.event.inputs.image-version }} -o ${{ github.event.inputs.organization }} -a ${{ github.event.inputs.architecture }} -u - - + + - name: Install preflight run: | curl -fLo preflight https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/latest/download/preflight-linux-amd64 @@ -62,4 +63,4 @@ jobs: - name: Run preflight shell: bash - run: python -m image_tools.preflight -p ${{ github.event.inputs.product }} -i ${{ github.event.inputs.image-version }} \ No newline at end of file + run: python -m image_tools.preflight -p ${{ github.event.inputs.product }} -i ${{ github.event.inputs.image-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index ef27f5096..f9deaa09c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ All notable changes to this project will be documented in this file. - Prometheus image, NodeExporter image, Antora ([#95]). - Retired Java 1.8.0 support ([#248]). - Tools image ([#325]). -- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([339]) +- Replace `build_product_images.py` with the `image_tools` package and add OpenShift preflight checks for images ([#339]) [#95]: https://github.com/stackabletech/docker-images/pull/95 [#248]: https://github.com/stackabletech/docker-images/pull/248 diff --git a/image_tools/args.py b/image_tools/args.py index 4520097ff..c49e4bbae 100644 --- a/image_tools/args.py +++ b/image_tools/args.py @@ -1,6 +1,6 @@ # This is the stackable release version from argparse import Namespace, ArgumentParser -from typing import List, Dict, Optional +from typing import List import re DEFAULT_IMAGE_VERSION_FORMATS = [re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+"), re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+-rc[1-9]\d?")] diff --git a/image_tools/bake.py b/image_tools/bake.py index d280a89c7..2a95f5380 100644 --- a/image_tools/bake.py +++ b/image_tools/bake.py @@ -11,7 +11,7 @@ from image_tools.args import parse from typing import List, Dict, Any from argparse import Namespace -from subprocess import run, CompletedProcess +from subprocess import run import json diff --git a/image_tools/lib.py b/image_tools/lib.py index 6ddb80caf..0aefd1908 100644 --- a/image_tools/lib.py +++ b/image_tools/lib.py @@ -1,8 +1,6 @@ """Library code for image tools.""" from dataclasses import dataclass, field from typing import List, Optional -from collections.abc import Sequence -from itertools import chain @dataclass(frozen=True) From e1ae8f444b3d9091fb9d2a86820b9cfa38168512 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 3 Mar 2023 20:37:13 +0100 Subject: [PATCH 4/5] Fix more lint warnings. --- image_tools/args.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/image_tools/args.py b/image_tools/args.py index c49e4bbae..bb5ea04c8 100644 --- a/image_tools/args.py +++ b/image_tools/args.py @@ -3,7 +3,10 @@ from typing import List import re -DEFAULT_IMAGE_VERSION_FORMATS = [re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+"), re.compile("[2-9][0-9]\.[1-9][0-2]?\.\d+-rc[1-9]\d?")] +DEFAULT_IMAGE_VERSION_FORMATS = [ + re.compile(r"[2-9][0-9]\.[1-9][0-2]?\.\d+"), + re.compile(r"[2-9][0-9]\.[1-9][0-2]?\.\d+-rc[1-9]\d?"), +] def parse() -> Namespace: From d27aaa8c4ac6498383f1a187d6a199e5f9de4662 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 3 Mar 2023 21:18:01 +0100 Subject: [PATCH 5/5] A bit of cleanup. --- image_tools/preflight.py | 52 +++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/image_tools/preflight.py b/image_tools/preflight.py index 9ac6fe516..13615063b 100644 --- a/image_tools/preflight.py +++ b/image_tools/preflight.py @@ -15,7 +15,7 @@ from image_tools.args import parse from typing import List, Dict, Any -from subprocess import CalledProcessError, run +import subprocess import json import sys import logging @@ -32,8 +32,25 @@ def get_images_for_target(product: str, bakefile: Dict[str, Any]) -> List[str]: return tags -def main(): - """Generate a Docker bake file from conf.py and build the given args.product images.""" +def get_preflight_failures(image_commands: Dict[str, Command]) -> Dict[str, List[Any]]: + """Run preflight commands for each image and return the failure field of the response.""" + failures = {} + for image, cmd in image_commands.items(): + try: + preflight_result = subprocess.run(cmd.args, input=cmd.input, check=True, capture_output=True) + preflight_json = json.loads(preflight_result.stdout) + failures[image] = preflight_json.get("results", {}).get("failed", []) + except subprocess.CalledProcessError as error: + failures[image] = [error.stderr.decode('utf-8')] + except FileNotFoundError: + failures[image] = ["preflight: command not found. Install from https://github.com/redhat-openshift-ecosystem/openshift-preflight"] + except json.JSONDecodeError as error: + failures[image] = [error.msg] + return failures + + +def main() -> int: + """Run OpenShift verification checks against the product images.""" logging.basicConfig(encoding="utf-8", level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') args = parse() @@ -44,25 +61,22 @@ def main(): logging.error("No images found for product [%s]", args.product) return 1 - preflight_cmds = {image: Command(args=["preflight", "check", "container", image]) for image in images} + # A mapping of image name to preflight command + image_commands = {image: Command(args=["preflight", "check", "container", image]) for image in images} - failures = {} - for image, cmd in preflight_cmds.items(): - if args.dry: - print(str(cmd)) - else: - try: - preflight_result = run(cmd.args, input=cmd.input, check=True, capture_output=True) - preflight_json = json.loads(preflight_result.stdout) - failures[image] = preflight_json.get("results", {}).get("failed", []) - except CalledProcessError as error: - failures[image] = [error.stderr.decode('utf-8')] - - for image, ifails in failures.items(): - if len(ifails) == 0: + if args.dry: + for _, cmd in image_commands.items(): + logging.info(str(cmd)) + return 0 + + # Run preflight and return failures + failures = get_preflight_failures(image_commands) + + for image, img_fails in failures.items(): + if len(img_fails) == 0: logging.info("Image [%s] preflight check successful.", image) else: - logging.error("Image [%s] preflight check failures:\n%s", image, "\n".join(ifails)) + logging.error("Image [%s] preflight check failures: %s", image, ",".join(img_fails)) fail_count = sum(map(lambda f: len(f), failures.values())) return fail_count