diff --git a/.flake8 b/.flake8 deleted file mode 100644 index ea6646f31..000000000 --- a/.flake8 +++ /dev/null @@ -1,2 +0,0 @@ -[flake8] -ignore = E111,E501,E114 diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 0e0dab1a1..931cec2dd 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -41,6 +41,7 @@ jobs: uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # tag=v2 - name: Install python 3 uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # tag=v4 + - run: pip install image-tools-stackabletech with: python-version: '3.x' - name: Login to Stackable Nexus @@ -52,7 +53,7 @@ jobs: - name: Publish dev images id: publish_images run: | - BAKE_OUTPUT=$(python -m image_tools.bake --product "${{ matrix.product }}" --image-version 0.0.0-dev --push 2>&1) + BAKE_OUTPUT=$(bake --product "${{ matrix.product }}" --image-version 0.0.0-dev --push 2>&1) echo "$BAKE_OUTPUT" # needed for multiline strings in github actions EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) @@ -104,9 +105,10 @@ jobs: uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # tag=v4 with: python-version: '3.x' + - run: pip install image-tools-stackabletech - name: Install preflight run: | wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/latest/download/preflight-linux-amd64 chmod +x preflight-linux-amd64 - name: OpenShift certification checks - run: python -m image_tools.preflight --product "${{ matrix.product }}" --image-version 0.0.0-dev --preflight-cmd ./preflight-linux-amd64 + run: check-container --product "${{ matrix.product }}" --image-version 0.0.0-dev --executable ./preflight-linux-amd64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7d59931ba..02a8da0db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,6 +48,7 @@ jobs: uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # tag=v4 with: python-version: '3.x' + - run: pip install image-tools-stackabletech - name: Login to Stackable Nexus uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # tag=v2 with: @@ -57,7 +58,7 @@ jobs: - name: Publish release product images id: publish_images run: | - BAKE_OUTPUT=$(python -m image_tools.bake --product "${{ matrix.product }}" --image-version "$GITHUB_REF_NAME" --organization stackable --architecture linux/amd64 --push 2>&1) + BAKE_OUTPUT=$(bake --product "${{ matrix.product }}" --image-version "$GITHUB_REF_NAME" --organization stackable --architecture linux/amd64 --push 2>&1) echo "$BAKE_OUTPUT" # needed for multiline strings in github actions EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) @@ -114,9 +115,10 @@ jobs: uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # tag=v4 with: python-version: '3.x' + - run: pip install image-tools-stackabletech - name: Install preflight run: | wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/1.6.2/preflight-linux-amd64 chmod +x preflight-linux-amd64 - name: Submit images to OpenShift certification projects - run: python -m image_tools.preflight --product "${{ matrix.product }}" --image-version "$GITHUB_REF_NAME" --preflight-cmd ./preflight-linux-amd64 --token "${{ secrets.RH_PYXIS_API_TOKEN }}" --submit + run: check-container --product "${{ matrix.product }}" --image-version "$GITHUB_REF_NAME" --executable ./preflight-linux-amd64 --token "${{ secrets.RH_PYXIS_API_TOKEN }}" --submit diff --git a/.github/workflows/reviewdog.yaml b/.github/workflows/reviewdog.yaml index 71c4a8aec..31bdf0d47 100644 --- a/.github/workflows/reviewdog.yaml +++ b/.github/workflows/reviewdog.yaml @@ -26,17 +26,6 @@ jobs: with: github_token: ${{ secrets.github_token }} - flake8: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # tag=v3.3.0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # tag=v4 - with: - python-version: "3.9" - - uses: reviewdog/action-flake8@1212bd6f1c67830dcff438cf39522d4b58407e71 # tag=v3.7.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - hadolint: runs-on: ubuntu-latest steps: diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 411bd0864..000000000 --- a/.pylintrc +++ /dev/null @@ -1,33 +0,0 @@ -[MASTER] - -# Files or directories to be skipped. They should be base names, not paths. -ignore=third_party - -# Files or directories matching the regex patterns are skipped. The regex -# matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# These rules are for missing docstrings which doesn't matter much for most of our simple scripts -disable=C0114,C0115,C0116 - -[FORMAT] - -max-line-length=999 -indent-string=' ' diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f08424b..8a31b00dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Changed +- Extract image tools their own [repository](https://github.com/stackabletech/image-tools) ([#437]) - Bump ubi8-rust-builder toolchain to 1.71.0 ([#419]). - BREAKING: Upgrade Vector in all product images to version 0.31.0. The integration tests of the operators must be adapted because the metric @@ -23,6 +24,7 @@ All notable changes to this project will be documented in this file. [#419]: https://github.com/stackabletech/docker-images/pull/419 [#429]: https://github.com/stackabletech/docker-images/pull/429 [#433]: https://github.com/stackabletech/docker-images/pull/433 +[#437]: https://github.com/stackabletech/docker-images/pull/437 ## [23.7.0] - 2023-07-14 diff --git a/README.adoc b/README.adoc index 17a377ab2..aaec4bc32 100644 --- a/README.adoc +++ b/README.adoc @@ -36,19 +36,27 @@ ENTRYPOINT ["/stackable-zookeeper-operator"] == Build Product Images Product images are published to the `docker.stackable.tech` registry under the `stackable` organization. -This uses the Docker CLI plugin `buildx` which is not necessarily installed by default when installing Docker. -Make sure that `docker buildx` works before running the following commands. + +Prerequisites: + +* Stackable Image Tools (`pip install image-tools-stackabletech`) https://github.com/stackabletech/image-tools +* Docker including the `buildx` plugin: https://github.com/docker/buildx To build and push product images to the default repository, use the `build_product_images.py` like this: - python -m image_tools.bake --product zookeeper --image 0.0.0-dev --push + bake --product zookeeper --image 0.0.0-dev --push -This will build images for Apache ZooKeeper versions as defined in the `image_tools/conf.py` and tag them with the `image-version` 0.0.0-dev +This will build images for Apache ZooKeeper versions as defined in the `conf.py` file and tag them with the `image-version` 0.0.0-dev The GitHub action called `Product images` can be triggered manually to do the same but not on the local machine. == Verify Product Images +Prerequisites: + +* Stackable Image Tools (`pip install image-tools-stackabletech`) https://github.com/stackabletech/image-tools +* OpenShift preflight tool: https://github.com/redhat-openshift-ecosystem/openshift-preflight + To verify if Apache Zookeeper validate against OpenShift preflight, run: - python -m image_tools.preflight --product zookeeper --image 0.0.0-dev + check-container --product zookeeper --image 0.0.0-dev diff --git a/image_tools/conf.py b/conf.py similarity index 99% rename from image_tools/conf.py rename to conf.py index c602a5ab9..8aa3ae2b4 100644 --- a/image_tools/conf.py +++ b/conf.py @@ -1,4 +1,6 @@ """ +Configuration file for the Stackable image-tools: https://github.com/stackabletech/image-tools. + Application images will be created for products and associated versions configured here. """ diff --git a/image_tools/__init__.py b/image_tools/__init__.py deleted file mode 100644 index 80d60700e..000000000 --- a/image_tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tools for image management.""" diff --git a/image_tools/args.py b/image_tools/args.py deleted file mode 100644 index 8eff3020b..000000000 --- a/image_tools/args.py +++ /dev/null @@ -1,81 +0,0 @@ -# This is the stackable release version -from argparse import Namespace, ArgumentParser -from typing import List -import re - -DEFAULT_IMAGE_VERSION_FORMATS = [ - re.compile(r"[2-9][0-9]\.[1-9][0-2]?\.\d+"), - re.compile(r"[2-9][0-9]\.[1-9][0-2]?\.\d+-rc[1-9]\d?"), - re.compile(r"0\.0\.0-dev"), -] - - -def parse() -> Namespace: - parser = ArgumentParser( - description="Build and publish product images. Requires docker and buildx (https://github.com/docker/buildx)." - ) - parser.add_argument( - "-i", - "--image-version", - help="Image version", - required=True, - type=check_image_version_format, - ) - parser.add_argument("-p", "--product", help="Product to build images for") - parser.add_argument("-u", "--push", help="Push images", action="store_true") - parser.add_argument("-d", "--dry", help="Dry run.", action="store_true") - parser.add_argument( - "-a", - "--architecture", - help="Target platform for image. Default: linux/amd64.", - nargs="+", - default=["linux/amd64"], - type=check_architecture_input, - ) - parser.add_argument( - "-o", - "--organization", - help="Organization name within the given registry. Default: stackable", - default="stackable", - ) - parser.add_argument( - "-r", - "--registry", - help="Image registry to publish to. Default: docker.stackable.tech", - default="docker.stackable.tech", - ) - return parser.parse_args() - - -def check_image_version_format(image_version) -> str: - """ - Check image version against allowed formats. - - >>> check_image_version_format("23.4.0") - '23.4.0' - >>> check_image_version_format("23.4.0-rc1") - '23.4.0-rc1' - >>> check_image_version_format("23.04.0") - Traceback (most recent call last): - ... - ValueError: Invalid image version: 23.04.0 - >>> check_image_version_format("23.4.0.prerelease") - Traceback (most recent call last): - ... - ValueError: Invalid image version: 23.4.0.prerelease - """ - for p in DEFAULT_IMAGE_VERSION_FORMATS: - if p.fullmatch(image_version): - return image_version - raise ValueError(f"Invalid image version: {image_version}") - - -def check_architecture_input(architecture) -> List[str]: - supported_arch = ["linux/amd64", "linux/arm64"] - - if architecture not in supported_arch: - raise ValueError( - f"Architecture {architecture} not supported. Supported: {supported_arch}" - ) - - return architecture diff --git a/image_tools/bake.py b/image_tools/bake.py deleted file mode 100644 index b067fdedc..000000000 --- a/image_tools/bake.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Image builder - -Requirements: docker and buildx. - -Usage: - - python -m image_tools.bake -p opa -i 22.12.0 -""" -from typing import List, Dict, Any -from argparse import Namespace -from subprocess import run -import json -import re - -from image_tools.lib import Command -from image_tools import conf -from image_tools.args import parse - - -def build_image_args(version, release_version): - """ - Returns a list of --build-arg command line arguments that are used by the - docker build command. - - Arguments: - - version: Can be a str, in which case it's considered the PRODUCT - or a dict. - """ - result = {} - - if isinstance(version, dict): - for k, v in version.items(): - result[k.upper()] = v - result["RELEASE"] = release_version - elif isinstance(version, str) and isinstance(release_version, str): - { - "PRODUCT": version, - "RELEASE": release_version, - } - else: - raise ValueError(f"Unsupported version object: {version}") - - return result - - -def build_image_tags( - image_name: str, image_version: str, product_version: str -) -> List[str]: - """ - Returns a list of --tag command line arguments that are used by the - docker build command. - Each image is tagged with two tags as follows: - 1. - - 2. - - """ - arr = re.split("\\.", image_version) - platform_version = arr[0] + "." + arr[1] - return [ - f"{image_name}:{product_version}-stackable{image_version}", - f"{image_name}:{product_version}-stackable{platform_version}", - ] - - -def generate_bakefile(args: Namespace) -> Dict[str, Any]: - """ - Generates a Bakefile (https://docs.docker.com/build/bake/file-definition/) describing how to build the whole image graph. - - build_and_publish_images() ensures that only the desired images are actually built. - """ - targets = {} - groups = {} - product_names = [product["name"] for product in conf.products] - for product in conf.products: - product_name: str = product["name"] - product_targets = {} - for version_dict in product.get("versions", []): - product_targets.update( - bakefile_product_version_targets( - args, product_name, version_dict, product_names - ) - ) - groups[product_name] = { - "targets": list(product_targets.keys()), - } - targets.update(product_targets) - groups["default"] = { - "targets": list(groups.keys()), - } - return { - "target": targets, - "group": groups, - } - - -def bakefile_target_name_for_product_version(product_name: str, version: str) -> str: - """ - Creates a normalized Bakefile target name for a given (product, version) combination. - """ - return f"{ product_name }-{ version.replace('.', '_') }" - - -def bakefile_product_version_targets( - args: Namespace, - product_name: str, - versions: Dict[str, str], - product_names: List[str], -): - """ - Creates Bakefile targets defining how to build a given product version. - - A product is assumed to depend on another if it defines a `versions` field with the same name as the other product. - """ - image_name = f"{args.registry}/{args.organization}/{product_name}" - tags = build_image_tags(image_name, args.image_version, versions["product"]) - build_args = build_image_args(versions, args.image_version) - - return { - bakefile_target_name_for_product_version(product_name, versions["product"]): { - "dockerfile": f"{ product_name }/Dockerfile", - "tags": tags, - "args": build_args, - "platforms": args.architecture, - "context": ".", - "contexts": { - f"stackable/image/{dep_name}": f"target:{bakefile_target_name_for_product_version(dep_name, dep_version)}" - for dep_name, dep_version in versions.items() - if dep_name in product_names - }, - }, - } - - -def bake_command(args: Namespace, product_name: str, bakefile) -> Command: - """ - Returns a list of commands that need to be run in order to build and - publish product images. - - For local building, builder instances are supported. - """ - - if args.push: - target_mode = ["--push"] - elif len(args.architecture) == 1: - target_mode = ["--load"] - else: - target_mode = [] - - return Command( - args=[ - "docker", - "buildx", - "bake", - "--file", - "-", - *([] if product_name is None else [product_name]), - *target_mode, - ], - stdin=json.dumps(bakefile), - ) - - -def main(): - """Generate a Docker bake file from conf.py and build the given args.product images.""" - args = parse() - bakefile = generate_bakefile(args) - - cmd = bake_command(args, args.product, bakefile) - - if args.dry: - print(cmd) - else: - run(cmd.args, input=cmd.input, check=True) - - -if __name__ == "__main__": - main() diff --git a/image_tools/lib.py b/image_tools/lib.py deleted file mode 100644 index 842d61393..000000000 --- a/image_tools/lib.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Library code for image tools.""" -from dataclasses import dataclass, field -from typing import List, Optional - - -@dataclass(frozen=True) -class Command: - """Command line program including arguments and optional standard input.""" - - args: List[str] = field(default_factory=list) - stdin: Optional[str] = field(default=None) - - @property - def input(self) -> Optional[bytes]: - """stdin input as UTF8 bytes.""" - if self.stdin: - return self.stdin.encode("utf-8") - else: - return None - - def __str__(self) -> str: - if self.stdin: - return f"{' '.join(self.args)} << List[str]: - tags = [] - - for target in bakefile.get("group", {}).get(product, {}).get("targets", []): - tags.extend(get_images_for_target(target, bakefile)) - - tags.extend(bakefile["target"].get(product, {}).get("tags", [])) - - return tags - - -def get_preflight_failures(image_commands: Dict[str, Command]) -> Dict[str, List[Any]]: - """Run preflight commands for each image and return the failure field of the response.""" - failures = {} - for image, cmd in image_commands.items(): - try: - preflight_result = subprocess.run( - cmd.args, input=cmd.input, check=True, stdout=subprocess.PIPE - ) - preflight_json = json.loads(preflight_result.stdout) - failures[image] = preflight_json.get("results", {}).get("failed", []) - except subprocess.CalledProcessError as error: - failures[image] = [error.stderr.decode("utf-8")] - except FileNotFoundError: - failures[image] = [ - "preflight: command not found. Install from https://github.com/redhat-openshift-ecosystem/openshift-preflight" - ] - except json.JSONDecodeError as error: - failures[image] = [error.msg] - return failures - - -def parse_args() -> Namespace: - parser = ArgumentParser( - description="Run OpenShift certification checks and submit results to RedHat Partner Connect portal" - ) - parser.add_argument( - "-i", - "--image-version", - help="Image version", - required=True, - type=check_image_version_format, - ) - parser.add_argument( - "-p", "--product", help="Product to build images for", required=True - ) - parser.add_argument("-s", "--submit", help="Submit results", action="store_true") - parser.add_argument("-d", "--dry", help="Dry run.", action="store_true") - parser.add_argument( - "-a", - "--architecture", - help="Target platform for image. Default: linux/amd64.", - nargs="+", - default=["linux/amd64"], - type=check_architecture_input, - ) - parser.add_argument( - "-t", - "--token", - help="RedHat portal API token", - ) - parser.add_argument( - "-o", - "--organization", - help="Organization name within the given registry. Default: stackable", - default="stackable", - ) - parser.add_argument( - "-r", - "--registry", - help="Image registry to publish to. Default: docker.stackable.tech", - default="docker.stackable.tech", - ) - parser.add_argument( - "-c", - "--preflight-cmd", - help="Name of the preflight program. Default: preflight", - default="preflight", - ) - - result = parser.parse_args() - - if result.submit and not result.token: - raise ValueError("Missing API token for submitting results.") - - return result - - -def preflight_commands(images: List[str], args: Namespace) -> Dict[str, Command]: - """A mapping of image name to preflight command""" - result = {} - for img in images: - cmd_args = [args.preflight_cmd, "check", "container", img] - if args.submit: - cmd_args.extend( - [ "--loglevel", - "trace", - "--submit", - "--pyxis-api-token", - args.token, - "--certification-project-id", - f"ospid-{open_shift_projects[args.product]['id']}", - ] - ) - result[img] = Command(args=cmd_args) - return result - - -def main() -> int: - """Run OpenShift verification checks against the product images.""" - logging.basicConfig( - encoding="utf-8", - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - ) - - args = parse_args() - bakefile = generate_bakefile(args) - - # List of images (with tags) to apply preflight checks to. - # Filter out images with platform release tags such as "23.4" and - # only check images with patch versions such as "23.4.0". - images = list( - filter( - lambda i: i.endswith(args.image_version), - get_images_for_target(args.product, bakefile), - ) - ) - if not images: - logging.error("No images found for product [%s]", args.product) - return 1 - - # A mapping of image name to preflight command - image_commands = preflight_commands(images, args) - - if args.dry: - for _, cmd in image_commands.items(): - logging.info(str(cmd)) - return 0 - - # Run preflight and return failures - failures = get_preflight_failures(image_commands) - - for image, img_fails in failures.items(): - if len(img_fails) == 0: - logging.info("Image [%s] preflight check successful.", image) - else: - logging.error( - "Image [%s] preflight check failures: %s", image, ",".join(img_fails) - ) - - fail_count = sum(map(len, failures.values())) - return fail_count - - -if __name__ == "__main__": - sys.exit(main())