Build 0.0.0-dev product images as multi-arch

.github/workflows/dev.yaml

@@ -6,16 +6,19 @@ on:
       - main
 
 jobs:
-  development:
-    name: ${{ matrix.product }} 0.0.0-dev
+  build_and_push:
+    name: ${{ matrix.product }} 0.0.0-dev ${{ matrix.runner }}
     permissions:
       id-token: write
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     strategy:
-      # fail-fast: true
+      # There is value to see which builds are working and which are actually broken
+      # The images will be pushed, but the (effective) manifest-list is only updated when *all* builds succeed.
+      fail-fast: false
       # This setting can be changed to throttle the build load
       # max-parallel: 1
       matrix:
+        runner: ["ubuntu-latest", "buildjet-2vcpu-ubuntu-2204-arm"]
         product:
           - airflow
           - druid
@@ -31,18 +34,26 @@ jobs:
           - superset
           - testing-tools
           - trino
+          - trino-cli
           - tools
           - zookeeper
         shard_count:
           - 5
         shard_index: [0, 1, 2, 3, 4] # between 0 and shard_count-1
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
-      - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # tag=v3.0.0
       - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # tag=v3.2.0
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # tag=v5.0.0
-        with:
-          python-version: '3.x'
+      # As of 2024-04-08 this throws
+      # Error: The version '3.x' with architecture 'arm64' was not found for Ubuntu 22.04.
+      # This is because it currently only bundles arm64 packages for darwin.
+      # So instead we install python via apt until this is resolved
+      # - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # tag=v5.0.0
+      #   with:
+      #     python-version: '3.x'
+      - name: Setup python
+        run: |
+          sudo apt update
+          sudo apt install --assume-yes python3
       - name: Set up Cosign
         uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # tag=v3.3.0
       - name: Set up syft
@@ -65,11 +76,15 @@ jobs:
         run: |
           # Bake images and load them to local docker repo for signing
           # TODO: buildx cannot --load and --push at the same time
+          # Tagging images with the architecture they were build on
+          ARCH_FOR_DOCKER="$(arch | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#')"
           bake --product "${{ matrix.product }}" \
-          --image-version 0.0.0-dev \
+          --image-version "0.0.0-dev-${ARCH_FOR_DOCKER}" \
+          --architecture "linux/${ARCH_FOR_DOCKER}" \
           --shard-count "${{matrix.shard_count}}" \
           --shard-index "${{matrix.shard_index}}" \
           --export-tags-file bake-target-tags
+
           # Push images to image repository
           if [ -f bake-target-tags ]; then
             echo "bake-target-tags: "$(< bake-target-tags)
@@ -125,3 +140,55 @@ jobs:
             # Attest the SBOM to the image
             cosign attest -y --predicate sbom.merged.json --type cyclonedx "$IMAGE_NAME@$DIGEST"
           fi
+  create_manifests:
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    needs: ["build_and_push"]
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
+      - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # tag=v3.1.0
+        with:
+          registry: docker.stackable.tech
+          username: github
+          password: ${{ secrets.NEXUS_PASSWORD }}
+      - name: Login to Stackable Harbor
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # tag=v3
+        with:
+          registry: oci.stackable.tech
+          username: robot$sdp+github-action-build
+          password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
+      - name: Set up Cosign
+        uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # tag=v3.3.0
+      - name: Build Manifest List
+        shell: bash
+        env:
+          DOCKER_USER: github
+          DOCKER_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          for product_and_version in $(python3 enumerate-product-versions.py); do
+            PRODUCT="$(echo "$product_and_version" | cut -d '#' -f 1)"
+            VERSION="$(echo "$product_and_version" | cut -d '#' -f 2)"
+
+            echo "Generating manifest list for $PRODUCT in version $VERSION"
+
+            MANIFEST_NAME="docker.stackable.tech/stackable/${PRODUCT}:${VERSION}-stackable0.0.0-dev"
+            # Create and push to Stackable Nexus
+            # `docker manifest push` directly returns the digest of the manifest list
+            # As it is an experimental feature, this might change in the future
+            # Further reading: https://docs.docker.com/reference/cli/docker/manifest/push/
+            # --amend because the manifest list would be updated since we use the same tag: 0.0.0-dev
+            docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64"
+            DIGEST=$(docker manifest push $MANIFEST_NAME)
+
+            # Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)
+            # This generates a signature and publishes it to the registry, next to the image
+            # Uses the keyless signing flow with Github Actions as identity provider
+            cosign sign -y "$MANIFEST_NAME@$DIGEST"
+
+            # Push to oci.stackable.tech as well
+            MANIFEST_NAME="oci.stackable.tech/sdp/${PRODUCT}:${VERSION}-stackable0.0.0-dev"
+            docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64"
+            DIGEST=$(docker manifest push $MANIFEST_NAME)
+            cosign sign -y "$MANIFEST_NAME@$DIGEST"
+          done

CHANGELOG.md

@@ -4,15 +4,22 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Build all `0.0.0-dev` product images as multi-arch and push them to Nexus and Harbor.
+  Also SBOMs are generated and everything is signed ([#614]).
+
 ### Fixed
 
 - Let Superset 3.1.0 build on ARM by adding `make` and `diffutils` ([#611]).
 - Let Airflow 2.8.x and 2.9.x build on ARM by adding `make` and `diffutils` ([#612]).
 - python:3.11 manifest list fixed. Added proper hash ([#613]).
+- trino-cli: Include the trino-cli in the CI build process ([#614]).
 
 [#611]: https://github.com/stackabletech/docker-images/pull/611
 [#612]: https://github.com/stackabletech/docker-images/pull/612
 [#613]: https://github.com/stackabletech/docker-images/pull/613
+[#614]: https://github.com/stackabletech/docker-images/pull/614
 
 ## [24.3.0] - 2024-03-20
 

enumerate-product-versions.py

@@ -0,0 +1,29 @@
+import conf
+
+PRODUCTS = [
+    "airflow",
+    "druid",
+    "hadoop",
+    "hbase",
+    "hive",
+    "kafka",
+    "kafka-testing-tools",
+    "krb5",
+    "nifi",
+    "opa",
+    "spark-k8s",
+    "superset",
+    "testing-tools",
+    "trino",
+    "trino-cli",
+    "tools",
+    "zookeeper",
+]
+
+for product in conf.products:
+    product_name = product['name']
+    if product_name not in PRODUCTS:
+        continue
+
+    for version in product['versions']:
+        print(f"{product_name}#{version['product']}")