diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 3ea9f4323..46511c6cd 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -6,16 +6,19 @@ on: - main jobs: - development: - name: ${{ matrix.product }} 0.0.0-dev + build_and_push: + name: ${{ matrix.product }} 0.0.0-dev ${{ matrix.runner }} permissions: id-token: write - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} strategy: - # fail-fast: true + # There is value to see which builds are working and which are actually broken + # The images will be pushed, but the (effective) manifest-list is only updated when *all* builds succeed. + fail-fast: false # This setting can be changed to throttle the build load # max-parallel: 1 matrix: + runner: ["ubuntu-latest", "buildjet-2vcpu-ubuntu-2204-arm"] product: - airflow - druid @@ -31,6 +34,7 @@ jobs: - superset - testing-tools - trino + - trino-cli - tools - zookeeper shard_count: @@ -38,11 +42,18 @@ jobs: shard_index: [0, 1, 2, 3, 4] # between 0 and shard_count-1 steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1 - - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # tag=v3.0.0 - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # tag=v3.2.0 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # tag=v5.0.0 - with: - python-version: '3.x' + # As of 2024-04-08 this throws + # Error: The version '3.x' with architecture 'arm64' was not found for Ubuntu 22.04. + # This is because it currently only bundles arm64 packages for darwin. + # So instead we install python via apt until this is resolved + # - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # tag=v5.0.0 + # with: + # python-version: '3.x' + - name: Setup python + run: | + sudo apt update + sudo apt install --assume-yes python3 - name: Set up Cosign uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # tag=v3.3.0 - name: Set up syft @@ -65,11 +76,15 @@ jobs: run: | # Bake images and load them to local docker repo for signing # TODO: buildx cannot --load and --push at the same time + # Tagging images with the architecture they were build on + ARCH_FOR_DOCKER="$(arch | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#')" bake --product "${{ matrix.product }}" \ - --image-version 0.0.0-dev \ + --image-version "0.0.0-dev-${ARCH_FOR_DOCKER}" \ + --architecture "linux/${ARCH_FOR_DOCKER}" \ --shard-count "${{matrix.shard_count}}" \ --shard-index "${{matrix.shard_index}}" \ --export-tags-file bake-target-tags + # Push images to image repository if [ -f bake-target-tags ]; then echo "bake-target-tags: "$(< bake-target-tags) @@ -125,3 +140,55 @@ jobs: # Attest the SBOM to the image cosign attest -y --predicate sbom.merged.json --type cyclonedx "$IMAGE_NAME@$DIGEST" fi + create_manifests: + permissions: + id-token: write + runs-on: ubuntu-latest + needs: ["build_and_push"] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # tag=v3.1.0 + with: + registry: docker.stackable.tech + username: github + password: ${{ secrets.NEXUS_PASSWORD }} + - name: Login to Stackable Harbor + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # tag=v3 + with: + registry: oci.stackable.tech + username: robot$sdp+github-action-build + password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} + - name: Set up Cosign + uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # tag=v3.3.0 + - name: Build Manifest List + shell: bash + env: + DOCKER_USER: github + DOCKER_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + for product_and_version in $(python3 enumerate-product-versions.py); do + PRODUCT="$(echo "$product_and_version" | cut -d '#' -f 1)" + VERSION="$(echo "$product_and_version" | cut -d '#' -f 2)" + + echo "Generating manifest list for $PRODUCT in version $VERSION" + + MANIFEST_NAME="docker.stackable.tech/stackable/${PRODUCT}:${VERSION}-stackable0.0.0-dev" + # Create and push to Stackable Nexus + # `docker manifest push` directly returns the digest of the manifest list + # As it is an experimental feature, this might change in the future + # Further reading: https://docs.docker.com/reference/cli/docker/manifest/push/ + # --amend because the manifest list would be updated since we use the same tag: 0.0.0-dev + docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64" + DIGEST=$(docker manifest push $MANIFEST_NAME) + + # Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...) + # This generates a signature and publishes it to the registry, next to the image + # Uses the keyless signing flow with Github Actions as identity provider + cosign sign -y "$MANIFEST_NAME@$DIGEST" + + # Push to oci.stackable.tech as well + MANIFEST_NAME="oci.stackable.tech/sdp/${PRODUCT}:${VERSION}-stackable0.0.0-dev" + docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64" + DIGEST=$(docker manifest push $MANIFEST_NAME) + cosign sign -y "$MANIFEST_NAME@$DIGEST" + done diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6b20c88..225727f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,22 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Build all `0.0.0-dev` product images as multi-arch and push them to Nexus and Harbor. + Also SBOMs are generated and everything is signed ([#614]). + ### Fixed - Let Superset 3.1.0 build on ARM by adding `make` and `diffutils` ([#611]). - Let Airflow 2.8.x and 2.9.x build on ARM by adding `make` and `diffutils` ([#612]). - python:3.11 manifest list fixed. Added proper hash ([#613]). +- trino-cli: Include the trino-cli in the CI build process ([#614]). [#611]: https://github.com/stackabletech/docker-images/pull/611 [#612]: https://github.com/stackabletech/docker-images/pull/612 [#613]: https://github.com/stackabletech/docker-images/pull/613 +[#614]: https://github.com/stackabletech/docker-images/pull/614 ## [24.3.0] - 2024-03-20 diff --git a/enumerate-product-versions.py b/enumerate-product-versions.py new file mode 100644 index 000000000..e89e0f353 --- /dev/null +++ b/enumerate-product-versions.py @@ -0,0 +1,29 @@ +import conf + +PRODUCTS = [ + "airflow", + "druid", + "hadoop", + "hbase", + "hive", + "kafka", + "kafka-testing-tools", + "krb5", + "nifi", + "opa", + "spark-k8s", + "superset", + "testing-tools", + "trino", + "trino-cli", + "tools", + "zookeeper", +] + +for product in conf.products: + product_name = product['name'] + if product_name not in PRODUCTS: + continue + + for version in product['versions']: + print(f"{product_name}#{version['product']}")