Assemble multi-arch image

.github/workflows/release.yaml

@@ -3,6 +3,10 @@ on:
   push:
     tags:
       - v[0-9]+.[0-9]+.[0-9]+*
+env:
+  IMAGE_BASE_NAME: ghcr.io/kamu-data/engine-datafusion
+  IMAGE_LABELS: |
+    org.opencontainers.image.vendor=Kamu Data Inc.
 jobs:
   build:
     strategy:
@@ -11,15 +15,32 @@ jobs:
         include:
           - runs-on: ubuntu-latest
             target: x86_64-unknown-linux-musl
-            image-platform: linux/amd64
+            platform: linux/amd64
+            qemu: false
           - runs-on: ubuntu-latest
             target: aarch64-unknown-linux-musl
-            image-platform: linux/arm/v8
+            platform: linux/arm64
+            qemu: true
     name: Build
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v4
 
+      # Collects metadata about build for image tags and labels
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_BASE_NAME }}
+          labels: ${{ env.IMAGE_LABELS }}
+
+      - name: Set up QEMU
+        if: matrix.qemu
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - uses: actions-rs/toolchain@v1
         with:
           target: ${{ matrix.target }}
@@ -29,12 +50,68 @@ jobs:
         run: cargo install cross --locked
 
       - name: Build binary
-        run: |
-          make build TARGET_ARCH=${{ matrix.target }}
+        run: make build TARGET_ARCH=${{ matrix.target }}
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Images with multiple architectures need to be pushed as one manifest. Pushing tagged images separately
+      # simply overrides the tag instead of merging the architectures. Since we are doing a matrix
+      # build where architectures are built on different runners in parallel we have to push
+      # individual images by digest and then assemble them together into one manifest that will be tagged.
+      #
+      # Source: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: image/Dockerfile
+          platforms: ${{ matrix.platform }}
+          build-args: target_arch=${{ matrix.target }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.IMAGE_BASE_NAME }},push-by-digest=true,name-canonical=true,push=true
 
-      - name: Build image
+      - name: Export digest
         run: |
-          make image IMAGE_PLATFORM=${{ matrix.image-platform }} TARGET_ARCH=${{ matrix.target }}
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.target }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Create multi-platform image
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_BASE_NAME }}
+          labels: ${{ env.IMAGE_LABELS }}
 
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v3
@@ -43,22 +120,30 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Publish image
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE_BASE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect final image
         run: |
-          make image-push IMAGE_PLATFORM=${{ matrix.image-platform }} TARGET_ARCH=${{ matrix.target }}
+          docker buildx imagetools inspect ${{ env.IMAGE_BASE_NAME }}:${{ steps.meta.outputs.version }}
 
   create_release:
     name: Create Release
     runs-on: ubuntu-latest
-    needs: [build]
+    needs: [merge]
     steps:
       - uses: actions/checkout@v4
+
       # Sets the header for the release description
       - run: echo "" > RELEASE_HEAD.md
+
       # Extracts relevant section from CHANGELOG.md and outputs it into RELEASE.md
       - uses: CSchoel/release-notes-from-changelog@v1
+
       - name: Create Release
-        id: create_release
         uses: actions/create-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -68,5 +153,3 @@ jobs:
           draft: false
           prerelease: false
           body_path: RELEASE.md
-    outputs:
-      upload_url: ${{ steps.create_release.outputs.upload_url }}

Makefile

@@ -1,5 +1,5 @@
 TARGET_ARCH = x86_64-unknown-linux-musl
-IMAGE_PLATFORM = linux/amd64
+PLATFORM = linux/amd64
 CRATE_VERSION = $(shell cargo metadata --format-version 1 | jq -r '.packages[] | select( .name == "kamu-engine-datafusion") | .version')
 IMAGE_TAG = $(CRATE_VERSION)
 IMAGE = ghcr.io/kamu-data/engine-datafusion:$(IMAGE_TAG)
@@ -35,12 +35,12 @@ build:
 
 .PHONY: image
 image:
-	docker build \
-	    --platform $(IMAGE_PLATFORM) \
+	docker buildx build \
+	    --platform $(PLATFORM) \
 		--build-arg target_arch=$(TARGET_ARCH) \
-		--build-arg version=$(CRATE_VERSION) \
 		-t $(IMAGE) \
 		-f image/Dockerfile \
+		--load \
 		.
 
 

image/Dockerfile

@@ -1,23 +1,13 @@
 # TODO: Consider using https://github.com/GoogleContainerTools/distroless
 FROM docker.io/library/alpine:3.19
 ARG target_arch
-ARG version
-LABEL name="Apache Arrow Datafusion ODF Engine" \
-      maintainer="info@kamu.dev" \
-      vendor="Kamu Data Inc." \
-      summary="Open Data Fabric Engine implementation using the Apache Arrow DataFusion framework" \
-      description="" \
-      version=${version}
-
 
 # Tini
 RUN apk add --no-cache tini
 
-
 # Engine
 COPY target/${target_arch}/release/kamu-engine-datafusion /opt/engine/bin/kamu-engine-datafusion
 
-
 ENV RUST_BACKTRACE=1
 ENV RUST_LOG=debug
 EXPOSE 2884/tcp