feat(arm): Optimize ETL to run on ARM

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.github/workflows/docker-build.yml

@@ -36,9 +36,30 @@ permissions:
   contents: read
 
 jobs:
+  metadata:
+    name: Compute Metadata
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+      labels: ${{ steps.meta.outputs.labels }}
+    steps:
+      - name: Docker Metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ inputs.image }}
+          tags: |
+            type=raw,value=latest
+            type=sha,format=long,prefix=
+            ${{ inputs.tag_with_version && format('type=raw,value=v{0}', inputs.version) || '' }}
+
   docker:
     name: Build Docker Image
+    needs: metadata
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
     steps:
       - name: Checkout (specific ref)
         if: inputs.checkout_ref != ''
@@ -53,34 +74,109 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
       - name: Log in to Docker Hub
         if: inputs.push == true
         uses: docker/login-action@v3
         with:
           username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Docker Metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ inputs.image }}
-          tags: |
-            type=raw,value=latest
-            type=sha,format=long,prefix=
-            ${{ inputs.tag_with_version && format('type=raw,value=v{0}', inputs.version) || '' }}
+      - name: Prepare Architecture Tags
+        id: arch_tags
+        shell: bash
+        run: |
+          set -euo pipefail
+          tags="${{ needs.metadata.outputs.tags }}"
+          arch="${{ matrix.arch }}"
+          if [ -z "$tags" ]; then
+            echo "tags=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          tmp=$(mktemp)
+          while IFS= read -r tag; do
+            if [ -n "$tag" ]; then
+              echo "${tag}-${arch}" >> "$tmp"
+            fi
+          done <<< "$tags"
+          printf 'tags<<EOF\n%s\nEOF\n' "$(cat "$tmp")" >> "$GITHUB_OUTPUT"
 
-      - name: Build and Push Image
+      - name: Build and Push Image (${{ matrix.arch }})
         id: build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ${{ inputs.context }}
           file: ${{ inputs.file }}
           push: ${{ inputs.push }}
-          tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          tags: ${{ steps.arch_tags.outputs.tags }}
+          labels: ${{ needs.metadata.outputs.labels }}
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha,scope=${{ inputs.image }}-${{ matrix.arch }}
+          cache-to: type=gha,scope=${{ inputs.image }}-${{ matrix.arch }},mode=max
+          outputs: type=image,name=${{ inputs.image }},push=${{ inputs.push }},push-by-digest=${{ inputs.push }},name-canonical=true
+
+      - name: Export Digest (${{ matrix.arch }})
+        if: inputs.push == true
+        run: |
+          set -euo pipefail
+          mkdir -p "${{ runner.temp }}/digests"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload Digest Artifact (${{ matrix.arch }})
+        if: inputs.push == true
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-linux-${{ matrix.arch }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  manifest:
+    name: Publish Multi-Arch Manifest
+    needs: [metadata, docker]
+    if: inputs.push == true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Download Digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Create Multi-Architecture Manifests
+        shell: bash
+        run: |
+          set -euo pipefail
+          tags="${{ needs.metadata.outputs.tags }}"
+          image="${{ inputs.image }}"
+          refs=""
+          for digest_file in "${{ runner.temp }}/digests"/*; do
+            digest="$(basename "$digest_file")"
+            refs="$refs ${image}@sha256:${digest}"
+          done
+          while IFS= read -r tag; do
+            if [ -z "$tag" ]; then
+              continue
+            fi
+            docker buildx imagetools create \
+              --tag "$tag" \
+              $refs
+          done <<< "$tags"
 
       - name: Output Image Information
         run: |

Cargo.toml

@@ -84,3 +84,8 @@ x509-cert = { version = "0.2.2", default-features = false }
 
 [profile.bench]
 debug = true
+
+[profile.release]
+opt-level = 3
+lto = "thin"
+codegen-units = 1

etl-api/Dockerfile

@@ -1,26 +1,100 @@
+# syntax=docker/dockerfile:1.6
+
 # Build stage with cargo-chef for better layer caching
 FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef
 WORKDIR /app
 
 # Install system dependencies
 FROM chef AS planner
 COPY . .
+RUN test -f Cargo.lock || cargo generate-lockfile
 RUN cargo chef prepare --recipe-path recipe.json
 
 # Build dependencies
 FROM chef AS builder
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM"
-RUN apt-get update && apt-get install -y pkg-config libssl-dev && rm -rf /var/lib/apt/lists/*
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then dpkg --add-architecture arm64; fi && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      pkg-config \
+      libssl-dev \
+      zlib1g-dev \
+      clang \
+      lld \
+      gcc-aarch64-linux-gnu \
+      g++-aarch64-linux-gnu \
+      libc6-dev-arm64-cross \
+      binutils-aarch64-linux-gnu \
+      $(if [ "$TARGETPLATFORM" = "linux/arm64" ]; then printf '%s ' "libssl-dev:arm64" "zlib1g-dev:arm64"; fi) && \
+    rm -rf /var/lib/apt/lists/*
+RUN case "$TARGETPLATFORM" in \
+      "linux/amd64") TARGET_TRIPLE="x86_64-unknown-linux-gnu"; LIB_ARCH="x86_64-linux-gnu" ;; \
+      "linux/arm64") TARGET_TRIPLE="aarch64-unknown-linux-gnu"; LIB_ARCH="aarch64-linux-gnu" ;; \
+      *) echo "Unsupported TARGETPLATFORM: $TARGETPLATFORM" >&2; exit 1 ;; \
+    esac && \
+    rustup target add "$TARGET_TRIPLE" && \
+    echo "$TARGET_TRIPLE" > /tmp/target-triple && \
+    echo "$LIB_ARCH" > /tmp/lib-arch
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/clang \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/aarch64-linux-gnu-gcc \
+    CC_x86_64_unknown_linux_gnu=/usr/bin/clang \
+    CXX_x86_64_unknown_linux_gnu=/usr/bin/clang++ \
+    AR_x86_64_unknown_linux_gnu=/usr/bin/ar \
+    CC_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-gcc \
+    CXX_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-g++ \
+    AR_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-ar
+RUN cat <<'EOF' >/usr/local/bin/with-cross-env
+#!/usr/bin/env bash
+set -euo pipefail
+target="$(cat /tmp/target-triple 2>/dev/null || true)"
+if [ "$target" = "aarch64-unknown-linux-gnu" ]; then
+  export PKG_CONFIG_ALLOW_CROSS=1
+  export PKG_CONFIG_LIBDIR=/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/share/pkgconfig
+  export PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu
+  export OPENSSL_DIR=/usr/aarch64-linux-gnu
+  export OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu
+  export OPENSSL_INCLUDE_DIR=/usr/include/aarch64-linux-gnu
+  export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig
+fi
+exec "$@"
+EOF
+RUN chmod +x /usr/local/bin/with-cross-env
 COPY --from=planner /app/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
+COPY --from=planner /app/Cargo.lock Cargo.lock
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    TARGET_TRIPLE=$(cat /tmp/target-triple) && \
+    with-cross-env cargo chef cook --release --target "$TARGET_TRIPLE" --recipe-path recipe.json
 
 # Build application
 COPY . .
 ENV SQLX_OFFLINE=true
-RUN cargo build --release -p etl-api && \
-    strip target/release/etl-api
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    TARGET_TRIPLE=$(cat /tmp/target-triple) && \
+    mkdir -p target/release && \
+    with-cross-env cargo build --release --target "$TARGET_TRIPLE" --locked -p etl-api && \
+    if [ "$TARGET_TRIPLE" = "aarch64-unknown-linux-gnu" ]; then \
+      STRIP_TOOL="aarch64-linux-gnu-strip"; \
+    else \
+      STRIP_TOOL="strip"; \
+    fi && \
+    "$STRIP_TOOL" "target/${TARGET_TRIPLE}/release/etl-api" && \
+    cp "target/${TARGET_TRIPLE}/release/etl-api" target/release/etl-api
+
+FROM builder AS runtime-deps
+RUN set -euo pipefail; \
+    LIB_ARCH="$(cat /tmp/lib-arch)"; \
+    mkdir -p "/runtime-libs/usr/lib/${LIB_ARCH}" "/runtime-libs/lib/${LIB_ARCH}"; \
+    cp "/usr/lib/${LIB_ARCH}/libssl.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \
+    cp "/usr/lib/${LIB_ARCH}/libcrypto.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \
+    if [ -f "/usr/lib/${LIB_ARCH}/libz.so.1" ]; then \
+      cp "/usr/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \
+    else \
+      cp "/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/lib/${LIB_ARCH}/"; \
+    fi
 
 # Runtime stage with distroless for security
 FROM gcr.io/distroless/cc-debian12:nonroot
@@ -31,6 +105,7 @@ USER nonroot:nonroot
 
 # Copy binary and configuration
 COPY --from=builder /app/target/release/etl-api ./etl-api
+COPY --from=runtime-deps /runtime-libs/ /
 COPY --chown=nonroot:nonroot etl-api/configuration ./configuration
 
 # Use exec form for proper signal handling

etl-replicator/Dockerfile

@@ -1,10 +1,13 @@
+# syntax=docker/dockerfile:1.6
+
 # Build stage with cargo-chef for better layer caching
 FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef
 WORKDIR /app
 
 # Install system dependencies
 FROM chef AS planner
 COPY . .
+RUN test -f Cargo.lock || cargo generate-lockfile
 RUN cargo chef prepare --recipe-path recipe.json
 
 # Build dependencies
@@ -13,14 +16,86 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM"
 # TODO: remove protobuf-compiler once the upstream gcp-bigquery-client remove it from its deps
-RUN apt-get update && apt-get install -y pkg-config libssl-dev protobuf-compiler clang && rm -rf /var/lib/apt/lists/*
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then dpkg --add-architecture arm64; fi && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      pkg-config \
+      libssl-dev \
+      zlib1g-dev \
+      protobuf-compiler \
+      clang \
+      lld \
+      gcc-aarch64-linux-gnu \
+      g++-aarch64-linux-gnu \
+      libc6-dev-arm64-cross \
+      binutils-aarch64-linux-gnu \
+      $(if [ "$TARGETPLATFORM" = "linux/arm64" ]; then printf '%s ' "libssl-dev:arm64" "zlib1g-dev:arm64"; fi) && \
+    rm -rf /var/lib/apt/lists/*
+RUN case "$TARGETPLATFORM" in \
+      "linux/amd64") TARGET_TRIPLE="x86_64-unknown-linux-gnu"; LIB_ARCH="x86_64-linux-gnu" ;; \
+      "linux/arm64") TARGET_TRIPLE="aarch64-unknown-linux-gnu"; LIB_ARCH="aarch64-linux-gnu" ;; \
+      *) echo "Unsupported TARGETPLATFORM: $TARGETPLATFORM" >&2; exit 1 ;; \
+    esac && \
+    rustup target add "$TARGET_TRIPLE" && \
+    echo "$TARGET_TRIPLE" > /tmp/target-triple && \
+    echo "$LIB_ARCH" > /tmp/lib-arch
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/clang \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/aarch64-linux-gnu-gcc \
+    CC_x86_64_unknown_linux_gnu=/usr/bin/clang \
+    CXX_x86_64_unknown_linux_gnu=/usr/bin/clang++ \
+    AR_x86_64_unknown_linux_gnu=/usr/bin/ar \
+    CC_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-gcc \
+    CXX_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-g++ \
+    AR_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-ar
+RUN cat <<'EOF' >/usr/local/bin/with-cross-env
+#!/usr/bin/env bash
+set -euo pipefail
+target="$(cat /tmp/target-triple 2>/dev/null || true)"
+if [ "$target" = "aarch64-unknown-linux-gnu" ]; then
+  export PKG_CONFIG_ALLOW_CROSS=1
+  export PKG_CONFIG_LIBDIR=/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/share/pkgconfig
+  export PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu
+  export OPENSSL_DIR=/usr/aarch64-linux-gnu
+  export OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu
+  export OPENSSL_INCLUDE_DIR=/usr/include/aarch64-linux-gnu
+  export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig
+fi
+exec "$@"
+EOF
+RUN chmod +x /usr/local/bin/with-cross-env
 COPY --from=planner /app/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
+COPY --from=planner /app/Cargo.lock Cargo.lock
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    TARGET_TRIPLE=$(cat /tmp/target-triple) && \
+    with-cross-env cargo chef cook --release --target "$TARGET_TRIPLE" --recipe-path recipe.json
 
 # Build application
 COPY . .
-RUN RUSTFLAGS="-C panic=abort" cargo build --release -p etl-replicator && \
-    strip target/release/etl-replicator
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    TARGET_TRIPLE=$(cat /tmp/target-triple) && \
+    mkdir -p target/release && \
+    RUSTFLAGS="-C panic=abort" with-cross-env cargo build --release --target "$TARGET_TRIPLE" --locked -p etl-replicator && \
+    if [ "$TARGET_TRIPLE" = "aarch64-unknown-linux-gnu" ]; then \
+      STRIP_TOOL="aarch64-linux-gnu-strip"; \
+    else \
+      STRIP_TOOL="strip"; \
+    fi && \
+    "$STRIP_TOOL" "target/${TARGET_TRIPLE}/release/etl-replicator" && \
+    cp "target/${TARGET_TRIPLE}/release/etl-replicator" target/release/etl-replicator
+
+FROM builder AS runtime-deps
+RUN set -euo pipefail; \
+    LIB_ARCH="$(cat /tmp/lib-arch)"; \
+    mkdir -p "/runtime-libs/usr/lib/${LIB_ARCH}" "/runtime-libs/lib/${LIB_ARCH}"; \
+    cp "/usr/lib/${LIB_ARCH}/libssl.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \
+    cp "/usr/lib/${LIB_ARCH}/libcrypto.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \
+    if [ -f "/usr/lib/${LIB_ARCH}/libz.so.1" ]; then \
+      cp "/usr/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \
+    else \
+      cp "/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/lib/${LIB_ARCH}/"; \
+    fi
 
 # Runtime stage with distroless for security
 FROM gcr.io/distroless/cc-debian12:nonroot
@@ -31,7 +106,8 @@ USER nonroot:nonroot
 
 # Copy binary and configuration
 COPY --from=builder /app/target/release/etl-replicator ./etl-replicator
+COPY --from=runtime-deps /runtime-libs/ /
 COPY --chown=nonroot:nonroot etl-replicator/configuration ./configuration
 
 # Use exec form for proper signal handling
-ENTRYPOINT ["./etl-replicator"]
+ENTRYPOINT ["./etl-replicator"]