diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..de20d2e0d --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,9 @@ +# Use lld linker for faster linking (1.5-2x faster than default ld) +# lld is part of the LLVM toolchain and widely available +[target.x86_64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-C", "link-arg=-fuse-ld=lld"] + +[target.aarch64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-C", "link-arg=-fuse-ld=lld"] diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 922a8a094..33bc80b06 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -34,12 +34,34 @@ on: permissions: contents: read + actions: write jobs: - docker: - name: Build Docker Image - runs-on: ubuntu-latest + # Build each platform natively on its own runner + build-platform: + name: Build ${{ matrix.platform }} Image + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + outputs: + digest-amd64: ${{ steps.export-digest.outputs.digest-amd64 }} + digest-arm64: ${{ steps.export-digest.outputs.digest-arm64 }} + image-name: ${{ steps.extract-name.outputs.name }} steps: + - name: Extract Image Name + id: extract-name + run: | + IMAGE="${{ inputs.image }}" + NAME="${IMAGE##*/}" + echo "name=$NAME" >> "$GITHUB_OUTPUT" + - name: Checkout (specific ref) if: inputs.checkout_ref != '' uses: actions/checkout@v4 @@ -60,39 +82,111 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker Metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ inputs.image }} - tags: | - type=raw,value=latest - type=sha,format=long,prefix= - ${{ inputs.tag_with_version && format('type=raw,value=v{0}', inputs.version) || '' }} - - - name: Build and Push Image + - name: Build and Push Single-Platform Image id: build uses: docker/build-push-action@v5 with: context: ${{ inputs.context }} file: ${{ inputs.file }} push: ${{ inputs.push }} - tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + provenance: false + outputs: type=image,name=${{ inputs.image }},push-by-digest=true,name-canonical=true + + - name: Export Digest + id: export-digest + if: inputs.push == true + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + echo "$digest" > "/tmp/digests/${{ matrix.arch }}.txt" + echo "digest-${{ matrix.arch }}=$digest" >> "$GITHUB_OUTPUT" + + - name: Upload Digest + if: inputs.push == true + uses: actions/upload-artifact@v4 + with: + name: digests-${{ steps.extract-name.outputs.name }}-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Create multi-arch manifest combining both platforms + create-manifest: + name: Create Multi-Arch Manifest + needs: build-platform + runs-on: ubuntu-latest + if: inputs.push == true + steps: + - name: Extract Image Name + id: extract-name + run: | + IMAGE="${{ inputs.image }}" + NAME="${IMAGE##*/}" + echo "name=$NAME" >> "$GITHUB_OUTPUT" + + - name: Download Digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-${{ steps.extract-name.outputs.name }}-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Checkout (specific ref) + if: inputs.checkout_ref != '' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.checkout_ref }} + + - name: Checkout (default) + if: inputs.checkout_ref == '' + uses: actions/checkout@v4 + + - name: Create and Push Multi-Arch Manifest + run: | + IMAGE="${{ inputs.image }}" + FULL_SHA=$(git rev-parse HEAD) + + # Prepare digest list for imagetools + DIGEST_LIST="" + for digest_file in /tmp/digests/*.txt; do + digest=$(cat "$digest_file") + DIGEST_LIST="${DIGEST_LIST} ${IMAGE}@${digest}" + done + + # Create manifest for latest + docker buildx imagetools create -t "${IMAGE}:latest" ${DIGEST_LIST} + + # Create manifest for SHA + docker buildx imagetools create -t "${IMAGE}:${FULL_SHA}" ${DIGEST_LIST} + + # Create manifest for version if needed + if [ "${{ inputs.tag_with_version }}" = "true" ] && [ -n "${{ inputs.version }}" ]; then + docker buildx imagetools create -t "${IMAGE}:v${{ inputs.version }}" ${DIGEST_LIST} + fi - name: Output Image Information run: | IMAGE="${{ inputs.image }}" NAME="${IMAGE##*/}" - # Derive SHA from the checked-out ref to ensure correctness. FULL_SHA=$(git rev-parse HEAD) TAGS="latest, ${FULL_SHA}" if [ "${{ inputs.tag_with_version }}" = "true" ] && [ -n "${{ inputs.version }}" ]; then TAGS="${TAGS}, v${{ inputs.version }}" fi HUB_PATH=$(echo "${IMAGE}" | sed -E 's@^docker\.io/@@') - echo "✅ Successfully built and pushed ${NAME} image" + echo "✅ Successfully built and pushed ${NAME} multi-arch image" echo "🏷️ Tags: ${TAGS}" + echo "🏗️ Platforms: linux/amd64, linux/arm64" echo "🔗 View at: https://hub.docker.com/r/${HUB_PATH}" diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index e032fa8ea..d40b42951 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -12,6 +12,7 @@ on: permissions: contents: read + actions: write jobs: resolve-ref: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7366661fd..56d9f3a45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,7 @@ on: permissions: contents: write + actions: write jobs: version: diff --git a/Cargo.toml b/Cargo.toml index c68c3fb32..095ee0ff5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,3 +84,12 @@ x509-cert = { version = "0.2.2", default-features = false } [profile.bench] debug = true + +[profile.release] +# Link-time optimization: enables better cross-crate optimizations +# "thin" provides ~80-90% of "fat" benefits with much faster compile times +lto = "thin" +# Default codegen-units for balanced build time and performance +codegen-units = 16 +# Maximum optimization level for best runtime performance +opt-level = 3 diff --git a/etl-api/Dockerfile b/etl-api/Dockerfile index 757819e88..02e92a6f1 100644 --- a/etl-api/Dockerfile +++ b/etl-api/Dockerfile @@ -1,5 +1,6 @@ # Build stage with cargo-chef for better layer caching -FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef +# Native build: each runner builds for its own architecture +FROM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef WORKDIR /app # Install system dependencies @@ -11,12 +12,25 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG TARGETPLATFORM ARG BUILDPLATFORM -RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM" -RUN apt-get update && apt-get install -y pkg-config libssl-dev && rm -rf /var/lib/apt/lists/* +RUN echo "Native build on $BUILDPLATFORM for $TARGETPLATFORM" + +# Install build dependencies including lld linker for faster linking +RUN apt-get update && \ + apt-get install -y \ + pkg-config \ + libssl-dev \ + clang \ + lld && \ + rm -rf /var/lib/apt/lists/* + +# Copy cargo config for build optimizations (lld linker, etc.) +COPY .cargo/config.toml /app/.cargo/config.toml + COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json -# Build application +# Build application with optimizations +# The release profile in Cargo.toml handles: thin LTO, strip, opt-level=3 COPY . . ENV SQLX_OFFLINE=true RUN cargo build --release -p etl-api && \ diff --git a/etl-replicator/Dockerfile b/etl-replicator/Dockerfile index b8e20a34f..56e8134e5 100644 --- a/etl-replicator/Dockerfile +++ b/etl-replicator/Dockerfile @@ -1,5 +1,6 @@ # Build stage with cargo-chef for better layer caching -FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef +# Native build: each runner builds for its own architecture +FROM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef WORKDIR /app # Install system dependencies @@ -11,13 +12,27 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG TARGETPLATFORM ARG BUILDPLATFORM -RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM" +RUN echo "Native build on $BUILDPLATFORM for $TARGETPLATFORM" + +# Install build dependencies including lld linker for faster linking # TODO: remove protobuf-compiler once the upstream gcp-bigquery-client remove it from its deps -RUN apt-get update && apt-get install -y pkg-config libssl-dev protobuf-compiler clang && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get install -y \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + clang \ + lld && \ + rm -rf /var/lib/apt/lists/* + +# Copy cargo config for build optimizations (lld linker, etc.) +COPY .cargo/config.toml /app/.cargo/config.toml + COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json -# Build application +# Build application with optimizations +# The release profile in Cargo.toml handles: thin LTO, strip, opt-level=3 COPY . . RUN RUSTFLAGS="-C panic=abort" cargo build --release -p etl-replicator && \ strip target/release/etl-replicator