diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..3c32d251 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 922a8a09..b6e47005 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -36,9 +36,30 @@ permissions: contents: read jobs: + metadata: + name: Compute Metadata + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + steps: + - name: Docker Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.image }} + tags: | + type=raw,value=latest + type=sha,format=long,prefix= + ${{ inputs.tag_with_version && format('type=raw,value=v{0}', inputs.version) || '' }} + docker: name: Build Docker Image + needs: metadata runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64, arm64] steps: - name: Checkout (specific ref) if: inputs.checkout_ref != '' @@ -53,6 +74,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Log in to Docker Hub if: inputs.push == true uses: docker/login-action@v3 @@ -60,27 +86,97 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker Metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ inputs.image }} - tags: | - type=raw,value=latest - type=sha,format=long,prefix= - ${{ inputs.tag_with_version && format('type=raw,value=v{0}', inputs.version) || '' }} + - name: Prepare Architecture Tags + id: arch_tags + shell: bash + run: | + set -euo pipefail + tags="${{ needs.metadata.outputs.tags }}" + arch="${{ matrix.arch }}" + if [ -z "$tags" ]; then + echo "tags=" >> "$GITHUB_OUTPUT" + exit 0 + fi + tmp=$(mktemp) + while IFS= read -r tag; do + if [ -n "$tag" ]; then + echo "${tag}-${arch}" >> "$tmp" + fi + done <<< "$tags" + printf 'tags<> "$GITHUB_OUTPUT" - - name: Build and Push Image + - name: Build and Push Image (${{ matrix.arch }}) id: build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ${{ inputs.context }} file: ${{ inputs.file }} push: ${{ inputs.push }} - tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max + tags: ${{ steps.arch_tags.outputs.tags }} + labels: ${{ needs.metadata.outputs.labels }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ inputs.image }}-${{ matrix.arch }} + cache-to: type=gha,scope=${{ inputs.image }}-${{ matrix.arch }},mode=max + outputs: type=image,name=${{ inputs.image }},push=${{ inputs.push }},push-by-digest=${{ inputs.push }},name-canonical=true + + - name: Export Digest (${{ matrix.arch }}) + if: inputs.push == true + run: | + set -euo pipefail + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload Digest Artifact (${{ matrix.arch }}) + if: inputs.push == true + uses: actions/upload-artifact@v4 + with: + name: digests-linux-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + manifest: + name: Publish Multi-Arch Manifest + needs: [metadata, docker] + if: inputs.push == true + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Download Digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + + - name: Create Multi-Architecture Manifests + shell: bash + run: | + set -euo pipefail + tags="${{ needs.metadata.outputs.tags }}" + image="${{ inputs.image }}" + refs="" + for digest_file in "${{ runner.temp }}/digests"/*; do + digest="$(basename "$digest_file")" + refs="$refs ${image}@sha256:${digest}" + done + while IFS= read -r tag; do + if [ -z "$tag" ]; then + continue + fi + docker buildx imagetools create \ + --tag "$tag" \ + $refs + done <<< "$tags" - name: Output Image Information run: | diff --git a/Cargo.toml b/Cargo.toml index c68c3fb3..fb8c0010 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,3 +84,8 @@ x509-cert = { version = "0.2.2", default-features = false } [profile.bench] debug = true + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 diff --git a/etl-api/Dockerfile b/etl-api/Dockerfile index 757819e8..2fd3d105 100644 --- a/etl-api/Dockerfile +++ b/etl-api/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.6 + # Build stage with cargo-chef for better layer caching FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef WORKDIR /app @@ -5,6 +7,7 @@ WORKDIR /app # Install system dependencies FROM chef AS planner COPY . . +RUN test -f Cargo.lock || cargo generate-lockfile RUN cargo chef prepare --recipe-path recipe.json # Build dependencies @@ -12,15 +15,86 @@ FROM chef AS builder ARG TARGETPLATFORM ARG BUILDPLATFORM RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM" -RUN apt-get update && apt-get install -y pkg-config libssl-dev && rm -rf /var/lib/apt/lists/* +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then dpkg --add-architecture arm64; fi && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + zlib1g-dev \ + clang \ + lld \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + binutils-aarch64-linux-gnu \ + $(if [ "$TARGETPLATFORM" = "linux/arm64" ]; then printf '%s ' "libssl-dev:arm64" "zlib1g-dev:arm64"; fi) && \ + rm -rf /var/lib/apt/lists/* +RUN case "$TARGETPLATFORM" in \ + "linux/amd64") TARGET_TRIPLE="x86_64-unknown-linux-gnu"; LIB_ARCH="x86_64-linux-gnu" ;; \ + "linux/arm64") TARGET_TRIPLE="aarch64-unknown-linux-gnu"; LIB_ARCH="aarch64-linux-gnu" ;; \ + *) echo "Unsupported TARGETPLATFORM: $TARGETPLATFORM" >&2; exit 1 ;; \ + esac && \ + rustup target add "$TARGET_TRIPLE" && \ + echo "$TARGET_TRIPLE" > /tmp/target-triple && \ + echo "$LIB_ARCH" > /tmp/lib-arch +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/clang \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/aarch64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=/usr/bin/clang \ + CXX_x86_64_unknown_linux_gnu=/usr/bin/clang++ \ + AR_x86_64_unknown_linux_gnu=/usr/bin/ar \ + CC_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-gcc \ + CXX_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-g++ \ + AR_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-ar +RUN cat <<'EOF' >/usr/local/bin/with-cross-env +#!/usr/bin/env bash +set -euo pipefail +target="$(cat /tmp/target-triple 2>/dev/null || true)" +if [ "$target" = "aarch64-unknown-linux-gnu" ]; then + export PKG_CONFIG_ALLOW_CROSS=1 + export PKG_CONFIG_LIBDIR=/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/share/pkgconfig + export PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu + export OPENSSL_DIR=/usr/aarch64-linux-gnu + export OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu + export OPENSSL_INCLUDE_DIR=/usr/include/aarch64-linux-gnu + export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig +fi +exec "$@" +EOF +RUN chmod +x /usr/local/bin/with-cross-env COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json +COPY --from=planner /app/Cargo.lock Cargo.lock +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + TARGET_TRIPLE=$(cat /tmp/target-triple) && \ + with-cross-env cargo chef cook --release --target "$TARGET_TRIPLE" --recipe-path recipe.json # Build application COPY . . ENV SQLX_OFFLINE=true -RUN cargo build --release -p etl-api && \ - strip target/release/etl-api +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + TARGET_TRIPLE=$(cat /tmp/target-triple) && \ + mkdir -p target/release && \ + with-cross-env cargo build --release --target "$TARGET_TRIPLE" --locked -p etl-api && \ + if [ "$TARGET_TRIPLE" = "aarch64-unknown-linux-gnu" ]; then \ + STRIP_TOOL="aarch64-linux-gnu-strip"; \ + else \ + STRIP_TOOL="strip"; \ + fi && \ + "$STRIP_TOOL" "target/${TARGET_TRIPLE}/release/etl-api" && \ + cp "target/${TARGET_TRIPLE}/release/etl-api" target/release/etl-api + +FROM builder AS runtime-deps +RUN set -euo pipefail; \ + LIB_ARCH="$(cat /tmp/lib-arch)"; \ + mkdir -p "/runtime-libs/usr/lib/${LIB_ARCH}" "/runtime-libs/lib/${LIB_ARCH}"; \ + cp "/usr/lib/${LIB_ARCH}/libssl.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \ + cp "/usr/lib/${LIB_ARCH}/libcrypto.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \ + if [ -f "/usr/lib/${LIB_ARCH}/libz.so.1" ]; then \ + cp "/usr/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \ + else \ + cp "/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/lib/${LIB_ARCH}/"; \ + fi # Runtime stage with distroless for security FROM gcr.io/distroless/cc-debian12:nonroot @@ -31,6 +105,7 @@ USER nonroot:nonroot # Copy binary and configuration COPY --from=builder /app/target/release/etl-api ./etl-api +COPY --from=runtime-deps /runtime-libs/ / COPY --chown=nonroot:nonroot etl-api/configuration ./configuration # Use exec form for proper signal handling diff --git a/etl-replicator/Dockerfile b/etl-replicator/Dockerfile index b8e20a34..d14f1a4e 100644 --- a/etl-replicator/Dockerfile +++ b/etl-replicator/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.6 + # Build stage with cargo-chef for better layer caching FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-1.88.0-slim-bookworm AS chef WORKDIR /app @@ -5,6 +7,7 @@ WORKDIR /app # Install system dependencies FROM chef AS planner COPY . . +RUN test -f Cargo.lock || cargo generate-lockfile RUN cargo chef prepare --recipe-path recipe.json # Build dependencies @@ -13,14 +16,86 @@ ARG TARGETPLATFORM ARG BUILDPLATFORM RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM" # TODO: remove protobuf-compiler once the upstream gcp-bigquery-client remove it from its deps -RUN apt-get update && apt-get install -y pkg-config libssl-dev protobuf-compiler clang && rm -rf /var/lib/apt/lists/* +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then dpkg --add-architecture arm64; fi && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + zlib1g-dev \ + protobuf-compiler \ + clang \ + lld \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + binutils-aarch64-linux-gnu \ + $(if [ "$TARGETPLATFORM" = "linux/arm64" ]; then printf '%s ' "libssl-dev:arm64" "zlib1g-dev:arm64"; fi) && \ + rm -rf /var/lib/apt/lists/* +RUN case "$TARGETPLATFORM" in \ + "linux/amd64") TARGET_TRIPLE="x86_64-unknown-linux-gnu"; LIB_ARCH="x86_64-linux-gnu" ;; \ + "linux/arm64") TARGET_TRIPLE="aarch64-unknown-linux-gnu"; LIB_ARCH="aarch64-linux-gnu" ;; \ + *) echo "Unsupported TARGETPLATFORM: $TARGETPLATFORM" >&2; exit 1 ;; \ + esac && \ + rustup target add "$TARGET_TRIPLE" && \ + echo "$TARGET_TRIPLE" > /tmp/target-triple && \ + echo "$LIB_ARCH" > /tmp/lib-arch +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/clang \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/aarch64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=/usr/bin/clang \ + CXX_x86_64_unknown_linux_gnu=/usr/bin/clang++ \ + AR_x86_64_unknown_linux_gnu=/usr/bin/ar \ + CC_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-gcc \ + CXX_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-g++ \ + AR_aarch64_unknown_linux_gnu=/usr/bin/aarch64-linux-gnu-ar +RUN cat <<'EOF' >/usr/local/bin/with-cross-env +#!/usr/bin/env bash +set -euo pipefail +target="$(cat /tmp/target-triple 2>/dev/null || true)" +if [ "$target" = "aarch64-unknown-linux-gnu" ]; then + export PKG_CONFIG_ALLOW_CROSS=1 + export PKG_CONFIG_LIBDIR=/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/share/pkgconfig + export PKG_CONFIG_SYSROOT_DIR=/usr/aarch64-linux-gnu + export OPENSSL_DIR=/usr/aarch64-linux-gnu + export OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu + export OPENSSL_INCLUDE_DIR=/usr/include/aarch64-linux-gnu + export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig +fi +exec "$@" +EOF +RUN chmod +x /usr/local/bin/with-cross-env COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json +COPY --from=planner /app/Cargo.lock Cargo.lock +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + TARGET_TRIPLE=$(cat /tmp/target-triple) && \ + with-cross-env cargo chef cook --release --target "$TARGET_TRIPLE" --recipe-path recipe.json # Build application COPY . . -RUN RUSTFLAGS="-C panic=abort" cargo build --release -p etl-replicator && \ - strip target/release/etl-replicator +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + TARGET_TRIPLE=$(cat /tmp/target-triple) && \ + mkdir -p target/release && \ + RUSTFLAGS="-C panic=abort" with-cross-env cargo build --release --target "$TARGET_TRIPLE" --locked -p etl-replicator && \ + if [ "$TARGET_TRIPLE" = "aarch64-unknown-linux-gnu" ]; then \ + STRIP_TOOL="aarch64-linux-gnu-strip"; \ + else \ + STRIP_TOOL="strip"; \ + fi && \ + "$STRIP_TOOL" "target/${TARGET_TRIPLE}/release/etl-replicator" && \ + cp "target/${TARGET_TRIPLE}/release/etl-replicator" target/release/etl-replicator + +FROM builder AS runtime-deps +RUN set -euo pipefail; \ + LIB_ARCH="$(cat /tmp/lib-arch)"; \ + mkdir -p "/runtime-libs/usr/lib/${LIB_ARCH}" "/runtime-libs/lib/${LIB_ARCH}"; \ + cp "/usr/lib/${LIB_ARCH}/libssl.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \ + cp "/usr/lib/${LIB_ARCH}/libcrypto.so.3" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \ + if [ -f "/usr/lib/${LIB_ARCH}/libz.so.1" ]; then \ + cp "/usr/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/usr/lib/${LIB_ARCH}/"; \ + else \ + cp "/lib/${LIB_ARCH}/libz.so.1" "/runtime-libs/lib/${LIB_ARCH}/"; \ + fi # Runtime stage with distroless for security FROM gcr.io/distroless/cc-debian12:nonroot @@ -31,7 +106,8 @@ USER nonroot:nonroot # Copy binary and configuration COPY --from=builder /app/target/release/etl-replicator ./etl-replicator +COPY --from=runtime-deps /runtime-libs/ / COPY --chown=nonroot:nonroot etl-replicator/configuration ./configuration # Use exec form for proper signal handling -ENTRYPOINT ["./etl-replicator"] \ No newline at end of file +ENTRYPOINT ["./etl-replicator"]