From 6df0c4d56886fe26e4fe98fb8541397188f4b6d3 Mon Sep 17 00:00:00 2001 From: Hiram Chirino Date: Mon, 3 Jun 2024 11:17:59 -0400 Subject: [PATCH] Simplify image builds by using QEMU to do multi-arch builds, but since they are slow, only do the multi-arch builds for releases. Also support passing cargo build args to the Docker image builder, allowing us to enable all features. Signed-off-by: Hiram Chirino --- .github/actions/build-image/action.yaml | 55 +++++++++++ .github/workflows/build-image.yaml | 96 +------------------ .github/workflows/build-multi-arch-image.yaml | 28 ++++++ Dockerfile | 18 +++- Dockerfile.aarch64 | 54 ----------- .../docker-compose-limitador-distributed.yaml | 33 +++++++ 6 files changed, 135 insertions(+), 149 deletions(-) create mode 100644 .github/actions/build-image/action.yaml create mode 100644 .github/workflows/build-multi-arch-image.yaml delete mode 100644 Dockerfile.aarch64 create mode 100644 limitador-server/sandbox/docker-compose-limitador-distributed.yaml diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml new file mode 100644 index 00000000..34269e98 --- /dev/null +++ b/.github/actions/build-image/action.yaml @@ -0,0 +1,55 @@ +name: 'Container Image Build' +description: 'Builds a container image for the project' +inputs: + platforms: + description: 'comma seperated list of platforms to build the image for' + required: true + default: 'linux/amd64' +runs: + using: "composite" + steps: + + - name: Set up QEMU for Multi-Arch Builds + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador + tags: | + # SHA tag for main branch + type=raw,value=${{ github.sha }},enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }} + # set latest tag for main branch + type=raw,value=latest,enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }} + # set ref name tag for non-main branches + type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != env.MAIN_BRANCH_NAME }} + + - name: Login to container registry + uses: docker/login-action@v2 + with: + username: ${{ secrets.IMG_REGISTRY_USERNAME }} + password: ${{ secrets.IMG_REGISTRY_TOKEN }} + registry: ${{ env.IMG_REGISTRY_HOST }} + + - name: Build and Push + id: build + uses: docker/build-push-action@v5 + with: + platforms: ${{ inputs.platforms }} + push: true + file: ./Dockerfile + tags: | + ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + GITHUB_SHA=${{ github.sha }} + + context: . + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador,push-by-digest=true,name-canonical=true,push=true + provenance: false diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 617f2f10..c7eb02d3 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -6,8 +6,6 @@ on: push: branches: - main - tags: - - "*" env: IMG_REGISTRY_HOST: quay.io @@ -20,97 +18,11 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - dockerfile: Dockerfile - platform: linux/amd64 - scope: build-amd - - dockerfile: Dockerfile.aarch64 - platform: linux/arm64 - scope: build-arm steps: - name: Check out code uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador - - name: Login to container registry - uses: docker/login-action@v2 - with: - username: ${{ secrets.IMG_REGISTRY_USERNAME }} - password: ${{ secrets.IMG_REGISTRY_TOKEN }} - registry: ${{ env.IMG_REGISTRY_HOST }} - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5 - with: - context: . - labels: ${{ steps.meta.outputs.labels }} - build-args: | - GITHUB_SHA=${{ github.sha }} - cache-from: type=gha,scope=${{ matrix.scope }} - cache-to: type=gha,mode=max,scope=${{ matrix.scope}} - outputs: type=image,name=${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador,push-by-digest=true,name-canonical=true,push=true - file: ${{ matrix.dockerfile }} - platforms: | - ${{ matrix.platform }} - provenance: false - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v3 - with: - name: digests - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: - - build - steps: - - name: Download digests - uses: actions/download-artifact@v3 - with: - name: digests - path: /tmp/digests - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador - tags: | - # SHA tag for main branch - type=raw,value=${{ github.sha }},enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }} - # set latest tag for main branch - type=raw,value=latest,enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }} - # set ref name tag for non-main branches - type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != env.MAIN_BRANCH_NAME }} - - name: Login to container registry - uses: docker/login-action@v2 + + - name: Build/Push the Container Image + uses: ./.github/actions/build-image with: - username: ${{ secrets.IMG_REGISTRY_USERNAME }} - password: ${{ secrets.IMG_REGISTRY_TOKEN }} - registry: ${{ env.IMG_REGISTRY_HOST }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador@sha256:%s ' *) - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }} - - name: Smoke Test - run: | - docker run --rm -t ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }} limitador-server --help \ No newline at end of file + platforms: linux/amd64 diff --git a/.github/workflows/build-multi-arch-image.yaml b/.github/workflows/build-multi-arch-image.yaml new file mode 100644 index 00000000..6883679e --- /dev/null +++ b/.github/workflows/build-multi-arch-image.yaml @@ -0,0 +1,28 @@ +--- +name: Build Image + +on: + workflow_dispatch: + push: + tags: + - "*" + +env: + IMG_REGISTRY_HOST: quay.io + IMG_REGISTRY_ORG: kuadrant + MAIN_BRANCH_NAME: main + +jobs: + build: + name: Build + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Build/Push the Container Image + uses: ./.github/actions/build-image + with: + platforms: linux/amd64,linux/arm64 diff --git a/Dockerfile b/Dockerfile index ee6f94de..77aab17a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # Use bullseye as build image instead of Bookworm as ubi9 does not not have GLIBCXX_3.4.30 # https://access.redhat.com/solutions/6969351 -FROM --platform=${BUILDPLATFORM} rust:1.78.0-bullseye as limitador-build +FROM rust:1.78.0-bullseye as limitador-build RUN apt update && apt upgrade -y \ && apt install -y protobuf-compiler clang @@ -12,12 +12,24 @@ RUN apt update && apt upgrade -y \ WORKDIR /usr/src/limitador ARG GITHUB_SHA +ARG CARGO_ARGS ENV GITHUB_SHA=${GITHUB_SHA:-unknown} ENV RUSTFLAGS="-C target-feature=-crt-static" -COPY . . +# We set the env here just to make sure that the build is invalidated if the args change +ENV CARGO_ARGS=${CARGO_ARGS} -RUN cargo build --release +# The following allows us to cache the Cargo dependency downloads with image layers +COPY Cargo.toml Cargo.lock ./ +COPY limitador/Cargo.toml ./limitador/ +COPY limitador-server/Cargo.toml ./limitador-server/ +RUN mkdir -p limitador-server/src && echo 'fn main() {}' > limitador-server/src/main.rs +RUN cargo build --release ${CARGO_ARGS} + +COPY ./limitador ./limitador +COPY ./limitador-server ./limitador-server + +RUN cargo build --release ${CARGO_ARGS} # ------------------------------------------------------------------------------ # Run Stage diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 deleted file mode 100644 index d2baa452..00000000 --- a/Dockerfile.aarch64 +++ /dev/null @@ -1,54 +0,0 @@ -# ------------------------------------------------------------------------------ -# Build Stage cross compiling -# ------------------------------------------------------------------------------ - -# Use bullseye as build image instead of Bookworm as ubi9 does not not have GLIBCXX_3.4.30 -# https://access.redhat.com/solutions/6969351 -FROM --platform=${BUILDPLATFORM} rust:1.78.0-bullseye as limitador-build - -RUN apt update && apt upgrade -y \ - && apt install -y protobuf-compiler clang g++-aarch64-linux-gnu libc6-dev-arm64-cross - -RUN rustup target add aarch64-unknown-linux-gnu \ - && rustup toolchain install stable-aarch64-unknown-linux-gnu - -WORKDIR /usr/src/limitador - -ARG GITHUB_SHA -ENV GITHUB_SHA=${GITHUB_SHA:-unknown} -ENV RUSTFLAGS="-C target-feature=-crt-static" \ - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ - CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \ - CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \ - # https://github.com/rust-lang/rust-bindgen/issues/1229 - BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" - -COPY . . - -RUN cargo build --release --target aarch64-unknown-linux-gnu - -# ------------------------------------------------------------------------------ -# Run Stage -# ------------------------------------------------------------------------------ - -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.2 - -# shadow-utils is required for `useradd` -RUN PKGS="libgcc libstdc++ shadow-utils" \ - && microdnf --assumeyes install --nodocs $PKGS \ - && rpm --verify --nogroup --nouser $PKGS \ - && microdnf -y clean all -RUN useradd -u 1000 -s /bin/sh -m -d /home/limitador limitador - -WORKDIR /home/limitador/bin/ -ENV PATH="/home/limitador/bin:${PATH}" - -COPY --from=limitador-build /usr/src/limitador/limitador-server/examples/limits.yaml ../ -COPY --from=limitador-build /usr/src/limitador/target/aarch64-unknown-linux-gnu/release/limitador-server ./limitador-server - -RUN chown -R limitador:root /home/limitador \ - && chmod -R 750 /home/limitador - -USER limitador - -CMD ["limitador-server"] \ No newline at end of file diff --git a/limitador-server/sandbox/docker-compose-limitador-distributed.yaml b/limitador-server/sandbox/docker-compose-limitador-distributed.yaml new file mode 100644 index 00000000..611e0f1b --- /dev/null +++ b/limitador-server/sandbox/docker-compose-limitador-distributed.yaml @@ -0,0 +1,33 @@ +--- +version: '3.8' +services: + limitador: + image: ${LIMITADOR_IMAGE:-limitador-testing} + depends_on: + - envoy + - redis + command: + - limitador-server + - --rls-ip + - 0.0.0.0 + - --rls-port + - "8081" + - --http-ip + - 0.0.0.0 + - --http-port + - "8080" + - -vvv + - --grpc-reflection-service + - /opt/kuadrant/limits/limits.yaml + - redis + - redis://redis:6379 + expose: + - "8080" + - "8081" + ports: + - "18080:8080" + - "18081:8081" + volumes: + - ./limits.yaml:/opt/kuadrant/limits/limits.yaml + redis: + image: redis:5