diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3ad41a2c1..99edd5fec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,6 +5,9 @@ on: push: branches: - master + - 'release-*' + tags: + - '**' pull_request: types: - opened @@ -12,6 +15,27 @@ on: - synchronize jobs: + define-scanner-job-matrix: + outputs: + matrix: ${{ steps.define-scanner-job-matrix.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Define the matrix for build jobs + id: define-scanner-job-matrix + run: | + # If goarch is updated, be sure to update architectures in push-manifests below. + matrix='{ "build_and_push": { "goos":["linux"], "goarch":["amd64", "arm64", "ppc64le", "s390x"] } }' + + jq <<< "$matrix" + + condensed="$(jq -c <<< "$matrix")" + echo "matrix=$condensed" >> "$GITHUB_OUTPUT" + pre-build-updater: runs-on: ubuntu-latest container: @@ -28,7 +52,7 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - - name: Build updater + - name: Build updater (amd64) run: make build-updater - name: Archive the build to preserve permissions @@ -40,7 +64,11 @@ jobs: path: updater-build.tgz pre-build-scanner: + needs: define-scanner-job-matrix runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push }} container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 steps: @@ -56,15 +84,15 @@ jobs: uses: ./.github/actions/cache-go-dependencies - name: Build Scanner - run: make scanner-build-nodeps + run: make GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} scanner-build-nodeps - name: Archive the build to preserve permissions - run: tar -cvzf scanner-build.tgz image/scanner/bin/scanner + run: tar -cvzf scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}.tgz image/scanner/bin/scanner - uses: actions/upload-artifact@v4 with: - name: scanner-build - path: scanner-build.tgz + name: scanner-build-${{ matrix.goos }}-${{ matrix.goarch }} + path: scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}.tgz style-check: runs-on: ubuntu-latest @@ -82,13 +110,11 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - - name: Run style checks + - name: Run style checks (amd64) run: ./scripts/ci/jobs/style-checks.sh unit-tests: runs-on: ubuntu-latest - needs: - - pre-build-scanner container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 steps: @@ -103,21 +129,11 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - - uses: actions/download-artifact@v4 - with: - name: scanner-build - - - name: Unpack scanner build - run: | - tar xvzf scanner-build.tgz - - - name: Run unit tests + - name: Run unit tests (amd64) run: ./scripts/ci/jobs/unit-tests.sh db-integration-tests: runs-on: ubuntu-latest - needs: - - pre-build-scanner container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 steps: @@ -132,18 +148,11 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - - uses: actions/download-artifact@v4 - with: - name: scanner-build - - - name: Unpack scanner build - run: | - tar xvzf scanner-build.tgz - - - name: Run db integration tests + - name: Run db integration tests (amd64) run: ./scripts/ci/jobs/db-integration-tests.sh generate-genesis-dump: + # Run this job if it's not a PR or the PR contains the `generate-dumps-on-pr` label if: | github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') @@ -187,6 +196,7 @@ jobs: path: /tmp/vuln-dump generate-db-dump: + # Run this job if it's not a PR or the PR contains the `generate-dumps-on-pr` label if: | github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') @@ -228,13 +238,19 @@ jobs: path: /tmp/postgres/pg-definitions.sql.gz generate-scanner-bundle: + # Run this job even if the generate-genesis-dump job was skipped, i.e., only skip this job if + # generate-genesis-dump failed + if: | + always() && + (needs.generate-genesis-dump.result == 'success' || needs.generate-genesis-dump.result == 'skipped') runs-on: ubuntu-latest needs: + - define-scanner-job-matrix - pre-build-scanner - generate-genesis-dump - if: | - always() && - (needs.generate-genesis-dump.result == 'success' || needs.generate-genesis-dump.result == 'skipped') + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push }} container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 steps: @@ -248,13 +264,15 @@ jobs: - uses: actions/download-artifact@v4 with: - name: scanner-build + name: scanner-build-${{ matrix.goos }}-${{ matrix.goarch }} - name: Unpack scanner build run: | - tar xvzf scanner-build.tgz + tar xvzf scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}.tgz - uses: actions/download-artifact@v4 + # Run this step if it's not a PR or the PR contains the `generate-dumps-on-pr` label + # When this step is skipped `get_genesis_dump` will pull the vulnerability data from our GCS bucket if: | github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') @@ -274,20 +292,22 @@ jobs: run: image/scanner/rhel/create-bundle.sh image/scanner image/scanner/rhel - name: Archive the bundle to preserve permissions - run: tar -cvzf bundle.tgz image/scanner/rhel + run: tar -cvzf scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}.tgz image/scanner/rhel - uses: actions/upload-artifact@v4 with: - name: scanner-bundle - path: bundle.tgz + name: scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }} + path: scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}.tgz generate-scanner-db-bundle: - runs-on: ubuntu-latest - needs: - - generate-db-dump + # Run this job even if the generate-db-dump job was skipped, i.e., only skip this job if + # generate-db-dump failed if: | always() && (needs.generate-db-dump.result == 'success' || needs.generate-db-dump.result == 'skipped') + runs-on: ubuntu-latest + needs: + - generate-db-dump container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 steps: @@ -321,6 +341,9 @@ jobs: path: image/db/rhel build-images: + # Run this job even if previous jobs were skipped, i.e., only skip this job if one of the previous jobs failed + # or was cancelled + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') env: QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} @@ -330,9 +353,12 @@ jobs: QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} runs-on: ubuntu-latest needs: + - define-scanner-job-matrix - generate-scanner-bundle - generate-scanner-db-bundle - if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push }} container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 steps: @@ -344,13 +370,19 @@ jobs: - uses: ./.github/actions/job-preamble + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/download-artifact@v4 with: - name: scanner-bundle + name: scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }} - name: Unpack bundle run: | - tar xvzf bundle.tgz + tar xvzf scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}.tgz - uses: actions/download-artifact@v4 with: @@ -359,19 +391,19 @@ jobs: - name: Build scanner image run: | - docker build -t scanner:"$(make --no-print-directory --quiet tag)" -f image/scanner/rhel/Dockerfile image/scanner/rhel + docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner:"$(make --no-print-directory --quiet tag)" -f image/scanner/rhel/Dockerfile image/scanner/rhel - name: Build scanner-slim image run: | - docker build -t scanner-slim:"$(make --no-print-directory --quiet tag)" -f image/scanner/rhel/Dockerfile.slim image/scanner/rhel + docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner-slim:"$(make --no-print-directory --quiet tag)" -f image/scanner/rhel/Dockerfile.slim image/scanner/rhel - name: Build scanner-db image run: | - docker build -t scanner-db:"$(make --no-print-directory --quiet tag)" -f image/db/rhel/Dockerfile image/db/rhel + docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner-db:"$(make --no-print-directory --quiet tag)" -f image/db/rhel/Dockerfile image/db/rhel - name: Build scanner-db-slim image run: | - docker build -t scanner-db-slim:"$(make --no-print-directory --quiet tag)" -f image/db/rhel/Dockerfile.slim image/db/rhel + docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner-db-slim:"$(make --no-print-directory --quiet tag)" -f image/db/rhel/Dockerfile.slim image/db/rhel - name: Docker login # Skip for external contributions. @@ -386,12 +418,55 @@ jobs: github.event_name == 'push' || !github.event.pull_request.head.repo.fork run: | source ./scripts/ci/lib.sh - push_image_set + push_scanner_image_set ${{ matrix.goarch }} + + push-manifests: + # Run this job even if previous jobs were skipped, i.e., only skip this job if one of the previous jobs failed + # or was cancelled + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + needs: + - define-scanner-job-matrix + - generate-scanner-bundle + - generate-scanner-db-bundle + - build-images + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.69 + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Push Scanner and ScannerDB image manifests + # Skip for external contributions. + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + run: | + source ./scripts/ci/lib.sh + + # If this is updated, be sure to update goarch in define-scanner-job-matrix above. + architectures="amd64,arm64,ppc64le,s390x" + + push_scanner_image_manifest_lists "$architectures" diff-dumps: + # Run this job if: + # - it's running on the master branch OR + # - it's in a PR context and the PR contains the `generate-dumps-on-pr` label + # Note that this doesn't run on tags if: | - github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') + github.ref == 'refs/heads/master' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr')) env: GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} SCANNER_GCP_SERVICE_ACCOUNT_CREDS: ${{ secrets.SCANNER_GCP_SERVICE_ACCOUNT_CREDS }} @@ -419,7 +494,6 @@ jobs: tar xvzf updater-build.tgz - uses: actions/download-artifact@v4 - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} with: name: genesis-dump path: /tmp/genesis-dump @@ -428,7 +502,8 @@ jobs: run: ./scripts/ci/jobs/diff-dumps.sh upload-db-dump: - # Only run on master branch + # Only run this step on the master branch + # Note that our scheduled jobs run on the master branch if: github.ref == 'refs/heads/master' env: GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} @@ -442,7 +517,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + ref: master - uses: ./.github/actions/job-preamble @@ -455,8 +530,9 @@ jobs: run: ./scripts/ci/jobs/upload-db-dump.sh upload-dumps-for-downstream: - # Only run on master branch - if: github.ref == 'refs/heads/master' + # Only run this step on the master branch or any tags + # Note that our scheduled jobs run on the master branch + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') env: GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} runs-on: ubuntu-latest @@ -469,7 +545,6 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble @@ -487,7 +562,8 @@ jobs: run: ./scripts/ci/jobs/upload-dumps-for-downstream.sh upload-dumps-for-embedding: - # Only run on master branch + # Only run this step on the master branch + # Note that our scheduled jobs run on the master branch if: github.ref == 'refs/heads/master' env: GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} @@ -501,7 +577,6 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble diff --git a/image/scanner/rhel/Dockerfile b/image/scanner/rhel/Dockerfile index 9fdb40218..9dcb5ea65 100644 --- a/image/scanner/rhel/Dockerfile +++ b/image/scanner/rhel/Dockerfile @@ -6,7 +6,7 @@ FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS extracted_bundle COPY bundle.tar.gz / WORKDIR /bundle -RUN microdnf install tar gzip && tar -zxf /bundle.tar.gz +RUN microdnf install -y tar gzip && tar -zxf /bundle.tar.gz FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS base @@ -24,9 +24,9 @@ COPY --from=extracted_bundle /bundle/scanner ./ COPY --from=extracted_bundle /bundle/THIRD_PARTY_NOTICES/ /THIRD_PARTY_NOTICES/ -RUN microdnf upgrade --nobest && \ - microdnf install xz && \ - microdnf clean all && \ +RUN microdnf upgrade -y --nobest && \ + microdnf install -y xz && \ + microdnf clean -y all && \ # (Optional) Remove line below to keep package management utilities # We don't uninstall rpm because scanner uses it to get packages installed in scanned images. rpm -e --nodeps $(rpm -qa curl '*dnf*' '*libsolv*' '*hawkey*' 'yum*') && \ diff --git a/image/scanner/rhel/Dockerfile.slim b/image/scanner/rhel/Dockerfile.slim index bf9356abb..68e08df76 100644 --- a/image/scanner/rhel/Dockerfile.slim +++ b/image/scanner/rhel/Dockerfile.slim @@ -6,7 +6,7 @@ FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS extracted_bundle COPY bundle.tar.gz / WORKDIR /bundle -RUN microdnf install tar gzip && tar -zxf /bundle.tar.gz +RUN microdnf install -y tar gzip && tar -zxf /bundle.tar.gz FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS base @@ -24,9 +24,9 @@ COPY --from=extracted_bundle /bundle/scanner ./ COPY --from=extracted_bundle /bundle/THIRD_PARTY_NOTICES/ /THIRD_PARTY_NOTICES/ -RUN microdnf upgrade --nobest && \ - microdnf install xz && \ - microdnf clean all && \ +RUN microdnf upgrade -y --nobest && \ + microdnf install -y xz && \ + microdnf clean -y all && \ # (Optional) Remove line below to keep package management utilities # We don't uninstall rpm because scanner uses it to get packages installed in scanned images. rpm -e --nodeps $(rpm -qa curl '*dnf*' '*libsolv*' '*hawkey*' 'yum*') && \ diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh index 4389c14d7..1ca87700d 100755 --- a/scripts/ci/lib.sh +++ b/scripts/ci/lib.sh @@ -94,63 +94,68 @@ create_exit_trap() { trap ci_exit_trap EXIT } -push_image_set() { - info "Pushing images" +push_scanner_image_manifest_lists() { + info "Pushing scanner and scanner-db images as manifest lists" - require_environment "QUAY_RHACS_ENG_RW_USERNAME" - require_environment "QUAY_RHACS_ENG_RW_PASSWORD" - require_environment "QUAY_STACKROX_IO_RW_USERNAME" - require_environment "QUAY_STACKROX_IO_RW_PASSWORD" - - local image_set=("scanner" "scanner-db" "scanner-slim" "scanner-db-slim") - if is_OPENSHIFT_CI; then - local image_srcs=("$SCANNER_IMAGE" "$SCANNER_DB_IMAGE" "$SCANNER_SLIM_IMAGE" "$SCANNER_DB_SLIM_IMAGE") - oc registry login + if [[ "$#" -ne 1 ]]; then + die "missing arg. usage: push_scanner_image_manifest_lists " fi - _push_image_set() { - local registry="$1" - local tag="$2" + local architectures="$1" + local scanner_image_set=("scanner" "scanner-db" "scanner-slim" "scanner-db-slim") + local registries=("quay.io/rhacs-eng" "quay.io/stackrox-io") - for image in "${image_set[@]}"; do - "$SCRIPTS_ROOT/scripts/push-as-manifest-list.sh" "${registry}/${image}:${tag}" | cat + local tag + tag="$(make --quiet --no-print-directory tag)" + for registry in "${registries[@]}"; do + registry_rw_login "$registry" + for image in "${scanner_image_set[@]}"; do + retry 5 true \ + "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat done - } + done +} - _tag_image_set() { +push_scanner_image_set() { + info "Pushing scanner and scanner-db images" + + if [[ "$#" -ne 1 ]]; then + die "missing arg. usage: push_scanner_image_set " + fi + + local arch="$1" + + local scanner_image_set=("scanner" "scanner-db" "scanner-slim" "scanner-db-slim") + + _push_scanner_image_set() { local registry="$1" local tag="$2" - for image in "${image_set[@]}"; do - docker tag "${image}:${tag}" "${registry}/${image}:${tag}" + for image in "${scanner_image_set[@]}"; do + retry 5 true \ + docker push "${registry}/${image}:${tag}" | cat done } - _mirror_image_set() { - local registry="$1" - local tag="$2" + _tag_scanner_image_set() { + local local_tag="$1" + local registry="$2" + local remote_tag="$3" - local idx=0 - for image in "${image_set[@]}"; do - oc_image_mirror "${image_srcs[$idx]}" "${registry}/${image}:${tag}" - (( idx++ )) || true + for image in "${scanner_image_set[@]}"; do + docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}" done } - local destination_registries=("quay.io/rhacs-eng" "quay.io/stackrox-io") + local registries=("quay.io/rhacs-eng" "quay.io/stackrox-io") local tag tag="$(make --quiet --no-print-directory tag)" - info "PUSHING IMAGES WITH TAG: ${tag}" - for registry in "${destination_registries[@]}"; do + for registry in "${registries[@]}"; do registry_rw_login "$registry" - if is_OPENSHIFT_CI; then - _mirror_image_set "$registry" "$tag" - else - _tag_image_set "$registry" "$tag" - _push_image_set "$registry" "$tag" - fi + _tag_scanner_image_set "$tag" "$registry" "$tag-$arch" + _push_scanner_image_set "$registry" "$tag-$arch" done } diff --git a/scripts/ci/push-as-multiarch-manifest-list.sh b/scripts/ci/push-as-multiarch-manifest-list.sh new file mode 100755 index 000000000..208094cb3 --- /dev/null +++ b/scripts/ci/push-as-multiarch-manifest-list.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)" +# shellcheck source=../../scripts/lib.sh +source "$ROOT/scripts/lib.sh" + +set -euo pipefail + +[[ "$#" == 2 ]] || die "Usage: $0 " + +image="$1" +IFS=',' read -ra architectures <<< "$2" + +[[ -n "$image" ]] || die "No image specified" +[[ "$image" == *:* ]] || die "Must specify a tagged image reference when using this script" + +image_list=() +for arch in "${architectures[@]}" +do + arch_image="${image}-${arch}" + docker pull "${arch_image}" + image_list+=("$arch_image") +done + +docker manifest create "$image" "${image_list[@]}" + +# Try pushing manifest a few times for the case when quay.io has issues +pushed=0 +for i in {1..5}; do + echo "Pushing manifest for ${image}. Attempt ${i}..." + echo docker manifest push "$image" + if docker manifest push "$image"; then + pushed=1 + break + fi + sleep 10 +done +(( pushed )) diff --git a/scripts/push-as-manifest-list.sh b/scripts/push-as-manifest-list.sh deleted file mode 100755 index 081fc2d19..000000000 --- a/scripts/push-as-manifest-list.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -die() { - echo >&2 "$@" - exit 1 -} - -[[ "$#" == 1 ]] || die "Usage: $0 " - -image="$1" - -[[ -n "$image" ]] || die "No image specified" -[[ "$image" == *:* ]] || die "Must specify a tagged image reference when using this script" - -arch_image="${image}-amd64" -docker tag "$image" "$arch_image" - -docker push "$arch_image" -docker manifest create "$image" "$arch_image" - -docker manifest push "$image"