diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index b8afea93e..ad3097d0a 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -38,6 +38,12 @@ on: type: boolean required: false default: true + push-dirty: + description: Push scanned images that have vulnerabilities? + type: boolean + required: false + # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures + default: true env: ANSIBLE_FORCE_COLOR: True @@ -109,7 +115,15 @@ jobs: - name: Install package dependencies run: | sudo apt update - sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget + + - name: Install gh + run: | + sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y - name: Checkout uses: actions/checkout@v4 @@ -127,6 +141,10 @@ jobs: run: | docker ps + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0 + - name: Install Kayobe run: | mkdir -p venvs && @@ -162,65 +180,124 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - name: Build and push kolla overcloud images + - name: Create build logs output directory + run: mkdir image-build-logs + + - name: Build kolla overcloud images + id: build_overcloud_images + continue-on-error: true run: | - args="${{ github.event.inputs.regexes }}" + args="${{ inputs.regexes }}" args="$args -e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}" args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" - if ${{ inputs.push }} == 'true'; then - args="$args --push" - fi source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe overcloud container image build $args env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: github.event.inputs.overcloud == 'true' + if: inputs.overcloud + + - name: Copy overcloud container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log + if: inputs.overcloud - - name: Build and push kolla seed images + - name: Build kolla seed images + id: build_seed_images + continue-on-error: true run: | args="-e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}" args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" - if ${{ inputs.push }} == 'true'; then - args="$args --push" - fi source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe seed container image build $args env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: github.event.inputs.seed == 'true' + if: inputs.seed + + - name: Copy seed container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log + if: inputs.seed - name: Get built container images - run: | - docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images - name: Fail if no images have been built run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi - - name: Upload container images artifact + - name: Scan built container images + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }} + + - name: Move image scan logs to output artifact + run: mv image-scan-output image-build-logs/image-scan-output + + - name: Fail if no images have passed scanning + run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi + if: ${{ !inputs.push-dirty }} + + - name: Copy clean images to push-attempt-images list + run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt + if: inputs.push + + - name: Append dirty images to push list + run: | + cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + if: ${{ inputs.push && inputs.push-dirty }} + + - name: Push images + run: | + touch image-build-logs/push-failed-images.txt + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml && + + while read -r image; do + # Retries! + for i in {1..5}; do + if docker push $image; then + echo "Pushed $image" + break + elif $i == 5; then + echo "Failed to push $image" + echo $image >> image-build-logs/push-failed-images.txt + else + echo "Failed on retry $i" + sleep 5 + fi; + done + done < image-build-logs/push-attempt-images.txt + shell: bash + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.push + + - name: Upload output artifact uses: actions/upload-artifact@v4 with: - name: ${{ matrix.distro }} container images - path: ${{ matrix.distro }}-container-images + name: ${{ matrix.distro }}-logs + path: image-build-logs retention-days: 7 + if: ${{ !cancelled() }} + + - name: Fail when images failed to build + run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1 + if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }} + + - name: Fail when images failed to push + run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi + if: ${{ !cancelled() }} + + - name: Fail when images failed scanning + run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi + if: ${{ !inputs.push-dirty && !cancelled() }} - sync-container-repositories: - name: Trigger container image repository sync - needs: - - container-image-build - if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push - runs-on: ubuntu-latest - permissions: {} - steps: # NOTE(mgoddard): Trigger another CI workflow in the # stackhpc-release-train repository. - name: Trigger container image repository sync run: | filter='${{ inputs.regexes }}' - if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then + if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then filter="$filter bifrost" fi gh workflow run \ @@ -231,7 +308,9 @@ jobs: -f sync-new-images=false env: GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} - name: Display link to container image repository sync workflows run: | echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml" + if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} diff --git a/etc/kayobe/ansible/docker-registry-login.yml b/etc/kayobe/ansible/docker-registry-login.yml new file mode 100644 index 000000000..39ad03600 --- /dev/null +++ b/etc/kayobe/ansible/docker-registry-login.yml @@ -0,0 +1,11 @@ +--- +- name: Login to docker registry + gather_facts: false + hosts: container-image-builders + tasks: + - name: Login to docker registry + docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: yes diff --git a/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml new file mode 100644 index 000000000..67a99f9c2 --- /dev/null +++ b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Kolla container images created using the + ``stackhpc-container-image-build.yml`` workflow are now automatically + scanned for vulnerablilities. diff --git a/tools/scan-images.sh b/tools/scan-images.sh new file mode 100755 index 000000000..50a04185a --- /dev/null +++ b/tools/scan-images.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Check correct usage +if [[ ! $2 ]]; then + echo "Usage: scan-images.sh " + exit 2 +fi + +set -u + +# Check that trivy is installed +if ! trivy --version; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1' +fi + +# Clear any previous outputs +rm -rf image-scan-output + +# Make a fresh output directory +mkdir -p image-scan-output + +# Get built container images +docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-scanned-container-images.txt + +# Make a file of imagename:tag +images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:) + +# Ensure output files exist +touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt + +# If Trivy detects no vulnerabilities, add the image name to clean-images.txt. +# If there are vulnerabilities detected, add it to dirty-images.txt and +# generate a csv summary +for image in $images; do + filename=$(basename $image | sed 's/:/\./g') + if $(trivy image \ + --quiet \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --output image-scan-output/${filename}.json \ + --ignore-unfixed \ + $image); then + # Clean up the output file for any images with no vulnerabilities + rm -f image-scan-output/${filename}.json + + # Add the image to the clean list + echo "${image}" >> image-scan-output/clean-images.txt + else + # Add the image to the dirty list + echo "${image}" >> image-scan-output/dirty-images.txt + + # Write a header for the summary CSV + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv + + # Write the summary CSV data + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + # Ignore packages with "kernel" in the PkgName + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv + fi +done