diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 285fd3b5b..67e9eda0c 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -38,7 +38,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT # Generate a tag to apply to all built overcloud host images. - name: Generate overcloud host image tag diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml index 449068f6f..a8a64f60f 100644 --- a/.github/workflows/overcloud-host-image-promote.yml +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -31,7 +31,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT working-directory: src/kayobe-config - name: Clone StackHPC Kayobe repository diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml index bb23b8ffe..b08f57381 100644 --- a/.github/workflows/overcloud-host-image-upload.yml +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -47,7 +47,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index d0da0c051..a769aa718 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -30,7 +30,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT - name: Install OpenStack client run: | diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 6a8055ded..4478e6fe5 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -33,6 +33,12 @@ on: type: boolean required: false default: true + push-dirty: + description: Push scanned images that have vulnerabilities? + type: boolean + required: false + # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures + default: true env: ANSIBLE_FORCE_COLOR: True @@ -54,7 +60,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT # Generate a tag to apply to all built container images. # Without this, each kayobe * container image build command would use a different tag. @@ -100,7 +106,15 @@ jobs: - name: Install package dependencies run: | sudo apt update - sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget + + - name: Install gh + run: | + sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y - name: Checkout uses: actions/checkout@v4 @@ -118,6 +132,10 @@ jobs: run: | docker ps + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0 + - name: Install Kayobe run: | mkdir -p venvs && @@ -132,6 +150,10 @@ jobs: - name: Install Docker Python SDK run: | sudo pip install docker + + - name: Get Kolla tag + id: write-kolla-tag + run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}" >> $GITHUB_OUTPUT - name: Configure localhost as a seed run: | @@ -153,67 +175,124 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - name: Build and push kolla overcloud images + - name: Create build logs output directory + run: mkdir image-build-logs + + - name: Build kolla overcloud images + id: build_overcloud_images + continue-on-error: true run: | - args="${{ github.event.inputs.regexes }}" + args="${{ inputs.regexes }}" args="$args -e kolla_base_distro=${{ matrix.distro }}" - args="$args -e kolla_tag=$KOLLA_TAG" + args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}" args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" - if ${{ inputs.push }} == 'true'; then - args="$args --push" - fi source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe overcloud container image build $args env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - KOLLA_TAG: "${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}" - if: github.event.inputs.overcloud == 'true' + if: inputs.overcloud + + - name: Copy overcloud container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log + if: inputs.overcloud - - name: Build and push kolla seed images + - name: Build kolla seed images + id: build_seed_images + continue-on-error: true run: | args="-e kolla_base_distro=${{ matrix.distro }}" - args="$args -e kolla_tag=$KOLLA_TAG" + args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}" args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" - if ${{ inputs.push }} == 'true'; then - args="$args --push" - fi source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe seed container image build $args env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - KOLLA_TAG: "${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}" - if: github.event.inputs.seed == 'true' + if: inputs.seed + + - name: Copy seed container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log + if: inputs.seed - name: Get built container images - run: | - docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:*${{ matrix.distro }}*${{ needs.generate-tag.outputs.datetime_tag }}" > ${{ matrix.distro }}-container-images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images - name: Fail if no images have been built run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi - - name: Upload container images artifact + - name: Scan built container images + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} + + - name: Move image scan logs to output artifact + run: mv image-scan-output image-build-logs/image-scan-output + + - name: Fail if no images have passed scanning + run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi + if: ${{ !inputs.push-dirty }} + + - name: Copy clean images to push-attempt-images list + run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt + if: inputs.push + + - name: Append dirty images to push list + run: | + cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + if: ${{ inputs.push && inputs.push-dirty }} + + - name: Push images + run: | + touch image-build-logs/push-failed-images.txt + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml && + + while read -r image; do + # Retries! + for i in {1..5}; do + if docker push $image; then + echo "Pushed $image" + break + elif $i == 5; then + echo "Failed to push $image" + echo $image >> image-build-logs/push-failed-images.txt + else + echo "Failed on retry $i" + sleep 5 + fi; + done + done < image-build-logs/push-attempt-images.txt + shell: bash + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.push + + - name: Upload output artifact uses: actions/upload-artifact@v4 with: - name: ${{ matrix.distro }} container images - path: ${{ matrix.distro }}-container-images + name: ${{ matrix.distro }}-logs + path: image-build-logs retention-days: 7 + if: ${{ !cancelled() }} + + - name: Fail when images failed to build + run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1 + if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }} + + - name: Fail when images failed to push + run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi + if: ${{ !cancelled() }} + + - name: Fail when images failed scanning + run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi + if: ${{ !inputs.push-dirty && !cancelled() }} - sync-container-repositories: - name: Trigger container image repository sync - needs: - - container-image-build - if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push - runs-on: ubuntu-latest - permissions: {} - steps: # NOTE(mgoddard): Trigger another CI workflow in the # stackhpc-release-train repository. - name: Trigger container image repository sync run: | filter='${{ inputs.regexes }}' - if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then + if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then filter="$filter bifrost" fi gh workflow run \ @@ -224,7 +303,9 @@ jobs: -f sync-old-images=false env: GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} - name: Display link to container image repository sync workflows run: | echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml" + if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index 0c3fb9457..d6bbaca8c 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -296,7 +296,9 @@ Configure Barbican [vault_plugin] vault_url = https://{{ kolla_internal_vip_address }}:8200 use_ssl = True - ssl_ca_crt_file = {% raw %}{{ openstack_cacert }}{% endraw %} + {% raw %} + ssl_ca_crt_file = {{ openstack_cacert }} + {% endraw %} approle_role_id = {{ secrets_barbican_approle_role_id }} approle_secret_id = {{ secrets_barbican_approle_secret_id }} kv_mountpoint = barbican diff --git a/doc/source/operations/tempest.rst b/doc/source/operations/tempest.rst index a5991097c..5f6fdb71f 100644 --- a/doc/source/operations/tempest.rst +++ b/doc/source/operations/tempest.rst @@ -277,7 +277,10 @@ command from the base of the ``kayobe-config`` directory: .. code-block:: bash - sudo -E docker run --detach -it --rm --network host -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + sudo -E docker run --name kayobe-automation --detach -it --rm --network host \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack By default, ``no_log`` is set to stop credentials from leaking. This can be disabled by adding ``-e rally_no_sensitive_log=false`` to the end. diff --git a/etc/kayobe/ansible/docker-registry-login.yml b/etc/kayobe/ansible/docker-registry-login.yml new file mode 100644 index 000000000..39ad03600 --- /dev/null +++ b/etc/kayobe/ansible/docker-registry-login.yml @@ -0,0 +1,11 @@ +--- +- name: Login to docker registry + gather_facts: false + hosts: container-image-builders + tasks: + - name: Login to docker registry + docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: yes diff --git a/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml b/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml index 36566b6a3..9ba469ce7 100644 --- a/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml +++ b/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml @@ -21,22 +21,25 @@ - name: Find OVN DB DB Leader hosts: "{{ ovn_nb_db_group | default('controllers') }}" tasks: - - name: Find the OVN NB DB leader - ansible.builtin.command: docker exec ovn_nb_db ovn-nbctl get-connection - changed_when: false - failed_when: false - register: ovn_check_result - check_mode: false + - name: Find OVN DB Leader + when: kolla_enable_ovn | bool + block: + - name: Find the OVN NB DB leader + ansible.builtin.command: docker exec ovn_nb_db ovn-nbctl get-connection + changed_when: false + failed_when: false + register: ovn_check_result + check_mode: false - - name: Group hosts by leader/follower role - ansible.builtin.group_by: - key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}" - changed_when: false + - name: Group hosts by leader/follower role + ansible.builtin.group_by: + key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}" + changed_when: false - - name: Assert one leader exists - ansible.builtin.assert: - that: - - groups['ovn_nb_leader'] | default([]) | length == 1 + - name: Assert one leader exists + ansible.builtin.assert: + that: + - groups['ovn_nb_leader'] | default([]) | length == 1 - name: Fix OVN chassis priorities hosts: ovn_nb_leader diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 3b477731c..928e1c52d 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -104,7 +104,3 @@ that: - ansible_facts.distribution_major_version == '22' - ansible_facts.distribution_release == 'jammy' - -- name: Run the OVN chassis priority fix playbook - import_playbook: "{{ lookup('ansible.builtin.env', 'KAYOBE_CONFIG_PATH') }}/ansible/ovn-fix-chassis-priorities.yml" - when: kolla_enable_ovn diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index a9eba19dd..d15d18613 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -116,6 +116,9 @@ # Ironic inspector deployment ramdisk location. #kolla_bifrost_inspector_deploy_ramdisk: +# Ironic inspector legacy deployment kernel location. +#kolla_bifrost_inspector_legacy_deploy_kernel: + # Timeout of hardware inspection on overcloud nodes, in seconds. Default is # {{ inspector_inspection_timeout }}. #kolla_bifrost_inspection_timeout: diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index b3940f649..75ff73c5f 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -63,15 +63,15 @@ ############################################################################### # Compute node LVM configuration. -# List of compute volume groups. See mrlesmithjr.manage-lvm role for +# List of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups: -# Default list of compute volume groups. See mrlesmithjr.manage-lvm role for +# Default list of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups_default: -# Additional list of compute volume groups. See mrlesmithjr.manage-lvm role +# Additional list of compute volume groups. See mrlesmithjr.manage_lvm role # for format. #compute_lvm_groups_extra: @@ -82,7 +82,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #compute_lvm_group_data_enabled: -# Compute LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Compute LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_group_data: diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml index fec1abb94..967c4e37d 100644 --- a/etc/kayobe/containers/pulp/post.yml +++ b/etc/kayobe/containers/pulp/post.yml @@ -27,3 +27,10 @@ when: - stackhpc_pulp_sync_for_local_container_build | bool - pulp_settings.changed + +- name: Login to docker registry + docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: yes diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index d51da1487..cdb26a592 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -72,15 +72,15 @@ ############################################################################### # Controller node LVM configuration. -# List of controller volume groups. See mrlesmithjr.manage-lvm role for +# List of controller volume groups. See mrlesmithjr.manage_lvm role for # format. #controller_lvm_groups: -# Default list of controller volume groups. See mrlesmithjr.manage-lvm role for +# Default list of controller volume groups. See mrlesmithjr.manage_lvm role for # format. #controller_lvm_groups_default: -# Additional list of controller volume groups. See mrlesmithjr.manage-lvm role +# Additional list of controller volume groups. See mrlesmithjr.manage_lvm role # for format. #controller_lvm_groups_extra: @@ -91,7 +91,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #controller_lvm_group_data_enabled: -# Controller LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Controller LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #controller_lvm_group_data: diff --git a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh index 0641a400e..4edcd01a4 100755 --- a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh +++ b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh @@ -87,7 +87,7 @@ kayobe seed vm provision kayobe seed host configure # Deploy local pulp server as a container on the seed VM -kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none -e deploy_containers_registry_attempt_login=False +kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none # Deploying the seed restarts networking interface, run configure-local-networking.sh again to re-add routes. $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh diff --git a/etc/kayobe/environments/aufn-ceph/tenks.yml b/etc/kayobe/environments/aufn-ceph/tenks.yml index 9b0e9e9f4..25eac0374 100644 --- a/etc/kayobe/environments/aufn-ceph/tenks.yml +++ b/etc/kayobe/environments/aufn-ceph/tenks.yml @@ -87,3 +87,9 @@ bridge_type: linuxbridge # No placement service. wait_for_placement: false + +# NOTE(priteau): Disable libvirt_vm_trust_guest_rx_filters, which when enabled +# triggers the following errors when booting baremetal instances with Tenks on +# Libvirt 9: Cannot set interface flags on 'macvtap1': Value too large for +# defined data type +libvirt_vm_trust_guest_rx_filters: false diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 3956dca36..50af1cb14 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -4,8 +4,7 @@ ############################################################################### # Local path configuration (Ansible control host). -# Path to Kayobe configuration directory on Ansible control host, with an -# environment path appended if kayobe_environment is set. +# Path to Kayobe configuration directory on Ansible control host. #kayobe_config_path: # Name of Kayobe environment to use. Default is $KAYOBE_ENVIRONMENT, or an diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index d89e3653f..4fb76ddf7 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -92,15 +92,15 @@ ############################################################################### # Infrastructure VM node LVM configuration. -# List of infrastructure vm volume groups. See mrlesmithjr.manage-lvm role for +# List of infrastructure vm volume groups. See mrlesmithjr.manage_lvm role for # format. #infra_vm_lvm_groups: -# Default list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# Default list of infrastructure vm volume groups. See mrlesmithjr.manage_lvm # role for format. #infra_vm_lvm_groups_default: -# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage_lvm # role for format. #infra_vm_lvm_groups_extra: @@ -111,7 +111,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #infra_vm_lvm_group_data_enabled: -# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage-lvm role +# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage_lvm role # for format. #infra_vm_lvm_group_data: diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index e61a8de3b..20bc11432 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -680,6 +680,10 @@ kolla_enable_prometheus: true # Kolla passwords file. #kolla_ansible_default_custom_passwords: +# Dictionary containing extra custom passwords to add or override in the Kolla +# passwords file. +#kolla_ansible_extra_custom_passwords: + # Dictionary containing custom passwords to add or override in the Kolla # passwords file. #kolla_ansible_custom_passwords: @@ -719,7 +723,7 @@ kolla_enable_prometheus: true # Path to a CA certificate file to use for the OS_CACERT environment variable # in public-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_external_fqdn_cacert: +#kolla_public_openrc_cacert: # Internal API certificate bundle. # @@ -732,7 +736,7 @@ kolla_enable_prometheus: true # Path to a CA certificate file to use for the OS_CACERT environment variable # in admin-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_internal_fqdn_cacert: +#kolla_admin_openrc_cacert: ############################################################################### # Proxy configuration diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json index e041d8ff0..e5258168a 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json @@ -1924,23 +1924,25 @@ } ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "ceph_cluster_total_objects", + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pool_objects * on(pool_id) group_left(instance,name) ceph_pool_metadata", "format": "time_series", "interval": "$interval", "intervalFactor": 1, - "legendFormat": "Total", + "legendFormat": "{{name}}", + "range": true, "refId": "A", "step": 300 } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Objects in the Cluster", "tooltip": { "msResolution": false, diff --git a/etc/kayobe/monitoring.yml b/etc/kayobe/monitoring.yml index 34ccf02e1..7b555bc82 100644 --- a/etc/kayobe/monitoring.yml +++ b/etc/kayobe/monitoring.yml @@ -63,15 +63,15 @@ ############################################################################### # Monitoring node LVM configuration. -# List of monitoring node volume groups. See mrlesmithjr.manage-lvm role for +# List of monitoring node volume groups. See mrlesmithjr.manage_lvm role for # format. #monitoring_lvm_groups: -# Default list of monitoring node volume groups. See mrlesmithjr.manage-lvm +# Default list of monitoring node volume groups. See mrlesmithjr.manage_lvm # role for format. #monitoring_lvm_groups_default: -# Additional list of monitoring node volume groups. See mrlesmithjr.manage-lvm +# Additional list of monitoring node volume groups. See mrlesmithjr.manage_lvm # role for format. #monitoring_lvm_groups_extra: diff --git a/etc/kayobe/seed-hypervisor.yml b/etc/kayobe/seed-hypervisor.yml index ba2d413b6..0a98f421a 100644 --- a/etc/kayobe/seed-hypervisor.yml +++ b/etc/kayobe/seed-hypervisor.yml @@ -36,7 +36,7 @@ ############################################################################### # Seed hypervisor node LVM configuration. -# List of seed hypervisor volume groups. See mrlesmithjr.manage-lvm role for +# List of seed hypervisor volume groups. See mrlesmithjr.manage_lvm role for # format. Set to "{{ seed_hypervisor_lvm_groups_with_data }}" to create a # volume group for libvirt storage. #seed_hypervisor_lvm_groups: @@ -45,7 +45,7 @@ # default. #seed_hypervisor_lvm_groups_with_data: -# Seed LVM volume group for data. See mrlesmithjr.manage-lvm role for format. +# Seed LVM volume group for data. See mrlesmithjr.manage_lvm role for format. #seed_hypervisor_lvm_group_data: # List of disks for use by seed hypervisor LVM data volume group. Default to an diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 5d0d817e8..2303006e3 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -36,14 +36,14 @@ ############################################################################### # Seed node LVM configuration. -# List of seed volume groups. See mrlesmithjr.manage-lvm role for format. +# List of seed volume groups. See mrlesmithjr.manage_lvm role for format. #seed_lvm_groups: -# Default list of seed volume groups. See mrlesmithjr.manage-lvm role for +# Default list of seed volume groups. See mrlesmithjr.manage_lvm role for # format. #seed_lvm_groups_default: -# Additional list of seed volume groups. See mrlesmithjr.manage-lvm role for +# Additional list of seed volume groups. See mrlesmithjr.manage_lvm role for # format. #seed_lvm_groups_extra: @@ -54,7 +54,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #seed_lvm_group_data_enabled: -# Seed LVM volume group for data. See mrlesmithjr.manage-lvm role for format. +# Seed LVM volume group for data. See mrlesmithjr.manage_lvm role for format. #seed_lvm_group_data: # List of disks for use by seed LVM data volume group. Default to an invalid @@ -152,6 +152,10 @@ seed_containers: >- seed_extra_containers: {} +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +seed_deploy_containers_registry_attempt_login: "{{ not seed_pulp_container_enabled | bool }}" + ############################################################################### # Seed node firewalld configuration. diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index 8270dba2f..32a7382aa 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -68,15 +68,15 @@ ############################################################################### # Storage node LVM configuration. -# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# List of storage volume groups. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_groups: -# Default list of storage volume groups. See mrlesmithjr.manage-lvm role for +# Default list of storage volume groups. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_groups_default: -# Additional list of storage volume groups. See mrlesmithjr.manage-lvm role +# Additional list of storage volume groups. See mrlesmithjr.manage_lvm role # for format. #storage_lvm_groups_extra: @@ -87,7 +87,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #storage_lvm_group_data_enabled: -# Storage LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Storage LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_group_data: diff --git a/kayobe-env b/kayobe-env index 5137927e5..28b1cccdb 100644 --- a/kayobe-env +++ b/kayobe-env @@ -30,8 +30,8 @@ export KOLLA_CONFIG_PATH=$KAYOBE_CONFIG_ROOT/etc/kolla # kayobe/ # kolla-ansible/ base_path=$(realpath $KAYOBE_CONFIG_ROOT/../../) -export KOLLA_SOURCE_PATH=${KOLLA_SOURCE_PATH:-${base_path}/src/kolla-ansible} -export KOLLA_VENV_PATH=${KOLLA_VENV_PATH:-${base_path}/venvs/kolla-ansible} +export KOLLA_SOURCE_PATH=${base_path}/src/kolla-ansible +export KOLLA_VENV_PATH=${base_path}/venvs/kolla-ansible function check_and_export_env { # Look for existing Kayobe environments diff --git a/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml new file mode 100644 index 000000000..67a99f9c2 --- /dev/null +++ b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Kolla container images created using the + ``stackhpc-container-image-build.yml`` workflow are now automatically + scanned for vulnerablilities. diff --git a/requirements.txt b/requirements.txt index 55300a079..7986a3c0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/zed -ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc-py39 +ansible-modules-hashivault>=5.2.1 jmespath diff --git a/tools/scan-images.sh b/tools/scan-images.sh new file mode 100755 index 000000000..74223ad90 --- /dev/null +++ b/tools/scan-images.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Check correct usage +if [[ ! $2 ]]; then + echo "Usage: scan-images.sh " + exit 2 +fi + +set -u + +# Check that trivy is installed +if ! trivy --version; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1' +fi + +# Clear any previous outputs +rm -rf image-scan-output + +# Make a fresh output directory +mkdir -p image-scan-output + +# Get built container images +docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt + +# Make a file of imagename:tag +images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:) + +# Ensure output files exist +touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt + +# If Trivy detects no vulnerabilities, add the image name to clean-images.txt. +# If there are vulnerabilities detected, add it to dirty-images.txt and +# generate a csv summary +for image in $images; do + filename=$(basename $image | sed 's/:/\./g') + if $(trivy image \ + --quiet \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --output image-scan-output/${filename}.json \ + --ignore-unfixed \ + $image); then + # Clean up the output file for any images with no vulnerabilities + rm -f image-scan-output/${filename}.json + + # Add the image to the clean list + echo "${image}" >> image-scan-output/clean-images.txt + else + # Add the image to the dirty list + echo "${image}" >> image-scan-output/dirty-images.txt + + # Write a header for the summary CSV + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv + + # Write the summary CSV data + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + # Ignore packages with "kernel" in the PkgName + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv + fi +done diff --git a/tools/ubuntu-upgrade-overcloud.sh b/tools/ubuntu-upgrade-overcloud.sh index 3e351d6d6..50959c263 100755 --- a/tools/ubuntu-upgrade-overcloud.sh +++ b/tools/ubuntu-upgrade-overcloud.sh @@ -31,4 +31,6 @@ set -x kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ovn-fix-chassis-priorities.yml + kayobe overcloud host configure --limit $1 --kolla-limit $1 -e os_release=jammy