From 9eab93aff179d4a678b4470cb4d3ab1916907014 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 25 Jan 2024 15:45:29 +0000 Subject: [PATCH 01/10] Update RL9 host image to 9.3 --- etc/kayobe/pulp-host-image-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index 1c421a52a..15ee74851 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -3,6 +3,6 @@ # These images must be in SMS, since they are used by our AIO CI runners stackhpc_centos_8_stream_overcloud_host_image_version: "yoga-20230525T095243" stackhpc_rocky_8_overcloud_host_image_version: "yoga-20230629T135322" -stackhpc_rocky_9_overcloud_host_image_version: "yoga-20231026T081556" +stackhpc_rocky_9_overcloud_host_image_version: "yoga-20240124T094316" stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20230609T120720" stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20231012T121552" From 845328a25a542a89aa43ae2d1f054d6e0a614a22 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 29 Jan 2024 14:32:27 +0000 Subject: [PATCH 02/10] Update checkout github action to v4 --- .github/workflows/overcloud-host-image-build.yml | 4 ++-- .github/workflows/overcloud-host-image-promote.yml | 4 ++-- .github/workflows/stackhpc-all-in-one.yml | 2 +- .github/workflows/stackhpc-build-kayobe-image.yml | 2 +- .github/workflows/stackhpc-container-image-build.yml | 6 +++--- .github/workflows/stackhpc-pull-request.yml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 27583ce9c..00952f2cb 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -42,7 +42,7 @@ jobs: runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] permissions: {} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: src/kayobe-config @@ -67,7 +67,7 @@ jobs: rm -f /tmp/updated_images.txt - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: stackhpc/kayobe ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml index eeced2ce6..2fd5e963c 100644 --- a/.github/workflows/overcloud-host-image-promote.yml +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -35,7 +35,7 @@ jobs: if: github.repository == 'stackhpc/stackhpc-kayobe-config' runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: src/kayobe-config @@ -46,7 +46,7 @@ jobs: echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: stackhpc/kayobe ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 7a0102ef3..f9554c31d 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -77,7 +77,7 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} KAYOBE_IMAGE: ${{ inputs.kayobe_image }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml index 323ba6993..0fbc5c7ce 100644 --- a/.github/workflows/stackhpc-build-kayobe-image.yml +++ b/.github/workflows/stackhpc-build-kayobe-image.yml @@ -47,7 +47,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout kayobe config - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index eb00fd33b..d899f20b3 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -53,7 +53,7 @@ jobs: openstack_release: ${{ steps.openstack_release.outputs.openstack_release }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Determine OpenStack release id: openstack_release @@ -106,12 +106,12 @@ jobs: needs: - generate-tag steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: src/kayobe-config - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: stackhpc/kayobe ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }} diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index b1e5695d1..2d71792a1 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -22,7 +22,7 @@ jobs: aio: ${{ steps.changes.outputs.aio }} steps: - name: GitHub Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check changed files uses: dorny/paths-filter@v2 @@ -47,7 +47,7 @@ jobs: if: github.repository == 'stackhpc/stackhpc-kayobe-config' steps: - name: GitHub Checkout 🛎 - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup Python ${{ matrix.python-version }} 🐍 From e567738829f1871602fd8f59c59f2da00a49455c Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 1 Feb 2024 13:58:52 +0000 Subject: [PATCH 03/10] fix wazuh docs formatting --- doc/source/configuration/wazuh.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/configuration/wazuh.rst b/doc/source/configuration/wazuh.rst index db9f9a37c..683d066db 100644 --- a/doc/source/configuration/wazuh.rst +++ b/doc/source/configuration/wazuh.rst @@ -226,11 +226,12 @@ You may need to modify some of the variables, including: .. note:: - NOTE: If you are using multiple environments, and you need to customise Wazuh in each environment, create override files in an appropriate directory, - for example `etc/kayobe/environments/production/inventory/group_vars/` + for example ``etc/kayobe/environments/production/inventory/group_vars/``. + Files which values can be overridden (in the context of Wazuh): + - etc/kayobe/inventory/group_vars/wazuh/wazuh-manager/wazuh-manager - etc/kayobe/wazuh-manager.yml - etc/kayobe/inventory/group_vars/wazuh/wazuh-agent/wazuh-agent From 39248f81ce6d2b9522cedac3173cdeaf6653e38d Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 1 Feb 2024 13:59:45 +0000 Subject: [PATCH 04/10] add wazuh keypaths for multiple environments to docs --- doc/source/configuration/wazuh.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/wazuh.rst b/doc/source/configuration/wazuh.rst index 683d066db..d1391db63 100644 --- a/doc/source/configuration/wazuh.rst +++ b/doc/source/configuration/wazuh.rst @@ -231,7 +231,7 @@ You may need to modify some of the variables, including: for example ``etc/kayobe/environments/production/inventory/group_vars/``. Files which values can be overridden (in the context of Wazuh): - + - etc/kayobe/inventory/group_vars/wazuh/wazuh-manager/wazuh-manager - etc/kayobe/wazuh-manager.yml - etc/kayobe/inventory/group_vars/wazuh/wazuh-agent/wazuh-agent @@ -317,6 +317,9 @@ If you are using the wazuh generated certificates, this will result in the creation of some certificates and keys (in case of custom certs adjust path to it). Encrypt the keys (and remember to commit to git): +``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/*.key`` + +if using the kayobe environments feature, otherwise: ``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key`` From 8ed7c54fe60047517a8a548b438ceca63fbf466f Mon Sep 17 00:00:00 2001 From: Steve Brasier <33413598+sjpb@users.noreply.github.com> Date: Thu, 1 Feb 2024 14:19:28 +0000 Subject: [PATCH 05/10] docs format nit Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> --- doc/source/configuration/wazuh.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/wazuh.rst b/doc/source/configuration/wazuh.rst index d1391db63..deceff077 100644 --- a/doc/source/configuration/wazuh.rst +++ b/doc/source/configuration/wazuh.rst @@ -319,7 +319,7 @@ Encrypt the keys (and remember to commit to git): ``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/*.key`` -if using the kayobe environments feature, otherwise: +If using the kayobe environments feature, otherwise: ``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key`` From f19ecd7717ea407666b753b83624cf032fb7b598 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 7 Feb 2024 10:50:43 +0000 Subject: [PATCH 06/10] Switch to github hosted runners for host image promote --- .github/workflows/overcloud-host-image-promote.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml index 2fd5e963c..a9b558258 100644 --- a/.github/workflows/overcloud-host-image-promote.yml +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -33,7 +33,7 @@ jobs: overcloud-host-image-promote: name: Promote overcloud host image if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: From 6c83584a0c48c01a44c4511ddc4efb1e2ffbf8d9 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 7 Feb 2024 11:20:22 +0000 Subject: [PATCH 07/10] Fix release detection step --- .github/workflows/overcloud-host-image-promote.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml index a9b558258..1bd777c8c 100644 --- a/.github/workflows/overcloud-host-image-promote.yml +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -44,6 +44,7 @@ jobs: run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + working-directory: src/kayobe-config - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 From 6302be723747a7e9e31ca0fb6077c5ce0078e70c Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 7 Feb 2024 10:16:00 +0000 Subject: [PATCH 08/10] Add workflow to upload Ark host images to glance --- .../workflows/overcloud-host-image-upload.yml | 260 ++++++++++++++++++ .../ansible/pulp-host-image-download.yml | 38 +++ 2 files changed, 298 insertions(+) create mode 100644 .github/workflows/overcloud-host-image-upload.yml create mode 100644 etc/kayobe/ansible/pulp-host-image-download.yml diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml new file mode 100644 index 000000000..b7f5a458e --- /dev/null +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -0,0 +1,260 @@ +--- +name: Upload overcloud host images +on: + workflow_dispatch: + inputs: + centos: + description: Upload CentOS Stream 8 + type: boolean + default: true + rocky8: + description: Upload Rocky Linux 8 + type: boolean + default: true + rocky9: + description: Upload Rocky Linux 9 + type: boolean + default: true + ubuntu-focal: + description: Upload Ubuntu 20.04 Focal + type: boolean + default: true + ubuntu-jammy: + description: Upload Ubuntu 22.04 Jammy + type: boolean + default: true + kayobe-environment: + description: Kayobe environment to use + type: string + default: "ci-builder" + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + overcloud-host-image-upload: + name: Upload overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: arc-skc-host-image-builder-runner + permissions: {} + steps: + - name: Install package dependencies + run: | + sudo apt update + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + + - uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + + - name: Clone StackHPC Kayobe repository + uses: actions/checkout@v4 + with: + repository: stackhpc/kayobe + ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} + path: src/kayobe + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install ../src/kayobe + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe control host bootstrap --skip-tags bootstrap + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + + - name: Install OpenStack client + run: | + source venvs/kayobe/bin/activate && + pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt + + - name: Download CentOS Stream 8 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="centos" \ + -e os_release="8-stream" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.centos + + - name: Output CentOS Stream 8 image tag + id: centos_8_stream_image_tag + run: | + echo image_tag=$(grep stackhpc_centos_8_stream_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload CentOS Stream 8 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-centos-8-stream-${{ steps.centos_8_stream_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/centos-8-stream.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.centos + + - name: Download Rocky Linux 8 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="rocky" \ + -e os_release="8" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky8 + + - name: Output Rocky Linux 8 image tag + id: rocky_8_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_8_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Rocky Linux 8 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-rocky-8-${{ steps.rocky_8_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/rocky-8.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky8 + + - name: Download Rocky Linux 9 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="rocky" \ + -e os_release="9" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky9 + + - name: Output Rocky Linux 9 image tag + id: rocky_9_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Rocky Linux 9 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/rocky-9.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky9 + + - name: Download Ubuntu Focal 20.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="ubuntu" \ + -e os_release="focal" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-focal + + - name: Output Ubuntu Focal image tag + id: ubuntu_focal_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_focal_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Ubuntu Focal 20.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-focal-${{ steps.ubuntu_focal_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-focal.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-focal + + - name: Download Ubuntu Jammy 22.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-jammy + + - name: Output Ubuntu Jammy image tag + id: ubuntu_jammy_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-jammy.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-jammy diff --git a/etc/kayobe/ansible/pulp-host-image-download.yml b/etc/kayobe/ansible/pulp-host-image-download.yml new file mode 100644 index 000000000..6b3494dfe --- /dev/null +++ b/etc/kayobe/ansible/pulp-host-image-download.yml @@ -0,0 +1,38 @@ +--- +- name: Download an overcloud host image from Ark + hosts: localhost + vars: + # This var is an edited version of stackhpc_overcloud_host_image_url + # without the auth credentials in it. Auth is handled by username and + # password in the get_url task of this playbook + stackhpc_overcloud_host_image_url_no_auth: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/\ + {{ openstack_release }}/{{ os_distribution }}/{{ os_release }}/\ + {{ 'ofed/' if stackhpc_overcloud_host_image_is_ofed else '' }}\ + {{ stackhpc_overcloud_host_image_version }}/\ + overcloud-{{ os_distribution }}-{{ os_release }}\ + {{ '-ofed' if stackhpc_overcloud_host_image_is_ofed else '' }}.qcow2" + + tasks: + - name: Print image information + debug: + msg: | + OS Distribution: {{ os_distribution }} + OS Release: {{ os_release }} + Image tag: {{ stackhpc_overcloud_host_image_version }} + OFED: {{ stackhpc_overcloud_host_image_is_ofed }} + + # TODO: Add checksum support + - name: Download image artifact + get_url: + url: "{{ stackhpc_overcloud_host_image_url_no_auth }}" + username: "{{ stackhpc_image_repository_username }}" + password: "{{ stackhpc_image_repository_password }}" + force_basic_auth: true + unredirected_headers: + - "Authorization" + dest: "/tmp/{{ os_distribution }}-{{ os_release }}.qcow2" + mode: "0644" + register: image_download_result + until: image_download_result.status_code == 200 + retries: 3 + delay: 60 From 25ebb3ff12e00489cde70b7e3fdb543f0e9f3b64 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 7 Feb 2024 15:39:24 +0000 Subject: [PATCH 09/10] Only upload host images when required --- .../workflows/overcloud-host-image-upload.yml | 130 +++++++++++++----- 1 file changed, 95 insertions(+), 35 deletions(-) diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml index b7f5a458e..f18f8ef4c 100644 --- a/.github/workflows/overcloud-host-image-upload.yml +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -94,6 +94,23 @@ jobs: source venvs/kayobe/bin/activate && pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt + - name: Output CentOS Stream 8 image tag + id: centos_8_stream_image_tag + run: | + echo image_tag=$(grep stackhpc_centos_8_stream_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: centos_8_stream_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-centos-8-stream-${{ steps.centos_8_stream_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true + - name: Download CentOS Stream 8 overcloud host image from Ark run: | source venvs/kayobe/bin/activate && @@ -104,12 +121,7 @@ jobs: -e os_release="8-stream" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.centos - - - name: Output CentOS Stream 8 image tag - id: centos_8_stream_image_tag - run: | - echo image_tag=$(grep stackhpc_centos_8_stream_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + if: inputs.centos && steps.centos_8_stream_image_exists.outcome == 'failure' - name: Upload CentOS Stream 8 overcloud host image to Cloud run: | @@ -125,7 +137,24 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.centos + if: inputs.centos && steps.centos_8_stream_image_exists.outcome == 'failure' + + - name: Output Rocky Linux 8 image tag + id: rocky_8_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_8_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: rocky_8_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-rocky-8-${{ steps.rocky_8_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true - name: Download Rocky Linux 8 overcloud host image from Ark run: | @@ -137,12 +166,7 @@ jobs: -e os_release="8" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.rocky8 - - - name: Output Rocky Linux 8 image tag - id: rocky_8_image_tag - run: | - echo image_tag=$(grep stackhpc_rocky_8_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + if: inputs.rocky8 && steps.rocky_8_image_exists.outcome == 'failure' - name: Upload Rocky Linux 8 overcloud host image to Cloud run: | @@ -158,7 +182,24 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.rocky8 + if: inputs.rocky8 && steps.rocky_8_image_exists.outcome == 'failure' + + - name: Output Rocky Linux 9 image tag + id: rocky_9_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: rocky_9_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true - name: Download Rocky Linux 9 overcloud host image from Ark run: | @@ -170,12 +211,7 @@ jobs: -e os_release="9" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.rocky9 - - - name: Output Rocky Linux 9 image tag - id: rocky_9_image_tag - run: | - echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' - name: Upload Rocky Linux 9 overcloud host image to Cloud run: | @@ -191,7 +227,24 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.rocky9 + if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' + + - name: Output Ubuntu Focal image tag + id: ubuntu_focal_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_focal_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: ubuntu_focal_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-ubuntu-focal-${{ steps.ubuntu_focal_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true - name: Download Ubuntu Focal 20.04 overcloud host image from Ark run: | @@ -203,12 +256,7 @@ jobs: -e os_release="focal" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.ubuntu-focal - - - name: Output Ubuntu Focal image tag - id: ubuntu_focal_image_tag - run: | - echo image_tag=$(grep stackhpc_ubuntu_focal_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + if: inputs.ubuntu-focal && steps.ubuntu_focal_image_exists.outcome == 'failure' - name: Upload Ubuntu Focal 20.04 overcloud host image to Cloud run: | @@ -224,7 +272,24 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.ubuntu-focal + if: inputs.ubuntu-focal && steps.ubuntu_focal_image_exists.outcome == 'failure' + + - name: Output Ubuntu Jammy image tag + id: ubuntu_jammy_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: ubuntu_jammy_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true - name: Download Ubuntu Jammy 22.04 overcloud host image from Ark run: | @@ -236,12 +301,7 @@ jobs: -e os_release="jammy" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.ubuntu-jammy - - - name: Output Ubuntu Jammy image tag - id: ubuntu_jammy_image_tag - run: | - echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud run: | @@ -257,4 +317,4 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.ubuntu-jammy + if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' From a88f4ada0d6f13616a8800efaa297d720be172e7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 7 Feb 2024 16:03:20 +0000 Subject: [PATCH 10/10] CI: Drop unsupported distributions from overcloud host image upload workflow --- .../workflows/overcloud-host-image-upload.yml | 147 ------------------ 1 file changed, 147 deletions(-) diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml index 38d643364..e1468df87 100644 --- a/.github/workflows/overcloud-host-image-upload.yml +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -3,22 +3,10 @@ name: Upload overcloud host images on: workflow_dispatch: inputs: - centos: - description: Upload CentOS Stream 8 - type: boolean - default: true - rocky8: - description: Upload Rocky Linux 8 - type: boolean - default: true rocky9: description: Upload Rocky Linux 9 type: boolean default: true - ubuntu-focal: - description: Upload Ubuntu 20.04 Focal - type: boolean - default: true ubuntu-jammy: description: Upload Ubuntu 22.04 Jammy type: boolean @@ -94,96 +82,6 @@ jobs: source venvs/kayobe/bin/activate && pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/zed/upper-constraints.txt - - name: Output CentOS Stream 8 image tag - id: centos_8_stream_image_tag - run: | - echo image_tag=$(grep stackhpc_centos_8_stream_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT - - - name: Check if image exists already - id: centos_8_stream_image_exists - run: | - source venvs/kayobe/bin/activate && - openstack image show \ - overcloud-centos-8-stream-${{ steps.centos_8_stream_image_tag.outputs.image_tag }} - env: - OS_CLOUD: openstack - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - continue-on-error: true - - - name: Download CentOS Stream 8 overcloud host image from Ark - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ - -e os_distribution="centos" \ - -e os_release="8-stream" - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.centos && steps.centos_8_stream_image_exists.outcome == 'failure' - - - name: Upload CentOS Stream 8 overcloud host image to Cloud - run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-centos-8-stream-${{ steps.centos_8_stream_image_tag.outputs.image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /tmp/centos-8-stream.qcow2 \ - --private \ - --progress - env: - OS_CLOUD: openstack - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.centos && steps.centos_8_stream_image_exists.outcome == 'failure' - - - name: Output Rocky Linux 8 image tag - id: rocky_8_image_tag - run: | - echo image_tag=$(grep stackhpc_rocky_8_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT - - - name: Check if image exists already - id: rocky_8_image_exists - run: | - source venvs/kayobe/bin/activate && - openstack image show \ - overcloud-rocky-8-${{ steps.rocky_8_image_tag.outputs.image_tag }} - env: - OS_CLOUD: openstack - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - continue-on-error: true - - - name: Download Rocky Linux 8 overcloud host image from Ark - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ - -e os_distribution="rocky" \ - -e os_release="8" - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.rocky8 && steps.rocky_8_image_exists.outcome == 'failure' - - - name: Upload Rocky Linux 8 overcloud host image to Cloud - run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-8-${{ steps.rocky_8_image_tag.outputs.image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /tmp/rocky-8.qcow2 \ - --private \ - --progress - env: - OS_CLOUD: openstack - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.rocky8 && steps.rocky_8_image_exists.outcome == 'failure' - - name: Output Rocky Linux 9 image tag id: rocky_9_image_tag run: | @@ -229,51 +127,6 @@ jobs: OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' - - name: Output Ubuntu Focal image tag - id: ubuntu_focal_image_tag - run: | - echo image_tag=$(grep stackhpc_ubuntu_focal_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT - - - name: Check if image exists already - id: ubuntu_focal_image_exists - run: | - source venvs/kayobe/bin/activate && - openstack image show \ - overcloud-ubuntu-focal-${{ steps.ubuntu_focal_image_tag.outputs.image_tag }} - env: - OS_CLOUD: openstack - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - continue-on-error: true - - - name: Download Ubuntu Focal 20.04 overcloud host image from Ark - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && - kayobe playbook run \ - src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ - -e os_distribution="ubuntu" \ - -e os_release="focal" - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.ubuntu-focal && steps.ubuntu_focal_image_exists.outcome == 'failure' - - - name: Upload Ubuntu Focal 20.04 overcloud host image to Cloud - run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-focal-${{ steps.ubuntu_focal_image_tag.outputs.image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /tmp/ubuntu-focal.qcow2 \ - --private \ - --progress - env: - OS_CLOUD: openstack - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.ubuntu-focal && steps.ubuntu_focal_image_exists.outcome == 'failure' - - name: Output Ubuntu Jammy image tag id: ubuntu_jammy_image_tag run: |