From b81ee745b261fd7fd61144af54cbf988cff883d6 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 5 Dec 2023 22:21:01 +0100 Subject: [PATCH 01/18] Add a note about network interfaces changes --- doc/source/operations/rocky-linux-9.rst | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/source/operations/rocky-linux-9.rst b/doc/source/operations/rocky-linux-9.rst index 829d45e6f..d83497caf 100644 --- a/doc/source/operations/rocky-linux-9.rst +++ b/doc/source/operations/rocky-linux-9.rst @@ -242,6 +242,13 @@ Potential issues - If you are using hyper-converged Ceph, please also note the potential issues in the Storage section below. +- Network interface names may change between CentOS Stream 8 and Rocky Linux + 9, in which case you will need to update Kayobe configuration. Note that the + configuration should remain correct for hosts not yet migrated, otherwise + fact gathering may fail. For example, this can be done using ``host_vars``. + Once all hosts are migrated, the change can be moved to ``group_vars`` and + the temporary ``host_vars`` deleted. + Full procedure for one host --------------------------- @@ -367,7 +374,12 @@ The possible batches depend on a number of things: Potential issues ---------------- -Nothing yet! +- Network interface names may change between CentOS Stream 8 and Rocky Linux + 9, in which case you will need to update Kayobe configuration. Note that the + configuration should remain correct for hosts not yet migrated, otherwise + fact gathering may fail. For example, this can be done using ``host_vars``. + Once all hosts are migrated, the change can be moved to ``group_vars`` and + the temporary ``host_vars`` deleted. Full procedure for one batch of hosts ------------------------------------- @@ -518,6 +530,13 @@ Potential issues - Commands starting with ``ceph`` are all run on the cephadm bootstrap host in a cephadm shell unless stated otherwise. +- Network interface names may change between CentOS Stream 8 and Rocky Linux + 9, in which case you will need to update Kayobe configuration. Note that the + configuration should remain correct for hosts not yet migrated, otherwise + fact gathering may fail. For example, this can be done using ``host_vars``. + Once all hosts are migrated, the change can be moved to ``group_vars`` and + the temporary ``host_vars`` deleted. + Full procedure for any storage host ----------------------------------- From 78c3257810533cd20a8e628aaebfa502d7618ac6 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Wed, 10 Jan 2024 16:11:50 +0000 Subject: [PATCH 02/18] Bump seed Pulp to 3.43.1 --- etc/kayobe/seed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 96fa86ac0..a63dad4a3 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,7 +106,7 @@ seed_pulp_container: image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" - tag: "3.24.0" + tag: "3.43.1" network_mode: host # Override deploy_containers_defaults.init == true to ensure # s6-overlay-suexec starts as pid 1 From 225710316547209979d9b5a56fd141676da03613 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 13 Feb 2024 16:25:35 +0000 Subject: [PATCH 03/18] CI: Fail aio job when Terraform reaches max attempts Previously the script exited 0 at the end of the loop, then failed in a later step. --- .github/workflows/stackhpc-all-in-one.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 704009412..de3c25f50 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -158,13 +158,15 @@ jobs: for attempt in $(seq 5); do if terraform apply -auto-approve; then echo "Created infrastructure on attempt $attempt" - break + exit 0 fi echo "Failed to create infrastructure on attempt $attempt" sleep 10 terraform destroy -auto-approve sleep 60 done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 working-directory: ${{ github.workspace }}/terraform/aio env: OS_CLOUD: ${{ inputs.OS_CLOUD }} From 56abaa7744f3278fd0a2ddb1763fbdd8852406a1 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 14 Feb 2024 14:46:36 +0000 Subject: [PATCH 04/18] Reduce Elasticsearch/OpenSearch heap size to 200m in ci-aio env This should free up some memory, allowing us to use a smaller flavor. --- etc/kayobe/environments/ci-aio/kolla/globals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/kolla/globals.yml b/etc/kayobe/environments/ci-aio/kolla/globals.yml index b717d6e54..6b42acdea 100644 --- a/etc/kayobe/environments/ci-aio/kolla/globals.yml +++ b/etc/kayobe/environments/ci-aio/kolla/globals.yml @@ -13,8 +13,8 @@ docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg" # Elasticsearch / OpenSearch memory tuning -es_heap_size: 1g -opensearch_heap_size: 1g +es_heap_size: 200m +opensearch_heap_size: 200m # Increase Grafana timeout grafana_start_first_node_retries: 20 From b293231e3fd96f2b4d0c01c95fdea1cc3a8c8fd7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 14 Feb 2024 14:46:53 +0000 Subject: [PATCH 05/18] Disable Heat in ci-aio env This should free up some memory, allowing us to use a smaller flavor. --- etc/kayobe/environments/ci-aio/stackhpc-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index 6050bffd2..d8f4ef7de 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -5,6 +5,9 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. kolla_docker_namespace: stackhpc-dev +# Disable some services to reduce memory footprint. +kolla_enable_heat: false + ############################################################################### # Network configuration. From 23247d1a2727a4121646036d8897a0c01d9ce2de Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 14 Feb 2024 14:47:30 +0000 Subject: [PATCH 06/18] Reduce aio flavor to en1.medium This flavor has 8G memory and is cheaper than en1.large. --- .github/workflows/stackhpc-all-in-one.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index de3c25f50..1155af7a1 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -38,7 +38,7 @@ on: vm_flavor: description: Flavor for the all-in-one VM type: string - default: en1.large + default: en1.medium vm_network: description: Network for the all-in-one VM type: string From 1d75be487e29cf7981eff474b8c6295091b23477 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 14 Feb 2024 13:57:39 +0000 Subject: [PATCH 07/18] CI: Add tags to aio VMs This will make it easier to identify them. --- .github/workflows/stackhpc-all-in-one.yml | 2 ++ terraform/aio/vm.tf | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index de3c25f50..211b781c0 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -134,6 +134,7 @@ jobs: aio_vm_flavor = "${{ env.VM_FLAVOR }}" aio_vm_network = "${{ env.VM_NETWORK }}" aio_vm_subnet = "${{ env.VM_SUBNET }}" + aio_vm_tags = ${{ env.VM_TAGS }} EOF working-directory: ${{ github.workspace }}/terraform/aio env: @@ -144,6 +145,7 @@ jobs: VM_NETWORK: ${{ inputs.vm_network }} VM_SUBNET: ${{ inputs.vm_subnet }} VM_INTERFACE: ${{ inputs.vm_interface }} + VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]' - name: Terraform Plan run: terraform plan diff --git a/terraform/aio/vm.tf b/terraform/aio/vm.tf index 676aff1c3..36dfa50a5 100644 --- a/terraform/aio/vm.tf +++ b/terraform/aio/vm.tf @@ -38,6 +38,11 @@ variable "aio_vm_volume_size" { default = 35 } +variable "aio_vm_tags" { + type = list(string) + default = [] +} + locals { image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.aio_vm_image)) > 0 } @@ -69,6 +74,8 @@ resource "openstack_compute_instance_v2" "kayobe-aio" { destination_type = "volume" delete_on_termination = true } + + tags = var.aio_vm_tags } # Wait for the instance to be accessible via SSH before progressing. From bf7649ec12857b86a610e64a3c3994d70d5f2574 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 14 Feb 2024 14:33:27 +0000 Subject: [PATCH 08/18] CI: Add a workflow to clean up stale instances Sometimes we can end up with aio instances left running indefinitely. This can lead to unnecessary cloud costs. This change adds a periodic workflow that runs every 2 hours and deletes instances with the skc-ci-aio tag that are over 3 hours old. --- .github/workflows/stackhpc-ci-cleanup.yml | 57 +++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/stackhpc-ci-cleanup.yml diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml new file mode 100644 index 000000000..672500ab1 --- /dev/null +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -0,0 +1,57 @@ +--- +name: Clean up stale CI resources +on: + schedule: + # Every 2 hours at quarter past + - cron: '15 0/2 * * *' + +jobs: + ci-cleanup: + name: Clean up stale CI resources + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Setup Python + uses: actions/setup-python@v5 + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + + - name: Install OpenStack client + run: | + pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt + + - name: Clean up aio instances over 3 hours old + run: | + result=0 + changes_before=$(date -Imin -d -3hours) + for status in ACTIVE ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column Name --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} From c338dd9b7cad77c14eb15eb0193d02b0c9ff78b4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 14 Feb 2024 15:52:55 +0000 Subject: [PATCH 09/18] CI: Use skc-ci-aio user for aio jobs This user only has read-only access to the package and container repositories, so is safer than using the release-train-ci user which has read/write permissions. For the container image build job we can use the skc-ci-aio user to access the package repositories, but must use the release-train-ci user to push container images. --- .../environments/ci-aio/stackhpc-ci.yml | 22 +++++++++---------- .../environments/ci-builder/stackhpc-ci.yml | 10 +++++++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index 6050bffd2..c41435c55 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -19,8 +19,14 @@ resolv_is_managed: false # Build and deploy the development Pulp service repositories. # Use Ark's package repositories to install packages. stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" -stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}" -stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}" +stackhpc_repo_mirror_username: "skc-ci-aio" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31386366383365666135336331663635396237623139306362633933636233613765663731666338 + 3633633736333936383439623066653663333964343234350a393137383537316164323837386437 + 36613139323161643766666565643739373037623363636234343965343436653261326238393566 + 3837336661653962340a316631366463623138623530373133336665376433633437306631383666 + 30333461333535363433363336663664316634343432633766346564323833346663 # Build and deploy released Pulp repository versions. stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" @@ -75,13 +81,5 @@ stackhpc_include_os_minor_version_in_repo_url: true # Host and port of container registry. # Push built images to the development Pulp service registry. stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" - -# Username and password of container registry. -stackhpc_docker_registry_username: "release-train-ci" -stackhpc_docker_registry_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 38356134376436656165303634626531653836366233383531343439646433376334396438373735 - 3135643664353934356237376134623235356137383263300a333165386562396134633534376532 - 34386133383366326639353432386235336132663839333337323739633434613934346462363031 - 3265323831663964360a643962346231386462323236373963633066393736323234303833363535 - 3664 +stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index f31629357..e17ccf69c 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -44,8 +44,14 @@ resolv_is_managed: false # Build against the development Pulp service repositories. # Use Ark's package repositories to install packages. stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}" -stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}" -stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}" +stackhpc_repo_mirror_username: "skc-ci-aio" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31386366383365666135336331663635396237623139306362633933636233613765663731666338 + 3633633736333936383439623066653663333964343234350a393137383537316164323837386437 + 36613139323161643766666565643739373037623363636234343965343436653261326238393566 + 3837336661653962340a316631366463623138623530373133336665376433633437306631383666 + 30333461333535363433363336663664316634343432633766346564323833346663 # Build against released Pulp repository versions. stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" From 3818ff9655fd48dfa529b6f8708bda03b0ce6fb0 Mon Sep 17 00:00:00 2001 From: Seunghun Lee <45145778+seunghun1ee@users.noreply.github.com> Date: Thu, 15 Feb 2024 14:16:46 +0000 Subject: [PATCH 10/18] Ubuntu Focal to Jammy migration support (#902) Add playbook to upgrade ubuntu focal hosts Co-authored-by: Alex-Welsh --- etc/kayobe/ansible/ubuntu-upgrade.yml | 110 ++++++++++++++++++ ...tu-migration-support-2d7e8a6e1a2103cc.yaml | 4 + tools/ubuntu-upgrade-infra-vm.sh | 34 ++++++ tools/ubuntu-upgrade-overcloud.sh | 34 ++++++ tools/ubuntu-upgrade-seed-hypervisor.sh | 29 +++++ tools/ubuntu-upgrade-seed.sh | 29 +++++ 6 files changed, 240 insertions(+) create mode 100644 etc/kayobe/ansible/ubuntu-upgrade.yml create mode 100644 releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml create mode 100755 tools/ubuntu-upgrade-infra-vm.sh create mode 100755 tools/ubuntu-upgrade-overcloud.sh create mode 100755 tools/ubuntu-upgrade-seed-hypervisor.sh create mode 100755 tools/ubuntu-upgrade-seed.sh diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml new file mode 100644 index 000000000..3b477731c --- /dev/null +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -0,0 +1,110 @@ +--- +# To prevent Ansible role dependency errors, this playbook requires that environment variable +# ANSIBLE_ROLES_PATH is defined and includes '$KAYOBE_PATH/ansible/roles' on the Ansible control host. +- name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + tasks: + - name: Assert that hosts are running Ubuntu Focal + assert: + that: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '20' + - ansible_facts.distribution_release == 'focal' + - os_distribution == 'ubuntu' + fail_msg: >- + This playbook is only designed for Ubuntu Focal 20.04 hosts. Ensure + that you are limiting it to only run on Focal hosts and + os_distribution is set to ubuntu. + + - name: Ensure apt packages are up to date + apt: + update_cache: true + upgrade: yes + become: true + + - name: Ensure do-release-upgrade is installed + package: + name: ubuntu-release-upgrader-core + state: latest + become: true + + - name: Check whether a reboot is required + stat: + path: /var/run/reboot-required + register: file_status + + - name: Reboot to apply updates + reboot: + reboot_timeout: 1200 + connect_timeout: 600 + become: true + when: file_status.stat.exists + + # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list + - name: Ensure Jammy repo definitions exist in sources.list + blockinfile: + path: /etc/apt/sources.list + block: | + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse + become: true + + - name: Do release upgrade + command: do-release-upgrade -f DistUpgradeViewNonInteractive + become: true + + - name: Ensure old venvs do not exist + file: + path: "/opt/kayobe/venvs/{{ item }}" + state: absent + loop: + - kayobe + - kolla-ansible + become: true + + - name: Update Python and current user facts before re-creating Kayobe venv + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + +- name: Run the Kayobe kayobe-target-venv playbook to ensure kayobe venv exists on remote host + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml" + +- name: Run the Kayobe network configuration playbook, to ensure definitions are not lost on reboot + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" + +- name: Reboot and confirm the host is upgraded to Jammy 22.04 + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + tasks: + - name: Ensure Jammy repo definitions do not exist in sources.list + blockinfile: + path: /etc/apt/sources.list + state: absent + become: true + + - name: Reboot and wait + reboot: + reboot_timeout: 1200 + connect_timeout: 600 + become: true + + - name: Update distribution facts + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + + - name: Assert that hosts are now using Ubuntu 22 + assert: + that: + - ansible_facts.distribution_major_version == '22' + - ansible_facts.distribution_release == 'jammy' + +- name: Run the OVN chassis priority fix playbook + import_playbook: "{{ lookup('ansible.builtin.env', 'KAYOBE_CONFIG_PATH') }}/ansible/ovn-fix-chassis-priorities.yml" + when: kolla_enable_ovn diff --git a/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml b/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml new file mode 100644 index 000000000..89216342d --- /dev/null +++ b/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for automated Ubuntu Focal to Jammy migration. diff --git a/tools/ubuntu-upgrade-infra-vm.sh b/tools/ubuntu-upgrade-infra-vm.sh new file mode 100755 index 000000000..8d5810174 --- /dev/null +++ b/tools/ubuntu-upgrade-infra-vm.sh @@ -0,0 +1,34 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $1 ]]; then + echo "Usage: infra-vm-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 + +kayobe infra vm host configure --limit $1 -e os_release=jammy diff --git a/tools/ubuntu-upgrade-overcloud.sh b/tools/ubuntu-upgrade-overcloud.sh new file mode 100755 index 000000000..3e351d6d6 --- /dev/null +++ b/tools/ubuntu-upgrade-overcloud.sh @@ -0,0 +1,34 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $1 ]]; then + echo "Usage: overcloud-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 + +kayobe overcloud host configure --limit $1 --kolla-limit $1 -e os_release=jammy diff --git a/tools/ubuntu-upgrade-seed-hypervisor.sh b/tools/ubuntu-upgrade-seed-hypervisor.sh new file mode 100755 index 000000000..ad09f2b34 --- /dev/null +++ b/tools/ubuntu-upgrade-seed-hypervisor.sh @@ -0,0 +1,29 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit seed-hypervisor + +kayobe seed hypervisor host configure diff --git a/tools/ubuntu-upgrade-seed.sh b/tools/ubuntu-upgrade-seed.sh new file mode 100755 index 000000000..4a48d5f36 --- /dev/null +++ b/tools/ubuntu-upgrade-seed.sh @@ -0,0 +1,29 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit seed + +kayobe seed host configure From 55672dfdd8e440876873999485e0f10d1ee4cc1b Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 16 Feb 2024 14:46:58 +0100 Subject: [PATCH 11/18] Stop warning about invalid group name characters This should silence the following Ansible warning [1]: [WARNING]: Invalid characters were found in group names but not replaced, use -vvvv to see details [1] https://docs.ansible.com/ansible/latest/reference_appendices/config.html#transform-invalid-group-chars --- etc/kayobe/ansible.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index 515af8d32..901001ad8 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -8,6 +8,8 @@ bin_ansible_callbacks = True inject_facts_as_vars = False # Add timing information to output callbacks_enabled = ansible.posix.profile_tasks +# Silence warning about invalid characters found in group names +force_valid_group_names = ignore [ssh_connection] pipelining = True From ca7fd1d17ef20cabc2f990e7fa09aae0d2de82d4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 19 Feb 2024 12:32:35 +0000 Subject: [PATCH 12/18] CI: Clean up instances in BUILD state, list by ID --- .github/workflows/stackhpc-ci-cleanup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index 672500ab1..df032fdac 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -40,8 +40,8 @@ jobs: run: | result=0 changes_before=$(date -Imin -d -3hours) - for status in ACTIVE ERROR SHUTOFF; do - for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column Name --changes-before $changes_before --status $status); do + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do echo "Cleaning up $status instance $instance" openstack server show $instance if ! openstack server delete $instance; then From 186a03c98b7e66ee6d8670234fbc3472e3d88db4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 19 Feb 2024 12:33:25 +0000 Subject: [PATCH 13/18] CI: Use correct URL for upper constraints This URL should not be affected by branch names etc. --- .github/workflows/overcloud-host-image-build.yml | 2 +- .github/workflows/overcloud-host-image-upload.yml | 2 +- .github/workflows/stackhpc-ci-cleanup.yml | 2 +- etc/kayobe/environments/ci-builder/inventory/hosts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 0c2fc3efb..cbccca23e 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -139,7 +139,7 @@ jobs: - name: Install OpenStack client run: | source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt + pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} - name: Build a CentOS Stream 8 overcloud host image id: build_centos_stream_8 diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml index f18f8ef4c..633f423b5 100644 --- a/.github/workflows/overcloud-host-image-upload.yml +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -92,7 +92,7 @@ jobs: - name: Install OpenStack client run: | source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt + pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} - name: Output CentOS Stream 8 image tag id: centos_8_stream_image_tag diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index df032fdac..d0da0c051 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -34,7 +34,7 @@ jobs: - name: Install OpenStack client run: | - pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt + pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} - name: Clean up aio instances over 3 hours old run: | diff --git a/etc/kayobe/environments/ci-builder/inventory/hosts b/etc/kayobe/environments/ci-builder/inventory/hosts index 33fda8b73..49b7be166 100644 --- a/etc/kayobe/environments/ci-builder/inventory/hosts +++ b/etc/kayobe/environments/ci-builder/inventory/hosts @@ -1,3 +1,3 @@ # A 'seed' host used for building images. [seed] -builder +localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 From 8ddb62df6dfb627351dc503df2542540ad35b96d Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas Date: Mon, 19 Feb 2024 16:52:06 +0100 Subject: [PATCH 14/18] bumping ansible vault --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ca7aab63..2089c4b3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/yoga ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc;python_version < "3.8" -ansible-modules-hashivault;python_version >= "3.8" +ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc-py39;python_version >= "3.8" jmespath From 0030f3edcdcdd4e8b22a25ae33833f65bc2521c1 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Feb 2024 12:13:33 +0000 Subject: [PATCH 15/18] docs: Update network interface note to mention group_vars --- doc/source/operations/rocky-linux-9.rst | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/source/operations/rocky-linux-9.rst b/doc/source/operations/rocky-linux-9.rst index d83497caf..b472942a4 100644 --- a/doc/source/operations/rocky-linux-9.rst +++ b/doc/source/operations/rocky-linux-9.rst @@ -245,9 +245,10 @@ Potential issues - Network interface names may change between CentOS Stream 8 and Rocky Linux 9, in which case you will need to update Kayobe configuration. Note that the configuration should remain correct for hosts not yet migrated, otherwise - fact gathering may fail. For example, this can be done using ``host_vars``. - Once all hosts are migrated, the change can be moved to ``group_vars`` and - the temporary ``host_vars`` deleted. + fact gathering may fail. For example, this can be done using ``group_vars`` + with a temporary group for the updated hosts or ``host_vars``. Once all + hosts are migrated, the change can be moved to the original group's + ``group_vars`` and the temporary changes reverted. Full procedure for one host --------------------------- @@ -377,9 +378,10 @@ Potential issues - Network interface names may change between CentOS Stream 8 and Rocky Linux 9, in which case you will need to update Kayobe configuration. Note that the configuration should remain correct for hosts not yet migrated, otherwise - fact gathering may fail. For example, this can be done using ``host_vars``. - Once all hosts are migrated, the change can be moved to ``group_vars`` and - the temporary ``host_vars`` deleted. + fact gathering may fail. For example, this can be done using ``group_vars`` + with a temporary group for the updated hosts or ``host_vars``. Once all + hosts are migrated, the change can be moved to the original group's + ``group_vars`` and the temporary changes reverted. Full procedure for one batch of hosts ------------------------------------- @@ -533,9 +535,10 @@ Potential issues - Network interface names may change between CentOS Stream 8 and Rocky Linux 9, in which case you will need to update Kayobe configuration. Note that the configuration should remain correct for hosts not yet migrated, otherwise - fact gathering may fail. For example, this can be done using ``host_vars``. - Once all hosts are migrated, the change can be moved to ``group_vars`` and - the temporary ``host_vars`` deleted. + fact gathering may fail. For example, this can be done using ``group_vars`` + with a temporary group for the updated hosts or ``host_vars``. Once all + hosts are migrated, the change can be moved to the original group's + ``group_vars`` and the temporary changes reverted. Full procedure for any storage host ----------------------------------- From b3e2c5601698789d6dadf65637b21b5f9c644523 Mon Sep 17 00:00:00 2001 From: "Max.Bed4d" Date: Fri, 9 Feb 2024 15:49:30 +0000 Subject: [PATCH 16/18] Enable AiO jobs to be cancelled even if they're underway. --- .github/workflows/stackhpc-all-in-one.yml | 2 +- .github/workflows/stackhpc-pull-request.yml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 26de98f66..59931e8cd 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -69,7 +69,7 @@ jobs: # NOTE: Runner needs unzip and nodejs packages. all-in-one: name: All in one - if: inputs.if + if: ${{ inputs.if && !cancelled() }} runs-on: arc-skc-aio-runner permissions: {} env: diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index 987e15356..ce9243ba2 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -88,7 +88,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-centos-ovn: name: aio (CentOS OVN) @@ -102,7 +102,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-rocky-8-ovs: name: aio (Rocky OVS) @@ -118,7 +118,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-rocky-8-ovn: name: aio (Rocky OVN) @@ -134,7 +134,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-ubuntu-ovs: name: aio (Ubuntu OVS) @@ -151,7 +151,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-ubuntu-ovn: name: aio (Ubuntu OVN) @@ -168,7 +168,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-ubuntu-jammy-ovs: name: aio (Ubuntu Jammy OVS) @@ -185,7 +185,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-ubuntu-jammy-ovn: name: aio (Ubuntu Jammy OVN) @@ -202,7 +202,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-rocky-9-ovs: name: aio (Rocky 9 OVS) @@ -219,7 +219,7 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-rocky-9-ovn: name: aio (Rocky 9 OVN) @@ -236,4 +236,4 @@ jobs: OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit - if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} From 2ce47a0129027950088016526399dd8a14d00e6c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Feb 2024 14:47:10 +0000 Subject: [PATCH 17/18] CI: Add cancellation support to check-tags job --- .github/workflows/stackhpc-check-tags.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml index f5a12a714..4016c00e9 100644 --- a/.github/workflows/stackhpc-check-tags.yml +++ b/.github/workflows/stackhpc-check-tags.yml @@ -23,7 +23,7 @@ env: jobs: check-tags: name: Check container image tags - if: inputs.if + if: ${{ inputs.if && ! cancelled() }} runs-on: arc-skc-aio-runner permissions: {} env: From dce592ca8c66a867f81a5c3c537f0385d94ba92e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 21 Feb 2024 08:59:43 +0000 Subject: [PATCH 18/18] Remove Ubuntu Jammy upgrade release note This feature is useful in Yoga only, so no need to announce it. --- ...d-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml diff --git a/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml b/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml deleted file mode 100644 index 89216342d..000000000 --- a/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -features: - - | - Add support for automated Ubuntu Focal to Jammy migration.