From ba63ccb9970feb0b5bd27210ede4b7d4035270e0 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 23 Jan 2024 12:18:32 +0000 Subject: [PATCH 01/17] Add playbook to upgrade ubuntu focal hosts --- etc/kayobe/ansible/ubuntu-upgrade.yml | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 etc/kayobe/ansible/ubuntu-upgrade.yml diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml new file mode 100644 index 000000000..7ddf14a92 --- /dev/null +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -0,0 +1,78 @@ +--- +- name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 + hosts: overcloud:infra-vms:seed + tasks: + - name: Assert that hosts are running Ubuntu Focal + assert: + that: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '20' + - ansible_facts.distribution_release == 'focal' + - os_distribution == 'ubuntu' + fail_msg: >- + This playbook is only designed for Ubuntu Focal 20.04 hosts. Ensure + that you are limiting it to only run on Focal hosts and + os_distribution in set to ubuntu. + + - name: Ensure apt packages are up to date + apt: + update_cache: true + upgrade: yes + + # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list + - name: Ensure Jammy repo definitions exist in sources.list + blockinfile: + path: /etc/apt/sources.list + block: | + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse + become: true + + - name: Do release upgrade + command: do-release-upgrade -f DistUpgradeViewNonInteractive + become: true + + - name: Print selection of Deities to pray to + debug: + msg: | + The following Deities are associated with luck or good fortune: + Hamingja - Norse guardian spirit + Fortuna - Roman goddess of fortune and luck + Tyche - Greek goddess of success, fortune, Luck, and prosperity + Nang Kwak - Thai folklore goddess deemed to bring good fortune and prosperity + + - name: Reboot and wait + reboot: + become: true + + - name: Gather facts + gather_facts: + + - name: Assert that hosts are now using Ubuntu 22 + assert: + that: + - ansible_facts.distribution_major_version == '22' + - ansible_facts.distribution_release == 'jammy' + + - name: Ensure Jammy repo definitions do not exist in sources.list + blockinfile: + path: /etc/apt/sources.list + state: absent + block: | + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse + become: true + + - name: Ensure old venvs do not exist + ansible_python_interpreter: /usr/bin/python + file: + path: "/opt/kayobe/venvs/{{ item }}" + state: absent + loop: + - kayobe + - kolla-ansible + become: true From 3bcd1fb5c5ced1cebb7936681c32891e5330096c Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 23 Jan 2024 14:22:40 +0000 Subject: [PATCH 02/17] add do-release-upgrade is installation task --- etc/kayobe/ansible/ubuntu-upgrade.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 7ddf14a92..00d75ebe6 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -30,6 +30,12 @@ deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse become: true + - name: Ensure do-release-upgrade is installed + package: + name: ubuntu-release-upgrader-core + state: latest + become: true + - name: Do release upgrade command: do-release-upgrade -f DistUpgradeViewNonInteractive become: true From 84e0dab629084f268905a812d2b71b9544a43903 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 23 Jan 2024 15:35:00 +0000 Subject: [PATCH 03/17] Fix ubuntu upgrade playbook --- etc/kayobe/ansible/ubuntu-upgrade.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 00d75ebe6..3edaebf2f 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -1,6 +1,9 @@ --- - name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 hosts: overcloud:infra-vms:seed + vars: + ansible_python_interpreter: /usr/bin/python3 + become: true tasks: - name: Assert that hosts are running Ubuntu Focal assert: @@ -28,17 +31,14 @@ deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse - become: true - name: Ensure do-release-upgrade is installed package: name: ubuntu-release-upgrader-core state: latest - become: true - name: Do release upgrade command: do-release-upgrade -f DistUpgradeViewNonInteractive - become: true - name: Print selection of Deities to pray to debug: @@ -51,7 +51,6 @@ - name: Reboot and wait reboot: - become: true - name: Gather facts gather_facts: @@ -71,14 +70,11 @@ deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse - become: true - name: Ensure old venvs do not exist - ansible_python_interpreter: /usr/bin/python file: path: "/opt/kayobe/venvs/{{ item }}" state: absent loop: - kayobe - kolla-ansible - become: true From 2d1359c7288c3abdc482c5cdd3d04d2724b33cd1 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 23 Jan 2024 16:15:42 +0000 Subject: [PATCH 04/17] Fix networking outage during ubuntu upgrade --- etc/kayobe/ansible/ubuntu-upgrade.yml | 33 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 3edaebf2f..5382b8029 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -49,18 +49,6 @@ Tyche - Greek goddess of success, fortune, Luck, and prosperity Nang Kwak - Thai folklore goddess deemed to bring good fortune and prosperity - - name: Reboot and wait - reboot: - - - name: Gather facts - gather_facts: - - - name: Assert that hosts are now using Ubuntu 22 - assert: - that: - - ansible_facts.distribution_major_version == '22' - - ansible_facts.distribution_release == 'jammy' - - name: Ensure Jammy repo definitions do not exist in sources.list blockinfile: path: /etc/apt/sources.list @@ -78,3 +66,24 @@ loop: - kayobe - kolla-ansible + + - name: debug + debug: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" + + - name: Run kayobe-target-venv playbook to ensure kayobe venv exists on remote host + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml" + + - name: Run network playbook to ensure networking config is not reliant on ifupdown + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" + + - name: Reboot and wait + reboot: + + - name: Gather facts + gather_facts: + + - name: Assert that hosts are now using Ubuntu 22 + assert: + that: + - ansible_facts.distribution_major_version == '22' + - ansible_facts.distribution_release == 'jammy' From 60dffea978bedac7b4bac2a9d899c40968bfe775 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Thu, 25 Jan 2024 13:52:57 +0000 Subject: [PATCH 05/17] Add updating subset of ansible facts after do-release-upgrade --- etc/kayobe/ansible/ubuntu-upgrade.yml | 39 ++++++++++++++++++++------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 5382b8029..5580b0b2d 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -3,7 +3,6 @@ hosts: overcloud:infra-vms:seed vars: ansible_python_interpreter: /usr/bin/python3 - become: true tasks: - name: Assert that hosts are running Ubuntu Focal assert: @@ -21,6 +20,7 @@ apt: update_cache: true upgrade: yes + become: true # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list - name: Ensure Jammy repo definitions exist in sources.list @@ -31,14 +31,17 @@ deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse + become: true - name: Ensure do-release-upgrade is installed package: name: ubuntu-release-upgrader-core state: latest + become: true - name: Do release upgrade command: do-release-upgrade -f DistUpgradeViewNonInteractive + become: true - name: Print selection of Deities to pray to debug: @@ -58,6 +61,7 @@ deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse + become: true - name: Ensure old venvs do not exist file: @@ -66,21 +70,38 @@ loop: - kayobe - kolla-ansible + become: true - - name: debug - debug: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" + - name: Update Python and current user facts before re-creating Kayobe venv + ansible.builtin.setup: + gather_subset: + - '!all' + - python + - python_version + - user_uid + - user_gid - - name: Run kayobe-target-venv playbook to ensure kayobe venv exists on remote host - import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml" +- name: Run kayobe-target-venv playbook to ensure kayobe venv exists on remote host + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml" - - name: Run network playbook to ensure networking config is not reliant on ifupdown - import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" +- name: Run network playbook to ensure networking config is not reliant on ifupdown + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" +- name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 + hosts: overcloud:infra-vms:seed + vars: + ansible_python_interpreter: /usr/bin/python3 + tasks: - name: Reboot and wait reboot: + become: true - - name: Gather facts - gather_facts: + - name: Update distribution facts + ansible.builtin.setup: + gather_subset: + - '!all' + - distribution_major_version + - distribution_release - name: Assert that hosts are now using Ubuntu 22 assert: From 73a038a777f47f5e36966311245127c778daae24 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 29 Jan 2024 11:43:48 +0000 Subject: [PATCH 06/17] Add shell script to automate ubuntu upgrade --- .../scripts/overcloud-ubuntu-upgrade.sh | 34 +++++++++++++++++++ .../ansible/scripts/seed-ubuntu-upgrade.sh | 34 +++++++++++++++++++ etc/kayobe/ansible/ubuntu-upgrade.yml | 12 ++++--- 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100755 etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh create mode 100755 etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh diff --git a/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh b/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh new file mode 100755 index 000000000..479a23a20 --- /dev/null +++ b/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh @@ -0,0 +1,34 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $1 ]]; then + echo "Usage: overcloud-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 + +kayobe overcloud host configure --limit $1 diff --git a/etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh b/etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh new file mode 100755 index 000000000..e9d4cee14 --- /dev/null +++ b/etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh @@ -0,0 +1,34 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $1 ]]; then + echo "Usage: seed-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 + +kayobe seed host configure --limit $1 diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 5580b0b2d..5a51748d5 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -1,4 +1,6 @@ --- +# To prevent Ansible role dependency errors, this playbook requires that environment variable +# ANSIBLE_ROLES_PATH is defined and includes '$KAYOBE_PATH/ansible/roles' at seed-hypervisor. - name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 hosts: overcloud:infra-vms:seed vars: @@ -39,10 +41,6 @@ state: latest become: true - - name: Do release upgrade - command: do-release-upgrade -f DistUpgradeViewNonInteractive - become: true - - name: Print selection of Deities to pray to debug: msg: | @@ -52,6 +50,10 @@ Tyche - Greek goddess of success, fortune, Luck, and prosperity Nang Kwak - Thai folklore goddess deemed to bring good fortune and prosperity + - name: Do release upgrade + command: do-release-upgrade -f DistUpgradeViewNonInteractive + become: true + - name: Ensure Jammy repo definitions do not exist in sources.list blockinfile: path: /etc/apt/sources.list @@ -87,7 +89,7 @@ - name: Run network playbook to ensure networking config is not reliant on ifupdown import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" -- name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 +- name: Reboot and confirm the host is upgraded to Jammy 22.04 hosts: overcloud:infra-vms:seed vars: ansible_python_interpreter: /usr/bin/python3 From c6bef3a63878af1d5c8ce255522d868cc66291d7 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 30 Jan 2024 09:25:20 +0000 Subject: [PATCH 07/17] Add release note --- ...d-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml diff --git a/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml b/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml new file mode 100644 index 000000000..89216342d --- /dev/null +++ b/releasenotes/notes/add-automated-ubuntu-migration-support-2d7e8a6e1a2103cc.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for automated Ubuntu Focal to Jammy migration. From cb4902b134f2fa0d2d8b69059c7238c96966dc8f Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 30 Jan 2024 13:49:01 +0000 Subject: [PATCH 08/17] fix trailing space --- etc/kayobe/ansible/ubuntu-upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 5a51748d5..830bad978 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -98,7 +98,7 @@ reboot: become: true - - name: Update distribution facts + - name: Update distribution facts ansible.builtin.setup: gather_subset: - '!all' From e625c48a648f11c1458bb0802800576d19b8da82 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 31 Jan 2024 15:34:45 +0000 Subject: [PATCH 09/17] Fix typo and praying --- etc/kayobe/ansible/ubuntu-upgrade.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 830bad978..dbb04736e 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -16,7 +16,7 @@ fail_msg: >- This playbook is only designed for Ubuntu Focal 20.04 hosts. Ensure that you are limiting it to only run on Focal hosts and - os_distribution in set to ubuntu. + os_distribution is set to ubuntu. - name: Ensure apt packages are up to date apt: @@ -41,15 +41,6 @@ state: latest become: true - - name: Print selection of Deities to pray to - debug: - msg: | - The following Deities are associated with luck or good fortune: - Hamingja - Norse guardian spirit - Fortuna - Roman goddess of fortune and luck - Tyche - Greek goddess of success, fortune, Luck, and prosperity - Nang Kwak - Thai folklore goddess deemed to bring good fortune and prosperity - - name: Do release upgrade command: do-release-upgrade -f DistUpgradeViewNonInteractive become: true From af51a2648f1bd3f117f8ef5e60ca5fa5f25fd24e Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 31 Jan 2024 15:58:05 +0000 Subject: [PATCH 10/17] Add kolla limit and override os_release to jammy as safety --- etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh b/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh index 479a23a20..750f1c7bd 100755 --- a/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh +++ b/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh @@ -31,4 +31,4 @@ set -x kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 -kayobe overcloud host configure --limit $1 +kayobe overcloud host configure --limit $1 --kolla-limit $1 -e os_release=jammy From 08d299b87fd36f7b9b27aeba3cfb8d32ab401c60 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 2 Feb 2024 15:18:50 +0000 Subject: [PATCH 11/17] Update ubuntu upgrade scripts after review --- etc/kayobe/ansible/ubuntu-upgrade.yml | 48 +++++++++---------- tools/infra-vm-ubuntu-upgrade.sh | 34 +++++++++++++ .../overcloud-ubuntu-upgrade.sh | 0 tools/seed-hypervisor-ubuntu-upgrade.sh | 29 +++++++++++ .../scripts => tools}/seed-ubuntu-upgrade.sh | 9 +--- 5 files changed, 87 insertions(+), 33 deletions(-) create mode 100755 tools/infra-vm-ubuntu-upgrade.sh rename {etc/kayobe/ansible/scripts => tools}/overcloud-ubuntu-upgrade.sh (100%) create mode 100755 tools/seed-hypervisor-ubuntu-upgrade.sh rename {etc/kayobe/ansible/scripts => tools}/seed-ubuntu-upgrade.sh (79%) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index dbb04736e..b24690b3d 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -1,8 +1,8 @@ --- # To prevent Ansible role dependency errors, this playbook requires that environment variable -# ANSIBLE_ROLES_PATH is defined and includes '$KAYOBE_PATH/ansible/roles' at seed-hypervisor. +# ANSIBLE_ROLES_PATH is defined and includes '$KAYOBE_PATH/ansible/roles' on the Ansible control host. - name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04 - hosts: overcloud:infra-vms:seed + hosts: overcloud:infra-vms:seed:seed-hypervisor vars: ansible_python_interpreter: /usr/bin/python3 tasks: @@ -18,12 +18,25 @@ that you are limiting it to only run on Focal hosts and os_distribution is set to ubuntu. + - name: Assert os_release is jammy + assert: + that: + - os_release == 'jammy' + fail_msg: >- + Ensure that os_release is set to jammy before upgrading hosts. + - name: Ensure apt packages are up to date apt: update_cache: true upgrade: yes become: true + - name: Ensure do-release-upgrade is installed + package: + name: ubuntu-release-upgrader-core + state: latest + become: true + # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list - name: Ensure Jammy repo definitions exist in sources.list blockinfile: @@ -35,12 +48,6 @@ deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse become: true - - name: Ensure do-release-upgrade is installed - package: - name: ubuntu-release-upgrader-core - state: latest - become: true - - name: Do release upgrade command: do-release-upgrade -f DistUpgradeViewNonInteractive become: true @@ -49,11 +56,6 @@ blockinfile: path: /etc/apt/sources.list state: absent - block: | - deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy main restricted universe multiverse - deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse - deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse - deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse become: true - name: Ensure old venvs do not exist @@ -67,21 +69,17 @@ - name: Update Python and current user facts before re-creating Kayobe venv ansible.builtin.setup: - gather_subset: - - '!all' - - python - - python_version - - user_uid - - user_gid + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" -- name: Run kayobe-target-venv playbook to ensure kayobe venv exists on remote host +- name: Run the Kayobe kayobe-target-venv playbook to ensure kayobe venv exists on remote host import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml" -- name: Run network playbook to ensure networking config is not reliant on ifupdown +- name: Run the Kayobe network configuration playbook, to ensure definitions are not lost on reboot import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" - name: Reboot and confirm the host is upgraded to Jammy 22.04 - hosts: overcloud:infra-vms:seed + hosts: overcloud:infra-vms:seed:seed-hypervisor vars: ansible_python_interpreter: /usr/bin/python3 tasks: @@ -91,10 +89,8 @@ - name: Update distribution facts ansible.builtin.setup: - gather_subset: - - '!all' - - distribution_major_version - - distribution_release + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" - name: Assert that hosts are now using Ubuntu 22 assert: diff --git a/tools/infra-vm-ubuntu-upgrade.sh b/tools/infra-vm-ubuntu-upgrade.sh new file mode 100755 index 000000000..2b796dc2a --- /dev/null +++ b/tools/infra-vm-ubuntu-upgrade.sh @@ -0,0 +1,34 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $1 ]]; then + echo "Usage: infra-vm-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 + +kayobe infra vm host configure --limit $1 --kolla-limit $1 -e os_release=jammy diff --git a/etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh b/tools/overcloud-ubuntu-upgrade.sh similarity index 100% rename from etc/kayobe/ansible/scripts/overcloud-ubuntu-upgrade.sh rename to tools/overcloud-ubuntu-upgrade.sh diff --git a/tools/seed-hypervisor-ubuntu-upgrade.sh b/tools/seed-hypervisor-ubuntu-upgrade.sh new file mode 100755 index 000000000..cf15c6935 --- /dev/null +++ b/tools/seed-hypervisor-ubuntu-upgrade.sh @@ -0,0 +1,29 @@ +#! /usr/bin/bash + +set -e + +if [[ ! $KAYOBE_PATH ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ ! $ANSIBLE_ROLES_PATH ]]; then + set -x + export ANSIBLE_ROLES_PATH=$KAYOBE_PATH/ansible/roles + set +x +else + set -x + export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles + set +x +fi + +set -x + +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit seed-hypervisor + +kayobe seed hypervisor host configure diff --git a/etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh b/tools/seed-ubuntu-upgrade.sh similarity index 79% rename from etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh rename to tools/seed-ubuntu-upgrade.sh index e9d4cee14..d13081d64 100755 --- a/etc/kayobe/ansible/scripts/seed-ubuntu-upgrade.sh +++ b/tools/seed-ubuntu-upgrade.sh @@ -2,11 +2,6 @@ set -e -if [[ ! $1 ]]; then - echo "Usage: seed-ubuntu-upgrade.sh " - exit 2 -fi - if [[ ! $KAYOBE_PATH ]]; then echo "Environment variable \$KAYOBE_PATH is not defined" exit 2 @@ -29,6 +24,6 @@ fi set -x -kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit seed -kayobe seed host configure --limit $1 +kayobe seed host configure From d8a4a4f21babb37186cd854c20cb48ee42c3b3ba Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 2 Feb 2024 15:28:49 +0000 Subject: [PATCH 12/17] Rename ubuntu upgrade scripts --- tools/{infra-vm-ubuntu-upgrade.sh => ubuntu-upgrade-infra-vm.sh} | 0 .../{overcloud-ubuntu-upgrade.sh => ubuntu-upgrade-overcloud.sh} | 0 ...rvisor-ubuntu-upgrade.sh => ubuntu-upgrade-seed-hypervisor.sh} | 0 tools/{seed-ubuntu-upgrade.sh => ubuntu-upgrade-seed.sh} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tools/{infra-vm-ubuntu-upgrade.sh => ubuntu-upgrade-infra-vm.sh} (100%) rename tools/{overcloud-ubuntu-upgrade.sh => ubuntu-upgrade-overcloud.sh} (100%) rename tools/{seed-hypervisor-ubuntu-upgrade.sh => ubuntu-upgrade-seed-hypervisor.sh} (100%) rename tools/{seed-ubuntu-upgrade.sh => ubuntu-upgrade-seed.sh} (100%) diff --git a/tools/infra-vm-ubuntu-upgrade.sh b/tools/ubuntu-upgrade-infra-vm.sh similarity index 100% rename from tools/infra-vm-ubuntu-upgrade.sh rename to tools/ubuntu-upgrade-infra-vm.sh diff --git a/tools/overcloud-ubuntu-upgrade.sh b/tools/ubuntu-upgrade-overcloud.sh similarity index 100% rename from tools/overcloud-ubuntu-upgrade.sh rename to tools/ubuntu-upgrade-overcloud.sh diff --git a/tools/seed-hypervisor-ubuntu-upgrade.sh b/tools/ubuntu-upgrade-seed-hypervisor.sh similarity index 100% rename from tools/seed-hypervisor-ubuntu-upgrade.sh rename to tools/ubuntu-upgrade-seed-hypervisor.sh diff --git a/tools/seed-ubuntu-upgrade.sh b/tools/ubuntu-upgrade-seed.sh similarity index 100% rename from tools/seed-ubuntu-upgrade.sh rename to tools/ubuntu-upgrade-seed.sh From 18df72be4d9453e2db2358e52452543b3e7df64d Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 5 Feb 2024 12:18:38 +0000 Subject: [PATCH 13/17] Improved ubuntu upgrade script Reboot prepatory apt update when required and always set -e os_release=jammy for the upgrade playbook --- etc/kayobe/ansible/ubuntu-upgrade.yml | 17 ++++++++++------- tools/ubuntu-upgrade-infra-vm.sh | 2 +- tools/ubuntu-upgrade-overcloud.sh | 2 +- tools/ubuntu-upgrade-seed-hypervisor.sh | 2 +- tools/ubuntu-upgrade-seed.sh | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index b24690b3d..189d71951 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -18,13 +18,6 @@ that you are limiting it to only run on Focal hosts and os_distribution is set to ubuntu. - - name: Assert os_release is jammy - assert: - that: - - os_release == 'jammy' - fail_msg: >- - Ensure that os_release is set to jammy before upgrading hosts. - - name: Ensure apt packages are up to date apt: update_cache: true @@ -37,6 +30,16 @@ state: latest become: true + - name: Check whether a reboot is required + stat: + path: /var/run/reboot-required + register: file_status + + - name: Reboot to apply updates + reboot: + become: true + when: file_status.stat.exists + # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list - name: Ensure Jammy repo definitions exist in sources.list blockinfile: diff --git a/tools/ubuntu-upgrade-infra-vm.sh b/tools/ubuntu-upgrade-infra-vm.sh index 2b796dc2a..155a5386a 100755 --- a/tools/ubuntu-upgrade-infra-vm.sh +++ b/tools/ubuntu-upgrade-infra-vm.sh @@ -29,6 +29,6 @@ fi set -x -kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 kayobe infra vm host configure --limit $1 --kolla-limit $1 -e os_release=jammy diff --git a/tools/ubuntu-upgrade-overcloud.sh b/tools/ubuntu-upgrade-overcloud.sh index 750f1c7bd..3e351d6d6 100755 --- a/tools/ubuntu-upgrade-overcloud.sh +++ b/tools/ubuntu-upgrade-overcloud.sh @@ -29,6 +29,6 @@ fi set -x -kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit $1 +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 kayobe overcloud host configure --limit $1 --kolla-limit $1 -e os_release=jammy diff --git a/tools/ubuntu-upgrade-seed-hypervisor.sh b/tools/ubuntu-upgrade-seed-hypervisor.sh index cf15c6935..ad09f2b34 100755 --- a/tools/ubuntu-upgrade-seed-hypervisor.sh +++ b/tools/ubuntu-upgrade-seed-hypervisor.sh @@ -24,6 +24,6 @@ fi set -x -kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit seed-hypervisor +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit seed-hypervisor kayobe seed hypervisor host configure diff --git a/tools/ubuntu-upgrade-seed.sh b/tools/ubuntu-upgrade-seed.sh index d13081d64..4a48d5f36 100755 --- a/tools/ubuntu-upgrade-seed.sh +++ b/tools/ubuntu-upgrade-seed.sh @@ -24,6 +24,6 @@ fi set -x -kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml --limit seed +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit seed kayobe seed host configure From 5203d01a3760d02817aac168783e59a80146f2b7 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 6 Feb 2024 09:57:19 +0000 Subject: [PATCH 14/17] Trim trailing whitespaces --- etc/kayobe/ansible/ubuntu-upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 189d71951..432fa7ab4 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -38,7 +38,7 @@ - name: Reboot to apply updates reboot: become: true - when: file_status.stat.exists + when: file_status.stat.exists # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list - name: Ensure Jammy repo definitions exist in sources.list From 75b77019be73ec3310fc98ff5dc45e530f04bf3e Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 6 Feb 2024 11:12:58 +0000 Subject: [PATCH 15/17] Run OVN chassis playbook after Jammy upgrade --- etc/kayobe/ansible/ubuntu-upgrade.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 432fa7ab4..fa86e8a53 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -100,3 +100,7 @@ that: - ansible_facts.distribution_major_version == '22' - ansible_facts.distribution_release == 'jammy' + +- name: Run the OVN chassis priority fix playbook + import_playbook: "{{ lookup('ansible.builtin.env', 'KAYOBE_CONFIG_PATH') }}/ansible/ovn-fix-chassis-priorities.yml" + when: kolla_enable_ovn From 421740bc11c5a9fd57aff3696609613a0fee8b97 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 13 Feb 2024 14:25:49 +0000 Subject: [PATCH 16/17] Improve ubuntu upgrade playbook reliability --- etc/kayobe/ansible/ubuntu-upgrade.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index fa86e8a53..3b477731c 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -37,6 +37,8 @@ - name: Reboot to apply updates reboot: + reboot_timeout: 1200 + connect_timeout: 600 become: true when: file_status.stat.exists @@ -55,12 +57,6 @@ command: do-release-upgrade -f DistUpgradeViewNonInteractive become: true - - name: Ensure Jammy repo definitions do not exist in sources.list - blockinfile: - path: /etc/apt/sources.list - state: absent - become: true - - name: Ensure old venvs do not exist file: path: "/opt/kayobe/venvs/{{ item }}" @@ -86,8 +82,16 @@ vars: ansible_python_interpreter: /usr/bin/python3 tasks: + - name: Ensure Jammy repo definitions do not exist in sources.list + blockinfile: + path: /etc/apt/sources.list + state: absent + become: true + - name: Reboot and wait reboot: + reboot_timeout: 1200 + connect_timeout: 600 become: true - name: Update distribution facts From 09da39ce84af0ce550fa3b3ba09d48e2a9ed0c86 Mon Sep 17 00:00:00 2001 From: Seunghun Lee <45145778+seunghun1ee@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:17:02 +0000 Subject: [PATCH 17/17] Remove kolla limit for infra vm host configure Co-authored-by: Mark Goddard --- tools/ubuntu-upgrade-infra-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ubuntu-upgrade-infra-vm.sh b/tools/ubuntu-upgrade-infra-vm.sh index 155a5386a..8d5810174 100755 --- a/tools/ubuntu-upgrade-infra-vm.sh +++ b/tools/ubuntu-upgrade-infra-vm.sh @@ -31,4 +31,4 @@ set -x kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 -kayobe infra vm host configure --limit $1 --kolla-limit $1 -e os_release=jammy +kayobe infra vm host configure --limit $1 -e os_release=jammy