From 4782c26e231cefedcb90b09f9bdae1e0e6b1eeb3 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 Dec 2021 16:03:44 +0000 Subject: [PATCH 1/3] Add playbooks and scripts to aid in performing maintainence on hypervisors These may be used to perform package updates that require a reboot: * disable compute service (nova-compute-disable.yml) * live migrate VMs away (nova-compute-drain.yml) * reboot (reboot.yml) * enable compute service (nova-compute-enable.yml) They may also be used to drain a compute node before a rebuild: * disable compute service (nova-compute-disable.yml) * live migrate VMs away (nova-compute-drain.yml) * stop services (kayobe overcloud service stop) --- etc/kayobe/ansible/nova-compute-disable.yml | 47 ++++++++++++++ etc/kayobe/ansible/nova-compute-drain.yml | 71 +++++++++++++++++++++ etc/kayobe/ansible/nova-compute-enable.yml | 50 +++++++++++++++ etc/kayobe/ansible/reboot.yml | 9 +++ 4 files changed, 177 insertions(+) create mode 100644 etc/kayobe/ansible/nova-compute-disable.yml create mode 100644 etc/kayobe/ansible/nova-compute-drain.yml create mode 100644 etc/kayobe/ansible/nova-compute-enable.yml create mode 100644 etc/kayobe/ansible/reboot.yml diff --git a/etc/kayobe/ansible/nova-compute-disable.yml b/etc/kayobe/ansible/nova-compute-disable.yml new file mode 100644 index 0000000000..98ab58e286 --- /dev/null +++ b/etc/kayobe/ansible/nova-compute-disable.yml @@ -0,0 +1,47 @@ +--- +- name: Disable nova compute service + hosts: compute + gather_facts: yes + tags: + - nova-compute-disable + vars: + venv: "{{ virtualenv_path }}/openstack" + disabled_reason: "Down for maintenance by nova-compute-disable.yml" + tasks: + - name: Set up openstack cli virtualenv + pip: + virtualenv: "{{ venv }}" + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Query nova compute services + command: > + {{ venv }}/bin/openstack + compute service list + --format json + --service nova-compute --host {{ ansible_facts.nodename }} + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + register: compute_services + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Disable nova compute service + command: > + {{ venv }}/bin/openstack + compute service set + {{ ansible_facts.nodename }} nova-compute + --disable --disable-reason '{{ disabled_reason }}' + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + vars: + service: "{{ compute_services.stdout | from_json | first }}" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + when: + - service.Status != 'disabled' diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml new file mode 100644 index 0000000000..2d47c0de31 --- /dev/null +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -0,0 +1,71 @@ +--- +- name: Drain a nova compute host of instances + hosts: compute + gather_facts: yes + tags: + - nova-compute-drain + vars: + venv: "{{ virtualenv_path }}/openstack" + live_migration_fatal: true + tasks: + - name: Set up openstack cli virtualenv + pip: + virtualenv: "{{ venv }}" + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + + - block: + - name: Query instances + command: > + {{ venv }}/bin/openstack + server list --host {{ ansible_facts.nodename }} + --all-projects + --status ACTIVE + --format json + register: instances + + - name: Live migrate instances + command: > + {{ venv }}/bin/openstack + server migrate + {{ instance_uuid }} + --live-migration + --wait + loop: "{{ instances.stdout | from_json }}" + loop_control: + label: "{{ instance_uuid }}" + vars: + instance_uuid: "{{ item.ID | default }}" + register: result + failed_when: + - live_migration_fatal | bool + - result is failed + + - name: Query instances + command: > + {{ venv }}/bin/openstack + server list --host {{ ansible_facts.nodename }} + --all-projects + --status ACTIVE + --format json + register: instances + + - name: Fail if there are instances still on the host + fail: + msg: > + Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} + when: + - live_migration_fatal | bool + - instances.stdout | from_json | length > 0 + + delegate_to: "{{ groups['controllers'][0] }}" + environment: "{{ openstack_auth_env }}" + when: + - "'compute' in group_names" + - groups['compute'] | length > 1 + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" diff --git a/etc/kayobe/ansible/nova-compute-enable.yml b/etc/kayobe/ansible/nova-compute-enable.yml new file mode 100644 index 0000000000..9d6d457202 --- /dev/null +++ b/etc/kayobe/ansible/nova-compute-enable.yml @@ -0,0 +1,50 @@ +--- +- name: Enable nova compute service + hosts: compute + gather_facts: yes + tags: + - nova-compute-enable + vars: + venv: "{{ virtualenv_path }}/openstack" + disabled_reason: "Down for maintenance by nova-compute-disable.yml" + tasks: + - name: Set up openstack cli virtualenv + pip: + virtualenv: "{{ venv }}" + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Query nova compute services + command: > + {{ venv }}/bin/openstack + compute service list + --format json + --service nova-compute --host {{ ansible_facts.nodename }} --long + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + register: compute_services + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Enable nova compute service + command: > + {{ venv }}/bin/openstack + compute service set + {{ ansible_facts.nodename }} nova-compute + --enable + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + vars: + service: "{{ compute_services.stdout | from_json | first }}" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + when: + # Don't enable the compute service if it was not disabled by + # nova-compute-disable.yml + - service.Status == 'disabled' + - service['Disabled Reason'] == disabled_reason diff --git a/etc/kayobe/ansible/reboot.yml b/etc/kayobe/ansible/reboot.yml new file mode 100644 index 0000000000..ed8217144e --- /dev/null +++ b/etc/kayobe/ansible/reboot.yml @@ -0,0 +1,9 @@ +--- +- name: Reboot the host + hosts: seed-hypervisor:seed:overcloud:infra-vms + tags: + - reboot + tasks: + - name: Reboot and wait + become: true + reboot: From dcdaa35dfba728373fd48f7474e109f62095ff2b Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 6 Jan 2022 15:55:16 +0100 Subject: [PATCH 2/3] Enable auto-configured block live migration Starting from python-openstackclient 5.5.0, it is not necessary to specify the --block-migration argument when instance disks are not on shared storage [1]. Instead, we can request the use of Nova API version 2.25, which added the auto value for block_migration. This was added to Mitaka so it is safe to use on all our deployments. This client version is only available from Wallaby. On Victoria, the change is a no-op. Note that a warning will be written to stderr about --disk-overcommit and --no-disk-overcommit options. This is a bug in openstackclient [2]. [1] https://review.opendev.org/c/openstack/python-openstackclient/+/771801 [2] https://storyboard.openstack.org/#!/story/2009657 --- etc/kayobe/ansible/nova-compute-drain.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml index 2d47c0de31..2296a0762b 100644 --- a/etc/kayobe/ansible/nova-compute-drain.yml +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -31,6 +31,7 @@ - name: Live migrate instances command: > {{ venv }}/bin/openstack + --os-compute-api-version 2.25 server migrate {{ instance_uuid }} --live-migration From 679d87225a6db6503245471875f6ed85ea1dcf66 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 10 Jan 2022 10:28:55 +0100 Subject: [PATCH 3/3] Allow to serialise critical upgrade tasks Note that `kayobe overcloud host package update` doesn't support serial execution. --- etc/kayobe/ansible/reboot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/ansible/reboot.yml b/etc/kayobe/ansible/reboot.yml index ed8217144e..af01e59448 100644 --- a/etc/kayobe/ansible/reboot.yml +++ b/etc/kayobe/ansible/reboot.yml @@ -1,6 +1,7 @@ --- - name: Reboot the host hosts: seed-hypervisor:seed:overcloud:infra-vms + serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(0) }}" tags: - reboot tasks: