From 18ec478aa55286ea8755aa9c662771fa4bb441cf Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 22 Jul 2022 10:38:23 +0100 Subject: [PATCH 1/3] Adds support for cold migration to drain script --- etc/kayobe/ansible/nova-compute-drain.yml | 85 ++++++------------- .../nova-compute-drain/defaults/main.yml | 4 + .../nova-compute-drain/tasks/cold-migrate.yml | 38 +++++++++ .../tasks/instance-info.yml | 15 ++++ .../nova-compute-drain/tasks/live-migrate.yml | 17 ++++ .../roles/nova-compute-drain/tasks/setup.yml | 29 +++++++ 6 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml create mode 100644 etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml create mode 100644 etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml create mode 100644 etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml create mode 100644 etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml index 2296a0762..93bfc9d0a 100644 --- a/etc/kayobe/ansible/nova-compute-drain.yml +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -5,68 +5,39 @@ tags: - nova-compute-drain vars: - venv: "{{ virtualenv_path }}/openstack" - live_migration_fatal: true + nova_compute_migration_fatal: true tasks: - - name: Set up openstack cli virtualenv - pip: - virtualenv: "{{ venv }}" - name: - - python-openstackclient - state: latest - extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + - include_role: + name: nova-compute-drain + tasks_from: setup.yml run_once: true - delegate_to: "{{ groups['controllers'][0] }}" - - block: - - name: Query instances - command: > - {{ venv }}/bin/openstack - server list --host {{ ansible_facts.nodename }} - --all-projects - --status ACTIVE - --format json - register: instances + - include_role: + name: nova-compute-drain + tasks_from: instance-info.yml - - name: Live migrate instances - command: > - {{ venv }}/bin/openstack - --os-compute-api-version 2.25 - server migrate - {{ instance_uuid }} - --live-migration - --wait - loop: "{{ instances.stdout | from_json }}" - loop_control: - label: "{{ instance_uuid }}" - vars: - instance_uuid: "{{ item.ID | default }}" - register: result - failed_when: - - live_migration_fatal | bool - - result is failed + - include_role: + name: nova-compute-drain + tasks_from: live-migrate.yml + loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'ACTIVE') | list }}" + loop_control: + label: "{{ item.ID | default }}" - - name: Query instances - command: > - {{ venv }}/bin/openstack - server list --host {{ ansible_facts.nodename }} - --all-projects - --status ACTIVE - --format json - register: instances + - include_role: + name: nova-compute-drain + tasks_from: cold-migrate.yml + loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'SHUTOFF') | list }}" + loop_control: + label: "{{ item.ID | default }}" - - name: Fail if there are instances still on the host - fail: - msg: > - Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} - when: - - live_migration_fatal | bool - - instances.stdout | from_json | length > 0 + - include_role: + name: nova-compute-drain + tasks_from: instance-info.yml - delegate_to: "{{ groups['controllers'][0] }}" - environment: "{{ openstack_auth_env }}" + - name: Fail if there are instances still on the host + fail: + msg: > + Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} when: - - "'compute' in group_names" - - groups['compute'] | length > 1 - vars: - ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + - nova_compute_migration_fatal | bool + - nova_compute_drain_instance_info | length > 0 diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml b/etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml new file mode 100644 index 000000000..3722cff60 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +nova_compute_drain_venv: "{{ virtualenv_path }}/openstack" +nova_compute_drain_delegate_host: "{{ groups['controllers'][0] }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml new file mode 100644 index 000000000..4a6288260 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml @@ -0,0 +1,38 @@ +--- + +- block: + - name: "Cold migrate instance: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack + --os-compute-api-version 2.25 + server migrate + {{ instance_uuid }} + --wait + register: result + + - name: "Wait for VERIFY_RESIZE: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack server show {{ instance_uuid }} -f value -c status + register: result + until: result.stdout == 'VERIFY_RESIZE' or result.stdout == 'SHUTOFF' + retries: 10 + delay: 30 + + - name: "Confirm resize: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack server migrate confirm {{ instance_uuid }} + when: result.stdout == 'VERIFY_RESIZE' + + - name: "Wait for SHUTOFF: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack server show {{ instance_uuid }} -f value -c status + register: result + until: result.stdout == 'SHUTOFF' + retries: 10 + delay: 30 + environment: "{{ openstack_auth_env }}" + vars: + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" + instance_uuid: "{{ item.ID | default }}" + rescue: + - meta: noop diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml new file mode 100644 index 000000000..599692536 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml @@ -0,0 +1,15 @@ +--- +- name: Query instances + command: > + {{ nova_compute_drain_venv }}/bin/openstack + server list --host {{ ansible_facts.fqdn }} + --all-projects + --format json + register: instances + environment: "{{ openstack_auth_env }}" + vars: + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" + +- name: Set fact containing list of instances + set_fact: + nova_compute_drain_instance_info: "{{ instances.stdout | from_json }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml new file mode 100644 index 000000000..906938588 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml @@ -0,0 +1,17 @@ +--- + +- name: "Live migrate instance: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack + --os-compute-api-version 2.25 + server migrate + {{ instance_uuid }} + --live-migration + --wait + environment: "{{ openstack_auth_env }}" + vars: + instance_uuid: "{{ item.ID | default }}" + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" + register: result + failed_when: + - result is failed diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml new file mode 100644 index 000000000..ad4b650d9 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml @@ -0,0 +1,29 @@ +--- + +- name: Initiate openstack cli virtualenv + pip: + virtualenv: "{{ nova_compute_drain_venv }}" + name: + - pip + - setuptools + state: latest + virtualenv_command: /usr/bin/python3.6 -m venv + run_once: true + delegate_to: "{{ nova_compute_drain_delegate_host }}" + vars: + # NOTE: Without this, the delegate ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host | default(nova_compute_drain_delegate_host) }}" + +- name: Install openstack CLI tools in virtualenv + pip: + virtualenv: "{{ nova_compute_drain_venv }}" + name: + - python-openstackclient + + run_once: true + delegate_to: "{{ nova_compute_drain_delegate_host }}" + vars: + # NOTE: Without this, the delegate's ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host | default(nova_compute_drain_delegate_host) }}" From 969de52dbeb2204560c9f240eacc1f2165d6a8ab Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 16 Nov 2022 09:37:35 +0000 Subject: [PATCH 2/3] Address code review comments --- etc/kayobe/ansible/nova-compute-drain.yml | 32 ------------------- .../nova-compute-drain/tasks/cold-migrate.yml | 1 + .../tasks/instance-info.yml | 3 +- .../nova-compute-drain/tasks/live-migrate.yml | 1 + .../roles/nova-compute-drain/tasks/setup.yml | 2 +- 5 files changed, 5 insertions(+), 34 deletions(-) diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml index 93bfc9d0a..a056deb52 100644 --- a/etc/kayobe/ansible/nova-compute-drain.yml +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -9,35 +9,3 @@ tasks: - include_role: name: nova-compute-drain - tasks_from: setup.yml - run_once: true - - - include_role: - name: nova-compute-drain - tasks_from: instance-info.yml - - - include_role: - name: nova-compute-drain - tasks_from: live-migrate.yml - loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'ACTIVE') | list }}" - loop_control: - label: "{{ item.ID | default }}" - - - include_role: - name: nova-compute-drain - tasks_from: cold-migrate.yml - loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'SHUTOFF') | list }}" - loop_control: - label: "{{ item.ID | default }}" - - - include_role: - name: nova-compute-drain - tasks_from: instance-info.yml - - - name: Fail if there are instances still on the host - fail: - msg: > - Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} - when: - - nova_compute_migration_fatal | bool - - nova_compute_drain_instance_info | length > 0 diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml index 4a6288260..08c3985e1 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml @@ -31,6 +31,7 @@ retries: 10 delay: 30 environment: "{{ openstack_auth_env }}" + delegate_to: "{{ nova_compute_drain_delegate_host }}" vars: ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" instance_uuid: "{{ item.ID | default }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml index 599692536..390639469 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml @@ -2,10 +2,11 @@ - name: Query instances command: > {{ nova_compute_drain_venv }}/bin/openstack - server list --host {{ ansible_facts.fqdn }} + server list --host {{ ansible_facts.nodename }} --all-projects --format json register: instances + delegate_to: "{{ nova_compute_drain_delegate_host }}" environment: "{{ openstack_auth_env }}" vars: ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml index 906938588..9c3a5ba4e 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml @@ -8,6 +8,7 @@ {{ instance_uuid }} --live-migration --wait + delegate_to: "{{ nova_compute_drain_delegate_host }}" environment: "{{ openstack_auth_env }}" vars: instance_uuid: "{{ item.ID | default }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml index ad4b650d9..9705c8980 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml @@ -20,7 +20,7 @@ virtualenv: "{{ nova_compute_drain_venv }}" name: - python-openstackclient - + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" run_once: true delegate_to: "{{ nova_compute_drain_delegate_host }}" vars: From e99b6e4a00c7f1a37573c603b3b6ff552658266b Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 16 Nov 2022 09:54:46 +0000 Subject: [PATCH 3/3] fixup! Adds support for cold migration to drain script --- etc/kayobe/ansible/nova-compute-drain.yml | 1 + .../nova-compute-drain/tasks/cold-migrate.yml | 1 - .../nova-compute-drain/tasks/live-migrate.yml | 1 - .../roles/nova-compute-drain/tasks/main.yml | 33 +++++++++++++++++++ .../roles/nova-compute-drain/tasks/setup.yml | 2 -- 5 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml index a056deb52..d06544f74 100644 --- a/etc/kayobe/ansible/nova-compute-drain.yml +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -9,3 +9,4 @@ tasks: - include_role: name: nova-compute-drain + run_once: true diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml index 08c3985e1..9f863d1cd 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml @@ -34,6 +34,5 @@ delegate_to: "{{ nova_compute_drain_delegate_host }}" vars: ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" - instance_uuid: "{{ item.ID | default }}" rescue: - meta: noop diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml index 9c3a5ba4e..fa73f78eb 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml @@ -11,7 +11,6 @@ delegate_to: "{{ nova_compute_drain_delegate_host }}" environment: "{{ openstack_auth_env }}" vars: - instance_uuid: "{{ item.ID | default }}" ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" register: result failed_when: diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml new file mode 100644 index 000000000..fa78da2e4 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- include_tasks: + name: setup.yml + +- include_tasks: + name: instance-info.yml + +- include_tasks: + name: live-migrate.yml + loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'ACTIVE') | list }}" + loop_control: + label: "{{ item.ID | default }}" + vars: + instance_uuid: "{{ item.ID | default }}" + +- include_tasks: + name: cold-migrate.yml + loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'SHUTOFF') | list }}" + loop_control: + label: "{{ item.ID | default }}" + vars: + instance_uuid: "{{ item.ID | default }}" + +- include_tasks: + name: instance-info.yml + +- name: Fail if there are instances still on the host + fail: + msg: > + Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} + when: + - nova_compute_migration_fatal | bool + - nova_compute_drain_instance_info | length > 0 \ No newline at end of file diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml index 9705c8980..5c7023408 100644 --- a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml @@ -8,7 +8,6 @@ - setuptools state: latest virtualenv_command: /usr/bin/python3.6 -m venv - run_once: true delegate_to: "{{ nova_compute_drain_delegate_host }}" vars: # NOTE: Without this, the delegate ansible_host variable will not @@ -21,7 +20,6 @@ name: - python-openstackclient extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" - run_once: true delegate_to: "{{ nova_compute_drain_delegate_host }}" vars: # NOTE: Without this, the delegate's ansible_host variable will not