From d06efcecc52b00f3e26dce3016accd04f6a52793 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Tue, 7 Mar 2017 22:03:50 +0800 Subject: [PATCH] Fix booting from volume failure Booting from volume require cinder's ceph client secret now. Move cinder before nova in site.yml, because nova depends on cinder ceph client key now. Change-Id: I01c9ed80843d98305b8963894c4917c21a35d3ac Closes-Bug: #1670676 --- ansible/roles/cinder/templates/cinder.conf.j2 | 2 +- ansible/roles/nova/tasks/ceph.yml | 39 +++++++++--- ansible/roles/nova/tasks/external-ceph.yml | 59 ++++++++++++++----- ansible/roles/nova/templates/secret.xml.j2 | 4 +- ansible/site.yml | 28 ++++----- etc/kolla/passwords.yml | 3 + kolla_ansible/cmd/genpwd.py | 10 +++- ...secret-uuid-password-f022e546930158ab.yaml | 4 ++ 8 files changed, 108 insertions(+), 41 deletions(-) create mode 100644 releasenotes/notes/require-cinder-rbd-secret-uuid-password-f022e546930158ab.yaml diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2 index e873cef400..99fc95b8dd 100644 --- a/ansible/roles/cinder/templates/cinder.conf.j2 +++ b/ansible/roles/cinder/templates/cinder.conf.j2 @@ -112,7 +112,7 @@ rbd_max_clone_depth = 5 rbd_store_chunk_size = 4 rados_connect_timeout = -1 rbd_user = cinder -rbd_secret_uuid = {{ rbd_secret_uuid }} +rbd_secret_uuid = {{ cinder_rbd_secret_uuid }} report_discard_supported = True {% endif %} diff --git a/ansible/roles/nova/tasks/ceph.yml b/ansible/roles/nova/tasks/ceph.yml index b7bd94839f..45b9d30d24 100644 --- a/ansible/roles/nova/tasks/ceph.yml +++ b/ansible/roles/nova/tasks/ceph.yml @@ -37,9 +37,16 @@ run_once: True # TODO(SamYaple): Improve failed_when and changed_when tests -- name: Pulling cephx keyring for libvirt +- name: Pulling nova cephx keyring for libvirt command: docker exec ceph_mon ceph auth get-key client.nova - register: cephx_raw_key + register: nova_cephx_raw_key + delegate_to: "{{ groups['ceph-mon'][0] }}" + changed_when: False + run_once: True + +- name: Pulling cinder cephx keyring for libvirt + command: docker exec ceph_mon ceph auth get-key client.cinder + register: cinder_cephx_raw_key delegate_to: "{{ groups['ceph-mon'][0] }}" changed_when: False run_once: True @@ -54,13 +61,31 @@ - name: Pushing secrets xml for libvirt template: src: "secret.xml.j2" - dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.xml" + dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.xml" mode: "0600" - when: inventory_hostname in groups['compute'] + when: + - inventory_hostname in groups['compute'] + - item.enabled | bool + with_items: + - uuid: "{{ rbd_secret_uuid }}" + name: client.nova secret + enabled: true + - uuid: "{{ cinder_rbd_secret_uuid }}" + name: client.cinder secret + enabled: "{{ cinder_backend_ceph }}" - name: Pushing secrets key for libvirt copy: - content: "{{ cephx_raw_key.stdout }}" - dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.base64" + content: "{{ item.content }}" + dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.base64" mode: "0600" - when: inventory_hostname in groups['compute'] + when: + - inventory_hostname in groups['compute'] + - item.enabled | bool + with_items: + - uuid: "{{ rbd_secret_uuid }}" + content: "{{ nova_cephx_raw_key.stdout }}" + enabled: true + - uuid: "{{ cinder_rbd_secret_uuid }}" + content: "{{ cinder_cephx_raw_key.stdout }}" + enabled: "{{ cinder_backend_ceph }}" diff --git a/ansible/roles/nova/tasks/external-ceph.yml b/ansible/roles/nova/tasks/external-ceph.yml index 74c6a0e158..4aa966a370 100644 --- a/ansible/roles/nova/tasks/external-ceph.yml +++ b/ansible/roles/nova/tasks/external-ceph.yml @@ -8,13 +8,21 @@ - "nova-libvirt/secrets" when: inventory_hostname in groups['compute'] -- name: Find keyring files - local_action: find paths="{{ node_custom_config }}/nova/" patterns="^ceph\.client\..*?\.keyring$" use_regex=True - register: cephx_keyring_files +- name: Check nova keyring file + local_action: stat path="{{ node_custom_config }}/nova/ceph.client.nova.keyring" + register: nova_cephx_keyring_file + failed_when: not nova_cephx_keyring_file.stat.exists -- name: Copy over ceph keyring file +- name: Check cinder keyring file + local_action: state path="{{ node_custom_config }}/nova/ceph.client.cinder.keyring" + register: cinder_cephx_keyring_file + failed_when: not cinder_cephx_keyring_file.stat.exists + when: cinder_backend_ceph | bool + +# NOTE: nova-compute and nova-libvirt only need ceph.client.nova.keyring. +- name: Copy over ceph nova keyring file copy: - src: "{{ cephx_keyring_files.files[0].path }}" + src: "{{ nova_cephx_keyring_file.stat.path }}" dest: "{{ node_config_directory }}/{{ item }}/" with_items: - nova-compute @@ -30,20 +38,43 @@ - nova-libvirt when: inventory_hostname in groups['compute'] -- name: Pushing secrets xml for libvirt +- name: Pushing nova secret xml for libvirt template: src: "secret.xml.j2" - dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.xml" + dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.xml" mode: "0600" - when: inventory_hostname in groups['compute'] + when: + - inventory_hostname in groups['compute'] + - item.enabled | bool + with_items: + - uuid: "{{ rbd_secret_uuid }}" + name: "client.nova secret" + enabled: true + - uuid: "{{ cinder_rbd_secret_uuid }}" + name: "client.cinder secret" + enabled: "{{ cinder_backend_ceph }}" -- name: Extract key from file - local_action: shell cat {{ cephx_keyring_files.files[0].path }} | grep -E 'key\s*=' | awk '{ print $3 }' - register: cephx_raw_key +- name: Extract nova key from file + local_action: shell cat "{{ nova_cephx_keyring_file.stat.path }}" | grep -E 'key\s*=' | awk '{ print $3 }' + register: nova_cephx_raw_key + +- name: Extract cinder key from file + local_action: shell cat "{{ cinder_cephx_keyring_file.stat.path }}" | grep -E 'key\s*=' | awk '{ print $3 }' + register: cinder_cephx_raw_key + when: cinder_backend_ceph | bool - name: Pushing secrets key for libvirt copy: - content: "{{ cephx_raw_key.stdout }}" - dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.base64" + content: "{{ item.content }}" + dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.base64" mode: "0600" - when: inventory_hostname in groups['compute'] + when: + - inventory_hostname in groups['compute'] + - item.enabled | bool + with_items: + - uuid: "{{ rbd_secret_uuid }}" + content: nova_cephx_raw_key + enabled: true + - uuid: "{{ cinder_rbd_secret_uuid }}" + content: cinder_cephx_raw_key + enabled: "{{ cinder_backend_ceph }}" diff --git a/ansible/roles/nova/templates/secret.xml.j2 b/ansible/roles/nova/templates/secret.xml.j2 index eab903be4f..9f63543a24 100644 --- a/ansible/roles/nova/templates/secret.xml.j2 +++ b/ansible/roles/nova/templates/secret.xml.j2 @@ -1,6 +1,6 @@ - {{ rbd_secret_uuid }} + {{ item.uuid }} - client.nova secret + {{ item.name }} diff --git a/ansible/site.yml b/ansible/site.yml index b5e05aa8b7..6065c9c4dc 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -245,6 +245,20 @@ tags: ironic, when: enable_ironic | bool } +- name: Apply role cinder + gather_facts: false + hosts: + - ceph-mon + - cinder-api + - cinder-backup + - cinder-scheduler + - cinder-volume + serial: '{{ serial|default("0") }}' + roles: + - { role: cinder, + tags: cinder, + when: enable_cinder | bool } + - name: Apply role nova gather_facts: false hosts: @@ -289,20 +303,6 @@ tags: kuryr, when: enable_kuryr | bool } -- name: Apply role cinder - gather_facts: false - hosts: - - ceph-mon - - cinder-api - - cinder-backup - - cinder-scheduler - - cinder-volume - serial: '{{ serial|default("0") }}' - roles: - - { role: cinder, - tags: cinder, - when: enable_cinder | bool } - - name: Apply role heat gather_facts: false hosts: diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 425f9a36e7..dec1a22a0e 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -5,7 +5,10 @@ # These options must be UUID4 values in string format # XXXXXXXX-XXXX-4XXX-XXXX-XXXXXXXXXXXX ceph_cluster_fsid: +# for backward compatible consideration, rbd_secret_uuid is only used for nova, +# cinder_rbd_secret_uuid is used for cinder rbd_secret_uuid: +cinder_rbd_secret_uuid: ################### # Database options diff --git a/kolla_ansible/cmd/genpwd.py b/kolla_ansible/cmd/genpwd.py index 11d15fbd80..cfd3319c43 100755 --- a/kolla_ansible/cmd/genpwd.py +++ b/kolla_ansible/cmd/genpwd.py @@ -51,9 +51,13 @@ def main(): passwords_file = os.path.expanduser(args.passwords) # These keys should be random uuids - uuid_keys = ['ceph_cluster_fsid', 'rbd_secret_uuid', - 'gnocchi_project_id', 'gnocchi_resource_id', - 'gnocchi_user_id', 'designate_pool_id', + uuid_keys = ['ceph_cluster_fsid', + 'rbd_secret_uuid', + 'cinder_rbd_secret_uuid', + 'gnocchi_project_id', + 'gnocchi_resource_id', + 'gnocchi_user_id', + 'designate_pool_id', 'karbor_openstack_infra_id'] # SSH key pair diff --git a/releasenotes/notes/require-cinder-rbd-secret-uuid-password-f022e546930158ab.yaml b/releasenotes/notes/require-cinder-rbd-secret-uuid-password-f022e546930158ab.yaml new file mode 100644 index 0000000000..81bd1fb900 --- /dev/null +++ b/releasenotes/notes/require-cinder-rbd-secret-uuid-password-f022e546930158ab.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + cinder_rbd_secret_uuid variable is requirement in passwords.yml file