Fix booting from volume failure

Booting from volume require cinder's ceph client secret now. Move cinder
before nova in site.yml, because nova depends on cinder ceph client key
now.

Change-Id: I01c9ed80843d98305b8963894c4917c21a35d3ac
Closes-Bug: #1670676

ansible/roles/cinder/templates/cinder.conf.j2

@@ -112,7 +112,7 @@ rbd_max_clone_depth = 5
 rbd_store_chunk_size = 4
 rados_connect_timeout = -1
 rbd_user = cinder
-rbd_secret_uuid = {{ rbd_secret_uuid }}
+rbd_secret_uuid = {{ cinder_rbd_secret_uuid }}
 report_discard_supported = True
 {% endif %}
 

ansible/roles/nova/tasks/ceph.yml

@@ -37,9 +37,16 @@
   run_once: True
 
 # TODO(SamYaple): Improve failed_when and changed_when tests
-- name: Pulling cephx keyring for libvirt
+- name: Pulling nova cephx keyring for libvirt
   command: docker exec ceph_mon ceph auth get-key client.nova
-  register: cephx_raw_key
+  register: nova_cephx_raw_key
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+- name: Pulling cinder cephx keyring for libvirt
+  command: docker exec ceph_mon ceph auth get-key client.cinder
+  register: cinder_cephx_raw_key
   delegate_to: "{{ groups['ceph-mon'][0] }}"
   changed_when: False
   run_once: True
@@ -54,13 +61,31 @@
 - name: Pushing secrets xml for libvirt
   template:
     src: "secret.xml.j2"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.xml"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.xml"
     mode: "0600"
-  when: inventory_hostname in groups['compute']
+  when:
+    - inventory_hostname in groups['compute']
+    - item.enabled | bool
+  with_items:
+    - uuid: "{{ rbd_secret_uuid }}"
+      name: client.nova secret
+      enabled: true
+    - uuid: "{{ cinder_rbd_secret_uuid }}"
+      name: client.cinder secret
+      enabled: "{{ cinder_backend_ceph }}"
 
 - name: Pushing secrets key for libvirt
   copy:
-    content: "{{ cephx_raw_key.stdout }}"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.base64"
+    content: "{{ item.content }}"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.base64"
     mode: "0600"
-  when: inventory_hostname in groups['compute']
+  when:
+    - inventory_hostname in groups['compute']
+    - item.enabled | bool
+  with_items:
+    - uuid: "{{ rbd_secret_uuid }}"
+      content: "{{ nova_cephx_raw_key.stdout }}"
+      enabled: true
+    - uuid: "{{ cinder_rbd_secret_uuid }}"
+      content: "{{ cinder_cephx_raw_key.stdout }}"
+      enabled: "{{ cinder_backend_ceph }}"

ansible/roles/nova/tasks/external-ceph.yml

@@ -8,13 +8,21 @@
     - "nova-libvirt/secrets"
   when: inventory_hostname in groups['compute']
 
-- name: Find keyring files
-  local_action: find paths="{{ node_custom_config }}/nova/" patterns="^ceph\.client\..*?\.keyring$" use_regex=True
-  register: cephx_keyring_files
+- name: Check nova keyring file
+  local_action: stat path="{{ node_custom_config }}/nova/ceph.client.nova.keyring"
+  register: nova_cephx_keyring_file
+  failed_when: not nova_cephx_keyring_file.stat.exists
 
-- name: Copy over ceph keyring file
+- name: Check cinder keyring file
+  local_action: state path="{{ node_custom_config }}/nova/ceph.client.cinder.keyring"
+  register: cinder_cephx_keyring_file
+  failed_when: not cinder_cephx_keyring_file.stat.exists
+  when: cinder_backend_ceph | bool
+
+# NOTE: nova-compute and nova-libvirt only need ceph.client.nova.keyring.
+- name: Copy over ceph nova keyring file
   copy:
-    src: "{{ cephx_keyring_files.files[0].path }}"
+    src: "{{ nova_cephx_keyring_file.stat.path }}"
     dest: "{{ node_config_directory }}/{{ item }}/"
   with_items:
     - nova-compute
@@ -30,20 +38,43 @@
     - nova-libvirt
   when: inventory_hostname in groups['compute']
 
-- name: Pushing secrets xml for libvirt
+- name: Pushing nova secret xml for libvirt
   template:
     src: "secret.xml.j2"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.xml"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.xml"
     mode: "0600"
-  when: inventory_hostname in groups['compute']
+  when:
+    - inventory_hostname in groups['compute']
+    - item.enabled | bool
+  with_items:
+    - uuid: "{{ rbd_secret_uuid }}"
+      name: "client.nova secret"
+      enabled: true
+    - uuid: "{{ cinder_rbd_secret_uuid }}"
+      name: "client.cinder secret"
+      enabled: "{{ cinder_backend_ceph }}"
 
-- name: Extract key from file
-  local_action: shell cat {{ cephx_keyring_files.files[0].path }} | grep -E 'key\s*=' | awk '{ print $3 }'
-  register: cephx_raw_key
+- name: Extract nova key from file
+  local_action: shell cat "{{ nova_cephx_keyring_file.stat.path }}" | grep -E 'key\s*=' | awk '{ print $3 }'
+  register: nova_cephx_raw_key
+
+- name: Extract cinder key from file
+  local_action: shell cat "{{ cinder_cephx_keyring_file.stat.path }}" | grep -E 'key\s*=' | awk '{ print $3 }'
+  register: cinder_cephx_raw_key
+  when: cinder_backend_ceph | bool
 
 - name: Pushing secrets key for libvirt
   copy:
-    content: "{{ cephx_raw_key.stdout }}"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.base64"
+    content: "{{ item.content }}"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.base64"
     mode: "0600"
-  when: inventory_hostname in groups['compute']
+  when:
+    - inventory_hostname in groups['compute']
+    - item.enabled | bool
+  with_items:
+    - uuid: "{{ rbd_secret_uuid }}"
+      content: nova_cephx_raw_key
+      enabled: true
+    - uuid: "{{ cinder_rbd_secret_uuid }}"
+      content: cinder_cephx_raw_key
+      enabled: "{{ cinder_backend_ceph }}"

ansible/roles/nova/templates/secret.xml.j2

@@ -1,6 +1,6 @@
 <secret ephemeral='no' private='no'>
-  <uuid>{{ rbd_secret_uuid }}</uuid>
+  <uuid>{{ item.uuid }}</uuid>
   <usage type='ceph'>
-    <name>client.nova secret</name>
+    <name>{{ item.name }}</name>
   </usage>
 </secret>

ansible/site.yml

@@ -245,6 +245,20 @@
         tags: ironic,
         when: enable_ironic | bool }
 
+- name: Apply role cinder
+  gather_facts: false
+  hosts:
+    - ceph-mon
+    - cinder-api
+    - cinder-backup
+    - cinder-scheduler
+    - cinder-volume
+  serial: '{{ serial|default("0") }}'
+  roles:
+    - { role: cinder,
+        tags: cinder,
+        when: enable_cinder | bool }
+
 - name: Apply role nova
   gather_facts: false
   hosts:
@@ -289,20 +303,6 @@
         tags: kuryr,
         when: enable_kuryr | bool }
 
-- name: Apply role cinder
-  gather_facts: false
-  hosts:
-    - ceph-mon
-    - cinder-api
-    - cinder-backup
-    - cinder-scheduler
-    - cinder-volume
-  serial: '{{ serial|default("0") }}'
-  roles:
-    - { role: cinder,
-        tags: cinder,
-        when: enable_cinder | bool }
-
 - name: Apply role heat
   gather_facts: false
   hosts:

etc/kolla/passwords.yml

@@ -5,7 +5,10 @@
 # These options must be UUID4 values in string format
 # XXXXXXXX-XXXX-4XXX-XXXX-XXXXXXXXXXXX
 ceph_cluster_fsid:
+# for backward compatible consideration, rbd_secret_uuid is only used for nova,
+# cinder_rbd_secret_uuid is used for cinder
 rbd_secret_uuid:
+cinder_rbd_secret_uuid:
 
 ###################
 # Database options

kolla_ansible/cmd/genpwd.py

@@ -51,9 +51,13 @@ def main():
     passwords_file = os.path.expanduser(args.passwords)
 
     # These keys should be random uuids
-    uuid_keys = ['ceph_cluster_fsid', 'rbd_secret_uuid',
-                 'gnocchi_project_id', 'gnocchi_resource_id',
-                 'gnocchi_user_id', 'designate_pool_id',
+    uuid_keys = ['ceph_cluster_fsid',
+                 'rbd_secret_uuid',
+                 'cinder_rbd_secret_uuid',
+                 'gnocchi_project_id',
+                 'gnocchi_resource_id',
+                 'gnocchi_user_id',
+                 'designate_pool_id',
                  'karbor_openstack_infra_id']
 
     # SSH key pair

releasenotes/notes/require-cinder-rbd-secret-uuid-password-f022e546930158ab.yaml

@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    cinder_rbd_secret_uuid variable is requirement in passwords.yml file