Multinode deploy ceph

etc/kayobe/ansible/cephadm-crush-rules.yml

@@ -0,0 +1,11 @@
+---
+- name: Define Ceph CRUSH rules
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-crush-rules
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.crush_rules

etc/kayobe/ansible/cephadm-deploy.yml

@@ -0,0 +1,11 @@
+---
+- name: Deploy Ceph using Cephadm
+  any_errors_fatal: true
+  gather_facts: true
+  hosts: ceph
+  tags:
+    - cephadm
+    - cephadm-deploy
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.cephadm

etc/kayobe/ansible/cephadm-ec-profiles.yml

@@ -0,0 +1,11 @@
+---
+- name: Define Ceph EC profiles
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-ec-profiles
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.ec_profiles

etc/kayobe/ansible/cephadm-gather-keys.yml

@@ -0,0 +1,88 @@
+---
+- name: Gather Ceph configuration and keys and populate kayobe-config
+  gather_facts: false
+  hosts: mons
+  run_once: true
+  tags:
+    - cephadm
+    - cephadm-gather-keys
+  vars:
+    # Map from an OpenStack service to the directory in which to store Ceph keys for it.
+    kolla_service_to_key_dir:
+      cinder-backup: cinder/cinder-backup
+      cinder-volume: cinder/cinder-volume
+      glance: glance
+      manila: manila
+      nova: nova
+    # Map from an OpenStack service to the directory in which to store Ceph configuration for it.
+    kolla_service_to_conf_dir:
+      cinder-backup: cinder
+      cinder-volume: cinder
+      glance: glance
+      manila: manila
+      nova: nova
+  tasks:
+    - name: Get Ceph keys
+      stackhpc.cephadm.cephadm_key:
+        name: "{{ item }}"
+        output_format: plain
+        state: info
+      register: cephadm_key_info
+      become: true
+      loop: "{{ kolla_ceph_services | selectattr('required') | map(attribute='keys') | flatten | unique }}"
+
+    - name: Generate ceph.conf
+      command: "cephadm shell -- ceph config generate-minimal-conf"
+      become: true
+      register: cephadm_ceph_conf
+      changed_when: false
+
+    - name: Ensure Kolla config directories are present
+      file:
+        state: directory
+        path: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.name] }}"
+      loop: "{{ kolla_ceph_services | selectattr('required') }}"
+      loop_control:
+        label:
+          service: "{{ item.name }}"
+      delegate_to: localhost
+
+    - name: Save Ceph keys to Kayobe configuration
+      vars:
+        key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}"
+        cephadm_key: "{{ key_info.stdout }}"
+        cephadm_user: "{{ item.1 }}"
+      copy:
+        # Include a trailing newline.
+        content: |
+          {{ cephadm_key }}
+        dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.0.name] }}/ceph.{{ cephadm_user }}.keyring"
+      loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}"
+      loop_control:
+        label:
+          service: "{{ item.0.name }}"
+          key: "{{ item.1 }}"
+      delegate_to: localhost
+      notify: Please add and commit the Kayobe configuration
+
+    - name: Save ceph.conf to Kayobe configuration
+      copy:
+        # Include a trailing newline.
+        # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf.
+        content: |
+          {{ cephadm_ceph_conf.stdout | regex_replace('\t') }}
+        dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf"
+      loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}"
+      loop_control:
+        label:
+          service: "{{ item.0.name }}"
+          key: "{{ item.1 }}"
+      delegate_to: localhost
+      notify: Please add and commit the Kayobe configuration
+
+  handlers:
+    - name: Please add and commit the Kayobe configuration
+      debug:
+        msg: >-
+          Please add and commit the Ceph configuration files and keys in Kayobe
+          configuration. Remember to encrypt the keys using Ansible Vault.

etc/kayobe/ansible/cephadm-keys.yml

@@ -0,0 +1,11 @@
+---
+- name: Generate Ceph keys
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-keys
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.keys

etc/kayobe/ansible/cephadm-pools.yml

@@ -0,0 +1,11 @@
+---
+- name: Define Ceph pools
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-keys
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.pools

etc/kayobe/ansible/cephadm.yml

@@ -0,0 +1,7 @@
+---
+# Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys.
+- import_playbook: cephadm-deploy.yml
+- import_playbook: cephadm-ec-profiles.yml
+- import_playbook: cephadm-crush-rules.yml
+- import_playbook: cephadm-pools.yml
+- import_playbook: cephadm-keys.yml

etc/kayobe/ansible/configure-vxlan.yml

@@ -1,6 +1,6 @@
 ---
 - name: Configure VXLAN
-  hosts: seed,controllers,compute
+  hosts: storage,seed,controllers,compute
   gather_facts: false
   vars:
     ansible_user: "{{ bootstrap_user }}"
@@ -10,7 +10,7 @@
     # Work around no known_hosts entry on first boot.
     ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
   roles:
-    - role: vxlan
+    - role: stackhpc.vxlan
       vars:
         vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}"
         vxlan_dstport: 4790

etc/kayobe/ansible/requirements.yml

@@ -4,7 +4,7 @@ collections:
     version: 0.3.0
   - name: pulp.squeezer
     version: 0.0.11
+  - name: stackhpc.cephadm
+    version: 1.8.0
 roles:
-  - name: vxlan
-    src: https://github.com/stackhpc/ansible-role-vxlan.git
-    version: b506e9817e1eb0a81ea66e0cddc40ab2ff66bc79
+  - src: stackhpc.vxlan

etc/kayobe/cephadm.yml

@@ -0,0 +1,140 @@
+---
+###############################################################################
+# Cephadm deployment configuration.
+
+# Ceph release name.
+cephadm_ceph_release: "pacific"
+
+# Ceph FSID.
+#cephadm_fsid:
+
+# Ceph container image.
+cephadm_image: "quay.io/ceph/ceph:v16.2.5"
+
+# Ceph container image tag.
+cephadm_image_tag: "v16.2.5"
+
+# Ceph container image registry URL.
+cephadm_registry_url: "{{ stackhpc_docker_registry }}"
+
+# Ceph container image registry username.
+cephadm_registry_username: "{{ stackhpc_docker_registry_username }}"
+
+# Ceph container image registry password.
+cephadm_registry_password: "{{ stackhpc_docker_registry_password }}"
+
+# Cephadm admin network interface.
+cephadm_admin_interface: "{{ admin_oc_net_name | net_interface }}"
+
+# Ceph public network interface.
+cephadm_public_interface: "{{ storage_net_name | net_interface }}"
+
+# Ceph public network subnet CIDR.
+cephadm_public_network: "{{ storage_net_name | net_cidr }}"
+
+# Ceph cluster network interface.
+cephadm_cluster_interface: "{{ storage_mgmt_net_name | net_interface }}"
+
+# Ceph cluster network subnet CIDR.
+cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}"
+
+# Whether to enable firewalld for Ceph storage hosts.
+#cephadm_enable_firewalld:
+
+# Ceph OSD specification.
+cephadm_osd_spec:
+  service_type: osd
+  service_id: osd_spec_default
+  placement:
+    host_pattern: "*"
+  data_devices:
+    all: true
+
+###############################################################################
+# Ceph post-deployment configuration.
+
+# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role
+# for format.
+cephadm_ec_profiles: []
+
+# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format.
+cephadm_crush_rules: []
+
+# List of Ceph pools. See stackhpc.cephadm.pools role for format.
+cephadm_pools:
+  - name: backups
+    application: rbd
+    state: present
+  - name: images
+    application: rbd
+    state: present
+  - name: volumes
+    application: rbd
+    state: present
+  - name: vms
+    application: rbd
+    state: present
+
+# List of Cephx keys. See stackhpc.cephadm.keys role for format.
+cephadm_keys:
+  - name: client.cinder
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images"
+      mgr: "profile rbd pool=volumes, profile rbd pool=vms"
+    state: present
+  - name: client.cinder-backup
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=volumes, profile rbd pool=backups"
+      mgr: "profile rbd pool=volumes, profile rbd pool=backups"
+    state: present
+  - name: client.glance
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=images"
+      mgr: "profile rbd pool=images"
+    state: present
+
+###############################################################################
+# Kolla Ceph auto-configuration.
+
+# List of Kolla Ansible services that require Ceph configuration files.
+kolla_ceph_services:
+  - name: cinder-backup
+    keys:
+      - client.cinder
+      - client.cinder-backup
+    required: "{{ kolla_ceph_cinder_backup_required | bool }}"
+  - name: cinder-volume
+    keys:
+      - client.cinder
+    required: "{{ kolla_ceph_cinder_volume_required | bool }}"
+  - name: glance
+    keys:
+      - client.glance
+    required: "{{ kolla_ceph_glance_required | bool }}"
+  - name: manila
+    keys:
+      - client.manila
+    required: "{{ kolla_ceph_manila_required | bool }}"
+  - name: nova
+    keys:
+      - client.cinder
+    required: "{{ kolla_ceph_nova_required | bool }}"
+
+# Whether to generate Ceph configuration for Cinder volume.
+kolla_ceph_cinder_volume_required: "{{ kolla_enable_cinder | bool }}"
+
+# Whether to generate Ceph configuration for Cinder backup.
+kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(false) | bool }}"
+
+# Whether to generate Ceph configuration for Glance.
+kolla_ceph_glance_required: "{{ kolla_enable_glance | bool }}"
+
+# Whether to generate Ceph configuration for Manila.
+kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}"
+
+# Whether to generate Ceph configuration for Nova.
+kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}"
+

etc/kayobe/environments/ci-multinode/kolla.yml

@@ -0,0 +1,3 @@
+---
+kolla_enable_cinder: true
+kolla_enable_cinder_backup: true

etc/kayobe/environments/ci-multinode/kolla/globals.yml

@@ -8,3 +8,12 @@ nova_compute_virt_type: qemu
 # processes to one per-service.
 openstack_service_workers: "1"
 openstack_service_rpc_workers: "1"
+
+# Glance Ceph configuration
+glance_backend_ceph: "yes"
+
+# Cinder Ceph configuration
+cinder_backend_ceph: "yes"
+
+# Nova Ceph configuration
+nova_backend_ceph: "yes"