diff --git a/etc/kayobe/ansible/cephadm-crush-rules.yml b/etc/kayobe/ansible/cephadm-crush-rules.yml new file mode 100644 index 000000000..b95b9f960 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-crush-rules.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph CRUSH rules + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-crush-rules + tasks: + - import_role: + name: stackhpc.cephadm.crush_rules diff --git a/etc/kayobe/ansible/cephadm-deploy.yml b/etc/kayobe/ansible/cephadm-deploy.yml new file mode 100644 index 000000000..9cb0f712b --- /dev/null +++ b/etc/kayobe/ansible/cephadm-deploy.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy Ceph using Cephadm + any_errors_fatal: true + gather_facts: true + hosts: ceph + tags: + - cephadm + - cephadm-deploy + tasks: + - import_role: + name: stackhpc.cephadm.cephadm diff --git a/etc/kayobe/ansible/cephadm-ec-profiles.yml b/etc/kayobe/ansible/cephadm-ec-profiles.yml new file mode 100644 index 000000000..5fb0f3bdc --- /dev/null +++ b/etc/kayobe/ansible/cephadm-ec-profiles.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph EC profiles + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-ec-profiles + tasks: + - import_role: + name: stackhpc.cephadm.ec_profiles diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml new file mode 100644 index 000000000..0a47de8d0 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -0,0 +1,88 @@ +--- +- name: Gather Ceph configuration and keys and populate kayobe-config + gather_facts: false + hosts: mons + run_once: true + tags: + - cephadm + - cephadm-gather-keys + vars: + # Map from an OpenStack service to the directory in which to store Ceph keys for it. + kolla_service_to_key_dir: + cinder-backup: cinder/cinder-backup + cinder-volume: cinder/cinder-volume + glance: glance + manila: manila + nova: nova + # Map from an OpenStack service to the directory in which to store Ceph configuration for it. + kolla_service_to_conf_dir: + cinder-backup: cinder + cinder-volume: cinder + glance: glance + manila: manila + nova: nova + tasks: + - name: Get Ceph keys + stackhpc.cephadm.cephadm_key: + name: "{{ item }}" + output_format: plain + state: info + register: cephadm_key_info + become: true + loop: "{{ kolla_ceph_services | selectattr('required') | map(attribute='keys') | flatten | unique }}" + + - name: Generate ceph.conf + command: "cephadm shell -- ceph config generate-minimal-conf" + become: true + register: cephadm_ceph_conf + changed_when: false + + - name: Ensure Kolla config directories are present + file: + state: directory + path: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.name] }}" + loop: "{{ kolla_ceph_services | selectattr('required') }}" + loop_control: + label: + service: "{{ item.name }}" + delegate_to: localhost + + - name: Save Ceph keys to Kayobe configuration + vars: + key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}" + cephadm_key: "{{ key_info.stdout }}" + cephadm_user: "{{ item.1 }}" + copy: + # Include a trailing newline. + content: | + {{ cephadm_key }} + dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.0.name] }}/ceph.{{ cephadm_user }}.keyring" + loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" + loop_control: + label: + service: "{{ item.0.name }}" + key: "{{ item.1 }}" + delegate_to: localhost + notify: Please add and commit the Kayobe configuration + + - name: Save ceph.conf to Kayobe configuration + copy: + # Include a trailing newline. + # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf. + content: | + {{ cephadm_ceph_conf.stdout | regex_replace('\t') }} + dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf" + loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" + loop_control: + label: + service: "{{ item.0.name }}" + key: "{{ item.1 }}" + delegate_to: localhost + notify: Please add and commit the Kayobe configuration + + handlers: + - name: Please add and commit the Kayobe configuration + debug: + msg: >- + Please add and commit the Ceph configuration files and keys in Kayobe + configuration. Remember to encrypt the keys using Ansible Vault. diff --git a/etc/kayobe/ansible/cephadm-keys.yml b/etc/kayobe/ansible/cephadm-keys.yml new file mode 100644 index 000000000..6f632f4ca --- /dev/null +++ b/etc/kayobe/ansible/cephadm-keys.yml @@ -0,0 +1,11 @@ +--- +- name: Generate Ceph keys + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-keys + tasks: + - import_role: + name: stackhpc.cephadm.keys diff --git a/etc/kayobe/ansible/cephadm-pools.yml b/etc/kayobe/ansible/cephadm-pools.yml new file mode 100644 index 000000000..938e6de19 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-pools.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph pools + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-keys + tasks: + - import_role: + name: stackhpc.cephadm.pools diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml new file mode 100644 index 000000000..96c667f62 --- /dev/null +++ b/etc/kayobe/ansible/cephadm.yml @@ -0,0 +1,7 @@ +--- +# Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. +- import_playbook: cephadm-deploy.yml +- import_playbook: cephadm-ec-profiles.yml +- import_playbook: cephadm-crush-rules.yml +- import_playbook: cephadm-pools.yml +- import_playbook: cephadm-keys.yml diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml index 12deba374..f07933c1b 100644 --- a/etc/kayobe/ansible/configure-vxlan.yml +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -1,6 +1,6 @@ --- - name: Configure VXLAN - hosts: seed,controllers,compute + hosts: storage,seed,controllers,compute gather_facts: false vars: ansible_user: "{{ bootstrap_user }}" @@ -10,7 +10,7 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" roles: - - role: vxlan + - role: stackhpc.vxlan vars: vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}" vxlan_dstport: 4790 diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 24b4b4fac..aab4877f5 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -4,7 +4,7 @@ collections: version: 0.3.0 - name: pulp.squeezer version: 0.0.11 + - name: stackhpc.cephadm + version: 1.8.0 roles: - - name: vxlan - src: https://github.com/stackhpc/ansible-role-vxlan.git - version: b506e9817e1eb0a81ea66e0cddc40ab2ff66bc79 + - src: stackhpc.vxlan diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml new file mode 100644 index 000000000..7e1b314d4 --- /dev/null +++ b/etc/kayobe/cephadm.yml @@ -0,0 +1,140 @@ +--- +############################################################################### +# Cephadm deployment configuration. + +# Ceph release name. +cephadm_ceph_release: "pacific" + +# Ceph FSID. +#cephadm_fsid: + +# Ceph container image. +cephadm_image: "quay.io/ceph/ceph:v16.2.5" + +# Ceph container image tag. +cephadm_image_tag: "v16.2.5" + +# Ceph container image registry URL. +cephadm_registry_url: "{{ stackhpc_docker_registry }}" + +# Ceph container image registry username. +cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" + +# Ceph container image registry password. +cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" + +# Cephadm admin network interface. +cephadm_admin_interface: "{{ admin_oc_net_name | net_interface }}" + +# Ceph public network interface. +cephadm_public_interface: "{{ storage_net_name | net_interface }}" + +# Ceph public network subnet CIDR. +cephadm_public_network: "{{ storage_net_name | net_cidr }}" + +# Ceph cluster network interface. +cephadm_cluster_interface: "{{ storage_mgmt_net_name | net_interface }}" + +# Ceph cluster network subnet CIDR. +cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" + +# Whether to enable firewalld for Ceph storage hosts. +#cephadm_enable_firewalld: + +# Ceph OSD specification. +cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + +############################################################################### +# Ceph post-deployment configuration. + +# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role +# for format. +cephadm_ec_profiles: [] + +# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. +cephadm_crush_rules: [] + +# List of Ceph pools. See stackhpc.cephadm.pools role for format. +cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + +# List of Cephx keys. See stackhpc.cephadm.keys role for format. +cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + state: present + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + state: present + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + +############################################################################### +# Kolla Ceph auto-configuration. + +# List of Kolla Ansible services that require Ceph configuration files. +kolla_ceph_services: + - name: cinder-backup + keys: + - client.cinder + - client.cinder-backup + required: "{{ kolla_ceph_cinder_backup_required | bool }}" + - name: cinder-volume + keys: + - client.cinder + required: "{{ kolla_ceph_cinder_volume_required | bool }}" + - name: glance + keys: + - client.glance + required: "{{ kolla_ceph_glance_required | bool }}" + - name: manila + keys: + - client.manila + required: "{{ kolla_ceph_manila_required | bool }}" + - name: nova + keys: + - client.cinder + required: "{{ kolla_ceph_nova_required | bool }}" + +# Whether to generate Ceph configuration for Cinder volume. +kolla_ceph_cinder_volume_required: "{{ kolla_enable_cinder | bool }}" + +# Whether to generate Ceph configuration for Cinder backup. +kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(false) | bool }}" + +# Whether to generate Ceph configuration for Glance. +kolla_ceph_glance_required: "{{ kolla_enable_glance | bool }}" + +# Whether to generate Ceph configuration for Manila. +kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}" + +# Whether to generate Ceph configuration for Nova. +kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}" + diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml new file mode 100644 index 000000000..168500c8b --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -0,0 +1,3 @@ +--- +kolla_enable_cinder: true +kolla_enable_cinder_backup: true diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 9c769d3ea..ebfe64897 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -8,3 +8,12 @@ nova_compute_virt_type: qemu # processes to one per-service. openstack_service_workers: "1" openstack_service_rpc_workers: "1" + +# Glance Ceph configuration +glance_backend_ceph: "yes" + +# Cinder Ceph configuration +cinder_backend_ceph: "yes" + +# Nova Ceph configuration +nova_backend_ceph: "yes" diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 9c0545c39..79235df38 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -311,7 +311,7 @@ stackhpc_pulp_distribution_rpm_production: stackhpc_release_pulp_registry_url: "{{ stackhpc_release_pulp_url }}" # List of all image names. -stackhpc_pulp_images: +stackhpc_pulp_images_kolla: - barbican-api - barbican-base - barbican-keystone-listener @@ -436,33 +436,65 @@ stackhpc_pulp_images: - skydive-base # Common parameters for container image repositories. -stackhpc_pulp_repository_container_repos_common: +stackhpc_pulp_repository_container_repos_kolla_common: url: "{{ stackhpc_release_pulp_registry_url }}" policy: on_demand remote_username: "{{ stackhpc_release_pulp_username }}" remote_password: "{{ stackhpc_release_pulp_password }}" state: present + required: true -# List of container image repositories. -stackhpc_pulp_repository_container_repos: >- +# List of Kolla container image repositories. +stackhpc_pulp_repository_container_repos_kolla: >- {%- set repos = [] -%} - {%- for image in stackhpc_pulp_images -%} + {%- for image in stackhpc_pulp_images_kolla -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set repo = {"name": image_repo} -%} - {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_common | combine(repo)) -%} + {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%} {%- endfor -%} {{ repos }} -# Common parameters for container image distributions. -stackhpc_pulp_distribution_container_common: +# Common parameters for Kolla container image distributions. +stackhpc_pulp_distribution_container_kolla_common: state: present + required: true -# List of container image distributions. -stackhpc_pulp_distribution_container: >- +# List of Kolla container image distributions. +stackhpc_pulp_distribution_container_kolla: >- {%- set distributions = [] -%} - {%- for image in stackhpc_pulp_images -%} + {%- for image in stackhpc_pulp_images_kolla -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%} - {%- set _ = distributions.append(stackhpc_pulp_distribution_container_common | combine(distribution)) -%} + {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%} {%- endfor -%} {{ distributions }} + +# Whether to sync Ceph container images. +stackhpc_sync_ceph_images: false + +# List of Ceph container image repositories. +stackhpc_pulp_repository_container_repos_ceph: + - name: "ceph/ceph" + url: "https://quay.io" + policy: on_demand + state: present + include_tags: "{{ cephadm_image_tag }}" + required: "{{ stackhpc_sync_ceph_images | bool }}" + +# List of Ceph container image distributions. +stackhpc_pulp_distribution_container_ceph: + - name: ceph + repository: ceph/ceph + base_path: ceph/ceph + state: present + required: "{{ stackhpc_sync_ceph_images | bool }}" + +# List of container image repositories. +stackhpc_pulp_repository_container_repos: >- + {{ (stackhpc_pulp_repository_container_repos_kolla + + stackhpc_pulp_repository_container_repos_ceph) | selectattr('required') }} + +# List of container image distributions. +stackhpc_pulp_distribution_container: >- + {{ (stackhpc_pulp_distribution_container_kolla + + stackhpc_pulp_distribution_container_ceph) | selectattr('required') }}