From 06ede41645550b1260f0c457aeca7ab7feffe918 Mon Sep 17 00:00:00 2001 From: Logan V Date: Fri, 9 Dec 2016 22:51:44 -0600 Subject: [PATCH] ceph-ansible integration Adds ceph-ansible integration to OSA using the upstream roles ceph-common, ceph-mon, and ceph-osd to support deployment of a Ceph cluster for use with OSA. The integration should be considered experimental at this point, however an integrated functional testing scenario named "ceph" is added which fully exercises the ceph-ansible deployment and configures OSA to use the deployed Ceph cluster. Once this is added to the OSA gate tests for the integrated build, that should help quickly stabilize any issues that arise with this integration. Change-Id: I275708096a71f66d73be1400c8a592c598382c1d --- ansible-role-requirements.yml | 16 ++++ deploy-guide/source/app-ceph.rst | 18 ++++ deploy-guide/source/app-resources.rst | 2 +- deploy-guide/source/app.rst | 1 + etc/openstack_deploy/conf.d/ceph.yml.aio | 9 ++ playbooks/ceph-install.yml | 90 +++++++++++++++++++ playbooks/inventory/env.d/ceph.yml | 53 +++++++++++ playbooks/inventory/group_vars/all.yml | 14 +++ playbooks/setup-infrastructure.yml | 1 + .../notes/ceph-ansible-d89e3a02b560f9b1.yaml | 15 ++++ tests/bootstrap-aio.yml | 12 ++- tests/roles/bootstrap-host/defaults/main.yml | 13 +++ tests/roles/bootstrap-host/tasks/main.yml | 7 ++ .../tasks/prepare_aio_config.yml | 2 +- .../bootstrap-host/tasks/prepare_ceph.yml | 74 +++++++++++++++ .../templates/user_variables.ceph.yml.j2 | 16 ++++ tests/test_inventory.py | 11 +++ 17 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 deploy-guide/source/app-ceph.rst create mode 100644 etc/openstack_deploy/conf.d/ceph.yml.aio create mode 100644 playbooks/ceph-install.yml create mode 100644 playbooks/inventory/env.d/ceph.yml create mode 100644 releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml create mode 100644 tests/roles/bootstrap-host/tasks/prepare_ceph.yml create mode 100644 tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 diff --git a/ansible-role-requirements.yml b/ansible-role-requirements.yml index c7abacd2f1..49dc8d0dfd 100644 --- a/ansible-role-requirements.yml +++ b/ansible-role-requirements.yml @@ -170,3 +170,19 @@ scm: git src: https://git.openstack.org/openstack/openstack-ansible-os_designate version: master +- name: ceph.ceph-common + scm: git + src: https://github.com/ceph/ansible-ceph-common + version: master +- name: ceph.ceph-docker-common + scm: git + src: https://github.com/ceph/ansible-ceph-docker-common + version: master +- name: ceph-mon + scm: git + src: https://github.com/ceph/ansible-ceph-mon + version: master +- name: ceph-osd + scm: git + src: https://github.com/ceph/ansible-ceph-osd + version: master diff --git a/deploy-guide/source/app-ceph.rst b/deploy-guide/source/app-ceph.rst new file mode 100644 index 0000000000..260dba6567 --- /dev/null +++ b/deploy-guide/source/app-ceph.rst @@ -0,0 +1,18 @@ +==================================== +Appendix G: Ceph-Ansible integration +==================================== + +OpenStack-Ansible allows `Ceph storage `_ cluster integration +using the roles maintained by the `Ceph-Ansible`_ project>. Deployers can +enable the ``ceph-install`` playbook by adding hosts to the +``ceph-mon_hosts`` and ``ceph-osd_hosts`` groups in +``openstack_user_config.yml``, and then configuring `Ceph-Ansible specific vars +`_ +in the OpenStack-Ansible ``user_variables.yml`` file. + +.. warning:: + + Ceph-Ansible integration in OpenStack-Ansible should be considered + experimental and for testing purposes only. + +.. _Ceph-Ansible: https://github.com/ceph/ceph-ansible/ diff --git a/deploy-guide/source/app-resources.rst b/deploy-guide/source/app-resources.rst index 9117f00897..b5285ea0f3 100644 --- a/deploy-guide/source/app-resources.rst +++ b/deploy-guide/source/app-resources.rst @@ -1,5 +1,5 @@ ================================= -Appendix G: Additional resources +Appendix H: Additional resources ================================= Ansible resources: diff --git a/deploy-guide/source/app.rst b/deploy-guide/source/app.rst index 2eb069078a..4272663259 100644 --- a/deploy-guide/source/app.rst +++ b/deploy-guide/source/app.rst @@ -11,4 +11,5 @@ Appendices app-security.rst app-networking.rst app-advanced-config-options.rst + app-ceph.rst app-resources.rst diff --git a/etc/openstack_deploy/conf.d/ceph.yml.aio b/etc/openstack_deploy/conf.d/ceph.yml.aio new file mode 100644 index 0000000000..b772765713 --- /dev/null +++ b/etc/openstack_deploy/conf.d/ceph.yml.aio @@ -0,0 +1,9 @@ +# The infra nodes where the Ceph mon services will run +ceph-mon_hosts: + aio1: + ip: 172.29.236.100 + +# The nodes that the Ceph OSD disks will be running on +ceph-osd_hosts: + aio1: + ip: 172.29.236.100 diff --git a/playbooks/ceph-install.yml b/playbooks/ceph-install.yml new file mode 100644 index 0000000000..d7d0e61d40 --- /dev/null +++ b/playbooks/ceph-install.yml @@ -0,0 +1,90 @@ +--- +# Copyright 2016, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install ceph mons + hosts: ceph-mon + max_fail_percentage: 20 + user: root + pre_tasks: + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-ceph" + dest: "/var/log/ceph" + - include: common-tasks/os-lxc-container-setup.yml + vars: + list_of_bind_mounts: + - bind_dir_path: "/var/lib/ceph" + mount_path: "/openstack/{{ inventory_hostname }}" + - include: common-tasks/package-cache-proxy.yml + roles: + - role: "ceph-mon" + tags: + - skip_ansible_lint + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: ceph_log_rotate + rsyslog_client_log_dir: "/var/log/ceph" + rsyslog_client_config_name: "99-ceph-rsyslog-client.conf" + tags: + - rsyslog + - role: "system_crontab_coordination" + tags: + - crontab + vars: + is_metal: "{{ properties.is_metal|default(false) }}" + tags: + - ceph + - ceph-mon + +- name: Gather ceph-mon facts + hosts: ceph-mon + user: root + gather_facts: no + tasks: + - name: Gather ceph-mon facts + action: setup + tags: + - ceph-mon-facts + tags: + - ceph + - ceph-osd + +- name: Install ceph osds + hosts: ceph-osd + max_fail_percentage: 20 + user: root + pre_tasks: + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-ceph" + dest: "/var/log/ceph" + - include: common-tasks/package-cache-proxy.yml + roles: + - role: "ceph-osd" + tags: + - skip_ansible_lint + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: ceph_log_rotate + rsyslog_client_log_dir: "/var/log/ceph" + rsyslog_client_config_name: "99-ceph-rsyslog-client.conf" + tags: + - rsyslog + - role: "system_crontab_coordination" + tags: + - crontab + tags: + - ceph + - ceph-osd diff --git a/playbooks/inventory/env.d/ceph.yml b/playbooks/inventory/env.d/ceph.yml new file mode 100644 index 0000000000..707e6e9841 --- /dev/null +++ b/playbooks/inventory/env.d/ceph.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2016, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +component_skel: + ceph-mon: + belongs_to: + - ceph_all + ceph-osd: + belongs_to: + - ceph_all + +container_skel: + ceph-mon_container: + belongs_to: + - ceph-mon_containers + contains: + - ceph-mon + properties: + service_name: ceph + ceph-osd_container: + belongs_to: + - ceph-osd_containers + contains: + - ceph-osd + properties: + is_metal: true + service_name: ceph + +physical_skel: + ceph-mon_containers: + belongs_to: + - all_containers + ceph-mon_hosts: + belongs_to: + - hosts + ceph-osd_containers: + belongs_to: + - all_containers + ceph-osd_hosts: + belongs_to: + - hosts diff --git a/playbooks/inventory/group_vars/all.yml b/playbooks/inventory/group_vars/all.yml index 9c7bc507ff..cbe4b5522f 100644 --- a/playbooks/inventory/group_vars/all.yml +++ b/playbooks/inventory/group_vars/all.yml @@ -452,3 +452,17 @@ ansible_ssh_extra_args: > -o ForwardX11=no -o ForwardAgent=yes -T + +## ceph-ansible configuration +mon_group_name: ceph-mon +osd_group_name: ceph-osd +ceph_stable: true +# The _stable_release var is used by both the OSA ceph_client role and the +# ceph-ansible roles. It is defaulted in ceph_client but set here to keep the +# OSA/ceph-ansible integrations in sync. +ceph_stable_release: jewel +fetch_directory: /etc/openstack_deploy/ceph-fetch/ +# tries to create /var/log/ceph as a directory and fails if the log link already +# exists. we handle the log dir creation so this is not something we need +# ceph-common to prepare for us. +rbd_client_directories: false diff --git a/playbooks/setup-infrastructure.yml b/playbooks/setup-infrastructure.yml index 820f80d3b2..96e0d5a0e8 100644 --- a/playbooks/setup-infrastructure.yml +++ b/playbooks/setup-infrastructure.yml @@ -20,5 +20,6 @@ - include: galera-install.yml - include: rabbitmq-install.yml - include: etcd-install.yml +- include: ceph-install.yml - include: utility-install.yml - include: rsyslog-install.yml diff --git a/releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml b/releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml new file mode 100644 index 0000000000..e947f236f2 --- /dev/null +++ b/releasenotes/notes/ceph-ansible-d89e3a02b560f9b1.yaml @@ -0,0 +1,15 @@ +--- +features: + - Playbooks for `ceph-ansible `_ have + been added to facilitate gate testing of the OpenStack-Ansible integration + with Ceph clusters, and can be used to integrate the two projects so that + OpenStack-Ansible can deploy and consume its own Ceph installation using + ceph-ansible. This should be considered an experimental integration until + further testing is been completed by deployers and the OpenStack-Ansible + gate to fine tune its stability and completeness. The ceph-install playbook + can be activated by adding hosts to the ``ceph-mon_hosts`` and + ``ceph-osd_hosts`` in the OSA inventory. A variety of ceph-ansible specific + variables will likely need to be configured in ``user_variables.yml`` to + configure ceph-ansible for your environment. Please reference the + `ceph-ansible repo `_ + for a list of variables the project supports. diff --git a/tests/bootstrap-aio.yml b/tests/bootstrap-aio.yml index f934409a79..f15baac324 100644 --- a/tests/bootstrap-aio.yml +++ b/tests/bootstrap-aio.yml @@ -22,7 +22,7 @@ - role: "pip_install" - role: "bootstrap-host" openstack_confd_entries: "{{ confd_overrides[scenario] }}" - scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}" + bootstrap_host_scenario: "{{ scenario }}" confd_overrides: aio: - name: aodh.yml.aio @@ -37,7 +37,17 @@ - name: neutron.yml.aio - name: nova.yml.aio - name: swift.yml.aio + ceph: + - name: ceph.yml.aio + - name: cinder.yml.aio + - name: glance.yml.aio + - name: heat.yml.aio + - name: horizon.yml.aio + - name: keystone.yml.aio + - name: neutron.yml.aio + - name: nova.yml.aio vars: + scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}" sshd: ListenAddress: - 0.0.0.0 diff --git a/tests/roles/bootstrap-host/defaults/main.yml b/tests/roles/bootstrap-host/defaults/main.yml index ea239bd6fd..39f0628412 100644 --- a/tests/roles/bootstrap-host/defaults/main.yml +++ b/tests/roles/bootstrap-host/defaults/main.yml @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Scenario used to bootstrap the host +bootstrap_host_scenario: aio + # Boolean option to implement OpenStack-Ansible configuration for an AIO # Switch to no for a multi-node configuration bootstrap_host_aio_config: yes @@ -44,6 +47,16 @@ bootstrap_host_loopback_nova: yes # Size of the Nova loopback disk in gigabytes (GB). bootstrap_host_loopback_nova_size: 1024 +# Boolean option to deploy the OSD loopback disks and cluster UUID for Ceph +bootstrap_host_ceph: "{{ (bootstrap_host_scenario == 'ceph') | bool }}" +# Size of the Ceph OSD loopbacks +bootstrap_host_loopback_ceph_size: 1024 +# Ceph OSDs to create on the AIO host +ceph_osd_images: + - 'ceph1' + - 'ceph2' + - 'ceph3' + ## Network configuration # The AIO bootstrap configures bridges for use with the AIO deployment. # By default, these bridges are configured to be independent of any physical diff --git a/tests/roles/bootstrap-host/tasks/main.yml b/tests/roles/bootstrap-host/tasks/main.yml index fa74d169df..de23ccc760 100644 --- a/tests/roles/bootstrap-host/tasks/main.yml +++ b/tests/roles/bootstrap-host/tasks/main.yml @@ -83,6 +83,13 @@ tags: - prepare-loopback-swift +# Prepare the Ceph cluster UUID and loopback disks +- include: prepare_ceph.yml + when: + - bootstrap_host_ceph | bool + tags: + - prepare-ceph + # Prepare the network interfaces - include: prepare_networking.yml tags: diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index 0eefdd3801..735ddb05e8 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -106,7 +106,7 @@ - name: Set the user_variables config_template: - src: user_variables.aio.yml.j2 + src: "user_variables.{{ bootstrap_host_scenario }}.yml.j2" dest: /etc/openstack_deploy/user_variables.yml config_overrides: "{{ user_variables_overrides | default({}) }}" config_type: yaml diff --git a/tests/roles/bootstrap-host/tasks/prepare_ceph.yml b/tests/roles/bootstrap-host/tasks/prepare_ceph.yml new file mode 100644 index 0000000000..e1794f3660 --- /dev/null +++ b/tests/roles/bootstrap-host/tasks/prepare_ceph.yml @@ -0,0 +1,74 @@ +--- +# Copyright 2016, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create sparse ceph OSD files + command: truncate -s {{ bootstrap_host_loopback_ceph_size }}G /openstack/{{ item }}.img + args: + creates: "/openstack/{{ item }}.img" + with_items: "{{ ceph_osd_images }}" + register: ceph_create + changed_when: false + tags: + - ceph-file-create + +- name: Create the ceph loopback device + command: losetup -f /openstack/{{ item.item }}.img --show + with_items: "{{ ceph_create.results }}" + register: ceph_create_loopback + when: not item|skipped + changed_when: false + tags: + - skip_ansible_lint + +- name: Ensure that rc.local exists + file: + path: /etc/rc.local + state: touch + mode: "u+x" + tags: + - ceph-rc-file + +- name: Create ceph loopback at boot time + lineinfile: + dest: /etc/rc.local + line: "losetup -f /openstack/{{ item }}.img" + with_items: "{{ ceph_osd_images }}" + +# TODO(logan): Move these vars to user_variables.ceph.yml.j2 once LP #1649381 +# is fixed and eliminate this task. +- name: Write ceph cluster config + copy: + content: | + --- + devices: {{ ceph_create_loopback.results | map(attribute='stdout') | list | to_yaml | trim }} + # The OSA ceph_client role does not support loading IPs from an inventory group, + # so we have to feed it a list of IPs + ceph_mons: "{% raw %}[ {% for host in groups[mon_group_name] %}'{{ hostvars[host]['ansible_host'] }}'{% if not loop.last %},{% endif %}{% endfor %} ]{% endraw %}" + cinder_backends: + "RBD": + volume_driver: cinder.volume.drivers.rbd.RBDDriver + rbd_pool: volumes + rbd_ceph_conf: /etc/ceph/ceph.conf + rbd_store_chunk_size: 8 + volume_backend_name: rbddriver + rbd_user: cinder + rbd_secret_uuid: "{% raw %}{{ cinder_ceph_client_uuid }}{% endraw %}" + report_discard_supported: true + dest: /etc/openstack_deploy/user_ceph_aio.yml + force: no + become: false + when: not ceph_create_loopback|skipped + tags: + - skip_ansible_lint diff --git a/tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 new file mode 100644 index 0000000000..212f2ad7c8 --- /dev/null +++ b/tests/roles/bootstrap-host/templates/user_variables.ceph.yml.j2 @@ -0,0 +1,16 @@ +{% include 'user_variables.aio.yml.j2' %} + +## ceph-ansible AIO settings +common_single_host_mode: true +monitor_interface: eth1 # Management network in the AIO +public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}" +journal_size: 100 +journal_collocation: true +pool_default_pg_num: 32 +openstack_config: true # Ceph ansible automatically creates pools & keys +cinder_ceph_client: cinder +cinder_default_volume_type: RBD +glance_ceph_client: glance +glance_default_store: rbd +glance_rbd_store_pool: images +nova_libvirt_images_rbd_pool: vms diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 97bdebae10..8f96f7dad6 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -152,6 +152,17 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase): 'ceilometer_api_container', 'ceilometer_collector', 'ceilometer_collector_container', + 'ceph_all', + 'ceph-mon_all', + 'ceph-mon_containers', + 'ceph-mon_container', + 'ceph-mon_hosts', + 'ceph-mon', + 'ceph-osd_all', + 'ceph-osd_containers', + 'ceph-osd_container', + 'ceph-osd_hosts', + 'ceph-osd', 'cinder_all', 'cinder_api', 'cinder_api_container',