From 1cae8a5e19bb9f4472dbf5eef757993375cdaa15 Mon Sep 17 00:00:00 2001 From: Piers Harding Date: Fri, 29 Jul 2022 07:49:19 +0100 Subject: [PATCH 1/4] Enable bootstrap for single host * add arbitrary bootstrap arguments * add generic post bootstrap config commands * parameterise distribution release (required on Jammy) * make ssh_user specification optional * add generic post bootstrap and osd commands Signed-off-by: Piers Harding --- roles/cephadm/README.md | 3 ++ roles/cephadm/defaults/main.yml | 3 ++ roles/cephadm/tasks/bootstrap.yml | 3 ++ roles/cephadm/tasks/config.yml | 12 +++++++ roles/cephadm/tasks/main.yml | 2 ++ roles/cephadm/tasks/pkg_debian.yml | 2 +- roles/cephadm/tasks/prereqs.yml | 1 + roles/commands/README.md | 28 +++++++++++++++ roles/commands/defaults/main.yml | 2 ++ roles/commands/tasks/commands.yml | 12 +++++++ roles/commands/tasks/main.yml | 2 ++ roles/commands/templates/cluster.yml.j2 | 46 +++++++++++++++++++++++++ roles/commands/vars/main.yml | 1 + 13 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 roles/cephadm/tasks/config.yml create mode 100644 roles/commands/README.md create mode 100644 roles/commands/defaults/main.yml create mode 100644 roles/commands/tasks/commands.yml create mode 100644 roles/commands/tasks/main.yml create mode 100644 roles/commands/templates/cluster.yml.j2 create mode 100644 roles/commands/vars/main.yml diff --git a/roles/cephadm/README.md b/roles/cephadm/README.md index 51e4a71..8ba9e34 100644 --- a/roles/cephadm/README.md +++ b/roles/cephadm/README.md @@ -41,6 +41,9 @@ All Ceph hosts must be in the `ceph` group. * `cephadm_ssh_public_key`: Location where ssh public key used by cephadm will be saved (default: /etc/ceph/cephadm.pub) * `cephadm_ssh_private_key`: Location where ssh private key used by cephadm will be saved (default: /etc/ceph/cephadm.id) * `cephadm_ssh_user`: Pre-existing user name that should be used for bootstrapping the cluster. User must have passwordless sudo enabled. Since 1.4.0 (default: `ansible_user`) + * `cephadm_bootstrap_additional_parameters`: additional arguments to pass to `cephadm bootstrap` + * `cephadm_ansible_distribution_release`: overide `ansible_distribution_release` for package repository + * `cephadm_bootstrap_configs: []`: array of `ceph config` commands to run after bootstrap (eg: `set mon mon_allow_pool_delete True`) * MONs and MGRs * `cephadm_mon_count`: Number of MONs to deploy (default: equals to number of hosts in `mons` Ansible group) * `cephadm_mgr_count`: Number of MGRs to deploy (default: equals to number of hosts in `mgrs` Ansible group) diff --git a/roles/cephadm/defaults/main.yml b/roles/cephadm/defaults/main.yml index 85b19fa..ed50927 100644 --- a/roles/cephadm/defaults/main.yml +++ b/roles/cephadm/defaults/main.yml @@ -22,6 +22,9 @@ cephadm_install_ceph_cli: False cephadm_ssh_public_key: "/etc/ceph/cephadm.pub" cephadm_ssh_private_key: "/etc/ceph/cephadm.id" cephadm_ssh_user: "{{ ansible_user }}" +cephadm_bootstrap_additional_parameters: "" +cephadm_ansible_distribution_release: "{{ ansible_distribution_release }}" +cephadm_bootstrap_configs: [] # MONs and MGRs cephadm_mon_count: "{{ groups.get('mons', []) | length }}" cephadm_mgr_count: "{{ groups.get('mgrs', []) | length }}" diff --git a/roles/cephadm/tasks/bootstrap.yml b/roles/cephadm/tasks/bootstrap.yml index f66e9af..a6f5ec8 100644 --- a/roles/cephadm/tasks/bootstrap.yml +++ b/roles/cephadm/tasks/bootstrap.yml @@ -19,7 +19,9 @@ {{ firewalld }} --ssh-private-key={{ cephadm_ssh_private_key }} --ssh-public-key={{ cephadm_ssh_public_key }} + {% if cephadm_ssh_user | length > 0 %} --ssh-user "{{ cephadm_ssh_user }}" + {% endif %} {% if cephadm_registry_url | length > 0 %} --registry-url={{ cephadm_registry_url }} --registry-username={{ cephadm_registry_username }} @@ -30,6 +32,7 @@ --fsid={{ cephadm_fsid }} {% endif %} --mon-ip={{ mon_ip }} + {{ cephadm_bootstrap_additional_parameters }} become: true when: not cephadm_check_ceph_conf.stat.exists diff --git a/roles/cephadm/tasks/config.yml b/roles/cephadm/tasks/config.yml new file mode 100644 index 0000000..a25e13f --- /dev/null +++ b/roles/cephadm/tasks/config.yml @@ -0,0 +1,12 @@ +--- +- name: Perform custom post bootstrap configs + block: + - name: Set configs + command: + cmd: "cephadm shell -- ceph config {{ item }}" + become: true + when: cephadm_bootstrap_configs | length > 0 + with_items: "{{ cephadm_bootstrap_configs }}" + + delegate_to: "{{ groups['mons'][0] }}" + run_once: True diff --git a/roles/cephadm/tasks/main.yml b/roles/cephadm/tasks/main.yml index c4ede44..762265f 100644 --- a/roles/cephadm/tasks/main.yml +++ b/roles/cephadm/tasks/main.yml @@ -13,3 +13,5 @@ when: cephadm_bootstrap | bool - include_tasks: "osds.yml" + +- include_tasks: "config.yml" diff --git a/roles/cephadm/tasks/pkg_debian.yml b/roles/cephadm/tasks/pkg_debian.yml index 8fbfe72..6f23dbe 100644 --- a/roles/cephadm/tasks/pkg_debian.yml +++ b/roles/cephadm/tasks/pkg_debian.yml @@ -7,7 +7,7 @@ - name: Ensure Ceph repositories are defined apt_repository: - repo: "deb https://download.ceph.com/debian-{{ item }}/ {{ ansible_distribution_release }} main" + repo: "deb https://download.ceph.com/debian-{{ item }}/ {{ cephadm_ansible_distribution_release }} main" state: "{{ 'present' if item == cephadm_ceph_release else 'absent' }}" when: not cephadm_custom_repos | bool become: true diff --git a/roles/cephadm/tasks/prereqs.yml b/roles/cephadm/tasks/prereqs.yml index 4202a1a..93357b1 100644 --- a/roles/cephadm/tasks/prereqs.yml +++ b/roles/cephadm/tasks/prereqs.yml @@ -55,6 +55,7 @@ user: "{{ cephadm_ssh_user }}" state: present key: "{{ content }}" + when: "cephadm_ssh_user | length > 0" become: true - name: Ensure the Logrotate package is installed diff --git a/roles/commands/README.md b/roles/commands/README.md new file mode 100644 index 0000000..522f959 --- /dev/null +++ b/roles/commands/README.md @@ -0,0 +1,28 @@ +# commands + +This role executes arbitrary commands against a Ceph cluster using `cephadm`. + +## Prerequisites + +### Host prerequisites + +* The role assumes target hosts connection over SSH with user that has passwordless sudo configured. +* Either direct Internet access or private registry with desired Ceph image accessible to all hosts is required. + +### Inventory + +This role assumes the existence of the following groups: + +* `mons` + +All Ceph hosts must be in the `ceph` group. + +## Role variables + +* `cephadm_commands`: A list of commands to pass to `cephadm shell -- ceph` + Example: + ``` + cephadm_commands: + - "fs new cephfs cephfs_metadata cephfs_data" + - "orch apply mds cephfs --placement 3" + ``` diff --git a/roles/commands/defaults/main.yml b/roles/commands/defaults/main.yml new file mode 100644 index 0000000..3c6a661 --- /dev/null +++ b/roles/commands/defaults/main.yml @@ -0,0 +1,2 @@ +--- +cephadm_commands: [] diff --git a/roles/commands/tasks/commands.yml b/roles/commands/tasks/commands.yml new file mode 100644 index 0000000..5629910 --- /dev/null +++ b/roles/commands/tasks/commands.yml @@ -0,0 +1,12 @@ +--- +- name: Perform custom commands + block: + - name: Execute commands + command: + cmd: "cephadm shell -- ceph {{ item }}" + become: true + when: cephadm_commands | length > 0 + with_items: "{{ cephadm_commands }}" + + delegate_to: "{{ groups['mons'][0] }}" + run_once: True diff --git a/roles/commands/tasks/main.yml b/roles/commands/tasks/main.yml new file mode 100644 index 0000000..f815b94 --- /dev/null +++ b/roles/commands/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: "commands.yml" diff --git a/roles/commands/templates/cluster.yml.j2 b/roles/commands/templates/cluster.yml.j2 new file mode 100644 index 0000000..9d69102 --- /dev/null +++ b/roles/commands/templates/cluster.yml.j2 @@ -0,0 +1,46 @@ +{% for host in groups['ceph'] %} +--- +service_type: host +hostname: {{ hostvars[host]['ansible_hostname'] }} +addr: {{ hostvars[host]['ansible_'~cephadm_admin_interface]['ipv4']['address'] }} +labels: +{% if host in groups['mons'] %} +- _admin +- mon +{% endif %} +{% if host in groups['mgrs'] %} +- mgr +{% endif %} +{% if host in groups['osds'] %} +- osd +{% endif %} +{% if host in groups.get('rgws', []) %} +- rgw +{% endif %} +{% endfor %} +--- +service_type: mon +placement: + count: {{ cephadm_mon_count }} + label: "mon" +--- +service_type: mgr +placement: + count: {{ cephadm_mgr_count }} + label: "mgr" +--- +service_type: crash +placement: + host_pattern: "*" +{% if groups.get('rgws', []) | length > 0 %} +{% for service in cephadm_radosgw_services %} +--- +service_type: rgw +service_id: {{ service.id }} +placement: + label: "rgw" +{% if service.port is defined %} +rgw_frontend_port: {{ service.port }} +{% endif %} +{% endfor %} +{% endif %} diff --git a/roles/commands/vars/main.yml b/roles/commands/vars/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/commands/vars/main.yml @@ -0,0 +1 @@ +--- From 39188adb0e452cc33bd92a8d414ca4ad55ea3150 Mon Sep 17 00:00:00 2001 From: Piers Harding Date: Mon, 1 Aug 2022 11:21:48 +0100 Subject: [PATCH 2/4] Fix for comments: * tidy up gratuitous copies in commands * rename variable specifically for debian * clear up documentation Signed-off-by: Piers Harding --- roles/cephadm/README.md | 2 +- roles/cephadm/defaults/main.yml | 2 +- roles/cephadm/tasks/pkg_debian.yml | 2 +- roles/commands/README.md | 2 +- roles/commands/templates/cluster.yml.j2 | 46 ------------------------- roles/commands/vars/main.yml | 1 - 6 files changed, 4 insertions(+), 51 deletions(-) delete mode 100644 roles/commands/templates/cluster.yml.j2 delete mode 100644 roles/commands/vars/main.yml diff --git a/roles/cephadm/README.md b/roles/cephadm/README.md index 8ba9e34..dd2469b 100644 --- a/roles/cephadm/README.md +++ b/roles/cephadm/README.md @@ -42,7 +42,7 @@ All Ceph hosts must be in the `ceph` group. * `cephadm_ssh_private_key`: Location where ssh private key used by cephadm will be saved (default: /etc/ceph/cephadm.id) * `cephadm_ssh_user`: Pre-existing user name that should be used for bootstrapping the cluster. User must have passwordless sudo enabled. Since 1.4.0 (default: `ansible_user`) * `cephadm_bootstrap_additional_parameters`: additional arguments to pass to `cephadm bootstrap` - * `cephadm_ansible_distribution_release`: overide `ansible_distribution_release` for package repository + * `cephadm_apt_repo_dist`: overide `ansible_distribution_release` for debian package repository * `cephadm_bootstrap_configs: []`: array of `ceph config` commands to run after bootstrap (eg: `set mon mon_allow_pool_delete True`) * MONs and MGRs * `cephadm_mon_count`: Number of MONs to deploy (default: equals to number of hosts in `mons` Ansible group) diff --git a/roles/cephadm/defaults/main.yml b/roles/cephadm/defaults/main.yml index ed50927..ad74724 100644 --- a/roles/cephadm/defaults/main.yml +++ b/roles/cephadm/defaults/main.yml @@ -23,7 +23,7 @@ cephadm_ssh_public_key: "/etc/ceph/cephadm.pub" cephadm_ssh_private_key: "/etc/ceph/cephadm.id" cephadm_ssh_user: "{{ ansible_user }}" cephadm_bootstrap_additional_parameters: "" -cephadm_ansible_distribution_release: "{{ ansible_distribution_release }}" +cephadm_apt_repo_dist: "{{ ansible_distribution_release }}" cephadm_bootstrap_configs: [] # MONs and MGRs cephadm_mon_count: "{{ groups.get('mons', []) | length }}" diff --git a/roles/cephadm/tasks/pkg_debian.yml b/roles/cephadm/tasks/pkg_debian.yml index 6f23dbe..88c4cde 100644 --- a/roles/cephadm/tasks/pkg_debian.yml +++ b/roles/cephadm/tasks/pkg_debian.yml @@ -7,7 +7,7 @@ - name: Ensure Ceph repositories are defined apt_repository: - repo: "deb https://download.ceph.com/debian-{{ item }}/ {{ cephadm_ansible_distribution_release }} main" + repo: "deb https://download.ceph.com/debian-{{ item }}/ {{ cephadm_apt_repo_dist }} main" state: "{{ 'present' if item == cephadm_ceph_release else 'absent' }}" when: not cephadm_custom_repos | bool become: true diff --git a/roles/commands/README.md b/roles/commands/README.md index 522f959..e805d8f 100644 --- a/roles/commands/README.md +++ b/roles/commands/README.md @@ -15,7 +15,7 @@ This role assumes the existence of the following groups: * `mons` -All Ceph hosts must be in the `ceph` group. +with at least one host in it - see the `cephadm` role for more details. ## Role variables diff --git a/roles/commands/templates/cluster.yml.j2 b/roles/commands/templates/cluster.yml.j2 deleted file mode 100644 index 9d69102..0000000 --- a/roles/commands/templates/cluster.yml.j2 +++ /dev/null @@ -1,46 +0,0 @@ -{% for host in groups['ceph'] %} ---- -service_type: host -hostname: {{ hostvars[host]['ansible_hostname'] }} -addr: {{ hostvars[host]['ansible_'~cephadm_admin_interface]['ipv4']['address'] }} -labels: -{% if host in groups['mons'] %} -- _admin -- mon -{% endif %} -{% if host in groups['mgrs'] %} -- mgr -{% endif %} -{% if host in groups['osds'] %} -- osd -{% endif %} -{% if host in groups.get('rgws', []) %} -- rgw -{% endif %} -{% endfor %} ---- -service_type: mon -placement: - count: {{ cephadm_mon_count }} - label: "mon" ---- -service_type: mgr -placement: - count: {{ cephadm_mgr_count }} - label: "mgr" ---- -service_type: crash -placement: - host_pattern: "*" -{% if groups.get('rgws', []) | length > 0 %} -{% for service in cephadm_radosgw_services %} ---- -service_type: rgw -service_id: {{ service.id }} -placement: - label: "rgw" -{% if service.port is defined %} -rgw_frontend_port: {{ service.port }} -{% endif %} -{% endfor %} -{% endif %} diff --git a/roles/commands/vars/main.yml b/roles/commands/vars/main.yml deleted file mode 100644 index ed97d53..0000000 --- a/roles/commands/vars/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- From ae75f6cddfec4d5d1748095469b2515c790ee9ca Mon Sep 17 00:00:00 2001 From: Piers Harding Date: Mon, 1 Aug 2022 12:11:51 +0100 Subject: [PATCH 3/4] remove the configs tasks Signed-off-by: Piers Harding --- roles/cephadm/README.md | 3 +-- roles/cephadm/defaults/main.yml | 1 - roles/cephadm/tasks/config.yml | 12 ------------ roles/cephadm/tasks/main.yml | 2 -- 4 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 roles/cephadm/tasks/config.yml diff --git a/roles/cephadm/README.md b/roles/cephadm/README.md index dd2469b..14af8f8 100644 --- a/roles/cephadm/README.md +++ b/roles/cephadm/README.md @@ -42,8 +42,7 @@ All Ceph hosts must be in the `ceph` group. * `cephadm_ssh_private_key`: Location where ssh private key used by cephadm will be saved (default: /etc/ceph/cephadm.id) * `cephadm_ssh_user`: Pre-existing user name that should be used for bootstrapping the cluster. User must have passwordless sudo enabled. Since 1.4.0 (default: `ansible_user`) * `cephadm_bootstrap_additional_parameters`: additional arguments to pass to `cephadm bootstrap` - * `cephadm_apt_repo_dist`: overide `ansible_distribution_release` for debian package repository - * `cephadm_bootstrap_configs: []`: array of `ceph config` commands to run after bootstrap (eg: `set mon mon_allow_pool_delete True`) + * `cephadm_apt_repo_dist`: overide (default) `ansible_distribution_release` for debian package repository * MONs and MGRs * `cephadm_mon_count`: Number of MONs to deploy (default: equals to number of hosts in `mons` Ansible group) * `cephadm_mgr_count`: Number of MGRs to deploy (default: equals to number of hosts in `mgrs` Ansible group) diff --git a/roles/cephadm/defaults/main.yml b/roles/cephadm/defaults/main.yml index ad74724..64d115d 100644 --- a/roles/cephadm/defaults/main.yml +++ b/roles/cephadm/defaults/main.yml @@ -24,7 +24,6 @@ cephadm_ssh_private_key: "/etc/ceph/cephadm.id" cephadm_ssh_user: "{{ ansible_user }}" cephadm_bootstrap_additional_parameters: "" cephadm_apt_repo_dist: "{{ ansible_distribution_release }}" -cephadm_bootstrap_configs: [] # MONs and MGRs cephadm_mon_count: "{{ groups.get('mons', []) | length }}" cephadm_mgr_count: "{{ groups.get('mgrs', []) | length }}" diff --git a/roles/cephadm/tasks/config.yml b/roles/cephadm/tasks/config.yml deleted file mode 100644 index a25e13f..0000000 --- a/roles/cephadm/tasks/config.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Perform custom post bootstrap configs - block: - - name: Set configs - command: - cmd: "cephadm shell -- ceph config {{ item }}" - become: true - when: cephadm_bootstrap_configs | length > 0 - with_items: "{{ cephadm_bootstrap_configs }}" - - delegate_to: "{{ groups['mons'][0] }}" - run_once: True diff --git a/roles/cephadm/tasks/main.yml b/roles/cephadm/tasks/main.yml index 762265f..c4ede44 100644 --- a/roles/cephadm/tasks/main.yml +++ b/roles/cephadm/tasks/main.yml @@ -13,5 +13,3 @@ when: cephadm_bootstrap | bool - include_tasks: "osds.yml" - -- include_tasks: "config.yml" From 2e63e3171fb55ecbc5af2721d6cb2f10089461e0 Mon Sep 17 00:00:00 2001 From: Piers Harding Date: Tue, 2 Aug 2022 19:11:25 +0100 Subject: [PATCH 4/4] collapse commands into main and remove block Signed-off-by: Piers Harding --- roles/commands/tasks/commands.yml | 12 ------------ roles/commands/tasks/main.yml | 11 ++++++++++- 2 files changed, 10 insertions(+), 13 deletions(-) delete mode 100644 roles/commands/tasks/commands.yml diff --git a/roles/commands/tasks/commands.yml b/roles/commands/tasks/commands.yml deleted file mode 100644 index 5629910..0000000 --- a/roles/commands/tasks/commands.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Perform custom commands - block: - - name: Execute commands - command: - cmd: "cephadm shell -- ceph {{ item }}" - become: true - when: cephadm_commands | length > 0 - with_items: "{{ cephadm_commands }}" - - delegate_to: "{{ groups['mons'][0] }}" - run_once: True diff --git a/roles/commands/tasks/main.yml b/roles/commands/tasks/main.yml index f815b94..4a4b867 100644 --- a/roles/commands/tasks/main.yml +++ b/roles/commands/tasks/main.yml @@ -1,2 +1,11 @@ --- -- include_tasks: "commands.yml" +- name: Execute custom commands + command: + cmd: "cephadm shell -- ceph {{ item }}" + register: cephadm_commands_result + with_items: "{{ cephadm_commands }}" + become: true + when: cephadm_commands | length > 0 + + delegate_to: "{{ groups['mons'][0] }}" + run_once: True