Add new container scenario

group_vars/all.yml.sample

@@ -543,6 +543,8 @@ dummy:
 #ceph_docker_enable_centos_extra_repo: false
 #ceph_docker_on_openstack: false
 #containerized_deployment: False
+#container_binary:
+
 
 ############
 # KV store #

group_vars/rhcs.yml.sample

@@ -543,6 +543,8 @@ ceph_docker_registry: "registry.access.redhat.com/rhceph/"
 #ceph_docker_enable_centos_extra_repo: false
 #ceph_docker_on_openstack: false
 #containerized_deployment: False
+#container_binary:
+
 
 ############
 # KV store #

infrastructure-playbooks/rolling_update.yml

@@ -177,7 +177,8 @@
         - not containerized_deployment
 
     - name: container | waiting for the containerized monitor to join the quorum...
-      command: docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json
+      command: >
+        {{ container_binary }} exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json
       register: ceph_health_raw
       until: >
         hostvars[mon_host]['ansible_hostname'] in (ceph_health_raw.stdout | from_json)["quorum_names"] or
@@ -214,8 +215,8 @@
       when: not containerized_deployment
 
     - name: set containerized osd flags
-      command: |
-        docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} osd set {{ item }}
+      command: >
+        {{ container_binary }} exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} osd set {{ item }}
       with_items:
         - noout
         - norebalance
@@ -232,6 +233,10 @@
   serial: 1
   become: True
   tasks:
+    - import_role:
+        name: ceph-defaults
+        private: false
+
     - name: non container - get current fsid
       command: "ceph --cluster {{ cluster }} fsid"
       register: cluster_uuid_non_container
@@ -240,7 +245,8 @@
         - not containerized_deployment
 
     - name: container - get current fsid
-      command: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} fsid"
+      command: >
+        {{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} fsid
       register: cluster_uuid_container
       delegate_to: "{{ groups[mon_group_name][0] }}"
       when:
@@ -275,7 +281,10 @@
           osd: allow *
           mds: allow *
         cluster: "{{ cluster }}"
-        containerized: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+        CEPH_UID: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
       when:
         - containerized_deployment
         - cephx
@@ -425,7 +434,7 @@
 
     - name: set_fact docker_exec_cmd_osd
       set_fact:
-        docker_exec_cmd_update_osd: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+        docker_exec_cmd_update_osd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
       when:
         - containerized_deployment
 
@@ -481,7 +490,7 @@
 
     - name: set_fact docker_exec_cmd_osd
       set_fact:
-        docker_exec_cmd_update_osd: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+        docker_exec_cmd_update_osd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
       when:
         - containerized_deployment
 
@@ -834,9 +843,10 @@
     - import_role:
         name: ceph-defaults
         private: false
+
     - name: set_fact docker_exec_cmd_status
       set_fact:
-        docker_exec_cmd_status: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+        docker_exec_cmd_status: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
       when:
         - containerized_deployment
 

infrastructure-playbooks/shrink-mon.yml

@@ -79,9 +79,9 @@
       when:
         - item != mon_to_kill
 
-    - name: set_fact docker_exec_cmd build docker exec command (containerized)
+    - name: "set_fact docker_exec_cmd build {{ container_binary }} exec command (containerized)"
       set_fact:
-        docker_exec_cmd: "docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }}"
+        docker_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }}"
       when: containerized_deployment
 
     - name: exit playbook, if can not connect to the cluster

infrastructure-playbooks/shrink-osd-ceph-disk.yml

@@ -57,9 +57,9 @@
         private: false
 
     # post-task for preceding import
-    - name: set_fact docker_exec_cmd build docker exec command (containerized)
+    - name: "set_fact docker_exec_cmd build {{ container_binary }} exec command (containerized)"
       set_fact:
-        docker_exec_cmd: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+        docker_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
       when: containerized_deployment
 
     - name: exit playbook, if can not connect to the cluster
@@ -102,7 +102,7 @@
     # NOTE(leseb): using '>' is the only way I could have the command working
     - name: find osd device based on the id
       shell: >
-        docker run --privileged=true -v /dev:/dev --entrypoint /usr/sbin/ceph-disk
+        {{ container_binary }} run --privileged=true -v /dev:/dev --entrypoint /usr/sbin/ceph-disk
         {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
         list | awk  -v pattern=osd.{{ item.0 }} '$0 ~ pattern {print $1}'
       with_together:
@@ -115,7 +115,7 @@
 
     - name: find osd dedicated devices - container
       shell: >
-        docker run --privileged=true -v /dev:/dev --entrypoint /usr/sbin/ceph-disk
+        {{ container_binary }} run --privileged=true -v /dev:/dev --entrypoint /usr/sbin/ceph-disk
         {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
         list | grep osd.{{ item.0 }} | grep -Eo '/dev/([hsv]d[a-z]{1,2})[0-9]{1,2}|/dev/nvme[0-9]n[0-9]p[0-9]'
       with_together:
@@ -162,7 +162,7 @@
 
     - name: zap ceph osd disks
       shell: |
-        docker run --rm \
+        {{ container_binary }} run --rm \
         --privileged=true \
         --name ceph-osd-zap-{{ hostvars[item.1]['ansible_hostname'] }}-{{ item.0.stdout }} \
         -v /dev/:/dev/ \
@@ -184,7 +184,7 @@
           pkname=$(lsblk --nodeps -no PKNAME "${osd}");
           then
             echo zapping ceph osd partitions "${osd}";
-            docker run --rm \
+            {{ container_binary }} run --rm \
             --privileged=true \
             --name ceph-osd-zap-{{ hostvars[item.0]['ansible_hostname'] }}-$(basename "${osd}") \
             -v /dev/:/dev/ \

infrastructure-playbooks/shrink-osd.yml

@@ -63,7 +63,7 @@
     # post-task for preceding import
     - name: set_fact docker_exec_cmd build docker exec command (containerized)
       set_fact:
-        docker_exec_cmd: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+        docker_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
       when: containerized_deployment
 
     - name: exit playbook, if can not connect to the cluster
@@ -91,6 +91,7 @@
       environment:
         CEPH_VOLUME_DEBUG: 1
         CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
       with_items: "{{ osd_hosts }}"
       delegate_to: "{{ item }}"
       register: osd_volumes
@@ -135,6 +136,7 @@
       environment:
         CEPH_VOLUME_DEBUG: 1
         CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
       delegate_to: "{{ item.host }}"
       with_items: "{{ osd_host_volumes_to_kill_non_container }}"
 

infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml

@@ -26,7 +26,7 @@
       when: ireallymeanit != 'yes'
 
 
-- name: make sure docker is present and started
+- name: gather facts
 
   hosts:
     - "{{ mon_group_name|default('mons') }}"
@@ -210,6 +210,10 @@
   become: true
   tasks:
 
+    - import_role:
+        name: ceph-defaults
+        private: false
+
     # pre-tasks for following importing
     - name: collect running osds and ceph-disk unit(s)
       shell: |
@@ -274,7 +278,7 @@
 
     - name: check if containerized osds are already running
       command: >
-        docker ps --filter='name=ceph-osd'
+        {{ container_binary }} ps --filter='name=ceph-osd'
       changed_when: false
       failed_when: false
       register: osd_running
@@ -313,12 +317,14 @@
 
     # post-task for preceding import -
     - name: get num_pgs
-      command: docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster "{{ cluster }}" -s --format json
+      command: >
+        {{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s --format json
       register: ceph_pgs
       delegate_to: "{{ groups[mon_group_name][0] }}"
 
     - name: container - waiting for clean pgs...
-      command: "docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s --format json"
+      command: >
+        {{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s --format json
       register: ceph_health_post
       until: >
         (((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) > 0)