[release-3.10] GlusterFS: Improve external cluster support

playbooks/init/vars/cluster_hosts.yml

@@ -22,5 +22,4 @@ g_glusterfs_registry_hosts: "{{ groups.glusterfs_registry | default(g_glusterfs_
 g_all_hosts: "{{ g_master_hosts | union(g_node_hosts) | union(g_etcd_hosts)
                  | union(g_new_etcd_hosts) | union(g_lb_hosts) | union(g_nfs_hosts)
                  | union(g_new_node_hosts)| union(g_new_master_hosts)
-                 | union(g_glusterfs_hosts) | union(g_glusterfs_registry_hosts)
                  | default([]) }}"

playbooks/openshift-glusterfs/config.yml

@@ -1,10 +1,10 @@
 ---
 - import_playbook: ../init/main.yml
   vars:
-    l_init_fact_hosts: "oo_masters_to_config:oo_glusterfs_to_config"
+    l_init_fact_hosts: "oo_masters_to_config:oo_nodes_to_config:oo_glusterfs_to_config"
     l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
     l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] | union(groups['oo_glusterfs_to_config']) }}"
     l_install_base_packages: True
-    l_base_packages_hosts: "oo_glusterfs_to_config"
+    l_base_packages_hosts: "oo_nodes_to_configi:&oo_glusterfs_to_config"
 
 - import_playbook: private/config.yml

roles/openshift_hosted/tasks/registry.yml

@@ -10,7 +10,7 @@
 - name: set openshift_hosted facts
   set_fact:
     # This determines the gluster_ips to use for the registry by looping over the glusterfs_registry group
-    openshift_hosted_registry_storage_glusterfs_ips: "{%- set gluster_ips = [] %}{% if groups.glusterfs_registry is defined %}{% for node in groups.glusterfs_registry %}{%- set _ = gluster_ips.append(hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip)) %}{% endfor %}{{ gluster_ips }}{% elif groups.glusterfs is defined %}{% for node in groups.glusterfs %}{%- set _ = gluster_ips.append(hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip)) %}{% endfor %}{{ gluster_ips }}{% else %}{{ openshift_hosted_registry_storage_glusterfs_ips }}{% endif %}"
+    openshift_hosted_registry_storage_glusterfs_ips: "{%- set gluster_ips = [] %}{% if groups.glusterfs_registry is defined %}{% for node in groups.glusterfs_registry %}{%- set _ = gluster_ips.append(hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip if 'openshift' in hostvars[node] else '')) %}{% endfor %}{{ gluster_ips }}{% elif groups.glusterfs is defined %}{% for node in groups.glusterfs %}{%- set _ = gluster_ips.append(hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip if 'openshift' in hostvars[node] else '')) %}{% endfor %}{{ gluster_ips }}{% else %}{{ openshift_hosted_registry_storage_glusterfs_ips }}{% endif %}"
 
 - name: Update registry environment variables when pushing via dns
   set_fact:

roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml

@@ -34,6 +34,16 @@
 - import_tasks: glusterfs_deploy.yml
   when: glusterfs_is_native
 
+- name: Start glusterd (external)
+  service:
+    name: glusterd
+    state: started
+    enabled: yes
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+  when:
+  - not glusterfs_is_native | bool
+
 - import_tasks: heketi_setup.yml
   when: glusterfs_heketi_is_native
 

roles/openshift_storage_glusterfs/tasks/glusterfs_uninstall.yml

@@ -48,37 +48,9 @@
     kind: node
     state: absent
     labels: "{{ glusterfs_nodeselector | lib_utils_oo_dict_to_list_of_dict }}"
-  with_items: "{{ groups.all }}"
+  with_items: "{{ groups.oo_nodes_to_config }}"
   when: "'openshift' in hostvars[item]"
 
-- name: Delete pre-existing GlusterFS config
-  file:
-    path: /var/lib/glusterd
-    state: absent
-  delegate_to: "{{ item }}"
-  with_items: "{{ glusterfs_nodes | default([]) }}"
-
-- name: Delete pre-existing additional GlusterFS config
-  file:
-    path: /etc/glusterfs
-    state: absent
-  delegate_to: "{{ item }}"
-  with_items: "{{ glusterfs_nodes | default([]) }}"
-
-- name: Delete pre-existing Heketi config
-  file:
-    path: /var/lib/heketi
-    state: absent
-  delegate_to: "{{ item }}"
-  with_items: "{{ glusterfs_nodes | default([]) }}"
-
-- name: Delete Glusterfs logs
-  file:
-    path: /var/log/glusterfs
-    state: absent
-  delegate_to: "{{ item }}"
-  with_items: "{{ glusterfs_nodes | default([]) }}"
-
 - name: Delete deploy resources
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
@@ -118,6 +90,58 @@
     selector: "gluster-s3"
   failed_when: False
 
+- name: Stop GlusterFS processes (external)
+  command: "/usr/share/glusterfs/scripts/stop-all-gluster-processes.sh"
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+  when:
+  - not glusterfs_is_native | bool
+
+- name: Unmount bricks (external)
+  shell: "for brick in `ls -d /var/lib/heketi/mounts/vg_*/*`; do umount $brick; done"
+  register: bricks
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+  when:
+  - not glusterfs_is_native | bool
+
+- name: Delete GlusterFS config
+  file:
+    path: /var/lib/glusterd
+    state: absent
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+
+- name: Delete additional GlusterFS config
+  file:
+    path: /etc/glusterfs
+    state: absent
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+
+- name: Delete Heketi config
+  file:
+    path: /var/lib/heketi
+    state: absent
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+
+- name: Delete Glusterfs logs
+  file:
+    path: /var/log/glusterfs
+    state: absent
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+
+# This recreates expected directory structures
+- name: Reinstall GlusterFS Server (external)
+  command: "yum reinstall -y glusterfs*"
+  delegate_to: "{{ item }}"
+  with_items: "{{ glusterfs_nodes | default([]) }}"
+  when:
+  - not glusterfs_is_native | bool
+  - not openshift_is_atomic | bool
+
 - name: Get GlusterFS storage devices state
   command: "pvdisplay -C --noheadings -o pv_name,vg_name {% for device in hostvars[item].glusterfs_devices %}{{ device }} {% endfor %}"
   register: devices_info

roles/openshift_storage_glusterfs/templates/heketi-endpoints.yml.j2

@@ -9,7 +9,7 @@ metadata:
 subsets:
 - addresses:
 {% for node in glusterfs_nodes %}
-  - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }}
+  - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip if "openshift" in hostvars[node] else "") }}
 {% endfor %}
   ports:
   - port: 1