Fixes for upgrade_ceph playbook

Mostly backported from upstream

Change-Id: Ifce6bfe0f5ed6f5ce2fdb434329c0becadfaeb04

plugins/playbook/upgrade_ceph/decapod_plugin_playbook_upgrade_ceph/playbook.yaml

@@ -18,11 +18,8 @@
 # Please check https://github.com/asheplyakov/mos9-ceph-upgrade as a
 # standalone solution.
 
-- hosts: all
-  become: true
-  gather_facts: false
-  tasks:
-  - include: "{{ decapod_common_playbooks }}/plays/init.yaml"
+
+- include: "{{ decapod_common_playbooks }}/plays/init.yaml"
 
 
 - hosts:
@@ -39,6 +36,13 @@
   - include: "{{ playbook_dir }}/tasks/timesync.yaml"
     when: do_timesync
 
+  - include: "{{ playbook_dir }}/tasks/packages_version_check_p1.yaml"
+
+- hosts:
+  - mons
+  - osds
+  tasks:
+  - include: "{{ playbook_dir }}/tasks/packages_version_check_p2.yaml"
   - include: "{{ playbook_dir }}/tasks/get_packages.yaml"
 
 
@@ -54,7 +58,7 @@
       daemon "mon.{{ ansible_hostname }}" version
     register: running_mon_version
 
-  - include: "{{ decapod_common_playbooks }}/tasks/restart_mon.yaml"
+  - include: "{{ playbook_dir }}/tasks/restart_mon.yaml"
     when: "{{ (running_mon_version.stdout | from_json).version != ceph_candidate_package_version.stdout }}"
 
 
@@ -69,7 +73,7 @@
   serial: 1
   tasks:
   - include: "{{ playbook_dir }}/tasks/install_packages.yaml"
-  - include: "{{ decapod_common_playbooks }}/tasks/restart_osd.yaml"
+  - include: "{{ playbook_dir }}/tasks/restart_osd.yaml"
 
 - hosts: clients
   become: true

plugins/playbook/upgrade_ceph/decapod_plugin_playbook_upgrade_ceph/tasks/packages_version_check_p1.yaml

@@ -0,0 +1,36 @@
+---
+# Copyright (c) 2017 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: find out currently installed ceph package version
+  shell: apt-cache policy ceph | awk '/Installed:/ { print $2 }'
+  register: ceph_package_version
+
+- name: find out ceph package candidate version
+  shell: apt-cache policy ceph | awk '/Candidate:/ { print $2 }'
+  register: ceph_candidate_package_version
+
+# Note: it's OK if the current and the candidate versions are the same,
+# such a node will be silently skipped. This is required to make the upgrade
+# procedure idempotent.
+- name: check if a newer version of Ceph is going to be installed
+  command: >
+    dpkg --compare-versions
+    "{{ ceph_candidate_package_version.stdout }}"
+    ge
+    "{{ ceph_package_version.stdout }}"
+  register: compare_pkg_versions
+  failed_when: false

plugins/playbook/upgrade_ceph/decapod_plugin_playbook_upgrade_ceph/tasks/packages_version_check_p2.yaml

@@ -0,0 +1,30 @@
+---
+# Copyright (c) 2017 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: fail on downgrade attempt
+  fail:
+    msg: Downgrading Ceph can destroy data, aborting
+  when: hostvars[item].compare_pkg_versions.rc != 0
+  with_items: "{{ groups.all }}"
+  run_once: true
+
+- name: fail if candidate package version is not the same on all hosts
+  fail:
+    msg: Different version of Ceph is going to be installed on {{ item }}, aborting
+  when: ceph_candidate_package_version.stdout != hostvars[item].ceph_candidate_package_version.stdout
+  with_items: "{{ groups.all }}"
+  run_once: true

plugins/playbook/upgrade_ceph/decapod_plugin_playbook_upgrade_ceph/tasks/restart_mon.yaml

@@ -0,0 +1,75 @@
+---
+# Copyright (c) 2017 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: find out monitor interface
+  shell: netstat -nptl | awk '/ceph-mon/ { sub(":.*", "", $4); print $4 }'
+  register: monitor_ip
+
+- name: find out mon data owner
+  stat:
+    path: "/var/lib/ceph/mon/{{ cluster }}-{{ ansible_hostname }}"
+  register: mon_dir_stat
+
+- name: find out ceph-mon user
+  command: >
+    sed -rne
+    '/^\s*exec\s+.*ceph-mon/ s/^.*--setuser\s+([^[:space:]]+).*$/\1/p'
+    /etc/init/ceph-mon.conf
+  register: ceph_mon_user
+
+- name: find out ceph-mon group
+  command: >
+    sed -rne
+    '/^\s*exec\s+.*ceph-mon/ s/^.*--setgroup\s+([^[:space:]]+).*$/\1/p'
+    /etc/init/ceph-mon.conf
+  register: ceph_mon_group
+
+- name: stop ceph monitor
+  service:
+    name: "ceph-mon@{{ ansible_hostname }}"
+    state: stopped
+
+- name: fix mon data ownership
+  file:
+    path: "/var/lib/ceph/mon/{{ cluster }}-{{ ansible_hostname }}"
+    state: directory
+    owner: "{{ ceph_mon_user.stdout }}"
+    group: "{{ ceph_mon_group.stdout }}"
+    mode: 0770
+    recurse: yes
+  when:
+  - ceph_mon_user.stdout != ""
+  - mon_dir_stat.stat.pw_name != ceph_mon_user.stdout
+
+- name: start ceph monitor
+  service:
+    name: "ceph-mon@{{ ansible_hostname }}"
+    state: started
+
+- name: wait for monitor to start up
+  wait_for:
+    host: "{{ monitor_ip.stdout }}"
+    port: 6789
+    state: started
+    timeout: 60
+
+- name: wait for monitor to join quorum
+  command: ceph --cluster "{{ cluster }}" -s --format json
+  register: ceph_health_raw
+  until: ansible_hostname in (ceph_health_raw.stdout | from_json)["quorum_names"]
+  retries: "{{ mon.restart_attempts }}"
+  delay: "{{ mon.restart_delay }}"

plugins/playbook/upgrade_ceph/decapod_plugin_playbook_upgrade_ceph/tasks/restart_osd.yaml

@@ -0,0 +1,86 @@
+---
+# Copyright (c) 2017 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- include: "{{ decapod_common_playbooks }}/tasks/set_ceph_facts.yaml"
+
+- name: find out ceph user
+  command: >
+    sed -rne
+    '/^\s*exec\s+.*ceph-osd/ s/^.*--setuser\s+([^[:space:]]+).*$/\1/p'
+    /etc/init/ceph-osd.conf
+  register: ceph_user
+
+- name: find out ceph group
+  command: >
+    sed -rne
+    '/^\s*exec\s+.*ceph-osd/ s/^.*--setgroup\s+([^[:space:]]+).*$/\1/p'
+    /etc/init/ceph-osd.conf
+  register: ceph_group
+
+- name: disable data rebalancing and scrubbing
+  command: ceph --cluster "{{ cluster }}" osd set "{{ item }}"
+  with_items:
+    - noout
+    - noscrub
+    - nodeep-scrub
+  delegate_to: "{{ mon_host }}"
+
+
+- block:
+  - name: Restart OSDs
+    include: "{{ playbook_dir }}/tasks/restart_single_osd.yaml"
+    with_items: "{{ osd_numbers }}"
+    loop_control:
+      loop_var: osd_number
+  always:
+  - block:
+    - name: find out final OSD map flags, part 1
+      command: ceph osd getmap -o /tmp/osdmap_fin.bin
+      delegate_to: "{{ mon_host }}"
+      when: inventory_hostname == groups.osds[-1]
+
+    - name: find out final OSD map flags, part 2
+      shell: >
+        osdmaptool --print /tmp/osdmap_fin.bin
+        | awk '/^flags/ { gsub(",", "\",\"", $2); print "[\"" $2 "\"]" }'
+      register: osdmap_final_flags
+      delegate_to: "{{ mon_host }}"
+      when: inventory_hostname == groups.osds[-1]
+
+    - name: reenable data rebalancing and scrubbing
+      command: ceph osd unset "{{ item }}"
+      delegate_to: "{{ mon_host }}"
+      with_items:
+      - noout
+      - noscrub
+      - nodeep-scrub
+      when:
+      - inventory_hostname == groups.osds[-1]
+      - "{{ item in (osdmap_final_flags.stdout | default([]) | from_json) }}"
+    always:
+    - name: remove OSD map temporary file
+      file:
+        path: /tmp/osdmap_fin.bin
+        state: absent
+
+- name: reenable data rebalancing and scrubbing
+  command: ceph --cluster "{{ cluster }}" osd unset "{{ item }}"
+  with_items:
+    - noout
+    - noscrub
+    - nodeep-scrub
+  delegate_to: "{{ mon_host }}"

plugins/playbook/upgrade_ceph/decapod_plugin_playbook_upgrade_ceph/tasks/restart_single_osd.yaml

@@ -0,0 +1,99 @@
+---
+# Copyright (c) 2017 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: find out osd.{{ osd_number }} version
+  command: ceph --cluster "{{ cluster }}" daemon "osd.{{ osd_number }}" version
+  register: running_osd_version
+
+- name: find out OSD data owner
+  stat:
+    path: "/var/lib/ceph/osd/{{ cluster }}-{{ osd_number }}"
+  register: osd_dir_stat
+
+- name: check if osd.{{ osd_number }} should be restarted
+  set_fact:
+    dont_restart_osd: true
+  when:
+  - "{{ (running_osd_version.stdout | from_json).version == ceph_candidate_package_version.stdout }}"
+  - "{{ (ceph_user.stdout == '') or osd_dir_stat.stat.pw_name == ceph_user.stdout }}"
+
+- block:
+  - name: find out current OSD map flags, part 1
+    command: ceph osd getmap -o /tmp/osdmap.bin
+    delegate_to: "{{ mon_host }}"
+    when: "{{ not(dont_restart_osd) }}"
+
+  - name: find out current OSD map flags, part 2
+    shell: >
+      osdmaptool --print /tmp/osdmap.bin
+      | awk '/^flags/ { gsub(",", "\",\"", $2); print "[\"" $2 "\"]" }'
+    register: osdmap_flags
+    delegate_to: "{{ mon_host }}"
+    when: "{{ not(dont_restart_osd) }}"
+
+  - name: disable data rebalancing and scrubbing
+    command: ceph osd set "{{ item }}"
+    delegate_to: "{{ mon_host }}"
+    with_items:
+    - noout
+    - noscrub
+    - nodeep-scrub
+    when:
+    - "{{ not(dont_restart_osd) }}"
+    - "{{ item not in (osdmap_flags.stdout | from_json) }}"
+  always:
+  - name: remove OSD map temporary file
+    file:
+      path: /tmp/osdmap.bin
+      state: absent
+
+- name: stop OSD item
+  service:
+    name: "ceph-osd@{{ osd_number }}"
+    state: stopped
+  when: "{{ not(dont_restart_osd) }}"
+
+- name: fix OSD data ownership
+  file:
+    path: "/var/lib/ceph/osd/{{ cluster }}-{{ osd_number }}"
+    state: directory
+    owner: "{{ ceph_user.stdout }}"
+    group: "{{ ceph_group.stdout }}"
+    mode: 0770
+    recurse: yes
+  when:
+  - ceph_user.stdout != ''
+  - osd_dir_stat.stat.pw_name != ceph_user.stdout
+  - "{{ not(dont_restart_osd) }}"
+
+- name: start OSD {{ osd_number }}
+  service:
+    name: "ceph-osd@{{ osd_number }}"
+    state: started
+  when: "{{ not(dont_restart_osd) }}"
+
+- name: wait for PGs to become active+clean
+  command: ceph --cluster "{{ cluster }}" -s --format json
+  register: ceph_health_post
+  delegate_to: "{{ mon_host }}"
+  until: >
+    ((ceph_health_post.stdout | from_json).pgmap.pgs_by_state | length) == 1
+    and
+    (ceph_health_post.stdout | from_json).pgmap.pgs_by_state.0.state_name == "active+clean"
+  retries: "{{ osd.restart_attempts }}"
+  delay: "{{ osd.restart_delay }}"
+  when: "{{ not(dont_restart_osd) }}"