Merge pull request #380 from git-harry/ceph-squash

Ceph support

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "openstack-ansible"]
 	path = openstack-ansible
 	url = https://git.openstack.org/openstack/openstack-ansible
+[submodule "rpcd/playbooks/roles/ceph-common"]
+	path = rpcd/playbooks/roles/ceph-common
+	url = https://github.com/ceph/ansible-ceph-common
+[submodule "rpcd/playbooks/roles/ceph-mon"]
+	path = rpcd/playbooks/roles/ceph-mon
+	url = https://github.com/ceph/ansible-ceph-mon
+[submodule "rpcd/playbooks/roles/ceph-osd"]
+	path = rpcd/playbooks/roles/ceph-osd
+	url = https://github.com/ceph/ansible-ceph-osd

maas/plugins/README.md

@@ -334,3 +334,58 @@ Returns metrics indicating the status of parts of HP hardware.
     metric hardware_memory_status uint32 1
     metric hardware_processors_status uint32 1
     metric hardware_disk_status uint32 1
+
+***
+#### ceph_monitoring.py
+
+##### Description:
+Connects to a Ceph cluster with the specified user/key and returns a number of metrics which can be used to monitor general health and availability of that cluster
+
+##### Mandatory Arguments:
+- {cluster,mon,osd}: Specify the type of data to return
+- --name NAME: Ceph client name to use when connecting to cluster
+- --keyring KEYRING: Ceph client keyring to use when connecting to cluster
+
+##### Mandatory Arguments (when osd argument given):
+- --osd_ids OSD_IDS: Space-separated list of OSDs, required when `--type osd`
+
+##### Mandatory Arguments (when mon argument given):
+- --host HOST: Specific MON to connect to, required when `--type mon`
+
+##### Example Output:
+Cluster:
+
+    metric cluster_health uint32 2
+    metric monmap_epoch uint32 1
+    metric osdmap_epoch uint32 16
+    metric osds_total uint32 3
+    metric osds_up uint32 3
+    metric osds_in uint32 3
+    metric osds_kb uint64 495007068
+    metric osds_kb_avail uint64 284983032
+    metric osds_kb_used uint64 189214596
+    metric pgs_active_clean uint32 512
+    metric pgs_total uint32 512
+
+MON:
+
+    metric mon_in_quorum uint32 1
+    metric mon_health uint32 2
+
+OSDs:
+
+    metric osd.0_up uint32 1
+    metric osd.0_in uint32 1
+    metric osd.0_kb uint64 165002356
+    metric osd.0_kb_used uint64 63071532
+    metric osd.0_kb_avail uint64 94994344
+    metric osd.1_up uint32 1
+    metric osd.1_in uint32 1
+    metric osd.1_kb uint64 165002356
+    metric osd.1_kb_used uint64 63071532
+    metric osd.1_kb_avail uint64 94994344
+    metric osd.2_up uint32 1
+    metric osd.2_in uint32 1
+    metric osd.2_kb uint64 165002356
+    metric osd.2_kb_used uint64 63071532
+    metric osd.2_kb_avail uint64 94994344

maas/plugins/ceph_monitoring.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import maas_common
+import subprocess
+
+
+STATUSES = {'HEALTH_OK': 2, 'HEALTH_WARN': 1, 'HEALTH_ERR': 0}
+
+
+def check_command(command):
+    output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+    lines = output.strip().split('\n')
+    return json.loads(lines[-1])
+
+
+def get_ceph_report(client, keyring, fmt='json'):
+    return check_command(('ceph', '--format', fmt, '--name', client,
+                          '--keyring', keyring, 'report'))
+
+
+def get_mon_statistics(report=None, host=None):
+    mon = [m for m in report['monmap']['mons']
+           if m['name'] == host]
+    mon_in = mon[0]['rank'] in report['quorum']
+    maas_common.metric_bool('mon_in_quorum', mon_in)
+    health_status = 0
+    for each in report['health']['health']['health_services'][0]['mons']:
+        if each['name'] == host:
+            health_status = STATUSES[each['health']]
+            break
+    maas_common.metric('mon_health', 'uint32', health_status)
+
+
+def get_osd_statistics(report=None, osd_ids=None):
+    for osd_id in osd_ids:
+        osd_ref = 'osd.%s' % osd_id
+        for _osd in report['osdmap']['osds']:
+            if _osd['osd'] == osd_id:
+                osd = _osd
+                break
+        else:
+            msg = 'The OSD ID %s does not exist.' % osd_id
+            raise maas_common.MaaSException(msg)
+        for key in ('up', 'in'):
+            name = '_'.join((osd_ref, key))
+            maas_common.metric_bool(name, osd[key])
+
+        for _osd in report['pgmap']['osd_stats']:
+            if _osd['osd'] == osd_id:
+                osd = _osd
+                break
+        for key in ('kb', 'kb_used', 'kb_avail'):
+            name = '_'.join((osd_ref, key))
+            maas_common.metric(name, 'uint64', osd[key])
+
+
+def get_cluster_statistics(report=None):
+    metrics = []
+
+    # Get overall cluster health
+    metrics.append({'name': 'cluster_health',
+                    'type': 'uint32',
+                    'value': STATUSES[report['health']['overall_status']]})
+
+    # Collect epochs for the mon and osd maps
+    for map_name in ('monmap', 'osdmap'):
+        metrics.append({'name': "%(map)s_epoch" % {'map': map_name},
+                        'type': 'uint32',
+                        'value': report[map_name]['epoch']})
+
+    # Collect OSDs per state
+    osds = {'total': 0, 'up': 0, 'in': 0}
+    for osd in report['osdmap']['osds']:
+        osds['total'] += 1
+        if osd['up'] == 1:
+            osds['up'] += 1
+        if osd['in'] == 1:
+            osds['in'] += 1
+    for k in osds:
+        metrics.append({'name': 'osds_%s' % k,
+                        'type': 'uint32',
+                        'value': osds[k]})
+
+    # Collect cluster size & utilisation
+    osds_stats = ('kb', 'kb_avail', 'kb_used')
+    for k in report['pgmap']['osd_stats_sum']:
+        if k in osds_stats:
+            metrics.append({'name': 'osds_%s' % k,
+                            'type': 'uint64',
+                            'value': report['pgmap']['osd_stats_sum'][k]})
+
+    # Collect num PGs and num healthy PGs
+    pgs = {'total': 0, 'active_clean': 0}
+    for pg in report['pgmap']['pg_stats']:
+        pgs['total'] += 1
+        if pg['state'] == 'active+clean':
+            pgs['active_clean'] += 1
+    for k in pgs:
+        metrics.append({'name': 'pgs_%s' % k,
+                        'type': 'uint32',
+                        'value': pgs[k]})
+
+    # Submit gathered metrics
+    for m in metrics:
+        maas_common.metric(m['name'], m['type'], m['value'])
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--name', required=True, help='Ceph client name')
+    parser.add_argument('--keyring', required=True, help='Ceph client keyring')
+
+    subparsers = parser.add_subparsers(dest='subparser_name')
+
+    parser_mon = subparsers.add_parser('mon')
+    parser_mon.add_argument('--host', required=True, help='Mon hostname')
+
+    parser_osd = subparsers.add_parser('osd')
+    parser_osd.add_argument('--osd_ids', required=True,
+                            help='Space separated list of OSD IDs')
+
+    subparsers.add_parser('cluster')
+    return parser.parse_args()
+
+
+def main(args):
+    get_statistics = {'cluster': get_cluster_statistics,
+                      'mon': get_mon_statistics,
+                      'osd': get_osd_statistics}
+    report = get_ceph_report(client=args.name, keyring=args.keyring)
+    kwargs = {'report': report}
+    if args.subparser_name == 'osd':
+        kwargs['osd_ids'] = [int(i) for i in args.osd_ids.split(' ')]
+    if args.subparser_name == 'mon':
+        kwargs['host'] = args.host
+    get_statistics[args.subparser_name](**kwargs)
+    maas_common.status_ok()
+
+
+if __name__ == '__main__':
+    with maas_common.print_output():
+        args = get_args()
+        main(args)

rpcd/etc/openstack_deploy/conf.d/ceph.yml.aio

@@ -0,0 +1,29 @@
+mons_hosts:
+  aio1:
+    ip: 172.29.236.100
+    affinity:
+      ceph_mon_container: 3
+osds_hosts:
+  aio1:
+    ip: 172.29.236.100
+    affinity:
+      ceph_osd_container: 3
+
+storage_hosts:
+  aio1:
+    ip: 172.29.236.100
+    container_vars:
+      cinder_backends:
+        limit_container_types: cinder_volume
+        ceph:
+          volume_driver: cinder.volume.drivers.rbd.RBDDriver
+          rbd_pool: volumes
+          rbd_ceph_conf: /etc/ceph/ceph.conf
+          rbd_flatten_volume_from_snapshot: 'false'
+          rbd_max_clone_depth: 5
+          rbd_store_chunk_size: 4
+          rados_connect_timeout: -1
+          glance_api_version: 2
+          volume_backend_name: ceph
+          rbd_user: "{{ cinder_ceph_client }}"
+          rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"

rpcd/etc/openstack_deploy/env.d/ceph.yml

@@ -0,0 +1,57 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+component_skel:
+  mons:
+    belongs_to:
+      - ceph_all
+  osds:
+    belongs_to:
+      - ceph_all
+
+
+container_skel:
+  ceph_mon_container:
+    belongs_to:
+      - mons_containers
+    contains:
+      - mons
+    properties:
+      container_release: trusty
+      service_name: ceph
+  ceph_osd_container:
+    belongs_to:
+      - osds_containers
+    contains:
+      - osds
+    properties:
+      is_metal: true
+      container_release: trusty
+      service_name: ceph
+
+
+physical_skel:
+  osds_containers:
+    belongs_to:
+      - all_containers
+  osds_hosts:
+    belongs_to:
+      - hosts
+  mons_containers:
+    belongs_to:
+      - all_containers
+  mons_hosts:
+    belongs_to:
+      - hosts

rpcd/etc/openstack_deploy/user_extras_secrets.yml

@@ -17,3 +17,4 @@ maas_keystone_password:
 rpc_support_holland_password:
 kibana_password:
 maas_rabbitmq_password:
+fsid_uuid:

rpcd/etc/openstack_deploy/user_extras_variables.yml

@@ -94,3 +94,19 @@ maas_filesystem_critical_threshold: 90.0
 #   - { package: "libvirt-bin", version: "1.2.2-0ubuntu13.1.9" }
 #   - { package: "rabbitmq-server", origin: "www.rabbitmq.com" }
 #   - { package: "*", release: "MariaDB" }
+
+# Ceph
+fsid: '{{ fsid_uuid }}'
+fetch_directory: /etc/openstack_deploy/ceph_fetch
+ceph_stable: true
+ceph_stable_release: hammer
+openstack_config: true
+raw_multi_journal: true
+journal_size: 80000
+pool_default_size: 3
+pool_default_min_size: 2
+mon_osd_full_ratio: .90
+mon_osd_nearfull_ratio: .80
+secure_cluster: true
+secure_cluster_flags:
+  - nodelete

rpcd/etc/openstack_deploy/user_variables.yml

@@ -37,3 +37,13 @@ keystone_token_driver: "keystone.token.persistence.backends.sql.Token"
 
 # Galera overrides
 galera_cluster_name: rpc_galera_cluster
+
+# Ceph overrides
+ceph_mons: >
+  {% set _var = [] -%}
+  {% if 'mons' in groups -%}
+  {% for mon in groups.mons -%}
+  {% if _var.append(hostvars[mon]['ansible_ssh_host']) -%}{% endif -%}
+  {% endfor -%}
+  {% endif -%}
+  {{ _var }}

rpcd/playbooks/beaver.yml

@@ -89,3 +89,10 @@
       - name: "auth"
         log_file: "/var/log/auth.log"
         tags: "auth,infrastructure"
+      - name: "ceph-mon"
+        log_file: "/var/log/ceph/ceph-mon.{{ inventory_hostname }}.log"
+        multiline_regex_before: '^[a-z_]*\s'
+        tags: "ceph-mon,ceph,infrastructure"
+      - name: "ceph-osd"
+        log_file: "/var/log/ceph/ceph-osd.*.log"
+        tags: "ceph-osd,ceph,infrastructure"

rpcd/playbooks/ceph-all.yml

@@ -0,0 +1,17 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: ceph-mon.yml
+- include: ceph-osd.yml