Add playbook for deployment of RBD/CLI clients

Change-Id: I668fd7754f6f87f5bfc36039e32890c84c3a8899

docker-compose.dev.yml

@@ -32,6 +32,7 @@ services:
     - ./backend/docker/decapod_docker:/usr/local/lib/python3.5/dist-packages/decapod_docker:ro
     - ./backend/monitoring/decapod_monitoring:/usr/local/lib/python2.7/dist-packages/decapod_monitoring:ro
     - ./plugins/alerts/emails/decapod_plugin_alerts_emails:/usr/local/lib/python3.5/dist-packages/decapod_plugin_alerts_emails:ro
+    - ./plugins/playbook/add_client/decapod_plugin_playbook_add_client:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_client:ro
     - ./plugins/playbook/add_mon/decapod_plugin_playbook_add_mon:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_mon:ro
     - ./plugins/playbook/add_nfs/decapod_plugin_playbook_add_nfs:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_nfs:ro
     - ./plugins/playbook/add_osd/decapod_plugin_playbook_add_osd:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_osd:ro
@@ -60,6 +61,7 @@ services:
     - ./backend/common/decapod_common:/usr/local/lib/python3.5/dist-packages/decapod_common:ro
     - ./backend/docker/decapod_docker:/usr/local/lib/python3.5/dist-packages/decapod_docker:ro
     - ./plugins/alerts/emails/decapod_plugin_alerts_emails:/usr/local/lib/python3.5/dist-packages/decapod_plugin_alerts_emails:ro
+    - ./plugins/playbook/add_client/decapod_plugin_playbook_add_client:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_client:ro
     - ./plugins/playbook/add_mon/decapod_plugin_playbook_add_mon:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_mon:ro
     - ./plugins/playbook/add_nfs/decapod_plugin_playbook_add_nfs:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_nfs:ro
     - ./plugins/playbook/add_osd/decapod_plugin_playbook_add_osd:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_osd:ro
@@ -89,6 +91,7 @@ services:
     - ./backend/controller/decapod_controller:/usr/local/lib/python3.5/dist-packages/decapod_controller:ro
     - ./backend/docker/decapod_docker:/usr/local/lib/python3.5/dist-packages/decapod_docker:ro
     - ./plugins/alerts/emails/decapod_plugin_alerts_emails:/usr/local/lib/python3.5/dist-packages/decapod_plugin_alerts_emails:ro
+    - ./plugins/playbook/add_client/decapod_plugin_playbook_add_client:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_client:ro
     - ./plugins/playbook/add_mon/decapod_plugin_playbook_add_mon:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_mon:ro
     - ./plugins/playbook/add_nfs/decapod_plugin_playbook_add_nfs:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_nfs:ro
     - ./plugins/playbook/add_osd/decapod_plugin_playbook_add_osd:/usr/local/lib/python3.5/dist-packages/decapod_plugin_playbook_add_osd:ro

plugins/playbook/add_client/MANIFEST.in

@@ -0,0 +1,2 @@
+include decapod_plugin_playbook_add_client/config.yaml
+include decapod_plugin_playbook_add_client/playbook.yaml

plugins/playbook/add_client/decapod_plugin_playbook_add_client/config.yaml

@@ -0,0 +1,15 @@
+---
+# Copyright (c) 2016 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

plugins/playbook/add_client/decapod_plugin_playbook_add_client/playbook.yaml

@@ -0,0 +1,24 @@
+---
+# Copyright (c) 2016 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- include: "{{ decapod_common_playbooks }}/plays/add_ceph_ansible_role.yaml"
+
+- hosts: clients
+  become: true
+  roles:
+  - role: ceph-client
+    user_config: false

plugins/playbook/add_client/decapod_plugin_playbook_add_client/plugin.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2016 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Playbook plugin for Add RBD and CLI clients to the hosts."""
+
+
+from decapod_common import log
+from decapod_common import playbook_plugin
+from decapod_common import playbook_plugin_hints
+from decapod_common.models import cluster_data
+from decapod_common.models import server
+
+
+DESCRIPTION = "Add RBD and CLI clients to the hosts"
+"""Plugin description."""
+
+HINTS_SCHEMA = {
+    "ceph_version_verify": {
+        "description": "Verify Ceph version consistency on install",
+        "typename": "boolean",
+        "type": "boolean",
+        "default_value": True
+    }
+}
+"""Schema for playbook hints."""
+
+LOG = log.getLogger(__name__)
+"""Logger."""
+
+
+class AddClient(playbook_plugin.CephAnsiblePlaybook):
+
+    NAME = "Add RBD and CLI clients to the hosts"
+    DESCRIPTION = DESCRIPTION
+    PUBLIC = True
+    REQUIRED_SERVER_LIST = True
+    SERVER_LIST_POLICY = playbook_plugin.ServerListPolicy.not_in_other_cluster
+    HINTS = playbook_plugin_hints.Hints(HINTS_SCHEMA)
+
+    def on_pre_execute(self, task):
+        super().on_pre_execute(task)
+
+        playbook_config = self.get_playbook_configuration(task)
+        config = playbook_config.configuration
+        cluster = playbook_config.cluster
+        servers = playbook_config.servers
+        servers = {srv.ip: srv for srv in servers}
+
+        for name, group_vars in config["inventory"].items():
+            if name != "clients" or not group_vars:
+                continue
+            group_servers = [servers[ip] for ip in group_vars]
+            cluster.add_servers(group_servers, name)
+
+        if cluster.configuration.changed:
+            cluster.save()
+
+        data = cluster_data.ClusterData.find_one(cluster.model_id)
+        data.global_vars = config["global_vars"]
+        hostvars = config["inventory"].get("_meta", {}).get("hostvars", {})
+        for hostname, values in hostvars.items():
+            data.update_host_vars(hostname, values)
+        data.save()
+
+    def make_playbook_configuration(self, cluster, servers, hints):
+        data = cluster_data.ClusterData.find_one(cluster.model_id)
+        global_vars = self.make_global_vars(cluster, data, servers, hints)
+        inventory = self.make_inventory(cluster, data, servers, hints)
+
+        return global_vars, inventory
+
+    def make_global_vars(self, cluster, data, servers, hints):
+        result = super().make_global_vars(cluster, servers, hints)
+        result.update(data.global_vars)
+
+        result["ceph_version_verify"] = bool(hints["ceph_version_verify"])
+
+        return result
+
+    def make_inventory(self, cluster, data, servers, hints):
+        groups = self.get_inventory_groups(cluster, servers, hints)
+        inventory = {"_meta": {"hostvars": {}}}
+
+        for name, group_servers in groups.items():
+            for srv in group_servers:
+                inventory.setdefault(name, []).append(srv.ip)
+
+                hostvars = inventory["_meta"]["hostvars"].setdefault(
+                    srv.ip, {})
+                hostvars.update(data.get_host_vars(srv.ip))
+                hostvars["ansible_user"] = srv.username
+
+        return inventory
+
+    def get_inventory_groups(self, cluster, servers, hints):
+        cluster_servers = server.ServerModel.cluster_servers(cluster.model_id)
+        cluster_servers = {item._id: item for item in cluster_servers}
+
+        mons = [
+            cluster_servers[item["server_id"]]
+            for item in cluster.configuration.state if item["role"] == "mons"
+        ]
+
+        return {
+            "mons": mons,
+            "clients": servers,
+            "already_deployed": list(cluster_servers.values())
+        }

plugins/playbook/add_client/setup.cfg

@@ -0,0 +1,9 @@
+[bdist_wheel]
+python-tag = py3
+
+[flake8]
+exclude = build,dist,docs/conf.py,*.egg-info,.eggs,.cache,.tox
+inline-quotes = "
+application-import-names = decapod_api,decapod_common,decapod_controller,decapodlib
+import-order-style = smarkets
+

plugins/playbook/add_client/setup.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2016 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Setup script for decapod_plugin_playbook_add_client plugin."""
+
+
+import setuptools
+
+
+setuptools.setup(
+    name="decapod-plugin-playbook-add-client",
+    description="Add RBD and CLI clients to the hosts",
+    version="1.1.0.dev1",
+    author="Sergey Arkhipov",
+    author_email="sarkhipov@mirantis.com",
+    url="https://github.com/Mirantis/ceph-lcm",
+    packages=setuptools.find_packages(),
+    entry_points={
+        "decapod.playbooks": [
+            "add_client = decapod_plugin_playbook_add_client.plugin:AddClient"  # NOQA
+        ]
+    },
+    python_requires=">= 3.4",
+    include_package_data=True,
+    package_data={
+        "decapod_plugin_playbook_add_client": [
+            "config.yaml",
+            "playbook.yaml"
+        ]
+    },
+    install_requires=[
+        "decapod-common~=1.1.dev1"
+    ],
+    zip_safe=False
+)

plugins/playbook/add_client/stdeb.cfg

@@ -0,0 +1,3 @@
+[DEFAULT]
+X-Python3-Version: >= 3.4
+Depends3: python3-decapod-common (>= 1.1), python3-decapod-common (<< 1.2)