Refactor add* plugins

Change-Id: Ie43fc40a5c97bade95eba4d48cdcd779d9f34e71

backend/common/decapod_common/playbook_plugin.py

@@ -32,6 +32,8 @@
 from decapod_common import pathutils
 from decapod_common import playbook_plugin_hints
 from decapod_common import process
+from decapod_common.models import cluster_data
+from decapod_common.models import server
 from decapod_common.models import task
 
 
@@ -487,6 +489,75 @@ def get_dynamic_inventory(self):
         return self.playbook_config.configuration["inventory"]
 
 
+class CephAnsibleNewWithVerification(CephAnsiblePlaybook):
+
+    PUBLIC = True
+    REQUIRED_SERVER_LIST = True
+    SERVER_LIST_POLICY = ServerListPolicy.not_in_other_cluster
+
+    def on_pre_execute(self, skip_roles, task):
+        super().on_pre_execute(task)
+
+        skip_roles = set(skip_roles)
+        playbook_config = self.get_playbook_configuration(task)
+        config = playbook_config.configuration["inventory"]
+        cluster = playbook_config.cluster
+        servers = playbook_config.servers
+        servers = {srv.ip: srv for srv in servers}
+
+        for name, group_vars in config.items():
+            if name not in skip_roles or not group_vars:
+                continue
+            group_servers = [servers[ip] for ip in group_vars]
+            cluster.add_servers(group_servers, name)
+
+        if cluster.configuration.changed:
+            cluster.save()
+
+    def make_playbook_configuration(self, cluster, servers, hints):
+        data = cluster_data.ClusterData.find_one(cluster.model_id)
+        global_vars = self.make_global_vars(cluster, data, servers, hints)
+        inventory = self.make_inventory(cluster, data, servers, hints)
+
+        return global_vars, inventory
+
+    def make_global_vars(self, cluster, data, servers, hints):
+        result = super().make_global_vars(cluster, servers, hints)
+        result.update(data.global_vars)
+
+        result["ceph_version_verify"] = bool(hints["ceph_version_verify"])
+
+        return result
+
+    def get_inventory_groups(self, cluster, servers, hints):
+        cluster_servers = server.ServerModel.cluster_servers(cluster.model_id)
+        cluster_servers = {item._id: item for item in cluster_servers}
+
+        mons = [
+            cluster_servers[item["server_id"]]
+            for item in cluster.configuration.state if item["role"] == "mons"]
+
+        return {
+            "mons": mons,
+            "already_deployed": list(cluster_servers.values())
+        }
+
+    def make_inventory(self, cluster, data, servers, hints):
+        groups = self.get_inventory_groups(cluster, servers, hints)
+        inventory = {"_meta": {"hostvars": {}}}
+
+        for name, group_servers in groups.items():
+            for srv in group_servers:
+                inventory.setdefault(name, []).append(srv.ip)
+
+                hostvars = inventory["_meta"]["hostvars"].setdefault(
+                    srv.ip, {})
+                hostvars.update(data.get_host_vars(srv.ip))
+                hostvars.setdefault("ansible_user", srv.username)
+
+        return inventory
+
+
 @functools.lru_cache()
 def load_config(filename):
     return config.yaml_load(filename)

plugins/playbook/add_client/decapod_plugin_playbook_add_client/plugin.py

@@ -20,7 +20,6 @@
 from decapod_common import playbook_plugin
 from decapod_common import playbook_plugin_hints
 from decapod_common.models import cluster_data
-from decapod_common.models import server
 
 
 DESCRIPTION = "Add RBD and CLI clients to the hosts"
@@ -40,32 +39,18 @@
 """Logger."""
 
 
-class AddClient(playbook_plugin.CephAnsiblePlaybook):
+class AddClient(playbook_plugin.CephAnsibleNewWithVerification):
 
     NAME = "Add RBD and CLI clients to the hosts"
     DESCRIPTION = DESCRIPTION
-    PUBLIC = True
-    REQUIRED_SERVER_LIST = True
-    SERVER_LIST_POLICY = playbook_plugin.ServerListPolicy.not_in_other_cluster
     HINTS = playbook_plugin_hints.Hints(HINTS_SCHEMA)
 
     def on_pre_execute(self, task):
-        super().on_pre_execute(task)
+        super().on_pre_execute(["clients"], task)
 
         playbook_config = self.get_playbook_configuration(task)
-        config = playbook_config.configuration
+        config = playbook_config.configuration["inventory"]
         cluster = playbook_config.cluster
-        servers = playbook_config.servers
-        servers = {srv.ip: srv for srv in servers}
-
-        for name, group_vars in config["inventory"].items():
-            if name != "clients" or not group_vars:
-                continue
-            group_servers = [servers[ip] for ip in group_vars]
-            cluster.add_servers(group_servers, name)
-
-        if cluster.configuration.changed:
-            cluster.save()
 
         data = cluster_data.ClusterData.find_one(cluster.model_id)
         data.global_vars = config["global_vars"]
@@ -74,47 +59,8 @@ def on_pre_execute(self, task):
             data.update_host_vars(hostname, values)
         data.save()
 
-    def make_playbook_configuration(self, cluster, servers, hints):
-        data = cluster_data.ClusterData.find_one(cluster.model_id)
-        global_vars = self.make_global_vars(cluster, data, servers, hints)
-        inventory = self.make_inventory(cluster, data, servers, hints)
-
-        return global_vars, inventory
-
-    def make_global_vars(self, cluster, data, servers, hints):
-        result = super().make_global_vars(cluster, servers, hints)
-        result.update(data.global_vars)
-
-        result["ceph_version_verify"] = bool(hints["ceph_version_verify"])
-
-        return result
-
-    def make_inventory(self, cluster, data, servers, hints):
-        groups = self.get_inventory_groups(cluster, servers, hints)
-        inventory = {"_meta": {"hostvars": {}}}
-
-        for name, group_servers in groups.items():
-            for srv in group_servers:
-                inventory.setdefault(name, []).append(srv.ip)
-
-                hostvars = inventory["_meta"]["hostvars"].setdefault(
-                    srv.ip, {})
-                hostvars.update(data.get_host_vars(srv.ip))
-                hostvars["ansible_user"] = srv.username
-
-        return inventory
-
     def get_inventory_groups(self, cluster, servers, hints):
-        cluster_servers = server.ServerModel.cluster_servers(cluster.model_id)
-        cluster_servers = {item._id: item for item in cluster_servers}
-
-        mons = [
-            cluster_servers[item["server_id"]]
-            for item in cluster.configuration.state if item["role"] == "mons"
-        ]
+        base = super().get_inventory_groups(cluster, servers, hints)
+        base["clients"] = servers
 
-        return {
-            "mons": mons,
-            "clients": servers,
-            "already_deployed": list(cluster_servers.values())
-        }
+        return base

plugins/playbook/add_mds/decapod_plugin_playbook_add_mds/plugin.py

@@ -19,7 +19,6 @@
 from decapod_common import log
 from decapod_common import playbook_plugin
 from decapod_common import playbook_plugin_hints
-from decapod_common.models import cluster_data
 from decapod_common.models import server
 
 
@@ -40,71 +39,20 @@
 """Logger."""
 
 
-class AddMds(playbook_plugin.CephAnsiblePlaybook):
+class AddMds(playbook_plugin.CephAnsibleNewWithVerification):
 
     NAME = "Add metadata server host"
     DESCRIPTION = DESCRIPTION
-    PUBLIC = True
-    REQUIRED_SERVER_LIST = True
-    SERVER_LIST_POLICY = playbook_plugin.ServerListPolicy.not_in_other_cluster
-
     HINTS = playbook_plugin_hints.Hints(HINTS_SCHEMA)
 
     def on_pre_execute(self, task):
-        super().on_pre_execute(task)
-
-        playbook_config = self.get_playbook_configuration(task)
-        config = playbook_config.configuration["inventory"]
-        cluster = playbook_config.cluster
-        servers = playbook_config.servers
-        servers = {srv.ip: srv for srv in servers}
-
-        for name, group_vars in config.items():
-            if name != "mdss" or not group_vars:
-                continue
-            group_servers = [servers[ip] for ip in group_vars]
-            cluster.add_servers(group_servers, name)
-
-        if cluster.configuration.changed:
-            cluster.save()
-
-    def make_playbook_configuration(self, cluster, servers, hints):
-        data = cluster_data.ClusterData.find_one(cluster.model_id)
-        global_vars = self.make_global_vars(cluster, data, servers, hints)
-        inventory = self.make_inventory(cluster, data, servers, hints)
-
-        return global_vars, inventory
-
-    def make_global_vars(self, cluster, data, servers, hints):
-        result = super().make_global_vars(cluster, servers, hints)
-        result.update(data.global_vars)
-
-        result["ceph_version_verify"] = bool(hints["ceph_version_verify"])
-
-        return result
-
-    def make_inventory(self, cluster, data, servers, hints):
-        groups = self.get_inventory_groups(cluster, servers, hints)
-        inventory = {"_meta": {"hostvars": {}}}
-
-        for name, group_servers in groups.items():
-            for srv in group_servers:
-                inventory.setdefault(name, []).append(srv.ip)
-
-                hostvars = inventory["_meta"]["hostvars"].setdefault(
-                    srv.ip, {})
-                hostvars.update(data.get_host_vars(srv.ip))
-                hostvars["ansible_user"] = srv.username
-
-        return inventory
+        super().on_pre_execute(["mdss"], task)
 
     def get_inventory_groups(self, cluster, servers, hints):
+        base = super().get_inventory_groups(cluster, servers, hints)
+
         cluster_servers = server.ServerModel.cluster_servers(cluster.model_id)
         cluster_servers = {item._id: item for item in cluster_servers}
-
-        mons = [
-            cluster_servers[item["server_id"]]
-            for item in cluster.configuration.state if item["role"] == "mons"]
         mdss = [
             cluster_servers[item["server_id"]]
             for item in cluster.configuration.state if item["role"] == "mdss"]
@@ -115,8 +63,6 @@ def get_inventory_groups(self, cluster, servers, hints):
                 mdss_ips.add(srv.ip)
                 mdss.append(srv)
 
-        return {
-            "mons": mons,
-            "mdss": servers,
-            "already_deployed": list(cluster_servers.values())
-        }
+        base["mdss"] = mdss
+
+        return base

plugins/playbook/add_mon/decapod_plugin_playbook_add_mon/plugin.py

@@ -28,9 +28,7 @@
 from . import exceptions
 
 
-DESCRIPTION = """\
-Add monitor to the cluster
-""".strip()
+DESCRIPTION = "Add monitor to the cluster"
 """Plugin description."""
 
 HINTS_SCHEMA = {
@@ -53,56 +51,25 @@ def get_monitor_secret(secret_id):
         return secret[0]
 
 
-class AddMon(playbook_plugin.CephAnsiblePlaybook):
+class AddMon(playbook_plugin.CephAnsibleNewWithVerification):
 
     NAME = "Add monitor to the cluster"
     DESCRIPTION = DESCRIPTION
-    PUBLIC = True
-    REQUIRED_SERVER_LIST = True
-    SERVER_LIST_POLICY = playbook_plugin.ServerListPolicy.not_in_other_cluster
-
     HINTS = playbook_plugin_hints.Hints(HINTS_SCHEMA)
 
     def on_pre_execute(self, task):
-        super().on_pre_execute(task)
+        super().on_pre_execute(["mons"], task)
 
         playbook_config = self.get_playbook_configuration(task)
         config = playbook_config.configuration["inventory"]
         cluster = playbook_config.cluster
-        servers = playbook_config.servers
-        servers = {srv.ip: srv for srv in servers}
-
-        for name, group_vars in config.items():
-            skip = {"_meta", "oldmons", "already_deployed"}
-            if name in skip or not group_vars:
-                continue
-            group_servers = [servers[ip] for ip in group_vars]
-            cluster.add_servers(group_servers, name)
-
-        if cluster.configuration.changed:
-            cluster.save()
 
         data = cluster_data.ClusterData.find_one(cluster.model_id)
         hostvars = config.get("_meta", {}).get("hostvars", {})
         for hostname, values in hostvars.items():
             data.update_host_vars(hostname, values)
         data.save()
 
-    def make_playbook_configuration(self, cluster, servers, hints):
-        data = cluster_data.ClusterData.find_one(cluster.model_id)
-        global_vars = self.make_global_vars(cluster, data, servers, hints)
-        inventory = self.make_inventory(cluster, data, servers, hints)
-
-        return global_vars, inventory
-
-    def make_global_vars(self, cluster, data, servers, hints):
-        result = super().make_global_vars(cluster, servers, hints)
-        result.update(data.global_vars)
-
-        result["ceph_version_verify"] = bool(hints["ceph_version_verify"])
-
-        return result
-
     def get_dynamic_inventory(self):
         if not self.playbook_config:
             raise exceptions.UnknownPlaybookConfiguration()