Merge 706eb78 into f170abd

kqueen/conftest.py

@@ -43,7 +43,8 @@ def __init__(self, test_provisioner=None, test_user=None):
             'state': config.get('CLUSTER_UNKNOWN_STATE'),
             'kubeconfig': yaml.load(open('kubeconfig_localhost', 'r').read()),
             'created_at': datetime.datetime.utcnow().replace(microsecond=0),
-            'owner': owner
+            'owner': owner,
+            'metadata': {}
         }
         self.obj = Cluster.create(owner.namespace, **create_kwargs)
 

kqueen/engines/openstack_kubespray.py

@@ -4,6 +4,7 @@
 from kqueen import kubeapi
 
 import base64
+import copy
 import ipaddress
 import json
 import logging
@@ -299,31 +300,41 @@ def _cleanup_pvc(self):
 
     def _scale_up(self, new_slave_count):
         try:
-            self.cluster.state = config.CLUSTER_UPDATING_STATE
-            self.cluster.save()
+            old_resources = copy.deepcopy(self.cluster.metadata["resources"])
+            resources = {}
             resources = self.os.grow(resources=self.cluster.metadata["resources"],
                                      new_slave_count=new_slave_count)
-            self.cluster.metadata["resources"] = resources
-            self.cluster.save()
             self.ks.scale(resources)
         except Exception as e:
             logger.exception("Failed to resize cluster: %s" % e)
+            if "slaves" in resources:
+                if len(resources["slaves"]) != len(old_resources["slaves"]):
+                    old_ids = [slave["hostname"] for slave in old_resources["slaves"]]
+                    ids = [slave["hostname"] for slave in resources["slaves"]]
+                    self.os.shrink(resources=resources,
+                                   remove_hostnames=set(ids) - set(old_ids))
+                    self.cluster.metadata["resources"] = old_resources
+        else:
+            self.cluster.metadata["resources"] = resources
+            self.cluster.metadata["slave_count"] = new_slave_count
+            self.cluster.metadata["node_count"] = new_slave_count + self.cluster.metadata["master_count"]
         finally:
             self.cluster.state = config.CLUSTER_OK_STATE
             self.cluster.save()
 
     def _scale_down(self, new_slave_count):
         try:
-            self.cluster.state = config.CLUSTER_UPDATING_STATE
-            self.cluster.save()
             resources = self.cluster.metadata["resources"]
             remove_hostnames = self.ks.shrink(resources,
                                               new_slave_count=new_slave_count)
             resources = self.os.shrink(resources=resources,
                                        remove_hostnames=remove_hostnames)
-            self.cluster.metadata["resources"] = resources
         except Exception as e:
             logger.exception("Failed to resize cluster: %s" % e)
+        else:
+            self.cluster.metadata["resources"] = resources
+            self.cluster.metadata["slave_count"] = new_slave_count
+            self.cluster.metadata["node_count"] = new_slave_count + self.cluster.metadata["master_count"]
         finally:
             self.cluster.state = config.CLUSTER_OK_STATE
             self.cluster.save()
@@ -334,19 +345,23 @@ def resize(self, node_count):
         node_count = int(node_count)
         master_count = len(self.cluster.metadata["resources"]["masters"])
         new_slave_count = node_count - master_count
-        if new_slave_count < 0:
-            return False, "Node count should be at least %s" % master_count
+        if new_slave_count <= 0:
+            return False, "There are should be at least one slave node besides master nodes"
         current_slave_count = len(self.cluster.metadata["resources"]["slaves"])
         delta = new_slave_count - current_slave_count
+
+        if delta == 0:
+            return False, "Cluster already has %s nodes" % node_count
+        self.cluster.state = config.CLUSTER_UPDATING_STATE
+        self.cluster.save()
         if delta > 0:
             logger.info("Scaling up %s -> %s slaves" % (current_slave_count, new_slave_count))
             app.executor.submit(self._scale_up, new_slave_count)
             return True, "Resizing started"
-        elif delta < 0:
+        else:
             logger.info("Scaling down %s -> %s slaves" % (current_slave_count, new_slave_count))
             app.executor.submit(self._scale_down, new_slave_count)
             return True, "Resizing started"
-        return False, "Cluster already has %s nodes" % node_count
 
     def get_kubeconfig(self):
         return self.cluster.kubeconfig
@@ -592,8 +607,11 @@ def _run_ansible(self, inventory="hosts.json", playbook="cluster.yml"):
                 env=env,
             )
             pipe.wait()
-            if pipe.returncode:
-                logger.warning("Non zero exit status from ansible (%s)" % pipe.returncode)
+        if pipe.returncode:
+            with open(self.ansible_log, "r") as log_file:
+                result = log_file.read()
+            if result.find("fatal:"):
+                raise RuntimeError("Ansible command execution failed ({})".format(" ".join(args)))
 
     def _get_kubeconfig(self, ip):
         cat_kubeconf = "sudo cat /etc/kubernetes/admin.conf"

kqueen/models.py

@@ -38,26 +38,25 @@ class Cluster(Model, metaclass=ModelMeta):
     owner = RelationField(required=True, remote_class_name='User')
 
     def update_state(self):
+        # Check for stale clusters
+        max_age = timedelta(seconds=config.get('PROVISIONER_TIMEOUT'))
+        provisioning_state = config.get('CLUSTER_PROVISIONING_STATE')
         try:
             remote_cluster = self.engine.cluster_get()
         except Exception as e:
             logger.exception('Unable to get data from backend for cluster {}'.format(self.name))
-            remote_cluster = {}
-
-        if 'state' in remote_cluster:
-            self.set_status(remote_cluster)
-            if remote_cluster['state'] == self.state:
-                return self.state
-            self.state = remote_cluster['state']
-        else:
             return config.get('CLUSTER_UNKNOWN_STATE')
 
-        # Check for stale clusters
-        max_age = timedelta(seconds=config.get('PROVISIONER_TIMEOUT'))
-        provisioning_state = config.get('CLUSTER_PROVISIONING_STATE')
-        if self.state == provisioning_state and datetime.utcnow() - self.created_at > max_age:
+        if remote_cluster['state'] == provisioning_state and datetime.utcnow() - self.created_at > max_age:
             self.state = config.get('CLUSTER_ERROR_STATE')
+            self.metadata['status_message'] = "Cluster deployment haven't finish in {}".format(max_age)
+            self.save()
+            return self.state
 
+        self.set_status(remote_cluster)
+        if remote_cluster['state'] == self.state:
+            return self.state
+        self.state = remote_cluster['state']
         self.save()
         return self.state
 

kqueen/tests/test_models.py

@@ -218,7 +218,8 @@ class TestClusterState:
     @pytest.fixture(autouse=True)
     def prepare(self, cluster, monkeypatch):
         def fake_cluster_get(self):
-            return {'state': config.get('CLUSTER_PROVISIONING_STATE')}
+            return {'state': config.get('CLUSTER_PROVISIONING_STATE'),
+                    'metadata': {}}
 
         monkeypatch.setattr(ManualEngine, 'cluster_get', fake_cluster_get)