Merge f517524 into 7195774

.gitignore

@@ -6,6 +6,9 @@ syntax: glob
 *.DS_Store
 *.pyc
 *.sqlite3
+*.log
+
+cloudman/cloudman_server.egg-info/*
 
 cloudman/db.sqlite3
 cloudman/cloudman/settings_local.py

Dockerfile

@@ -4,7 +4,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED 1
 
 ENV KUBE_LATEST_VERSION=v1.16.2
-ENV HELM_VERSION=v2.15.2
+ENV HELM_VERSION=v3.0.2
 ENV HELM_FILENAME=helm-${HELM_VERSION}-linux-amd64.tar.gz
 
 RUN set -xe; \
@@ -20,7 +20,7 @@ RUN set -xe; \
         python3-setuptools \
         curl \
     && curl -L https://storage.googleapis.com/kubernetes-release/release/${KUBE_LATEST_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl \
-    && curl -L https://storage.googleapis.com/kubernetes-helm/${HELM_FILENAME} | tar xz && mv linux-amd64/helm /usr/local/bin/helm && rm -rf linux-amd64 \
+    && curl -L https://get.helm.sh/${HELM_FILENAME} | tar xz && mv linux-amd64/helm /usr/local/bin/helm && rm -rf linux-amd64 \
     && apt-get autoremove -y && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* \
     && mkdir -p /app \

cloudman/clusterman/migrations/0001_initial.py

@@ -9,7 +9,7 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('cloudlaunch', '0003_deployment_credentials_relationship'),
+        ('cloudlaunch', '0001_initial'),
     ]
 
     operations = [

cloudman/helmsman/api.py

@@ -1,8 +1,9 @@
 """HelmsMan Service API."""
 import jsonmerge
 
-from .helm.client import HelmClient
-from .helm.client import HelmValueHandling
+from .clients.helm_client import HelmClient
+from .clients.k8s_client import KubernetesClient
+from .clients.helm_client import HelmValueHandling
 
 
 class HelmsmanException(Exception):
@@ -13,6 +14,14 @@ class ChartExistsException(HelmsmanException):
     pass
 
 
+class NamespaceNotFoundException(HelmsmanException):
+    pass
+
+
+class NamespaceExistsException(HelmsmanException):
+    pass
+
+
 class HMServiceContext(object):
     """
     A class to contain contextual information when processing a
@@ -52,6 +61,7 @@ def __init__(self, context):
         super(HelmsManAPI, self).__init__(context)
         self._repo_svc = HMChartRepoService(context)
         self._chart_svc = HMChartService(context)
+        self._namespace_svc = HMNamespaceService(context)
 
     @classmethod
     def from_request(cls, request):
@@ -66,6 +76,43 @@ def repositories(self):
     def charts(self):
         return self._chart_svc
 
+    @property
+    def namespaces(self):
+        return self._namespace_svc
+
+
+class HMNamespaceService(HelmsManService):
+
+    def __init__(self, context):
+        super(HMNamespaceService, self).__init__(context)
+
+    def list(self):
+        return [KubectlNamespace(self, **namespace)
+                for namespace in KubernetesClient().namespaces.list()]
+
+    def get(self, namespace):
+        namespaces = (n for n in self.list() if n.name == namespace)
+        return next(namespaces, None)
+
+    def create(self, namespace):
+        client = KubernetesClient()
+        existing = self.get(namespace)
+        if existing:
+            raise NamespaceExistsException(
+                f"Namespace '{namespace}' already exists.")
+        else:
+            client.namespaces.create(namespace)
+        return self.get(namespace)
+
+    def delete(self, namespace):
+        client = KubernetesClient()
+        existing = self.get(namespace)
+        if not existing:
+            raise NamespaceNotFoundException(
+                f"Namespace {namespace} cannot be found.")
+        else:
+            client.namespaces.delete(namespace)
+
 
 class HMChartRepoService(HelmsManService):
 
@@ -105,7 +152,7 @@ def list(self):
                 state=release.get("STATUS"),
                 updated=release.get("UPDATED"),
                 values=HelmClient().releases.get_values(
-                    release.get("NAME"), get_all=True)
+                    release.get("NAME"), get_all=True, namespace=release.get("NAMESPACE"))
             )
             for release in releases
         ]
@@ -191,3 +238,15 @@ def __init__(self, service, id, name, namespace, **kwargs):
 
     def delete(self):
         self.service.delete(self)
+
+
+class KubectlNamespace(HelmsManResource):
+
+    def __init__(self, service, **kwargs):
+        super().__init__(service)
+        self.name = kwargs.get('NAME')
+        self.status = kwargs.get('STATUS')
+        self.age = kwargs.get('AGE')
+
+    def delete(self):
+        self.service.delete(self.name)

cloudman/helmsman/helm/client.py → cloudman/helmsman/clients/helm_client.py

@@ -1,11 +1,8 @@
 """A wrapper around the helm commandline client"""
-import logging as log
-import shlex
 import shutil
-import subprocess
 import tempfile
 import yaml
-from . import helpers
+from helmsman.clients import helpers
 from enum import Enum
 
 
@@ -27,36 +24,11 @@ def __init__(self):
         self._repo_svc = HelmRepositoryService(self)
         self._chart_svc = HelmChartService(self)
 
-    def _check_environment(self):
+    @staticmethod
+    def _check_environment():
         if not shutil.which("helm"):
             raise Exception("Could not find helm executable in path")
 
-    def install_helm(self):
-        # FIXME: Check whether tiller role exists instead of ignoring exception
-        try:
-            cmd = (
-                "kubectl create serviceaccount --namespace kube-system tiller"
-                " && kubectl create clusterrolebinding tiller-cluster-role"
-                " --clusterrole=cluster-admin"
-                " --serviceaccount=kube-system:tiller")
-            helpers.run_command(cmd, shell=True)
-        except subprocess.CalledProcessError as e:
-            log.exception("Could not create tiller role bindings. "
-                          "Reason: {0}".format(e.output))
-        print("Initializing tiller...")
-        self.helm_init(service_account="tiller", wait=True)
-        print("Tiller initialized.")
-
-    def helm_init(self, service_account=None, upgrade=False, wait=False):
-        cmd = ["helm", "init"]
-        if service_account:
-            cmd += ["--service-account", service_account]
-        if upgrade:
-            cmd += ["--upgrade"]
-        if wait:
-            cmd += ["--wait"]
-        return helpers.run_command(cmd)
-
     @property
     def releases(self):
         return self._release_svc
@@ -82,7 +54,7 @@ def __init__(self, client):
         super(HelmReleaseService, self).__init__(client)
 
     def list(self):
-        data = helpers.run_list_command(["helm", "list"])
+        data = helpers.run_list_command(["helm", "list", "--all-namespaces"])
         return data
 
     def get(self, release_name):
@@ -103,12 +75,14 @@ def _set_values_and_run_command(self, cmd, values):
 
     def create(self, chart, namespace, release_name=None,
                version=None, values=None):
-        cmd = ["helm", "install", chart]
+        cmd = ["helm", "install"]
 
+        if release_name:
+            cmd += [release_name, chart]
+        else:
+            cmd += [chart, "--generate-name"]
         if namespace:
             cmd += ["--namespace", namespace]
-        if release_name:
-            cmd += ["--name", release_name]
         if version:
             cmd += ["--version", version]
         return self._set_values_and_run_command(cmd, values)
@@ -143,19 +117,22 @@ def rollback(self, release_name, revision=None):
                 revision = history[-2].get('REVISION')
             else:
                 return
-        return helpers.run_command(["helm", "rollback", release_name, revision])
+        return helpers.run_command(["helm", "rollback",
+                                    release_name, revision])
 
     def delete(self, release_name):
         return helpers.run_command(["helm", "delete", release_name])
 
-    def get_values(self, release_name, get_all=True):
+    def get_values(self, release_name, get_all=True, namespace=None):
         """
         get_all=True will also dump chart default values.
         get_all=False will only return user overridden values.
         """
         cmd = ["helm", "get", "values", release_name]
         if get_all:
             cmd += ["--all"]
+        if namespace:
+            cmd += ["--namespace", namespace]
         return yaml.safe_load(helpers.run_command(cmd))
 
     @staticmethod

cloudman/helmsman/helm/helpers.py → cloudman/helmsman/clients/helpers.py

@@ -11,13 +11,13 @@ def run_command(command, shell=False):
                                    shell=shell, encoding='utf-8')
 
 
-def run_list_command(command):
+def run_list_command(command, delimiter="\t", skipinitialspace=True):
     """
     Runs a command, and parses the output as
     tab separated columnar output. First row must be column names."
     """
     output = run_command(command)
-    reader = csv.DictReader(io.StringIO(output), delimiter="\t")
+    reader = csv.DictReader(io.StringIO(output), delimiter=delimiter, skipinitialspace=skipinitialspace)
     output = []
     for row in reader:
         data = {key.strip(): val.strip() for key, val in row.items()}
@@ -27,12 +27,12 @@ def run_list_command(command):
 
 # based on: https://codereview.stackexchange.com/questions/21033/flatten-dic
 # tionary-in-python-functional-style
-def flatten_dict(d):
-    def items():
-        for key, value in d.items():
-            if isinstance(value, dict):
-                for subkey, subvalue in flatten_dict(value).items():
-                    yield key + "." + subkey, subvalue
-            else:
-                yield key, value
-    return dict(items())
+# def flatten_dict(d):
+#     def items():
+#         for key, value in d.items():
+#             if isinstance(value, dict):
+#                 for subkey, subvalue in flatten_dict(value).items():
+#                     yield key + "." + subkey, subvalue
+#             else:
+#                 yield key, value
+#     return dict(items())

cloudman/helmsman/clients/k8s_client.py

@@ -0,0 +1,57 @@
+"""A wrapper around the kubectl commandline client"""
+import shutil
+from . import helpers
+
+
+class KubernetesService(object):
+    """Marker interface for CloudMan services"""
+    def __init__(self, client):
+        self._client = client
+
+    def client(self):
+        return self._client
+
+
+class KubernetesClient(KubernetesService):
+
+    def __init__(self):
+        self._check_environment()
+        super(KubernetesClient, self).__init__(self)
+        self._namespace_svc = KubernetesNamespaceService(self)
+
+    @staticmethod
+    def _check_environment():
+        if not shutil.which("kubectl"):
+            raise Exception("Could not find kubectl executable in path")
+
+    @property
+    def namespaces(self):
+        return self._namespace_svc
+
+
+class KubernetesNamespaceService(KubernetesService):
+
+    def __init__(self, client):
+        super(KubernetesNamespaceService, self).__init__(client)
+
+    def list(self):
+        data = helpers.run_list_command(["kubectl", "get", "namespaces"],
+                                        delimiter=" ", skipinitialspace=True)
+        return data
+
+    # def _list_names(self):
+    #     data = self.list()
+    #     output = [each.get('NAME') for each in data]
+    #     return output
+
+    def create(self, namespace_name):
+        return helpers.run_command(["kubectl", "create",
+                                    "namespace", namespace_name])
+
+    # def _create_if_not_exists(self, namespace_name):
+    #     if namespace_name not in self._list_names():
+    #         return self.create(namespace_name)
+
+    def delete(self, namespace_name):
+        return helpers.run_command(["kubectl", "delete",
+                                    "namespace", namespace_name])