Merge 0586b5d into 38b4519

cloudman/clusterman/api.py

@@ -1,9 +1,13 @@
 """CloudMan Service API."""
 import uuid
+
+from django.db import IntegrityError
+
 from rest_framework.exceptions import PermissionDenied
 
 from cloudlaunch import models as cl_models
 from cloudlaunch_cli.api.client import APIClient
+from . import exceptions
 from . import models
 from . import resources
 
@@ -109,14 +113,18 @@ def get(self, cluster_id):
 
     def create(self, name, cluster_type, connection_settings, autoscale=True):
         self.check_permissions('clusters.add_cluster')
-        obj = models.CMCluster.objects.create(
-            name=name, cluster_type=cluster_type,
-            connection_settings=connection_settings,
-            autoscale=autoscale)
-        cluster = self.to_api_object(obj)
-        template = cluster.get_cluster_template()
-        template.setup()
-        return cluster
+        try:
+            obj = models.CMCluster.objects.create(
+                name=name, cluster_type=cluster_type,
+                connection_settings=connection_settings,
+                autoscale=autoscale)
+            cluster = self.to_api_object(obj)
+            template = cluster.get_cluster_template()
+            template.setup()
+            return cluster
+        except IntegrityError as e:
+            raise exceptions.CMDuplicateNameException(
+                "A cluster with name: %s already exists" % name)
 
     def update(self, cluster):
         self.check_permissions('clusters.change_cluster', cluster)

cloudman/clusterman/cluster_templates.py

@@ -117,7 +117,10 @@ def add_node(self, name, vm_type=None, zone=None):
             'config_app': {
                 'rancher_action': 'add_node',
                 'config_rancher_kube': {
+                    'rancher_url': self.rancher_url,
+                    'rancher_api_key': self.rancher_api_key,
                     'rancher_cluster_id': self.rancher_cluster_id,
+                    'rancher_project_id': self.rancher_project_id,
                     'rancher_node_command': (
                         self.rancher_client.get_cluster_registration_command()
                         + " --worker")

cloudman/clusterman/exceptions.py

@@ -1,5 +1,5 @@
 # Exception hierarchy for cloudman
 
 
-class InvalidStateException(Exception):
+class CMDuplicateNameException(Exception):
     pass

cloudman/clusterman/management/commands/create_autoscale_user.py

@@ -4,6 +4,8 @@
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand
 
+from clusterman.models import GlobalSettings
+
 
 class Command(BaseCommand):
     help = 'Creates a user for managing autoscaling. This user has permissions to scale' \
@@ -16,12 +18,17 @@ def add_arguments(self, parser):
         parser.add_argument(
             '--password', required=False,
             help='Password for this user, autogenerated if not specified')
+        parser.add_argument(
+            '--impersonate_account', required=False,
+            help='User account to impersonate when scaling. This account is assumed to have stored'
+                 ' cloud credentials or IAM access. Defaults to the first super admin found.')
 
     def handle(self, *args, **options):
         username = options['username']
         password = options['password']
+        account = options['impersonate_account']
 
-        return self.create_autoscale_user(username, password)
+        return self.create_autoscale_user(username, password, account)
 
     @staticmethod
     def _add_permissions(user, perm_names):
@@ -31,7 +38,7 @@ def _add_permissions(user, perm_names):
         return user
 
     @staticmethod
-    def create_autoscale_user(username, password):
+    def create_autoscale_user(username, password, account):
         try:
             print("Creating autoscale user: {0}".format(username))
             user, created = User.objects.get_or_create(username=username)
@@ -41,6 +48,11 @@ def create_autoscale_user(username, password):
                     user, ['view_cmcluster', 'add_cmclusternode',
                            'delete_cmclusternode'])
                 user.save()
+                if account:
+                    impersonate_user = User.objects.get(username=account)
+                else:
+                    impersonate_user = User.objects.filter(is_superuser=True).first()
+                GlobalSettings().settings.autoscale_impersonate = impersonate_user.username
                 return "Autoscale user created successfully."
             else:
                 return "Autoscale user already exists."

cloudman/clusterman/management/commands/create_cluster.py

@@ -50,4 +50,4 @@ def create_cluster(name, cluster_type, settings):
             print("cluster created successfully.")
         except Exception as e:
             log.exception("An error occurred while creating the initial cluster!!:")
-            print("An error occurred while creating the initial cluster!!:", e)
+            print("An error occurred while creating the initial cluster!!:", str(e))

cloudman/clusterman/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 2.2.10 on 2020-02-17 18:49
+# Generated by Django 2.2.10 on 2020-02-23 17:26
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -34,7 +34,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('added', models.DateTimeField(auto_now_add=True)),
                 ('updated', models.DateTimeField(auto_now=True)),
-                ('name', models.CharField(max_length=60)),
+                ('name', models.CharField(max_length=60, unique=True)),
                 ('cluster_type', models.CharField(max_length=255)),
                 ('autoscale', models.BooleanField(default=True, help_text='Whether autoscaling is activated')),
                 ('_connection_settings', models.TextField(blank=True, db_column='connection_settings', help_text='External provider specific settings for this cluster.', max_length=16384, null=True)),
@@ -44,6 +44,14 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': 'Clusters',
             },
         ),
+        migrations.CreateModel(
+            name='GlobalSettings_SettingsStore',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('key', models.CharField(max_length=255)),
+                ('value', models.TextField()),
+            ],
+        ),
         migrations.CreateModel(
             name='CMClusterNode',
             fields=[

cloudman/clusterman/models.py

@@ -1,17 +1,27 @@
 from django.db import models
 
+from hierarkey.models import GlobalSettingsBase, Hierarkey
+
 from cloudlaunch import models as cl_models
 from djcloudbridge import models as cb_models
 import yaml
 
 
+hierarkey = Hierarkey(attribute_name='settings')
+
+
+@hierarkey.set_global()
+class GlobalSettings(GlobalSettingsBase):
+    pass
+
+
 class CMCluster(models.Model):
     """CloudMan cluster details."""
     # Automatically add timestamps when object is created
     added = models.DateTimeField(auto_now_add=True)
     # Automatically add timestamps when object is updated
     updated = models.DateTimeField(auto_now=True)
-    name = models.CharField(max_length=60)
+    name = models.CharField(max_length=60, unique=True)
     cluster_type = models.CharField(max_length=255, blank=False, null=False)
     autoscale = models.BooleanField(
         default=True, help_text="Whether autoscaling is activated")

cloudman/clusterman/plugins/rancher_kubernetes_app.py

@@ -1,10 +1,14 @@
 """Plugin implementation for a simple web application."""
+import time
+
 from celery.utils.log import get_task_logger
 
 from cloudlaunch.backend_plugins.base_vm_app import BaseVMAppPlugin
 from cloudlaunch.backend_plugins.cloudman2_app import get_iam_handler_for
 from cloudlaunch.configurers import AnsibleAppConfigurer
 
+from clusterman.rancher import RancherClient
+
 from rest_framework.serializers import ValidationError
 
 log = get_task_logger('cloudlaunch')
@@ -48,6 +52,12 @@ def deploy(self, name, task, app_config, provider_config, **kwargs):
             name, task, app_config, provider_config)
         return result
 
+    def _create_rancher_client(self, rancher_cfg):
+        return RancherClient(rancher_cfg.get('rancher_url'),
+                             rancher_cfg.get('rancher_api_key'),
+                             rancher_cfg.get('rancher_cluster_id'),
+                             rancher_cfg.get('rancher_project_id'))
+
     def delete(self, provider, deployment):
         """
         Delete resource(s) associated with the supplied deployment.
@@ -58,8 +68,20 @@ def delete(self, provider, deployment):
         *Note* that this method will delete resource(s) associated with
         the deployment - this is an un-recoverable action.
         """
-        # key += get_required_val(rancher_config, "RANCHER_API_KEY")
-        # Contact rancher API and delete node
+        app_config = deployment.get('app_config')
+        rancher_cfg = app_config.get('config_rancher_kube')
+        rancher_client = self._create_rancher_client(rancher_cfg)
+        node_ip = deployment.get(
+            'launch_result', {}).get('cloudLaunch', {}).get('publicIP')
+        rancher_node_id = rancher_client.find_node(ip=node_ip)
+        if rancher_node_id:
+            rancher_client.drain_node(rancher_node_id)
+            # during tests, node_ip is None, so skip sleep if so
+            if node_ip:
+                time.sleep(60)
+            # remove node from rancher
+            rancher_client.delete_node(rancher_node_id)
+        # delete the VM
         return super().delete(provider, deployment)
 
     def _get_configurer(self, app_config):

cloudman/clusterman/rancher.py

@@ -1,4 +1,5 @@
 import requests
+from string import Template
 from requests.auth import AuthBase
 
 
@@ -17,44 +18,47 @@ def __call__(self, r):
 
 class RancherClient(object):
 
-    KUBE_CONFIG_URL = ("{rancher_url}/v3/clusters/{cluster_id}"
+    KUBE_CONFIG_URL = ("$rancher_url/v3/clusters/$cluster_id"
                        "?action=generateKubeconfig")
-    INSTALLED_APP_URL = ("{rancher_url}/v3/projects/{project_id}/app"
+    INSTALLED_APP_URL = ("$rancher_url/v3/projects/$project_id/app"
                          "?targetNamespace=galaxy-ns")
-    NODE_COMMAND_URL = "{rancher_url}/v3/clusterregistrationtoken"
+    NODE_COMMAND_URL = "$rancher_url/v3/clusterregistrationtoken"
+    NODE_LIST_URL = "$rancher_url/v3/nodes/?clusterId=$cluster_id"
+    NODE_DRAIN_URL = "$rancher_url/v3/nodes/$node_id?action=drain"
+    NODE_DELETE_URL = "$rancher_url/v3/nodes/$node_id"
 
     def __init__(self, rancher_url, api_key, cluster_id, project_id):
         self.rancher_url = rancher_url
         self.api_key = api_key
         self.cluster_id = cluster_id
         self.project_id = project_id
 
-    def format_url(self, url):
-        return url.format(rancher_url=self.rancher_url,
-                          cluster_id=self.cluster_id,
-                          project_id=self.project_id)
+    def _format_url(self, url):
+        result = Template(url).safe_substitute({
+            'rancher_url': self.rancher_url,
+            'cluster_id': self.cluster_id,
+            'project_id': self.project_id
+        })
+        return result
 
-    def get_auth(self):
+    def _get_auth(self):
         return RancherAuth(self)
 
-    def _api_get(self, url):
-        return requests.get(self.format_url(url), auth=self.get_auth(),
-                            verify=False).json()
-
-    def _api_post(self, url, data):
-        return requests.post(self.format_url(url), auth=self.get_auth(),
-                             verify=False, json=data).json()
-
-    def _api_put(self, url, data):
-        return requests.put(self.format_url(url), auth=self.get_auth(),
+    def _api_get(self, url, data):
+        return requests.get(self._format_url(url), auth=self._get_auth(),
                             verify=False, json=data).json()
 
-    def list_installed_charts(self):
-        return self._api_get(self.INSTALLED_APP_URL).get('data')
+    def _api_post(self, url, data, json_response=True):
+        r = requests.post(self._format_url(url), auth=self._get_auth(),
+                          verify=False, json=data)
+        if json_response:
+            return r.json()
+        else:
+            return r
 
-    def update_installed_chart(self, data):
-        r = self._api_put(data.get('links').get('self'), data)
-        return r
+    def _api_delete(self, url, data):
+        return requests.delete(self._format_url(url), auth=self._get_auth(),
+                               verify=False, json=data).json()
 
     def fetch_kube_config(self):
         return self._api_post(self.KUBE_CONFIG_URL, data=None).get('config')
@@ -65,3 +69,30 @@ def get_cluster_registration_command(self):
             data={"type": "clusterRegistrationToken",
                   "clusterId": f"{self.cluster_id}"}
         ).get('nodeCommand')
+
+    def get_nodes(self):
+        return self._api_get(self.NODE_LIST_URL, data=None)
+
+    def find_node(self, ip):
+        matches = [n for n in self.get_nodes()['data']
+                   if n.get('ipAddress') == ip or
+                   n.get('externalIpAddress') == ip]
+        return matches[0]['id'] if matches else None
+
+    def drain_node(self, node_id):
+        node_url = Template(self.NODE_DRAIN_URL).safe_substitute({
+            'node_id': node_id
+        })
+        return self._api_post(node_url, data={
+            "deleteLocalData": True,
+            "force": True,
+            "ignoreDaemonSets": True,
+            "gracePeriod": "-1",
+            "timeout": "60"
+        }, json_response=False)
+
+    def delete_node(self, node_id):
+        node_url = Template(self.NODE_DELETE_URL).safe_substitute({
+            'node_id': node_id
+        })
+        return self._api_delete(node_url, data=None)

cloudman/clusterman/rules.py

@@ -35,3 +35,5 @@ def has_autoscale_permissions(user, obj):
 rules.add_perm('clusternodes.add_clusternode', is_node_owner | has_autoscale_permissions | rules.is_staff)
 rules.add_perm('clusternodes.change_clusternode', is_node_owner | has_autoscale_permissions | rules.is_staff)
 rules.add_perm('clusternodes.delete_clusternode', is_node_owner | has_autoscale_permissions | rules.is_staff)
+
+rules.add_perm('autoscalers.can_autoscale', has_autoscale_permissions | rules.is_staff)

cloudman/clusterman/serializers.py

@@ -1,11 +1,15 @@
 """DRF serializers for the CloudMan Create API endpoints."""
 
 from rest_framework import serializers
+from rest_framework import status
+from rest_framework.exceptions import ValidationError
+
 from cloudlaunch import serializers as cl_serializers
 from djcloudbridge import models as cb_models
 from djcloudbridge.drf_helpers import CustomHyperlinkedIdentityField
+
 from .api import CloudManAPI
-from rest_framework.exceptions import ValidationError
+from .exceptions import CMDuplicateNameException
 
 
 class CMClusterSerializer(serializers.Serializer):
@@ -21,10 +25,14 @@ class CMClusterSerializer(serializers.Serializer):
                                            lookup_url_kwarg='cluster_pk')
 
     def create(self, valid_data):
-        return CloudManAPI.from_request(self.context['request']).clusters.create(
-            valid_data.get('name'), valid_data.get('cluster_type'),
-            valid_data.get('connection_settings'),
-            autoscale=valid_data.get('autoscale'))
+        try:
+            cmapi = CloudManAPI.from_request(self.context['request'])
+            return cmapi.clusters.create(
+                valid_data.get('name'), valid_data.get('cluster_type'),
+                valid_data.get('connection_settings'),
+                autoscale=valid_data.get('autoscale'))
+        except CMDuplicateNameException as e:
+            raise ValidationError(detail=str(e))
 
     def update(self, instance, valid_data):
         instance.name = valid_data.get('name') or instance.name