When deleting nodes, also drain and delete from rancher

cloudman/clusterman/cluster_templates.py

@@ -117,7 +117,10 @@ def add_node(self, name, vm_type=None, zone=None):
             'config_app': {
                 'rancher_action': 'add_node',
                 'config_rancher_kube': {
+                    'rancher_url': self.rancher_url,
+                    'rancher_api_key': self.rancher_api_key,
                     'rancher_cluster_id': self.rancher_cluster_id,
+                    'rancher_project_id': self.rancher_project_id,
                     'rancher_node_command': (
                         self.rancher_client.get_cluster_registration_command()
                         + " --worker")

cloudman/clusterman/plugins/rancher_kubernetes_app.py

@@ -1,10 +1,14 @@
 """Plugin implementation for a simple web application."""
+import time
+
 from celery.utils.log import get_task_logger
 
 from cloudlaunch.backend_plugins.base_vm_app import BaseVMAppPlugin
 from cloudlaunch.backend_plugins.cloudman2_app import get_iam_handler_for
 from cloudlaunch.configurers import AnsibleAppConfigurer
 
+from clusterman.rancher import RancherClient
+
 from rest_framework.serializers import ValidationError
 
 log = get_task_logger('cloudlaunch')
@@ -48,6 +52,12 @@ def deploy(self, name, task, app_config, provider_config, **kwargs):
             name, task, app_config, provider_config)
         return result
 
+    def _create_rancher_client(self, rancher_cfg):
+        return RancherClient(rancher_cfg.get('rancher_url'),
+                             rancher_cfg.get('rancher_api_key'),
+                             rancher_cfg.get('rancher_cluster_id'),
+                             rancher_cfg.get('rancher_project_id'))
+
     def delete(self, provider, deployment):
         """
         Delete resource(s) associated with the supplied deployment.
@@ -58,8 +68,20 @@ def delete(self, provider, deployment):
         *Note* that this method will delete resource(s) associated with
         the deployment - this is an un-recoverable action.
         """
-        # key += get_required_val(rancher_config, "RANCHER_API_KEY")
-        # Contact rancher API and delete node
+        app_config = deployment.get('app_config')
+        rancher_cfg = app_config.get('config_rancher_kube')
+        rancher_client = self._create_rancher_client(rancher_cfg)
+        node_ip = deployment.get(
+            'launch_result', {}).get('cloudLaunch', {}).get('publicIP')
+        rancher_node_id = rancher_client.find_node(ip=node_ip)
+        if rancher_node_id:
+            rancher_client.drain_node(rancher_node_id)
+            # during tests, node_ip is None, so skip sleep if so
+            if node_ip:
+                time.sleep(60)
+            # remove node from rancher
+            rancher_client.delete_node(rancher_node_id)
+        # delete the VM
         return super().delete(provider, deployment)
 
     def _get_configurer(self, app_config):

cloudman/clusterman/rancher.py

@@ -1,4 +1,5 @@
 import requests
+from string import Template
 from requests.auth import AuthBase
 
 
@@ -17,44 +18,47 @@ def __call__(self, r):
 
 class RancherClient(object):
 
-    KUBE_CONFIG_URL = ("{rancher_url}/v3/clusters/{cluster_id}"
+    KUBE_CONFIG_URL = ("$rancher_url/v3/clusters/$cluster_id"
                        "?action=generateKubeconfig")
-    INSTALLED_APP_URL = ("{rancher_url}/v3/projects/{project_id}/app"
+    INSTALLED_APP_URL = ("$rancher_url/v3/projects/$project_id/app"
                          "?targetNamespace=galaxy-ns")
-    NODE_COMMAND_URL = "{rancher_url}/v3/clusterregistrationtoken"
+    NODE_COMMAND_URL = "$rancher_url/v3/clusterregistrationtoken"
+    NODE_LIST_URL = "$rancher_url/v3/nodes/?clusterId=$cluster_id"
+    NODE_DRAIN_URL = "$rancher_url/v3/nodes/$node_id?action=drain"
+    NODE_DELETE_URL = "$rancher_url/v3/nodes/$node_id"
 
     def __init__(self, rancher_url, api_key, cluster_id, project_id):
         self.rancher_url = rancher_url
         self.api_key = api_key
         self.cluster_id = cluster_id
         self.project_id = project_id
 
-    def format_url(self, url):
-        return url.format(rancher_url=self.rancher_url,
-                          cluster_id=self.cluster_id,
-                          project_id=self.project_id)
+    def _format_url(self, url):
+        result = Template(url).safe_substitute({
+            'rancher_url': self.rancher_url,
+            'cluster_id': self.cluster_id,
+            'project_id': self.project_id
+        })
+        return result
 
-    def get_auth(self):
+    def _get_auth(self):
         return RancherAuth(self)
 
-    def _api_get(self, url):
-        return requests.get(self.format_url(url), auth=self.get_auth(),
-                            verify=False).json()
-
-    def _api_post(self, url, data):
-        return requests.post(self.format_url(url), auth=self.get_auth(),
-                             verify=False, json=data).json()
-
-    def _api_put(self, url, data):
-        return requests.put(self.format_url(url), auth=self.get_auth(),
+    def _api_get(self, url, data):
+        return requests.get(self._format_url(url), auth=self._get_auth(),
                             verify=False, json=data).json()
 
-    def list_installed_charts(self):
-        return self._api_get(self.INSTALLED_APP_URL).get('data')
+    def _api_post(self, url, data, json_response=True):
+        r = requests.post(self._format_url(url), auth=self._get_auth(),
+                          verify=False, json=data)
+        if json_response:
+            return r.json()
+        else:
+            return r
 
-    def update_installed_chart(self, data):
-        r = self._api_put(data.get('links').get('self'), data)
-        return r
+    def _api_delete(self, url, data):
+        return requests.delete(self._format_url(url), auth=self._get_auth(),
+                               verify=False, json=data).json()
 
     def fetch_kube_config(self):
         return self._api_post(self.KUBE_CONFIG_URL, data=None).get('config')
@@ -65,3 +69,30 @@ def get_cluster_registration_command(self):
             data={"type": "clusterRegistrationToken",
                   "clusterId": f"{self.cluster_id}"}
         ).get('nodeCommand')
+
+    def get_nodes(self):
+        return self._api_get(self.NODE_LIST_URL, data=None)
+
+    def find_node(self, ip):
+        matches = [n for n in self.get_nodes()['data']
+                   if n.get('ipAddress') == ip or
+                   n.get('externalIpAddress') == ip]
+        return matches[0]['id'] if matches else None
+
+    def drain_node(self, node_id):
+        node_url = Template(self.NODE_DRAIN_URL).safe_substitute({
+            'node_id': node_id
+        })
+        return self._api_post(node_url, data={
+            "deleteLocalData": True,
+            "force": True,
+            "ignoreDaemonSets": True,
+            "gracePeriod": "-1",
+            "timeout": "60"
+        }, json_response=False)
+
+    def delete_node(self, node_id):
+        node_url = Template(self.NODE_DELETE_URL).safe_substitute({
+            'node_id': node_id
+        })
+        return self._api_delete(node_url, data=None)

cloudman/clusterman/tests/test_cluster_api.py

@@ -226,7 +226,6 @@ def create_mock_provider(self, name, config):
         responses.add_passthru('http://localhost')
         responses.add(responses.POST, 'https://127.0.0.1:4430/v3/clusterregistrationtoken',
                       json={'nodeCommand': 'docker run rancher --worker'}, status=200)
-
         super().setUp()
 
 
@@ -267,6 +266,18 @@ def _check_cluster_node_exists(self, cluster_id, node_id):
         return response.data['id']
 
     def _delete_cluster_node(self, cluster_id, node_id):
+        responses.add(responses.GET, 'https://127.0.0.1:4430/v3/nodes/?clusterId=c-abcd1',
+                      json=[
+                          {'id': 'c-ph9ck:m-01606aca4649',
+                           'ipAddress': '10.1.1.1',
+                           'externalIpAddress': None
+                           }
+                      ],
+                      status=200)
+        responses.add(responses.POST, 'https://127.0.0.1:4430/v3/nodes/c-ph9ck:m-01606aca4649?action=drain',
+                      json={}, status=200)
+        responses.add(responses.DELETE, 'https://127.0.0.1:4430/v3/nodes/c-ph9ck:m-01606aca4649',
+                      json={}, status=200)
         url = reverse('clusterman:node-detail', args=[cluster_id, node_id])
         return self.client.delete(url)