Fixing the stop_models functions by calling explicitly the removal …

…of the replica sets and pods (#732) * Add removal of replicasets to the stop functions * Removing replica set is not enough - explicitly remove pods * Refactoring after feedback
ucbrise · Jun 13, 2019 · df96eb4 · df96eb4
1 parent a72957e
commit df96eb4
Showing 1 changed file with 32 additions and 12 deletions.
diff --git a/clipper_admin/clipper_admin/kubernetes/kubernetes_container_manager.py b/clipper_admin/clipper_admin/kubernetes/kubernetes_container_manager.py
@@ -168,7 +168,7 @@ def __init__(self,
         self._k8s_v1 = client.CoreV1Api()
         self._k8s_beta = client.ExtensionsV1beta1Api()
         self._k8s_rbac = client.RbacAuthorizationV1beta1Api()
-        
+
 
         # Create the template engine
         # Config: Any variable missing -> Error
@@ -624,28 +624,48 @@ def stop_models(self, models):
         try:
             for m in models:
                 for v in models[m]:
+                    label_selector="{label}={val}, {cluster_label}={cluster_name}".format(
+                        label=CLIPPER_MODEL_CONTAINER_LABEL,
+                        val=create_model_container_label(m, v),
+                        cluster_label=CLIPPER_DOCKER_LABEL,
+                        cluster_name=self.cluster_name)
+
                     self._k8s_beta.delete_collection_namespaced_deployment(
                         namespace=self.k8s_namespace,
-                        label_selector=
-                        "{label}={val}, {cluster_label}={cluster_name}".format(
-                            label=CLIPPER_MODEL_CONTAINER_LABEL,
-                            val=create_model_container_label(m, v),
-                            cluster_label=CLIPPER_DOCKER_LABEL,
-                            cluster_name=self.cluster_name))
+                        label_selector=label_selector)
+
+                    self._k8s_beta.delete_collection_namespaced_replica_set(
+                        namespace=self.k8s_namespace,
+                        label_selector=label_selector)
+
+                    self._k8s_v1.delete_collection_namespaced_pod(
+                        namespace=self.k8s_namespace,
+                        label_selector=label_selector)
+
         except ApiException as e:
             self.logger.warning(
                 "Exception deleting kubernetes deployments: {}".format(e))
             raise e
 
     def stop_all_model_containers(self):
         try:
+            label_selector="{label}, {cluster_label}={cluster_name}".format(
+                label=CLIPPER_MODEL_CONTAINER_LABEL,
+                cluster_label=CLIPPER_DOCKER_LABEL,
+                cluster_name=self.cluster_name)
+
             self._k8s_beta.delete_collection_namespaced_deployment(
                 namespace=self.k8s_namespace,
-                label_selector="{label}, {cluster_label}={cluster_name}".
-                format(
-                    label=CLIPPER_MODEL_CONTAINER_LABEL,
-                    cluster_label=CLIPPER_DOCKER_LABEL,
-                    cluster_name=self.cluster_name))
+                label_selector=label_selector)
+
+            self._k8s_beta.delete_collection_namespaced_replica_set(
+                namespace=self.k8s_namespace,
+                label_selector=label_selector)
+
+            self._k8s_v1.delete_collection_namespaced_pod(
+                namespace=self.k8s_namespace,
+                label_selector=label_selector)
+
         except ApiException as e:
             self.logger.warning(
                 "Exception deleting kubernetes deployments: {}".format(e))