Parallel cleanup of kubernetes objects.

Same code can be used for any other contrail object.

This code will help reduce clenup time

Change-Id: I77bae0ce394e38d70a4d1fffeaff51692e2e6d96
Closes-bug: #1773624

common/k8s/base.py

@@ -6,6 +6,7 @@
 import vnc_api_test
 import uuid
 from tcutils.util import get_random_name, retry
+from tcutils.gevent_lib import exec_in_parallel
 from tcutils.verification_util import *
 from lxml import etree
 from k8s.pod import PodFixture
@@ -19,6 +20,7 @@
 from common import create_public_vn
 from common.base import GenericTestBase
 from vn_test import VNFixture
+import gevent
 
 K8S_SERVICE_IPAM = ['default-domain', 'default', 'service-ipam']
 K8S_PUBLIC_VN_NAME = '__public__'
@@ -1118,4 +1120,25 @@ def cleanup_fabric_gw(cls):
         cls.vnc_h.delete_bgp_router(cls.name)
     #end fabric_fabric_gw
 
+    def delete_in_parallel(self, fixture_type_list):
+        ''' Populates list of objects to be deleted parallely based on
+            Fixture Type as input like VmFixture
+            Then It deletes the objects in the list parallely
+        '''
+        parallel_cleanup_list = list()
+        for fn in self._cleanups[:]:
+            try:
+                if fn[0].im_self.__class__.__name__ in fixture_type_list:
+                    index = self._cleanups.index(fn)
+                    parallel_cleanup_list.append(fn)
+                    self._cleanups.pop(index)
+            except AttributeError:
+                pass
+        if parallel_cleanup_list:
+            greenlets = exec_in_parallel(parallel_cleanup_list)
+            gevent.joinall(greenlets)
+        else:
+            self.logger.warn("Nothing to delete parallely")
+            return
+    #end delete_in_parallel
 

fixtures/k8s/pod.py

@@ -143,7 +143,6 @@ def delete_only(self):
     def delete(self):
         if not self.already_exists:
             resp = self.delete_only()
-
             assert self.verify_on_cleanup()
     # end delete
 

scripts/k8s_scripts/test_deployment.py

@@ -4,7 +4,11 @@
 import test
 
 class TestPodDeployment(BaseK8sTest):
-
+
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @preposttest_wrapper
     def test_deployment_1(self):
         '''

scripts/k8s_scripts/test_fabric_forwarding.py

@@ -14,6 +14,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestFabricFWD, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_namespaces_pods_for_fabric_test(self, isolation=False,ip_fabric_forwarding=False):
         """ common routine to create the namesapces and the pods  by enabling the fabric forwarding
             1.create 2 namespaces (ns1,ns2 enable fabric forwarding)

scripts/k8s_scripts/test_ingress.py

@@ -14,6 +14,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestIngress, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @skip_because(mx_gw = False)
     @preposttest_wrapper
     def test_ingress_1(self):

scripts/k8s_scripts/test_ingress_tls.py

@@ -14,6 +14,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestIngressTLS, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['k8s_sanity']) 
     @skip_because(mx_gw = False)
     @preposttest_wrapper

scripts/k8s_scripts/test_isolation.py

@@ -16,6 +16,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestNSIsolation, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self, prov_service = False):
         service_ns1 = None
         service_ns2 = None
@@ -144,6 +148,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestCustomIsolation, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self, prov_service = False, prov_ingress = False):
         service_ns1, ingress_ns1 = None, None
         service_ns2, ingress_ns2 = None, None

scripts/k8s_scripts/test_namespace.py

@@ -14,6 +14,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestNamespace, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['openshift_1'])
     @preposttest_wrapper
     def test_namespace_1(self):

scripts/k8s_scripts/test_pod.py

@@ -12,6 +12,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestPod, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['openshift_1'])
     @preposttest_wrapper
     def test_add_delete_pod(self):

scripts/k8s_scripts/test_policy.py

@@ -10,8 +10,11 @@
 import test
 from tcutils.util import skip_because
 
-class TestNetworkPolicy(BaseK8sTest):
+import gevent
+from gevent import greenlet
 
+class TestNetworkPolicy(BaseK8sTest):
+
     @classmethod
     def setUpClass(cls):
         super(TestNetworkPolicy, cls).setUpClass()
@@ -115,24 +118,26 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
+        cleanup_list = list()
         if getattr(cls, 'web_pod_ns1', None):
-            cls.web_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.web_pod_ns1.cleanUp))
         if getattr(cls, 'web_pod_ns2', None):
-            cls.web_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.web_pod_ns2.cleanUp))
         if getattr(cls, 'client1_pod_ns1', None):
-            cls.client1_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client1_pod_ns1.cleanUp))
         if getattr(cls, 'client2_pod_ns1', None):
-            cls.client2_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client2_pod_ns1.cleanUp))
         if getattr(cls, 'client1_pod_ns2', None):
-            cls.client1_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client1_pod_ns2.cleanUp))
         if getattr(cls, 'client2_pod_ns2', None):
-            cls.client2_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client2_pod_ns2.cleanUp))
         if getattr(cls, 'client1_pod_ns3', None):
-            cls.client1_pod_ns3.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client1_pod_ns3.cleanUp))
         if getattr(cls, 'client2_pod_ns3', None):
-            cls.client2_pod_ns3.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client2_pod_ns3.cleanUp))
         if getattr(cls, 'client3_pod_ns3', None):
-            cls.client3_pod_ns3.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client3_pod_ns3.cleanUp))
+        gevent.joinall(cleanup_list)
         if getattr(cls, 'ns2', None):
             cls.ns2.cleanUp()
         if getattr(cls, 'ns3', None):
@@ -1967,7 +1972,11 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         super(TestNetworkPolicyNSIsolation, cls).tearDownClass()
-
+
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self):
         namespace1 = self.setup_namespace(name = get_random_name("ns1"))
         namespace2 = self.setup_namespace(name = get_random_name("ns2"))
@@ -2201,6 +2210,10 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         super(TestNetworkPolicyRandom, cls).tearDownClass()
+
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
 
     @test.attr(type=['k8s_sanity'])
     @preposttest_wrapper
@@ -2825,22 +2838,24 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
+        cleanup_list = list()
         if getattr(cls, 'web1_pod_ns1', None):
-            cls.web1_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.web1_pod_ns1.cleanUp))
         if getattr(cls, 'web2_pod_ns1', None):
-            cls.web2_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.web2_pod_ns1.cleanUp))
         if getattr(cls, 'web1_pod_ns2', None):
-            cls.web1_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.web1_pod_ns2.cleanUp))
         if getattr(cls, 'web2_pod_ns2', None):
-            cls.web2_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.web2_pod_ns2.cleanUp))
         if getattr(cls, 'client1_pod_ns1', None):
-            cls.client1_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client1_pod_ns1.cleanUp))
         if getattr(cls, 'client2_pod_ns1', None):
-            cls.client2_pod_ns1.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client2_pod_ns1.cleanUp))
         if getattr(cls, 'client1_pod_ns2', None):
-            cls.client1_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client1_pod_ns2.cleanUp))
         if getattr(cls, 'client2_pod_ns2', None):
-            cls.client2_pod_ns2.cleanUp()
+            cleanup_list.append(gevent.spawn(cls.client2_pod_ns2.cleanUp))
+        gevent.joinall(cleanup_list)
         if getattr(cls, 'ns2', None):
             cls.ns2.cleanUp()
         if getattr(cls, 'ns1', None):

scripts/k8s_scripts/test_service.py

@@ -18,6 +18,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestService, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['openshift_1'])
     @preposttest_wrapper
     def test_service_1(self):
@@ -207,6 +211,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestServiceExternalIP, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['k8s_sanity'])
     @skip_because(mx_gw = False)
     @preposttest_wrapper

scripts/k8s_scripts/test_snat.py

@@ -5,6 +5,7 @@
 import time
 
 class TestSNAT(BaseK8sTest):
+
     @classmethod
     def setUpClass(cls):
         super(TestSNAT, cls).setUpClass()
@@ -14,6 +15,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestSNAT, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_namespaces_pods_for_snat_test(self, isolation=False ,ip_fabric_snat=False):
         """ common routine to create the namesapces and the pods  by enabling snat
             1.create 2 namespaces (ns1,ns2 enable snat )

serial_scripts/k8s_scripts/test_deployment.py

@@ -15,6 +15,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestDeployment, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['k8s_sanity'])
     @preposttest_wrapper
     def test_deployment_with_kube_manager_restart(self):

serial_scripts/k8s_scripts/test_fabric_forwarding.py

@@ -14,6 +14,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestFabricFWDRestarts, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_namespaces_pods_for_fabric_restart(self, isolation=False,ip_fabric_forwarding=False):
         """ common routine to create the namesapces and the pods  by enabling the fabric forwarding
             1.create 2 namespaces (ns1,ns2:enable fabric forwarding)

serial_scripts/k8s_scripts/test_fabric_snat_restart_scenarios.py

@@ -20,6 +20,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestFabricSNATRestarts, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self, isolation=False, ip_fabric_snat=False,
                                      ip_fabric_forwarding=False):
         """ common routine to create the namesapces and the pods  by enabling the fabric snat

serial_scripts/k8s_scripts/test_ingress.py

@@ -16,6 +16,10 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         super(TestIngress, cls).tearDownClass()
+
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
 
     @test.attr(type=['k8s_sanity'])
     @skip_because(mx_gw = False)

serial_scripts/k8s_scripts/test_isolation.py

@@ -17,7 +17,11 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         super(TestNSIsolationSerial, cls).tearDownClass()
-
+
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self, prov_service = False, prov_ingress = False):
         service_ns1, ingress_ns1 = None, None
         service_ns2, ingress_ns2 = None, None
@@ -184,7 +188,11 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         super(TestCustomIsolationSerial, cls).tearDownClass()
-
+
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self, prov_service = False):
         service_ns1 = None
         service_ns2 = None
@@ -327,6 +335,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestProjectIsolationSerial, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     def setup_common_namespaces_pods(self, prov_service = False,
                                     prov_ingress = False,
                                     isolation = False):

serial_scripts/k8s_scripts/test_pod.py

@@ -13,6 +13,10 @@ def setUpClass(cls):
     def tearDownClass(cls):
         super(TestPodScale, cls).tearDownClass()
 
+    def parallel_cleanup(self):
+        parallelCleanupCandidates = ["PodFixture"]
+        self.delete_in_parallel(parallelCleanupCandidates)
+
     @test.attr(type=['openshift_1'])
     @preposttest_wrapper
     def test_many_pods(self):