[DM]: Handle gracefully when two successive BGP unlink/delete PR and …

…create PR happens

    Problem:

       T0: DM receives delete "bgp router" link removal request. As a result of this, DM tries to delete config from device in a separate Device Greenlet. Thi

       T1: Immediately, DM receives another request “physical-router” delete, in the context of PR object delete, DM deletes the config from device (this happ

       T2: Device Greenlet fails to delete config from Device (since config was already deleted (T1)), and hence goes into “RETRY” mode.

       T3: DM receives new PR create/BGP router update events. ==> New PR Object gets created locally, and this will create a new Device Greenlet and ultimate

       T4: When timer expires, Old Device Greenlet retries to delete the config, and this time delete will be successful. Config will be gone from Device.

       Solution:
           Terminate Pending Device Greenlet in the context of PR delete

Change-Id: I0d68d585230c503ed81650190d6e7aa4ba594875
Closes-Bug: #1714004

src/config/device-manager/device_manager/db.py

@@ -169,6 +169,8 @@ def delete(cls, uuid):
         if uuid not in cls._dict:
             return
         obj = cls._dict[uuid]
+        if obj.nc_handler_gl:
+            gevent.kill(obj.nc_handler_gl)
         if obj.is_vnc_managed() and obj.is_conf_sent():
             obj.config_manager.push_conf(is_delete=True)
             obj.config_manager.clear()
@@ -185,7 +187,8 @@ def delete(cls, uuid):
     @classmethod
     def reset(cls):
         for obj in cls._dict.values():
-            obj.config_manager.clear()
+            if obj.config_manager:
+                obj.config_manager.clear()
         cls._dict = {}
     # end reset
 

src/config/device-manager/test/test_case.py

@@ -64,32 +64,35 @@ def _get_ip_fabric_ri_obj(self):
         return rt_inst_obj
     # end _get_ip_fabric_ri_obj
 
-    def create_router(self, name, mgmt_ip, vendor='juniper', product='mx', ignore_pr=False, role=None):
-        bgp_router = BgpRouter(name, parent_obj=self._get_ip_fabric_ri_obj())
-        params = BgpRouterParams()
-        params.address = mgmt_ip
-        params.identifier = '1.1.1.1'
-        params.address_families = AddressFamilies(['route-target', 'inet-vpn', 'e-vpn',
+    def create_router(self, name, mgmt_ip, vendor='juniper', product='mx', ignore_pr=False, role=None, ignore_bgp=False):
+        bgp_router, pr = None, None
+        if not ignore_bgp:
+            bgp_router = BgpRouter(name, parent_obj=self._get_ip_fabric_ri_obj())
+            params = BgpRouterParams()
+            params.address = mgmt_ip
+            params.identifier = '1.1.1.1'
+            params.address_families = AddressFamilies(['route-target', 'inet-vpn', 'e-vpn',
                                              'inet6-vpn'])
-        params.autonomous_system = randint(0, 64512)
-        bgp_router.set_bgp_router_parameters(params)
-        self._vnc_lib.bgp_router_create(bgp_router)
+            params.autonomous_system = randint(0, 64512)
+            bgp_router.set_bgp_router_parameters(params)
+            self._vnc_lib.bgp_router_create(bgp_router)
 
-        if ignore_pr:
-            return bgp_router, None
-        pr = PhysicalRouter(name)
-        pr.physical_router_management_ip = mgmt_ip
-        pr.physical_router_vendor_name = vendor
-        pr.physical_router_product_name = product
-        pr.physical_router_vnc_managed = True
-        if role:
-            pr.physical_router_role = role
-        uc = UserCredentials('user', 'pw')
-        pr.set_physical_router_user_credentials(uc)
-        pr.set_bgp_router(bgp_router)
-        pr_id = self._vnc_lib.physical_router_create(pr)
+        if not ignore_pr:
+            pr = PhysicalRouter(name)
+            pr.physical_router_management_ip = mgmt_ip
+            pr.physical_router_vendor_name = vendor
+            pr.physical_router_product_name = product
+            pr.physical_router_vnc_managed = True
+            if role:
+                pr.physical_router_role = role
+            uc = UserCredentials('user', 'pw')
+            pr.set_physical_router_user_credentials(uc)
+            if not ignore_bgp:
+                pr.set_bgp_router(bgp_router)
+            self._vnc_lib.physical_router_create(pr)
 
         return bgp_router, pr
+    # end create_router
 
     def delete_routers(self, bgp_router=None, pr=None):
         if pr:

src/config/device-manager/test/test_dm_infra.py

@@ -51,6 +51,30 @@ def test_dm_xml_generation(self):
         self.delete_routers(bgp_router, pr)
         self.wait_for_routers_delete(bgp_router_fq, pr_fq)
 
+    # check for greenlets, bug: #1714004
+    def test_dm_greenlets(self):
+        bgp_router, pr = self.create_router('router1' + self.id(), '1.1.1.1',
+                                    product=self.product)
+        self.check_if_xml_is_generated()
+        bgp_router_fq = bgp_router.get_fq_name()
+        pr_fq = pr.get_fq_name()
+        pr.del_bgp_router(bgp_router)
+        # delete bgp link, update and immediately delete PR
+        self._vnc_lib.physical_router_update(pr)
+        self._vnc_lib.physical_router_delete(fq_name=pr_fq)
+        # create again PR with same FQ/ip and link it with BGP
+        _, pr = self.create_router('router1' + self.id(), '1.1.1.1',
+                                   product=self.product, ignore_bgp=True)
+        pr.set_bgp_router(bgp_router)
+        gevent.sleep(5)
+        self._vnc_lib.physical_router_update(pr)
+
+        # netconf push should happen as expected
+        self.check_if_xml_is_generated()
+        self.delete_routers(bgp_router, pr)
+        self.wait_for_routers_delete(bgp_router_fq, pr_fq)
+    # end test_dm_greenlets
+
     # no crash if bgp router paramters are not configured
     def test_dm_no_bgp_params(self):
         bgp_router, pr = self.create_router('router1' + self.id(), '1.1.1.1',