Handling new portgroups when CVM is running

Change-Id: Ibfb663a2a107bd0c20f80e66c8cc6d5b9540e598
Closes-Bug: #1778895

cvm/__main__.py

@@ -6,7 +6,6 @@
 
 import gevent
 import yaml
-from pysandesh.sandesh_base import Sandesh
 
 import cvm.constants as const
 from cvm.clients import (ESXiAPIClient, VCenterAPIClient, VNCAPIClient,
@@ -19,6 +18,7 @@
 from cvm.services import (VirtualMachineInterfaceService,
                           VirtualMachineService, VirtualNetworkService,
                           VRouterPortService)
+from pysandesh.sandesh_base import Sandesh
 
 gevent.monkey.patch_all()
 
@@ -67,10 +67,12 @@ def build_monitor(config_file, database):
         database=database
     )
     vm_renamed_handler = VmRenamedHandler(vm_service, vmi_service, vrouter_port_service)
-    vm_reconfigured_handler = VmReconfiguredHandler(vm_service, vmi_service, vrouter_port_service)
+    vm_reconfigured_handler = VmReconfiguredHandler(vm_service, vn_service,
+                                                    vmi_service, vrouter_port_service)
     vm_removed_handler = VmRemovedHandler(vm_service, vmi_service, vrouter_port_service)
     handlers = [vm_renamed_handler, vm_reconfigured_handler, vm_removed_handler]
-    vmware_controller = VmwareController(vm_service, vn_service, vmi_service, vrouter_port_service, handlers)
+    vmware_controller = VmwareController(vm_service, vn_service,
+                                         vmi_service, vrouter_port_service, handlers)
     return VMwareMonitor(esxi_api_client, vmware_controller)
 
 

cvm/clients.py

@@ -172,9 +172,9 @@ def __enter__(self):
     def __exit__(self, *args):
         Disconnect(self._si)
 
-    def get_dpg_by_name(self, name):
+    def get_dpg_by_key(self, key):
         for dpg in self._datacenter.network:
-            if dpg.name == name and isinstance(dpg, vim.dvs.DistributedVirtualPortgroup):
+            if isinstance(dpg, vim.dvs.DistributedVirtualPortgroup) and dpg.key == key:
                 return dpg
         return None
 
@@ -300,12 +300,12 @@ def read_vmi(self, uuid):
             logger.error('Virtual Machine Interface not found %s', uuid)
             return None
 
-    def get_vns_by_project(self, project):
-        vns = self.vnc_lib.virtual_networks_list(parent_id=project.uuid).get('virtual-networks')
-        return [self._read_vn(vn['fq_name']) for vn in vns]
-
-    def _read_vn(self, fq_name):
-        return self.vnc_lib.virtual_network_read(fq_name)
+    def read_vn(self, fq_name):
+        try:
+            return self.vnc_lib.virtual_network_read(fq_name)
+        except NoIdError:
+            logger.error('Not found VN with fq_name: %s', str(fq_name))
+        return None
 
     def read_or_create_project(self):
         try:

cvm/controllers.py

@@ -1,5 +1,5 @@
-from abc import abstractmethod, ABCMeta
 import logging
+from abc import ABCMeta, abstractmethod
 
 from pyVmomi import vim, vmodl  # pylint: disable=no-name-in-module
 
@@ -17,8 +17,8 @@ def __init__(self, vm_service, vn_service, vmi_service, vrouter_port_service, ha
 
     def initialize_database(self):
         logger.info('Initializing database...')
-        self._vn_service.sync_vns()
         self._vm_service.get_vms_from_vmware()
+        self._vn_service.update_vns()
         self._vmi_service.sync_vmis()
         self._vm_service.delete_unused_vms_in_vnc()
         self._vrouter_port_service.sync_ports()
@@ -70,6 +70,7 @@ def _handle_vm_updated_event(self, event):
         vmware_vm = event.vm.vm
         try:
             self._vm_service.update(vmware_vm)
+            self._vn_service.update_vns()
             self._vmi_service.update_vmis()
             self._vrouter_port_service.sync_ports()
         except vmodl.fault.ManagedObjectNotFound:
@@ -129,8 +130,9 @@ def _handle_event(self, event):
 class VmReconfiguredHandler(AbstractEventHandler):
     EVENTS = (vim.event.VmReconfiguredEvent,)
 
-    def __init__(self, vm_service, vmi_service, vrouter_port_service):
+    def __init__(self, vm_service, vn_service, vmi_service, vrouter_port_service):
         self._vm_service = vm_service
+        self._vn_service = vn_service
         self._vmi_service = vmi_service
         self._vrouter_port_service = vrouter_port_service
 
@@ -142,6 +144,7 @@ def _handle_event(self, event):
             if isinstance(device, vim.vm.device.VirtualVmxnet3):
                 logger.info('Detected VmReconfiguredEvent with %s device', type(device))
                 self._vm_service.update_vm_models_interfaces(vmware_vm)
+                self._vn_service.update_vns()
                 self._vmi_service.update_vmis()
                 self._vrouter_port_service.sync_ports()
             else:

cvm/sandesh_handler.py

@@ -1,9 +1,12 @@
-from cvm.sandesh.vcenter_manager.ttypes import (
-    VirtualMachineData, VirtualMachineInterfaceData, VirtualNetworkData,
-    VirtualMachineRequest, VirtualMachineResponse,
-    VirtualNetworkRequest, VirtualNetworkResponse,
-    VirtualMachineInterfaceRequest, VirtualMachineInterfaceResponse
-)
+from cvm.sandesh.vcenter_manager.ttypes import (VirtualMachineData,
+                                                VirtualMachineInterfaceData,
+                                                VirtualMachineInterfaceRequest,
+                                                VirtualMachineInterfaceResponse,
+                                                VirtualMachineRequest,
+                                                VirtualMachineResponse,
+                                                VirtualNetworkData,
+                                                VirtualNetworkRequest,
+                                                VirtualNetworkResponse)
 
 
 class SandeshHandler(object):

cvm/services.py

@@ -2,7 +2,8 @@
 import logging
 
 from cvm.constants import (CONTRAIL_VM_NAME, VLAN_ID_RANGE_END,
-                           VLAN_ID_RANGE_START)
+                           VLAN_ID_RANGE_START, VNC_ROOT_DOMAIN,
+                           VNC_VCENTER_PROJECT)
 from cvm.models import (VirtualMachineInterfaceModel, VirtualMachineModel,
                         VirtualNetworkModel, VlanIdPool)
 
@@ -136,14 +137,25 @@ def __init__(self, vcenter_api_client, vnc_api_client, database):
         super(VirtualNetworkService, self).__init__(vnc_api_client, database)
         self._vcenter_api_client = vcenter_api_client
 
-    def sync_vns(self):
-        with self._vcenter_api_client:
-            for vn in self._vnc_api_client.get_vns_by_project(self._project):
-                dpg = self._vcenter_api_client.get_dpg_by_name(vn.name)
-                if vn and dpg:
-                    vn_model = VirtualNetworkModel(dpg, vn, VlanIdPool(VLAN_ID_RANGE_START, VLAN_ID_RANGE_END))
+    def update_vns(self):
+        for vmi_model in self._database.vmis_to_update:
+            portgroup_key = vmi_model.vcenter_port.portgroup_key
+            if self._database.get_vn_model_by_key(portgroup_key) is not None:
+                continue
+            logger.info('Fetching new portgroup for key: %s', portgroup_key)
+            with self._vcenter_api_client:
+                dpg = self._vcenter_api_client.get_dpg_by_key(portgroup_key)
+                fq_name = [VNC_ROOT_DOMAIN, VNC_VCENTER_PROJECT, dpg.name]
+                vnc_vn = self._vnc_api_client.read_vn(fq_name)
+                if dpg and vnc_vn:
+                    logger.info('Fetched new portgroup key: %s name: %s', dpg.key, vnc_vn.name)
+                    vn_model = VirtualNetworkModel(dpg, vnc_vn,
+                                                   VlanIdPool(VLAN_ID_RANGE_START, VLAN_ID_RANGE_END))
                     self._vcenter_api_client.enable_vlan_override(vn_model.vmware_vn)
                     self._database.save(vn_model)
+                    logger.info('Successfully saved new portgroup key: %s name: %s', dpg.key, vnc_vn.name)
+                else:
+                    logger.error('Unable to fetch new portgroup for key: %s', portgroup_key)
 
 
 class VirtualMachineInterfaceService(Service):

setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 
 def requirements(filename):

tests/test_database.py

@@ -3,8 +3,8 @@
 from mock import Mock
 
 from cvm.database import Database
-from cvm.models import (VirtualMachineInterfaceModel)
-from tests.utils import create_vn_model, create_vm_model
+from cvm.models import VirtualMachineInterfaceModel
+from tests.utils import create_vm_model, create_vn_model
 
 
 class TestFindVirtualMachineIpAddress(TestCase):

tests/test_events.py

@@ -222,9 +222,10 @@ def test_vm_created(vcenter_api_client, vn_model_1, vm_created_update,
     vrouter_api_client = Mock()
     database = Database()
     vm_service = VirtualMachineService(esxi_api_client, vnc_api_client, database)
+    vn_service = VirtualNetworkService(esxi_api_client, vnc_api_client, database)
     vmi_service = VirtualMachineInterfaceService(vcenter_api_client, vnc_api_client, database)
     vrouter_port_service = VRouterPortService(vrouter_api_client, database)
-    controller = VmwareController(vm_service, None, vmi_service, vrouter_port_service, [])
+    controller = VmwareController(vm_service, vn_service, vmi_service, vrouter_port_service, [])
 
     # Virtual Networks are already created for us and after synchronization,
     # their models are stored in our database
@@ -287,14 +288,15 @@ def test_vm_renamed(vcenter_api_client, vn_model_1, vm_created_update,
     vrouter_api_client = Mock()
     database = Database()
     vm_service = VirtualMachineService(esxi_api_client, vnc_api_client, database)
+    vn_service = VirtualNetworkService(esxi_api_client, vnc_api_client, database)
     vmi_service = VirtualMachineInterfaceService(
         vcenter_api_client,
         vnc_api_client,
         database
     )
     vrouter_port_service = VRouterPortService(vrouter_api_client, database)
     vm_renamed_handler = VmRenamedHandler(vm_service, vmi_service, vrouter_port_service)
-    controller = VmwareController(vm_service, None, vmi_service, vrouter_port_service, [vm_renamed_handler])
+    controller = VmwareController(vm_service, vn_service, vmi_service, vrouter_port_service, [vm_renamed_handler])
 
     # Virtual Networks are already created for us and after synchronization,
     # their models are stored in our database
@@ -368,7 +370,7 @@ def test_vm_reconfigured(vcenter_api_client, vn_model_1, vn_model_2, vm_created_
         database
     )
     vrouter_port_service = VRouterPortService(vrouter_api_client, database)
-    vm_reconfigure_handler = VmReconfiguredHandler(vm_service, vmi_service, vrouter_port_service)
+    vm_reconfigure_handler = VmReconfiguredHandler(vm_service, vn_service, vmi_service, vrouter_port_service)
     controller = VmwareController(vm_service, vn_service, vmi_service, vrouter_port_service, [vm_reconfigure_handler])
 
     # Virtual Networks are already created for us and after synchronization,
@@ -452,9 +454,10 @@ def test_vm_created_vlan_id(vcenter_api_client, vn_model_1, vm_created_update,
     vrouter_api_client = Mock()
     database = Database()
     vm_service = VirtualMachineService(esxi_api_client, vnc_api_client, database)
+    vn_service = VirtualNetworkService(esxi_api_client, vnc_api_client, database)
     vmi_service = VirtualMachineInterfaceService(vcenter_api_client, vnc_api_client, database)
     vrouter_port_service = VRouterPortService(vrouter_api_client, database)
-    controller = VmwareController(vm_service, None, vmi_service, vrouter_port_service, [])
+    controller = VmwareController(vm_service, vn_service, vmi_service, vrouter_port_service, [])
 
     # Virtual Networks are already created for us and after synchronization,
     # their models are stored in our database
@@ -492,10 +495,11 @@ def test_contrail_vm(vcenter_api_client, vm_created_update, esxi_api_client,
     vrouter_api_client = Mock()
     database = Database()
     vm_service = VirtualMachineService(esxi_api_client, vnc_api_client, database)
+    vn_service = VirtualNetworkService(esxi_api_client, vnc_api_client, database)
     vmi_service = VirtualMachineInterfaceService(vcenter_api_client, vnc_api_client, database)
     vrouter_port_service = VRouterPortService(vrouter_api_client, database)
     esxi_api_client.read_vm_properties.return_value = contrail_vm_properties
-    controller = VmwareController(vm_service, None, vmi_service, vrouter_port_service, [])
+    controller = VmwareController(vm_service, vn_service, vmi_service, vrouter_port_service, [])
 
     # A new update containing VmCreatedEvent arrives and is being handled by the controller
     controller.handle_update(vm_created_update)

tests/test_services.py

@@ -161,6 +161,50 @@ def test_rename_vm(self):
         self.database.save.assert_called_once_with(vm_model)
 
 
+class TestVirtualNetworkService(TestCase):
+
+    def setUp(self):
+        self.database = Database()
+        self.vnc_client = create_vnc_client_mock()
+        self.vcenter_client = create_vcenter_client_mock()
+        self.vn_service = VirtualNetworkService(self.vcenter_client,
+                                                self.vnc_client, self.database)
+
+    def test_update_vns_no_vns(self):
+        self.database.vmis_to_update = []
+
+        self.vn_service.update_vns()
+
+        assert len(self.database.vn_models) == 0
+        self.vcenter_client.get_dpg_by_key.assert_not_called()
+        self.vnc_client.read_vn.assert_not_called()
+
+    def test_update_vns(self):
+        vmi_model = Mock()
+        vmi_model.vcenter_port.portgroup_key = 'dvportgroup-62'
+        self.database.vmis_to_update = [vmi_model]
+
+        dpg_mock = create_dpg_mock(key='dvportgroup-62', name='network_name')
+        self.vcenter_client.get_dpg_by_key.return_value = dpg_mock
+
+        vnc_vn_mock = Mock()
+        self.vnc_client.read_vn.return_value = vnc_vn_mock
+
+        self.vn_service.update_vns()
+
+        vn_model = self.database.get_vn_model_by_key('dvportgroup-62')
+        assert vn_model is not None
+        assert vn_model.vnc_vn == vnc_vn_mock
+        assert vn_model.vmware_vn == dpg_mock
+        assert vn_model.key == 'dvportgroup-62'
+
+        self.vcenter_client.get_dpg_by_key.called_once_with('dvportgroup-62')
+        self.vcenter_client.enable_vlan_override.called_once_with(dpg_mock)
+
+        fq_name = [VNC_ROOT_DOMAIN, VNC_VCENTER_PROJECT, 'network_name']
+        self.vnc_client.read_vn.called_once_with(fq_name)
+
+
 class TestVirtualMachineInterfaceService(TestCase):
 
     def setUp(self):
@@ -318,38 +362,6 @@ def test_rename_vmis(self):
         self.assertIn(vmi_model, self.database.ports_to_update)
 
 
-class TestVirtualNetworkService(TestCase):
-
-    def setUp(self):
-        self.vcenter_api_client = create_vcenter_client_mock()
-        self.vnc_api_client = create_vnc_client_mock()
-        self.database = Database()
-        self.vn_service = VirtualNetworkService(self.vcenter_api_client,
-                                                self.vnc_api_client, self.database)
-
-    def test_sync_no_vns(self):
-        """ Syncing when there's no VNC VNs doesn't save anything to the database. """
-        self.vnc_api_client.get_all_vns.return_value = None
-
-        self.vn_service.sync_vns()
-
-        self.assertFalse(self.database.vn_models)
-
-    def test_sync_vns(self):
-        first_vnc_vn = vnc_api.VirtualNetwork('VM Portgroup')
-        second_vnc_vn = vnc_api.VirtualNetwork(VirtualNetworkModel.get_uuid('DPortgroup'))
-        self.vnc_api_client.get_vns_by_project.return_value = [first_vnc_vn, second_vnc_vn]
-
-        first_vmware_dpg = create_dpg_mock(name='VM Portgroup', key='dportgroup-50')
-        second_vmware_dpg = create_dpg_mock(name='DPortgroup', key='dportgroup-51')
-        self.vcenter_api_client.get_dpg_by_name.side_effect = [first_vmware_dpg, second_vmware_dpg]
-
-        self.vn_service.sync_vns()
-
-        self.assertEqual(first_vnc_vn, self.database.get_vn_model_by_key('dportgroup-50').vnc_vn)
-        self.assertEqual(second_vnc_vn, self.database.get_vn_model_by_key('dportgroup-51').vnc_vn)
-
-
 class TestVMIInstanceIp(TestCase):
     def setUp(self):
         self.instance_ip = Mock()