Add relevant infrastructure for performing NDR-related data plane tests and actual testing

zaza/openstack/charm_tests/dragent/configure.py

@@ -18,46 +18,15 @@
 
 import logging
 import zaza.model
-from zaza.openstack.configure import (
-    network,
-    bgp_speaker,
-)
+from zaza.openstack.configure import bgp_speaker
 from zaza.openstack.utilities import (
-    cli as cli_utils,
     generic as generic_utils,
     openstack as openstack_utils,
 )
+from zaza.openstack.charm_tests.neutron.setup import basic_overcloud_network
 
 DEFAULT_PEER_APPLICATION_NAME = "osci-frr"
 
-# The overcloud network configuration settings are declared.
-# These are the network configuration settings under test.
-OVERCLOUD_NETWORK_CONFIG = {
-    "network_type": "gre",
-    "router_name": openstack_utils.PROVIDER_ROUTER,
-    "ip_version": "4",
-    "address_scope": "public",
-    "external_net_name": openstack_utils.EXT_NET,
-    "external_subnet_name": openstack_utils.EXT_NET_SUBNET,
-    "prefix_len": "24",
-    "subnetpool_name": "pooled_subnets",
-    "subnetpool_prefix": "192.168.0.0/16",
-}
-
-# The undercloud network configuration settings are substrate specific to
-# the environment where the tests are being executed. These settings may be
-# overridden by environment variables. See the doc string documentation for
-# zaza.openstack.utilities.generic_utils.get_undercloud_env_vars for the
-# environment variables required to be exported and available to zaza.
-# These are default settings provided as an example.
-DEFAULT_UNDERCLOUD_NETWORK_CONFIG = {
-    "start_floating_ip": "10.5.150.0",
-    "end_floating_ip": "10.5.150.254",
-    "external_dns": "10.5.0.2",
-    "external_net_cidr": "10.5.0.0/16",
-    "default_gateway": "10.5.0.1",
-}
-
 
 def setup():
     """Run setup for BGP networking.
@@ -72,23 +41,13 @@ def setup():
     :returns: None
     :rtype: None
     """
-    cli_utils.setup_logging()
+    # Reuse the existing network configuration code but ask for a separate
+    # service subnet to be created for FIPs.
+    basic_overcloud_network(use_separate_fip_subnet=True)
 
-    # Get network configuration settings
-    network_config = {}
-    # Declared overcloud settings
-    network_config.update(OVERCLOUD_NETWORK_CONFIG)
-    # Default undercloud settings
-    network_config.update(DEFAULT_UNDERCLOUD_NETWORK_CONFIG)
-    # Environment specific settings
-    network_config.update(generic_utils.get_undercloud_env_vars())
-
-    # Get keystone session
+    # Get a keystone session
     keystone_session = openstack_utils.get_overcloud_keystone_session()
 
-    # Confugre the overcloud network
-    network.setup_sdn(network_config, keystone_session=keystone_session)
-
     # LP Bugs #1784083 and #1841459, require a late restart of the
     # neutron-bgp-dragent service
     logging.warning("Due to LP Bugs #1784083 and #1841459, we require a late "

zaza/openstack/charm_tests/dragent/tests.py

@@ -16,25 +16,124 @@
 
 """Define class of BGP tests."""
 
+import logging
+import tenacity
 import unittest
+import zaza.openstack.charm_tests.neutron.tests as neutron_tests
 
-from zaza.openstack.utilities import cli as cli_utils
-from zaza.openstack.charm_tests.dragent import test
+from zaza import model
+from zaza.openstack.utilities import (
+    cli as cli_utils,
+    juju as juju_utils,
+)
+from zaza.openstack.configure.bgp_speaker import NDR_TEST_FIP
 
 
-class DRAgentTest(unittest.TestCase):
+class DRAgentTest(neutron_tests.NeutronNetworkingBase):
     """Class to encapsulate BPG tests."""
 
     BGP_PEER_APPLICATION = 'osci-frr'
 
+    def setUp(self):
+        """Run setup actions specific to the class."""
+        super().setUp()
+        self._peer_unit = model.get_units(
+            self.BGP_PEER_APPLICATION)[0].entity_id
+
     @classmethod
     def setUpClass(cls):
-        """Run setup for BGP tests."""
+        """Run setup actions specific to the class."""
+        super().setUpClass()
         cli_utils.setup_logging()
 
+    @staticmethod
+    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
+                    reraise=True, stop=tenacity.stop_after_attempt(10))
+    def _assert_cidr_in_peer_routing_table(peer_unit, cidr):
+        logging.debug("Checking for {} on BGP peer {}"
+                      .format(cidr, peer_unit))
+        # Run show ip route bgp on BGP peer
+        routes = juju_utils.remote_run(
+            peer_unit, remote_cmd='vtysh -c "show ip route bgp"')
+        logging.info(routes)
+        assert cidr in routes, (
+            "CIDR, {}, not found in BGP peer's routing table: {}"
+            .format(cidr, routes))
+
+    @staticmethod
+    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
+                    reraise=True, stop=tenacity.stop_after_attempt(10))
+    def _assert_ip_reachable_via_peer(peer_unit, address):
+        logging.debug(f"Checking if {peer_unit} can reach {address} using "
+                      f"routes adverised by NDR")
+        # Ping with -w specified will return an exit code if there is no
+        # response after the number of seconds specified. This is to ignore the
+        # first ping that may not arrive due to an ARP resolution.
+        juju_utils.remote_run(peer_unit, fatal=True,
+                              remote_cmd=f'ping -w4 {address}')
+
     def test_bgp_routes(self):
-        """Run bgp tests."""
-        test.test_bgp_routes(peer_application_name=self.BGP_PEER_APPLICATION)
+        """Test BGP routes.
+
+        A test that checks only the control plane of Neutron Dynamic Routing.
+
+        :raises: AssertionError if expected BGP routes are not found
+        :returns: None
+        :rtype: None
+        """
+        # Get expected advertised routes
+        private_cidr = self.project_subnet['cidr']
+        floating_ip_cidr = "{}/32".format(
+            self.neutron_client.list_floatingips(name=NDR_TEST_FIP)
+            ["floatingips"][0]["floating_ip_address"])
+
+        # This test may run immediately after configuration.
+        # It may take time for routes to propagate via BGP. Do a
+        # binary backoff.
+        self._assert_cidr_in_peer_routing_table(self._peer_unit, private_cidr)
+        logging.info("Private subnet CIDR, {}, found in routing table"
+                     .format(private_cidr))
+        self._assert_cidr_in_peer_routing_table(self._peer_unit,
+                                                floating_ip_cidr)
+        logging.info("Floating IP CIDR, {}, found in routing table"
+                     .format(floating_ip_cidr))
+
+    def test_instance_connectivity(self):
+        """Test connectivity to instances via dynamic routes.
+
+        Make sure that with routes advertised via NDR it is actually possible
+        to reach instances from a unit that gets those routes programmed into
+        its routing table.
+        """
+        # Get an instance but do not perform connectivity checks as the machine
+        # running those tests does not have the dynamic routes advertised to
+        # the peer unit by NDR.
+        fip_instance = self.launch_guest('fip-instance', instance_key='jammy',
+                                         perform_connectivity_check=False)
+
+        @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
+                        reraise=True, stop=tenacity.stop_after_attempt(10))
+        def get_fip(nova_client, instance_id):
+            """Try to get a FIP from an instance.
+
+            Instance FIPs may not be immediately accessible from the Nova
+            object after the instance creation so a retry logic is necessary.
+            """
+            # The reason for looking up an instance object again is that the
+            # Nova client does not refresh address information after the
+            # initial retrieval.
+            instance = nova_client.servers.find(id=instance_id)
+            fips = neutron_tests.floating_ips_from_instance(instance)
+            if not fips:
+                raise tenacity.TryAgain
+            return fips[0]
+
+        fip = get_fip(self.nova_client, fip_instance.id)
+
+        # First check that the FIP is present in the peer unit's routing table.
+        self._assert_cidr_in_peer_routing_table(self._peer_unit, f'{fip}/32')
+        # Once it is, check if it is actually reachable.
+        self._assert_ip_reachable_via_peer(self._peer_unit, fip)
 
 
 if __name__ == "__main__":

zaza/openstack/charm_tests/neutron/setup.py

@@ -68,6 +68,17 @@
     "default_gateway": "10.5.0.1",
 }
 
+# For Neutron Dynamic Tests it is useful to avoid relying on the directly
+# connected routes and instead using the advertised routes on the southbound
+# path and default routes on the northbound path. To do that, a separate
+# service subnet may be optionally created to force Neutron to use that instead
+# of the external network subnet without concrete service IPs which is used as
+# a fallback only.
+DEFAULT_FIP_SERVICE_SUBNET_CONFIG = {
+    "fip_service_subnet_name": openstack_utils.FIP_SERVICE_SUBNET_NAME,
+    "fip_service_subnet_cidr": "100.64.0.0/24"
+}
+
 
 def undercloud_and_charm_setup(limit_gws=None):
     """Perform undercloud and charm setup for network plumbing.
@@ -111,14 +122,18 @@ def undercloud_and_charm_setup(limit_gws=None):
                         .format(provider_type))
 
 
-def basic_overcloud_network(limit_gws=None):
+def basic_overcloud_network(limit_gws=None, use_separate_fip_subnet=False):
     """Run setup for neutron networking.
 
     Configure the following:
         The overcloud network using subnet pools
 
     :param limit_gws: Limit the number of gateways that get a port attached
     :type limit_gws: int
+    :param use_separate_fip_subnet: Use a separate service subnet for floating
+                                    ips instead of relying on the external
+                                    network subnet for FIP allocations.
+    :type use_separate_fip_subnet: bool
     """
     cli_utils.setup_logging()
 
@@ -128,6 +143,10 @@ def basic_overcloud_network(limit_gws=None):
     network_config.update(OVERCLOUD_NETWORK_CONFIG)
     # Default undercloud settings
     network_config.update(DEFAULT_UNDERCLOUD_NETWORK_CONFIG)
+
+    if use_separate_fip_subnet:
+        network_config.update(DEFAULT_FIP_SERVICE_SUBNET_CONFIG)
+
     # Environment specific settings
     network_config.update(generic_utils.get_undercloud_env_vars())
 

zaza/openstack/charm_tests/test_utils.py

@@ -796,7 +796,7 @@ def resource_cleanup(self):
     def launch_guest(self, guest_name, userdata=None, use_boot_volume=False,
                      instance_key=None, flavor_name=None,
                      attach_to_external_network=False,
-                     keystone_session=None):
+                     keystone_session=None, perform_connectivity_check=True):
         """Launch one guest to use in tests.
 
         Note that it is up to the caller to have set the RESOURCE_PREFIX class
@@ -817,6 +817,9 @@ def launch_guest(self, guest_name, userdata=None, use_boot_volume=False,
                                            network.
         :type attach_to_external_network: bool
         :param keystone_session: Keystone session to use.
+        :param perform_connectivity_check: Whether to perform a connectivity
+                                           check.
+        :type perform_connectivity_check: bool
         :type keystone_session: Optional[keystoneauth1.session.Session]
         :returns: Nova instance objects
         :rtype: Server
@@ -848,7 +851,9 @@ def launch_guest(self, guest_name, userdata=None, use_boot_volume=False,
                     userdata=userdata,
                     flavor_name=flavor_name,
                     attach_to_external_network=attach_to_external_network,
-                    keystone_session=keystone_session)
+                    keystone_session=keystone_session,
+                    perform_connectivity_check=perform_connectivity_check
+                )
 
     def launch_guests(self, userdata=None, attach_to_external_network=False,
                       flavor_name=None):