Adds scenario for hotplug nic in network_basic_ops

New scenario test verifying add network to an active VM.

Test is added to network_basic_ops module as most of the code already exists
there.

Partially Implements: blueprint neutron-advanced-scenarios

Change-Id: I5946134de7038a3a7da236dac7018c211cbcce97

tempest/common/utils/linux/remote_client.py

@@ -93,3 +93,18 @@ def ping_host(self, host):
     def get_mac_address(self):
         cmd = "/sbin/ifconfig | awk '/HWaddr/ {print $5}'"
         return self.ssh_client.exec_command(cmd)
+
+    def get_ip_list(self):
+        cmd = "/bin/ip address"
+        return self.ssh_client.exec_command(cmd)
+
+    def assign_static_ip(self, nic, addr):
+        cmd = "sudo /bin/ip addr add {ip}/{mask} dev {nic}".format(
+            ip=addr, mask=CONF.network.tenant_network_mask_bits,
+            nic=nic
+        )
+        return self.ssh_client.exec_command(cmd)
+
+    def turn_nic_on(self, nic):
+        cmd = "sudo /bin/ip link set {nic} up".format(nic=nic)
+        return self.ssh_client.exec_command(cmd)

tempest/scenario/manager.py

@@ -490,7 +490,7 @@ def _get_tenant_own_port_num(self, tenant_id):
         ports = self._list_ports(tenant_id=tenant_id)
         return len(ports)
 
-    def _create_subnet(self, network, namestart='subnet-smoke-'):
+    def _create_subnet(self, network, namestart='subnet-smoke-', **kwargs):
         """
         Create a subnet for the given network within the cidr block
         configured for tenant networks.
@@ -523,6 +523,7 @@ def cidr_in_use(cidr, tenant_id):
                     cidr=str_cidr,
                 ),
             )
+            body['subnet'].update(kwargs)
             try:
                 result = self.network_client.create_subnet(body=body)
                 break
@@ -680,6 +681,28 @@ def _check_vm_connectivity(self, ip_address,
                                                   private_key)
             linux_client.validate_authentication()
 
+    def _check_remote_connectivity(self, source, dest, should_succeed=True):
+        """
+        check ping server via source ssh connection
+
+        :param source: RemoteClient: an ssh connection from which to ping
+        :param dest: and IP to ping against
+        :param should_succeed: boolean should ping succeed or not
+        :returns: boolean -- should_succeed == ping
+        :returns: ping is false if ping failed
+        """
+        def ping_remote():
+            try:
+                source.ping_host(dest)
+            except exceptions.SSHExecCommandFailed:
+                LOG.exception('Failed to ping host via ssh connection')
+                return not should_succeed
+            return should_succeed
+
+        return tempest.test.call_until_true(ping_remote,
+                                            CONF.compute.ping_timeout,
+                                            1)
+
     def _create_security_group_nova(self, client=None,
                                     namestart='secgroup-smoke-',
                                     tenant_id=None):

tempest/scenario/test_network_basic_ops.py

@@ -15,6 +15,9 @@
 #    under the License.
 import collections
 
+import re
+
+from tempest.api.network import common as net_common
 from tempest.common import debug
 from tempest.common.utils import data_utils
 from tempest import config
@@ -36,36 +39,6 @@ class TestNetworkBasicOps(manager.NetworkScenarioTest):
     boot VM's with Neutron-managed networking, and attempts to
     verify network connectivity as follows:
 
-     * For a freshly-booted VM with an IP address ("port") on a given network:
-
-       - the Tempest host can ping the IP address.  This implies, but
-         does not guarantee (see the ssh check that follows), that the
-         VM has been assigned the correct IP address and has
-         connectivity to the Tempest host.
-
-       - the Tempest host can perform key-based authentication to an
-         ssh server hosted at the IP address.  This check guarantees
-         that the IP address is associated with the target VM.
-
-       - detach the floating-ip from the VM and verify that it becomes
-       unreachable
-
-       - associate detached floating ip to a new VM and verify connectivity.
-       VMs are created with unique keypair so connectivity also asserts that
-       floating IP is associated with the new VM instead of the old one
-
-       # TODO(mnewby) - Need to implement the following:
-       - the Tempest host can ssh into the VM via the IP address and
-         successfully execute the following:
-
-         - ping an external IP address, implying external connectivity.
-
-         - ping an external hostname, implying that dns is correctly
-           configured.
-
-         - ping an internal IP address, implying connectivity to another
-           VM on the same network.
-
      There are presumed to be two types of networks: tenant and
      public.  A tenant network may or may not be reachable from the
      Tempest host.  A public network is assumed to be reachable from
@@ -250,9 +223,125 @@ def _reassociate_floating_ips(self):
         self.floating_ip_tuple = Floating_IP_tuple(
             floating_ip, serv_dict['server'])
 
+    def _create_new_network(self):
+        self.new_net = self._create_network(self.tenant_id)
+        self.addCleanup(self.cleanup_wrapper, self.new_net)
+        self.new_subnet = self._create_subnet(
+            network=self.new_net,
+            gateway_ip=None)
+        self.addCleanup(self.cleanup_wrapper, self.new_subnet)
+
+    def _hotplug_server(self):
+        old_floating_ip, server = self.floating_ip_tuple
+        ip_address = old_floating_ip.floating_ip_address
+        private_key = self.servers[server].private_key
+        ssh_client = self.get_remote_client(ip_address,
+                                            private_key=private_key)
+        old_nic_list = self._get_server_nics(ssh_client)
+        # get a port from a list of one item
+        port_list = self._list_ports(device_id=server.id)
+        self.assertEqual(1, len(port_list))
+        old_port = port_list[0]
+        self.compute_client.servers.interface_attach(server=server,
+                                                     net_id=self.new_net.id,
+                                                     port_id=None,
+                                                     fixed_ip=None)
+        # move server to the head of the cleanup list
+        self.addCleanup(self.cleanup_wrapper, server)
+
+        def check_ports():
+            port_list = [port for port in
+                         self._list_ports(device_id=server.id)
+                         if port != old_port]
+            return len(port_list) == 1
+
+        test.call_until_true(check_ports, 60, 1)
+        new_port_list = [p for p in
+                         self._list_ports(device_id=server.id)
+                         if p != old_port]
+        self.assertEqual(1, len(new_port_list))
+        new_port = new_port_list[0]
+        new_port = net_common.DeletablePort(client=self.network_client,
+                                            **new_port)
+        new_nic_list = self._get_server_nics(ssh_client)
+        diff_list = [n for n in new_nic_list if n not in old_nic_list]
+        self.assertEqual(1, len(diff_list))
+        num, new_nic = diff_list[0]
+        ssh_client.assign_static_ip(nic=new_nic,
+                                    addr=new_port.fixed_ips[0]['ip_address'])
+        ssh_client.turn_nic_on(nic=new_nic)
+
+    def _get_server_nics(self, ssh_client):
+        reg = re.compile(r'(?P<num>\d+): (?P<nic_name>\w+):')
+        ipatxt = ssh_client.get_ip_list()
+        return reg.findall(ipatxt)
+
+    def _check_network_internal_connectivity(self):
+        """
+        via ssh check VM internal connectivity:
+        - ping internal DHCP port, implying in-tenant connectivty
+        """
+        floating_ip, server = self.floating_ip_tuple
+        # get internal ports' ips:
+        # get all network ports in the new network
+        internal_ips = (p['fixed_ips'][0]['ip_address'] for p in
+                        self._list_ports(tenant_id=server.tenant_id,
+                                         network_id=self.new_net.id)
+                        if p['device_owner'].startswith('network'))
+
+        ip_address = floating_ip.floating_ip_address
+        private_key = self.servers[server].private_key
+        ssh_source = self._ssh_to_server(ip_address, private_key)
+
+        for remote_ip in internal_ips:
+            try:
+                self.assertTrue(self._check_remote_connectivity(ssh_source,
+                                                                remote_ip),
+                                "Timed out waiting for %s to become "
+                                "reachable" % remote_ip)
+            except Exception:
+                LOG.exception("Unable to access {dest} via ssh to "
+                              "floating-ip {src}".format(dest=remote_ip,
+                                                         src=floating_ip))
+                debug.log_ip_ns()
+                raise
+
     @test.attr(type='smoke')
     @test.services('compute', 'network')
     def test_network_basic_ops(self):
+        """
+        For a freshly-booted VM with an IP address ("port") on a given
+            network:
+
+        - the Tempest host can ping the IP address.  This implies, but
+         does not guarantee (see the ssh check that follows), that the
+         VM has been assigned the correct IP address and has
+         connectivity to the Tempest host.
+
+        - the Tempest host can perform key-based authentication to an
+         ssh server hosted at the IP address.  This check guarantees
+         that the IP address is associated with the target VM.
+
+        - detach the floating-ip from the VM and verify that it becomes
+        unreachable
+
+        - associate detached floating ip to a new VM and verify connectivity.
+        VMs are created with unique keypair so connectivity also asserts that
+        floating IP is associated with the new VM instead of the old one
+
+        # TODO(mnewby) - Need to implement the following:
+        - the Tempest host can ssh into the VM via the IP address and
+         successfully execute the following:
+
+         - ping an external IP address, implying external connectivity.
+
+         - ping an external hostname, implying that dns is correctly
+           configured.
+
+         - ping an internal IP address, implying connectivity to another
+           VM on the same network.
+
+        """
         self._check_public_network_connectivity(should_connect=True)
         self._disassociate_floating_ips()
         self._check_public_network_connectivity(should_connect=False,
@@ -262,3 +351,20 @@ def test_network_basic_ops(self):
         self._check_public_network_connectivity(should_connect=True,
                                                 msg="after re-associate "
                                                     "floating ip")
+
+    @test.attr(type='smoke')
+    @test.services('compute', 'network')
+    def test_hotplug_nic(self):
+        """
+        1. create a new network, with no gateway (to prevent overwriting VM's
+            gateway)
+        2. connect VM to new network
+        3. set static ip and bring new nic up
+        4. check VM can ping new network dhcp port
+
+        """
+
+        self._check_public_network_connectivity(should_connect=True)
+        self._create_new_network()
+        self._hotplug_server()
+        self._check_network_internal_connectivity()

tempest/scenario/test_security_groups_basic_ops.py

@@ -17,7 +17,6 @@
 from tempest.common import debug
 from tempest.common.utils import data_utils
 from tempest import config
-from tempest import exceptions
 from tempest.openstack.common import log as logging
 from tempest.scenario import manager
 from tempest import test
@@ -332,28 +331,6 @@ def _connect_to_access_point(self, tenant):
                                                private_key=private_key)
         return access_point_ssh
 
-    def _test_remote_connectivity(self, source, dest, should_succeed=True):
-        """
-        check ping server via source ssh connection
-
-        :param source: RemoteClient: an ssh connection from which to ping
-        :param dest: and IP to ping against
-        :param should_succeed: boolean should ping succeed or not
-        :returns: boolean -- should_succeed == ping
-        :returns: ping is false if ping failed
-        """
-        def ping_remote():
-            try:
-                source.ping_host(dest)
-            except exceptions.SSHExecCommandFailed as ex:
-                LOG.debug(ex)
-                return not should_succeed
-            return should_succeed
-
-        return test.call_until_true(ping_remote,
-                                    CONF.compute.ping_timeout,
-                                    1)
-
     def _check_connectivity(self, access_point, ip, should_succeed=True):
         if should_succeed:
             msg = "Timed out waiting for %s to become reachable" % ip
@@ -362,8 +339,8 @@ def _check_connectivity(self, access_point, ip, should_succeed=True):
             return True
             msg = "%s is reachable" % ip
         try:
-            self.assertTrue(self._test_remote_connectivity(access_point, ip,
-                                                           should_succeed),
+            self.assertTrue(self._check_remote_connectivity(access_point, ip,
+                                                            should_succeed),
                             msg)
         except Exception:
             debug.log_ip_ns()