Fix how nc is called in qos test

We have already nc_listen method in base scenario tests class.
It was since now used only in port_forwarding tests but we can
reuse it in QoS tests also.

There was also problem with spawning ncat process, that sometimes,
without any clear reason for me, process wasn't spawned at all.
That caused failure of test.

So this patch adds new method ensure_nc_listen() which spawns ncat
process on remote host and checkes if process is really spawned. That
way we can avoid problems with not spawned ncat process.

This patch also makes "server" attribute to be optional in nc_listen
method. It is used only to log console output in case when ssh to the
server wouldn't work as expected. And if server is not given,
_log_console_output() method will list all servers which belongs to
tenant and log console for each of them.

Closes-Bug: #1868100

Change-Id: I54c9f041f2f971219c32005b3fa573c06f0110ef

neutron_tempest_plugin/common/utils.py

@@ -117,6 +117,14 @@ def kill_nc_process(ssh_client):
         pass
 
 
+def process_is_running(ssh_client, process_name):
+    try:
+        ssh_client.exec_command("pidof %s" % process_name)
+        return True
+    except exceptions.SSHExecCommandFailed:
+        return False
+
+
 def spawn_http_server(ssh_client, port, message):
     cmd = ("(echo -e 'HTTP/1.1 200 OK\r\n'; echo '%(msg)s') "
            "| sudo nc -lp %(port)d &" % {'msg': message, 'port': port})

neutron_tempest_plugin/scenario/base.py

@@ -31,6 +31,7 @@
 from neutron_tempest_plugin.common import ip as ip_utils
 from neutron_tempest_plugin.common import shell
 from neutron_tempest_plugin.common import ssh
+from neutron_tempest_plugin.common import utils
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin import exceptions
 from neutron_tempest_plugin.scenario import constants
@@ -53,16 +54,19 @@ def get_ncat_version(ssh_client=None):
     return distutils.version.StrictVersion(m.group(1) if m else '7.60')
 
 
-def get_ncat_server_cmd(port, protocol, msg):
+def get_ncat_server_cmd(port, protocol, msg=None):
     udp = ''
     if protocol.lower() == neutron_lib_constants.PROTO_NAME_UDP:
         udp = '-u'
     cmd = "nc %(udp)s -p %(port)s -lk " % {
         'udp': udp, 'port': port}
-    if CONF.neutron_plugin_options.default_image_is_advanced:
-        cmd += "-c 'echo %s' &" % msg
+    if msg:
+        if CONF.neutron_plugin_options.default_image_is_advanced:
+            cmd += "-c 'echo %s' &" % msg
+        else:
+            cmd += "-e echo %s &" % msg
     else:
-        cmd += "-e echo %s &" % msg
+        cmd += "< /dev/zero &"
     return cmd
 
 
@@ -468,7 +472,20 @@ def check_servers_hostnames(self, servers, log_errors=True):
                 self._log_console_output(servers)
             raise
 
-    def nc_listen(self, server, ssh_client, port, protocol, echo_msg):
+    def ensure_nc_listen(self, ssh_client, port, protocol, echo_msg=None,
+                         servers=None):
+        """Ensure that nc server listening on the given TCP/UDP port is up.
+
+        Listener is created always on remote host.
+        """
+        def spawn_and_check_process():
+            self.nc_listen(ssh_client, port, protocol, echo_msg, servers)
+            return utils.process_is_running(ssh_client, "nc")
+
+        utils.wait_until_true(spawn_and_check_process)
+
+    def nc_listen(self, ssh_client, port, protocol, echo_msg=None,
+                  servers=None):
         """Create nc server listening on the given TCP/UDP port.
 
         Listener is created always on remote host.
@@ -479,7 +496,7 @@ def nc_listen(self, server, ssh_client, port, protocol, echo_msg):
                 become_root=True)
         except lib_exc.SSHTimeout as ssh_e:
             LOG.debug(ssh_e)
-            self._log_console_output([server])
+            self._log_console_output(servers)
             raise
 
     def nc_client(self, ip_address, port, protocol):

neutron_tempest_plugin/scenario/test_port_forwardings.py

@@ -83,11 +83,11 @@ def _prepare_resources(self, num_servers, internal_tcp_port, protocol):
     def _test_udp_port_forwarding(self, servers):
 
         def _message_received(server, ssh_client, expected_msg):
-            self.nc_listen(server,
-                           ssh_client,
+            self.nc_listen(ssh_client,
                            server['port_forwarding_udp']['internal_port'],
                            constants.PROTO_NAME_UDP,
-                           expected_msg)
+                           expected_msg,
+                           [server])
             received_msg = self.nc_client(
                 self.fip['floating_ip_address'],
                 server['port_forwarding_udp']['external_port'],

neutron_tempest_plugin/scenario/test_qos.py

@@ -81,8 +81,7 @@ class QoSTestMixin(object):
 
     def _check_bw(self, ssh_client, host, port, expected_bw=LIMIT_BYTES_SEC):
         utils.kill_nc_process(ssh_client)
-        cmd = ("(nc -ll -p %d < /dev/zero > /dev/null &)" % port)
-        ssh_client.exec_command(cmd, timeout=5)
+        self.ensure_nc_listen(ssh_client, port, "tcp")
 
         # Open TCP socket to remote VM and download big file
         start_time = time.time()