Address test_zmq setup/teardown races

Each test now consumes a new port number (9500+),
and socket directory based on an auto-incrementing
global counter (TESTCNT).

Also makes minor cleanup to test receiver setup logic.

fixes bug 1096223

Change-Id: Ie6a88c535c81f2fc12b97d32c2674a6184325d2a

tests/unit/rpc/test_zmq.py

@@ -44,10 +44,14 @@
 
 
 class _RpcZmqBaseTestCase(common.BaseRpcTestCase):
-    @testutils.skip_if(True, "Zmq tests broken on jenkins")
+    # TESTCNT needs to be a class var as each run
+    # by subclasses must have a unique identifier
+    TESTCNT = 0
+
+    @testutils.skip_if(not impl_zmq, "ZeroMQ library required")
     def setUp(self, topic='test', topic_nested='nested'):
-        if not impl_zmq:
-            return None
+        _RpcZmqBaseTestCase.TESTCNT += 1
+        testcnt = _RpcZmqBaseTestCase.TESTCNT
 
         self.reactor = None
         self.rpc = impl_zmq
@@ -60,21 +64,19 @@ def setUp(self, topic='test', topic_nested='nested'):
         # We'll change this if we detect no daemon running.
         ipc_dir = FLAGS.rpc_zmq_ipc_dir
 
-        # Our IPC dir, if no nova-rpc-zmq_receiver is running system-wide.
-        internal_ipc_dir = '/tmp/openstack-zmq.ipc.test'
-
         # Only launch the router if it isn't running.
         if not os.path.exists(os.path.join(ipc_dir, "zmq_topic_zmq_replies")):
-            LOG.info(_("Running internal zmq receiver."))
-            # The normal ipc_dir default needs to run as root,
-            # /tmp is easier within a testing environment.
+            # NOTE(ewindisch): rpc_zmq_port and internal_ipc_dir must
+            #                  increment to avoid async socket
+            #                  closing/wait delays causing races
+            #                  between tearDown() and setUp()
+            self.config(rpc_zmq_port=9500 + testcnt)
+            internal_ipc_dir = "/tmp/openstack-zmq.ipc.test.%s" % testcnt
             self.config(rpc_zmq_ipc_dir=internal_ipc_dir)
 
-            # Value has changed.
-            ipc_dir = FLAGS.rpc_zmq_ipc_dir
-
-            self.setup_receiver(ipc_dir)
-        elif ipc_dir != internal_ipc_dir:
+            LOG.info(_("Running internal zmq receiver."))
+            self.setup_receiver(internal_ipc_dir)
+        else:
             LOG.warning(_("Detected zmq-receiver socket."))
             LOG.warning(_("Assuming nova-rpc-zmq-receiver is running."))
             LOG.warning(_("Using system zmq receiver deamon."))
@@ -108,9 +110,8 @@ def setup_receiver(self, ipc_dir):
                         "Socket may already be in use."))
             raise
 
+    @testutils.skip_if(not impl_zmq, "ZeroMQ library required")
     def tearDown(self):
-        if not impl_zmq:
-            return None
         if self.reactor:
             self.reactor.close()