Use of logging in native thread causes deadlock connecting to libvirtd

When connecting to libvirtd daemon we were calling the _connect
method in a native thread. The use of logging in this method hit this
bug in eventlet https://bitbucket.org/eventlet/eventlet/issue/137/
causing nova-compute to hang.

Closes-Bug: #1252409

Change-Id: I0d22c3ed1b3288aeb1afae130f11a4f38dc6440a

nova/tests/virt/libvirt/test_libvirt.py

@@ -7296,6 +7296,19 @@ def test_connection_to_primitive(self):
         connection.set_host_enabled = mock.Mock()
         jsonutils.to_primitive(connection._conn, convert_instances=True)
 
+    def test_tpool_execute_calls_libvirt(self):
+        self.mox.StubOutWithMock(eventlet.tpool, 'execute')
+        conn = libvirt.virConnect()
+        conn.is_expected = True
+        eventlet.tpool.execute(
+            libvirt.openAuth, 'test:///default',
+            mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(conn)
+        self.mox.ReplayAll()
+
+        driver = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        c = driver._get_connection()
+        self.assertEqual(True, c.is_expected)
+
 
 class LibvirtVolumeSnapshotTestCase(test.TestCase):
     """Tests for libvirtDriver.volume_snapshot_create/delete."""

nova/virt/libvirt/driver.py

@@ -627,13 +627,9 @@ def _get_new_connection(self):
         # call with _wrapped_conn_lock held
         LOG.debug(_('Connecting to libvirt: %s'), self.uri())
         wrapped_conn = None
+
         try:
-            if not CONF.libvirt.api_thread_pool:
-                wrapped_conn = self._connect(self.uri(), self.read_only)
-            else:
-                wrapped_conn = tpool.proxy_call(
-                        (libvirt.virDomain, libvirt.virConnect),
-                        self._connect, self.uri(), self.read_only)
+            wrapped_conn = self._connect(self.uri(), self.read_only)
         finally:
             # Enabling the compute service, in case it was disabled
             # since the connection was successful.
@@ -740,7 +736,15 @@ def _connect_auth_cb(creds, opaque):
             flags = 0
             if read_only:
                 flags = libvirt.VIR_CONNECT_RO
-            return libvirt.openAuth(uri, auth, flags)
+            if not CONF.libvirt.api_thread_pool:
+                return libvirt.openAuth(uri, auth, flags)
+            else:
+                # tpool.proxy_call creates a native thread. Due to limitations
+                # with eventlet locking we cannot use the logging API inside
+                # the called function.
+                return tpool.proxy_call(
+                    (libvirt.virDomain, libvirt.virConnect),
+                    libvirt.openAuth, uri, auth, flags)
         except libvirt.libvirtError as ex:
             LOG.exception(_("Connection to libvirt failed: %s"), ex)
             payload = dict(ip=LibvirtDriver.get_host_ip_addr(),