hyperv: Retry after WMI query fails to find dev

During long stress runs the WMI query that is looking for
the iSCSI device number can incorrectly return no data.
If the query is retried the appropriate data can then be
obtained.

This commit adds a retry loop, calling
basevolumeutils.get_device_number_for_target to avoid this situation.
It also handles the case where the devices list returned in
get_device_number_for_target is empty.  The retry loop is
implemented with new mounted_disk_query_retry_count and
mounted_disk_query_retry_interval configuration options.

Unit tests have been added to check the good and bad paths for
get_mounted_disk_from_lun.

DocImpact
Closes-bug: 1247901
Change-Id: I082c4b1694efcd20cce65293cd330b7a0cf7d470

etc/nova/nova.conf.sample

@@ -2592,6 +2592,14 @@
 # (integer value)
 #volume_attach_retry_interval=5
 
+# The number of times to retry checking for a disk mounted via
+# iSCSI. (integer value)
+#mounted_disk_query_retry_count=10
+
+# Interval between checks for a mounted iSCSI disk, in
+# seconds. (integer value)
+#mounted_disk_query_retry_interval=5
+
 
 [image_file_url]
 

nova/tests/virt/hyperv/test_hypervapi.py

@@ -18,6 +18,7 @@
 Test suite for the Hyper-V driver and related APIs.
 """
 
+import contextlib
 import io
 import os
 import platform
@@ -60,6 +61,7 @@
 from nova.virt.hyperv import vhdutilsv2
 from nova.virt.hyperv import vmutils
 from nova.virt.hyperv import vmutilsv2
+from nova.virt.hyperv import volumeops
 from nova.virt.hyperv import volumeutils
 from nova.virt.hyperv import volumeutilsv2
 from nova.virt import images
@@ -1636,3 +1638,52 @@ def test_rollback_live_migration_at_destination(self):
                     self._test_spawn_instance, [], None)
             mock_destroy.assert_called_once_with(self._context,
                     self._test_spawn_instance, [], None)
+
+
+class VolumeOpsTestCase(HyperVAPITestCase):
+    """Unit tests for VolumeOps class."""
+
+    def setUp(self):
+        super(VolumeOpsTestCase, self).setUp()
+        self.volumeops = volumeops.VolumeOps()
+
+    def test_get_mounted_disk_from_lun(self):
+        with contextlib.nested(
+            mock.patch.object(self.volumeops._volutils,
+                              'get_device_number_for_target'),
+            mock.patch.object(self.volumeops._vmutils,
+                              'get_mounted_disk_by_drive_number')
+            ) as (mock_get_device_number_for_target,
+                  mock_get_mounted_disk_by_drive_number):
+
+            mock_get_device_number_for_target.return_value = 0
+            mock_get_mounted_disk_by_drive_number.return_value = 'disk_path'
+
+            block_device_info = db_fakes.get_fake_block_device_info(
+                self._volume_target_portal, self._volume_id)
+
+            mapping = driver.block_device_info_get_mapping(block_device_info)
+            data = mapping[0]['connection_info']['data']
+            target_lun = data['target_lun']
+            target_iqn = data['target_iqn']
+
+            disk = self.volumeops._get_mounted_disk_from_lun(target_iqn,
+                                                             target_lun)
+            self.assertEqual(disk, 'disk_path')
+
+    def test_get_mounted_disk_from_lun_failure(self):
+        with mock.patch.object(self.volumeops._volutils,
+                               'get_device_number_for_target') as m_device_num:
+            m_device_num.return_value = None
+
+            block_device_info = db_fakes.get_fake_block_device_info(
+                self._volume_target_portal, self._volume_id)
+
+            mapping = driver.block_device_info_get_mapping(block_device_info)
+            data = mapping[0]['connection_info']['data']
+            target_lun = data['target_lun']
+            target_iqn = data['target_iqn']
+
+            self.assertRaises(exception.NotFound,
+                              self.volumeops._get_mounted_disk_from_lun,
+                              target_iqn, target_lun)

nova/virt/hyperv/basevolumeutils.py

@@ -127,6 +127,10 @@ def get_device_number_for_target(self, target_iqn, target_lun):
             return None
 
         devices = initiator_sessions[0].Devices
+
+        if not devices:
+            return None
+
         for device in devices:
             if device.ScsiLun == target_lun:
                 return device.DeviceNumber

nova/virt/hyperv/volumeops.py

@@ -39,6 +39,14 @@
     cfg.IntOpt('volume_attach_retry_interval',
                default=5,
                help='Interval between volume attachment attempts, in seconds'),
+    cfg.IntOpt('mounted_disk_query_retry_count',
+               default=10,
+               help='The number of times to retry checking for a disk mounted '
+                    'via iSCSI.'),
+    cfg.IntOpt('mounted_disk_query_retry_interval',
+               default=5,
+               help='Interval between checks for a mounted iSCSI '
+                    'disk, in seconds.'),
 ]
 
 CONF = cfg.CONF
@@ -192,8 +200,21 @@ def get_volume_connector(self, instance):
 
     def _get_mounted_disk_from_lun(self, target_iqn, target_lun,
                                    wait_for_device=False):
-        device_number = self._volutils.get_device_number_for_target(target_iqn,
-                                                                    target_lun)
+        # The WMI query in get_device_number_for_target can incorrectly
+        # return no data when the system is under load.  This issue can
+        # be avoided by adding a retry.
+        for i in xrange(CONF.hyperv.mounted_disk_query_retry_count):
+            device_number = self._volutils.get_device_number_for_target(
+                            target_iqn, target_lun)
+            if device_number is None:
+                attempt = i + 1
+                LOG.debug(_('Attempt %d to get device_number '
+                          'from get_device_number_for_target failed. '
+                          'Retrying...') % attempt)
+                time.sleep(CONF.hyperv.mounted_disk_query_retry_interval)
+            else:
+                break
+
         if device_number is None:
             raise exception.NotFound(_('Unable to find a mounted disk for '
                                        'target_iqn: %s') % target_iqn)