Netapp drivers support for pool-aware scheduling

Adds pools support for all NetApp drivers: eseries, 7mode (iscsi and nfs), and
cmode (iscsi and nfs). With 7mode and cmode drivers, a pool is one-to-one with a
Ontap flexvol. With eseries, a pool is one-to-one with a dynamic disk pool.

DocImpact
Implements: blueprint pool-aware-cinder-scheduler-support-in-netapp-drivers

Change-Id: Ie6f155df7bc1ae2cd5f7fa39f1b1a0ad38075988

cinder/scheduler/filters/capacity_filter.py

@@ -54,11 +54,17 @@ def host_passes(self, host_state, filter_properties):
             return True
         reserved = float(host_state.reserved_percentage) / 100
         free = math.floor(free_space * (1 - reserved))
+
+        msg_args = {"host": host_state.host,
+                    "requested": volume_size,
+                    "available": free}
         if free < volume_size:
             LOG.warning(_("Insufficient free space for volume creation "
-                          "(requested / avail): "
-                          "%(requested)s/%(available)s")
-                        % {'requested': volume_size,
-                           'available': free})
+                          "on host %(host)s (requested / avail): "
+                          "%(requested)s/%(available)s") % msg_args)
+        else:
+            LOG.debug("Sufficient free space for volume creation "
+                      "on host %(host)s (requested / avail): "
+                      "%(requested)s/%(available)s" % msg_args)
 
         return free >= volume_size

cinder/tests/test_netapp.py

@@ -495,7 +495,7 @@ class NetAppDirectCmodeISCSIDriverTestCase(test.TestCase):
               'os_type': 'linux', 'provider_location': 'lun1',
               'id': 'lun1', 'provider_auth': None, 'project_id': 'project',
               'display_name': None, 'display_description': 'lun1',
-              'volume_type_id': None}
+              'volume_type_id': None, 'host': 'hostname@backend#vol1'}
     snapshot = {'name': 'snapshot1', 'size': 2, 'volume_name': 'lun1',
                 'volume_size': 2, 'project_id': 'project',
                 'display_name': None, 'display_description': 'lun1',
@@ -524,7 +524,7 @@ class NetAppDirectCmodeISCSIDriverTestCase(test.TestCase):
                 'os_type': 'linux', 'provider_location': 'lun1',
                 'id': 'lun1', 'provider_auth': None, 'project_id': 'project',
                 'display_name': None, 'display_description': 'lun1',
-                'volume_type_id': None}
+                'volume_type_id': None, 'host': 'hostname@backend#vol1'}
     vol1 = ssc_utils.NetAppVolume('lun1', 'openstack')
     vol1.state['vserver_root'] = False
     vol1.state['status'] = 'online'
@@ -623,12 +623,11 @@ def test_map_by_creating_igroup(self):
         if not properties:
             raise AssertionError('Target portal is none')
 
-    def test_fail_create_vol(self):
-        self.assertRaises(exception.VolumeBackendAPIException,
-                          self.driver.create_volume, self.vol_fail)
-
     def test_vol_stats(self):
         self.driver.get_volume_stats(refresh=True)
+        stats = self.driver._stats
+        self.assertEqual(stats['vendor_name'], 'NetApp')
+        self.assertTrue(stats['pools'][0]['pool_name'])
 
     def test_create_vol_snapshot_diff_size_resize(self):
         self.driver.create_volume(self.volume)
@@ -1133,6 +1132,7 @@ def _custom_setup(self):
         client = driver.client
         client.set_api_version(1, 9)
         self.driver = driver
+        self.driver.root_volume_name = 'root'
 
     def _set_config(self, configuration):
         configuration.netapp_storage_family = 'ontap_7mode'
@@ -1150,19 +1150,6 @@ def test_create_on_select_vol(self):
         self.driver.delete_volume(self.volume)
         self.driver.volume_list = []
 
-    def test_create_fail_on_select_vol(self):
-        self.driver.volume_list = ['vol2', 'vol3']
-        success = False
-        try:
-            self.driver.create_volume(self.volume)
-        except exception.VolumeBackendAPIException:
-            success = True
-            pass
-        finally:
-            self.driver.volume_list = []
-        if not success:
-            raise AssertionError('Failed creating on selected volumes')
-
     def test_check_for_setup_error_version(self):
         drv = self.driver
         delattr(drv.client, '_api_version')
@@ -1195,6 +1182,7 @@ def _custom_setup(self):
         client = driver.client
         client.set_api_version(1, 9)
         self.driver = driver
+        self.driver.root_volume_name = 'root'
 
     def _set_config(self, configuration):
         configuration.netapp_storage_family = 'ontap_7mode'

cinder/tests/test_netapp_eseries_iscsi.py

@@ -16,6 +16,7 @@
 Tests for NetApp e-series iscsi volume driver.
 """
 
+import copy
 import json
 import re
 
@@ -27,8 +28,12 @@
 from cinder import test
 from cinder.volume import configuration as conf
 from cinder.volume.drivers.netapp import common
+from cinder.volume.drivers.netapp.eseries import client
+from cinder.volume.drivers.netapp.eseries import iscsi
+from cinder.volume.drivers.netapp.eseries.iscsi import LOG as driver_log
 from cinder.volume.drivers.netapp.options import netapp_basicauth_opts
 from cinder.volume.drivers.netapp.options import netapp_eseries_opts
+import cinder.volume.drivers.netapp.utils as na_utils
 
 
 LOG = logging.getLogger(__name__)
@@ -562,7 +567,7 @@ class NetAppEseriesIscsiDriverTestCase(test.TestCase):
     """Test case for NetApp e-series iscsi driver."""
 
     volume = {'id': '114774fb-e15a-4fae-8ee2-c9723e3645ef', 'size': 1,
-              'volume_name': 'lun1',
+              'volume_name': 'lun1', 'host': 'hostname@backend#DDP',
               'os_type': 'linux', 'provider_location': 'lun1',
               'id': '114774fb-e15a-4fae-8ee2-c9723e3645ef',
               'provider_auth': 'provider a b', 'project_id': 'project',
@@ -597,7 +602,10 @@ class NetAppEseriesIscsiDriverTestCase(test.TestCase):
                           'project_id': 'project', 'display_name': None,
                           'display_description': 'lun1',
                           'volume_type_id': None}
+    fake_eseries_volume_label = na_utils.convert_uuid_to_es_fmt(volume['id'])
     connector = {'initiator': 'iqn.1998-01.com.vmware:localhost-28a58148'}
+    fake_size_gb = volume['size']
+    fake_eseries_pool_label = 'DDP'
 
     def setUp(self):
         super(NetAppEseriesIscsiDriverTestCase, self).setUp()
@@ -745,3 +753,84 @@ def test_create_vol_snapshot_diff_size_subclone(self):
             self.volume_clone_large, self.snapshot)
         self.driver.delete_snapshot(self.snapshot)
         self.driver.delete_volume(self.volume)
+
+    @mock.patch.object(iscsi.Driver, '_get_volume',
+                       mock.Mock(return_value={'volumeGroupRef': 'fake_ref'}))
+    def test_get_pool(self):
+        self.driver._objects['pools'] = [{'volumeGroupRef': 'fake_ref',
+                                          'label': 'ddp1'}]
+        pool = self.driver.get_pool({'id': 'fake-uuid'})
+        self.assertEqual(pool, 'ddp1')
+
+    @mock.patch.object(iscsi.Driver, '_get_volume',
+                       mock.Mock(return_value={'volumeGroupRef': 'fake_ref'}))
+    def test_get_pool_no_pools(self):
+        self.driver._objects['pools'] = []
+        pool = self.driver.get_pool({'id': 'fake-uuid'})
+        self.assertEqual(pool, None)
+
+    @mock.patch.object(iscsi.Driver, '_get_volume',
+                       mock.Mock(return_value={'volumeGroupRef': 'fake_ref'}))
+    def test_get_pool_no_match(self):
+        self.driver._objects['pools'] = [{'volumeGroupRef': 'fake_ref2',
+                                          'label': 'ddp2'}]
+        pool = self.driver.get_pool({'id': 'fake-uuid'})
+        self.assertEqual(pool, None)
+
+    @mock.patch.object(iscsi.Driver, '_create_volume', mock.Mock())
+    def test_create_volume(self):
+        self.driver.create_volume(self.volume)
+        self.driver._create_volume.assert_called_with(
+            'DDP', self.fake_eseries_volume_label, self.volume['size'])
+
+    def test_create_volume_no_pool_provided_by_scheduler(self):
+        volume = copy.deepcopy(self.volume)
+        volume['host'] = "host@backend"  # missing pool
+        self.assertRaises(exception.InvalidHost, self.driver.create_volume,
+                          volume)
+
+    @mock.patch.object(client.RestClient, 'list_storage_pools')
+    def test_helper_create_volume_fail(self, fake_list_pools):
+        fake_pool = {}
+        fake_pool['label'] = self.fake_eseries_pool_label
+        fake_pool['volumeGroupRef'] = 'foo'
+        fake_pools = [fake_pool]
+        fake_list_pools.return_value = fake_pools
+        wrong_eseries_pool_label = 'hostname@backend'
+        self.assertRaises(exception.NetAppDriverException,
+                          self.driver._create_volume, wrong_eseries_pool_label,
+                          self.fake_eseries_volume_label, self.fake_size_gb)
+
+    @mock.patch.object(driver_log, 'info')
+    @mock.patch.object(client.RestClient, 'list_storage_pools')
+    @mock.patch.object(client.RestClient, 'create_volume',
+                       mock.MagicMock(return_value='CorrectVolume'))
+    def test_helper_create_volume(self, storage_pools, log_info):
+        fake_pool = {}
+        fake_pool['label'] = self.fake_eseries_pool_label
+        fake_pool['volumeGroupRef'] = 'foo'
+        fake_pools = [fake_pool]
+        storage_pools.return_value = fake_pools
+        drv = self.driver
+        storage_vol = drv.driver._create_volume(self.fake_eseries_pool_label,
+                                                self.fake_eseries_volume_label,
+                                                self.fake_size_gb)
+        log_info.assert_called_once_with("Created volume with label %s.",
+                                         self.fake_eseries_volume_label)
+        self.assertEqual('CorrectVolume', storage_vol)
+
+    @mock.patch.object(client.RestClient, 'list_storage_pools')
+    @mock.patch.object(client.RestClient, 'create_volume',
+                       mock.MagicMock(
+                           side_effect=exception.NetAppDriverException))
+    @mock.patch.object(driver_log, 'info', mock.Mock())
+    def test_create_volume_check_exception(self, fake_list_pools):
+        fake_pool = {}
+        fake_pool['label'] = self.fake_eseries_pool_label
+        fake_pool['volumeGroupRef'] = 'foo'
+        fake_pools = [fake_pool]
+        fake_list_pools.return_value = fake_pools
+        self.assertRaises(exception.NetAppDriverException,
+                          self.driver._create_volume,
+                          self.fake_eseries_pool_label,
+                          self.fake_eseries_volume_label, self.fake_size_gb)

cinder/tests/test_netapp_nfs.py

@@ -50,10 +50,11 @@ def create_configuration():
 
 
 class FakeVolume(object):
-    def __init__(self, size=0):
+    def __init__(self, host='', size=0):
         self.size = size
         self.id = hash(self)
         self.name = None
+        self.host = host
 
     def __getitem__(self, key):
         return self.__dict__[key]
@@ -110,10 +111,11 @@ def test_create_volume_from_snapshot(self):
         """Tests volume creation from snapshot."""
         drv = self._driver
         mox = self.mox
-        volume = FakeVolume(1)
+        location = '127.0.0.1:/nfs'
+        host = 'hostname@backend#' + location
+        volume = FakeVolume(host, 1)
         snapshot = FakeSnapshot(1)
 
-        location = '127.0.0.1:/nfs'
         expected_result = {'provider_location': location}
         mox.StubOutWithMock(drv, '_clone_volume')
         mox.StubOutWithMock(drv, '_get_volume_location')
@@ -797,6 +799,10 @@ def test_construct_image_url_direct(self):
         if location != "nfs://host/path/image-id":
             self.fail("Unexpected direct url.")
 
+    def test_get_pool(self):
+        pool = self._driver.get_pool({'provider_location': 'fake-share'})
+        self.assertEqual(pool, 'fake-share')
+
 
 class NetappDirectCmodeNfsDriverOnlyTestCase(test.TestCase):
     """Test direct NetApp C Mode driver only and not inherit."""
@@ -820,37 +826,43 @@ def test_create_volume(self, mock_volume_extra_specs):
         extra_specs = {}
         mock_volume_extra_specs.return_value = extra_specs
         fake_share = 'localhost:myshare'
+        host = 'hostname@backend#' + fake_share
         with mock.patch.object(drv, '_ensure_shares_mounted'):
-            with mock.patch.object(drv, '_find_shares',
-                                   return_value=['localhost:myshare']):
-                with mock.patch.object(drv, '_do_create_volume'):
-                    volume_info = self._driver.create_volume(FakeVolume(1))
-                    self.assertEqual(volume_info.get('provider_location'),
-                                     fake_share)
+            with mock.patch.object(drv, '_do_create_volume'):
+                volume_info = self._driver.create_volume(FakeVolume(host, 1))
+                self.assertEqual(volume_info.get('provider_location'),
+                                 fake_share)
+
+    def test_create_volume_no_pool_specified(self):
+        drv = self._driver
+        drv.ssc_enabled = False
+        host = 'hostname@backend'  # missing pool
+        with mock.patch.object(drv, '_ensure_shares_mounted'):
+            self.assertRaises(exception.InvalidHost,
+                              self._driver.create_volume, FakeVolume(host, 1))
 
     @mock.patch.object(netapp_nfs, 'get_volume_extra_specs')
     def test_create_volume_with_qos_policy(self, mock_volume_extra_specs):
         drv = self._driver
         drv.ssc_enabled = False
         extra_specs = {'netapp:qos_policy_group': 'qos_policy_1'}
-        fake_volume = FakeVolume(1)
         fake_share = 'localhost:myshare'
+        host = 'hostname@backend#' + fake_share
+        fake_volume = FakeVolume(host, 1)
         fake_qos_policy = 'qos_policy_1'
         mock_volume_extra_specs.return_value = extra_specs
 
         with mock.patch.object(drv, '_ensure_shares_mounted'):
-            with mock.patch.object(drv, '_find_shares',
-                                   return_value=['localhost:myshare']):
-                with mock.patch.object(drv, '_do_create_volume'):
-                    with mock.patch.object(drv,
-                                           '_set_qos_policy_group_on_volume'
-                                           ) as mock_set_qos:
-                        volume_info = self._driver.create_volume(fake_volume)
-                        self.assertEqual(volume_info.get('provider_location'),
-                                         'localhost:myshare')
-                        mock_set_qos.assert_called_once_with(fake_volume,
-                                                             fake_share,
-                                                             fake_qos_policy)
+            with mock.patch.object(drv, '_do_create_volume'):
+                with mock.patch.object(drv,
+                                       '_set_qos_policy_group_on_volume'
+                                       ) as mock_set_qos:
+                    volume_info = self._driver.create_volume(fake_volume)
+                    self.assertEqual(volume_info.get('provider_location'),
+                                     'localhost:myshare')
+                    mock_set_qos.assert_called_once_with(fake_volume,
+                                                         fake_share,
+                                                         fake_qos_policy)
 
     def test_copy_img_to_vol_copyoffload_success(self):
         drv = self._driver
@@ -1089,6 +1101,14 @@ def _prepare_delete_snapshot_mock(self, snapshot_exists):
 
         return mox
 
+    def test_create_volume_no_pool_specified(self):
+        drv = self._driver
+        drv.ssc_enabled = False
+        host = 'hostname@backend'  # missing pool
+        with mock.patch.object(drv, '_ensure_shares_mounted'):
+            self.assertRaises(exception.InvalidHost,
+                              self._driver.create_volume, FakeVolume(host, 1))
+
     def test_check_for_setup_error_version(self):
         drv = self._driver
         drv._client = api.NaServer("127.0.0.1")
@@ -1196,3 +1216,7 @@ def test_clone_volume_clear(self):
                 raise
 
         mox.VerifyAll()
+
+    def test_get_pool(self):
+        pool = self._driver.get_pool({'provider_location': 'fake-share'})
+        self.assertEqual(pool, 'fake-share')