Make API part of instance boot use new BDM format

This patch makes the API part of the instance boot process rely on the
new BDM format. It adds additional validations that are enabled due to
using the new format.

Since the new format will allow to specify devices that will be created
as images on the hypervisor (if the hypervisor enables it), in order
to prevent a DOS, a new config option 'max_local_block_devices' was
added. This option allows the operator to set a limit on the number of
destination_type='local' block devices a user can specify per VM.

This patch also refactors and simplifies how block devices are handled
during boot, and makes sure that validation errors are caught and
transformed to expected HTTP errors.

Change-Id: I62e0807fcff0a284ff72555d107cac1c9864e46d
blueprint: improve-block-device-handling

etc/nova/nova.conf.sample

@@ -631,6 +631,12 @@
 # keys for the template are: name, uuid, count. (string value)
 #multi_instance_display_name_template=%(name)s-%(uuid)s
 
+# Maximum number of devices that will result in a local image
+# being created on the hypervisor node. Setting this to 0
+# means nova will allow only boot from volume. A negative
+# number means unlimited. (integer value)
+#max_local_block_devices=3
+
 
 #
 # Options defined in nova.compute.flavors

nova/api/openstack/compute/servers.py

@@ -977,7 +977,8 @@ def create(self, req, body):
                 exception.InvalidMetadata,
                 exception.InvalidRequest,
                 exception.PortNotFound,
-                exception.SecurityGroupNotFound) as error:
+                exception.SecurityGroupNotFound,
+                exception.InvalidBDM) as error:
             raise exc.HTTPBadRequest(explanation=error.format_message())
         except exception.PortInUse as error:
             raise exc.HTTPConflict(explanation=error.format_message())

nova/block_device.py

@@ -273,6 +273,41 @@ def legacy_mapping(block_device_mapping):
     return legacy_block_device_mapping
 
 
+def from_legacy_mapping(legacy_block_device_mapping, image_uuid='',
+                        root_device_name=None):
+    """Transform a legacy list of block devices to the new data format."""
+
+    new_bdms = [BlockDeviceDict.from_legacy(legacy_bdm)
+                for legacy_bdm in legacy_block_device_mapping]
+    image_bdm = None
+    volume_backed = False
+
+    # Try to assign boot_device
+    if not root_device_name and not image_uuid:
+        # NOTE (ndipanov): If there is no root_device, pick the first non
+        #                  blank one.
+        non_blank = [bdm for bdm in new_bdms if bdm['source_type'] != 'blank']
+        if non_blank:
+            non_blank[0]['boot_index'] = 0
+    else:
+        for bdm in new_bdms:
+            if (bdm['source_type'] in ('volume', 'snapshot', 'image') and
+                    root_device_name is not None and
+                    (strip_dev(bdm.get('device_name')) ==
+                     strip_dev(root_device_name))):
+                bdm['boot_index'] = 0
+                volume_backed = True
+            elif not bdm['no_device']:
+                bdm['boot_index'] = -1
+            else:
+                bdm['boot_index'] = None
+
+        if not volume_backed and image_uuid:
+            image_bdm = create_image_bdm(image_uuid, boot_index=0)
+
+    return ([image_bdm] if image_bdm else []) + new_bdms
+
+
 def properties_root_device_name(properties):
     """get root device name from image meta data.
     If it isn't specified, return None.

nova/exception.py

@@ -229,6 +229,24 @@ class InvalidBDMVolume(InvalidBDM):
                 "failed to get volume %(id)s.")
 
 
+class InvalidBDMImage(InvalidBDM):
+    msg_fmt = _("Block Device Mapping is Invalid: "
+                "failed to get image %(id)s.")
+
+
+class InvalidBDMBootSequence(InvalidBDM):
+    msg_fmt = _("Block Device Mapping is Invalid: "
+                "Boot sequence for the instance "
+                "and image/block device mapping "
+                "combination is not valid.")
+
+
+class InvalidBDMLocalsLimit(InvalidBDM):
+    msg_fmt = _("Block Device Mapping is Invalid: "
+                "You specified more local devices than the "
+                "limit allows")
+
+
 class InvalidBDMFormat(InvalidBDM):
     msg_fmt = _("Block Device Mapping is Invalid: "
                 "%(details)s")

nova/tests/api/ec2/test_cinder_cloud.py

@@ -586,6 +586,7 @@ def test_describe_instances_bdm(self):
         self._tearDownBlockDeviceMapping(inst1, inst2, volumes)
 
     def _setUpImageSet(self, create_volumes_and_snapshots=False):
+        self.flags(max_local_block_devices=-1)
         mappings1 = [
             {'device': '/dev/sda1', 'virtual': 'root'},
 

nova/tests/api/ec2/test_cloud.py

@@ -1288,6 +1288,7 @@ def assertDictListUnorderedMatch(self, L1, L2, key):
                     self.assertThat(d1, matchers.DictMatches(d2))
 
     def _setUpImageSet(self, create_volumes_and_snapshots=False):
+        self.flags(max_local_block_devices=-1)
         mappings1 = [
             {'device': '/dev/sda1', 'virtual': 'root'},
 

nova/tests/api/openstack/compute/plugins/v3/test_availability_zone.py

@@ -326,6 +326,7 @@ def instance_create(context, inst):
                 "fixed_ips": [],
                 "task_state": "",
                 "vm_state": "",
+                "root_device_name": inst.get('root_device_name', 'vda'),
             })
 
             self.instance_cache_by_id[instance['id']] = instance

nova/tests/api/openstack/compute/plugins/v3/test_config_drive.py

@@ -126,6 +126,7 @@ def instance_create(context, inst):
                 "fixed_ips": [],
                 "task_state": "",
                 "vm_state": "",
+                "root_device_name": inst.get('root_device_name', 'vda'),
             })
 
             self.instance_cache_by_id[instance['id']] = instance

nova/tests/api/openstack/compute/plugins/v3/test_scheduler_hints.py

@@ -161,6 +161,7 @@ def instance_create(context, inst):
                 "fixed_ips": [],
                 "task_state": "",
                 "vm_state": "",
+                "root_device_name": inst.get('root_device_name', 'vda'),
             })
 
             self.instance_cache_by_id[instance['id']] = instance

nova/tests/api/openstack/compute/plugins/v3/test_servers.py

@@ -1721,7 +1721,7 @@ def instance_create(context, inst):
                 "fixed_ips": [],
                 "task_state": "",
                 "vm_state": "",
-                "security_groups": inst['security_groups'],
+                "root_device_name": inst.get('root_device_name', 'vda'),
             })
 
             self.instance_cache_by_id[instance['id']] = instance

nova/tests/api/openstack/compute/plugins/v3/test_user_data.py

@@ -89,6 +89,7 @@ def instance_create(context, inst):
                 "fixed_ips": [],
                 "task_state": "",
                 "vm_state": "",
+                "root_device_name": inst.get('root_device_name', 'vda'),
             })
 
             self.instance_cache_by_id[instance['id']] = instance

nova/tests/api/openstack/compute/test_servers.py

@@ -2429,7 +2429,8 @@ def test_create_instance_with_volumes_enabled_and_bdms_no_image(self):
         }]
         volume = bdm[0]
         compute_api.API._validate_bdm(mox.IgnoreArg(),
-                                      mox.IgnoreArg()).AndReturn(True)
+                mox.IgnoreArg(), mox.IgnoreArg(),
+                mox.IgnoreArg()).AndReturn(True)
         compute_api.API._get_volume_image_metadata(mox.IgnoreArg(),
                                                    bdm).AndReturn(volume)
         params = {'block_device_mapping': bdm}
@@ -2472,8 +2473,9 @@ def create(*args, **kwargs):
         self.mox.StubOutWithMock(compute_api.API, '_validate_bdm')
         self.mox.StubOutWithMock(compute_api.API, '_get_volume_image_metadata')
 
-        compute_api.API._validate_bdm(mox.IgnoreArg(),
-                                      mox.IgnoreArg()).AndReturn(True)
+        compute_api.API._validate_bdm(
+            mox.IgnoreArg(), mox.IgnoreArg(),
+            mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True)
         compute_api.API._get_volume_image_metadata(
             mox.IgnoreArg(), mox.IgnoreArg()).AndReturn({})
         self.mox.ReplayAll()
@@ -2566,6 +2568,46 @@ def create(*args, **kwargs):
         self.assertRaises(webob.exc.HTTPBadRequest,
                           self._test_create_extra, params)
 
+    def test_create_instance_bdm_api_validation_fails(self):
+        self.ext_mgr.extensions = {'os-volumes': 'fake',
+                                   'os-block-device-mapping-v2-boot': 'fake'}
+        bdm = {'delete_on_termination': 1,
+                'device_name': 'vda',
+                'source_type': 'volume',
+                'destination_type': 'volume',
+                'volume_size': 1,
+                'boot_index': 0,
+                'uuid': '11111111-1111-1111-1111-111111111111'}
+        self.validation_fail_test_validate_called = False
+        self.validation_fail_instance_destroy_called = False
+
+        bdm_exceptions = ((exception.InvalidBDMSnapshot, {'id': 'fake'}),
+                          (exception.InvalidBDMVolume, {'id': 'fake'}),
+                          (exception.InvalidBDMImage, {'id': 'fake'}),
+                          (exception.InvalidBDMBootSequence, {}),
+                          (exception.InvalidBDMLocalsLimit, {}))
+
+        ex_iter = iter(bdm_exceptions)
+
+        def _validate_bdm(*args, **kwargs):
+            self.validation_fail_test_validate_called = True
+            ex, kargs = ex_iter.next()
+            raise ex(**kargs)
+
+        def _instance_destroy(*args, **kwargs):
+            self.validation_fail_instance_destroy_called = True
+
+        self.stubs.Set(compute_api.API, '_validate_bdm', _validate_bdm)
+        self.stubs.Set(compute_api.db, 'instance_destroy', _instance_destroy)
+
+        for _ in xrange(len(bdm_exceptions)):
+            params = {'block_device_mapping_v2': [bdm.copy()]}
+            self.assertRaises(webob.exc.HTTPBadRequest,
+                              self._test_create_extra, params)
+            self.assertTrue(self.validation_fail_test_validate_called)
+            self.validation_fail_test_validate_called = False
+            self.validation_fail_test_validate_called = True
+
     def test_create_instance_with_bdm_delete_on_termination(self):
         self.ext_mgr.extensions = {'os-volumes': 'fake'}
         bdm = [{'device_name': 'foo1', 'volume_id': 'fake_vol',

nova/tests/cells/test_cells_scheduler.py

@@ -19,6 +19,7 @@
 
 from oslo.config import cfg
 
+from nova import block_device
 from nova.cells import filters
 from nova.cells import weights
 from nova.compute import vm_states
@@ -106,6 +107,8 @@ def test_create_instances_here(self):
                           'project_id': self.ctxt.project_id}
 
         call_info = {'uuids': []}
+        block_device_mapping = [block_device.create_image_bdm(
+            'fake_image_ref')]
 
         def _fake_instance_update_at_top(_ctxt, instance):
             call_info['uuids'].append(instance['uuid'])
@@ -114,7 +117,8 @@ def _fake_instance_update_at_top(_ctxt, instance):
                        _fake_instance_update_at_top)
 
         self.scheduler._create_instances_here(self.ctxt, instance_uuids,
-                instance_props, inst_type, image, ['default'], [])
+                instance_props, inst_type, image,
+                ['default'], block_device_mapping)
         self.assertEqual(instance_uuids, call_info['uuids'])
 
         for instance_uuid in instance_uuids: