Add persistence to the RequestSpec object

In order to use the request spec for an instance boot during later
scheduling operations, like resize/migrate, it needs to be persisted.
This adds a db migration to store the request spec and methods to the
object to create/save/get the request spec.

The RequestSpec object is also important for cells v2 so that boot
requests can have their information persisted before the creation of an
instance record in the database.

Change-Id: I94cd50ad29e10eb320d1d78f3081db6519815ff9
Implements: bp persist-request-spec

nova/db/sqlalchemy/api_migrations/migrate_repo/versions/004_add_request_spec.py

@@ -0,0 +1,46 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from migrate import UniqueConstraint
+from sqlalchemy import Column
+from sqlalchemy import DateTime
+from sqlalchemy import Index
+from sqlalchemy import Integer
+from sqlalchemy import MetaData
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy import Text
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    request_specs = Table('request_specs', meta,
+        Column('created_at', DateTime),
+        Column('updated_at', DateTime),
+        Column('id', Integer, primary_key=True, nullable=False),
+        Column('instance_uuid', String(36), nullable=False),
+        Column('spec', Text, nullable=False),
+        UniqueConstraint('instance_uuid',
+            name='uniq_request_specs0instance_uuid'),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
+    # NOTE(mriedem): DB2 creates an index when a unique constraint is created
+    # so trying to add a second index on the host column will fail with error
+    # SQL0605W, so omit the index in the case of DB2.
+    if migrate_engine.name != 'ibm_db_sa':
+        Index('request_spec_instance_uuid_idx', request_specs.c.instance_uuid)
+
+    request_specs.create(checkfirst=True)

nova/db/sqlalchemy/api_models.py

@@ -69,3 +69,18 @@ class HostMapping(API_BASE):
     cell_id = Column(Integer, ForeignKey('cell_mappings.id'),
             nullable=False)
     host = Column(String(255), nullable=False)
+
+
+class RequestSpec(API_BASE):
+    """Represents the information passed to the scheduler."""
+
+    __tablename__ = 'request_specs'
+    __table_args__ = (
+        Index('request_spec_instance_uuid_idx', 'instance_uuid'),
+        schema.UniqueConstraint('instance_uuid',
+            name='uniq_request_specs0instance_uuid'),
+        )
+
+    id = Column(Integer, primary_key=True)
+    instance_uuid = Column(String(36), nullable=False)
+    spec = Column(Text, nullable=False)

nova/exception.py

@@ -1966,3 +1966,7 @@ class HostMappingNotFound(Invalid):
 class RealtimeConfigurationInvalid(Invalid):
     msg_fmt = _("Cannot set realtime policy in a non dedicated "
                 "cpu pinning policy")
+
+
+class RequestSpecNotFound(NotFound):
+    msg_fmt = _("RequestSpec not found for instance %(instance_uuid)s")

nova/objects/request_spec.py

@@ -12,14 +12,21 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
 import six
 
+from nova.db.sqlalchemy import api as db
+from nova.db.sqlalchemy import api_models
+from nova import exception
 from nova import objects
 from nova.objects import base
 from nova.objects import fields
 from nova.objects import instance as obj_instance
 from nova.virt import hardware
 
+LOG = logging.getLogger(__name__)
+
 
 @base.NovaObjectRegistry.register
 class RequestSpec(base.NovaObject):
@@ -28,7 +35,8 @@ class RequestSpec(base.NovaObject):
     # Version 1.2: SchedulerRetries version 1.1
     # Version 1.3: InstanceGroup version 1.10
     # Version 1.4: ImageMeta version 1.7
-    VERSION = '1.4'
+    # Version 1.5: Added get_by_instance_uuid(), create(), save()
+    VERSION = '1.5'
 
     fields = {
         'id': fields.IntegerField(),
@@ -311,6 +319,85 @@ def to_legacy_filter_properties_dict(self):
                 hint) for hint in self.scheduler_hints}
         return filt_props
 
+    @staticmethod
+    def _from_db_object(context, spec, db_spec):
+        spec = spec.obj_from_primitive(jsonutils.loads(db_spec['spec']))
+        spec._context = context
+        spec.obj_reset_changes()
+        return spec
+
+    @staticmethod
+    def _get_by_instance_uuid_from_db(context, instance_uuid):
+        session = db.get_api_session()
+
+        with session.begin():
+            db_spec = session.query(api_models.RequestSpec).filter_by(
+                    instance_uuid=instance_uuid).first()
+            if not db_spec:
+                raise exception.RequestSpecNotFound(
+                        instance_uuid=instance_uuid)
+        return db_spec
+
+    @base.remotable_classmethod
+    def get_by_instance_uuid(cls, context, instance_uuid):
+        db_spec = cls._get_by_instance_uuid_from_db(context, instance_uuid)
+        return cls._from_db_object(context, cls(), db_spec)
+
+    @staticmethod
+    @db.api_context_manager.writer
+    def _create_in_db(context, updates):
+        db_spec = api_models.RequestSpec()
+        db_spec.update(updates)
+        db_spec.save(context.session)
+        return db_spec
+
+    def _get_update_primitives(self):
+        """Serialize object to match the db model.
+
+        We store copies of embedded objects rather than
+        references to these objects because we want a snapshot of the request
+        at this point.  If the references changed or were deleted we would
+        not be able to reschedule this instance under the same conditions as
+        it was originally scheduled with.
+        """
+        updates = self.obj_get_changes()
+        # NOTE(alaski): The db schema is the full serialized object in a
+        # 'spec' column.  If anything has changed we rewrite the full thing.
+        if updates:
+            db_updates = {'spec': jsonutils.dumps(self.obj_to_primitive())}
+            if 'instance_uuid' in updates:
+                db_updates['instance_uuid'] = updates['instance_uuid']
+        return db_updates
+
+    @base.remotable
+    def create(self):
+        if self.obj_attr_is_set('id'):
+            raise exception.ObjectActionError(action='create',
+                                              reason='already created')
+
+        updates = self._get_update_primitives()
+
+        db_spec = self._create_in_db(self._context, updates)
+        self._from_db_object(self._context, self, db_spec)
+
+    @staticmethod
+    @db.api_context_manager.writer
+    def _save_in_db(context, instance_uuid, updates):
+        # FIXME(sbauza): Provide a classmethod when oslo.db bug #1520195 is
+        # fixed and released
+        db_spec = RequestSpec._get_by_instance_uuid_from_db(context,
+                                                            instance_uuid)
+        db_spec.update(updates)
+        db_spec.save(context.session)
+        return db_spec
+
+    @base.remotable
+    def save(self):
+        updates = self._get_update_primitives()
+        db_spec = self._save_in_db(self._context, self.instance_uuid, updates)
+        self._from_db_object(self._context, self, db_spec)
+        self.obj_reset_changes()
+
 
 @base.NovaObjectRegistry.register
 class SchedulerRetries(base.NovaObject):

nova/tests/functional/db/api/test_migrations.py

@@ -188,6 +188,17 @@ def _check_003(self, engine, data):
         self.assertEqual(['id'], fk['referred_columns'])
         self.assertEqual(['cell_id'], fk['constrained_columns'])
 
+    def _check_004(self, engine, data):
+        columns = ['created_at', 'updated_at', 'id', 'instance_uuid', 'spec']
+        for column in columns:
+            self.assertColumnExists(engine, 'request_specs', column)
+
+        self.assertUniqueConstraintExists(engine, 'request_specs',
+                ['instance_uuid'])
+        if engine.name != 'ibm_db_sa':
+            self.assertIndexExists(engine, 'request_specs',
+                    'request_spec_instance_uuid_idx')
+
 
 class TestNovaAPIMigrationsWalkSQLite(NovaAPIMigrationsWalk,
                                       test_base.DbTestCase,

nova/tests/functional/db/test_request_spec.py

@@ -0,0 +1,63 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova import context
+from nova import exception
+from nova.objects import base as obj_base
+from nova.objects import request_spec
+from nova import test
+from nova.tests import fixtures
+from nova.tests.unit import fake_request_spec
+
+
+class RequestSpecTestCase(test.NoDBTestCase):
+    def setUp(self):
+        super(RequestSpecTestCase, self).setUp()
+        self.useFixture(fixtures.Database(database='api'))
+        self.context = context.RequestContext('fake-user', 'fake-project')
+        self.spec_obj = request_spec.RequestSpec()
+        self.instance_uuid = None
+
+    def _create_spec(self):
+        args = fake_request_spec.fake_db_spec()
+        args.pop('id', None)
+        self.instance_uuid = args['instance_uuid']
+        spec = request_spec.RequestSpec._from_db_object(self.context,
+                self.spec_obj,
+                self.spec_obj._create_in_db(self.context, args))
+        return spec
+
+    def test_get_by_instance_uuid_not_found(self):
+        self.assertRaises(exception.RequestSpecNotFound,
+                self.spec_obj._get_by_instance_uuid_from_db, self.context,
+                self.instance_uuid)
+
+    def test_get_by_uuid(self):
+        spec = self._create_spec()
+        db_spec = self.spec_obj.get_by_instance_uuid(self.context,
+                self.instance_uuid)
+        self.assertTrue(obj_base.obj_equal_prims(spec, db_spec))
+
+    def test_save_in_db(self):
+        spec = self._create_spec()
+
+        old_az = spec.availability_zone
+        spec.availability_zone = '%s-new' % old_az
+        spec.save()
+        db_spec = self.spec_obj.get_by_instance_uuid(self.context,
+                spec.instance_uuid)
+        self.assertTrue(obj_base.obj_equal_prims(spec, db_spec))
+        self.assertNotEqual(old_az, db_spec.availability_zone)
+
+    def test_double_create(self):
+        spec = self._create_spec()
+        self.assertRaises(exception.ObjectActionError, spec.create)

nova/tests/unit/fake_request_spec.py

@@ -0,0 +1,90 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_serialization import jsonutils
+from oslo_utils import uuidutils
+
+from nova import context
+from nova import objects
+from nova.tests.unit import fake_flavor
+
+
+INSTANCE_NUMA_TOPOLOGY = objects.InstanceNUMATopology(
+        cells=[objects.InstanceNUMACell(id=0, cpuset=set([1, 2]), memory=512),
+              objects.InstanceNUMACell(id=1, cpuset=set([3, 4]), memory=512)])
+INSTANCE_NUMA_TOPOLOGY.obj_reset_changes(recursive=True)
+
+IMAGE_META = objects.ImageMeta.from_dict(
+        {'status': 'active',
+         'container_format': 'bare',
+         'min_ram': 0,
+         'updated_at': '2014-12-12T11:16:36.000000',
+         'min_disk': '0',
+         'owner': '2d8b9502858c406ebee60f0849486222',
+         'protected': 'yes',
+         'properties': {
+             'os_type': 'Linux',
+             'hw_video_model': 'vga',
+             'hw_video_ram': '512',
+             'hw_qemu_guest_agent': 'yes',
+             'hw_scsi_model': 'virtio-scsi',
+         },
+         'size': 213581824,
+         'name': 'f16-x86_64-openstack-sda',
+         'checksum': '755122332caeb9f661d5c978adb8b45f',
+         'created_at': '2014-12-10T16:23:14.000000',
+         'disk_format': 'qcow2',
+         'id': 'c8b1790e-a07d-4971-b137-44f2432936cd',
+        }
+    )
+IMAGE_META.obj_reset_changes(recursive=True)
+
+PCI_REQUESTS = objects.InstancePCIRequests(
+        requests=[objects.InstancePCIRequest(count=1),
+                  objects.InstancePCIRequest(count=2)])
+PCI_REQUESTS.obj_reset_changes(recursive=True)
+
+
+def fake_db_spec():
+    req_obj = fake_spec_obj()
+    db_request_spec = {
+            'id': 1,
+            'instance_uuid': req_obj.instance_uuid,
+            'spec': jsonutils.dumps(req_obj.obj_to_primitive()),
+    }
+
+    return db_request_spec
+
+
+def fake_spec_obj(remove_id=False):
+    ctxt = context.RequestContext('fake', 'fake')
+    req_obj = objects.RequestSpec(ctxt)
+    if not remove_id:
+        req_obj.id = 42
+    req_obj.instance_uuid = uuidutils.generate_uuid()
+    req_obj.image = IMAGE_META
+    req_obj.numa_topology = INSTANCE_NUMA_TOPOLOGY
+    req_obj.pci_requests = PCI_REQUESTS
+    req_obj.flavor = fake_flavor.fake_flavor_obj(ctxt)
+    req_obj.retry = objects.SchedulerRetries()
+    req_obj.limits = objects.SchedulerLimits()
+    req_obj.instance_group = objects.InstanceGroup()
+    req_obj.project_id = 'fake'
+    req_obj.num_instances = 1
+    req_obj.availability_zone = None
+    req_obj.ignore_hosts = ['host2', 'host4']
+    req_obj.force_hosts = ['host1', 'host3']
+    req_obj.force_nodes = ['node1', 'node2']
+    req_obj.scheduler_hints = {'hint': ['over-there']}
+    # This should never be a changed field
+    req_obj.obj_reset_changes(['id'])
+    return req_obj

nova/tests/unit/objects/test_objects.py

@@ -1253,7 +1253,7 @@ def obj_name(cls):
     'PciDevicePoolList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',
     'Quotas': '1.2-1fe4cd50593aaf5d36a6dc5ab3f98fb3',
     'QuotasNoOp': '1.2-e041ddeb7dc8188ca71706f78aad41c1',
-    'RequestSpec': '1.4-6922fe208b5d1186bdd825513f677921',
+    'RequestSpec': '1.5-576a249869c161e17b7cd6d55f9d85f3',
     'S3ImageMapping': '1.0-7dd7366a890d82660ed121de9092276e',
     'SchedulerLimits': '1.0-249c4bd8e62a9b327b7026b7f19cc641',
     'SchedulerRetries': '1.1-3c9c8b16143ebbb6ad7030e999d14cc0',