Use manifest to backport OVOs during upgrades

To make objects that have other objects as fields compatible to an
earlier version, oslo versioned objects uses either a manifest passed to
obj_to_primitive or the object's obj_relationships mapping.

Which means that if we don't have any of those mechanisms in place our
rolling upgrades mechanism will fail whenever we try to backport a
Versioned Object that has set an ObjectField field because Oslo
Versioned Object will not know how to backport that related object.

This patch introduces the usage of manifests on backports when we are
doing rolling upgrades.

For the manifest, we use the data in our Objects History.  Which means
that as long as we keep history in OBJ_VERSIONS right we will not have
to create and worry about keeping lists' child_versions field or our
versioned object's obj_relationships for fields with types
ListOfObjectsField and ObjectField.

We also don't have to worry about cascade version bumping, as in
changing the List OVO version whenever the OVO it contains gets bumped,
or bumping our OVO whenever one of the related OVO fields is bumped.

Closes-Bug: #1571566
Change-Id: Ibc1a1257830c925c10696c0b5aedd5f471c538d0
(cherry picked from commit f0d34b7)

cinder/exception.py

@@ -758,6 +758,11 @@ class LockingFailed(CinderException):
 ObjectFieldInvalid = obj_exc.ObjectFieldInvalid
 
 
+class CappedVersionUnknown(CinderException):
+    message = _('Unrecoverable Error: Versioned Objects in DB are capped to '
+                'unknown version %(version)s.')
+
+
 class VolumeGroupNotFound(CinderException):
     message = _('Unable to find Volume Group: %(vg_name)s')
 

cinder/objects/backup.py

@@ -163,9 +163,6 @@ class BackupList(base.ObjectListBase, base.CinderObject):
     fields = {
         'objects': fields.ListOfObjectsField('Backup'),
     }
-    child_versions = {
-        '1.0': '1.0'
-    }
 
     @base.remotable_classmethod
     def get_all(cls, context, filters=None, marker=None, limit=None,

cinder/objects/base.py

@@ -401,6 +401,17 @@ def __init__(self, version_cap=None):
         super(CinderObjectSerializer, self).__init__()
         self.version_cap = version_cap
 
+        # NOTE(geguileo): During upgrades we will use a manifest to ensure that
+        # all objects are properly backported.  This allows us to properly
+        # backport child objects to the right version even if parent version
+        # has not been bumped.
+        if not version_cap or version_cap == OBJ_VERSIONS.get_current():
+            self.manifest = None
+        else:
+            if version_cap not in OBJ_VERSIONS:
+                raise exception.CappedVersionUnknown(version=version_cap)
+            self.manifest = OBJ_VERSIONS[version_cap]
+
     def _get_capped_obj_version(self, obj):
         objname = obj.obj_name()
         version_dict = OBJ_VERSIONS.get(self.version_cap, {})
@@ -426,5 +437,5 @@ def serialize_entity(self, context, entity):
               callable(entity.obj_to_primitive)):
             # NOTE(dulek): Backport outgoing object to the capped version.
             backport_ver = self._get_capped_obj_version(entity)
-            entity = entity.obj_to_primitive(backport_ver)
+            entity = entity.obj_to_primitive(backport_ver, self.manifest)
         return entity

cinder/objects/cgsnapshot.py

@@ -127,9 +127,6 @@ class CGSnapshotList(base.ObjectListBase, base.CinderObject):
     fields = {
         'objects': fields.ListOfObjectsField('CGSnapshot')
     }
-    child_version = {
-        '1.0': '1.0'
-    }
 
     @base.remotable_classmethod
     def get_all(cls, context, filters=None):

cinder/objects/consistencygroup.py

@@ -143,10 +143,6 @@ class ConsistencyGroupList(base.ObjectListBase, base.CinderObject):
     fields = {
         'objects': fields.ListOfObjectsField('ConsistencyGroup')
     }
-    child_version = {
-        '1.0': '1.0',
-        '1.1': '1.1',
-    }
 
     @base.remotable_classmethod
     def get_all(cls, context, filters=None, marker=None, limit=None,

cinder/objects/service.py

@@ -142,10 +142,6 @@ class ServiceList(base.ObjectListBase, base.CinderObject):
     fields = {
         'objects': fields.ListOfObjectsField('Service'),
     }
-    child_versions = {
-        '1.0': '1.0',
-        '1.1': '1.2',
-    }
 
     @base.remotable_classmethod
     def get_all(cls, context, filters=None):

cinder/objects/snapshot.py

@@ -225,9 +225,6 @@ class SnapshotList(base.ObjectListBase, base.CinderObject):
     fields = {
         'objects': fields.ListOfObjectsField('Snapshot'),
     }
-    child_versions = {
-        '1.0': '1.0'
-    }
 
     @base.remotable_classmethod
     def get_all(cls, context, search_opts, marker=None, limit=None,

cinder/objects/volume.py

@@ -442,11 +442,6 @@ class VolumeList(base.ObjectListBase, base.CinderObject):
         'objects': fields.ListOfObjectsField('Volume'),
     }
 
-    child_versions = {
-        '1.0': '1.0',
-        '1.1': '1.1',
-    }
-
     @classmethod
     def _get_expected_attrs(cls, context):
         expected_attrs = ['metadata', 'volume_type']

cinder/objects/volume_attachment.py

@@ -71,10 +71,6 @@ class VolumeAttachmentList(base.ObjectListBase, base.CinderObject):
         'objects': fields.ListOfObjectsField('VolumeAttachment'),
     }
 
-    child_versions = {
-        '1.0': '1.0',
-    }
-
     @base.remotable_classmethod
     def get_all_by_volume_id(cls, context, volume_id):
         attachments = db.volume_attachment_get_used_by_volume_id(context,

cinder/objects/volume_type.py

@@ -108,11 +108,6 @@ class VolumeTypeList(base.ObjectListBase, base.CinderObject):
         'objects': fields.ListOfObjectsField('VolumeType'),
     }
 
-    child_versions = {
-        '1.0': '1.0',
-        '1.1': '1.0',
-    }
-
     @base.remotable_classmethod
     def get_all(cls, context, inactive=0, filters=None, marker=None,
                 limit=None, sort_keys=None, sort_dirs=None, offset=None):

cinder/tests/unit/fake_objects.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2016 Red Hat Inc.
+# Copyright (c) 2016 Intel Corp.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_utils import versionutils
+
+from cinder import objects
+
+
+@objects.base.CinderObjectRegistry.register_if(False)
+class ChildObject(objects.base.CinderObject):
+    VERSION = '1.2'
+
+    fields = {
+        'scheduled_at': objects.base.fields.DateTimeField(nullable=True),
+        'uuid': objects.base.fields.UUIDField(),
+        'text': objects.base.fields.StringField(nullable=True),
+        'integer': objects.base.fields.IntegerField(nullable=True),
+    }
+
+    def obj_make_compatible(self, primitive, target_version):
+        super(ChildObject, self).obj_make_compatible(primitive,
+                                                     target_version)
+        target_version = versionutils.convert_version_to_tuple(target_version)
+        if target_version < (1, 1):
+            primitive.pop('text', None)
+        if target_version < (1, 2):
+            primitive.pop('integer', None)
+
+
+@objects.base.CinderObjectRegistry.register_if(False)
+class ParentObject(objects.base.CinderObject):
+    VERSION = '1.1'
+
+    fields = {
+        'uuid': objects.base.fields.UUIDField(),
+        'child': objects.base.fields.ObjectField('ChildObject', nullable=True),
+        'scheduled_at': objects.base.fields.DateTimeField(nullable=True),
+    }
+
+    def obj_make_compatible(self, primitive, target_version):
+        super(ParentObject, self).obj_make_compatible(primitive,
+                                                      target_version)
+        target_version = versionutils.convert_version_to_tuple(target_version)
+        if target_version < (1, 1):
+            primitive.pop('scheduled_at', None)
+
+
+@objects.base.CinderObjectRegistry.register_if(False)
+class ParentObjectList(objects.base.CinderObject, objects.base.ObjectListBase):
+    VERSION = ParentObject.VERSION
+
+    fields = {
+        'objects': objects.base.fields.ListOfObjectsField('ParentObject'),
+    }
+
+
+class MyHistory(objects.base.CinderObjectVersionsHistory):
+    linked_objects = {'ParentObject': 'ParentObjectList'}
+
+    def __init__(self):
+        self.versions = ['1.0']
+        self['1.0'] = {'ChildObject': '1.0'}
+        self.add('1.1', {'ChildObject': '1.1'})
+        self.add('1.2', {'ParentObject': '1.0'})
+        self.add('1.3', {'ParentObjectList': '1.0'})
+        self.add('1.4', {'ParentObject': '1.1'})
+        self.add('1.5', {'ParentObjectList': '1.1'})
+        self.add('1.6', {'ChildObject': '1.2'})

cinder/tests/unit/objects/test_base.py

@@ -13,47 +13,30 @@
 #    under the License.
 
 import datetime
-import mock
 import uuid
 
 from iso8601 import iso8601
-from oslo_utils import versionutils
+import mock
 from oslo_versionedobjects import fields
 from sqlalchemy import sql
 
 from cinder import context
 from cinder import db
 from cinder.db.sqlalchemy import models
+from cinder import exception
 from cinder import objects
 from cinder import test
 from cinder.tests.unit import fake_constants as fake
+from cinder.tests.unit import fake_objects
 from cinder.tests.unit import objects as test_objects
 
 
-@objects.base.CinderObjectRegistry.register_if(False)
-class TestObject(objects.base.CinderObject):
-    VERSION = '1.1'
-
-    fields = {
-        'scheduled_at': objects.base.fields.DateTimeField(nullable=True),
-        'uuid': objects.base.fields.UUIDField(),
-        'text': objects.base.fields.StringField(nullable=True),
-    }
-
-    def obj_make_compatible(self, primitive, target_version):
-        super(TestObject, self).obj_make_compatible(primitive,
-                                                    target_version)
-        target_version = versionutils.convert_version_to_tuple(target_version)
-        if target_version < (1, 1):
-            primitive.pop('text', None)
-
-
 class TestCinderObject(test_objects.BaseObjectsTestCase):
     """Tests methods from CinderObject."""
 
     def setUp(self):
         super(TestCinderObject, self).setUp()
-        self.obj = TestObject(
+        self.obj = fake_objects.ChildObject(
             scheduled_at=None,
             uuid=uuid.uuid4(),
             text='text')
@@ -609,21 +592,107 @@ def test_dict_objects(self):
         self.assertFalse('def' in obj)
 
 
-@mock.patch('cinder.objects.base.OBJ_VERSIONS', {'1.0': {'TestObject': '1.0'},
-                                                 '1.1': {'TestObject': '1.1'},
-                                                 })
+@mock.patch('cinder.objects.base.OBJ_VERSIONS', fake_objects.MyHistory())
 class TestCinderObjectSerializer(test_objects.BaseObjectsTestCase):
     def setUp(self):
         super(TestCinderObjectSerializer, self).setUp()
-        self.obj = TestObject(scheduled_at=None, uuid=uuid.uuid4(),
-                              text='text')
-
-    def test_serialize_entity_backport(self):
-        serializer = objects.base.CinderObjectSerializer('1.0')
-        primitive = serializer.serialize_entity(self.context, self.obj)
-        self.assertEqual('1.0', primitive['versioned_object.version'])
+        self.obj = fake_objects.ChildObject(scheduled_at=None,
+                                            uuid=uuid.uuid4(),
+                                            text='text',
+                                            integer=1)
+        self.parent = fake_objects.ParentObject(uuid=uuid.uuid4(),
+                                                child=self.obj,
+                                                scheduled_at=None)
+        self.parent_list = fake_objects.ParentObjectList(objects=[self.parent])
+
+    def test_serialize_init_current_has_no_manifest(self):
+        """Test that pinned to current version we have no manifest."""
+        serializer = objects.base.CinderObjectSerializer('1.6')
+        # Serializer should not have a manifest
+        self.assertIsNone(serializer.manifest)
+
+    def test_serialize_init_no_cap_has_no_manifest(self):
+        """Test that without cap we have no manifest."""
+        serializer = objects.base.CinderObjectSerializer()
+        # Serializer should not have a manifest
+        self.assertIsNone(serializer.manifest)
+
+    def test_serialize_init_pinned_has_manifest(self):
+        """Test that pinned to older version we have manifest."""
+        objs_version = '1.5'
+        serializer = objects.base.CinderObjectSerializer(objs_version)
+        # Serializer should have the right manifest
+        self.assertDictEqual(fake_objects.MyHistory()[objs_version],
+                             serializer.manifest)
 
     def test_serialize_entity_unknown_version(self):
-        serializer = objects.base.CinderObjectSerializer('0.9')
+        """Test that bad cap version will prevent serializer creation."""
+        self.assertRaises(exception.CappedVersionUnknown,
+                          objects.base.CinderObjectSerializer, '0.9')
+
+    def test_serialize_entity_basic_no_backport(self):
+        """Test single element serializer with no backport."""
+        serializer = objects.base.CinderObjectSerializer('1.6')
+        primitive = serializer.serialize_entity(self.context, self.obj)
+        self.assertEqual('1.2', primitive['versioned_object.version'])
+        data = primitive['versioned_object.data']
+        self.assertEqual(1, data['integer'])
+        self.assertEqual('text', data['text'])
+
+    def test_serialize_entity_basic_backport(self):
+        """Test single element serializer with backport."""
+        serializer = objects.base.CinderObjectSerializer('1.5')
         primitive = serializer.serialize_entity(self.context, self.obj)
         self.assertEqual('1.1', primitive['versioned_object.version'])
+        data = primitive['versioned_object.data']
+        self.assertNotIn('integer', data)
+        self.assertEqual('text', data['text'])
+
+    def test_serialize_entity_full_no_backport(self):
+        """Test related elements serialization with no backport."""
+        serializer = objects.base.CinderObjectSerializer('1.6')
+        primitive = serializer.serialize_entity(self.context, self.parent_list)
+        self.assertEqual('1.1', primitive['versioned_object.version'])
+        parent = primitive['versioned_object.data']['objects'][0]
+        self.assertEqual('1.1', parent['versioned_object.version'])
+        child = parent['versioned_object.data']['child']
+        self.assertEqual('1.2', child['versioned_object.version'])
+
+    def test_serialize_entity_full_backport_last_children(self):
+        """Test related elements serialization with backport of the last child.
+
+        Test that using the manifest we properly backport a child object even
+        when all its parents have not changed their version.
+        """
+        serializer = objects.base.CinderObjectSerializer('1.5')
+        primitive = serializer.serialize_entity(self.context, self.parent_list)
+        self.assertEqual('1.1', primitive['versioned_object.version'])
+        parent = primitive['versioned_object.data']['objects'][0]
+        self.assertEqual('1.1', parent['versioned_object.version'])
+        # Only the child has been backported
+        child = parent['versioned_object.data']['child']
+        self.assertEqual('1.1', child['versioned_object.version'])
+        # Check that the backport has been properly done
+        data = child['versioned_object.data']
+        self.assertNotIn('integer', data)
+        self.assertEqual('text', data['text'])
+
+    def test_serialize_entity_full_backport(self):
+        """Test backport of the whole tree of related elements."""
+        serializer = objects.base.CinderObjectSerializer('1.3')
+        primitive = serializer.serialize_entity(self.context, self.parent_list)
+        # List has been backported
+        self.assertEqual('1.0', primitive['versioned_object.version'])
+        parent = primitive['versioned_object.data']['objects'][0]
+        # Parent has been backported as well
+        self.assertEqual('1.0', parent['versioned_object.version'])
+        # And the backport has been properly done
+        data = parent['versioned_object.data']
+        self.assertNotIn('scheduled_at', data)
+        # And child as well
+        child = parent['versioned_object.data']['child']
+        self.assertEqual('1.1', child['versioned_object.version'])
+        # Check that the backport has been properly done
+        data = child['versioned_object.data']
+        self.assertNotIn('integer', data)
+        self.assertEqual('text', data['text'])