Merge pull request #482 from pkediyal/schema_versioning_final

M156: add conditional persisted_version update during deserialization

pyon/core/interfaces/object_model_generator.py

@@ -348,12 +348,12 @@ def generate_objects(self, opts):
         init_lines = []
         first_time = True
         decorators = ''
-        class_decorators = ''
+        class_decorators = {}
         description = ''
         csv_description = ''
         class_comment = ''
         version_num_pattern = re.compile("[0-9]+")
-        type_version_pattern = re.compile("TypeVersion\":\s\"[0-9]+")
+
         for line in self.data_yaml_text.split('\n'):
             if line.isspace():
                 continue
@@ -435,11 +435,7 @@ def generate_objects(self, opts):
                     dec = line.strip()[2:].split("=")
                     key = dec[0]
                     value = dec[1] if len(dec) == 2 else ""
-                    # Add it to the decorator list
-                    if not class_decorators:
-                        class_decorators = '"' + key + '": "' + value + '"'
-                    else:
-                        class_decorators = class_decorators + ', "' + key + '": "' + value + '"'
+                    class_decorators[key]=value
 
                 #Handle class level comments
                 if line.startswith('#'):
@@ -518,44 +514,39 @@ def generate_objects(self, opts):
                 self.dataobject_output_text += "class " + line + "):" + class_comment_temp + "\n\n"
 
                 # get current_type_version (ctv) from decorator
-                ctv = (re.findall(version_num_pattern,str(re.findall(type_version_pattern,class_decorators))))
+                #ctv = (re.findall(version_num_pattern,str(re.findall(type_version_pattern,class_decorators))))
+                ctv = 0
+                if "TypeVersion" in class_decorators:
+                    ctv = int(class_decorators["TypeVersion"])
+                    # if yml specified TypeVersion then it comes in as string and needs to be made int
+                    class_decorators["TypeVersion"]=ctv
 
-                # if found
-                if len(ctv)>0:
-                    # convert to an int
-                    ctv = int(ctv[0])
-                else:
-                    # else note that the class has no ctv
-                    ctv = 0
                 # however, if the super class has ctv
                 if super_class in self.class_args_dict and 'cumulative_version' in self.class_args_dict[super_class]:
                     # and current class does not
-                    if ctv == 0:
-                        # then give it at least a ctv of 1 to differentiate from classes where we don't track versions
+                    if not ctv:
+                        # then give it at least a ctv of 1 to differentiate from classes where we don't track versions for
                         ctv = 1
+
                     # add the type version from super; because versions in parents imply them in children
                     ctv = ctv + self.class_args_dict[super_class]['cumulative_version'] -1
-                    # if current class did not have a decorator give it one
-                    #  because we determined that it has one via its super
-                    if not re.match(type_version_pattern,class_decorators):
-                        class_decorators="\"TypeVersion\": \"1\""
-                # for all current classes that have a TypeVersion; upgrade it with information from super
-                # (note the schema yml files  will be modified by a human
-                # who won't manually copy, paste, and add the inherited version info, so we do it)
-                class_decorators=re.sub(type_version_pattern,"TypeVersion\": \""+str(ctv),class_decorators)
+                    # for all current classes that have a TypeVersion; upgrade it with information from super
+                    # (note the schema yml files  will be modified by a human
+                    # who won't manually copy, paste, and add the inherited version info, so we do it)
+                    class_decorators["TypeVersion"]=ctv
 
                 # for current classes that have a ctv (non-zero)
                 if ctv:
                     # save ctv so that its children may inherit
                     self.class_args_dict[current_class]={}
                     self.class_args_dict[current_class]['cumulative_version']=ctv
 
-                self.dataobject_output_text += "    _class_info = {'name': '" + "', 'decorators': {" + class_decorators + \
-                                               "}, 'docstring': '"+ re.escape(class_comment)+"'}\n\n"
+                self.dataobject_output_text += "    _class_info = {'name': '" + "', 'decorators': " + str(class_decorators) + \
+                                               ", 'docstring': '"+ re.escape(class_comment)+"'}\n\n"
                 self.dataobject_output_text += "    def __init__(self"
                 current_class_comment = class_comment
                 class_comment = ''
-                class_decorators = ''
+                class_decorators = {}
         if len(args) > 0:
             for arg in args:
                self.dataobject_output_text += arg

pyon/core/object.py

@@ -424,24 +424,33 @@ class IonObjectSerializer(IonObjectSerializationBase):
 
     Used when being written to Datastore.
     """
-    def _transform(self, obj):
-        if isinstance(obj, IonObjectBase):
-            res = {k:v for k, v in obj.__dict__.iteritems() if k in obj._schema or k in built_in_attrs}
-            if not 'type_' in res:
-                res['type_'] = obj._get_type()
 
-            # update persisted_version if serializing for persistence
-            if 'TypeVersion' in obj._class_info['decorators']:
+    def _transform(self, update_version=False):
 
-                # convert TypeVersion in decorator from string to int
-                # this is a hack because the object_model_generator converts TypeVersion to string
-                res['persisted_version'] = int(obj._class_info['decorators']['TypeVersion'])
+        def _transform(obj):
 
-            return res
+            if isinstance(obj, IonObjectBase):
+                res = {k:v for k, v in obj.__dict__.iteritems() if k in obj._schema or k in built_in_attrs}
+                if not 'type_' in res:
+                    res['type_'] = obj._get_type()
 
-        return obj
+                # update persisted_version if serializing for persistence
+                if update_version and 'TypeVersion' in obj._class_info['decorators']:
+
+                    # convert TypeVersion in decorator from string to int
+                    # because the object_model_generator converts TypeVersion to string
+                    res['persisted_version'] = obj._class_info['decorators']['TypeVersion']
+                return res
+
+            return obj
+        return _transform
+
+
+    def serialize(self, obj, update_version=False):
+
+        self._transform_method = self._transform(update_version)
 
-    serialize = IonObjectSerializationBase.operate
+        return IonObjectSerializationBase.operate(self, obj)
 
 class IonObjectBlameSerializer(IonObjectSerializer):
 

pyon/core/registry.py

@@ -186,6 +186,5 @@ def validating_setattr(self, name, value):
             obj = clzz(**keywordargs)
         else:
             obj = clzz(**kwargs)
-        if hasattr(clzz,'_class_info') and 'TypeVersion' in obj._class_info['decorators']:
-            obj.persisted_version = clzz._class_info['decorators']['TypeVersion']
+
         return obj

pyon/core/test/test_object.py

@@ -58,7 +58,7 @@ def test_persisted_version(self):
         # create an initial version of SampleResource
         io_serializer = IonObjectSerializer()
         obj = IonObject('SampleResource', {'num': 9, 'other_field': 'test value'})
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         self.assertEquals(obj_dict['persisted_version'], 1)
         # verify that the simulated previous version does not have new_attribute
         self.assertEquals('new_attribute' in obj_dict, False)
@@ -78,7 +78,7 @@ def test_persisted_version(self):
         self.assertEquals(obj_dict['persisted_version'], 1)
 
         # simulate update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is updated
         self.assertEquals(obj_dict['persisted_version'], 2)
 
@@ -92,7 +92,7 @@ def test_version_del_attrib(self):
             IonObject('SampleResource_V2', {'num': 9, 'other_field': 'test value','more_new_attribute': {'key':'value'}})
         # simulate creating a version 2 of SampleResource that has "new_attribute"
         obj = IonObject('SampleResource_V2', {'num': 9, 'other_field': 'test value','new_attribute': {'key':'value'}})
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is 2
         self.assertEquals(obj_dict['persisted_version'], 2)
         # verify that the simulated version 2 data does have new_attribute
@@ -119,7 +119,7 @@ def test_version_del_attrib(self):
         self.assertEquals(obj_dict['persisted_version'], 2)
 
         # simulate update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is updated
         self.assertEquals(obj_dict['persisted_version'], 3)
 
@@ -128,7 +128,7 @@ def test_event_version(self):
 
         io_serializer = IonObjectSerializer()
         obj = IonObject('SampleEvent', {'num': 9, 'other_field': 'test value'})
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         self.assertEquals(obj_dict['persisted_version'], 1)
         # simulate a previous version data of SampleEvent_V2
         obj_dict['type_'] = 'SampleEvent_V2'
@@ -148,7 +148,7 @@ def test_event_version(self):
         self.assertEquals(obj_dict['persisted_version'], 1)
 
         # simulate create/update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is updated
         self.assertEquals(obj_dict['persisted_version'], 2)
 
@@ -162,7 +162,7 @@ def test_event_version_del_attrib(self):
             IonObject('SampleEvent_V2', {'num': 9, 'other_field': 'test value','more_new_attribute': {'key':'value'}})
 
         obj = IonObject('SampleEvent_V2', {'num': 9, 'other_field': 'test value','new_attribute': {'key':'value'}})
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         self.assertEquals(obj_dict['persisted_version'], 2)
         # simulate a next version data of SampleEvent_V2
         obj_dict['type_'] = 'SampleEvent_V3'
@@ -186,15 +186,15 @@ def test_event_version_del_attrib(self):
         self.assertEquals(obj_dict['persisted_version'], 2)
 
         # simulate create/update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is updated
         self.assertEquals(obj_dict['persisted_version'], 3)
 
     def test_complex_version(self):
 
         io_serializer = IonObjectSerializer()
         obj = IonObject('SampleComplexEvent', {'num': 9, 'other_field': 'test value'})
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         self.assertEquals(obj_dict['persisted_version'], 1)
         # simulate a previous version data of SampleComplexEvent_V2
         obj_dict['type_'] = 'SampleComplexEvent_V2'
@@ -216,7 +216,7 @@ def test_complex_version(self):
         self.assertEquals(obj_dict['persisted_version'], 1)
 
         # simulate create/update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is updated
         self.assertEquals(obj_dict['persisted_version'], 2)
 
@@ -234,7 +234,7 @@ def test_complex_version_del_attrib(self):
 
         obj = IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','new_resource': {'num': 9, 'other_field': 'test value','new_attribute':{'key':'value'}}})
         # create simulated saved data
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         self.assertEquals(obj_dict['persisted_version'], 2)
         # simulate a next version data of SampleComplexEvent_V2
         obj_dict['type_'] = 'SampleComplexEvent_V3'
@@ -271,7 +271,7 @@ def test_complex_version_del_attrib(self):
 
 
         # simulate create/update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that version is updated
         self.assertEquals(obj_dict['persisted_version'], 3)
         # verify that version is updated fo the subsumed object
@@ -290,7 +290,7 @@ def test_attribute_version(self):
 
         obj = IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','new_resource':
             {'num': 9, 'other_field': 'test value','new_attribute':{'key':'value'}}})
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         self.assertEquals(obj_dict['persisted_version'], 2)
 
         # verify that the simulated previous version data does have new_resource
@@ -325,7 +325,7 @@ def test_attribute_version(self):
         self.assertEquals(obj.new_resource.persisted_version, 2)
 
         # simulate create/update
-        obj_dict = io_serializer.serialize(obj)
+        obj_dict = io_serializer.serialize(obj,True)
         # verify that versions are unchanged
         self.assertEquals(obj_dict['persisted_version'], 3)
         # verify that versions are updated in the subsumed object

pyon/datastore/postgresql/datastore.py

@@ -720,7 +720,7 @@ def _ion_object_to_persistence_dict(self, ion_object):
         if ion_object is None:
             return None
 
-        obj_dict = self._io_serializer.serialize(ion_object)
+        obj_dict = self._io_serializer.serialize(ion_object, update_version=True)
         return obj_dict
 
     def _persistence_dict_to_ion_object(self, obj_dict):