Add support for data_relation with late document versioning

eve/io/mongo/validation.py

@@ -17,7 +17,8 @@
 from flask import current_app as app
 from cerberus import Validator
 from werkzeug.datastructures import FileStorage
-from eve.versioning import get_data_version_relation_document
+from eve.versioning import get_data_version_relation_document, \
+    missing_version_field
 
 
 class Validator(Validator):
@@ -131,8 +132,14 @@ def _validate_data_relation(self, data_relation, field, value):
                         " data_relation if '%s' isn't versioned" %
                         data_relation['resource'])
                 else:
-                    search = get_data_version_relation_document(data_relation,
-                                                                value)
+                    # support late versioning
+                    if value[version_field] == 0:
+                        # there is a chance this document hasn't been saved
+                        # since versioning was turned on
+                        search = missing_version_field(data_relation, value)
+                    else:
+                        search = get_data_version_relation_document(
+                            data_relation, value)
                     if not search:
                         self._error(
                             field, "value '%s' must exist in resource"

eve/methods/get.py

@@ -23,7 +23,7 @@
     querydef, config, debug_error_message, resource_uri
 from eve.versioning import resolve_document_version, \
     synthesize_versioned_document, versioned_id_field, get_old_document, \
-    diff_document, get_data_version_relation_document
+    diff_document, get_data_version_relation_document, missing_version_field
 from flask import current_app as app, abort, request
 
 
@@ -413,13 +413,39 @@ def _resolve_embedded_documents(document, resource, embedded_fields):
         data_relation = schema[field]['data_relation']
         # Retrieve and serialize the requested document
         if 'version' in data_relation and data_relation['version'] is True:
-            # grab the specific version
-            embedded_doc = get_data_version_relation_document(
-                data_relation, document[field])
-
-            # grab the latest version
-            latest_embedded_doc = get_data_version_relation_document(
-                data_relation, document[field], latest=True)
+            # support late versioning
+            if document[field][config.VERSION] == 0:
+                # there is a chance this document hasn't been saved
+                # since versioning was turned on
+                embedded_doc = missing_version_field(
+                    data_relation, document[field])
+
+                if embedded_doc is None:
+                    # this document has been saved since the data_relation was
+                    # made - we basically do not have the copy of the document
+                    # that existed when the data relation was made, but we'll
+                    # try the next best thing - the first version
+                    document[field][config.VERSION] = 1
+                    embedded_doc = get_data_version_relation_document(
+                        data_relation, document[field])
+
+                latest_embedded_doc = embedded_doc
+            else:
+                # grab the specific version
+                embedded_doc = get_data_version_relation_document(
+                    data_relation, document[field])
+
+                # grab the latest version
+                latest_embedded_doc = get_data_version_relation_document(
+                    data_relation, document[field], latest=True)
+
+            # make sure we got the documents
+            if embedded_doc is None or latest_embedded_doc is None:
+                # your database is not consistent!!! that is bad
+                abort(404, description=debug_error_message(
+                    "Unable to locate embedded documents for '%s'" %
+                    field
+                ))
 
             # build the response document
             _build_response_document(

eve/tests/versioning.py

@@ -39,6 +39,26 @@ def enableVersioning(self, partial=False):
             settings['datasource'].pop('projection', None)
             self.app.register_resource(resource, settings)
 
+    def enableDataVersionRelation(self, embeddable=True):
+        field = {
+            'type': 'dict',
+            'schema': {
+                self.app.config['VERSION']: {'type': 'integer'}
+            },
+            'data_relation': {
+                'version': True,
+                'resource': 'contacts'
+            }
+        }
+        if embeddable is True:
+            field['schema'][self.app.config['ID_FIELD']] = {'type': 'objectid'}
+            field['data_relation']['embeddable'] = True
+            #field['data_relation']['field'] = '_id' is auto filled
+        else:
+            field['schema']['ref'] = {'type': 'string'}
+            field['data_relation']['field'] = 'ref'
+        self.domain['invoices']['schema']['person'] = field
+
     def assertEqualFields(self, obj1, obj2, fields):
         for field in fields:
             self.assertEqual(obj1[field], obj2[field])
@@ -534,20 +554,7 @@ def setUp(self):
         super(TestDataRelationVersionNotVersioned, self).setUp()
 
         # enable versioning in the invoice data_relation definition
-        invoice_schema = self.domain['invoices']['schema']
-        invoice_schema['person'] = {
-            'type': 'dict',
-            'schema': {
-                '_id': {'type': 'objectid'},
-                self.app.config['VERSION']: {'type': 'integer'}
-            },
-            'data_relation': {
-                'version': True,
-                'resource': 'contacts',
-                'embeddable': True
-                #'field': '_id' is auto filled
-            }
-        }
+        self.enableDataVersionRelation(embeddable=True)
 
         # turn on version after data has been inserted into the db
         self.enableVersioning()
@@ -638,7 +645,7 @@ def test_embedded(self):
         # add embeddable data relation
         data = {"person": {value_field: self.item_id, self.version_field: 1}}
         response, status = self.post('/invoices/', data=data)
-        invoice_id = response[self.app.config['ID_FIELD']]
+        invoice_id = response[value_field]
         self.assert201(status)
 
         # test that it works
@@ -654,19 +661,7 @@ def setUp(self):
         super(TestDataRelationVersionVersioned, self).setUp()
 
         # enable versioning in the invoice data_relation definition
-        invoice_schema = self.domain['invoices']['schema']
-        invoice_schema['person'] = {
-            'type': 'dict',
-            'schema': {
-                'ref': {'type': 'string'},
-                self.app.config['VERSION']: {'type': 'integer'}
-            },
-            'data_relation': {
-                'version': True,
-                'resource': 'contacts',
-                'field': 'ref'
-            }
-        }
+        self.enableDataVersionRelation(embeddable=False)
 
         # turn on version after data has been inserted into the db
         self.enableVersioning()
@@ -755,6 +750,9 @@ class TestLateVersioning(TestVersioningBase):
     def setUp(self):
         super(TestLateVersioning, self).setUp()
 
+        # enable versioning in the invoice data_relation definition
+        self.enableDataVersionRelation(embeddable=True)
+
         # turn on version after data has been inserted into the db
         self.enableVersioning()
 
@@ -852,4 +850,47 @@ def test_referential_integrity(self):
         version 0 of a document. This should only be allowed if the shadow
         collection it empty.
         """
-        pass  # TODO
+        data_relation = \
+            self.domain['invoices']['schema']['person']['data_relation']
+        value_field = data_relation['field']
+        version_field = self.app.config['VERSION']
+
+        # verify that Eve will take version = 0 if no shadow docs exist
+        data = {"person": {value_field: self.item_id, version_field: 0}}
+        response, status = self.post('/invoices/', data=data)
+        invoice_id = response[value_field]
+        self.assert201(status)
+
+        # verify that we can embed across the data_relation w/ version = 0
+        response, status = self.get(
+            self.domain['invoices']['url'],
+            item=invoice_id, query='?embedded={"person": 1}')
+        self.assert200(status)
+        self.assertTrue('ref' in response['person'])
+
+        # put a change to the document (will be version = 1)
+        changes = {"ref": "this is a different value"}
+        response, status = self.put(self.item_id_url, data=changes,
+                                    headers=[('If-Match', self.item_etag)])
+        self.assertGoodPutPatch(response, status)
+        self.assertDocumentVersions(response, 1)
+
+        # verify that Eve will not take version = 0 now
+        data = {"person": {value_field: self.item_id, version_field: 0}}
+        r, status = self.post('/invoices/', data=data)
+        self.assert200(status)
+        self.assertValidationError(
+            r, {'person': "value '%s' must exist in "
+                "resource '%s', field '%s' at version '%s'." %
+                (self.item_id, 'contacts', value_field, 0)})
+
+        # verify that we can still embed with out-of-date version = 0
+        response, status = self.get(
+            self.domain['invoices']['url'],
+            item=invoice_id, query='?embedded={"person": 1}')
+        self.assert200(status)
+        self.assertTrue('ref' in response['person'])
+
+        # The test for data_relation with version == 1 and embedding across a
+        # data relation with version > 0 is the normal behavior. This is tested
+        # in TestDataRelationVersionNotVersioned.test_referential_integrity().

eve/versioning.py

@@ -282,3 +282,23 @@ def get_data_version_relation_document(data_relation, reference, latest=False):
             query[version_field] = {'$gte': reference[version_field]}
 
     return app.data.find_one(collection, None, **query)
+
+
+def missing_version_field(data_relation, reference):
+    """ Returns a document if it matches the value_field but doesn't have a
+    _version field. This is the scenario when there is data in the database
+    before document versioning is turned on.
+
+    :param data_relation: the schema definition describing the data_relation.
+    :param reference: a dictionary with a value_field and a version_field.
+
+    .. versionadded:: 0.4
+    """
+    value_field = data_relation['field']
+    version_field = app.config['VERSION']
+    collection = data_relation['resource']
+    query = {}
+    query[value_field] = reference[value_field]
+    query[version_field] = {'$exists': False}
+
+    return app.data.find_one(collection, None, **query)