Backwards compatibility with older style structured properties. (#126)

When implementing structured properties the first time, I just used
Datastore's native embedded entity functionality, not realizing that NDB
had originally used dotted property names instead. (Probably GAE
Datastore didn't have embedded entities when NDB was originally
written.) The problem is that users migrating from GAE NDB can't load
entities with structured properties from their existing datastore. This
PR makes NDB backwards compatible with older, dotted name style
structured properties so that existing repositories still work with the
new NDB.

Fixes #122.

src/google/cloud/ndb/model.py

@@ -514,7 +514,7 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
 
     Args:
         ds_entity (google.cloud.datastore_v1.types.Entity): An entity to be
-        deserialized.
+            deserialized.
 
     Returns:
         .Model: The deserialized entity.
@@ -523,8 +523,47 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
     entity = model_class()
     if ds_entity.key:
         entity._key = key_module.Key._from_ds_key(ds_entity.key)
+
     for name, value in ds_entity.items():
         prop = getattr(model_class, name, None)
+
+        # Backwards compatibility shim. NDB previously stored structured
+        # properties as sets of dotted name properties. Datastore now has
+        # native support for embedded entities and NDB now uses that, by
+        # default. This handles the case of reading structured properties from
+        # older NDB datastore instances.
+        if prop is None and "." in name:
+            supername, subname = name.split(".", 1)
+            structprop = getattr(model_class, supername, None)
+            if isinstance(structprop, StructuredProperty):
+                subvalue = value
+                value = structprop._get_base_value(entity)
+                if value in (None, []):  # empty list for repeated props
+                    kind = structprop._model_class._get_kind()
+                    key = key_module.Key(kind, None)
+                    if structprop._repeated:
+                        value = [
+                            _BaseValue(entity_module.Entity(key._key))
+                            for _ in subvalue
+                        ]
+                    else:
+                        value = entity_module.Entity(key._key)
+                        value = _BaseValue(value)
+
+                    structprop._store_value(entity, value)
+
+                if structprop._repeated:
+                    # Branch coverage bug,
+                    # See: https://github.com/nedbat/coveragepy/issues/817
+                    for subentity, subsubvalue in zip(  # pragma no branch
+                        value, subvalue
+                    ):
+                        subentity.b_val.update({subname: subsubvalue})
+                else:
+                    value.b_val.update({subname: subvalue})
+
+                continue
+
         if not (prop is not None and isinstance(prop, Property)):
             if value is not None and isinstance(  # pragma: no branch
                 entity, Expando
@@ -538,6 +577,7 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
                     value = _BaseValue(value)
                 setattr(entity, name, value)
             continue
+
         if value is not None:
             if prop._repeated:
                 value = [
@@ -546,6 +586,7 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
                 ]
             else:
                 value = _BaseValue(value)
+
         prop._store_value(entity, value)
 
     return entity

src/google/cloud/ndb/query.py

@@ -242,12 +242,41 @@ def __init__(self, name, match_keys, entity_pb):
         self.match_values = [entity_pb.properties[key] for key in match_keys]
 
     def __call__(self, entity_pb):
-        subentities = entity_pb.properties.get(self.name).array_value.values
-        for subentity in subentities:
-            properties = subentity.entity_value.properties
-            values = [properties.get(key) for key in self.match_keys]
-            if values == self.match_values:
-                return True
+        prop_pb = entity_pb.properties.get(self.name)
+        if prop_pb:
+            subentities = prop_pb.array_value.values
+            for subentity in subentities:
+                properties = subentity.entity_value.properties
+                values = [properties.get(key) for key in self.match_keys]
+                if values == self.match_values:
+                    return True
+
+        else:
+            # Backwards compatibility. Legacy NDB, rather than using
+            # Datastore's ability to embed subentities natively, used dotted
+            # property names.
+            prefix = self.name + "."
+            subentities = ()
+            for prop_name, prop_pb in entity_pb.properties.items():
+                if not prop_name.startswith(prefix):
+                    continue
+
+                subprop_name = prop_name.split(".", 1)[1]
+                if not subentities:
+                    subentities = [
+                        {subprop_name: value}
+                        for value in prop_pb.array_value.values
+                    ]
+                else:
+                    for subentity, value in zip(
+                        subentities, prop_pb.array_value.values
+                    ):
+                        subentity[subprop_name] = value
+
+            for subentity in subentities:
+                values = [subentity.get(key) for key in self.match_keys]
+                if values == self.match_values:
+                    return True
 
         return False
 

tests/system/test_crud.py

@@ -417,6 +417,59 @@ class SomeKind(ndb.Model):
     dispose_of(key._key)
 
 
+@pytest.mark.usefixtures("client_context")
+def test_retrieve_entity_with_legacy_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND, entity_id, **{"foo": 42, "bar.one": "hi", "bar.two": "mom"}
+    )
+
+    key = ndb.Key(KIND, entity_id)
+    retrieved = key.get()
+    assert retrieved.foo == 42
+    assert retrieved.bar.one == "hi"
+    assert retrieved.bar.two == "mom"
+
+    assert isinstance(retrieved.bar, OtherKind)
+
+
+@pytest.mark.usefixtures("client_context")
+def test_retrieve_entity_with_legacy_repeated_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind, repeated=True)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{"foo": 42, "bar.one": ["hi", "hello"], "bar.two": ["mom", "dad"]}
+    )
+
+    key = ndb.Key(KIND, entity_id)
+    retrieved = key.get()
+    assert retrieved.foo == 42
+    assert retrieved.bar[0].one == "hi"
+    assert retrieved.bar[0].two == "mom"
+    assert retrieved.bar[1].one == "hello"
+    assert retrieved.bar[1].two == "dad"
+
+    assert isinstance(retrieved.bar[0], OtherKind)
+    assert isinstance(retrieved.bar[1], OtherKind)
+
+
 @pytest.mark.usefixtures("client_context")
 def test_insert_expando(dispose_of):
     class SomeKind(ndb.Expando):

tests/system/test_query.py

@@ -603,6 +603,58 @@ def make_entities():
     assert results[1].foo == 2
 
 
+@pytest.mark.skip("Requires an index")
+@pytest.mark.usefixtures("client_context")
+def test_query_legacy_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+        three = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{"foo": 1, "bar.one": "pish", "bar.two": "posh", "bar.three": "pash"}
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{"foo": 2, "bar.one": "pish", "bar.two": "posh", "bar.three": "push"}
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 3,
+            "bar.one": "pish",
+            "bar.two": "moppish",
+            "bar.three": "pass the peas",
+        }
+    )
+
+    eventually(SomeKind.query().fetch, _length_equals(3))
+
+    query = (
+        SomeKind.query()
+        .filter(SomeKind.bar.one == "pish", SomeKind.bar.two == "posh")
+        .order(SomeKind.foo)
+    )
+
+    results = query.fetch()
+    assert len(results) == 2
+    assert results[0].foo == 1
+    assert results[1].foo == 2
+
+
 @pytest.mark.skip("Requires an index")
 @pytest.mark.usefixtures("client_context")
 def test_query_repeated_structured_property_with_properties(dispose_of):
@@ -723,3 +775,67 @@ def make_entities():
     results = query.fetch()
     assert len(results) == 1
     assert results[0].foo == 1
+
+
+@pytest.mark.skip("Requires an index")
+@pytest.mark.usefixtures("client_context")
+def test_query_legacy_repeated_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+        three = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind, repeated=True)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 1,
+            "bar.one": ["pish", "bish"],
+            "bar.two": ["posh", "bosh"],
+            "bar.three": ["pash", "bash"],
+        }
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 2,
+            "bar.one": ["bish", "pish"],
+            "bar.two": ["bosh", "posh"],
+            "bar.three": ["bass", "pass"],
+        }
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 3,
+            "bar.one": ["pish", "bish"],
+            "bar.two": ["fosh", "posh"],
+            "bar.three": ["fash", "bash"],
+        }
+    )
+
+    eventually(SomeKind.query().fetch, _length_equals(3))
+
+    query = (
+        SomeKind.query()
+        .filter(
+            SomeKind.bar == OtherKind(one="pish", two="posh"),
+            SomeKind.bar == OtherKind(two="posh", three="pash"),
+        )
+        .order(SomeKind.foo)
+    )
+
+    results = query.fetch()
+    assert len(results) == 1
+    assert results[0].foo == 1

tests/unit/test_model.py

@@ -4351,6 +4351,66 @@ class ThisKind(model.Model):
         assert entity._key.kind() == "ThisKind"
         assert entity._key.id() == 123
 
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    def test_legacy_structured_property():
+        class OtherKind(model.Model):
+            foo = model.IntegerProperty()
+            bar = model.StringProperty()
+
+        class ThisKind(model.Model):
+            baz = model.StructuredProperty(OtherKind)
+            copacetic = model.BooleanProperty()
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        datastore_entity.update(
+            {
+                "baz.foo": 42,
+                "baz.bar": "himom",
+                "copacetic": True,
+                "super.fluous": "whocares?",
+            }
+        )
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        assert isinstance(entity, ThisKind)
+        assert entity.baz.foo == 42
+        assert entity.baz.bar == "himom"
+        assert entity.copacetic is True
+
+        assert not hasattr(entity, "super")
+        assert not hasattr(entity, "super.fluous")
+
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    def test_legacy_repeated_structured_property():
+        class OtherKind(model.Model):
+            foo = model.IntegerProperty()
+            bar = model.StringProperty()
+
+        class ThisKind(model.Model):
+            baz = model.StructuredProperty(OtherKind, repeated=True)
+            copacetic = model.BooleanProperty()
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        datastore_entity.update(
+            {
+                "baz.foo": [42, 144],
+                "baz.bar": ["himom", "hellodad"],
+                "copacetic": True,
+            }
+        )
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        assert isinstance(entity, ThisKind)
+        assert entity.baz[0].foo == 42
+        assert entity.baz[0].bar == "himom"
+        assert entity.baz[1].foo == 144
+        assert entity.baz[1].bar == "hellodad"
+        assert entity.copacetic is True
+
 
 class Test_entity_to_protobuf:
     @staticmethod