Backwards compatibility with older style structured properties.

src/google/cloud/ndb/model.py

@@ -514,7 +514,7 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
 
     Args:
         ds_entity (google.cloud.datastore_v1.types.Entity): An entity to be
-        deserialized.
+            deserialized.
 
     Returns:
         .Model: The deserialized entity.
@@ -523,8 +523,47 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
     entity = model_class()
     if ds_entity.key:
         entity._key = key_module.Key._from_ds_key(ds_entity.key)
+
     for name, value in ds_entity.items():
         prop = getattr(model_class, name, None)
+
+        # Backwards compatibility shim. NDB previously stored structured
+        # properties as sets of dotted name properties. Datastore now has
+        # native support for embedded entities and NDB now uses that, by
+        # default. This handles the case of reading structured properties from
+        # older NDB datastore instances.
+        if prop is None and "." in name:
+            supername, subname = name.split(".", 1)
+            structprop = getattr(model_class, supername, None)
+            if isinstance(structprop, StructuredProperty):
+                subvalue = value
+                value = structprop._get_base_value(entity)
+                if value in (None, []):  # empty list for repeated props
+                    kind = structprop._model_class._get_kind()
+                    key = key_module.Key(kind, None)
+                    if structprop._repeated:
+                        value = [
+                            _BaseValue(entity_module.Entity(key._key))
+                            for _ in subvalue
+                        ]
+                    else:
+                        value = entity_module.Entity(key._key)
+                        value = _BaseValue(value)
+
+                    structprop._store_value(entity, value)
+
+                if structprop._repeated:
+                    # Branch coverage bug,
+                    # See: https://github.com/nedbat/coveragepy/issues/817
+                    for subentity, subsubvalue in zip(  # pragma no branch
+                        value, subvalue
+                    ):
+                        subentity.b_val.update({subname: subsubvalue})
+                else:
+                    value.b_val.update({subname: subvalue})
+
+                continue
+
         if not (prop is not None and isinstance(prop, Property)):
             if value is not None and isinstance(  # pragma: no branch
                 entity, Expando
@@ -538,6 +577,7 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
                     value = _BaseValue(value)
                 setattr(entity, name, value)
             continue
+
         if value is not None:
             if prop._repeated:
                 value = [
@@ -546,6 +586,7 @@ def _entity_from_ds_entity(ds_entity, model_class=None):
                 ]
             else:
                 value = _BaseValue(value)
+
         prop._store_value(entity, value)
 
     return entity

src/google/cloud/ndb/query.py

@@ -242,12 +242,41 @@ def __init__(self, name, match_keys, entity_pb):
         self.match_values = [entity_pb.properties[key] for key in match_keys]
 
     def __call__(self, entity_pb):
-        subentities = entity_pb.properties.get(self.name).array_value.values
-        for subentity in subentities:
-            properties = subentity.entity_value.properties
-            values = [properties.get(key) for key in self.match_keys]
-            if values == self.match_values:
-                return True
+        prop_pb = entity_pb.properties.get(self.name)
+        if prop_pb:
+            subentities = prop_pb.array_value.values
+            for subentity in subentities:
+                properties = subentity.entity_value.properties
+                values = [properties.get(key) for key in self.match_keys]
+                if values == self.match_values:
+                    return True
+
+        else:
+            # Backwards compatibility. Legacy NDB, rather than using
+            # Datastore's ability to embed subentities natively, used dotted
+            # property names.
+            prefix = self.name + "."
+            subentities = ()
+            for prop_name, prop_pb in entity_pb.properties.items():
+                if not prop_name.startswith(prefix):
+                    continue
+
+                subprop_name = prop_name.split(".", 1)[1]
+                if not subentities:
+                    subentities = [
+                        {subprop_name: value}
+                        for value in prop_pb.array_value.values
+                    ]
+                else:
+                    for subentity, value in zip(
+                        subentities, prop_pb.array_value.values
+                    ):
+                        subentity[subprop_name] = value
+
+            for subentity in subentities:
+                values = [subentity.get(key) for key in self.match_keys]
+                if values == self.match_values:
+                    return True
 
         return False
 

tests/system/test_crud.py

@@ -417,6 +417,59 @@ class SomeKind(ndb.Model):
     dispose_of(key._key)
 
 
+@pytest.mark.usefixtures("client_context")
+def test_retrieve_entity_with_legacy_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND, entity_id, **{"foo": 42, "bar.one": "hi", "bar.two": "mom"}
+    )
+
+    key = ndb.Key(KIND, entity_id)
+    retrieved = key.get()
+    assert retrieved.foo == 42
+    assert retrieved.bar.one == "hi"
+    assert retrieved.bar.two == "mom"
+
+    assert isinstance(retrieved.bar, OtherKind)
+
+
+@pytest.mark.usefixtures("client_context")
+def test_retrieve_entity_with_legacy_repeated_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind, repeated=True)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{"foo": 42, "bar.one": ["hi", "hello"], "bar.two": ["mom", "dad"]}
+    )
+
+    key = ndb.Key(KIND, entity_id)
+    retrieved = key.get()
+    assert retrieved.foo == 42
+    assert retrieved.bar[0].one == "hi"
+    assert retrieved.bar[0].two == "mom"
+    assert retrieved.bar[1].one == "hello"
+    assert retrieved.bar[1].two == "dad"
+
+    assert isinstance(retrieved.bar[0], OtherKind)
+    assert isinstance(retrieved.bar[1], OtherKind)
+
+
 @pytest.mark.usefixtures("client_context")
 def test_insert_expando(dispose_of):
     class SomeKind(ndb.Expando):

tests/system/test_query.py

@@ -603,6 +603,58 @@ def make_entities():
     assert results[1].foo == 2
 
 
+@pytest.mark.skip("Requires an index")
+@pytest.mark.usefixtures("client_context")
+def test_query_legacy_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+        three = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{"foo": 1, "bar.one": "pish", "bar.two": "posh", "bar.three": "pash"}
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{"foo": 2, "bar.one": "pish", "bar.two": "posh", "bar.three": "push"}
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 3,
+            "bar.one": "pish",
+            "bar.two": "moppish",
+            "bar.three": "pass the peas",
+        }
+    )
+
+    eventually(SomeKind.query().fetch, _length_equals(3))
+
+    query = (
+        SomeKind.query()
+        .filter(SomeKind.bar.one == "pish", SomeKind.bar.two == "posh")
+        .order(SomeKind.foo)
+    )
+
+    results = query.fetch()
+    assert len(results) == 2
+    assert results[0].foo == 1
+    assert results[1].foo == 2
+
+
 @pytest.mark.skip("Requires an index")
 @pytest.mark.usefixtures("client_context")
 def test_query_repeated_structured_property_with_properties(dispose_of):
@@ -723,3 +775,67 @@ def make_entities():
     results = query.fetch()
     assert len(results) == 1
     assert results[0].foo == 1
+
+
+@pytest.mark.skip("Requires an index")
+@pytest.mark.usefixtures("client_context")
+def test_query_legacy_repeated_structured_property(ds_entity):
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+        three = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.StructuredProperty(OtherKind, repeated=True)
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 1,
+            "bar.one": ["pish", "bish"],
+            "bar.two": ["posh", "bosh"],
+            "bar.three": ["pash", "bash"],
+        }
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 2,
+            "bar.one": ["bish", "pish"],
+            "bar.two": ["bosh", "posh"],
+            "bar.three": ["bass", "pass"],
+        }
+    )
+
+    entity_id = test_utils.system.unique_resource_id()
+    ds_entity(
+        KIND,
+        entity_id,
+        **{
+            "foo": 3,
+            "bar.one": ["pish", "bish"],
+            "bar.two": ["fosh", "posh"],
+            "bar.three": ["fash", "bash"],
+        }
+    )
+
+    eventually(SomeKind.query().fetch, _length_equals(3))
+
+    query = (
+        SomeKind.query()
+        .filter(
+            SomeKind.bar == OtherKind(one="pish", two="posh"),
+            SomeKind.bar == OtherKind(two="posh", three="pash"),
+        )
+        .order(SomeKind.foo)
+    )
+
+    results = query.fetch()
+    assert len(results) == 1
+    assert results[0].foo == 1

tests/unit/test_model.py

@@ -4351,6 +4351,66 @@ class ThisKind(model.Model):
         assert entity._key.kind() == "ThisKind"
         assert entity._key.id() == 123
 
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    def test_legacy_structured_property():
+        class OtherKind(model.Model):
+            foo = model.IntegerProperty()
+            bar = model.StringProperty()
+
+        class ThisKind(model.Model):
+            baz = model.StructuredProperty(OtherKind)
+            copacetic = model.BooleanProperty()
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        datastore_entity.update(
+            {
+                "baz.foo": 42,
+                "baz.bar": "himom",
+                "copacetic": True,
+                "super.fluous": "whocares?",
+            }
+        )
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        assert isinstance(entity, ThisKind)
+        assert entity.baz.foo == 42
+        assert entity.baz.bar == "himom"
+        assert entity.copacetic is True
+
+        assert not hasattr(entity, "super")
+        assert not hasattr(entity, "super.fluous")
+
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    def test_legacy_repeated_structured_property():
+        class OtherKind(model.Model):
+            foo = model.IntegerProperty()
+            bar = model.StringProperty()
+
+        class ThisKind(model.Model):
+            baz = model.StructuredProperty(OtherKind, repeated=True)
+            copacetic = model.BooleanProperty()
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        datastore_entity.update(
+            {
+                "baz.foo": [42, 144],
+                "baz.bar": ["himom", "hellodad"],
+                "copacetic": True,
+            }
+        )
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        assert isinstance(entity, ThisKind)
+        assert entity.baz[0].foo == 42
+        assert entity.baz[0].bar == "himom"
+        assert entity.baz[1].foo == 144
+        assert entity.baz[1].bar == "hellodad"
+        assert entity.copacetic is True
+
 
 class Test_entity_to_protobuf:
     @staticmethod