Use fields for keys/values in DictField

tests/test_data_proxy.py

@@ -462,16 +462,22 @@ class MyEmbedded(EmbeddedDocument):
 
         class MySchema(EmbeddedSchema):
             # EmbeddedField need instance to retrieve implementation
-            listed = fields.ListField(fields.EmbeddedField(MyEmbedded, instance=self.instance))
-            embedded = fields.EmbeddedField(MyEmbedded, instance=self.instance)
             required = fields.IntField(required=True)
+            embedded = fields.EmbeddedField(MyEmbedded, instance=self.instance)
+            listed = fields.ListField(fields.EmbeddedField(MyEmbedded, instance=self.instance))
+            dicted = fields.DictField(fields.EmbeddedField(MyEmbedded, instance=self.instance))
 
         MyDataProxy = data_proxy_factory('My', MySchema())
         d = MyDataProxy()
 
-        d.load({'embedded': {'required': 42}, 'required': 42, 'listed': [{'required': 42}]})
+        d.load({
+            'required': 42,
+            'embedded': {'required': 42},
+            'listed': [{'required': 42}],
+            'dicted': {'a': {'required': 42}}
+        })
         d.required_validate()
-        # Empty list should not trigger required if embedded field has required fields
+        # Empty list/dict should not trigger required if embedded field has required fields
         d.load({'embedded': {'required': 42}, 'required': 42})
         d.required_validate()
 
@@ -489,10 +495,13 @@ class MySchema(EmbeddedSchema):
             d.required_validate()
         assert exc.value.messages == {'embedded': {'required': ['Missing data for required field.']}}
 
-        d.load({'embedded': {'required': 42}, 'required': 42, 'listed': [{}]})
+        d.load({'embedded': {'required': 42}, 'required': 42, 'listed': [{}], 'dicted': {'a': {}}})
         with pytest.raises(ValidationError) as exc:
             d.required_validate()
-        assert exc.value.messages == {'listed': {0: {'required': ['Missing data for required field.']}}}
+        assert exc.value.messages == {
+            'listed': {0: {'required': ['Missing data for required field.']}},
+            'dicted': {'a': {'value': {'required': ['Missing data for required field.']}}},
+        }
 
     def test_unkown_field_in_db(self):
         class MySchema(EmbeddedSchema):

tests/test_fields.py

@@ -45,8 +45,10 @@ class MyDoc(Document):
             embedded = fields.EmbeddedField(MyEmbedded)
             embedded_required = fields.EmbeddedField(MyEmbedded, required=True)
             embedded_list = fields.ListField(fields.EmbeddedField(MyEmbedded))
+            embedded_dict = fields.DictField(fields.EmbeddedField(MyEmbedded))
 
-        MyDoc(embedded={}, embedded_list=[{}])  # Required fields are check on commit
+        # Required fields are checked on commit
+        MyDoc(embedded={}, embedded_list=[{}], embedded_dict={'a': {}})
         # Don't check required fields in not required missing embedded
         MyDoc(embedded_required={'required_field': 42}).required_validate()
         # Now trigger required fails
@@ -57,17 +59,32 @@ class MyDoc(Document):
             MyDoc(embedded_required={'optional_field': 1}).required_validate()
         assert exc.value.messages == {'embedded_required': {'required_field': ['Missing data for required field.']}}
         with pytest.raises(ValidationError) as exc:
-            MyDoc(embedded={'optional_field': 1},
-                  embedded_required={'required_field': 42}).required_validate()
+            MyDoc(
+                embedded={'optional_field': 1},
+                embedded_required={'required_field': 42}
+            ).required_validate()
         assert exc.value.messages == {'embedded': {'required_field': ['Missing data for required field.']}}
         with pytest.raises(ValidationError) as exc:
-            MyDoc(embedded={'required_field': 1}, embedded_list=[{'optional_field': 1}], embedded_required={'required_field': 42}).required_validate()
-        assert exc.value.messages == {'embedded_list': {0: {'required_field': ['Missing data for required field.']}}}
+            MyDoc(
+                embedded={'required_field': 1},
+                embedded_list=[{'optional_field': 1}],
+                embedded_dict={'a': {'optional_field': 1}},
+                embedded_required={'required_field': 42}
+            ).required_validate()
+        assert exc.value.messages == {
+            'embedded_list': {0: {'required_field': ['Missing data for required field.']}},
+            'embedded_dict': {'a': {'value': {'required_field': ['Missing data for required field.']}}},
+        }
 
         # Check valid constructions
-        doc = MyDoc(embedded={'required_field': 1}, embedded_list=[], embedded_required={'required_field': 42})
+        doc = MyDoc(embedded={'required_field': 1}, embedded_list=[], embedded_dict={}, embedded_required={'required_field': 42})
         doc.required_validate()
-        doc = MyDoc(embedded={'required_field': 1}, embedded_list=[{'required_field': 1}], embedded_required={'required_field': 42})
+        doc = MyDoc(
+            embedded={'required_field': 1},
+            embedded_list=[{'required_field': 1}],
+            embedded_dict={'a': {'required_field': 1}},
+            embedded_required={'required_field': 42},
+        )
         doc.required_validate()
 
 
@@ -226,28 +243,32 @@ class MySchema(EmbeddedSchema):
     def test_dict(self):
 
         class MySchema(Schema):
-            dict = fields.DictField(attribute='in_mongo_dict', allow_none=True)
+            dict = fields.DictField(fields.IntField, attribute='in_mongo_dict', allow_none=True)
 
         MyDataProxy = data_proxy_factory('My', MySchema())
         d = MyDataProxy()
-        d.from_mongo({'in_mongo_dict': {'a': 1, 'b': {'c': True}}})
+        d.from_mongo({'in_mongo_dict': {'a': 1, 'b': 2}})
         with pytest.raises(KeyError):
             d.get('in_mongo_dict')
-        assert d.dump() == {'dict': {'a': 1, 'b': {'c': True}}}
-        assert d.get('dict') == {'a': 1, 'b': {'c': True}}
-        assert d.to_mongo() == {'in_mongo_dict': {'a': 1, 'b': {'c': True}}}
+        assert d.dump() == {'dict': {'a': 1, 'b': 2}}
+        assert d.get('dict') == {'a': 1, 'b': 2}
+        assert d.to_mongo() == {'in_mongo_dict': {'a': 1, 'b': 2}}
 
-        # Must manually set_dirty to take the changes into account
         dict_ = d.get('dict')
         dict_['a'] = 1
-        assert d.to_mongo(update=True) is None
-        dict_.set_modified()
-        assert d.to_mongo(update=True) == {'$set': {'in_mongo_dict': {'a': 1, 'b': {'c': True}}}}
+        assert d.to_mongo(update=True) == {'$set': {'in_mongo_dict': {'a': 1, 'b': 2}}}
         dict_.clear_modified()
         assert d.to_mongo(update=True) is None
 
-        d2 = MyDataProxy({'dict': {'a': 1, 'b': {'c': True}}})
-        assert d2.to_mongo() == {'in_mongo_dict': {'a': 1, 'b': {'c': True}}}
+        # Test repr readability
+        repr_d = repr(d.get('dict'))
+        assert any(
+            repr_d == "<object umongo.data_objects.Dict({})>".format(d)
+            for d in ("{'a': 1, 'b': 2}", "{'b': 2, 'a': 1}")
+        )
+
+        d2 = MyDataProxy({'dict': {'a': 1, 'b': 2}})
+        assert d2.to_mongo() == {'in_mongo_dict': {'a': 1, 'b': 2}}
 
         d2.set('dict', {})
         assert d2.to_mongo() == {'in_mongo_dict': {}}
@@ -264,11 +285,9 @@ class MySchema(Schema):
 
         d3.from_mongo({'in_mongo_dict': {}})
         assert d3._data.get('in_mongo_dict') == {}
-        d3.get('dict')['field'] = 'value'
-        assert d3.to_mongo(update=True) is None
-        d3.get('dict').set_modified()
-        assert d3.to_mongo(update=True) == {'$set': {'in_mongo_dict': {'field': 'value'}}}
-        assert d3.to_mongo() == {'in_mongo_dict': {'field': 'value'}}
+        d3.get('dict')['c'] = 3
+        assert d3.to_mongo(update=True) == {'$set': {'in_mongo_dict': {'c': 3}}}
+        assert d3.to_mongo() == {'in_mongo_dict': {'c': 3}}
 
         d4 = MyDataProxy({'dict': None})
         assert d4.to_mongo() == {'in_mongo_dict': None}
@@ -279,8 +298,8 @@ def test_dict_default(self):
 
         class MySchema(Schema):
             # Passing a mutable as default is a bad idea in real life
-            d_dict = fields.DictField(default={'1': 1, '2': 2})
-            c_dict = fields.DictField(default=lambda: {'1': 1, '2': 2})
+            d_dict = fields.DictField(fields.IntField, default={'1': 1, '2': 2})
+            c_dict = fields.DictField(fields.IntField, default=lambda: {'1': 1, '2': 2})
 
         MyDataProxy = data_proxy_factory('My', MySchema())
         d = MyDataProxy()
@@ -302,6 +321,68 @@ class MySchema(Schema):
         assert isinstance(d.get('d_dict'), Dict)
         assert isinstance(d.get('c_dict'), Dict)
 
+    def test_complex_dict(self):
+
+        @self.instance.register
+        class MyEmbeddedDocument(EmbeddedDocument):
+            field = fields.IntField()
+
+        @self.instance.register
+        class ToRefDoc(Document):
+            pass
+
+        @self.instance.register
+        class MyDoc(Document):
+            embeds = fields.DictField(
+                fields.EmbeddedField(MyEmbeddedDocument))
+            refs = fields.DictField(fields.ReferenceField(ToRefDoc))
+
+        MySchema = MyDoc.Schema
+
+        obj_id1 = ObjectId()
+        obj_id2 = ObjectId()
+        to_ref_doc1 = ToRefDoc.build_from_mongo(data={'_id': obj_id1})
+        MyDataProxy = data_proxy_factory('My', MySchema())
+        d = MyDataProxy()
+        d.load({
+            'embeds': {
+                'a': MyEmbeddedDocument(field=1),
+                'b': {'field': 2},
+            },
+            'refs': {
+                '1': to_ref_doc1,
+                '2': Reference(ToRefDoc, obj_id2),
+            }
+        })
+        assert d.to_mongo() == {
+            'embeds': {'a': {'field': 1}, 'b': {'field': 2}},
+            'refs': {'1': obj_id1, '2': obj_id2},
+        }
+        assert isinstance(d.get('embeds'), Dict)
+        assert isinstance(d.get('refs'), Dict)
+        for e in d.get('refs').values():
+            assert isinstance(e, Reference)
+        for e in d.get('embeds').values():
+            assert isinstance(e, MyEmbeddedDocument)
+        # Test dict modification as well
+        refs_dict = d.get('refs')
+        refs_dict.update({'3': to_ref_doc1, '4': Reference(ToRefDoc, obj_id2)})
+        for e in refs_dict.values():
+            assert isinstance(e, Reference)
+        embeds_dict = d.get('embeds')
+        embeds_dict.update({'c': MyEmbeddedDocument(field=3), 'd': {'field': 4}})
+        for e in embeds_dict.values():
+            assert isinstance(e, MyEmbeddedDocument)
+        # Modifying an EmbeddedDocument inside a dict should count a dict modification
+        d.clear_modified()
+        d.get('refs')['1'] = obj_id2
+        assert d.to_mongo(update=True) == {'$set': {'refs': {
+            '1': obj_id2, '2': obj_id2, '3': obj_id1, '4': obj_id2}}}
+        d.clear_modified()
+        d.get('embeds')['b'].field = 42
+        assert d.to_mongo(update=True) == {'$set': {'embeds': {
+            'a': {'field': 1}, 'b': {'field': 42}, 'c': {'field': 3}, 'd': {'field': 4}}}}
+
     def test_list(self):
 
         class MySchema(Schema):

tests/test_marshmallow.py

@@ -81,8 +81,10 @@ class Accessory(EmbeddedDocument):
         @self.instance.register
         class Bag(Document):
             id = fields.EmbeddedField(Accessory, attribute='_id', required=True)
-            names = fields.ListField(fields.StringField())
+            names = fields.ListField(fields.StringField)
             content = fields.ListField(fields.EmbeddedField(Accessory))
+            relations = fields.DictField(fields.StringField)
+            inventory = fields.DictField(fields.EmbeddedField(Accessory))
 
         ma_field = Bag.schema.fields['id'].as_marshmallow_field(params={
             'load_only': True,
@@ -100,6 +102,17 @@ class Bag(Document):
         assert ma_field.load_only is True
         assert ma_field.inner.required is True
         assert ma_field.inner.nested._declared_fields['value'].dump_only
+        ma_field = Bag.schema.fields['relations'].as_marshmallow_field(params={
+            'load_only': True,
+            'params': {'dump_only': True}})
+        assert ma_field.load_only is True
+        assert ma_field.value_field.dump_only is True
+        ma_field = Bag.schema.fields['inventory'].as_marshmallow_field(params={
+            'load_only': True,
+            'params': {'required': True, 'params': {'value': {'dump_only': True}}}})
+        assert ma_field.load_only is True
+        assert ma_field.value_field.required is True
+        assert ma_field.value_field.nested._declared_fields['value'].dump_only
 
     def test_pass_meta_attributes(self):
         @self.instance.register
@@ -111,6 +124,7 @@ class Accessory(EmbeddedDocument):
         class Bag(Document):
             id = fields.EmbeddedField(Accessory, attribute='_id', required=True)
             content = fields.ListField(fields.EmbeddedField(Accessory))
+            inventory = fields.DictField(fields.EmbeddedField(Accessory))
 
         ma_schema = Bag.schema.as_marshmallow_schema(meta={'exclude': ('id',)})
         assert ma_schema.Meta.exclude == ('id',)
@@ -120,6 +134,9 @@ class Bag(Document):
         ma_schema = Bag.schema.as_marshmallow_schema(params={
             'content': {'params': {'meta': {'exclude': ('value',)}}}})
         assert ma_schema._declared_fields['content'].inner.nested.Meta.exclude == ('value',)
+        ma_schema = Bag.schema.as_marshmallow_schema(params={
+            'inventory': {'params': {'meta': {'exclude': ('value',)}}}})
+        assert ma_schema._declared_fields['inventory'].value_field.nested.Meta.exclude == ('value',)
 
         class DumpOnlyIdSchema(marshmallow.Schema):
             class Meta:

umongo/builder.py

@@ -12,7 +12,7 @@
 from .exceptions import DocumentDefinitionError, NotRegisteredDocumentError
 from .schema import Schema, on_need_add_id_field, add_child_field
 from .indexes import parse_index
-from .fields import ListField, EmbeddedField
+from .fields import ListField, DictField, EmbeddedField
 
 
 def camel_to_snake(name):
@@ -200,7 +200,11 @@ def _patch_field(self, field):
         field.instance = self.instance
         if isinstance(field, ListField):
             self._patch_field(field.inner)
-        if isinstance(field, EmbeddedField):
+        elif isinstance(field, DictField):
+            # TODO: Do we need to also patch key_field?
+            self._patch_field(field.key_field)
+            self._patch_field(field.value_field)
+        elif isinstance(field, EmbeddedField):
             for embedded_field in field.schema.fields.values():
                 self._patch_field(embedded_field)
 

umongo/data_objects.py

@@ -66,42 +66,80 @@ def __repr__(self):
         return '<object %s.%s(%s)>' % (
             self.__module__, self.__class__.__name__, list(self))
 
-    def set_modified(self):
-        self._modified = True
-
     def is_modified(self):
         if self._modified:
             return True
         if len(self) and isinstance(self[0], BaseDataObject):
-            # Recursive handling needed
             return any(obj.is_modified() for obj in self)
         return False
 
+    def set_modified(self):
+        self._modified = True
+
     def clear_modified(self):
         self._modified = False
         if len(self) and isinstance(self[0], BaseDataObject):
-            # Recursive handling needed
             for obj in self:
                 obj.clear_modified()
 
 
-# TODO: Dict is to much raw: you need to use `set_modified` by hand !
 class Dict(BaseDataObject, dict):
 
-    __slots__ = ('_modified', )
+    __slots__ = ('key_field', 'value_field', '_modified')
 
-    def __init__(self, *args, **kwargs):
-        self._modified = False
+    def __init__(self, key_field, value_field, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self._modified = False
+        self.key_field = key_field
+        self.value_field = value_field
+
+    def __setitem__(self, key, obj):
+        obj = self.value_field.deserialize(obj)
+        super().__setitem__(key, obj)
+        self.set_modified()
+
+    def __delitem__(self, key):
+        super().__delitem__(key)
+        self.set_modified()
+
+    def pop(self, *args, **kwargs):
+        ret = super().pop(*args, **kwargs)
+        self.set_modified()
+        return ret
+
+    def popitem(self, *args, **kwargs):
+        ret = super().popitem(*args, **kwargs)
+        self.set_modified()
+        return ret
+
+    def setdefault(self, key, obj=None):
+        obj = self.value_field.deserialize(obj)
+        ret = super().setdefault(key, obj)
+        self.set_modified()
+        return ret
+
+    def update(self, other):
+        new = {k: self.value_field.deserialize(v) for k, v in other.items()}
+        super().update(new)
+        self.set_modified()
+
+    def __repr__(self):
+        return '<object %s.%s(%s)>' % (
+            self.__module__, self.__class__.__name__, dict(self))
 
     def is_modified(self):
+        if self and any(isinstance(v, BaseDataObject) for v in self.values()):
+            return any(obj.is_modified() for obj in self.values())
         return self._modified
 
     def set_modified(self):
         self._modified = True
 
     def clear_modified(self):
         self._modified = False
+        if self and any(isinstance(v, BaseDataObject) for v in self.values()):
+            for obj in self.values():
+                obj.clear_modified()
 
 
 class Reference: