Merge 0c22e5b into 77512b2

.travis.yml

@@ -25,9 +25,9 @@ jobs:
   include:
 
     - { python: '3.8', env: TOXENV=lint }
-    - { python: '3.6', env: TOXENV=py36-pymongo }
-    - { python: '3.6', env: TOXENV=py36-motor1 }
-    - { python: '3.6', env: TOXENV=py36-txmongo }
+    - { python: '3.7', env: TOXENV=py37-pymongo }
+    - { python: '3.7', env: TOXENV=py37-motor1 }
+    - { python: '3.7', env: TOXENV=py37-txmongo }
     - { python: '3.8', env: TOXENV=py38-pymongo }
     - { python: '3.8', env: TOXENV=py38-motor2 }
     - { python: '3.8', env: TOXENV=py38-txmongo }

CONTRIBUTING.rst

@@ -122,6 +122,6 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
    feature to the list in README.rst.
-3. The pull request should work for Python 3.6, 3.7 and 3.8. Check
+3. The pull request should work for Python 3.7 and 3.8. Check
    https://travis-ci.org/touilleMan/umongo/pull_requests
    and make sure that the tests pass for all supported Python versions.

setup.py

@@ -29,7 +29,7 @@
     url='https://github.com/touilleMan/umongo',
     packages=['umongo', 'umongo.frameworks'],
     include_package_data=True,
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     install_requires=requirements,
     extras_require={
         'motor': ['motor>=1.1,<3.0'],
@@ -45,7 +45,6 @@
         'License :: OSI Approved :: MIT License',
         'Natural Language :: English',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
     ],

tests/test_document.py

@@ -8,9 +8,8 @@
 
 from umongo import (
     Document, EmbeddedDocument, MixinDocument,
-    Schema, fields, exceptions, post_dump, pre_load, validates_schema
+    Schema, fields, exceptions, post_dump, pre_load, validates_schema, ExposeMissing
 )
-
 from .common import BaseTest
 
 
@@ -434,6 +433,14 @@ class User(Document):
         with pytest.raises(exceptions.AlreadyCreatedError):
             john.update({'primary_key': ObjectId()})
 
+    def test_expose_missing(self):
+        john = self.Student(name='John Doe')
+        assert john.name == 'John Doe'
+        assert john.birthday is None
+        with ExposeMissing():
+            assert john.name == 'John Doe'
+            assert john.birthday is ma.missing
+
     def test_mixin(self):
 
         @self.instance.register

tests/test_embedded_document.py

@@ -5,7 +5,7 @@
 import marshmallow as ma
 
 from umongo.data_proxy import data_proxy_factory
-from umongo import Document, EmbeddedDocument, MixinDocument, fields, exceptions
+from umongo import Document, EmbeddedDocument, MixinDocument, fields, exceptions, ExposeMissing
 
 from .common import BaseTest
 
@@ -414,6 +414,26 @@ class Parent(EmbeddedDocument):
         assert jane.child == john.child
         assert jane.child is not john.child
 
+    def test_expose_missing(self):
+        @self.instance.register
+        class Child(EmbeddedDocument):
+            name = fields.StrField()
+            age = fields.IntField()
+
+        @self.instance.register
+        class Parent(EmbeddedDocument):
+            name = fields.StrField()
+            child = fields.EmbeddedField(Child)
+
+        parent = Parent(**{'child': {'age': 42}})
+        assert parent.name is None
+        assert parent.child.name is None
+        assert parent.child.age == 42
+        with ExposeMissing():
+            assert parent.name is ma.missing
+            assert parent.child.name is ma.missing
+            assert parent.child.age == 42
+
     def test_mixin(self):
 
         @self.instance.register

tests/test_marshmallow.py

@@ -9,7 +9,7 @@
 from umongo import Document, EmbeddedDocument, fields, set_gettext, validate, missing
 from umongo import marshmallow_bonus as ma_bonus_fields, Schema
 from umongo.abstract import BaseField
-from umongo.schema import schema_from_umongo_get_attribute, SchemaFromUmongo
+from umongo.schema import RemoveMissingSchema
 
 from .common import BaseTest
 
@@ -52,10 +52,16 @@ class Meta:
         class MyDocument(Document):
             MA_BASE_SCHEMA_CLS = ExcludeBaseSchema
 
+            class Meta:
+                abstract = True
+
         @self.instance.register
         class MyEmbeddedDocument(EmbeddedDocument):
             MA_BASE_SCHEMA_CLS = ExcludeBaseSchema
 
+            class Meta:
+                abstract = True
+
         # Now, all our objects will generate "exclude" marshmallow schemas
         @self.instance.register
         class Accessory(MyEmbeddedDocument):
@@ -139,11 +145,6 @@ class MyDoc(Document):
 
     def test_as_marshmallow_schema_cache(self):
         ma_schema_cls = self.User.schema.as_marshmallow_schema()
-
-        new_ma_schema_cls = self.User.schema.as_marshmallow_schema(
-            mongo_world=True)
-        assert new_ma_schema_cls != ma_schema_cls
-
         new_ma_schema_cls = self.User.schema.as_marshmallow_schema()
         assert new_ma_schema_cls == ma_schema_cls
 
@@ -202,16 +203,11 @@ class Dog(Document):
 
         payload = {'name': 'Scruffy', 'age': 2}
         ma_schema_cls = Dog.schema.as_marshmallow_schema()
-        ma_mongo_schema_cls = Dog.schema.as_marshmallow_schema(mongo_world=True)
 
         ret = ma_schema_cls().load(payload)
         assert ret == {'name': 'Scruffy', 'age': 2}
         assert ma_schema_cls().dump(ret) == payload
 
-        ret = ma_mongo_schema_cls().load(payload)
-        assert ret == {'_id': 'Scruffy', 'age': 2}
-        assert ma_mongo_schema_cls().dump(ret) == payload
-
     def test_i18n(self):
         # i18n support should be kept, because it's pretty cool to have this !
         def my_gettext(message):
@@ -300,15 +296,11 @@ class Bag(Document):
         # (no ObjectId to str conversion needed for example)
 
         ma_schema = Bag.schema.as_marshmallow_schema()()
-        ma_mongo_schema = Bag.schema.as_marshmallow_schema(mongo_world=True)()
 
         bag = Bag(**data)
         assert ma_schema.dump(bag) == data
         assert ma_schema.load(data) == data
 
-        assert ma_mongo_schema.dump(bag.to_mongo()) == data
-        assert ma_mongo_schema.load(data) == bag.to_mongo()
-
     def test_marshmallow_bonus_fields(self):
         # Fields related to mongodb provided for marshmallow
         @self.instance.register
@@ -337,9 +329,6 @@ class Doc(Document):
             "gen_ref": {'cls': 'Doc', 'id': "57c1a71113adf27ab96b2c4f"}
         }
         doc = Doc(**oo_data)
-        mongo_data = doc.to_mongo()
-
-        # schema to OO world
         ma_schema_cls = Doc.schema.as_marshmallow_schema()
         ma_schema = ma_schema_cls()
         # Dump uMongo object
@@ -349,16 +338,6 @@ class Doc(Document):
         # Load serialized data
         assert ma_schema.load(serialized) == oo_data
 
-        # schema to mongo world
-        ma_mongo_schema_cls = Doc.schema.as_marshmallow_schema(mongo_world=True)
-        ma_mongo_schema = ma_mongo_schema_cls()
-        assert ma_mongo_schema.dump(mongo_data) == serialized
-        assert ma_mongo_schema.load(serialized) == mongo_data
-        # Cannot load mongo form
-        with pytest.raises(ma.ValidationError) as excinfo:
-            ma_mongo_schema.load({"gen_ref": {'_cls': 'Doc', '_id': "57c1a71113adf27ab96b2c4f"}})
-        assert excinfo.value.messages == {'gen_ref': ['Generic reference must have `id` and `cls` fields.']}
-
     def test_marshmallow_bonus_objectid_field(self):
 
         class DocSchema(ma.Schema):
@@ -377,7 +356,7 @@ class Meta:
                 schema.load({"id": invalid_id})
             assert exc.value.messages == {"id": ["Invalid ObjectId."]}
 
-    def test_marshmallow_schema_helpers(self):
+    def test_marshmallow_remove_missing_schema(self):
 
         @self.instance.register
         class Doc(Document):
@@ -387,32 +366,80 @@ class Doc(Document):
             def prop(self):
                 return "I'm a property !"
 
-        class VanillaSchema(ma.Schema):
+        class VanillaDocSchema(ma.Schema):
             a = ma.fields.Int()
 
-        class CustomGetAttributeSchema(VanillaSchema):
-            get_attribute = schema_from_umongo_get_attribute
+        class RemoveMissingDocSchema(RemoveMissingSchema):
+            a = ma.fields.Int()
 
-        data = VanillaSchema().dump(Doc())
+        data = VanillaDocSchema().dump(Doc())
         assert data == {'a': None}
 
-        data = CustomGetAttributeSchema().dump(Doc())
+        data = RemoveMissingDocSchema().dump(Doc())
         assert data == {}
 
-        data = CustomGetAttributeSchema().dump(Doc(a=1))
+        data = RemoveMissingDocSchema().dump(Doc(a=1))
         assert data == {'a': 1}
 
-        class MySchemaFromUmongo(SchemaFromUmongo):
-            a = ma.fields.Int()
-            prop = ma.fields.String(dump_only=True)
+    @pytest.mark.parametrize("base_schema", (ma.Schema, RemoveMissingSchema))
+    def test_marshmallow_remove_missing_schema_as_base_schema(self, base_schema):
+        """Test RemoveMissingSchema used as base marshmallow Schema"""
 
-        with pytest.raises(ma.ValidationError) as excinfo:
-            MySchemaFromUmongo().load({'a': 1, 'dummy': 2})
-        assert excinfo.value.messages == {'dummy': ['Unknown field.']}
+        # Typically, we'll use it in all our schemas, so let's define base
+        # Document and EmbeddedDocument classes using this base schema class
+        @self.instance.register
+        class MyDocument(Document):
+            MA_BASE_SCHEMA_CLS = base_schema
 
-        with pytest.raises(ma.ValidationError) as excinfo:
-            MySchemaFromUmongo().load({'a': 1, 'prop': '2'})
-        assert excinfo.value.messages == {'prop': ['Unknown field.']}
+            class Meta:
+                abstract = True
+
+        @self.instance.register
+        class MyEmbeddedDocument(EmbeddedDocument):
+            MA_BASE_SCHEMA_CLS = base_schema
+
+            class Meta:
+                abstract = True
+
+        @self.instance.register
+        class Accessory(MyEmbeddedDocument):
+            brief = fields.StrField()
+            value = fields.IntField()
+
+        @self.instance.register
+        class Bag(MyDocument):
+            item = fields.EmbeddedField(Accessory)
+            content = fields.ListField(fields.EmbeddedField(Accessory))
+
+        data = {
+            'item': {'brief': 'sportbag'},
+            'content': [
+                {'brief': 'cellphone'},
+                {'brief': 'lighter'}]
+        }
+        dump = {
+            'id': None,
+            'content': [
+                {'brief': 'cellphone', 'value': None},
+                {'brief': 'lighter', 'value': None}
+            ],
+            'item': {'brief': 'sportbag', 'value': None}
+        }
+        remove_missing_dump = {
+            'item': {'brief': 'sportbag'},
+            'content': [
+                {'brief': 'cellphone'},
+                {'brief': 'lighter'}
+            ]
+        }
+        expected_dump = {
+            ma.Schema: dump,
+            RemoveMissingSchema: remove_missing_dump,
+        }[base_schema]
+
+        bag = Bag(**data)
+        ma_schema = Bag.schema.as_marshmallow_schema()
+        assert ma_schema().dump(bag) == expected_dump
 
     def test_marshmallow_access_custom_attributes(self):
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = lint,{py36,py37,py38}-{motor1,motor2,pymongo,txmongo}
+envlist = lint,{py37,py38}-{motor1,motor2,pymongo,txmongo}
 
 [testenv]
 setenv =

umongo/__init__.py

@@ -29,6 +29,7 @@
 from .data_objects import Reference
 from .embedded_document import EmbeddedDocument
 from .mixin import MixinDocument
+from .expose_missing import ExposeMissing
 from .i18n import set_gettext
 
 
@@ -52,6 +53,7 @@
     'validates_schema',
     'EmbeddedDocument',
     'MixinDocument',
+    'ExposeMissing',
 
     'UMongoError',
     'ValidationError',

umongo/abstract.py

@@ -146,31 +146,22 @@ def _serialize_to_mongo(self, obj):
     def _deserialize_from_mongo(self, value):
         return value
 
-    def _extract_marshmallow_field_params(self, mongo_world):
+    def _extract_marshmallow_field_params(self):
         params = {
             attribute: getattr(self, attribute)
             for attribute in (
                 'validate', 'required', 'allow_none',
                 'load_only', 'dump_only', 'error_messages'
             )
         }
-        if mongo_world and self.attribute:
-            params['attribute'] = self.attribute
-
         # Override uMongo attributes with marshmallow_ prefixed attributes
         params.update(self._ma_kwargs)
-
         params.update(self.metadata)
         return params
 
-    def as_marshmallow_field(self, *, mongo_world=False, **kwargs):
-        """
-        Return a pure-marshmallow version of this field.
-
-        :param mongo_world: If True the field will work against the mongo world
-            instead of the OO world (default: False)
-        """
-        field_kwargs = self._extract_marshmallow_field_params(mongo_world)
+    def as_marshmallow_field(self):
+        """Return a pure-marshmallow version of this field"""
+        field_kwargs = self._extract_marshmallow_field_params()
         # Retrieve the marshmallow class we inherit from
         for m_class in type(self).mro():
             if (not issubclass(m_class, BaseField) and

umongo/embedded_document.py

@@ -3,6 +3,7 @@
 
 from .template import Implementation, Template
 from .data_objects import BaseDataObject
+from .expose_missing import EXPOSE_MISSING
 from .exceptions import AbstractDocumentError
 
 
@@ -159,7 +160,7 @@ def items(self):
 
     def __getitem__(self, name):
         value = self._data.get(name)
-        return value if value is not ma.missing else None
+        return None if value is ma.missing and not EXPOSE_MISSING.get() else value
 
     def __delitem__(self, name):
         self._data.delete(name)
@@ -182,7 +183,7 @@ def __getattr__(self, name):
         if name[:2] == name[-2:] == '__':
             raise AttributeError(name)
         value = self._data.get(name, to_raise=AttributeError)
-        return value if value is not ma.missing else None
+        return None if value is ma.missing and not EXPOSE_MISSING.get() else value
 
     def __delattr__(self, name):
         if not self.__real_attributes: