Merge ded7dd1 into c6b6525

.travis.yml

@@ -27,9 +27,9 @@ jobs:
   include:
 
     - { python: '3.8', env: TOXENV=lint }
-    - { python: '3.6', env: TOXENV=py36-pymongo }
-    - { python: '3.6', env: TOXENV=py36-motor1 }
-    - { python: '3.6', env: TOXENV=py36-txmongo }
+    - { python: '3.7', env: TOXENV=py37-pymongo }
+    - { python: '3.7', env: TOXENV=py37-motor1 }
+    - { python: '3.7', env: TOXENV=py37-txmongo }
     - { python: '3.8', env: TOXENV=py38-pymongo }
     - { python: '3.8', env: TOXENV=py38-motor2 }
     - { python: '3.8', env: TOXENV=py38-txmongo }

CONTRIBUTING.rst

@@ -122,6 +122,6 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
    feature to the list in README.rst.
-3. The pull request should work for Python 3.6, 3.7 and 3.8. Check
+3. The pull request should work for Python 3.7 and 3.8. Check
    https://travis-ci.org/touilleMan/umongo/pull_requests
    and make sure that the tests pass for all supported Python versions.

setup.py

@@ -29,7 +29,7 @@
     url='https://github.com/touilleMan/umongo',
     packages=['umongo', 'umongo.frameworks'],
     include_package_data=True,
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     install_requires=requirements,
     extras_require={
         'motor': ['motor>=1.1,<3.0'],
@@ -45,7 +45,6 @@
         'License :: OSI Approved :: MIT License',
         'Natural Language :: English',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
     ],

tests/test_document.py

@@ -6,8 +6,8 @@
 from bson import ObjectId, DBRef
 
 from umongo import (Document, EmbeddedDocument, Schema, fields, exceptions,
-                    post_dump, pre_load, validates_schema)
-
+                    post_dump, pre_load, validates_schema, missing)
+from umongo.document import ExposeMissing
 from .common import BaseTest
 
 
@@ -440,6 +440,13 @@ class User(Document):
         with pytest.raises(exceptions.AlreadyCreatedError):
             john.update({'primary_key': ObjectId()})
 
+    def test_expose_missing(self):
+        john = self.Student(name='John Doe')
+        assert john.name == 'John Doe'
+        assert john.birthday is None
+        with ExposeMissing():
+            assert john.name == 'John Doe'
+            assert john.birthday is missing
 
 class TestConfig(BaseTest):
 

tests/test_embedded_document.py

@@ -6,6 +6,7 @@
 
 from umongo.data_proxy import data_proxy_factory
 from umongo import Document, EmbeddedDocument, fields, exceptions
+from umongo.document import ExposeMissing
 
 from .common import BaseTest
 
@@ -423,3 +424,23 @@ class Parent(EmbeddedDocument):
         assert jane.name == john.name
         assert jane.child == john.child
         assert jane.child is not john.child
+
+    def test_expose_missing(self):
+        @self.instance.register
+        class Child(EmbeddedDocument):
+            name = fields.StrField()
+            age = fields.IntField()
+
+        @self.instance.register
+        class Parent(EmbeddedDocument):
+            name = fields.StrField()
+            child = fields.EmbeddedField(Child)
+
+        parent = Parent(**{'child': {'age': 42}})
+        assert parent.name is None
+        assert parent.child.name is None
+        assert parent.child.age == 42
+        with ExposeMissing():
+            assert parent.name is missing
+            assert parent.child.name is missing
+            assert parent.child.age == 42

tests/test_marshmallow.py

@@ -8,7 +8,7 @@
 from umongo import Document, EmbeddedDocument, fields, set_gettext, validate, missing
 from umongo import marshmallow_bonus as ma_bonus_fields, Schema
 from umongo.abstract import BaseField
-from umongo.schema import schema_from_umongo_get_attribute, SchemaFromUmongo
+from umongo.schema import RemoveMissingSchema
 
 from .common import BaseTest
 
@@ -385,7 +385,7 @@ class Meta:
                 schema.load({"id": invalid_id})
             assert exc.value.messages == {"id": ["Invalid ObjectId."]}
 
-    def test_marshmallow_schema_helpers(self):
+    def test_marshmallow_remove_missing_schema(self):
 
         @self.instance.register
         class Doc(Document):
@@ -395,32 +395,82 @@ class Doc(Document):
             def prop(self):
                 return "I'm a property !"
 
-        class VanillaSchema(marshmallow.Schema):
+        class VanillaDocSchema(marshmallow.Schema):
             a = marshmallow.fields.Int()
 
-        class CustomGetAttributeSchema(VanillaSchema):
-            get_attribute = schema_from_umongo_get_attribute
+        class RemoveMissingDocSchema(RemoveMissingSchema):
+            a = marshmallow.fields.Int()
 
-        data = VanillaSchema().dump(Doc())
+        data = VanillaDocSchema().dump(Doc())
         assert data == {'a': None}
 
-        data = CustomGetAttributeSchema().dump(Doc())
+        data = RemoveMissingDocSchema().dump(Doc())
         assert data == {}
 
-        data = CustomGetAttributeSchema().dump(Doc(a=1))
+        data = RemoveMissingDocSchema().dump(Doc(a=1))
         assert data == {'a': 1}
 
-        class MySchemaFromUmongo(SchemaFromUmongo):
-            a = marshmallow.fields.Int()
-            prop = marshmallow.fields.String(dump_only=True)
+    @pytest.mark.parametrize("base_schema", (marshmallow.Schema, RemoveMissingSchema))
+    def test_marshmallow_remove_missing_schema_as_base_schema(self, base_schema):
+        """Test RemoveMissingSchema used as base marshmallow Schema"""
 
-        with pytest.raises(marshmallow.ValidationError) as excinfo:
-            MySchemaFromUmongo().load({'a': 1, 'dummy': 2})
-        assert excinfo.value.messages == {'dummy': ['Unknown field.']}
+        # Typically, we'll use it in all our schemas, so let's define base
+        # Document and EmbeddedDocument classes using this base schema class
+        @self.instance.register
+        class MyDocument(Document):
+            MA_BASE_SCHEMA_CLS = base_schema
 
-        with pytest.raises(marshmallow.ValidationError) as excinfo:
-            MySchemaFromUmongo().load({'a': 1, 'prop': '2'})
-        assert excinfo.value.messages == {'prop': ['Unknown field.']}
+            class Meta:
+                allow_inheritance = True
+
+        @self.instance.register
+        class MyEmbeddedDocument(EmbeddedDocument):
+            MA_BASE_SCHEMA_CLS = base_schema
+
+            class Meta:
+                allow_inheritance = True
+
+        @self.instance.register
+        class Accessory(MyEmbeddedDocument):
+            brief = fields.StrField()
+            value = fields.IntField()
+
+        @self.instance.register
+        class Bag(MyDocument):
+            item = fields.EmbeddedField(Accessory)
+            content = fields.ListField(fields.EmbeddedField(Accessory))
+
+        data = {
+            'item': {'brief': 'sportbag'},
+            'content': [
+                {'brief': 'cellphone'},
+                {'brief': 'lighter'}]
+        }
+        dump = {
+            'id': None,
+            'cls': 'Bag',
+            'content': [
+                {'brief': 'cellphone', 'cls': 'Accessory', 'value': None},
+                {'brief': 'lighter', 'cls': 'Accessory', 'value': None}
+            ],
+            'item': {'brief': 'sportbag', 'cls': 'Accessory', 'value': None}
+        }
+        remove_missing_dump = {
+            'cls': 'Bag',
+            'item': {'cls': 'Accessory', 'brief': 'sportbag'},
+            'content': [
+                {'cls': 'Accessory', 'brief': 'cellphone'},
+                {'cls': 'Accessory', 'brief': 'lighter'}
+            ]
+        }
+        expected_dump = {
+            marshmallow.Schema: dump,
+            RemoveMissingSchema: remove_missing_dump,
+        }[base_schema]
+
+        bag = Bag(**data)
+        ma_schema = Bag.schema.as_marshmallow_schema()
+        assert ma_schema().dump(bag) == expected_dump
 
     def test_marshmallow_access_custom_attributes(self):
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = lint,{py36,py37,py38}-{motor1,motor2,pymongo,txmongo}
+envlist = lint,{py37,py38}-{motor1,motor2,pymongo,txmongo}
 
 [testenv]
 setenv =

umongo/document.py

@@ -1,4 +1,6 @@
 from copy import deepcopy
+from contextlib import AbstractContextManager
+from contextvars import ContextVar
 
 from bson import DBRef
 from marshmallow import (
@@ -28,6 +30,24 @@
 )
 
 
+EXPOSE_MISSING = ContextVar("expose_missing", default=False)
+
+
+class ExposeMissing(AbstractContextManager):
+    """Let Document expose missing values rather than returning None
+
+    By default, getting a document item returns None if the value is missing.
+    Inside this context manager, the missing singleton is returned. This can
+    be useful is cases where the user want to distinguish between None and
+    missing value.
+    """
+    def __enter__(self):
+        self.token = EXPOSE_MISSING.set(True)
+
+    def __exit__(self, *args, **kwargs):
+        EXPOSE_MISSING.reset(self.token)
+
+
 class DocumentTemplate(Template):
     """
     Base class to define a umongo document.
@@ -274,7 +294,7 @@ def items(self):
 
     def __getitem__(self, name):
         value = self._data.get(name)
-        return value if value is not missing else None
+        return None if value is missing and not EXPOSE_MISSING.get() else value
 
     def __setitem__(self, name, value):
         if self.is_created and name == self.pk_field:
@@ -290,7 +310,7 @@ def __getattr__(self, name):
         if name[:2] == name[-2:] == '__':
             raise AttributeError(name)
         value = self._data.get(name, to_raise=AttributeError)
-        return value if value is not missing else None
+        return None if value is missing and not EXPOSE_MISSING.get() else value
 
     def __setattr__(self, name, value):
         # Try to retrieve name among class's attributes and __slots__

umongo/embedded_document.py

@@ -1,6 +1,7 @@
 import marshmallow as ma
 
-from .document import Implementation, Template
+from .document import EXPOSE_MISSING
+from .template import Implementation, Template
 from .data_objects import BaseDataObject
 from .data_proxy import missing
 from .exceptions import DocumentDefinitionError, AbstractDocumentError
@@ -161,7 +162,7 @@ def items(self):
 
     def __getitem__(self, name):
         value = self._data.get(name)
-        return value if value is not missing else None
+        return None if value is missing and not EXPOSE_MISSING.get() else value
 
     def __delitem__(self, name):
         self._data.delete(name)
@@ -184,7 +185,7 @@ def __getattr__(self, name):
         if name[:2] == name[-2:] == '__':
             raise AttributeError(name)
         value = self._data.get(name, to_raise=AttributeError)
-        return value if value is not missing else None
+        return None if value is missing and not EXPOSE_MISSING.get() else value
 
     def __delattr__(self, name):
         if not self.__real_attributes:

umongo/schema.py

@@ -1,47 +1,24 @@
 """Schema used in Document"""
-from marshmallow import Schema as MaSchema, missing
+from marshmallow import Schema as MaSchema
 
 from .abstract import BaseSchema
 from .i18n import gettext as _
-
+from .document import ExposeMissing
 
 __all__ = (
     'Schema',
-    'schema_from_umongo_get_attribute',
-    'SchemaFromUmongo',
+    'RemoveMissingSchema',
 )
 
 
-def schema_from_umongo_get_attribute(self, obj, attr, default):
-    """
-    Overwrite default `Schema.get_attribute` method by this one to access
-        umongo missing fields instead of returning `None`.
-
-    example::
-
-        class MySchema(marshsmallow.Schema):
-            get_attribute = schema_from_umongo_get_attribute
-
-            # Define the rest of your schema
-            ...
-
+class RemoveMissingSchema(MaSchema):
     """
-    ret = MaSchema.get_attribute(self, obj, attr, default)
-    if ret is None and ret is not default and attr in obj.schema.fields:
-        raw_ret = obj._data.get(attr)
-        return default if raw_ret is missing else raw_ret
-    return ret
-
-
-class SchemaFromUmongo(MaSchema):
-    """
-    Custom :class:`marshmallow.Schema` subclass providing unknown fields
-    checking and custom get_attribute for umongo documents.
-
-    .. note: It is not mandatory to use this schema with umongo document.
-        This is just a helper providing usefull behaviors.
+    Custom :class:`marshmallow.Schema` subclass returning missing rather than
+    None for missing fields in umongo :class:`umongo.Document`s.
     """
-    get_attribute = schema_from_umongo_get_attribute
+    def dump(self, *args, **kwargs):
+        with ExposeMissing():
+            return super().dump(*args, **kwargs)
 
 
 class Schema(BaseSchema):
@@ -69,8 +46,6 @@ def as_marshmallow_schema(self, *, mongo_world=False):
         name = 'Marshmallow%s' % type(self).__name__
         # By default OO world returns `missing` fields as `None`,
         # disable this behavior here to let marshmallow deal with it
-        if not mongo_world:
-            nmspc['get_attribute'] = schema_from_umongo_get_attribute
         m_schema = type(name, (self.MA_BASE_SCHEMA_CLS, ), nmspc)
         # Add i18n support to the schema
         # We can't use I18nErrorDict here because __getitem__ is not called