Added proper Document validation

docs/persistence.rst

@@ -67,9 +67,8 @@ Common field options:
 ``multi``
   If set to ``True`` the field's value will be set to ``[]`` at first access.
 
-``blank``
-  Defaults to ``False`` and if eneabled will cause access to a field with no
-  value to return the field's empty value (``''`` or ``None``).
+``required``
+  Indicates if a field requires a value for the document to be valid.
 
 Analysis
 --------

elasticsearch_dsl/document.py

@@ -195,7 +195,10 @@ def to_dict(self, include_meta=False):
             d = meta
         return d
 
-    def save(self, using=None, index=None, **kwargs):
+    def save(self, using=None, index=None, validate=True, **kwargs):
+        if validate:
+            self.full_clean()
+
         es = self._get_connection(using)
         if index is None:
             index = getattr(self.meta, 'index', self._doc_type.index)

elasticsearch_dsl/field.py

@@ -43,7 +43,7 @@ class Field(DslBase):
 
     def __init__(self, *args, **kwargs):
         self._multi = kwargs.pop('multi', False)
-        self._allow_blank = kwargs.pop('blank', self._multi)
+        self._required = kwargs.pop('required', False)
         super(Field, self).__init__(*args, **kwargs)
 
     def _to_python(self, data):
@@ -54,15 +54,23 @@ def _empty(self):
 
     def empty(self):
         if self._multi:
-            return []
+            return AttrList([])
         return self._empty()
 
     def to_python(self, data):
+        if not data:
+            return data
         if isinstance(data, (list, AttrList)):
             data[:] = map(self._to_python, data)
             return data
         return self._to_python(data)
 
+    def clean(self, data):
+        data = self.to_python(data)
+        if not data and self._required:
+            raise ValidationException("Value required for this field.")
+        return data
+
     def to_dict(self):
         d = super(Field, self).to_dict()
         name, value = d.popitem()
@@ -139,16 +147,24 @@ def _to_python(self, data):
             data[:] = list(map(self._to_python, data))
             return data
 
+        if isinstance(data, AttrDict):
+            data = data._d_
+
         return self._doc_class(self.properties, **data)
 
+    def clean(self, data):
+        data = super(InnerObject, self).clean(data)
+        if isinstance(data, (list, AttrList)):
+            for d in data:
+                d.full_clean()
+        else:
+            data.full_clean()
+        return data
+
+
 class Object(InnerObject, Field):
     name = 'object'
 
-    def __init__(self, *args, **kwargs):
-        # change the default for Object fields
-        kwargs.setdefault('blank', True)
-        super(Object, self).__init__(*args, **kwargs)
-
 class Nested(InnerObject, Field):
     name = 'nested'
 

elasticsearch_dsl/utils.py

@@ -3,7 +3,9 @@
 from six import iteritems, add_metaclass
 from six.moves import map
 
-from .exceptions import UnknownDslObject
+from .exceptions import UnknownDslObject, ValidationException
+
+SKIP_VALUES = ('', None)
 
 def _wrap(val, obj_wrapper=None):
     if isinstance(val, dict):
@@ -416,9 +418,9 @@ def __getattr__(self, name):
         except AttributeError:
             if name in self._doc_type.mapping:
                 f = self._doc_type.mapping[name]
-                if hasattr(f, 'empty') and f._allow_blank:
+                if hasattr(f, 'empty'):
                     value = f.empty()
-                    if value is not None:
+                    if value not in SKIP_VALUES:
                         setattr(self, name, value)
                         value = getattr(self, name)
                     return value
@@ -436,5 +438,32 @@ def to_dict(self):
                 v = [i.to_dict() if hasattr(i, 'to_dict') else i for i in v]
             else:
                 v = v.to_dict() if hasattr(v, 'to_dict') else v
+
+            # don't serialize empty values
+            # careful not to include numeric zeros
+            if not isinstance(v, (int, float)) and not v:
+                continue
+
             out[k] = v
         return out
+
+    def clean_fields(self):
+        errors = {}
+        for name in self._doc_type.mapping:
+            field = self._doc_type.mapping[name]
+            data = getattr(self, name, None)
+            try:
+                data = field.clean(data)
+            except ValidationException as e:
+                errors.setdefault(name, []).append(e)
+
+        if errors:
+            raise ValidationException(errors)
+
+    def clean(self):
+        pass
+
+    def full_clean(self):
+        self.clean_fields()
+        self.clean()
+

test_elasticsearch_dsl/test_document.py

@@ -55,8 +55,6 @@ def test_attribute_can_be_removed():
 
     del d.title
     assert 'title' not in d._d_
-    with raises(AttributeError):
-        d.title
 
 def test_doc_type_can_be_correctly_pickled():
     d = DocWithNested(title='Hello World!', comments=[{'title': 'hellp'}], meta={'id': 42})
@@ -103,16 +101,6 @@ class Blog(document.DocType):
     b.tags.append('python')
     assert ['search', 'python'] == b.tags
 
-def test_blank_enabled_fields():
-    class Blog(document.DocType):
-        published = field.Date(blank=True)
-        published_not_blank = field.Date()
-
-    b = Blog()
-    with raises(AttributeError):
-        b.published_not_blank
-    assert None is b.published
-
 def test_docs_with_properties():
     class User(document.DocType):
         pwd_hash = field.String()
@@ -225,7 +213,7 @@ def test_document_can_be_created_dynamicaly():
 def test_invalid_date_will_raise_exception():
     md = MyDoc()
     with raises(ValidationException):
-        md.created_at = None
+        md.created_at = 'not-a-date'
 
 def test_document_inheritance():
     assert issubclass(MySubDoc, MyDoc)

test_elasticsearch_dsl/test_validation.py

@@ -0,0 +1,60 @@
+from elasticsearch_dsl import DocType, Nested, String, Date
+from elasticsearch_dsl.field import InnerObjectWrapper
+from elasticsearch_dsl.exceptions import ValidationException
+
+from pytest import raises
+
+class Author(InnerObjectWrapper):
+    def clean(self):
+        if self.name.lower() not in self.email:
+            raise ValidationException('Invalid email!')
+
+class BlogPost(DocType):
+    authors = Nested(
+        required=True,
+        doc_class=Author,
+        properties={
+            'name': String(required=True),
+            'email': String(required=True)
+        }
+    )
+    created = Date()
+
+def test_missing_required_field_raises_validation_exception():
+    d = BlogPost()
+    with raises(ValidationException):
+        d.full_clean()
+
+    d = BlogPost()
+    d.authors.append({'name': 'Honza'})
+    with raises(ValidationException):
+        d.full_clean()
+
+    d = BlogPost()
+    d.authors.append({'name': 'Honza', 'email': 'honza@elastic.co'})
+    d.full_clean()
+
+def test_custom_validation_on_nested_gets_run():
+    d = BlogPost(authors=[{'name': 'Honza', 'email': 'king@example.com'}], created=None)
+
+    assert isinstance(d.authors[0], Author)
+
+    with raises(ValidationException):
+        d.full_clean()
+
+def test_accessing_known_fields_returns_empty_value():
+    d = BlogPost()
+
+    assert [] == d.authors
+
+    d.authors.append({})
+    assert '' == d.authors[0].name
+    assert '' == d.authors[0].email 
+
+def test_empty_values_are_not_serialized():
+    d = BlogPost(authors=[{'name': 'Honza', 'email': 'honza@elastic.co'}], created=None)
+
+    d.full_clean()
+    assert d.to_dict() == {
+        'authors': [{'name': 'Honza', 'email': 'honza@elastic.co'}]
+    }