Merge pull request #17 from catholabs/improve_mapping_settings

Improve mapping settings - half of the #5

esengine/fields.py

@@ -25,7 +25,7 @@ class LongField(BaseField):
 
 class StringField(BaseField):
     _type = unicode
-    _default_mapping = {'type': 'string'}
+    _default_mapping = {"index": "analyzed", "store": "yes", 'type': 'string'}
 
 
 class FloatField(BaseField):

esengine/mapping.py

@@ -1,3 +1,4 @@
+import collections
 
 
 class Mapping(object):
@@ -13,26 +14,44 @@ class Mapping(object):
     obj_mapping = Mapping(Obj)
     obj_mapping.save()
 
+    Adicionally this class handle index settings configuration. However this
+    operation must be done at elasticsearch index creation.
+
     """
-    def __init__(self, document_class, enable_all=True):
+    def __init__(self, document_class=None, enable_all=True):
         self.document_class = document_class
         self.enable_all = enable_all
 
-    def generate(self):
+    def _generate(self, doc_class):
+        """
+        Generate the mapping acording to doc_class.
+
+        Args:
+            doc_class: esengine.Document object containing the model to be
+            mapped to elasticsearch.
+        """
         m = {
-            self.document_class._doctype: {
+            doc_class._doctype: {
                 "_all": {"enabled": self.enable_all},
                 "properties": {
                     field_name: field_instance.mapping
-                    for field_name, field_instance in
-                    self.document_class._fields.items()
+                    for field_name, field_instance in doc_class._fields.items()
                     if field_name != "id"
                 }
             }
         }
         return m
 
+    def generate(self):
+        return self._generate(self.document_class)
+
     def save(self, es=None):
+        """
+        Save the mapping to index.
+
+        Args:
+            es: elasticsearch client intance.
+        """
         es = self.document_class.get_es(es)
         if not es.indices.exists(index=self.document_class._index):
             return es.indices.create(
@@ -45,3 +64,47 @@ def save(self, es=None):
                 index=self.document_class._index,
                 body=self.generate()
             )
+
+    def configure(self, models_to_mapping, custom_settings=None, es=None):
+        """
+        Add custon settings like filters and analizers to index.
+
+        Add custon settings, like filters and analizers, to index. Be aware
+        that elasticsearch only allow this operation on index creation.
+
+        Args:
+            models_to_mapping: A list with the esengine.Document objects that
+            we want generate mapping.
+
+            custom_settings: a dict containing the configuration that will be
+            sent to elasticsearch/_settings (www.elastic.co/guide/en/
+                elasticsearch/reference/current/indices-update-settings.html)
+
+            es: elasticsearch client intance.
+        """
+        if not isinstance(models_to_mapping, collections.Iterable):
+            raise AttributeError('models_to_mapping must be iterable')
+
+        mapped_models = [x for x in models_to_mapping]
+        if custom_settings:
+            indexes = set()
+            for model in mapped_models:
+                indexes.add(model._index)
+                es = model.get_es(es)
+            for index in indexes:
+                if es.indices.exists(index=index):
+                    msg = 'Settings are supported only on index creation'
+                    raise ValueError(msg)
+            mappings_by_index = collections.defaultdict(dict)
+            for model in mapped_models:
+                mapping = self._generate(model)
+                mappings_by_index[model._index].update(mapping)
+            for index, mappings in mappings_by_index.items():
+                settings = {
+                    "settings": custom_settings,
+                    "mappings": mappings
+                }
+                es.indices.create(index=index, body=settings)
+        else:
+            for model in mapped_models:
+                model.put_mapping()

tests/test_mapping.py

@@ -4,8 +4,16 @@
     DateField, BooleanField, GeoPointField
 )
 
-class Doc(Document):
+
+class BaseDoc(Document):
     _index = 'index'
+
+    @classmethod
+    def put_mapping(cls, *args, **kwargs):
+        cls.called = True
+
+
+class Doc(BaseDoc):
     _doctype = 'doc_type'
 
     integerfield = IntegerField()
@@ -17,6 +25,11 @@ class Doc(Document):
     geopointfield = GeoPointField()
 
 
+class Doc1(BaseDoc):
+    _doctype = 'doc_type1'
+    integerfield = IntegerField()
+
+
 class DocDate(Doc):
     datefield = DateField(mapping={'format': 'yyyy-MM-dd||epoch_millis'})
 
@@ -37,14 +50,117 @@ def test_mapping():
                 'geopointfield': {'type': 'geo_point'},
                 'integerfield': {'type': 'integer'},
                 'longfield': {'type': 'long'},
-                'stringfield': {'type': 'string'}
+                'stringfield': {
+                    "index": "analyzed",
+                    "store": "yes",
+                    'type': 'string'
+                }
             }
         }
     }
 
 
-
 def test_change_format():
     mapping = Mapping(DocDate, enable_all=False).generate()
+    pattern = 'yyyy-MM-dd||epoch_millis'
     assert mapping['doc_type']['_all']['enabled'] is False
-    assert mapping['doc_type']['properties']['datefield']['format'] == 'yyyy-MM-dd||epoch_millis'
+    assert mapping['doc_type']['properties']['datefield']['format'] == pattern
+
+
+def test_configure_prerequiriments():
+    mapping = Mapping()
+    try:
+        mapping.configure(10, None)
+    except AttributeError as e:
+        assert str(e) == 'models_to_mapping must be iterable'
+
+
+def test_configure_prerequiriments_throw_on_index_existence():
+    mapping = Mapping()
+    try:
+        models = [Doc, Doc1]
+        es = ESMock()
+        es.indices.exists_ret = True
+        mapping.configure(models, True, es)
+    except ValueError as e:
+        assert str(e) == 'Settings are supported only on index creation'
+
+
+def test_configure_without_settings():
+    mapping = Mapping()
+    models = [Doc, Doc1]
+    mapping.configure(models, None)
+    for model in models:
+        assert model.called
+
+
+def test_configure():
+    mapping = Mapping()
+    models = [Doc, Doc1]
+    es = ESMock()
+    es.indices.exists_ret = False
+    settings = {
+        "asdf": 'This is a test',
+        "analyzer": {
+            "my_analizer": "Another test"
+        }
+    }
+    mapping.configure(models, settings, es)
+    expected_mappings = {
+        'doc_type': {
+            '_all': {'enabled': True},
+            'properties': {
+                'booleanfield': {'type': 'boolean'},
+                'datefield': {
+                    'type': 'date'
+                },
+                'floatfield': {'type': 'float'},
+                'geopointfield': {'type': 'geo_point'},
+                'integerfield': {'type': 'integer'},
+                'longfield': {'type': 'long'},
+                'stringfield': {
+                    "index": "analyzed",
+                    "store": "yes",
+                    'type': 'string'
+                }
+            }
+        },
+        'doc_type1': {
+            '_all': {'enabled': True},
+            'properties': {
+                'integerfield': {'type': 'integer'},
+            }
+        }
+    }
+    expected_output = {
+        "mappings": expected_mappings,
+        "settings": settings
+    }
+    assert es.indices.create_return['index'] == expected_output
+
+
+class ESMock(object):
+
+    class Indice(object):
+        exists_ret = False
+
+        def exists(self, *args, **kwargs):
+            return self.exists_ret
+
+        def create(self, index, body):
+            try:
+                self.create_return[index] = body
+            except:
+                self.create_return = {}
+                self.create_return[index] = body
+
+    indices = Indice()
+
+    def index(self, *args, **kwargs):
+        pass
+
+    def search(self, *args, **kwargs):
+        pass
+
+    def get(self, *args, **kwargs):
+        pass