Merge pull request #46 from brandicted/develop

release 0.3.1

VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

docs/source/changelog.rst

@@ -1,6 +1,10 @@
 Changelog
 =========
 
+* :release:`0.3.1 <2015-05-27>`
+* :bug:`-` fixed PUT to replace all fields and PATCH to update some
+* :bug:`-` fixed posting to singular resources e.d. /api/users/<username>/profile
+
 * :release:`0.3.0 <2015-05-18>`
 * :support:`-` Step-by-step 'Getting started' guide
 * :bug:`- major` Fixed several issues related to ElasticSearch indexing

docs/source/development_tools.rst

@@ -8,12 +8,12 @@ Indexing in ElasticSearch
 
 You can run it like so::
 
-    $ nefertari.index --config local.ini --models example_api.model.Story
+    $ nefertari.index --config local.ini --models Model
 
 The available options are:
 
 --config        specify ini file to use (required)
---models        list of models to index (e.g. User). Models must subclass ESBaseDocument.
+--models        list of models to index. Models must subclass ESBaseDocument.
 --params        URL-encoded parameters for each module
 --quiet         "quiet mode" (surpress output)
 --index         Specify name of index. E.g. the slug at the end of http://localhost:9200/example_api

nefertari/authentication/models.py

@@ -147,15 +147,15 @@ def get_authuser_by_name(cls, request):
             return cls.get_resource(username=username)
 
 
-def lower_strip(value):
-    return (value or '').lower().strip()
+def lower_strip(instance, new_value):
+    return (new_value or '').lower().strip()
 
 
-def encrypt_password(password):
-    """ Crypt :password: if it's not crypted yet. """
-    if password and not crypt.match(password):
-        password = unicode(crypt.encode(password))
-    return password
+def encrypt_password(instance, new_value):
+    """ Crypt :new_value: if it's not crypted yet. """
+    if new_value and not crypt.match(new_value):
+        new_value = unicode(crypt.encode(new_value))
+    return new_value
 
 
 class AuthUser(AuthModelDefaultMixin, engine.BaseDocument):

nefertari/elasticsearch.py

@@ -5,7 +5,8 @@
 
 from nefertari.utils import (
     dictset, dict2obj, process_limit, split_strip)
-from nefertari.json_httpexceptions import JHTTPBadRequest, JHTTPNotFound, exception_response
+from nefertari.json_httpexceptions import (
+    JHTTPBadRequest, JHTTPNotFound, exception_response)
 from nefertari import engine
 
 log = logging.getLogger(__name__)
@@ -37,6 +38,7 @@ def perform_request(self, *args, **kw):
 
             return super(ESHttpConnection, self).perform_request(*args, **kw)
         except Exception as e:
+            log.error(e.error)
             status_code = e.status_code
             if status_code == 404:
                 raise IndexNotFoundException()
@@ -51,12 +53,35 @@ def perform_request(self, *args, **kw):
 def includeme(config):
     Settings = dictset(config.registry.settings)
     ES.setup(Settings)
+    ES.create_index()
 
 
 def _bulk_body(body):
     return ES.api.bulk(body=body)
 
 
+def process_fields_param(fields):
+    """ Process 'fields' ES param.
+
+    * Fields list is split if needed
+    * '_type' field is added, if not present, so the actual value is
+      displayed instead of 'None'
+    * '_source=False' is returned as well, so document source is not
+      loaded from ES. This is done because source is not used when
+      'fields' param is provided
+    """
+    if not fields:
+        return fields
+    if isinstance(fields, basestring):
+        fields = split_strip(fields)
+    if '_type' not in fields:
+        fields.append('_type')
+    return {
+        'fields': fields,
+        '_source': False,
+    }
+
+
 def apply_sort(_sort):
     _sort_param = []
 
@@ -137,11 +162,37 @@ def setup(cls, settings):
             raise Exception(
                 'Bad or missing settings for elasticsearch. %s' % e)
 
+    @classmethod
+    def create_index(cls, index_name=None):
+        index_name = index_name or ES.settings.index_name
+        try:
+            ES.api.indices.exists([index_name])
+        except IndexNotFoundException:
+            ES.api.indices.create(index_name)
+
+    @classmethod
+    def setup_mappings(cls):
+        models = engine.get_document_classes()
+        try:
+            for model_name, model_cls in models.items():
+                if getattr(model_cls, '_index_enabled', False):
+                    es = ES(model_cls.__name__)
+                    es.put_mapping(body=model_cls.get_es_mapping())
+        except JHTTPBadRequest as ex:
+            raise Exception(ex.json['extra']['data'])
+
     def __init__(self, source='', index_name=None, chunk_size=100):
         self.doc_type = self.src2type(source)
         self.index_name = index_name or ES.settings.index_name
         self.chunk_size = chunk_size
 
+    def put_mapping(self, body, **kwargs):
+        ES.api.indices.put_mapping(
+            doc_type=self.doc_type,
+            body=body,
+            index=self.index_name,
+            **kwargs)
+
     def process_chunks(self, documents, operation, chunk_size):
         """ Apply `operation` to chunks of `documents` of size `chunk_size`.
 
@@ -290,7 +341,9 @@ def get_by_ids(self, ids, **params):
             body=dict(docs=docs)
         )
         if fields:
-            params['fields'] = fields
+            fields_params = process_fields_param(fields)
+            params.update(fields_params)
+
         documents = _ESDocs()
         documents._nefertari_meta = dict(
             start=_start,
@@ -394,13 +447,15 @@ def get_collection(self, **params):
         if '_count' in params:
             return self.do_count(_params)
 
-        # pop the fields before passing to search.
-        # ES does not support passing names of nested structures
-        _fields = _params.pop('fields', '')
+        fields = _params.pop('fields', '')
+        if fields:
+            fields_params = process_fields_param(fields)
+            _params.update(fields_params)
+
         documents = _ESDocs()
         documents._nefertari_meta = dict(
             start=_params['from_'],
-            fields=_fields)
+            fields=fields)
 
         try:
             data = ES.api.search(**_params)
@@ -414,7 +469,7 @@ def get_collection(self, **params):
             return documents
 
         for da in data['hits']['hits']:
-            _d = da['fields'] if _fields else da['_source']
+            _d = da['fields'] if fields else da['_source']
             _d['_score'] = da['_score']
             documents.append(dict2obj(_d))
 

nefertari/resource.py

@@ -106,7 +106,7 @@ def add_route_and_view(config, action, route_name, path, request_method,
         'GET', traverse=_traverse)
 
     add_route_and_view(
-        config, 'update', name_prefix + member_name, path + id_name,
+        config, 'replace', name_prefix + member_name, path + id_name,
         'PUT', traverse=_traverse)
 
     add_route_and_view(

nefertari/view.py

@@ -142,6 +142,24 @@ def _run_init_actions(self):
         self.setup_default_wrappers()
         self.convert_ids2objects()
         self.set_public_limits()
+        if self.request.method == 'PUT':
+            self.fill_null_values()
+
+    def fill_null_values(self, model_cls=None):
+        """ Fill missing model fields in JSON with {key: None}.
+
+        Only run for PUT requests.
+        """
+        if model_cls is None:
+            model_cls = self._model_class
+        if not model_cls:
+            log.info("%s has no model defined" % self.__class__.__name__)
+            return
+
+        empty_values = model_cls.get_null_values()
+        for field, value in empty_values.items():
+            if field not in self._json_params:
+                self._json_params[field] = value
 
     def set_public_limits(self):
         """ Set public limits if auth is enabled and user is not
@@ -152,20 +170,22 @@ def set_public_limits(self):
         if auth_enabled and not getattr(self.request, 'user', None):
             wrappers.set_public_limits(self)
 
-    def convert_ids2objects(self):
+    def convert_ids2objects(self, model_cls=None):
         """ Convert object IDs from `self._json_params` to objects if needed.
 
         Only IDs tbat belong to relationship field of `self._model_class`
         are converted.
         """
-        if not self._model_class:
+        if model_cls is None:
+            model_cls = self._model_class
+        if not model_cls:
             log.info("%s has no model defined" % self.__class__.__name__)
             return
 
         for field in self._json_params.keys():
-            if not engine.is_relationship_field(field, self._model_class):
+            if not engine.is_relationship_field(field, model_cls):
                 continue
-            model_cls = engine.get_relationship_cls(field, self._model_class)
+            model_cls = engine.get_relationship_cls(field, model_cls)
             self.id2obj(field, model_cls)
 
     def get_debug(self, package=None):

tests/test_authentication/test_models.py

@@ -9,15 +9,15 @@ class TestModelHelpers(object):
 
     def test_lower_strip(self, engine_mock):
         from nefertari.authentication import models
-        assert models.lower_strip('Foo   ') == 'foo'
-        assert models.lower_strip(None) == ''
+        assert models.lower_strip(None, 'Foo   ') == 'foo'
+        assert models.lower_strip(None, None) == ''
 
     def test_encrypt_password(self, engine_mock):
         from nefertari.authentication import models
-        encrypted = models.encrypt_password('foo')
+        encrypted = models.encrypt_password(None, 'foo')
         assert models.crypt.match(encrypted)
         assert encrypted != 'foo'
-        assert encrypted == models.encrypt_password(encrypted)
+        assert encrypted == models.encrypt_password(None, encrypted)
 
     @patch('nefertari.authentication.models.uuid.uuid4')
     def test_create_apikey_token(self, mock_uuid, engine_mock):

tests/test_elasticsearch.py

@@ -40,6 +40,21 @@ def test_perform_request_no_index(self, mock_log):
 
 
 class TestHelperFunctions(object):
+    def test_process_fields_param_no_fields(self):
+        assert es.process_fields_param(None) is None
+
+    def test_process_fields_param_string(self):
+        assert es.process_fields_param('foo,bar') == {
+            'fields': ['foo', 'bar', '_type'],
+            '_source': False
+        }
+
+    def test_process_fields_param_list(self):
+        assert es.process_fields_param(['foo', 'bar']) == {
+            'fields': ['foo', 'bar', '_type'],
+            '_source': False
+        }
+
     @patch('nefertari.elasticsearch.ES')
     def test_includeme(self, mock_es):
         config = Mock()
@@ -368,15 +383,15 @@ def test_get_by_ids_fields(self, mock_mget):
         docs = obj.get_by_ids(documents, _limit=1, _fields=['name'])
         mock_mget.assert_called_once_with(
             body={'docs': [{'_index': 'foondex', '_type': 'story', '_id': 1}]},
-            fields=['name']
+            fields=['name', '_type'], _source=False
         )
         assert len(docs) == 1
         assert not hasattr(docs[0], '_id')
         assert not hasattr(docs[0], '_type')
         assert docs[0].name == 'bar'
         assert docs._nefertari_meta['total'] == 1
         assert docs._nefertari_meta['start'] == 0
-        assert docs._nefertari_meta['fields'] == ['name']
+        assert docs._nefertari_meta['fields'] == ['name', '_type']
 
     @patch('nefertari.elasticsearch.ES.api.mget')
     def test_get_by_ids_no_index_raise(self, mock_mget):
@@ -525,14 +540,16 @@ def test_get_collection_fields(self, mock_search):
         }
         docs = obj.get_collection(
             fields=['foo'], body={'foo': 'bar'}, from_=0)
-        mock_search.assert_called_once_with(body={'foo': 'bar'}, from_=0)
+        mock_search.assert_called_once_with(
+            body={'foo': 'bar'}, fields=['foo', '_type'], from_=0,
+            _source=False)
         assert len(docs) == 1
         assert docs[0].id == 1
         assert docs[0]._score == 2
         assert docs[0].foo == 'bar'
         assert docs._nefertari_meta['total'] == 4
         assert docs._nefertari_meta['start'] == 0
-        assert docs._nefertari_meta['fields'] == ['foo']
+        assert docs._nefertari_meta['fields'] == ['foo', '_type']
         assert docs._nefertari_meta['took'] == 2.8
 
     @patch('nefertari.elasticsearch.ES.api.search')

tests/test_resource.py

@@ -32,6 +32,9 @@ def __getattr__(self, attr):
         def convert_ids2objects(self, *args, **kwargs):
             pass
 
+        def fill_null_values(self, *args, **kwargs):
+            pass
+
     return View
 
 
@@ -218,6 +221,10 @@ def test_get_member(self):
 
     def test_put_member(self):
         result = self.app.put('/messages/1').body
+        self.assertEqual(result, 'replace')
+
+    def test_patch_member(self):
+        result = self.app.patch('/messages/1').body
         self.assertEqual(result, 'update')
 
     def test_delete_member(self):
@@ -303,11 +310,15 @@ def test_singular_resource(self, *a):
 
         self.assertEqual(app.delete('/grandpas/1').body, '"delete"')
 
-        self.assertEqual(app.put('/thing').body, '"update"')
+        self.assertEqual(app.put('/thing').body, '"replace"')
+
+        self.assertEqual(app.patch('/thing').body, '"update"')
 
         self.assertEqual(app.delete('/thing').body, '"delete"')
 
-        self.assertEqual(app.put('/grandpas/1/wife').body, '"update"')
+        self.assertEqual(app.put('/grandpas/1/wife').body, '"replace"')
+
+        self.assertEqual(app.patch('/grandpas/1/wife').body, '"update"')
 
         self.assertEqual(app.delete('/grandpas/1/wife').body, '"delete"')
 

tests/test_view.py

@@ -207,6 +207,21 @@ def test_run_init_actions(self, limit, conv, setpub):
         conv.assert_called_once_with()
         setpub.assert_called_once_with()
 
+    @patch('nefertari.view.BaseView._run_init_actions')
+    def test_fill_null_values(self, run):
+        request = Mock(content_type='', method='', accept=[''])
+        view = BaseView(
+            context={}, request=request,
+            _query_params={'foo': 'bar'})
+        view._model_class = Mock()
+        view._model_class.get_null_values.return_value = {
+            'name': None, 'email': 1, 'foo': None}
+        view._json_params = {'foo': 'bar'}
+        view.fill_null_values()
+        assert view._json_params == {
+            'foo': 'bar', 'name': None, 'email': 1
+        }
+
     @patch('nefertari.view.wrappers')
     @patch('nefertari.view.BaseView._run_init_actions')
     def test_set_public_limits_no_root(self, run, wrap):