Merge pull request #127 from postatum/engine-refactor

Engine refactor

nefertari/elasticsearch.py

@@ -372,7 +372,7 @@ def get_by_ids(self, ids, **params):
         if not ids:
             return _ESDocs()
 
-        __raise_on_empty = params.pop('__raise_on_empty', False)
+        _raise_on_empty = params.pop('_raise_on_empty', False)
         fields = params.pop('_fields', [])
 
         _limit = params.pop('_limit', len(ids))
@@ -406,7 +406,7 @@ def get_by_ids(self, ids, **params):
         try:
             data = self.api.mget(**params)
         except IndexNotFoundException:
-            if __raise_on_empty:
+            if _raise_on_empty:
                 raise JHTTPNotFound(
                     '{}({}) resource not found (Index does not exist)'.format(
                         self.doc_type, params))
@@ -420,7 +420,7 @@ def get_by_ids(self, ids, **params):
             except KeyError:
                 msg = "ES: '%s(%s)' resource not found" % (
                     found_doc['_type'], found_doc['_id'])
-                if __raise_on_empty:
+                if _raise_on_empty:
                     raise JHTTPNotFound(msg)
                 else:
                     log.error(msg)
@@ -500,15 +500,15 @@ def aggregate(self, **params):
         Arguments:
             :_aggregations_params: Dict of aggregation params. Root key is an
                 aggregation name. Required.
-            :__raise_on_empty: Boolean indicating whether to raise exception
+            :_raise_on_empty: Boolean indicating whether to raise exception
                 when IndexNotFoundException exception happens. Optional,
                 defaults to False.
             :_search_type: Type of search to use. Optional, defaults to
                 'count'. You might want to provide this argument explicitly
                 when performing nested aggregations on buckets.
         """
         _aggregations_params = params.pop('_aggregations_params', None)
-        __raise_on_empty = params.pop('__raise_on_empty', False)
+        _raise_on_empty = params.pop('_raise_on_empty', False)
         _search_type = params.pop('_search_type', 'count')
 
         if not _aggregations_params:
@@ -528,7 +528,7 @@ def aggregate(self, **params):
         try:
             response = self.api.search(**search_params)
         except IndexNotFoundException:
-            if __raise_on_empty:
+            if _raise_on_empty:
                 raise JHTTPNotFound(
                     'Aggregation failed: Index does not exist')
             return {}
@@ -539,7 +539,7 @@ def aggregate(self, **params):
             raise JHTTPNotFound('No aggregations returned from ES')
 
     def get_collection(self, **params):
-        __raise_on_empty = params.pop('__raise_on_empty', False)
+        _raise_on_empty = params.pop('_raise_on_empty', False)
 
         if 'body' in params:
             _params = params
@@ -562,7 +562,7 @@ def get_collection(self, **params):
         try:
             data = self.api.search(**_params)
         except IndexNotFoundException:
-            if __raise_on_empty:
+            if _raise_on_empty:
                 raise JHTTPNotFound(
                     '{}({}) resource not found (Index does not exist)'.format(
                         self.doc_type, params))
@@ -583,15 +583,15 @@ def get_collection(self, **params):
 
         if not documents:
             msg = "%s(%s) resource not found" % (self.doc_type, params)
-            if __raise_on_empty:
+            if _raise_on_empty:
                 raise JHTTPNotFound(msg)
             else:
                 log.debug(msg)
 
         return documents
 
     def get_item(self, **kw):
-        __raise_on_empty = kw.pop('__raise_on_empty', True)
+        _raise_on_empty = kw.pop('_raise_on_empty', True)
 
         params = dict(
             index=self.index_name,
@@ -604,15 +604,15 @@ def get_item(self, **kw):
         try:
             data = self.api.get_source(**params)
         except IndexNotFoundException:
-            if __raise_on_empty:
+            if _raise_on_empty:
                 raise JHTTPNotFound("{} (Index does not exist)".format(
                     not_found_msg, self.doc_type, params))
             data = {}
         except JHTTPNotFound:
             data = {}
 
         if not data:
-            if __raise_on_empty:
+            if _raise_on_empty:
                 raise JHTTPNotFound(not_found_msg)
             else:
                 log.debug(not_found_msg)

nefertari/engine.py

@@ -32,18 +32,35 @@
 
 nefertari relies on 'nefertari.engine' being included when configuring the app.
 """
-
+import sys
 from zope.dottedname.resolve import resolve
+from pyramid.settings import aslist
 
 
 def includeme(config):
-    def _valid_global(g):
-        ignored = ('log', 'includeme')
-        return (not g.startswith('__') and g not in ignored)
-
-    engine_path = config.registry.settings['nefertari.engine']
-    config.include(engine_path)
-    engine_module = resolve(engine_path)
-    engine_globals = {k: v for k, v in engine_module.__dict__.items()
-                      if _valid_global(k)}
-    globals().update(engine_globals)
+    engine_paths = aslist(config.registry.settings['nefertari.engine'])
+    for path in engine_paths:
+        config.include(path)
+    _load_engines(config)
+    main_engine_module = engines[0]
+    _import_public_names(main_engine_module)
+
+
+# replaced by registered engine modules during configuration
+engines = ()
+
+
+def _load_engines(config):
+    global engines
+    engine_paths = aslist(config.registry.settings['nefertari.engine'])
+    engines = tuple([resolve(path) for path in engine_paths])
+
+
+def _import_public_names(module):
+    "Import public names from module into this module, like import *"
+    self = sys.modules[__name__]
+    for name in module.__all__:
+        if hasattr(self, name):
+            # don't overwrite existing names
+            continue
+        setattr(self, name, getattr(module, name))

nefertari/view.py

@@ -352,7 +352,7 @@ def _get_object(id_):
                 return id_
 
             obj = model.get_item(
-                **{pk_field: id_, '__raise_on_empty': False})
+                **{pk_field: id_, '_raise_on_empty': False})
             if setdefault:
                 return obj or setdefault
             else:

tests/test_elasticsearch.py

@@ -457,7 +457,7 @@ def test_get_by_ids_no_index_raise(self, mock_mget):
         documents = [{'_id': 1, '_type': 'Story'}]
         mock_mget.side_effect = es.IndexNotFoundException()
         with pytest.raises(JHTTPNotFound) as ex:
-            obj.get_by_ids(documents, __raise_on_empty=True)
+            obj.get_by_ids(documents, _raise_on_empty=True)
         assert 'resource not found (Index does not exist)' in str(ex.value)
 
     @patch('nefertari.elasticsearch.ES.api.mget')
@@ -466,7 +466,7 @@ def test_get_by_ids_no_index_not_raise(self, mock_mget):
         documents = [{'_id': 1, '_type': 'Story'}]
         mock_mget.side_effect = es.IndexNotFoundException()
         try:
-            docs = obj.get_by_ids(documents, __raise_on_empty=False)
+            docs = obj.get_by_ids(documents, _raise_on_empty=False)
         except JHTTPNotFound:
             raise Exception('Unexpected error')
         assert len(docs) == 0
@@ -477,15 +477,15 @@ def test_get_by_ids_not_found_raise(self, mock_mget):
         documents = [{'_id': 1, '_type': 'Story'}]
         mock_mget.return_value = {'docs': [{'_type': 'foo', '_id': 1}]}
         with pytest.raises(JHTTPNotFound):
-            obj.get_by_ids(documents, __raise_on_empty=True)
+            obj.get_by_ids(documents, _raise_on_empty=True)
 
     @patch('nefertari.elasticsearch.ES.api.mget')
     def test_get_by_ids_not_found_not_raise(self, mock_mget):
         obj = es.ES('Foo', 'foondex')
         documents = [{'_id': 1, '_type': 'Story'}]
         mock_mget.return_value = {'docs': [{'_type': 'foo', '_id': 1}]}
         try:
-            docs = obj.get_by_ids(documents, __raise_on_empty=False)
+            docs = obj.get_by_ids(documents, _raise_on_empty=False)
         except JHTTPNotFound:
             raise Exception('Unexpected error')
         assert len(docs) == 0
@@ -652,7 +652,7 @@ def test_aggregation_index_not_exists(self, mock_search, mock_build):
         obj = es.ES('Foo', 'foondex')
         with pytest.raises(JHTTPNotFound) as ex:
             obj.aggregate(_aggregations_params={'zoo': 5}, param1=6,
-                          __raise_on_empty=True)
+                          _raise_on_empty=True)
         assert 'Aggregation failed: Index does not exist' in str(ex.value)
 
     @patch('nefertari.elasticsearch.ES.build_search_params')
@@ -731,7 +731,7 @@ def test_get_collection_no_index_raise(self, mock_search):
         mock_search.side_effect = es.IndexNotFoundException()
         with pytest.raises(JHTTPNotFound) as ex:
             obj.get_collection(
-                body={'foo': 'bar'}, __raise_on_empty=True,
+                body={'foo': 'bar'}, _raise_on_empty=True,
                 from_=0)
         assert 'resource not found (Index does not exist)' in str(ex.value)
 
@@ -741,7 +741,7 @@ def test_get_collection_no_index_not_raise(self, mock_search):
         mock_search.side_effect = es.IndexNotFoundException()
         try:
             docs = obj.get_collection(
-                body={'foo': 'bar'}, __raise_on_empty=False,
+                body={'foo': 'bar'}, _raise_on_empty=False,
                 from_=0)
         except JHTTPNotFound:
             raise Exception('Unexpected error')
@@ -759,7 +759,7 @@ def test_get_collection_not_found_raise(self, mock_search):
         }
         with pytest.raises(JHTTPNotFound):
             obj.get_collection(
-                body={'foo': 'bar'}, __raise_on_empty=True,
+                body={'foo': 'bar'}, _raise_on_empty=True,
                 from_=0)
 
     @patch('nefertari.elasticsearch.ES.api.search')
@@ -774,7 +774,7 @@ def test_get_collection_not_found_not_raise(self, mock_search):
         }
         try:
             docs = obj.get_collection(
-                body={'foo': 'bar'}, __raise_on_empty=False,
+                body={'foo': 'bar'}, _raise_on_empty=False,
                 from_=0)
         except JHTTPNotFound:
             raise Exception('Unexpected error')
@@ -803,7 +803,7 @@ def test_get_item_no_index_not_raise(self, mock_get):
         obj = es.ES('Foo', 'foondex')
         mock_get.side_effect = es.IndexNotFoundException()
         try:
-            obj.get_item(name='foo', __raise_on_empty=False)
+            obj.get_item(name='foo', _raise_on_empty=False)
         except JHTTPNotFound:
             raise Exception('Unexpected error')
 
@@ -819,7 +819,7 @@ def test_get_item_not_found_not_raise(self, mock_get):
         obj = es.ES('Foo', 'foondex')
         mock_get.return_value = {}
         try:
-            obj.get_item(name='foo', __raise_on_empty=False)
+            obj.get_item(name='foo', _raise_on_empty=False)
         except JHTTPNotFound:
             raise Exception('Unexpected error')
 

tests/test_engine.py

@@ -10,6 +10,8 @@ def test_includeme(self, mock_resolve):
         module.log = 1
         module.__testvar__ = 3
         module.another_var = 4
+        module.includeme = 42
+        module.__all__ = ['another_var', 'includeme']
         mock_resolve.return_value = module
         from nefertari import engine
         assert not hasattr(engine, 'log')
@@ -19,7 +21,30 @@ def test_includeme(self, mock_resolve):
         engine.includeme(config)
 
         config.include.assert_called_once_with('foo')
-        mock_resolve.assert_called_once_with('foo')
+        mock_resolve.assert_called_with('foo')
         assert not hasattr(engine, 'log')
         assert not hasattr(engine, '__testvar__')
         assert hasattr(engine, 'another_var')
+        assert engine.engines == (module, )
+
+    @patch('nefertari.engine.resolve')
+    def test_multiple_engines(self, mock_resolve):
+        from nefertari import engine
+        foo = Mock()
+        bar = Mock()
+        foo.__all__ = ['one', 'two']
+        bar.__all__ = ['three', 'four']
+        config = Mock()
+        config.registry.settings = {'nefertari.engine': ['foo', 'bar']}
+        mock_resolve.side_effect = lambda m: foo if m == 'foo' else bar
+        engine.includeme(config)
+
+        config.include.assert_any_call('foo')
+        config.include.assert_any_call('bar')
+        mock_resolve.assert_any_call('foo')
+        mock_resolve.assert_any_call('bar')
+        assert not hasattr(engine, 'three')
+        assert not hasattr(engine, 'four')
+        assert hasattr(engine, 'one')
+        assert hasattr(engine, 'two')
+        assert engine.engines == (foo, bar)

tests/test_view.py

@@ -563,7 +563,7 @@ def test_id2obj(self, run):
         assert view._json_params['user'] == 'foo'
         model.pk_field.assert_called_once_with()
         model.get_item.assert_called_once_with(
-            idname='1', __raise_on_empty=False)
+            idname='1', _raise_on_empty=False)
 
     @patch('nefertari.view.BaseView._run_init_actions')
     def test_id2obj_list(self, run):
@@ -579,7 +579,7 @@ def test_id2obj_list(self, run):
         assert view._json_params['user'] == ['foo']
         model.pk_field.assert_called_once_with()
         model.get_item.assert_called_once_with(
-            idname='1', __raise_on_empty=False)
+            idname='1', _raise_on_empty=False)
 
     @patch('nefertari.view.BaseView._run_init_actions')
     def test_id2obj_not_in_params(self, run):
@@ -606,7 +606,7 @@ def test_id2obj_setdefault(self, run):
         assert view._json_params['user'] == 123
         model.pk_field.assert_called_once_with()
         model.get_item.assert_called_once_with(
-            idname='1', __raise_on_empty=False)
+            idname='1', _raise_on_empty=False)
 
     @patch('nefertari.view.BaseView._run_init_actions')
     def test_id2obj_value_none(self, run):