[master][#1708][config,controllers,lib]: Fix bug causing SOLR setting…

…s being lost.

ckan/config/environment.py

@@ -81,8 +81,11 @@ def find_controller(self, controller):
                           'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
-    # Check if SOLR schema is compatible
-    from ckan.lib.search import check_solr_schema_version
+    # Init SOLR settings and check if the schema is compatible
+    from ckan.lib.search import SolrSettings, check_solr_schema_version
+    SolrSettings.init(config.get('solr_url'),
+                      config.get('solr_user'),
+                      config.get('solr_password'))
     check_solr_schema_version()
 
     config['routes.map'] = make_map()

ckan/controllers/group.py

@@ -9,7 +9,6 @@
 import ckan.authz as authz
 from ckan.authz import Authorizer
 from ckan.lib.helpers import Page
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.plugins import PluginImplementations, IGroupController, IGroupForm
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.logic import NotFound, NotAuthorized, ValidationError
@@ -203,6 +202,7 @@ def index(self):
 
 
     def read(self, id):
+        from ckan.lib.search import SearchError
         group_type = self._get_group_type(id.split('@')[0])        
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
@@ -298,6 +298,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Group search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])

ckan/controllers/package.py

@@ -17,7 +17,6 @@
 from ckan.lib.base import request, c, BaseController, model, abort, h, g, render
 from ckan.lib.base import response, redirect, gettext
 from ckan.authz import Authorizer
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.lib.package_saver import PackageSaver, ValidationException
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.lib.helpers import json
@@ -206,6 +205,7 @@ def _setup_template_variables(self, context, data_dict, package_type=None):
     authorizer = ckan.authz.Authorizer()
 
     def search(self):
+        from ckan.lib.search import SearchError
         try:
             context = {'model':model,'user': c.user or c.author}
             check_access('site_read',context)
@@ -277,6 +277,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Package search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])
@@ -593,6 +594,7 @@ def _get_package_type(self, id):
         return data['type']
 
     def _save_new(self, context, package_type=None):
+        from ckan.lib.search import SearchIndexError
         try:
             data_dict = clean_dict(unflatten(
                 tuplize_dict(parse_params(request.POST))))
@@ -616,6 +618,7 @@ def _save_new(self, context, package_type=None):
             return self.new(data_dict, errors, error_summary)
 
     def _save_edit(self, id, context):
+        from ckan.lib.search import SearchIndexError
         try:
             package_type = self._get_package_type(id)
             data_dict = clean_dict(unflatten(

ckan/lib/hash.py

@@ -3,9 +3,15 @@
 
 from pylons import config, request
 
-secret = config['beaker.session.secret']
+global secret
+secret = None
 
 def get_message_hash(value):
+    if not secret:
+        global secret
+        # avoid getting config value at module scope since config may
+        # not be read in yet
+        secret = config['beaker.session.secret']
     return hmac.new(secret, value, hashlib.sha1).hexdigest()
 
 def get_redirect():

ckan/lib/search/__init__.py

@@ -7,7 +7,7 @@
 from ckan.logic import get_action
 
 from common import (SearchIndexError, SearchError, SearchQueryError,
-                    make_connection, is_available, DEFAULT_SOLR_URL)
+                    make_connection, is_available, SolrSettings)
 from index import PackageSearchIndex, NoopSearchIndex
 from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions, convert_legacy_parameters_to_solr
 
@@ -197,15 +197,13 @@ def check_solr_schema_version(schema_file=None):
 
     # Try to get the schema XML file to extract the version
     if not schema_file:
-        solr_user = config.get('solr_user')
-        solr_password = config.get('solr_password')
+        solr_url, solr_user, solr_password = SolrSettings.get()
 
         http_auth = None
         if solr_user is not None and solr_password is not None:
             http_auth = solr_user + ':' + solr_password
             http_auth = 'Basic ' + http_auth.encode('base64').strip()
 
-        solr_url = config.get('solr_url', DEFAULT_SOLR_URL)
         url = solr_url.strip('/') + SOLR_SCHEMA_FILE_OFFSET
 
         req = urllib2.Request(url = url)

ckan/lib/search/common.py

@@ -8,9 +8,29 @@ class SearchQueryError(SearchError): pass
 
 DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr'
 
-solr_url = config.get('solr_url', DEFAULT_SOLR_URL)
-solr_user = config.get('solr_user')
-solr_password = config.get('solr_password')
+class SolrSettings(object):
+    _is_initialised = False
+    _url = None
+    _user = None
+    _password = None
+
+    @classmethod
+    def init(cls, url, user=None, password=None):
+        if url is not None:
+            cls._url = url
+            cls._user = user
+            cls._password = password
+        else:
+            cls._url = DEFAULT_SOLR_URL
+        cls._is_initialised = True
+
+    @classmethod
+    def get(cls):
+        if not cls._is_initialised:
+            raise SearchIndexError('SOLR URL not initialised')
+        if not cls._url:
+            raise SearchIndexError('SOLR URL is blank')
+        return (cls._url, cls._user, cls._password)
 
 def is_available():
     """
@@ -23,12 +43,15 @@ def is_available():
         log.exception(e)
         return False
     finally:
-        conn.close()
+        if 'conn' in dir():
+            conn.close()
 
     return True
 
 def make_connection():
     from solr import SolrConnection
+    solr_url, solr_user, solr_password = SolrSettings.get()
+    assert solr_url is not None
     if solr_user is not None and solr_password is not None:
         return SolrConnection(solr_url, http_user=solr_user, http_pass=solr_password)
     else:

ckan/lib/search/index.py

@@ -19,14 +19,21 @@
 RELATIONSHIP_TYPES = PackageRelationship.types
 
 def clear_index():
+    import solr.core
     conn = make_connection()
     query = "+site_id:\"%s\"" % (config.get('ckan.site_id'))
     try:
         conn.delete_query(query)
         conn.commit()
     except socket.error, e:
-        log.error('Could not connect to SOLR: %r' % e)
+        err = 'Could not connect to SOLR %r: %r' % (conn.url, e)
+        log.error(err)
+        raise SearchIndexError(err)
         raise
+##    except solr.core.SolrException, e:
+##        err = 'SOLR %r exception: %r' % (conn.url, e)
+##        log.error(err)
+##        raise SearchIndexError(err)
     finally:
         conn.close()
 

ckan/lib/search/query.py

@@ -292,7 +292,6 @@ def run(self, query):
 
         conn = make_connection()
         log.debug('Package query: %r' % query)
-
         try:
             solr_response = conn.raw_query(**query)
         except SolrException, e:

ckan/tests/functional/api/model/test_package.py

@@ -5,6 +5,7 @@
 from ckan.lib.create_test_data import CreateTestData
 from ckan import plugins
 import ckan.lib.search as search
+from ckan.lib.search.common import SolrSettings
 
 from ckan.tests.functional.api.base import BaseModelApiTestCase
 from ckan.tests.functional.api.base import Api1TestCase as Version1TestCase
@@ -266,9 +267,9 @@ def test_register_post_indexerror(self):
         Test that we can't add a package if Solr is down.
         """
         bad_solr_url = 'http://127.0.0.1/badsolrurl'
-        solr_url = search.common.solr_url
+        original_settings = SolrSettings.get()[0]
         try:
-            search.common.solr_url = bad_solr_url
+            SolrSettings.init(bad_solr_url)
             plugins.load('synchronous_search')
 
             assert not self.get_package_by_name(self.package_fixture_data['name'])
@@ -279,7 +280,7 @@ def test_register_post_indexerror(self):
             model.Session.remove()
         finally:
             plugins.unload('synchronous_search')
-            search.common.solr_url = solr_url
+            SolrSettings.init(original_settings)
 
     def test_register_post_tag_too_long(self):
         pkg = {'name': 'test_tag_too_long',
@@ -655,17 +656,17 @@ def test_entity_update_indexerror(self):
         Test that we can't update a package if Solr is down.
         """
         bad_solr_url = 'http://127.0.0.1/badsolrurl'
-        solr_url = search.common.solr_url
+        original_settings = SolrSettings.get()[0]
         try:
-            search.common.solr_url = bad_solr_url
+            SolrSettings.init(bad_solr_url)
             plugins.load('synchronous_search')
 
             assert_raises(
                 search.SearchIndexError, self.assert_package_update_ok, 'name', 'post'
             )
         finally:
             plugins.unload('synchronous_search')
-            search.common.solr_url = solr_url
+            SolrSettings.init(original_settings)
 
     def test_entity_delete_ok(self):
         # create a package with package_fixture_data

ckan/tests/functional/test_package.py

@@ -22,6 +22,7 @@
 from ckan.plugins import SingletonPlugin, implements, IPackageController
 from ckan import plugins
 from ckan.rating import set_rating
+from ckan.lib.search.common import SolrSettings
 
 class MockPackageControllerPlugin(SingletonPlugin):
     implements(IPackageController)
@@ -997,9 +998,9 @@ def test_edit_404(self):
 
     def test_edit_indexerror(self):
         bad_solr_url = 'http://127.0.0.1/badsolrurl'
-        solr_url = search.common.solr_url
+        solr_url = SolrSettings.get()[0]
         try:
-            search.common.solr_url = bad_solr_url
+            SolrSettings.init(bad_solr_url)
             plugins.load('synchronous_search')
 
             fv = self.res.forms['dataset-edit']
@@ -1009,7 +1010,7 @@ def test_edit_indexerror(self):
             assert 'Unable to update search index' in res, res
         finally:
             plugins.unload('synchronous_search')
-            search.common.solr_url = solr_url
+            SolrSettings.init(solr_url)
 
 
 class TestNew(TestPackageForm):
@@ -1248,9 +1249,9 @@ def test_new_plugin_hook(self):
 
     def test_new_indexerror(self):
         bad_solr_url = 'http://127.0.0.1/badsolrurl'
-        solr_url = search.common.solr_url
+        solr_url = SolrSettings.get()[0]
         try:
-            search.common.solr_url = bad_solr_url
+            SolrSettings.init(bad_solr_url)
             plugins.load('synchronous_search')
             new_package_name = u'new-package-missing-solr'
 
@@ -1267,7 +1268,7 @@ def test_new_indexerror(self):
             assert 'Unable to add package to search index' in res, res
         finally:
             plugins.unload('synchronous_search')
-            search.common.solr_url = solr_url
+            SolrSettings.init(solr_url)
 
 class TestSearch(TestPackageForm):
     pkg_names = []

ckan/tests/functional/test_pagination.py

@@ -64,6 +64,7 @@ def test_package_search_p2(self):
     def test_group_read_p1(self):
         res = self.app.get(url_for(controller='group', action='read', id='group_00'))
         assert 'href="/group/group_00?page=2' in res
+        print res.body
         pkg_numbers = scrape_search_results(res, 'dataset')
         assert_equal(['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29'], pkg_numbers)
 

ckan/tests/lib/test_solr_search_index.py

@@ -96,6 +96,10 @@ def test_index_clear(self):
         search.index_for('Package').clear()
         response = self.solr.query('title:penguin', fq=self.fq)
         assert len(response) == 0
+        # clear whilst empty
+        search.index_for('Package').clear()
+        response = self.solr.query('title:penguin', fq=self.fq)
+        assert len(response) == 0
 
 
 class TestSolrSearch:

doc/configuration.rst

@@ -493,14 +493,28 @@ solr_url
 
 Example::
 
- solr_url = http://solr.okfn.org:8983/solr
+ solr_url = http://solr.okfn.org:8983/solr/ckan-schema-1.3
+
+Default value:  ``http://solr.okfn.org:8983/solr``
 
-This configures the Solr server used for search. The SOLR schema must be one of the ones in ``ckan/config/solr`` (generally the last one).
+This configures the Solr server used for search. The Solr schema found at that URL must be one of the ones in ``ckan/config/solr`` (generally the most recent one). A check of the schema version number occurs when CKAN starts.
 
-Optionally, ``solr_user`` and ``solr_password`` can also be passed along to specify HTTP Basic authentication details for all Solr requests. 
+Optionally, ``solr_user`` and ``solr_password`` can also be configured to specify HTTP Basic authentication details for all Solr requests. 
 
 Note, if you change this value, you need to rebuild the search index.
 
+simple_search
+^^^^^^^^^^^^^
+
+Example::
+
+ ckan.simple_search = true
+
+Default value:  ``false``
+
+Switching this on tells CKAN search functionality to just query the database, (rather than using Solr). In this setup, search is crude and limited, e.g. no full-text search, no faceting, etc. However, this might be very useful for getting up and running quickly with CKAN.
+
+
 Site Settings
 -------------
 
@@ -593,7 +607,7 @@ plugins
 
 Example::
 
-  ckan.plugins = disqus synchronous_search datapreview googleanalytics stats storage follower
+  ckan.plugins = disqus datapreview googleanalytics follower
 
 Specify which CKAN extensions are to be enabled.