Merge from master

ckan/config/environment.py

@@ -81,8 +81,11 @@ def find_controller(self, controller):
                           'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
-    # Check if SOLR schema is compatible
-    from ckan.lib.search import check_solr_schema_version
+    # Init SOLR settings and check if the schema is compatible
+    from ckan.lib.search import SolrSettings, check_solr_schema_version
+    SolrSettings.init(config.get('solr_url'),
+                      config.get('solr_user'),
+                      config.get('solr_password'))
     check_solr_schema_version()
 
     config['routes.map'] = make_map()

ckan/config/routing.py

@@ -8,14 +8,17 @@
 from pylons import config
 from routes import Mapper
 from ckan.plugins import PluginImplementations, IRoutes
-from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour
-from ckan.controllers.group   import register_pluggable_behaviour as register_group_behaviour
 
 
 routing_plugins = PluginImplementations(IRoutes)
 
 def make_map():
     """Create, configure and return the routes Mapper"""
+    # import controllers here rather than at root level because 
+    # pylons config is initialised by this point.
+    from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour
+    from ckan.controllers.group   import register_pluggable_behaviour as register_group_behaviour
+
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
     map.minimization = False
@@ -273,6 +276,7 @@ def make_map():
     map.connect('/tag/{id}', controller='tag', action='read')
     # users
     map.redirect("/users/{url:.*}", "/user/{url}")
+    map.redirect("/user/", "/user")
     map.connect('/user/edit', controller='user', action='edit')
     # Note: openid users have slashes in their ids, so need the wildcard
     # in the route.

ckan/controllers/group.py

@@ -1,3 +1,4 @@
+import logging
 import genshi
 import datetime
 from urllib import urlencode
@@ -9,7 +10,6 @@
 import ckan.authz as authz
 from ckan.authz import Authorizer
 from ckan.lib.helpers import Page
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.plugins import PluginImplementations, IGroupController, IGroupForm
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.logic import NotFound, NotAuthorized, ValidationError
@@ -19,6 +19,7 @@
 from ckan.lib.dictization.model_dictize import package_dictize
 import ckan.forms
 
+log = logging.getLogger(__name__)
 
 # Mapping from group-type strings to IDatasetForm instances
 _controller_behaviour_for = dict()
@@ -199,6 +200,7 @@ def index(self):
 
 
     def read(self, id):
+        from ckan.lib.search import SearchError
         group_type = self._get_group_type(id.split('@')[0])        
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
@@ -294,6 +296,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Group search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])

ckan/controllers/package.py

@@ -17,7 +17,6 @@
 from ckan.lib.base import request, c, BaseController, model, abort, h, g, render
 from ckan.lib.base import response, redirect, gettext
 from ckan.authz import Authorizer
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.lib.package_saver import PackageSaver, ValidationException
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.lib.helpers import json
@@ -31,7 +30,7 @@
 import ckan.rating
 import ckan.misc
 
-log = logging.getLogger('ckan.controllers')
+log = logging.getLogger(__name__)
 
 def search_url(params):
     url = h.url_for(controller='package', action='search')
@@ -198,6 +197,7 @@ def _setup_template_variables(self, context, data_dict, package_type=None):
     authorizer = ckan.authz.Authorizer()
 
     def search(self):
+        from ckan.lib.search import SearchError
         try:
             context = {'model':model,'user': c.user or c.author}
             check_access('site_read',context)
@@ -269,6 +269,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Package search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])
@@ -587,6 +588,7 @@ def _get_package_type(self, id):
         return data['type']
 
     def _save_new(self, context, package_type=None):
+        from ckan.lib.search import SearchIndexError
         try:
             data_dict = clean_dict(unflatten(
                 tuplize_dict(parse_params(request.POST))))
@@ -610,6 +612,7 @@ def _save_new(self, context, package_type=None):
             return self.new(data_dict, errors, error_summary)
 
     def _save_edit(self, id, context):
+        from ckan.lib.search import SearchIndexError
         try:
             package_type = self._get_package_type(id)
             data_dict = clean_dict(unflatten(

ckan/lib/cli.py

@@ -68,8 +68,9 @@ class ManageDb(CkanCommand):
     db version # returns current version of data schema
     db dump {file-path} # dump to a pg_dump file
     db dump-rdf {dataset-name} {file-path}
-    db simple-dump-csv {file-path}
-    db simple-dump-json {file-path}
+    db simple-dump-csv {file-path} # dump just datasets in CSV format
+    db simple-dump-json {file-path} # dump just datasets in JSON format
+    db user-dump-csv {file-path} # dump user information to a CSV file
     db send-rdf {talis-store} {username} {password}
     db load {file-path} # load a pg_dump from a file
     db load-only {file-path} # load a pg_dump from a file but don\'t do
@@ -115,6 +116,8 @@ def command(self):
             self.simple_dump_json()
         elif cmd == 'dump-rdf':
             self.dump_rdf()
+        elif cmd == 'user-dump-csv':
+            self.user_dump_csv()
         elif cmd == 'create-from-model':
             model.repo.create_db()
             if self.verbose:
@@ -238,6 +241,15 @@ def dump_rdf(self):
         f.write(rdf)
         f.close()
 
+    def user_dump_csv(self):
+        if len(self.args) < 2:
+            print 'Need csv file path'
+            return
+        dump_filepath = self.args[1]
+        import ckan.lib.dumper as dumper
+        dump_file = open(dump_filepath, 'w')
+        dumper.UserDumper().dump(dump_file)
+
     def send_rdf(self):
         if len(self.args) < 4:
             print 'Need all arguments: {talis-store} {username} {password}'

ckan/lib/dumper.py

@@ -4,7 +4,7 @@
 
 import ckan.model as model
 import ckan.model
-from helpers import json
+from helpers import json, OrderedDict
 
 class SimpleDumper(object):
     '''Dumps just package data but including tags, groups, license text etc'''
@@ -40,7 +40,7 @@ def dump_csv(self, dump_file_obj, query):
                         pkg_dict[name_] = value_
                     del pkg_dict[name]
             row_dicts.append(pkg_dict)
-        writer = PackagesCsvWriter(row_dicts)
+        writer = CsvWriter(row_dicts)
         writer.save(dump_file_obj)
 
     def dump_json(self, dump_file_obj, query):
@@ -201,7 +201,7 @@ def migrate_06_to_07(self):
                         values={'name': record.name})
                 update.execute()
 
-class PackagesCsvWriter:
+class CsvWriter:
     def __init__(self, package_dict_list=None):
         self._rows = []
         self._col_titles = []
@@ -301,3 +301,25 @@ def pkg_to_xl_dict(pkg):
                     dict_[key_] = value_
                 del dict_[key]
         return dict_
+
+class UserDumper(object):
+    def dump(self, dump_file_obj):
+        query = model.Session.query(model.User)
+        query = query.order_by(model.User.created.asc())
+
+        columns = (('id', 'name', 'openid', 'fullname', 'email', 'created', 'about'))
+        row_dicts = []
+        for user in query:
+            row = OrderedDict()
+            for col in columns:
+                value = getattr(user, col)
+                if not value:
+                    value = ''
+                if col == 'created':
+                    value = str(value) # or maybe dd/mm/yyyy?
+                row[col] = value
+            row_dicts.append(row)
+
+        writer = CsvWriter(row_dicts)
+        writer.save(dump_file_obj)
+        dump_file_obj.close()

ckan/lib/hash.py

@@ -3,9 +3,15 @@
 
 from pylons import config, request
 
-secret = config['beaker.session.secret']
+global secret
+secret = None
 
 def get_message_hash(value):
+    if not secret:
+        global secret
+        # avoid getting config value at module scope since config may
+        # not be read in yet
+        secret = config['beaker.session.secret']
     return hmac.new(secret, value, hashlib.sha1).hexdigest()
 
 def get_redirect():

ckan/lib/navl/dictization_functions.py

@@ -60,9 +60,10 @@ def flatten_schema(schema, flattened=None, key=None):
     return flattened
 
 def get_all_key_combinations(data, flattented_schema):
-    '''compare the schema agaist the given data and get all valid tuples that
-    match the schema ignoring the last value in the tuple.'''
+    '''Compare the schema against the given data and get all valid tuples that
+    match the schema ignoring the last value in the tuple.
 
+    '''
     schema_prefixes = set([key[:-1] for key in flattented_schema])
     combinations = set([()])
 
@@ -206,7 +207,7 @@ def _remove_blank_keys(schema):
     return schema
 
 def validate(data, schema, context=None):
-    '''validate an unflattened nested dict agiast a schema'''
+    '''Validate an unflattened nested dict against a schema.'''
 
     context = context or {}
 

ckan/lib/search/__init__.py

@@ -7,7 +7,7 @@
 from ckan.logic import get_action
 
 from common import (SearchIndexError, SearchError, SearchQueryError,
-                    make_connection, is_available, DEFAULT_SOLR_URL)
+                    make_connection, is_available, SolrSettings)
 from index import PackageSearchIndex, NoopSearchIndex
 from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions, convert_legacy_parameters_to_solr
 
@@ -197,15 +197,13 @@ def check_solr_schema_version(schema_file=None):
 
     # Try to get the schema XML file to extract the version
     if not schema_file:
-        solr_user = config.get('solr_user')
-        solr_password = config.get('solr_password')
+        solr_url, solr_user, solr_password = SolrSettings.get()
 
         http_auth = None
         if solr_user is not None and solr_password is not None:
             http_auth = solr_user + ':' + solr_password
             http_auth = 'Basic ' + http_auth.encode('base64').strip()
 
-        solr_url = config.get('solr_url', DEFAULT_SOLR_URL)
         url = solr_url.strip('/') + SOLR_SCHEMA_FILE_OFFSET
 
         req = urllib2.Request(url = url)

ckan/lib/search/common.py

@@ -8,9 +8,29 @@ class SearchQueryError(SearchError): pass
 
 DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr'
 
-solr_url = config.get('solr_url', DEFAULT_SOLR_URL)
-solr_user = config.get('solr_user')
-solr_password = config.get('solr_password')
+class SolrSettings(object):
+    _is_initialised = False
+    _url = None
+    _user = None
+    _password = None
+
+    @classmethod
+    def init(cls, url, user=None, password=None):
+        if url is not None:
+            cls._url = url
+            cls._user = user
+            cls._password = password
+        else:
+            cls._url = DEFAULT_SOLR_URL
+        cls._is_initialised = True
+
+    @classmethod
+    def get(cls):
+        if not cls._is_initialised:
+            raise SearchIndexError('SOLR URL not initialised')
+        if not cls._url:
+            raise SearchIndexError('SOLR URL is blank')
+        return (cls._url, cls._user, cls._password)
 
 def is_available():
     """
@@ -23,12 +43,15 @@ def is_available():
         log.exception(e)
         return False
     finally:
-        conn.close()
+        if 'conn' in dir():
+            conn.close()
 
     return True
 
 def make_connection():
     from solr import SolrConnection
+    solr_url, solr_user, solr_password = SolrSettings.get()
+    assert solr_url is not None
     if solr_user is not None and solr_password is not None:
         return SolrConnection(solr_url, http_user=solr_user, http_pass=solr_password)
     else:

ckan/lib/search/index.py

@@ -19,14 +19,21 @@
 RELATIONSHIP_TYPES = PackageRelationship.types
 
 def clear_index():
+    import solr.core
     conn = make_connection()
     query = "+site_id:\"%s\"" % (config.get('ckan.site_id'))
     try:
         conn.delete_query(query)
         conn.commit()
     except socket.error, e:
-        log.error('Could not connect to SOLR: %r' % e)
+        err = 'Could not connect to SOLR %r: %r' % (conn.url, e)
+        log.error(err)
+        raise SearchIndexError(err)
         raise
+##    except solr.core.SolrException, e:
+##        err = 'SOLR %r exception: %r' % (conn.url, e)
+##        log.error(err)
+##        raise SearchIndexError(err)
     finally:
         conn.close()
 

ckan/lib/search/query.py

@@ -292,7 +292,6 @@ def run(self, query):
 
         conn = make_connection()
         log.debug('Package query: %r' % query)
-
         try:
             solr_response = conn.raw_query(**query)
         except SolrException, e:

ckan/lib/search/sql.py

@@ -24,7 +24,7 @@ def run(self, query):
         def makelike(field):
             _attr = getattr(model.Package, field)
             return _attr.ilike('%' + term + '%')
-        if q and not (q == '""' or q == "''"):
+        if q and q not in ('""', "''", '*:*'):
             terms = q.split()
             # TODO: tags ...?
             fields = ['name', 'title', 'notes']

ckan/logic/schema.py

@@ -39,7 +39,7 @@ def default_resource_schema():
 
     schema = {
         'id': [ignore_empty, unicode],
-        'revistion_id': [ignore_missing, unicode],
+        'revision_id': [ignore_missing, unicode],
         'resource_group_id': [ignore],
         'package_id': [ignore],
         'url': [ignore_empty, unicode],#, URL(add_http=False)],