Merge branch 'master' of github.com:okfn/ckan into feature-1698-tag-t…

…axonomies

Conflicts:
	ckan/logic/action/create.py
	ckan/logic/action/delete.py
	ckan/logic/action/update.py

ckan/config/environment.py

@@ -81,8 +81,11 @@ def find_controller(self, controller):
                           'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
-    # Check if SOLR schema is compatible
-    from ckan.lib.search import check_solr_schema_version
+    # Init SOLR settings and check if the schema is compatible
+    from ckan.lib.search import SolrSettings, check_solr_schema_version
+    SolrSettings.init(config.get('solr_url'),
+                      config.get('solr_user'),
+                      config.get('solr_password'))
     check_solr_schema_version()
 
     config['routes.map'] = make_map()

ckan/config/routing.py

@@ -79,6 +79,10 @@ def make_map():
         controller='api', action='list',
         requirements=dict(register=register_list_str),
         conditions=dict(method=['GET']))
+    map.connect('/api/{ver:1|2}/rest/{register}/{id}/:subregister',
+        controller='api', action='create',
+        requirements=dict(register=register_list_str),
+        conditions=dict(method=['POST']))
     map.connect('/api/{ver:1|2}/rest/{register}/{id}/:subregister/{id2}',
         controller='api', action='create',
         requirements=dict(register=register_list_str),
@@ -135,6 +139,10 @@ def make_map():
         controller='api', action='list',
         requirements=dict(register=register_list_str),
         conditions=dict(method=['GET']))
+    map.connect('/api/rest/{register}/{id}/:subregister',
+        controller='api', action='create',
+        requirements=dict(register=register_list_str),
+        conditions=dict(method=['POST']))
     map.connect('/api/rest/{register}/{id}/:subregister/{id2}',
         controller='api', action='create',
         requirements=dict(register=register_list_str),

ckan/controllers/api.py

@@ -266,17 +266,17 @@ def _represent_package(self, package):
     def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
 
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_create'),
-            ('package', 'relationships'): get_action('package_relationship_create'),
+            ('dataset', 'relationships'): get_action('package_relationship_create_rest'),
+            ('package', 'relationships'): get_action('package_relationship_create_rest'),
              'group': get_action('group_create_rest'),
              'dataset': get_action('package_create_rest'),
              'package': get_action('package_create_rest'),
              'rating': get_action('rating_create'),
         }
 
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_create')
-            action_map[('package', type)] = get_action('package_relationship_create')
+            action_map[('dataset', type)] = get_action('package_relationship_create_rest')
+            action_map[('package', type)] = get_action('package_relationship_create_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}
@@ -296,6 +296,7 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
             return self._finish_bad_request(
                 gettext('Cannot create new entity of this type: %s %s') % \
                 (register, subregister))
+
         try:
             response_data = action(context, data_dict)
             location = None
@@ -325,17 +326,16 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
             raise
 
     def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
-
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_update'),
-            ('package', 'relationships'): get_action('package_relationship_update'),
+            ('dataset', 'relationships'): get_action('package_relationship_update_rest'),
+            ('package', 'relationships'): get_action('package_relationship_update_rest'),
              'dataset': get_action('package_update_rest'),
              'package': get_action('package_update_rest'),
              'group': get_action('group_update_rest'),
         }
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_update')
-            action_map[('package', type)] = get_action('package_relationship_update')
+            action_map[('dataset', type)] = get_action('package_relationship_update_rest')
+            action_map[('package', type)] = get_action('package_relationship_update_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver, 'id': id}
@@ -376,15 +376,15 @@ def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
 
     def delete(self, ver=None, register=None, subregister=None, id=None, id2=None):
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_delete'),
-            ('package', 'relationships'): get_action('package_relationship_delete'),
+            ('dataset', 'relationships'): get_action('package_relationship_delete_rest'),
+            ('package', 'relationships'): get_action('package_relationship_delete_rest'),
              'group': get_action('group_delete'),
              'dataset': get_action('package_delete'),
              'package': get_action('package_delete'),
         }
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_delete')
-            action_map[('package', type)] = get_action('package_relationship_delete')
+            action_map[('dataset', type)] = get_action('package_relationship_delete_rest')
+            action_map[('package', type)] = get_action('package_relationship_delete_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}

ckan/controllers/group.py

@@ -9,7 +9,6 @@
 import ckan.authz as authz
 from ckan.authz import Authorizer
 from ckan.lib.helpers import Page
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.plugins import PluginImplementations, IGroupController, IGroupForm
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.logic import NotFound, NotAuthorized, ValidationError
@@ -203,6 +202,7 @@ def index(self):
 
 
     def read(self, id):
+        from ckan.lib.search import SearchError
         group_type = self._get_group_type(id.split('@')[0])        
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
@@ -298,6 +298,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Group search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])

ckan/controllers/package.py

@@ -17,7 +17,6 @@
 from ckan.lib.base import request, c, BaseController, model, abort, h, g, render
 from ckan.lib.base import response, redirect, gettext
 from ckan.authz import Authorizer
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.lib.package_saver import PackageSaver, ValidationException
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.lib.helpers import json
@@ -206,6 +205,7 @@ def _setup_template_variables(self, context, data_dict, package_type=None):
     authorizer = ckan.authz.Authorizer()
 
     def search(self):
+        from ckan.lib.search import SearchError
         try:
             context = {'model':model,'user': c.user or c.author}
             check_access('site_read',context)
@@ -277,6 +277,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Package search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])
@@ -593,6 +594,7 @@ def _get_package_type(self, id):
         return data['type']
 
     def _save_new(self, context, package_type=None):
+        from ckan.lib.search import SearchIndexError
         try:
             data_dict = clean_dict(unflatten(
                 tuplize_dict(parse_params(request.POST))))
@@ -616,6 +618,7 @@ def _save_new(self, context, package_type=None):
             return self.new(data_dict, errors, error_summary)
 
     def _save_edit(self, id, context):
+        from ckan.lib.search import SearchIndexError
         try:
             package_type = self._get_package_type(id)
             data_dict = clean_dict(unflatten(

ckan/forms/group.py

@@ -49,7 +49,7 @@ def render(self, **kwargs):
 
 def build_group_form(is_admin=False, with_packages=False):
     builder = FormBuilder(model.Group)
-    builder.set_field_text('name', _('Name'), literal("<br/><strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'"))
+    builder.set_field_text('name', _('Name'), literal("<strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'<p></p>"))
     builder.set_field_option('name', 'validate', common.group_name_validator)
     builder.set_field_option('description', 'textarea', {'size':'60x15'})
     builder.add_field(ExtrasField('extras', hidden_label=True))

ckan/lib/cli.py

@@ -68,8 +68,9 @@ class ManageDb(CkanCommand):
     db version # returns current version of data schema
     db dump {file-path} # dump to a pg_dump file
     db dump-rdf {dataset-name} {file-path}
-    db simple-dump-csv {file-path}
-    db simple-dump-json {file-path}
+    db simple-dump-csv {file-path} # dump just datasets in CSV format
+    db simple-dump-json {file-path} # dump just datasets in JSON format
+    db user-dump-csv {file-path} # dump user information to a CSV file
     db send-rdf {talis-store} {username} {password}
     db load {file-path} # load a pg_dump from a file
     db load-only {file-path} # load a pg_dump from a file but don\'t do
@@ -115,6 +116,8 @@ def command(self):
             self.simple_dump_json()
         elif cmd == 'dump-rdf':
             self.dump_rdf()
+        elif cmd == 'user-dump-csv':
+            self.user_dump_csv()
         elif cmd == 'create-from-model':
             model.repo.create_db()
             if self.verbose:
@@ -238,6 +241,15 @@ def dump_rdf(self):
         f.write(rdf)
         f.close()
 
+    def user_dump_csv(self):
+        if len(self.args) < 2:
+            print 'Need csv file path'
+            return
+        dump_filepath = self.args[1]
+        import ckan.lib.dumper as dumper
+        dump_file = open(dump_filepath, 'w')
+        dumper.UserDumper().dump(dump_file)
+
     def send_rdf(self):
         if len(self.args) < 4:
             print 'Need all arguments: {talis-store} {username} {password}'

ckan/lib/dumper.py

@@ -4,7 +4,7 @@
 
 import ckan.model as model
 import ckan.model
-from helpers import json
+from helpers import json, OrderedDict
 
 class SimpleDumper(object):
     '''Dumps just package data but including tags, groups, license text etc'''
@@ -40,7 +40,7 @@ def dump_csv(self, dump_file_obj, query):
                         pkg_dict[name_] = value_
                     del pkg_dict[name]
             row_dicts.append(pkg_dict)
-        writer = PackagesCsvWriter(row_dicts)
+        writer = CsvWriter(row_dicts)
         writer.save(dump_file_obj)
 
     def dump_json(self, dump_file_obj, query):
@@ -201,7 +201,7 @@ def migrate_06_to_07(self):
                         values={'name': record.name})
                 update.execute()
 
-class PackagesCsvWriter:
+class CsvWriter:
     def __init__(self, package_dict_list=None):
         self._rows = []
         self._col_titles = []
@@ -301,3 +301,25 @@ def pkg_to_xl_dict(pkg):
                     dict_[key_] = value_
                 del dict_[key]
         return dict_
+
+class UserDumper(object):
+    def dump(self, dump_file_obj):
+        query = model.Session.query(model.User)
+        query = query.order_by(model.User.created.asc())
+
+        columns = (('id', 'name', 'openid', 'fullname', 'email', 'created', 'about'))
+        row_dicts = []
+        for user in query:
+            row = OrderedDict()
+            for col in columns:
+                value = getattr(user, col)
+                if not value:
+                    value = ''
+                if col == 'created':
+                    value = str(value) # or maybe dd/mm/yyyy?
+                row[col] = value
+            row_dicts.append(row)
+
+        writer = CsvWriter(row_dicts)
+        writer.save(dump_file_obj)
+        dump_file_obj.close()

ckan/lib/hash.py

@@ -3,9 +3,15 @@
 
 from pylons import config, request
 
-secret = config['beaker.session.secret']
+global secret
+secret = None
 
 def get_message_hash(value):
+    if not secret:
+        global secret
+        # avoid getting config value at module scope since config may
+        # not be read in yet
+        secret = config['beaker.session.secret']
     return hmac.new(secret, value, hashlib.sha1).hexdigest()
 
 def get_redirect():

ckan/lib/search/__init__.py

@@ -7,7 +7,7 @@
 from ckan.logic import get_action
 
 from common import (SearchIndexError, SearchError, SearchQueryError,
-                    make_connection, is_available, DEFAULT_SOLR_URL)
+                    make_connection, is_available, SolrSettings)
 from index import PackageSearchIndex, NoopSearchIndex
 from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions, convert_legacy_parameters_to_solr
 
@@ -197,15 +197,13 @@ def check_solr_schema_version(schema_file=None):
 
     # Try to get the schema XML file to extract the version
     if not schema_file:
-        solr_user = config.get('solr_user')
-        solr_password = config.get('solr_password')
+        solr_url, solr_user, solr_password = SolrSettings.get()
 
         http_auth = None
         if solr_user is not None and solr_password is not None:
             http_auth = solr_user + ':' + solr_password
             http_auth = 'Basic ' + http_auth.encode('base64').strip()
 
-        solr_url = config.get('solr_url', DEFAULT_SOLR_URL)
         url = solr_url.strip('/') + SOLR_SCHEMA_FILE_OFFSET
 
         req = urllib2.Request(url = url)

ckan/lib/search/common.py

@@ -8,9 +8,29 @@ class SearchQueryError(SearchError): pass
 
 DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr'
 
-solr_url = config.get('solr_url', DEFAULT_SOLR_URL)
-solr_user = config.get('solr_user')
-solr_password = config.get('solr_password')
+class SolrSettings(object):
+    _is_initialised = False
+    _url = None
+    _user = None
+    _password = None
+
+    @classmethod
+    def init(cls, url, user=None, password=None):
+        if url is not None:
+            cls._url = url
+            cls._user = user
+            cls._password = password
+        else:
+            cls._url = DEFAULT_SOLR_URL
+        cls._is_initialised = True
+
+    @classmethod
+    def get(cls):
+        if not cls._is_initialised:
+            raise SearchIndexError('SOLR URL not initialised')
+        if not cls._url:
+            raise SearchIndexError('SOLR URL is blank')
+        return (cls._url, cls._user, cls._password)
 
 def is_available():
     """
@@ -23,12 +43,15 @@ def is_available():
         log.exception(e)
         return False
     finally:
-        conn.close()
+        if 'conn' in dir():
+            conn.close()
 
     return True
 
 def make_connection():
     from solr import SolrConnection
+    solr_url, solr_user, solr_password = SolrSettings.get()
+    assert solr_url is not None
     if solr_user is not None and solr_password is not None:
         return SolrConnection(solr_url, http_user=solr_user, http_pass=solr_password)
     else: