Merge branch 'master' into feature-1413-ask-users-to-add-email-address

bin/copy-ckan-2-ckan.py

@@ -16,7 +16,8 @@
 def copy_packages(source_ckan_uri,
                   dest_ckan_uri, dest_api_key,
                   dest_group_name, dest_group_title,
-                  site_name):
+                  site_name, filter,
+                  ):
     ckan1 = ckanclient.CkanClient(base_location=source_ckan_uri)
     ckan2 = ckanclient.CkanClient(base_location=dest_ckan_uri,
                                   api_key=dest_api_key)
@@ -53,9 +54,11 @@ def add_group_if_needed(group_name, group_title=None, fetch_title=False):
     package_list = ckan1.package_register_get()
     print 'Found %i packages to copy' % len(package_list)
 
-##    #HACK
-##    package_list = [pkg for pkg in package_list if pkg.startswith('a')]
-##    print 'Found %i packages beginning with A' % len(package_list)
+    if filter:
+        filter_re = re.compile(filter)
+        package_list = [pkg for pkg in package_list \
+                        if filter_re.match(pkg)]
+        print 'Filtered down to %i packages' % len(package_list)
 
     for package_ref in package_list[:]:
         try:
@@ -98,12 +101,13 @@ def add_group_if_needed(group_name, group_title=None, fetch_title=False):
             ckan2.group_entity_put(group)
             print '...and added to group %s' % group_ref
 
-    group = ckan2.group_entity_get(dest_group_name)
-    pkgs_to_add_to_group = list(set(package_list) - set(group['packages']))
-    if pkgs_to_add_to_group:
-        print 'Adding %i packages to group %s: %r' % (len(pkgs_to_add_to_group), dest_group_name, pkgs_to_add_to_group)
-        group['packages'].extend(pkgs_to_add_to_group)
-        ckan2.group_entity_put(group)
+    if dest_group_name:
+        group = ckan2.group_entity_get(dest_group_name)
+        pkgs_to_add_to_group = list(set(package_list) - set(group['packages']))
+        if pkgs_to_add_to_group:
+            print 'Adding %i packages to group %s: %r' % (len(pkgs_to_add_to_group), dest_group_name, pkgs_to_add_to_group)
+            group['packages'].extend(pkgs_to_add_to_group)
+            ckan2.group_entity_put(group)
 
 def _munge_to_length(string, min_length, max_length):
     '''Pad/truncates a string'''
@@ -132,6 +136,8 @@ def munge_tag(tag):
                   help="Destination CKAN group's title")
 parser.add_option("-s", "--site-name", dest="site_name",
                   help="Name of source CKAN site - so source can be tagged")
+parser.add_option("-f", "--filter", dest="filter",
+                  help="Filter package names (regex format)")
 
 (options, args) = parser.parse_args()
 
@@ -143,4 +149,4 @@ def munge_tag(tag):
               destination_ckan_uri,
               options.destination_ckan_api_key,
               options.group_name, options.group_title,
-              options.site_name)
+              options.site_name, options.filter)

ckan/config/environment.py

@@ -2,9 +2,12 @@
 import os
 from urlparse import urlparse
 import logging
+import warnings
 
 from paste.deploy.converters import asbool
 
+# Suppress benign warning 'Unbuilt egg for setuptools'
+warnings.simplefilter('ignore', UserWarning) 
 import pylons
 from sqlalchemy import engine_from_config
 from pylons import config
@@ -74,6 +77,8 @@ def find_controller(self, controller):
     if config.get('ckan.site_id') is None:
         if ':' in ckan_host:
             ckan_host, port = ckan_host.split(':')
+        assert ckan_host, 'You need to configure ckan.site_url or ' \
+                          'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
     config['routes.map'] = make_map()

ckan/config/routing.py

@@ -157,6 +157,8 @@ def make_map():
         action='user_autocomplete')
     map.connect('/api/2/util/is_slug_valid', controller='api', action='is_slug_valid',
                 conditions=dict(method=['GET']))
+    map.connect('/api/2/util/dataset/autocomplete', controller='api', action='dataset_autocomplete',
+                conditions=dict(method=['GET']))
     map.connect('/api/2/util/tag/autocomplete', controller='api', action='tag_autocomplete',
                 conditions=dict(method=['GET']))
     map.connect('/api/2/util/resource/format_autocomplete', controller='api', action='format_autocomplete',

ckan/config/schema.xml

@@ -131,10 +131,10 @@
     <field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
     <field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
     <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
-    <field name="extras_*" type="text" indexed="true" stored="false" multiValued="true"/>
 
     <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
 
+    <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
     <dynamicField name="*" type="string" indexed="true"  stored="false"/>
 </fields>
 

ckan/controllers/api.py

@@ -7,7 +7,9 @@
 from ckan.lib.helpers import json, date_str_to_datetime
 import ckan.model as model
 import ckan.rating
-from ckan.lib.search import query_for, QueryOptions, SearchIndexError, SearchError, DEFAULT_OPTIONS, convert_legacy_parameters_to_solr
+from ckan.lib.search import (query_for, QueryOptions, SearchIndexError, SearchError,
+                             SearchQueryError, DEFAULT_OPTIONS,
+                             convert_legacy_parameters_to_solr)
 from ckan.plugins import PluginImplementations, IGroupController
 from ckan.lib.navl.dictization_functions import DataError
 from ckan.lib.munge import munge_name, munge_title_to_name, munge_tag
@@ -147,6 +149,13 @@ def action(self, logic_function):
             log.error('Bad request data: %s' % str(inst))
             return self._finish_bad_request(
                 gettext('JSON Error: %s') % str(inst))
+        if not isinstance(request_data, dict):
+            # this occurs if request_data is blank
+            log.error('Bad request data - not dict: %r' % request_data)
+            return self._finish_bad_request(
+                gettext('Bad request data: %s') % \
+                'Request data JSON decoded to %r but ' \
+                'it needs to be a dictionary.' % request_data)
         try:
             result = function(context, request_data)
             return_dict['success'] = True
@@ -173,6 +182,16 @@ def action(self, logic_function):
             return_dict['success'] = False
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, return_dict, content_type='json')
+        except SearchQueryError, e:
+            return_dict['error'] = {'__type': 'Search Query Error',
+                                    'message': 'Search Query is invalid: %r' % e.args }
+            return_dict['success'] = False
+            return self._finish(400, return_dict, content_type='json')        
+        except SearchError, e:
+            return_dict['error'] = {'__type': 'Search Error',
+                                    'message': 'Search error: %r' % e.args }
+            return_dict['success'] = False
+            return self._finish(409, return_dict, content_type='json')        
         return self._finish_ok(return_dict)
 
     def list(self, ver=None, register=None, subregister=None, id=None):
@@ -582,6 +601,22 @@ def is_slug_valid(self):
         return self._finish_bad_request(gettext('Bad slug type: %s') % slugtype)
 
 
+    def dataset_autocomplete(self):
+        q = request.params.get('incomplete', '')
+        q_lower = q.lower()
+        limit = request.params.get('limit', 10)
+        tag_names = []
+        if q:
+            context = {'model': model, 'session': model.Session,
+                       'user': c.user or c.author}
+
+            data_dict = {'q': q, 'limit': limit}
+
+            package_dicts = get_action('package_autocomplete')(context, data_dict)
+
+        resultSet = {'ResultSet': {'Result': package_dicts}}
+        return self._finish_ok(resultSet)
+
     def tag_autocomplete(self):
         q = request.params.get('incomplete', '')
         limit = request.params.get('limit', 10)
@@ -590,9 +625,9 @@ def tag_autocomplete(self):
             context = {'model': model, 'session': model.Session,
                        'user': c.user or c.author}
 
-            data_dict = {'q':q,'limit':limit}
+            data_dict = {'q': q, 'limit': limit}
 
-            tag_names = get_action('tag_autocomplete')(context,data_dict)
+            tag_names = get_action('tag_autocomplete')(context, data_dict)
 
         resultSet = {
             'ResultSet': {

ckan/controllers/package.py

@@ -553,6 +553,7 @@ def _get_userobjectroles(self, pkg_id):
 
 
     def autocomplete(self):
+        # DEPRECATED in favour of /api/2/util/dataset/autocomplete
         q = unicode(request.params.get('q', ''))
         if not len(q): 
             return ''
@@ -566,10 +567,7 @@ def autocomplete(self):
 
         pkg_list = []
         for pkg in packages:
-            if pkg['name'].lower().startswith(q.lower()):
-                pkg_list.append('%s|%s' % (pkg['name'], pkg['name']))
-            else:
-                pkg_list.append('%s (%s)|%s' % (pkg['title'].replace('|', ' '), pkg['name'], pkg['name']))
+            pkg_list.append('%s|%s' % (pkg['match_displayed'].replace('|', ' '), pkg['name']))
         return '\n'.join(pkg_list)
 
     def _render_edit_form(self, fs, params={}, clear_session=False):

ckan/lib/alphabet_paginate.py

@@ -58,7 +58,7 @@ def pager(self, q=None):
         letters = [char for char in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'] + [self.other_text]
         for letter in letters:
             if letter != page:
-                page = HTML.a(class_='pager_link', href=url_for(page=letter), c=letter)
+                page = HTML.a(class_='pager_link', href=url_for(controller='tag', action='index', page=letter), c=letter)
             else:
                 page = HTML.span(class_='pager_curpage', c=letter)
             pages.append(page)                           

ckan/lib/base.py

@@ -182,12 +182,13 @@ def _get_request_data(cls):
             try:
                 request_data = json.loads(request_data, encoding='utf8')
             except ValueError, e:
-                raise ValueError, 'Error parsing JSON data. ' \
+                raise ValueError, 'Error decoding JSON data. ' \
                                     'Error: %r ' \
-                                    'JSON (Decoded and re-encoded): %r' % \
+                                    'JSON data extracted from the request: %r' % \
                                     (e, request_data)
             if not isinstance(request_data, dict):
-                raise ValueError, "Request params must be in form of a json encoded dictionary."
+                raise ValueError, 'Request data JSON decoded to %r but ' \
+                                  'it needs to be a dictionary.' % request_data
             # ensure unicode values
             for key, val in request_data.items():
                 # if val is str then assume it is ascii, since json converts