Merge branch 'master' of github.com:okfn/ckan into feature-1515-activ…

…ity-streams

Conflicts:
	ckan/controllers/group.py
	ckan/logic/action/create.py
	ckan/logic/action/update.py

ckan/config/environment.py

@@ -81,8 +81,11 @@ def find_controller(self, controller):
                           'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
-    # Check if SOLR schema is compatible
-    from ckan.lib.search import check_solr_schema_version
+    # Init SOLR settings and check if the schema is compatible
+    from ckan.lib.search import SolrSettings, check_solr_schema_version
+    SolrSettings.init(config.get('solr_url'),
+                      config.get('solr_user'),
+                      config.get('solr_password'))
     check_solr_schema_version()
 
     config['routes.map'] = make_map()

ckan/config/routing.py

@@ -8,14 +8,17 @@
 from pylons import config
 from routes import Mapper
 from ckan.plugins import PluginImplementations, IRoutes
-from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour
-from ckan.controllers.group   import register_pluggable_behaviour as register_group_behaviour
 
 
 routing_plugins = PluginImplementations(IRoutes)
 
 def make_map():
     """Create, configure and return the routes Mapper"""
+    # import controllers here rather than at root level because 
+    # pylons config is initialised by this point.
+    from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour
+    from ckan.controllers.group   import register_pluggable_behaviour as register_group_behaviour
+
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
     map.minimization = False
@@ -81,6 +84,10 @@ def make_map():
         controller='api', action='list',
         requirements=dict(register=register_list_str),
         conditions=dict(method=['GET']))
+    map.connect('/api/{ver:1|2}/rest/{register}/{id}/:subregister',
+        controller='api', action='create',
+        requirements=dict(register=register_list_str),
+        conditions=dict(method=['POST']))
     map.connect('/api/{ver:1|2}/rest/{register}/{id}/:subregister/{id2}',
         controller='api', action='create',
         requirements=dict(register=register_list_str),
@@ -137,6 +144,10 @@ def make_map():
         controller='api', action='list',
         requirements=dict(register=register_list_str),
         conditions=dict(method=['GET']))
+    map.connect('/api/rest/{register}/{id}/:subregister',
+        controller='api', action='create',
+        requirements=dict(register=register_list_str),
+        conditions=dict(method=['POST']))
     map.connect('/api/rest/{register}/{id}/:subregister/{id2}',
         controller='api', action='create',
         requirements=dict(register=register_list_str),

ckan/controllers/api.py

@@ -271,17 +271,17 @@ def _represent_package(self, package):
     def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
 
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_create'),
-            ('package', 'relationships'): get_action('package_relationship_create'),
+            ('dataset', 'relationships'): get_action('package_relationship_create_rest'),
+            ('package', 'relationships'): get_action('package_relationship_create_rest'),
              'group': get_action('group_create_rest'),
              'dataset': get_action('package_create_rest'),
              'package': get_action('package_create_rest'),
              'rating': get_action('rating_create'),
         }
 
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_create')
-            action_map[('package', type)] = get_action('package_relationship_create')
+            action_map[('dataset', type)] = get_action('package_relationship_create_rest')
+            action_map[('package', type)] = get_action('package_relationship_create_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}
@@ -301,6 +301,7 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
             return self._finish_bad_request(
                 gettext('Cannot create new entity of this type: %s %s') % \
                 (register, subregister))
+
         try:
             response_data = action(context, data_dict)
             location = None
@@ -330,17 +331,16 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
             raise
 
     def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
-
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_update'),
-            ('package', 'relationships'): get_action('package_relationship_update'),
+            ('dataset', 'relationships'): get_action('package_relationship_update_rest'),
+            ('package', 'relationships'): get_action('package_relationship_update_rest'),
              'dataset': get_action('package_update_rest'),
              'package': get_action('package_update_rest'),
              'group': get_action('group_update_rest'),
         }
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_update')
-            action_map[('package', type)] = get_action('package_relationship_update')
+            action_map[('dataset', type)] = get_action('package_relationship_update_rest')
+            action_map[('package', type)] = get_action('package_relationship_update_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver, 'id': id}
@@ -381,15 +381,15 @@ def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
 
     def delete(self, ver=None, register=None, subregister=None, id=None, id2=None):
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_delete'),
-            ('package', 'relationships'): get_action('package_relationship_delete'),
+            ('dataset', 'relationships'): get_action('package_relationship_delete_rest'),
+            ('package', 'relationships'): get_action('package_relationship_delete_rest'),
              'group': get_action('group_delete'),
              'dataset': get_action('package_delete'),
              'package': get_action('package_delete'),
         }
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_delete')
-            action_map[('package', type)] = get_action('package_relationship_delete')
+            action_map[('dataset', type)] = get_action('package_relationship_delete_rest')
+            action_map[('package', type)] = get_action('package_relationship_delete_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}

ckan/controllers/group.py

@@ -1,8 +1,9 @@
 import genshi
 import datetime
+from urllib import urlencode
 
 from sqlalchemy.orm import eagerload_all
-from ckan.lib.base import BaseController, c, model, request, render, h
+from ckan.lib.base import BaseController, c, model, request, render, h, g
 from ckan.lib.base import ValidationException, abort, gettext
 from pylons.i18n import get_lang, _
 import ckan.authz as authz
@@ -177,7 +178,7 @@ def _setup_template_variables(self, context, data_dict, group_type=None):
         return _lookup_plugin(group_type).setup_template_variables(context,data_dict)
 
     ## end hooks
-    
+
     def index(self):
 
         context = {'model': model, 'session': model.Session,
@@ -202,47 +203,106 @@ def index(self):
 
 
     def read(self, id):
+        from ckan.lib.search import SearchError
         group_type = self._get_group_type(id.split('@')[0])        
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
                    'schema': self._form_to_db_schema(group_type=type)}
         data_dict = {'id': id}
+        q = c.q = request.params.get('q', '') # unicode format (decoded from utf8)
+
         try:
             c.group_dict = get_action('group_show')(context, data_dict)
             c.group = context['group']
         except NotFound:
             abort(404, _('Group not found'))
         except NotAuthorized:
             abort(401, _('Unauthorized to read group %s') % id)
+
+        # Search within group
+        q += ' groups: "%s"' % c.group_dict.get('name')
+
         try:
-            description_formatted = ckan.misc.MarkdownFormat().to_html(c.group.get('description',''))
+            description_formatted = ckan.misc.MarkdownFormat().to_html(c.group_dict.get('description',''))
             c.description_formatted = genshi.HTML(description_formatted)
         except Exception, e:
             error_msg = "<span class='inline-warning'>%s</span>" % _("Cannot render description")
             c.description_formatted = genshi.HTML(error_msg)
 
-        try:
-            desc_formatted = ckan.misc.MarkdownFormat().to_html(c.group.description)
-            desc_formatted = genshi.HTML(desc_formatted)
-        except genshi.ParseError, e:
-            desc_formatted = 'Error: Could not parse group description'
-        c.group_description_formatted = desc_formatted
         c.group_admins = self.authorizer.get_admins(c.group)
 
         context['return_query'] = True
-        results = get_action('group_package_show')(context, data_dict)
 
-        c.page = Page(
-            collection=results,
-            page=request.params.get('page', 1),
-            url=h.pager_url,
-            items_per_page=30
-        )
+        limit = 20
+        try:
+            page = int(request.params.get('page', 1))
+        except ValueError, e:
+            abort(400, ('"page" parameter must be an integer'))
+
+        # most search operations should reset the page counter:
+        params_nopage = [(k, v) for k,v in request.params.items() if k != 'page']
+
+        def search_url(params):
+            url = h.url_for(controller='group', action='read', id=c.group_dict.get('name'))
+            params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) \
+                            for k, v in params]
+            return url + u'?' + urlencode(params)
+
+        def drill_down_url(**by):
+            params = list(params_nopage)
+            params.extend(by.items())
+            return search_url(set(params))
+
+        c.drill_down_url = drill_down_url 
+
+        def remove_field(key, value):
+            params = list(params_nopage)
+            params.remove((key, value))
+            return search_url(params)
 
-        result = []
-        for pkg_rev in c.page.items:
-            result.append(package_dictize(pkg_rev, context))
-        c.page.items = result
+        c.remove_field = remove_field
+
+        def pager_url(q=None, page=None):
+            params = list(params_nopage)
+            params.append(('page', page))
+            return search_url(params)
+
+        try:
+            c.fields = []
+            search_extras = {}
+            for (param, value) in request.params.items():
+                if not param in ['q', 'page'] \
+                        and len(value) and not param.startswith('_'):
+                    if not param.startswith('ext_'):
+                        c.fields.append((param, value))
+                        q += ' %s: "%s"' % (param, value)
+                    else:
+                        search_extras[param] = value
+
+            data_dict = {
+                'q':q,
+                'facet.field':g.facets,
+                'rows':limit,
+                'start':(page-1)*limit,
+                'extras':search_extras
+            }
+
+            query = get_action('package_search')(context,data_dict)
+
+            c.page = h.Page(
+                collection=query['results'],
+                page=page,
+                url=pager_url,
+                item_count=query['count'],
+                items_per_page=limit
+            )
+            c.facets = query['facets']
+            c.page.items = query['results']
+        except SearchError, se:
+            log.error('Group search error: %r', se.args)
+            c.query_error = True
+            c.facets = {}
+            c.page = h.Page(collection=[])
 
         # Add the group's activity stream (already rendered to HTML) to the
         # template context for the group/read.html template to retrieve later.

ckan/controllers/package.py

@@ -17,7 +17,6 @@
 from ckan.lib.base import request, c, BaseController, model, abort, h, g, render
 from ckan.lib.base import response, redirect, gettext
 from ckan.authz import Authorizer
-from ckan.lib.search import SearchIndexError, SearchError
 from ckan.lib.package_saver import PackageSaver, ValidationException
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.lib.helpers import json
@@ -207,6 +206,7 @@ def _setup_template_variables(self, context, data_dict, package_type=None):
     authorizer = ckan.authz.Authorizer()
 
     def search(self):
+        from ckan.lib.search import SearchError
         try:
             context = {'model':model,'user': c.user or c.author}
             check_access('site_read',context)
@@ -278,6 +278,7 @@ def pager_url(q=None, page=None):
             c.facets = query['facets']
             c.page.items = query['results']
         except SearchError, se:
+            log.error('Package search error: %r', se.args)
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])
@@ -456,14 +457,20 @@ def new(self, data=None, errors=None, error_summary=None):
         if context['save'] and not data:
             return self._save_new(context)
 
-        data = data or dict(request.params) 
+        data = data or clean_dict(unflatten(tuplize_dict(parse_params(request.params))))
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
 
         self._setup_template_variables(context, {'id': id})
-        c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
 
+        # TODO: This check is to maintain backwards compatibility with the old way of creating
+        # custom forms. This behaviour is now deprecated.
+        if hasattr(self, 'package_form'):
+            c.form = render(self.package_form, extra_vars=vars)
+        else:
+            c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
         return render('package/new.html')
 
 
@@ -504,7 +511,12 @@ def edit(self, id, data=None, errors=None, error_summary=None):
 
         self._setup_template_variables(context, {'id': id}, package_type=package_type)
 
-        c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
+        # TODO: This check is to maintain backwards compatibility with the old way of creating
+        # custom forms. This behaviour is now deprecated.
+        if hasattr(self, 'package_form'):
+            c.form = render(self.package_form, extra_vars=vars)
+        else:
+            c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
         return render('package/edit.html')
 
     def read_ajax(self, id, revision=None):
@@ -590,6 +602,7 @@ def _get_package_type(self, id):
         return data['type']
 
     def _save_new(self, context, package_type=None):
+        from ckan.lib.search import SearchIndexError
         try:
             data_dict = clean_dict(unflatten(
                 tuplize_dict(parse_params(request.POST))))
@@ -613,6 +626,7 @@ def _save_new(self, context, package_type=None):
             return self.new(data_dict, errors, error_summary)
 
     def _save_edit(self, id, context):
+        from ckan.lib.search import SearchIndexError
         try:
             package_type = self._get_package_type(id)
             data_dict = clean_dict(unflatten(

ckan/forms/group.py

@@ -49,7 +49,7 @@ def render(self, **kwargs):
 
 def build_group_form(is_admin=False, with_packages=False):
     builder = FormBuilder(model.Group)
-    builder.set_field_text('name', _('Name'), literal("<br/><strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'"))
+    builder.set_field_text('name', _('Name'), literal("<strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'<p></p>"))
     builder.set_field_option('name', 'validate', common.group_name_validator)
     builder.set_field_option('description', 'textarea', {'size':'60x15'})
     builder.add_field(ExtrasField('extras', hidden_label=True))