[1669] Merge from master

ckan/config/routing.py

@@ -79,6 +79,10 @@ def make_map():
         controller='api', action='list',
         requirements=dict(register=register_list_str),
         conditions=dict(method=['GET']))
+    map.connect('/api/{ver:1|2}/rest/{register}/{id}/:subregister',
+        controller='api', action='create',
+        requirements=dict(register=register_list_str),
+        conditions=dict(method=['POST']))
     map.connect('/api/{ver:1|2}/rest/{register}/{id}/:subregister/{id2}',
         controller='api', action='create',
         requirements=dict(register=register_list_str),
@@ -135,6 +139,10 @@ def make_map():
         controller='api', action='list',
         requirements=dict(register=register_list_str),
         conditions=dict(method=['GET']))
+    map.connect('/api/rest/{register}/{id}/:subregister',
+        controller='api', action='create',
+        requirements=dict(register=register_list_str),
+        conditions=dict(method=['POST']))
     map.connect('/api/rest/{register}/{id}/:subregister/{id2}',
         controller='api', action='create',
         requirements=dict(register=register_list_str),

ckan/controllers/api.py

@@ -266,17 +266,17 @@ def _represent_package(self, package):
     def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
 
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_create'),
-            ('package', 'relationships'): get_action('package_relationship_create'),
+            ('dataset', 'relationships'): get_action('package_relationship_create_rest'),
+            ('package', 'relationships'): get_action('package_relationship_create_rest'),
              'group': get_action('group_create_rest'),
              'dataset': get_action('package_create_rest'),
              'package': get_action('package_create_rest'),
              'rating': get_action('rating_create'),
         }
 
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_create')
-            action_map[('package', type)] = get_action('package_relationship_create')
+            action_map[('dataset', type)] = get_action('package_relationship_create_rest')
+            action_map[('package', type)] = get_action('package_relationship_create_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}
@@ -296,6 +296,7 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
             return self._finish_bad_request(
                 gettext('Cannot create new entity of this type: %s %s') % \
                 (register, subregister))
+
         try:
             response_data = action(context, data_dict)
             location = None
@@ -325,17 +326,16 @@ def create(self, ver=None, register=None, subregister=None, id=None, id2=None):
             raise
 
     def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
-
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_update'),
-            ('package', 'relationships'): get_action('package_relationship_update'),
+            ('dataset', 'relationships'): get_action('package_relationship_update_rest'),
+            ('package', 'relationships'): get_action('package_relationship_update_rest'),
              'dataset': get_action('package_update_rest'),
              'package': get_action('package_update_rest'),
              'group': get_action('group_update_rest'),
         }
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_update')
-            action_map[('package', type)] = get_action('package_relationship_update')
+            action_map[('dataset', type)] = get_action('package_relationship_update_rest')
+            action_map[('package', type)] = get_action('package_relationship_update_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver, 'id': id}
@@ -376,15 +376,15 @@ def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
 
     def delete(self, ver=None, register=None, subregister=None, id=None, id2=None):
         action_map = {
-            ('dataset', 'relationships'): get_action('package_relationship_delete'),
-            ('package', 'relationships'): get_action('package_relationship_delete'),
+            ('dataset', 'relationships'): get_action('package_relationship_delete_rest'),
+            ('package', 'relationships'): get_action('package_relationship_delete_rest'),
              'group': get_action('group_delete'),
              'dataset': get_action('package_delete'),
              'package': get_action('package_delete'),
         }
         for type in model.PackageRelationship.get_all_types():
-            action_map[('dataset', type)] = get_action('package_relationship_delete')
-            action_map[('package', type)] = get_action('package_relationship_delete')
+            action_map[('dataset', type)] = get_action('package_relationship_delete_rest')
+            action_map[('package', type)] = get_action('package_relationship_delete_rest')
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}

ckan/controllers/group.py

@@ -1,13 +1,15 @@
 import genshi
 import datetime
+from urllib import urlencode
 
 from sqlalchemy.orm import eagerload_all
-from ckan.lib.base import BaseController, c, model, request, render, h
+from ckan.lib.base import BaseController, c, model, request, render, h, g
 from ckan.lib.base import ValidationException, abort, gettext
 from pylons.i18n import get_lang, _
 import ckan.authz as authz
 from ckan.authz import Authorizer
 from ckan.lib.helpers import Page
+from ckan.lib.search import SearchIndexError, SearchError
 from ckan.plugins import PluginImplementations, IGroupController, IGroupForm
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.logic import NotFound, NotAuthorized, ValidationError
@@ -172,7 +174,7 @@ def _setup_template_variables(self, context, data_dict, group_type=None):
         return _lookup_plugin(group_type).setup_template_variables(context,data_dict)
 
     ## end hooks
-    
+
     def index(self):
 
         context = {'model': model, 'session': model.Session,
@@ -202,42 +204,99 @@ def read(self, id):
                    'user': c.user or c.author,
                    'schema': self._form_to_db_schema(group_type=type)}
         data_dict = {'id': id}
+        q = c.q = request.params.get('q', '') # unicode format (decoded from utf8)
+
         try:
             c.group_dict = get_action('group_show')(context, data_dict)
             c.group = context['group']
         except NotFound:
             abort(404, _('Group not found'))
         except NotAuthorized:
             abort(401, _('Unauthorized to read group %s') % id)
+
+        # Search within group
+        q += ' groups: "%s"' % c.group_dict.get('name')
+
         try:
-            description_formatted = ckan.misc.MarkdownFormat().to_html(c.group.get('description',''))
+            description_formatted = ckan.misc.MarkdownFormat().to_html(c.group_dict.get('description',''))
             c.description_formatted = genshi.HTML(description_formatted)
         except Exception, e:
             error_msg = "<span class='inline-warning'>%s</span>" % _("Cannot render description")
             c.description_formatted = genshi.HTML(error_msg)
 
-        try:
-            desc_formatted = ckan.misc.MarkdownFormat().to_html(c.group.description)
-            desc_formatted = genshi.HTML(desc_formatted)
-        except genshi.ParseError, e:
-            desc_formatted = 'Error: Could not parse group description'
-        c.group_description_formatted = desc_formatted
         c.group_admins = self.authorizer.get_admins(c.group)
 
         context['return_query'] = True
-        results = get_action('group_package_show')(context, data_dict)
 
-        c.page = Page(
-            collection=results,
-            page=request.params.get('page', 1),
-            url=h.pager_url,
-            items_per_page=30
-        )
+        limit = 20
+        try:
+            page = int(request.params.get('page', 1))
+        except ValueError, e:
+            abort(400, ('"page" parameter must be an integer'))
+
+        # most search operations should reset the page counter:
+        params_nopage = [(k, v) for k,v in request.params.items() if k != 'page']
+
+        def search_url(params):
+            url = h.url_for(controller='group', action='read', id=c.group_dict.get('name'))
+            params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) \
+                            for k, v in params]
+            return url + u'?' + urlencode(params)
+
+        def drill_down_url(**by):
+            params = list(params_nopage)
+            params.extend(by.items())
+            return search_url(set(params))
+
+        c.drill_down_url = drill_down_url 
+
+        def remove_field(key, value):
+            params = list(params_nopage)
+            params.remove((key, value))
+            return search_url(params)
 
-        result = []
-        for pkg_rev in c.page.items:
-            result.append(package_dictize(pkg_rev, context))
-        c.page.items = result
+        c.remove_field = remove_field
+
+        def pager_url(q=None, page=None):
+            params = list(params_nopage)
+            params.append(('page', page))
+            return search_url(params)
+
+        try:
+            c.fields = []
+            search_extras = {}
+            for (param, value) in request.params.items():
+                if not param in ['q', 'page'] \
+                        and len(value) and not param.startswith('_'):
+                    if not param.startswith('ext_'):
+                        c.fields.append((param, value))
+                        q += ' %s: "%s"' % (param, value)
+                    else:
+                        search_extras[param] = value
+
+            data_dict = {
+                'q':q,
+                'facet.field':g.facets,
+                'rows':limit,
+                'start':(page-1)*limit,
+                'extras':search_extras
+            }
+
+            query = get_action('package_search')(context,data_dict)
+
+            c.page = h.Page(
+                collection=query['results'],
+                page=page,
+                url=pager_url,
+                item_count=query['count'],
+                items_per_page=limit
+            )
+            c.facets = query['facets']
+            c.page.items = query['results']
+        except SearchError, se:
+            c.query_error = True
+            c.facets = {}
+            c.page = h.Page(collection=[])
 
         return render('group/read.html')
 

ckan/controllers/package.py

@@ -448,14 +448,20 @@ def new(self, data=None, errors=None, error_summary=None):
         if context['save'] and not data:
             return self._save_new(context)
 
-        data = data or dict(request.params) 
+        data = data or clean_dict(unflatten(tuplize_dict(parse_params(request.params))))
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
 
         self._setup_template_variables(context, {'id': id})
-        c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
 
+        # TODO: This check is to maintain backwards compatibility with the old way of creating
+        # custom forms. This behaviour is now deprecated.
+        if hasattr(self, 'package_form'):
+            c.form = render(self.package_form, extra_vars=vars)
+        else:
+            c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
         return render('package/new.html')
 
 
@@ -496,7 +502,12 @@ def edit(self, id, data=None, errors=None, error_summary=None):
 
         self._setup_template_variables(context, {'id': id}, package_type=package_type)
 
-        c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
+        # TODO: This check is to maintain backwards compatibility with the old way of creating
+        # custom forms. This behaviour is now deprecated.
+        if hasattr(self, 'package_form'):
+            c.form = render(self.package_form, extra_vars=vars)
+        else:
+            c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
         return render('package/edit.html')
 
     def read_ajax(self, id, revision=None):

ckan/forms/group.py

@@ -49,7 +49,7 @@ def render(self, **kwargs):
 
 def build_group_form(is_admin=False, with_packages=False):
     builder = FormBuilder(model.Group)
-    builder.set_field_text('name', _('Name'), literal("<br/><strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'"))
+    builder.set_field_text('name', _('Name'), literal("<strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'<p></p>"))
     builder.set_field_option('name', 'validate', common.group_name_validator)
     builder.set_field_option('description', 'textarea', {'size':'60x15'})
     builder.add_field(ExtrasField('extras', hidden_label=True))

ckan/lib/cli.py

@@ -32,6 +32,9 @@ class CkanCommand(paste.script.command.Command):
     group_name = 'ckan'
 
     def _load_config(self):
+        # Avoids vdm logging warning
+        logging.basicConfig(level=logging.ERROR)
+
         from paste.deploy import appconfig
         from ckan.config.environment import load_environment
         if not self.options.config:
@@ -78,10 +81,7 @@ class ManageDb(CkanCommand):
     max_args = None
     min_args = 1
 
-    def command(self):
-        # Avoids vdm logging warning
-        logging.basicConfig(level=logging.ERROR)
-
+    def command(self):        
         self._load_config()
         from ckan import model
         import ckan.lib.search as search

ckan/lib/helpers.py

@@ -206,8 +206,7 @@ def linked_user(user, maxlength=0):
         _name = user.name if model.User.VALID_NAME.match(user.name) else user.id
         # Absolute URL of default user icon
         from pylons import config 
-        _site_url = config.get('ckan.site_url', '')
-        _icon_url_default = _site_url + icon_url("user")
+        _icon_url_default = icon_url("user")
         _icon = gravatar(user.email_hash, 16, _icon_url_default)+" "
         displayname = user.display_name
         if maxlength and len(user.display_name) > maxlength:
@@ -230,7 +229,7 @@ def markdown_extract(text, extract_length=190):
     return unicode(truncate(plain, length=extract_length, indicator='...', whole_word=True))
 
 def icon_url(name):
-    return '/images/icons/%s.png' % name
+    return url_for('/images/icons/%s.png' % name)
 
 def icon_html(url, alt=None):
     return literal('<img src="%s" height="16px" width="16px" alt="%s" /> ' % (url, alt))

ckan/lib/search/index.py

@@ -6,6 +6,7 @@
 from pylons import config
 
 from common import SearchIndexError, make_connection
+from ckan.model import PackageRelationship
 
 log = logging.getLogger(__name__)
 
@@ -15,13 +16,7 @@
 SOLR_FIELDS = [TYPE_FIELD, "res_url", "text", "urls", "indexed_ts", "site_id"]
 RESERVED_FIELDS = SOLR_FIELDS + ["tags", "groups", "res_description", 
                                  "res_format", "res_url"]
-# HACK: this is copied over from model.PackageRelationship 
-RELATIONSHIP_TYPES = [
-    (u'depends_on', u'dependency_of'),
-    (u'derives_from', u'has_derivation'),
-    (u'links_to', u'linked_from'),
-    (u'child_of', u'parent_of'),
-]
+RELATIONSHIP_TYPES = PackageRelationship.types
 
 def clear_index():
     conn = make_connection()

ckan/logic/action/__init__.py

@@ -0,0 +1,21 @@
+from copy import deepcopy
+
+def rename_keys(dict_, key_map, reverse=False, destructive=False):
+    '''Returns a dict that has particular keys renamed,
+    according to the key_map.
+
+    Rename is by default non-destructive, so if the intended new
+    key name already exists, it won\'t do that rename.
+
+    To reverse the change, set reverse=True.'''
+    new_dict = deepcopy(dict_)
+    for key, mapping in key_map.items():
+        if reverse:
+            key, mapping = (mapping, key)
+        if (not destructive) and new_dict.has_key(mapping):
+            continue
+        if dict_.has_key(key):
+            value = dict_[key]
+            new_dict[mapping] = value
+            del new_dict[key]
+    return new_dict