Merge remote-tracking branch 'origin/master' into 3467-datastore-total

ckan/controllers/group.py

@@ -270,8 +270,9 @@ def drill_down_url(**by):
         c.drill_down_url = drill_down_url
 
         def remove_field(key, value=None, replace=None):
+            controller = lookup_group_controller(group_type)
             return h.remove_url_param(key, value=value, replace=replace,
-                                      controller='group', action='read',
+                                      controller=controller, action='read',
                                       extras=dict(id=c.group_dict.get('name')))
 
         c.remove_field = remove_field
@@ -283,13 +284,18 @@ def pager_url(q=None, page=None):
 
         try:
             c.fields = []
+            c.fields_grouped = {}
             search_extras = {}
             for (param, value) in request.params.items():
                 if param not in ['q', 'page', 'sort'] \
                         and len(value) and not param.startswith('_'):
                     if not param.startswith('ext_'):
                         c.fields.append((param, value))
                         q += ' %s: "%s"' % (param, value)
+                        if param not in c.fields_grouped:
+                            c.fields_grouped[param] = [value]
+                        else:
+                            c.fields_grouped[param].append(value)
                     else:
                         search_extras[param] = value
 

ckan/controllers/package.py

@@ -579,12 +579,7 @@ def resource_edit(self, id, resource_id, data=None, errors=None,
             # dataset has not yet been fully created
             resource_dict = get_action('resource_show')(context,
                                                         {'id': resource_id})
-            fields = ['url', 'resource_type', 'format', 'name', 'description',
-                      'id']
-            data = {}
-            for field in fields:
-                data[field] = resource_dict[field]
-            return self.new_resource(id, data=data)
+            return self.new_resource(id, data=resource_dict)
         # resource is fully created
         try:
             resource_dict = get_action('resource_show')(context,
@@ -1041,7 +1036,12 @@ def resource_delete(self, id, resource_id):
             if request.method == 'POST':
                 get_action('resource_delete')(context, {'id': resource_id})
                 h.flash_notice(_('Resource has been deleted.'))
-                h.redirect_to(controller='package', action='read', id=id)
+                pkg_dict = get_action('package_show')(None, {'id': id})
+                if pkg_dict['state'].startswith('draft'):
+                    h.redirect_to(controller='package', action='new_resource',
+                                  id=id)
+                else:
+                    h.redirect_to(controller='package', action='read', id=id)
             c.resource_dict = get_action('resource_show')(
                 context, {'id': resource_id})
             c.pkg_id = id

ckan/lib/mailer.py

@@ -144,11 +144,11 @@ def get_invite_body(user, group_dict=None, role=None):
 
 
 def get_reset_link(user):
-    return urljoin(config.get('site_url'),
-                   h.url_for(controller='user',
-                             action='perform_reset',
-                             id=user.id,
-                             key=user.reset_key))
+    return h.url_for(controller='user',
+                     action='perform_reset',
+                     id=user.id,
+                     key=user.reset_key,
+                     qualified=True)
 
 
 def send_reset_link(user):

ckan/logic/action/get.py

@@ -336,7 +336,7 @@ def _group_or_org_list(context, data_dict, is_org=False):
             data_dict, logic.schema.default_pagination_schema(), context)
         if errors:
             raise ValidationError(errors)
-    sort = data_dict.get('sort') or 'name'
+    sort = data_dict.get('sort') or 'title'
     q = data_dict.get('q')
 
     all_fields = asbool(data_dict.get('all_fields', None))
@@ -1831,8 +1831,9 @@ def package_search(context, data_dict):
 
     fl
         The parameter that controls which fields are returned in the solr
-        query cannot be changed.  CKAN always returns the matched datasets as
-        dictionary objects.
+        query.
+        fl can be  None or a list of result fields, such as ['id', 'extras_custom_field'].
+        if fl = None, datasets are returned as a list of full dictionary.
     '''
     # sometimes context['schema'] is None
     schema = (context.get('schema') or
@@ -1875,8 +1876,12 @@ def package_search(context, data_dict):
         else:
             data_source = 'validated_data_dict'
         data_dict.pop('use_default_schema', None)
-        # return a list of package ids
-        data_dict['fl'] = 'id {0}'.format(data_source)
+
+        result_fl = data_dict.get('fl')
+        if not result_fl:
+            data_dict['fl'] = 'id {0}'.format(data_source)
+        else:
+            data_dict['fl'] = ' '.join(result_fl)
 
         # Remove before these hit solr FIXME: whitelist instead
         include_private = asbool(data_dict.pop('include_private', False))
@@ -1903,21 +1908,28 @@ def package_search(context, data_dict):
         # Add them back so extensions can use them on after_search
         data_dict['extras'] = extras
 
-        for package in query.results:
-            # get the package object
-            package_dict = package.get(data_source)
-            ## use data in search index if there
-            if package_dict:
-                # the package_dict still needs translating when being viewed
-                package_dict = json.loads(package_dict)
-                if context.get('for_view'):
-                    for item in plugins.PluginImplementations(
-                            plugins.IPackageController):
-                        package_dict = item.before_view(package_dict)
-                results.append(package_dict)
-            else:
-                log.error('No package_dict is coming from solr for package '
-                          'id %s', package['id'])
+        if result_fl:
+            for package in query.results:
+                if package.get('extras'):
+                    package.update(package['extras'] )
+                    package.pop('extras')
+                results.append(package)
+        else:
+            for package in query.results:
+                # get the package object
+                package_dict = package.get(data_source)
+                ## use data in search index if there
+                if package_dict:
+                    # the package_dict still needs translating when being viewed
+                    package_dict = json.loads(package_dict)
+                    if context.get('for_view'):
+                        for item in plugins.PluginImplementations(
+                                plugins.IPackageController):
+                            package_dict = item.before_view(package_dict)
+                    results.append(package_dict)
+                else:
+                    log.error('No package_dict is coming from solr for package '
+                              'id %s', package['id'])
 
         count = query.count
         facets = query.facets

ckan/logic/schema.py

@@ -587,6 +587,7 @@ def default_autocomplete_schema():
 def default_package_search_schema():
     schema = {
         'q': [ignore_missing, unicode],
+        'fl': [ignore_missing, list_of_strings],
         'fq': [ignore_missing, unicode],
         'rows': [ignore_missing, natural_number_validator],
         'sort': [ignore_missing, unicode],

ckan/model/activity.py

@@ -3,7 +3,16 @@
 import datetime
 
 from sqlalchemy import (
-    orm, types, Column, Table, ForeignKey, desc, or_, union_all)
+    orm,
+    types,
+    Column,
+    Table,
+    ForeignKey,
+    desc,
+    or_,
+    and_,
+    union_all
+)
 
 import ckan.model
 import meta
@@ -182,14 +191,33 @@ def _group_activity_query(group_id):
         # Return a query with no results.
         return model.Session.query(model.Activity).filter("0=1")
 
-    dataset_ids = [dataset.id for dataset in group.packages()]
+    q = model.Session.query(
+        model.Activity
+    ).outerjoin(
+        model.Member,
+        and_(
+            model.Activity.object_id == model.Member.table_id,
+            model.Member.state == 'active'
+        )
+    ).outerjoin(
+        model.Package,
+        and_(
+            model.Package.id == model.Member.table_id,
+            model.Package.private == False,
+            model.Package.state == 'active'
+        )
+    ).filter(
+        # We only care about activity either on the the group itself or on
+        # packages within that group.
+        # FIXME: This means that activity that occured while a package belonged
+        # to a group but was then removed will not show up. This may not be
+        # desired but is consistent with legacy behaviour.
+        or_(
+            model.Member.group_id == group_id,
+            model.Activity.object_id == group_id
+        ),
+    )
 
-    q = model.Session.query(model.Activity)
-    if dataset_ids:
-        q = q.filter(or_(model.Activity.object_id == group_id,
-            model.Activity.object_id.in_(dataset_ids)))
-    else:
-        q = q.filter(model.Activity.object_id == group_id)
     return q
 
 

ckan/templates/organization/index.html

@@ -17,7 +17,7 @@
 {% block primary_content_inner %}
   <h1 class="hide-heading">{% block page_heading %}{{ _('Organizations') }}{% endblock %}</h1>
   {% block organizations_search_form %}
-    {% snippet 'snippets/search_form.html', form_id='organization-search-form', type='organization', query=c.q, sorting_selected=c.sort_by_selected, count=c.page.item_count, placeholder=_('Search organizations...'), show_empty=request.params, no_bottom_border=true if c.page.items %}
+    {% snippet 'snippets/search_form.html', form_id='organization-search-form', type='organization', query=c.q, sorting_selected=c.sort_by_selected, count=c.page.item_count, placeholder=_('Search organizations...'), show_empty=request.params, no_bottom_border=true if c.page.items, sorting = [(_('Name Ascending'), 'title asc'), (_('Name Descending'), 'title desc')] %}
   {% endblock %}
   {% block organizations_list %}
     {% if c.page.items or request.params %}

ckan/templates/package/resource_edit_base.html

@@ -22,9 +22,7 @@
 
 {% block content_primary_nav %}
   {{ h.build_nav_icon('resource_edit', _('Edit resource'), id=pkg.name, resource_id=res.id) }}
-  {% if 'datapusher' in g.plugins %}
-    {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
-  {% endif %}
+  {% block inner_primary_nav %}{% endblock %}
   {{ h.build_nav_icon('views', _('Views'), id=pkg.name, resource_id=res.id) }}
 {% endblock %}
 

ckan/tests/logic/action/test_get.py

@@ -852,6 +852,14 @@ def test_search(self):
         eq(search_result['results'][0]['title'], 'Rivers')
         eq(search_result['count'], 1)
 
+    def test_search_fl(self):
+        factories.Dataset(title='Rivers', name='test_ri')
+        factories.Dataset(title='Lakes')
+
+        search_result = helpers.call_action('package_search', q='rivers', fl=['title', 'name'])
+
+        eq(search_result['results'], [{'title': 'Rivers', 'name': 'test_ri'}])
+
     def test_search_all(self):
         factories.Dataset(title='Rivers')
         factories.Dataset(title='Lakes')

ckanext/datapusher/plugin.py

@@ -67,11 +67,12 @@ def resource_data(self, id, resource_id):
         except logic.NotAuthorized:
             base.abort(403, _('Not authorized to see this page'))
 
-        return base.render('package/resource_data.html',
+        return base.render('datapusher/resource_data.html',
                            extra_vars={'status': datapusher_status})
 
 
 class DatapusherPlugin(p.SingletonPlugin):
+    p.implements(p.IConfigurer, inherit=True)
     p.implements(p.IConfigurable, inherit=True)
     p.implements(p.IActions)
     p.implements(p.IAuthFunctions)
@@ -83,6 +84,9 @@ class DatapusherPlugin(p.SingletonPlugin):
     legacy_mode = False
     resource_show_action = None
 
+    def update_config(self, config):
+        p.toolkit.add_template_directory(config, 'templates')
+
     def configure(self, config):
         self.config = config
 

ckanext/datapusher/templates/package/resource_edit_base.html

@@ -0,0 +1,6 @@
+{% ckan_extends %}
+
+{% block inner_primary_nav %}
+  {{ super() }}
+  {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
+{% endblock %}

ckanext/datastore/controller.py

@@ -1,19 +1,24 @@
 # encoding: utf-8
 
 import StringIO
+import md5
 
 import pylons
 
 from ckan.plugins.toolkit import (
     Invalid,
     ObjectNotFound,
+    NotAuthorized,
     get_action,
     get_validator,
     _,
     request,
     response,
     BaseController,
     abort,
+    render,
+    c,
+    h,
 )
 from ckanext.datastore.writer import (
     csv_writer,
@@ -84,3 +89,42 @@ def result_page(offset, limit):
                     limit -= PAGINATE_BY
 
                 result = result_page(offset, limit)
+
+    def dictionary(self, id, resource_id):
+        u'''data dictionary view: show/edit field labels and descriptions'''
+
+        try:
+            # resource_edit_base template uses these
+            c.pkg_dict = get_action('package_show')(
+                None, {'id': id})
+            c.resource = get_action('resource_show')(
+                None, {'id': resource_id})
+            rec = get_action('datastore_search')(None, {
+                'resource_id': resource_id,
+                'limit': 0})
+        except (ObjectNotFound, NotAuthorized):
+            abort(404, _('Resource not found'))
+
+        fields = [f for f in rec['fields'] if not f['id'].startswith('_')]
+
+        if request.method == 'POST':
+            get_action('datastore_create')(None, {
+                'resource_id': resource_id,
+                'force': True,
+                'fields': [{
+                    'id': f['id'],
+                    'type': f['type'],
+                    'info': {
+                        'label': request.POST.get('f{0}label'.format(i)),
+                        'notes': request.POST.get('f{0}notes'.format(i)),
+                        }} for i, f in enumerate(fields, 1)]})
+
+            h.redirect_to(
+                controller='ckanext.datastore.controller:DatastoreController',
+                action='dictionary',
+                id=id,
+                resource_id=resource_id)
+
+        return render(
+            'datastore/dictionary.html',
+            extra_vars={'fields': fields})