Merge branch 'release-v2.5.5' into release-v2.5-latest

CHANGELOG.rst

@@ -7,6 +7,14 @@
 Changelog
 ---------
 
+v2.5.5 2017-03-22
+=================
+
+* Use fully qualified urls for reset emails (#3486)
+* Fix edit_resource for resource with draft state (#3480)
+* Tag fix for group/organization pages (#3460)
+* Setting of datastore_active flag moved to separate function (#3481)
+
 v2.5.4 2017-02-22
 =================
 

ckan/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.5.4'
+__version__ = '2.5.5'
 
 __description__ = 'CKAN Software'
 __long_description__ = \

ckan/controllers/group.py

@@ -274,8 +274,9 @@ def drill_down_url(**by):
         c.drill_down_url = drill_down_url
 
         def remove_field(key, value=None, replace=None):
+            controller = lookup_group_controller(group_type)
             return h.remove_url_param(key, value=value, replace=replace,
-                                      controller='group', action='read',
+                                      controller=controller, action='read',
                                       extras=dict(id=c.group_dict.get('name')))
 
         c.remove_field = remove_field
@@ -287,13 +288,18 @@ def pager_url(q=None, page=None):
 
         try:
             c.fields = []
+            c.fields_grouped = {}
             search_extras = {}
             for (param, value) in request.params.items():
                 if not param in ['q', 'page', 'sort'] \
                         and len(value) and not param.startswith('_'):
                     if not param.startswith('ext_'):
                         c.fields.append((param, value))
                         q += ' %s: "%s"' % (param, value)
+                        if param not in c.fields_grouped:
+                            c.fields_grouped[param] = [value]
+                        else:
+                            c.fields_grouped[param].append(value)
                     else:
                         search_extras[param] = value
 

ckan/controllers/package.py

@@ -586,12 +586,7 @@ def resource_edit(self, id, resource_id, data=None, errors=None,
             # dataset has not yet been fully created
             resource_dict = get_action('resource_show')(context,
                                                         {'id': resource_id})
-            fields = ['url', 'resource_type', 'format', 'name', 'description',
-                      'id']
-            data = {}
-            for field in fields:
-                data[field] = resource_dict[field]
-            return self.new_resource(id, data=data)
+            return self.new_resource(id, data=resource_dict)
         # resource is fully created
         try:
             resource_dict = get_action('resource_show')(context,

ckan/controllers/revision.py

@@ -157,6 +157,15 @@ def diff(self, id=None):
 
         c.diff_entity = request.params.get('diff_entity')
         if c.diff_entity == 'package':
+            try:
+                logic.check_access('package_show',  {
+                    'model': model,
+                    'user': c.user or c.author,
+                    'auth_user_obj': c.userobj
+                }, {'id': id})
+            except logic.NotAuthorized:
+                base.abort(401)
+
             c.pkg = model.Package.by_name(id)
             diff = c.pkg.diff(c.revision_to, c.revision_from)
         elif c.diff_entity == 'group':

ckan/lib/mailer.py

@@ -133,11 +133,12 @@ def get_invite_body(user):
     return invite_message.format(**d)
 
 def get_reset_link(user):
-    return urljoin(g.site_url,
-                   h.url_for(controller='user',
-                           action='perform_reset',
-                           id=user.id,
-                           key=user.reset_key))
+    return h.url_for(controller='user',
+                     action='perform_reset',
+                     id=user.id,
+                     key=user.reset_key,
+                     qualified=True)
+
 
 def send_reset_link(user):
     create_reset_key(user)
@@ -164,6 +165,3 @@ def verify_reset_link(user, key):
     if not user.reset_key or len(user.reset_key) < 5:
         return False
     return key.strip() == user.reset_key
-
-
-

ckanext/datastore/controller.py

@@ -1,3 +1,4 @@
+import json
 import StringIO
 import unicodecsv as csv
 
@@ -10,6 +11,14 @@
 from ckan.common import request
 
 
+def _json_dump_nested(value):
+    is_nested = isinstance(value, (list, dict))
+
+    if is_nested:
+        return json.dumps(value)
+    return value
+
+
 class DatastoreController(base.BaseController):
     def dump(self, resource_id):
         context = {
@@ -40,5 +49,8 @@ def dump(self, resource_id):
         wr.writerow(header)
 
         for record in result['records']:
-            wr.writerow([record[column] for column in header])
+            wr.writerow([
+                _json_dump_nested(record[column])
+                for column in header])
+
         return f.getvalue()

ckanext/datastore/logic/action.py

@@ -148,47 +148,7 @@ def datastore_create(context, data_dict):
         log.debug(
             'Setting datastore_active=True on resource {0}'.format(resource.id)
         )
-        # issue #3245: race condition
-        update_dict = {'datastore_active': True}
-
-        # get extras(for entity update) and package_id(for search index update)
-        res_query = model.Session.query(
-            model.resource_table.c.extras,
-            model.resource_table.c.package_id
-        ).filter(
-            model.Resource.id == data_dict['resource_id']
-        )
-        extras, package_id = res_query.one()
-
-        # update extras in database for record and its revision
-        extras.update(update_dict)
-        res_query.update({'extras': extras}, synchronize_session=False)
-
-        model.Session.query(model.resource_revision_table).filter(
-            model.ResourceRevision.id == data_dict['resource_id'],
-            model.ResourceRevision.current is True
-        ).update({'extras': extras}, synchronize_session=False)
-
-        model.Session.commit()
-
-        # get package with  updated resource from solr
-        # find changed resource, patch it and reindex package
-        psi = search.PackageSearchIndex()
-        solr_query = search.PackageSearchQuery()
-        q = {
-            'q': 'id:"{0}"'.format(package_id),
-            'fl': 'data_dict',
-            'wt': 'json',
-            'fq': 'site_id:"%s"' % pylons.config.get('ckan.site_id'),
-            'rows': 1
-        }
-        for record in solr_query.run(q)['results']:
-            solr_data_dict = json.loads(record['data_dict'])
-            for resource in solr_data_dict['resources']:
-                if resource['id'] == data_dict['resource_id']:
-                    resource.update(update_dict)
-                    psi.index_package(solr_data_dict)
-                    break
+        set_datastore_active_flag(model, data_dict, True)
 
     result.pop('id', None)
     result.pop('private', None)
@@ -395,11 +355,9 @@ def datastore_delete(context, data_dict):
     if (not data_dict.get('filters') and
             resource.extras.get('datastore_active') is True):
         log.debug(
-            'Setting datastore_active=True on resource {0}'.format(resource.id)
+            'Setting datastore_active=False on resource {0}'.format(resource.id)
         )
-        p.toolkit.get_action('resource_patch')(
-            context, {'id': data_dict['resource_id'],
-                      'datastore_active': False})
+        set_datastore_active_flag(model, data_dict, False)
 
     result.pop('id', None)
     result.pop('connection_url')
@@ -597,6 +555,53 @@ def datastore_make_public(context, data_dict):
     db.make_public(context, data_dict)
 
 
+def set_datastore_active_flag(model, data_dict, flag):
+    '''
+    Set appropriate datastore_active flag on CKAN resource.
+
+    Called after creation or deletion of DataStore table.
+    '''
+    update_dict = {'datastore_active': flag}
+
+    # get extras(for entity update) and package_id(for search index update)
+    res_query = model.Session.query(
+        model.resource_table.c.extras,
+        model.resource_table.c.package_id
+    ).filter(
+        model.Resource.id == data_dict['resource_id']
+    )
+    extras, package_id = res_query.one()
+
+    # update extras in database for record and its revision
+    extras.update(update_dict)
+    res_query.update({'extras': extras}, synchronize_session=False)
+    model.Session.query(model.resource_revision_table).filter(
+        model.ResourceRevision.id == data_dict['resource_id'],
+        model.ResourceRevision.current is True
+    ).update({'extras': extras}, synchronize_session=False)
+
+    model.Session.commit()
+
+    # get package with  updated resource from solr
+    # find changed resource, patch it and reindex package
+    psi = search.PackageSearchIndex()
+    solr_query = search.PackageSearchQuery()
+    q = {
+        'q': 'id:"{0}"'.format(package_id),
+        'fl': 'data_dict',
+        'wt': 'json',
+        'fq': 'site_id:"%s"' % pylons.config.get('ckan.site_id'),
+        'rows': 1
+    }
+    for record in solr_query.run(q)['results']:
+        solr_data_dict = json.loads(record['data_dict'])
+        for resource in solr_data_dict['resources']:
+            if resource['id'] == data_dict['resource_id']:
+                resource.update(update_dict)
+                psi.index_package(solr_data_dict)
+                break
+
+
 def _resource_exists(context, data_dict):
     ''' Returns true if the resource exists in CKAN and in the datastore '''
     model = _get_or_bust(context, 'model')

ckanext/datastore/tests/test_dump.py

@@ -2,6 +2,7 @@
 
 import nose
 from nose.tools import assert_equals
+from ckan.tests.helpers import assert_in
 from pylons import config
 import sqlalchemy.orm as orm
 import paste.fixture
@@ -37,14 +38,16 @@ def setup_class(cls):
             'fields': [{'id': u'b\xfck', 'type': 'text'},
                        {'id': 'author', 'type': 'text'},
                        {'id': 'published'},
-                       {'id': u'characters', u'type': u'_text'}],
+                       {'id': u'characters', u'type': u'_text'},
+                       {'id': 'random_letters', 'type': 'text[]'}],
             'records': [{u'b\xfck': 'annakarenina',
-                        'author': 'tolstoy',
-                        'published': '2005-03-01',
-                        'nested': ['b', {'moo': 'moo'}],
-                        u'characters': [u'Princess Anna', u'Sergius']},
+                         'author': 'tolstoy',
+                         'published': '2005-03-01',
+                         'nested': ['b', {'moo': 'moo'}],
+                         u'characters': [u'Princess Anna', u'Sergius'],
+                         'random_letters': ['a', 'e', 'x']},
                         {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
-                         'nested': {'a': 'b'}}]
+                         'nested': {'a': 'b'}, 'random_letters': []}]
         }
         postparams = '%s=1' % json.dumps(cls.data)
         auth = {'Authorization': str(cls.sysadmin_user.apikey)}
@@ -67,10 +70,13 @@ def test_dump_basic(self):
         res = self.app.get('/datastore/dump/{0}'.format(str(
             self.data['resource_id'])), extra_environ=auth)
         content = res.body.decode('utf-8')
-        expected = u'_id,b\xfck,author,published,characters,nested'
+
+        expected = (
+            u'_id,b\xfck,author,published'
+            u',characters,random_letters,nested')
         assert_equals(content[:len(expected)], expected)
-        assert 'warandpeace' in content
-        assert "[u'Princess Anna', u'Sergius']" in content
+        assert_in('warandpeace', content)
+        assert_in('"[""Princess Anna"", ""Sergius""]"', content)
 
         # get with alias instead of id
         res = self.app.get('/datastore/dump/{0}'.format(str(
@@ -86,6 +92,15 @@ def test_dump_limit(self):
         res = self.app.get('/datastore/dump/{0}?limit=1'.format(str(
             self.data['resource_id'])), extra_environ=auth)
         content = res.body.decode('utf-8')
-        expected = u'_id,b\xfck,author,published,characters,nested'
+
+        expected = (u'_id,b\xfck,author,published'
+                    u',characters,random_letters,nested')
         assert_equals(content[:len(expected)], expected)
-        assert_equals(len(content), 148)
+
+        expected_content = (
+            u'_id,b\xfck,author,published,characters,random_letters,'
+            u'nested\r\n1,annakarenina,tolstoy,2005-03-01T00:00:00,'
+            u'"[""Princess Anna"", ""Sergius""]",'
+            u'"[""a"", ""e"", ""x""]","[""b"", '
+            u'{""moo"": ""moo""}]"\r\n')
+        assert_equals(content, expected_content)

dev-requirements.txt

@@ -1,7 +1,7 @@
 # These are packages that required when running ckan tests and building the docs
 
 docutils==0.8.1
-httpretty==0.8.3
+httpretty==0.8.7
 # nose==1.3.0  # already in requirements.txt
 pep8==1.4.6
 Sphinx==1.2.3