From 890afc75c7447a799fd2d47ffb222b066434eb94 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 22 Feb 2017 13:24:45 +0000 Subject: [PATCH 01/10] Minor change in CHANGELOG --- CHANGELOG.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 32e57264be2..522cb628df6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -239,7 +239,7 @@ v2.5.0 2015-12-17 Cancelled release -v2.4.5 2017-02-22 +v2.4.6 2017-02-22 ================= * Use the url_for() helper for datapusher URLs (#2866) @@ -254,6 +254,11 @@ v2.4.5 2017-02-22 * Remove idle database connection (#3260) * Fix package_owner_org_update action when called via the API (#2661) +v2.4.5 2017-02-22 +================= + +Cancelled release + v2.4.4 2016-11-02 ================= From 6c3a61329ce832261bd4c2948c0a45a980f3bcdb Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 22 Mar 2017 15:43:29 +0200 Subject: [PATCH 02/10] Change version number for 2.6.2 --- ckan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/__init__.py b/ckan/__init__.py index b76d5b7e010..2e0d9268809 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,6 +1,6 @@ # encoding: utf-8 -__version__ = '2.6.1' +__version__ = '2.6.2' __description__ = 'CKAN Software' __long_description__ = \ From 61eaff6ed1cfb840ce9e20b888bdf32ceea911e7 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 15 Mar 2017 18:03:20 +0200 Subject: [PATCH 03/10] Use h.url_for and qualified=True for reset mails --- ckan/lib/mailer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index 9eaf2fd6813..c75e3435e96 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -144,11 +144,11 @@ def get_invite_body(user, group_dict=None, role=None): def get_reset_link(user): - return urljoin(config.get('site_url'), - h.url_for(controller='user', - action='perform_reset', - id=user.id, - key=user.reset_key)) + return h.url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key, + qualified=True) def send_reset_link(user): From 1989eed3f1885b9584744adcdbda67f1b16e09f4 Mon Sep 17 00:00:00 2001 From: Jana Sloukova Date: Tue, 14 Mar 2017 17:15:26 +0100 Subject: [PATCH 04/10] Setting of datastore_active flag moved to separate function --- ckanext/datastore/logic/action.py | 95 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index e519670bf19..8d45f837b91 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -149,47 +149,7 @@ def datastore_create(context, data_dict): log.debug( 'Setting datastore_active=True on resource {0}'.format(resource.id) ) - # issue #3245: race condition - update_dict = {'datastore_active': True} - - # get extras(for entity update) and package_id(for search index update) - res_query = model.Session.query( - model.resource_table.c.extras, - model.resource_table.c.package_id - ).filter( - model.Resource.id == data_dict['resource_id'] - ) - extras, package_id = res_query.one() - - # update extras in database for record and its revision - extras.update(update_dict) - res_query.update({'extras': extras}, synchronize_session=False) - - model.Session.query(model.resource_revision_table).filter( - model.ResourceRevision.id == data_dict['resource_id'], - model.ResourceRevision.current is True - ).update({'extras': extras}, synchronize_session=False) - - model.Session.commit() - - # get package with updated resource from solr - # find changed resource, patch it and reindex package - psi = search.PackageSearchIndex() - solr_query = search.PackageSearchQuery() - q = { - 'q': 'id:"{0}"'.format(package_id), - 'fl': 'data_dict', - 'wt': 'json', - 'fq': 'site_id:"%s"' % config.get('ckan.site_id'), - 'rows': 1 - } - for record in solr_query.run(q)['results']: - solr_data_dict = json.loads(record['data_dict']) - for resource in solr_data_dict['resources']: - if resource['id'] == data_dict['resource_id']: - resource.update(update_dict) - psi.index_package(solr_data_dict) - break + set_datastore_active_flag(model, data_dict, True) result.pop('id', None) result.pop('private', None) @@ -396,11 +356,9 @@ def datastore_delete(context, data_dict): if (not data_dict.get('filters') and resource.extras.get('datastore_active') is True): log.debug( - 'Setting datastore_active=True on resource {0}'.format(resource.id) + 'Setting datastore_active=False on resource {0}'.format(resource.id) ) - p.toolkit.get_action('resource_patch')( - context, {'id': data_dict['resource_id'], - 'datastore_active': False}) + set_datastore_active_flag(model, data_dict, False) result.pop('id', None) result.pop('connection_url') @@ -598,6 +556,53 @@ def datastore_make_public(context, data_dict): db.make_public(context, data_dict) +def set_datastore_active_flag(model, data_dict, flag): + ''' + Set appropriate datastore_active flag on CKAN resource. + + Called after creation or deletion of DataStore table. + ''' + update_dict = {'datastore_active': flag} + + # get extras(for entity update) and package_id(for search index update) + res_query = model.Session.query( + model.resource_table.c.extras, + model.resource_table.c.package_id + ).filter( + model.Resource.id == data_dict['resource_id'] + ) + extras, package_id = res_query.one() + + # update extras in database for record and its revision + extras.update(update_dict) + res_query.update({'extras': extras}, synchronize_session=False) + model.Session.query(model.resource_revision_table).filter( + model.ResourceRevision.id == data_dict['resource_id'], + model.ResourceRevision.current is True + ).update({'extras': extras}, synchronize_session=False) + + model.Session.commit() + + # get package with updated resource from solr + # find changed resource, patch it and reindex package + psi = search.PackageSearchIndex() + solr_query = search.PackageSearchQuery() + q = { + 'q': 'id:"{0}"'.format(package_id), + 'fl': 'data_dict', + 'wt': 'json', + 'fq': 'site_id:"%s"' % config.get('ckan.site_id'), + 'rows': 1 + } + for record in solr_query.run(q)['results']: + solr_data_dict = json.loads(record['data_dict']) + for resource in solr_data_dict['resources']: + if resource['id'] == data_dict['resource_id']: + resource.update(update_dict) + psi.index_package(solr_data_dict) + break + + def _resource_exists(context, data_dict): ''' Returns true if the resource exists in CKAN and in the datastore ''' model = _get_or_bust(context, 'model') From 84a80337186ab2318163ad79aa9f7737477cb7b8 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Mon, 13 Mar 2017 14:30:32 -0400 Subject: [PATCH 05/10] fix edit resource of draft dataset --- ckan/controllers/package.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 6b65a4514af..9e5c80328dc 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -586,12 +586,7 @@ def resource_edit(self, id, resource_id, data=None, errors=None, # dataset has not yet been fully created resource_dict = get_action('resource_show')(context, {'id': resource_id}) - fields = ['url', 'resource_type', 'format', 'name', 'description', - 'id'] - data = {} - for field in fields: - data[field] = resource_dict[field] - return self.new_resource(id, data=data) + return self.new_resource(id, data=resource_dict) # resource is fully created try: resource_dict = get_action('resource_show')(context, From dbbc48bf1040553732b36bb162499644c54f1b5c Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Fri, 24 Feb 2017 10:43:26 +0200 Subject: [PATCH 06/10] Fix tags on org/group read pages --- ckan/controllers/group.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 087209f07c4..d0fc32a7470 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -271,8 +271,9 @@ def drill_down_url(**by): c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): + controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, - controller='group', action='read', + controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field @@ -284,6 +285,7 @@ def pager_url(q=None, page=None): try: c.fields = [] + c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ @@ -291,6 +293,10 @@ def pager_url(q=None, page=None): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) + if param not in c.fields_grouped: + c.fields_grouped[param] = [value] + else: + c.fields_grouped[param].append(value) else: search_extras[param] = value From 4340736c0dde24abb3e80eb05a3882476a876e64 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 22 Mar 2017 15:55:42 +0200 Subject: [PATCH 07/10] auth check in revision controller --- ckan/controllers/revision.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ckan/controllers/revision.py b/ckan/controllers/revision.py index dd3527cc4ea..9c35a553977 100644 --- a/ckan/controllers/revision.py +++ b/ckan/controllers/revision.py @@ -159,6 +159,15 @@ def diff(self, id=None): c.diff_entity = request.params.get('diff_entity') if c.diff_entity == 'package': + try: + logic.check_access('package_show', { + 'model': model, + 'user': c.user or c.author, + 'auth_user_obj': c.userobj + }, {'id': id}) + except logic.NotAuthorized: + base.abort(401) + c.pkg = model.Package.by_name(id) diff = c.pkg.diff(c.revision_to, c.revision_from) elif c.diff_entity == 'group': From 682bae68f280efc612f09de0f7111553b7dfa329 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 22 Mar 2017 17:18:06 +0200 Subject: [PATCH 08/10] Updated changelog --- CHANGELOG.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 522cb628df6..d7cee95d673 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Changelog --------- +v2.6.2 2017-03-22 +================= + +* Use fully qualified urls for reset emails (#3486) +* Fix edit_resource for resource with draft state (#3480) +* Tag fix for group/organization pages (#3460) +* Setting of datastore_active flag moved to separate function (#3481) + v2.6.1 2017-02-22 ================= @@ -394,8 +402,8 @@ Changes and deprecations Custom templates or users of this API call will need to pass ``include_datasets=True`` to include datasets in the response. -* The ``vocabulary_show`` and ``tag_show`` API calls no longer returns the - ``packages`` key - i.e. datasets that use the vocabulary or tag. +* The ``vocabulary_show`` and ``tag_show`` API calls no longer returns the + ``packages`` key - i.e. datasets that use the vocabulary or tag. However ``tag_show`` now has an ``include_datasets`` option. (#1886) * Config option ``site_url`` is now required - CKAN will not abort during From 7e2430a8fac88cf69bb33876f10c4556719f1f1a Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Mar 2017 16:33:40 +0200 Subject: [PATCH 09/10] Use json.dumps for nested fields in datastore_dump --- ckanext/datastore/controller.py | 24 +++++++++++++++++- ckanext/datastore/tests/test_dump.py | 38 +++++++++++++++++++--------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index b872c1d3a78..8f1398ec8c7 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -1,5 +1,6 @@ # encoding: utf-8 +import json import StringIO import unicodecsv as csv @@ -12,6 +13,20 @@ from ckan.common import request +def _dump_nested(column, record): + name, ctype = column + value = record[name] + + is_nested = ( + ctype == 'json' or + ctype.startswith('_') or + ctype.endswith(']') + ) + if is_nested: + return json.dumps(value) + return value + + class DatastoreController(base.BaseController): def dump(self, resource_id): context = { @@ -41,6 +56,13 @@ def dump(self, resource_id): header = [x['id'] for x in result['fields']] wr.writerow(header) + columns = [ + (x['id'], x['type']) + for x in result['fields']] + for record in result['records']: - wr.writerow([record[column] for column in header]) + wr.writerow([ + _dump_nested(column, record) + for column in columns]) + return f.getvalue() diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index b966bedf7fa..f488011d8fe 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -3,7 +3,7 @@ import json import nose -from nose.tools import assert_equals +from nose.tools import assert_equals, assert_in from ckan.common import config import sqlalchemy.orm as orm import paste.fixture @@ -39,14 +39,16 @@ def setup_class(cls): 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, {'id': 'published'}, - {'id': u'characters', u'type': u'_text'}], + {'id': u'characters', u'type': u'_text'}, + {'id': 'random_letters', 'type': 'text[]'}], 'records': [{u'b\xfck': 'annakarenina', - 'author': 'tolstoy', - 'published': '2005-03-01', - 'nested': ['b', {'moo': 'moo'}], - u'characters': [u'Princess Anna', u'Sergius']}, + 'author': 'tolstoy', + 'published': '2005-03-01', + 'nested': ['b', {'moo': 'moo'}], + u'characters': [u'Princess Anna', u'Sergius'], + 'random_letters': ['a', 'e', 'x']}, {u'b\xfck': 'warandpeace', 'author': 'tolstoy', - 'nested': {'a': 'b'}}] + 'nested': {'a': 'b'}, 'random_letters': []}] } postparams = '%s=1' % json.dumps(cls.data) auth = {'Authorization': str(cls.sysadmin_user.apikey)} @@ -69,10 +71,13 @@ def test_dump_basic(self): res = self.app.get('/datastore/dump/{0}'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = u'_id,b\xfck,author,published,characters,nested' + + expected = ( + u'_id,b\xfck,author,published' + u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) - assert 'warandpeace' in content - assert "[u'Princess Anna', u'Sergius']" in content + assert_in('warandpeace', content) + assert_in('"[""Princess Anna"", ""Sergius""]"', content) # get with alias instead of id res = self.app.get('/datastore/dump/{0}'.format(str( @@ -88,6 +93,15 @@ def test_dump_limit(self): res = self.app.get('/datastore/dump/{0}?limit=1'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = u'_id,b\xfck,author,published,characters,nested' + + expected = (u'_id,b\xfck,author,published' + u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) - assert_equals(len(content), 148) + + expected_content = ( + u'_id,b\xfck,author,published,characters,random_letters,' + u'nested\r\n1,annakarenina,tolstoy,2005-03-01T00:00:00,' + u'"[""Princess Anna"", ""Sergius""]",' + u'"[""a"", ""e"", ""x""]","[""b"", ' + u'{""moo"": ""moo""}]"\r\n') + assert_equals(content, expected_content) From a6fc30979dd75890c3322006554a350102f6b10b Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 3 Apr 2017 15:24:18 +0300 Subject: [PATCH 10/10] updated _dump_nested --- ckanext/datastore/controller.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 8f1398ec8c7..418b669775a 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -13,15 +13,9 @@ from ckan.common import request -def _dump_nested(column, record): - name, ctype = column - value = record[name] +def _json_dump_nested(value): + is_nested = isinstance(value, (list, dict)) - is_nested = ( - ctype == 'json' or - ctype.startswith('_') or - ctype.endswith(']') - ) if is_nested: return json.dumps(value) return value @@ -56,13 +50,9 @@ def dump(self, resource_id): header = [x['id'] for x in result['fields']] wr.writerow(header) - columns = [ - (x['id'], x['type']) - for x in result['fields']] - for record in result['records']: wr.writerow([ - _dump_nested(column, record) - for column in columns]) + _json_dump_nested(record[column]) + for column in header]) return f.getvalue()