From a47f7dd40ff82519096535d70b940dd3e363b746 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 10 Feb 2017 17:09:53 +0000 Subject: [PATCH 01/76] Update version number for 2.5.4b --- ckan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/__init__.py b/ckan/__init__.py index d0db9a01b54..f017397a7c0 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.5.3' +__version__ = '2.5.4b' __description__ = 'CKAN Software' __long_description__ = \ From f51238570276d2771d3e9d9f41d2e13991402998 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 24 Jun 2016 14:07:50 +0100 Subject: [PATCH 02/76] [#2661] Add revision if owner-org updated. package_owner_org_update is called during a package_update to handle updating the owner organization. A package revision is created during this by package_update. If package_owner_org_update is called in isolation, no revision is created and an error occurs in vdm. This commit ensures a revision is created when package_owner_org_update is called outside of a package_update. --- ckan/logic/action/update.py | 13 +++++- ckan/tests/logic/action/test_update.py | 55 +++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index b16eac1057d..81b492cb419 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -354,6 +354,7 @@ def package_update(context, data_dict): context_org_update = context.copy() context_org_update['ignore_auth'] = True context_org_update['defer_commit'] = True + context_org_update['add_revision'] = False _get_action('package_owner_org_update')(context_org_update, {'id': pkg.id, 'organization_id': pkg.owner_org}) @@ -1082,6 +1083,7 @@ def package_owner_org_update(context, data_dict): :type id: string ''' model = context['model'] + user = context['user'] name_or_id = data_dict.get('id') organization_id = data_dict.get('organization_id') @@ -1101,10 +1103,17 @@ def package_owner_org_update(context, data_dict): org = None pkg.owner_org = None + if context.get('add_revision', True): + rev = model.repo.new_revision() + rev.author = user + if 'message' in context: + rev.message = context['message'] + else: + rev.message = _(u'REST API: Update object %s') % pkg.get("name") members = model.Session.query(model.Member) \ - .filter(model.Member.table_id == pkg.id) \ - .filter(model.Member.capacity == 'organization') + .filter(model.Member.table_id == pkg.id) \ + .filter(model.Member.capacity == 'organization') need_update = True for member_obj in members: diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 24a195f2c51..c85a4e8910c 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -858,7 +858,7 @@ def test_user_create_password_hash_not_for_normal_users(self): assert user_obj.password != 'pretend-this-is-a-valid-hash' -class TestBulkOperations(object): +class TestPackageOwnerOrgUpdate(object): @classmethod def teardown_class(cls): @@ -867,6 +867,59 @@ def teardown_class(cls): def setup(self): helpers.reset_db() + def test_package_owner_org_added(self): + '''A package without an owner_org can have one added.''' + sysadmin = factories.Sysadmin() + org = factories.Organization() + dataset = factories.Dataset() + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] is None + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=org['id']) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org == org['id'] + + def test_package_owner_org_changed(self): + '''A package with an owner_org can have it changed.''' + + sysadmin = factories.Sysadmin() + org_1 = factories.Organization() + org_2 = factories.Organization() + dataset = factories.Dataset(owner_org=org_1['id']) + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] == org_1['id'] + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=org_2['id']) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org == org_2['id'] + + def test_package_owner_org_removed(self): + '''A package with an owner_org can have it removed.''' + sysadmin = factories.Sysadmin() + org = factories.Organization() + dataset = factories.Dataset(owner_org=org['id']) + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] == org['id'] + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=None) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org is None + + +class TestBulkOperations(object): + def test_bulk_make_private(self): org = factories.Organization() From 0a8fee66e982baa171ecbd23b35a2c249373aac0 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 24 Jun 2016 15:34:16 +0100 Subject: [PATCH 03/76] [#2661] Add 'add_revision' to ctx in pkg create. package_create also calls package_owner_org_update, and requires the 'add_revision' property adding to the context to prevent package_owner_org_update from creating an unecessary revision. --- ckan/logic/action/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index fa81cd9896a..6ef480331be 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -195,6 +195,7 @@ def package_create(context, data_dict): context_org_update = context.copy() context_org_update['ignore_auth'] = True context_org_update['defer_commit'] = True + context_org_update['add_revision'] = False _get_action('package_owner_org_update')(context_org_update, {'id': pkg.id, 'organization_id': pkg.owner_org}) From 50098e7441cf64e119c68d6cbab7b096fae4e376 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 21 Nov 2016 13:55:41 +0200 Subject: [PATCH 04/76] 3260 removed idle connection Session, that used for querying config varibales diring startup was not finished and one idle connection appeared in postgres. Now session committed in the end of environment setup. --- ckan/config/environment.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 0bd9416d4df..18d66d83fe4 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -231,6 +231,11 @@ def genshi_lookup_attr(cls, obj, key): # load all CKAN plugins p.load_all(config) + # issue #3260: remove idle transaction + # Session that was used for getting all config params nor committed, + # neither removed and we have idle connection as result + model.Session.commit() + # A mapping of config settings that can be overridden by env vars. # Note: Do not remove the following lines, they are used in the docs From 423cfd20d8501d168025acea09744c6866c7531e Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 21 Oct 2016 13:50:00 +0100 Subject: [PATCH 05/76] [#3265] Add tests for user_delete --- ckan/tests/logic/action/test_delete.py | 68 ++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index ab2b25b9539..36ca97ef8b1 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -456,3 +456,71 @@ def test_missing_id_returns_error(self): def test_bad_id_returns_404(self): assert_raises(logic.NotFound, helpers.call_action, 'dataset_purge', id='123') + + +class TestUserDelete(object): + def setup(self): + helpers.reset_db() + + def test_user_delete(self): + user = factories.User() + context = {} + params = {u'id': user[u'id']} + + helpers.call_action(u'user_delete', context, **params) + + # It is still there but with state=deleted + user_obj = model.User.get(user[u'id']) + assert_equals(user_obj.state, u'deleted') + + def test_user_delete_removes_memberships(self): + user = factories.User() + factories.Organization( + users=[{u'name': user[u'id'], u'capacity': u'admin'}]) + + factories.Group( + users=[{u'name': user[u'id'], u'capacity': u'admin'}]) + + user_memberships = model.Session.query(model.Member).filter( + model.Member.table_id == user[u'id']).all() + + assert_equals(len(user_memberships), 2) + + assert_equals([m.state for m in user_memberships], + [u'active', u'active']) + + context = {} + params = {u'id': user[u'id']} + + helpers.call_action(u'user_delete', context, **params) + + user_memberships = model.Session.query(model.Member).filter( + model.Member.table_id == user[u'id']).all() + + # Member objects are still there, but flagged as deleted + assert_equals(len(user_memberships), 2) + + assert_equals([m.state for m in user_memberships], + [u'deleted', u'deleted']) + + def test_user_delete_removes_memberships_when_using_name(self): + user = factories.User() + factories.Organization( + users=[{u'name': user[u'id'], u'capacity': u'admin'}]) + + factories.Group( + users=[{u'name': user[u'id'], u'capacity': u'admin'}]) + + context = {} + params = {u'id': user[u'name']} + + helpers.call_action(u'user_delete', context, **params) + + user_memberships = model.Session.query(model.Member).filter( + model.Member.table_id == user[u'id']).all() + + # Member objects are still there, but flagged as deleted + assert_equals(len(user_memberships), 2) + + assert_equals([m.state for m in user_memberships], + [u'deleted', u'deleted']) From 5dcd2b6ea8709968c41a814591a6ea63991a3443 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 21 Oct 2016 13:50:42 +0100 Subject: [PATCH 06/76] [#3265] Use user id to delete memberships Otherwise when passing the user name on `user_delete` the memberships where not deleted --- ckan/logic/action/delete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 1a46b63f805..87707bd0bf2 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -48,7 +48,7 @@ def user_delete(context, data_dict): user.delete() user_memberships = model.Session.query(model.Member).filter( - model.Member.table_id == user_id).all() + model.Member.table_id == user.id).all() for membership in user_memberships: membership.delete() From 03b7ebed9a1c0115f5d993110bcf879f9cfca03b Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 21 Oct 2016 13:52:11 +0100 Subject: [PATCH 07/76] [#3265] Use action on user remove CLI command Otherwise we miss all the membership logic --- ckan/lib/cli.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 8845577f3d6..487203d9d10 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -872,12 +872,9 @@ def remove(self): return username = self.args[1] - user = model.User.by_name(unicode(username)) - if not user: - print 'Error: user "%s" not found!' % username - return - user.delete() - model.repo.commit_and_remove() + p.toolkit.get_action('user_delete')( + {'model': model, 'ignore_auth': True}, + {'id': username}) print('Deleted user: %s' % username) From d70db78f5af2d5f0b101241d7701e560f89aefde Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 24 Nov 2016 13:58:04 +0200 Subject: [PATCH 08/76] 3245 datastore_active race condition `datastore_create` directly updates database and solr index and this reduces possibility of conflicts inside simultaneous calls --- ckanext/datastore/logic/action.py | 47 ++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 2baa02fba6b..f2d44c34ab7 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -1,9 +1,10 @@ import logging +import json import pylons import sqlalchemy -import ckan.lib.base as base +import ckan.lib.search as search import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p @@ -147,9 +148,47 @@ def datastore_create(context, data_dict): log.debug( 'Setting datastore_active=True on resource {0}'.format(resource.id) ) - p.toolkit.get_action('resource_patch')( - context, - {'id': data_dict['resource_id'], 'datastore_active': True}) + # issue #3245: race condition + update_dict = {'datastore_active': True} + + # get extras(for entity update) and package_id(for search index update) + res_query = model.Session.query( + model.resource_table.c.extras, + model.resource_table.c.package_id + ).filter( + model.Resource.id == data_dict['resource_id'] + ) + extras, package_id = res_query.one() + + # update extras in database for record and its revision + extras.update(update_dict) + res_query.update({'extras': extras}, synchronize_session=False) + + model.Session.query(model.resource_revision_table).filter( + model.ResourceRevision.id == data_dict['resource_id'], + model.ResourceRevision.current is True + ).update({'extras': extras}, synchronize_session=False) + + model.Session.commit() + + # get package with updated resource from solr + # find changed resource, patch it and reindex package + psi = search.PackageSearchIndex() + solr_query = search.PackageSearchQuery() + q = { + 'q': 'id:"{0}"'.format(package_id), + 'fl': 'data_dict', + 'wt': 'json', + 'fq': 'site_id:"%s"' % config.get('ckan.site_id'), + 'rows': 1 + } + for record in solr_query.run(q)['results']: + solr_data_dict = json.loads(record['data_dict']) + for resource in solr_data_dict['resources']: + if resource['id'] == data_dict['resource_id']: + resource.update(update_dict) + psi.index_package(solr_data_dict) + break result.pop('id', None) result.pop('private', None) From 30fb0d2c5e32d925c6feca7a9ec81b4c98937a66 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 29 Nov 2016 14:04:42 +0200 Subject: [PATCH 09/76] \#3189 Datastore doesn't add site_url to resource created via API Small change in datastore plugin, that generates fully-qualified url during datastore creation --- ckanext/datapusher/tests/test.py | 2 +- ckanext/datastore/plugin.py | 3 ++- ckanext/datastore/tests/test_create.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index c3c1d8069f7..3fe5b247d9c 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -78,7 +78,7 @@ def test_create_ckan_resource_in_package(self): res = tests.call_action_api( self.app, 'resource_show', id=res_dict['result']['resource_id']) - assert res['url'] == '/datastore/dump/' + res['id'], res + assert res['url'].endswith('/datastore/dump/' + res['id']), res @httpretty.activate def test_providing_res_with_url_calls_datapusher_correctly(self): diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 2175b28ebec..34ba456b10e 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -280,7 +280,8 @@ def before_show(self, resource_dict): if resource_dict.get('url_type') == 'datastore': resource_dict['url'] = p.toolkit.url_for( controller='ckanext.datastore.controller:DatastoreController', - action='dump', resource_id=resource_dict['id']) + action='dump', resource_id=resource_dict['id'], + qualified=True) if 'datastore_active' not in resource_dict: resource_dict[u'datastore_active'] = False diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 2cc6a80a0d3..5d75e677a1b 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -50,6 +50,20 @@ def test_create_creates_index_on_primary_key(self): index_names = self._get_index_names(resource_id) assert resource_id + '_pkey' in index_names + def test_create_creates_url_with_site_name(self): + package = factories.Dataset() + data = { + 'resource': { + 'boo%k': 'crime', + 'package_id': package['id'] + }, + } + result = helpers.call_action('datastore_create', **data) + resource_id = result['resource_id'] + resource = helpers.call_action('resource_show', id=resource_id) + url = resource['url'] + assert url.startswith(config.get('ckan.site_url')) + def test_create_index_on_specific_fields(self): package = factories.Dataset() data = { From c125131b9b6354811e771f0767f51aaf51b9712c Mon Sep 17 00:00:00 2001 From: Fabio Anderegg Date: Tue, 15 Nov 2016 16:18:28 +0100 Subject: [PATCH 10/76] ckan datastore: convert data error to unicode instead of str to avoid encoding errors --- ckanext/datastore/logic/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index f2d44c34ab7..5f670b7183e 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -141,7 +141,7 @@ def datastore_create(context, data_dict): try: result = db.create(context, data_dict) except db.InvalidDataError as err: - raise p.toolkit.ValidationError(str(err)) + raise p.toolkit.ValidationError(unicode(err)) # Set the datastore_active flag on the resource if necessary if resource.extras.get('datastore_active') is not True: From 21c1d6ce3fbc8fabc240ebf2f7b9939d32f509d7 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 14 Feb 2017 13:06:00 +0000 Subject: [PATCH 11/76] Change order of Solr filters --- ckan/logic/action/get.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index da897b78665..53887d9ce8d 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1917,7 +1917,7 @@ def package_search(context, data_dict): fq = data_dict.get('fq', '') if not context.get('ignore_capacity_check', False): fq = ' '.join(p for p in fq.split() if 'capacity:' not in p) - data_dict['fq'] = fq + ' capacity:"public"' + data_dict['fq'] = 'capacity:"public" ' + fq # Solr doesn't need 'include_drafts`, so pop it. include_drafts = data_dict.pop('include_drafts', False) @@ -1925,15 +1925,15 @@ def package_search(context, data_dict): if include_drafts: user_id = authz.get_user_id_for_username(user, allow_none=True) if authz.is_sysadmin(user): - data_dict['fq'] = fq + ' +state:(active OR draft)' + data_dict['fq'] = '+state:(active OR draft) ' + fq elif user_id: # Query to return all active datasets, and all draft datasets # for this user. - data_dict['fq'] = fq + \ - ' ((creator_user_id:{0} AND +state:(draft OR active))' \ - ' OR state:active)'.format(user_id) + u_fq = ' ((creator_user_id:{0} AND +state:(draft OR active))' \ + ' OR state:active) '.format(user_id) + data_dict['fq'] = u_fq + ' ' + fq elif not authz.is_sysadmin(user): - data_dict['fq'] = fq + ' +state:active' + data_dict['fq'] = '+state:active ' + fq # Pop these ones as Solr does not need them extras = data_dict.pop('extras', None) From 6f27eb5ca2ed83f2cd57f05cdacf4ec4cdc9cc6a Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 14 Feb 2017 13:30:48 +0000 Subject: [PATCH 12/76] Clean language url --- ckan/controllers/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/util.py b/ckan/controllers/util.py index 840c6833a04..2f59103610c 100644 --- a/ckan/controllers/util.py +++ b/ckan/controllers/util.py @@ -15,7 +15,7 @@ def redirect(self): if not url: base.abort(400, _('Missing Value') + ': url') - if h.url_is_local(url): + if h.url_is_local(url) and '\r' not in url and '\n' not in url: return base.redirect(url) else: base.abort(403, _('Redirecting to external site is not allowed.')) From 76b76f5db6bde9be088779a9219814e24ce182c0 Mon Sep 17 00:00:00 2001 From: Yan Date: Tue, 20 Dec 2016 12:13:41 +0200 Subject: [PATCH 13/76] [#3373]Dashboard_mark_activities_old is set to utcnow() format The problem was that dashboard_mark_activities_old method was not changed to utcnow() format, so when datasets were updated, because of the time difference, user have seen them before he actually entered the page. I've adde utcnow() format for this method and now the time works fine. Also added untnow() to Dashboard model, so the time can be saved correctly. --- ckan/logic/action/update.py | 2 +- ckan/model/dashboard.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 81b492cb419..1ca9b9caa46 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -1045,7 +1045,7 @@ def dashboard_mark_activities_old(context, data_dict): model = context['model'] user_id = model.User.get(context['user']).id model.Dashboard.get(user_id).activity_stream_last_viewed = ( - datetime.datetime.now()) + datetime.datetime.utcnow()) if not context.get('defer_commit'): model.repo.commit() diff --git a/ckan/model/dashboard.py b/ckan/model/dashboard.py index 6813f3ce133..3a9c75f3011 100644 --- a/ckan/model/dashboard.py +++ b/ckan/model/dashboard.py @@ -19,8 +19,8 @@ class Dashboard(object): def __init__(self, user_id): self.user_id = user_id - self.activity_stream_last_viewed = datetime.datetime.now() - self.email_last_sent = datetime.datetime.now() + self.activity_stream_last_viewed = datetime.datetime.utcnow() + self.email_last_sent = datetime.datetime.utcnow() @classmethod def get(cls, user_id): From 0b1d3e1e1ae99e102bf6a222aa1d49a5d907a467 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 5 Dec 2016 17:35:41 +0200 Subject: [PATCH 14/76] check group name and id during package creation --- ckan/logic/auth/create.py | 6 ++---- ckan/tests/legacy/functional/api/test_activity.py | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 31f67c35907..0a89ae3749d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -194,10 +194,8 @@ def _check_group_auth(context, data_dict): for group_blob in group_blobs: # group_blob might be a dict or a group_ref if isinstance(group_blob, dict): - if api_version == '1': - id = group_blob.get('name') - else: - id = group_blob.get('id') + # use group id by default, but we can accept name as well + id = group_blob.get('id') or group_blob.get('name') if not id: continue else: diff --git a/ckan/tests/legacy/functional/api/test_activity.py b/ckan/tests/legacy/functional/api/test_activity.py index 2a6d1bb5538..6f7dc4967c2 100644 --- a/ckan/tests/legacy/functional/api/test_activity.py +++ b/ckan/tests/legacy/functional/api/test_activity.py @@ -289,6 +289,9 @@ def _create_package(self, user, name=None): # Create a new package. request_data = make_package(name) + # quick fix for #3351 + request_data['groups'] = [] + before = self.record_details(user_id=user_id, group_ids=[group['name'] for group in request_data['groups']], apikey=apikey) From 71a637b9aac3df54997d103272c6c0ebf81930ca Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 8 Dec 2016 16:21:31 +0200 Subject: [PATCH 15/76] normal user added to "roger" group in order to pass auth checks --- ckan/tests/legacy/functional/api/test_activity.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ckan/tests/legacy/functional/api/test_activity.py b/ckan/tests/legacy/functional/api/test_activity.py index 6f7dc4967c2..3752a7a970d 100644 --- a/ckan/tests/legacy/functional/api/test_activity.py +++ b/ckan/tests/legacy/functional/api/test_activity.py @@ -19,6 +19,7 @@ from nose import SkipTest from ckan.common import json import ckan.tests.legacy as tests +from ckan.tests.helpers import call_action ##def package_update(context, data_dict): @@ -188,6 +189,14 @@ def setup_class(self): 'name': sysadmin_user.name, } normal_user = model.User.get('annafan') + + call_action( + 'member_create', + id='roger', + object=normal_user.id, + object_type='user', capacity='admin' + ) + self.normal_user = { 'id': normal_user.id, 'apikey': normal_user.apikey, @@ -289,8 +298,6 @@ def _create_package(self, user, name=None): # Create a new package. request_data = make_package(name) - # quick fix for #3351 - request_data['groups'] = [] before = self.record_details(user_id=user_id, group_ids=[group['name'] for group in request_data['groups']], From 139724c7a11e29a34ffd08327706a28be5062693 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 12 Dec 2016 14:12:40 +0200 Subject: [PATCH 16/76] inside `_create_package` user becomes admin of group --- .../legacy/functional/api/test_activity.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ckan/tests/legacy/functional/api/test_activity.py b/ckan/tests/legacy/functional/api/test_activity.py index 3752a7a970d..bfb0bd7ab92 100644 --- a/ckan/tests/legacy/functional/api/test_activity.py +++ b/ckan/tests/legacy/functional/api/test_activity.py @@ -190,13 +190,6 @@ def setup_class(self): } normal_user = model.User.get('annafan') - call_action( - 'member_create', - id='roger', - object=normal_user.id, - object_type='user', capacity='admin' - ) - self.normal_user = { 'id': normal_user.id, 'apikey': normal_user.apikey, @@ -283,7 +276,7 @@ def record_details(self, user_id, package_id=None, group_ids=None, details['recently changed datasets stream'] = \ self.recently_changed_datasets_stream(apikey) - details['time'] = datetime.datetime.now() + details['time'] = datetime.datetime.utcnow() return details def _create_package(self, user, name=None): @@ -303,6 +296,12 @@ def _create_package(self, user, name=None): group_ids=[group['name'] for group in request_data['groups']], apikey=apikey) extra_environ = {'Authorization': str(user['apikey'])} + + call_action('member_create', + capacity='admin', + object=user['id'], + object_type='user', + id='roger') response = self.app.post('/api/action/package_create', json.dumps(request_data), extra_environ=extra_environ) response_dict = json.loads(response.body) @@ -1412,7 +1411,7 @@ def test_create_user(self): a new user is created. """ - before = datetime.datetime.now() + before = datetime.datetime.utcnow() # Create a new user. user_dict = {'name': 'testuser', From 741a26494f464748f4da801776490db0fa572fd0 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 12 Dec 2016 14:33:32 +0200 Subject: [PATCH 17/76] Updated value for `ckan.auth.user_create_groups Now documentation contains correct default value - False. Also this option mentioned in writhing extension guide --- doc/extensions/tutorial.rst | 5 +++-- doc/maintaining/configuration.rst | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/extensions/tutorial.rst b/doc/extensions/tutorial.rst index f0e4bf4ea13..6af7f777524 100644 --- a/doc/extensions/tutorial.rst +++ b/doc/extensions/tutorial.rst @@ -257,8 +257,9 @@ dictionary: Whenever a user tries to create a new group via the web interface or the API, CKAN calls the :func:`~ckan.logic.auth.create.group_create` authorization function to decide whether to allow the action. Let's override this function -and simply prevent anyone from creating new groups. Edit your ``plugin.py`` -file so that it looks like this: +and simply prevent anyone from creating new groups(Note: this is default behavior. +In order to go further, you need to change ``ckan.auth.user_create_groups`` to `True` +in configuration file). Edit your ``plugin.py`` file so that it looks like this: .. literalinclude:: ../../ckanext/example_iauthfunctions/plugin_v2.py diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index ede281f19ad..96f07efc3e8 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -454,9 +454,9 @@ ckan.auth.user_create_groups Example:: - ckan.auth.user_create_groups = False + ckan.auth.user_create_groups = True -Default value: ``True`` +Default value: ``False`` Allow users to create groups. From e6e755e0a0c1be24fa5f02e3850751558679ce15 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 12 Dec 2016 15:02:23 +0200 Subject: [PATCH 18/76] `render_markdown` breaks links with ampersands Added few additional allowed tags to `bleach.clean` function and changed sanitization sequence so that markdown applied first and only after that result cleaned --- ckan/lib/helpers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index bba18104ccc..47b75f07a99 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -27,7 +27,7 @@ from webhelpers.text import truncate import webhelpers.date as date from markdown import markdown -from bleach import clean as clean_html +from bleach import clean as clean_html, ALLOWED_TAGS from pylons import url as _pylons_default_url from pylons.decorators.cache import beaker_cache from pylons import config @@ -45,11 +45,18 @@ import ckan.logic as logic import ckan.lib.uploader as uploader import ckan.authz as authz - from ckan.common import ( _, ungettext, g, c, request, session, json, OrderedDict ) + +MARKDOWN_TAGS = set([ + 'del', 'dd', 'dl', 'dt', 'h1', 'h2', + 'h3', 'img', 'kbd', 'p', 'pre', 's', + 'sup', 'sub', 'strike', 'br', 'hr' +]).union(ALLOWED_TAGS) + + get_available_locales = i18n.get_available_locales get_locales_dict = i18n.get_locales_dict @@ -1727,7 +1734,7 @@ def render_markdown(data, auto_link=True, allow_html=False): data = markdown(data.strip()) else: data = RE_MD_HTML_TAGS.sub('', data.strip()) - data = markdown(clean_html(data, strip=True)) + data = clean_html(markdown(data), strip=True, tags=MARKDOWN_TAGS) # tags can be added by tag:... or tag:"...." and a link will be made # from it if auto_link: From ce65ab68d15f1ffaa446937b655559d23f5c02fc Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 16 Dec 2016 11:20:18 +0200 Subject: [PATCH 19/76] added some tests and src attr to img --- ckan/lib/helpers.py | 9 ++++++-- ckan/tests/lib/test_helpers.py | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 47b75f07a99..5541981f7d9 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -27,7 +27,7 @@ from webhelpers.text import truncate import webhelpers.date as date from markdown import markdown -from bleach import clean as clean_html, ALLOWED_TAGS +from bleach import clean as clean_html, ALLOWED_TAGS, ALLOWED_ATTRIBUTES from pylons import url as _pylons_default_url from pylons.decorators.cache import beaker_cache from pylons import config @@ -56,6 +56,9 @@ 'sup', 'sub', 'strike', 'br', 'hr' ]).union(ALLOWED_TAGS) +MARKDOWN_ATTRIBUTES = copy.deepcopy(ALLOWED_ATTRIBUTES) +MARKDOWN_ATTRIBUTES.setdefault('img', []).extend(['src', 'alt', 'title']) + get_available_locales = i18n.get_available_locales get_locales_dict = i18n.get_locales_dict @@ -1734,7 +1737,9 @@ def render_markdown(data, auto_link=True, allow_html=False): data = markdown(data.strip()) else: data = RE_MD_HTML_TAGS.sub('', data.strip()) - data = clean_html(markdown(data), strip=True, tags=MARKDOWN_TAGS) + data = clean_html( + markdown(data), strip=True, + tags=MARKDOWN_TAGS, attributes=MARKDOWN_ATTRIBUTES) # tags can be added by tag:... or tag:"...." and a link will be made # from it if auto_link: diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py index 669624564c6..964b8bfb2bf 100644 --- a/ckan/tests/lib/test_helpers.py +++ b/ckan/tests/lib/test_helpers.py @@ -155,6 +155,45 @@ def test_render_naughty_markdown(self): output = u'' eq_(h.render_markdown(data), output) + def test_render_markdown_with_js(self): + data = u'[text](javascript: alert(1))' + output = u'

text

' + eq_(h.render_markdown(data), output) + + def test_event_attributes(self): + data = u'

and text

' + output = u'

and text

' + eq_(h.render_markdown(data), output) + + def test_ampersand_in_links(self): + data = u'[link](/url?a=1&b=2)' + output = u'

link

' + eq_(h.render_markdown(data), output) + + data = u'http://example.com/page?a=1&b=2' + output = u'

http://example.com/page?a=1&b=2

' + eq_(h.render_markdown(data), output) + + def test_tags_h1(self): + data = u'#heading' + output = u'

heading

' + eq_(h.render_markdown(data), output) + + def test_tags_h2(self): + data = u'##heading' + output = u'

heading

' + eq_(h.render_markdown(data), output) + + def test_tags_h3(self): + data = u'###heading' + output = u'

heading

' + eq_(h.render_markdown(data), output) + + def test_tags_img(self): + data = u'![image](/image.png)' + output = u'

image

' + eq_(h.render_markdown(data), output) + class TestHelpersRemoveLineBreaks(object): From 1b327e87498a08b45fa29a4dcaa03137bd3b23d4 Mon Sep 17 00:00:00 2001 From: Marc Fortier Date: Wed, 27 Jan 2016 14:53:50 -0500 Subject: [PATCH 20/76] fix ckan.root_path The language selector is passing a babel Locale object but _add_i18n_to_url expects a str. --- ckan/lib/helpers.py | 3 +-- ckan/tests/lib/test_helpers.py | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 5541981f7d9..cd48dc2465a 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -13,7 +13,6 @@ import pytz import tzlocal import urllib -import urlparse import pprint import copy import urlparse @@ -301,7 +300,7 @@ def _add_i18n_to_url(url_to_amend, **kw): if default_locale: root_path = re.sub('/{{LANG}}', '', root_path) else: - root_path = re.sub('{{LANG}}', locale, root_path) + root_path = re.sub('{{LANG}}', str(locale), root_path) # make sure we don't have a trailing / on the root if root_path[-1] == '/': root_path = root_path[:-1] diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py index 964b8bfb2bf..19b8a4362a8 100644 --- a/ckan/tests/lib/test_helpers.py +++ b/ckan/tests/lib/test_helpers.py @@ -1,5 +1,6 @@ import nose import i18n +from babel import Locale import ckan.lib.helpers as h import ckan.exceptions @@ -74,6 +75,14 @@ def test_url_for_with_locale(self): locale='de') eq_(generated_url, url) + @helpers.change_config('ckan.site_url', 'http://example.com') + @helpers.change_config('ckan.root_path', '/foo/{{LANG}}') + def test_url_for_with_locale_object(self): + url = '/foo/de/dataset/my_dataset' + generated_url = h.url_for('/dataset/my_dataset', + locale=Locale('de')) + eq_(generated_url, url) + @helpers.change_config('ckan.site_url', 'http://example.com') def test_url_for_not_qualified(self): url = '/dataset/my_dataset' From 6d47ecfac1be47d0bdf17baafce1bbe385df9e67 Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 20 Feb 2017 15:36:25 +0200 Subject: [PATCH 21/76] [#3447] Resource creation date use datetime.utcnow() --- ckan/model/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/model/resource.py b/ckan/model/resource.py index ceca568898a..451b1c07af2 100644 --- a/ckan/model/resource.py +++ b/ckan/model/resource.py @@ -47,7 +47,7 @@ Column('mimetype', types.UnicodeText), Column('mimetype_inner', types.UnicodeText), Column('size', types.BigInteger), - Column('created', types.DateTime, default=datetime.datetime.now), + Column('created', types.DateTime, default=datetime.datetime.utcnow), Column('last_modified', types.DateTime), Column('cache_url', types.UnicodeText), Column('cache_last_updated', types.DateTime), From e766b676f4cb1efed3b5aa747aa4073a7da17c15 Mon Sep 17 00:00:00 2001 From: Jukka Heino Date: Fri, 17 Feb 2017 16:38:20 +0200 Subject: [PATCH 22/76] Use the url_for() helper for datapusher URLs --- ckanext/datapusher/logic/action.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 5afb4b0b33a..ead02701dff 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -8,6 +8,7 @@ import pylons import requests +import ckan.lib.helpers as h import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p @@ -57,8 +58,8 @@ def datapusher_submit(context, data_dict): datapusher_url = pylons.config.get('ckan.datapusher.url') - site_url = pylons.config['ckan.site_url'] - callback_url = site_url.rstrip('/') + '/api/3/action/datapusher_hook' + site_url = h.url_for('/', qualified=True) + callback_url = h.url_for('/api/3/action/datapusher_hook', qualified=True) user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) From 7ed2d0ddd9b3dfe6597db28b80d6422bc0418069 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 22 Feb 2017 10:31:54 +0000 Subject: [PATCH 23/76] Update CHANGELOG ahead of 2.5.4 --- CHANGELOG.rst | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4c1bdea58b4..578ee2ff662 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,24 @@ Changelog --------- +v2.5.4 2017-02-22 +================= + + + * Fix DataPusher being fired multiple times (#3245) + * Use the url_for() helper for datapusher URLs (#2866) + * Resource creation date use datetime.utcnow() (#3447) + * Fix locale error when using fix ckan.root_path + * `render_markdown` breaks links with ampersands + * Check group name and id during package creation + * Use utcnow() on dashboard_mark_activities_old (#3373) + * Fix encoding error on DataStore exception + * Datastore doesn't add site_url to resource created via API (#3189) + * Fix memberships after user deletion (#3265) + * Remove idle database connection (#3260) + * Fix package_owner_org_update action when called via the API (#2661) + + v2.5.3 2016-11-02 ================= @@ -108,6 +126,21 @@ v2.5.0 2015-12-17 Cancelled release +v2.4.5 2017-02-22 +================= + + * Use the url_for() helper for datapusher URLs (#2866) + * Resource creation date use datetime.utcnow() (#3447) + * Fix locale error when using fix ckan.root_path + * `render_markdown` breaks links with ampersands + * Check group name and id during package creation + * Use utcnow() on dashboard_mark_activities_old (#3373) + * Fix encoding error on DataStore exception + * Datastore doesn't add site_url to resource created via API (#3189) + * Fix memberships after user deletion (#3265) + * Remove idle database connection (#3260) + * Fix package_owner_org_update action when called via the API (#2661) + v2.4.4 2016-11-02 ================= From 8abbb3b4be6aef95b253d69f625f833f3a9713b7 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 22 Feb 2017 10:32:17 +0000 Subject: [PATCH 24/76] Update version number for 2.5.4 --- ckan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/__init__.py b/ckan/__init__.py index f017397a7c0..1e402c2e840 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.5.4b' +__version__ = '2.5.4' __description__ = 'CKAN Software' __long_description__ = \ From 03664c851d0c6e13ab066553996954c969495c70 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 22 Feb 2017 11:48:39 +0000 Subject: [PATCH 25/76] Fix exception caused by bad merged on d70db78 --- ckanext/datastore/logic/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 5f670b7183e..0ec9b9f349f 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -179,7 +179,7 @@ def datastore_create(context, data_dict): 'q': 'id:"{0}"'.format(package_id), 'fl': 'data_dict', 'wt': 'json', - 'fq': 'site_id:"%s"' % config.get('ckan.site_id'), + 'fq': 'site_id:"%s"' % pylons.config.get('ckan.site_id'), 'rows': 1 } for record in solr_query.run(q)['results']: From 39ba8a43971ec1913728cc4e04c43f3006f5ea5e Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 22 Feb 2017 12:56:28 +0000 Subject: [PATCH 26/76] Change version number for 2.5.5 --- ckan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/__init__.py b/ckan/__init__.py index 1e402c2e840..67428c8c5d2 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.5.4' +__version__ = '2.5.5b' __description__ = 'CKAN Software' __long_description__ = \ From bdd591c9fef84e6cbfce37ebf16f9a960f16fcfc Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 15 Mar 2017 18:03:20 +0200 Subject: [PATCH 27/76] Use h.url_for and qualified=True for reset mails --- ckan/lib/mailer.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index b98ceb5ba38..0cbf8eb2bed 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -133,11 +133,12 @@ def get_invite_body(user): return invite_message.format(**d) def get_reset_link(user): - return urljoin(g.site_url, - h.url_for(controller='user', - action='perform_reset', - id=user.id, - key=user.reset_key)) + return h.url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key, + qualified=True) + def send_reset_link(user): create_reset_key(user) @@ -164,6 +165,3 @@ def verify_reset_link(user, key): if not user.reset_key or len(user.reset_key) < 5: return False return key.strip() == user.reset_key - - - From e2f809ed7c27546c385d16c12595d4dbcd2f2f68 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 22 Mar 2017 15:26:40 +0200 Subject: [PATCH 28/76] Setting of datastore_active flag moved to separate function --- ckanext/datastore/logic/action.py | 95 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 0ec9b9f349f..cef492dff31 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -148,47 +148,7 @@ def datastore_create(context, data_dict): log.debug( 'Setting datastore_active=True on resource {0}'.format(resource.id) ) - # issue #3245: race condition - update_dict = {'datastore_active': True} - - # get extras(for entity update) and package_id(for search index update) - res_query = model.Session.query( - model.resource_table.c.extras, - model.resource_table.c.package_id - ).filter( - model.Resource.id == data_dict['resource_id'] - ) - extras, package_id = res_query.one() - - # update extras in database for record and its revision - extras.update(update_dict) - res_query.update({'extras': extras}, synchronize_session=False) - - model.Session.query(model.resource_revision_table).filter( - model.ResourceRevision.id == data_dict['resource_id'], - model.ResourceRevision.current is True - ).update({'extras': extras}, synchronize_session=False) - - model.Session.commit() - - # get package with updated resource from solr - # find changed resource, patch it and reindex package - psi = search.PackageSearchIndex() - solr_query = search.PackageSearchQuery() - q = { - 'q': 'id:"{0}"'.format(package_id), - 'fl': 'data_dict', - 'wt': 'json', - 'fq': 'site_id:"%s"' % pylons.config.get('ckan.site_id'), - 'rows': 1 - } - for record in solr_query.run(q)['results']: - solr_data_dict = json.loads(record['data_dict']) - for resource in solr_data_dict['resources']: - if resource['id'] == data_dict['resource_id']: - resource.update(update_dict) - psi.index_package(solr_data_dict) - break + set_datastore_active_flag(model, data_dict, True) result.pop('id', None) result.pop('private', None) @@ -395,11 +355,9 @@ def datastore_delete(context, data_dict): if (not data_dict.get('filters') and resource.extras.get('datastore_active') is True): log.debug( - 'Setting datastore_active=True on resource {0}'.format(resource.id) + 'Setting datastore_active=False on resource {0}'.format(resource.id) ) - p.toolkit.get_action('resource_patch')( - context, {'id': data_dict['resource_id'], - 'datastore_active': False}) + set_datastore_active_flag(model, data_dict, False) result.pop('id', None) result.pop('connection_url') @@ -597,6 +555,53 @@ def datastore_make_public(context, data_dict): db.make_public(context, data_dict) +def set_datastore_active_flag(model, data_dict, flag): + ''' + Set appropriate datastore_active flag on CKAN resource. + + Called after creation or deletion of DataStore table. + ''' + update_dict = {'datastore_active': flag} + + # get extras(for entity update) and package_id(for search index update) + res_query = model.Session.query( + model.resource_table.c.extras, + model.resource_table.c.package_id + ).filter( + model.Resource.id == data_dict['resource_id'] + ) + extras, package_id = res_query.one() + + # update extras in database for record and its revision + extras.update(update_dict) + res_query.update({'extras': extras}, synchronize_session=False) + model.Session.query(model.resource_revision_table).filter( + model.ResourceRevision.id == data_dict['resource_id'], + model.ResourceRevision.current is True + ).update({'extras': extras}, synchronize_session=False) + + model.Session.commit() + + # get package with updated resource from solr + # find changed resource, patch it and reindex package + psi = search.PackageSearchIndex() + solr_query = search.PackageSearchQuery() + q = { + 'q': 'id:"{0}"'.format(package_id), + 'fl': 'data_dict', + 'wt': 'json', + 'fq': 'site_id:"%s"' % pylons.config.get('ckan.site_id'), + 'rows': 1 + } + for record in solr_query.run(q)['results']: + solr_data_dict = json.loads(record['data_dict']) + for resource in solr_data_dict['resources']: + if resource['id'] == data_dict['resource_id']: + resource.update(update_dict) + psi.index_package(solr_data_dict) + break + + def _resource_exists(context, data_dict): ''' Returns true if the resource exists in CKAN and in the datastore ''' model = _get_or_bust(context, 'model') From c1dd3acafe917551c98b146b1602d4dd6af743e1 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Mon, 13 Mar 2017 14:30:32 -0400 Subject: [PATCH 29/76] fix edit resource of draft dataset --- ckan/controllers/package.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index c8f9b34325b..f0ba12ee6bb 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -586,12 +586,7 @@ def resource_edit(self, id, resource_id, data=None, errors=None, # dataset has not yet been fully created resource_dict = get_action('resource_show')(context, {'id': resource_id}) - fields = ['url', 'resource_type', 'format', 'name', 'description', - 'id'] - data = {} - for field in fields: - data[field] = resource_dict[field] - return self.new_resource(id, data=data) + return self.new_resource(id, data=resource_dict) # resource is fully created try: resource_dict = get_action('resource_show')(context, From b1e46f19108f3680cffdfc6b5f4138ff51da3ee5 Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Fri, 24 Feb 2017 10:43:26 +0200 Subject: [PATCH 30/76] Fix tags on org/group read pages --- ckan/controllers/group.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 22da9d11f52..8483ea07c0b 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -274,8 +274,9 @@ def drill_down_url(**by): c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): + controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, - controller='group', action='read', + controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field @@ -287,6 +288,7 @@ def pager_url(q=None, page=None): try: c.fields = [] + c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ @@ -294,6 +296,10 @@ def pager_url(q=None, page=None): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) + if param not in c.fields_grouped: + c.fields_grouped[param] = [value] + else: + c.fields_grouped[param].append(value) else: search_extras[param] = value From 8131b466945cdc58e60f21822062a2c49348be5c Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 22 Mar 2017 15:56:01 +0200 Subject: [PATCH 31/76] auth check in revision controller --- ckan/controllers/revision.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ckan/controllers/revision.py b/ckan/controllers/revision.py index 171d7dbaad4..9884cee7287 100644 --- a/ckan/controllers/revision.py +++ b/ckan/controllers/revision.py @@ -157,6 +157,15 @@ def diff(self, id=None): c.diff_entity = request.params.get('diff_entity') if c.diff_entity == 'package': + try: + logic.check_access('package_show', { + 'model': model, + 'user': c.user or c.author, + 'auth_user_obj': c.userobj + }, {'id': id}) + except logic.NotAuthorized: + base.abort(401) + c.pkg = model.Package.by_name(id) diff = c.pkg.diff(c.revision_to, c.revision_from) elif c.diff_entity == 'group': From d7ddce721cc6965bb587e844f2e579855acfd3fc Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 22 Mar 2017 17:14:28 +0200 Subject: [PATCH 32/76] Updated changelog --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 578ee2ff662..2b2b625771b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Changelog --------- +v2.5.5 2017-03-22 +================= + +* Use fully qualified urls for reset emails (#3486) +* Fix edit_resource for resource with draft state (#3480) +* Tag fix for group/organization pages (#3460) +* Setting of datastore_active flag moved to separate function (#3481) + v2.5.4 2017-02-22 ================= From 5e0ddde76dd58431330df5246ea24942878e1de4 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 28 Mar 2017 16:53:52 +0300 Subject: [PATCH 33/76] Updated version of httpretty in dev-requirements --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 983ca49868d..69a7cbfdce4 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # These are packages that required when running ckan tests and building the docs docutils==0.8.1 -httpretty==0.8.3 +httpretty==0.8.7 # nose==1.3.0 # already in requirements.txt pep8==1.4.6 Sphinx==1.2.3 From 21cf8e6d8ccd6276f377e9f983be11070cf8526e Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Mar 2017 16:33:40 +0200 Subject: [PATCH 34/76] Use json.dumps for nested fields in datastore_dump --- ckanext/datastore/controller.py | 24 +++++++++++++++++- ckanext/datastore/tests/test_dump.py | 37 +++++++++++++++++++--------- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 3bb5def4e73..d050fa96ec8 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -1,3 +1,4 @@ +import json import StringIO import unicodecsv as csv @@ -10,6 +11,20 @@ from ckan.common import request +def _dump_nested(column, record): + name, ctype = column + value = record[name] + + is_nested = ( + ctype == 'json' or + ctype.startswith('_') or + ctype.endswith(']') + ) + if is_nested: + return json.dumps(value) + return value + + class DatastoreController(base.BaseController): def dump(self, resource_id): context = { @@ -39,6 +54,13 @@ def dump(self, resource_id): header = [x['id'] for x in result['fields']] wr.writerow(header) + columns = [ + (x['id'], x['type']) + for x in result['fields']] + for record in result['records']: - wr.writerow([record[column] for column in header]) + wr.writerow([ + _dump_nested(column, record) + for column in columns]) + return f.getvalue() diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 53190240bb6..272152e6cb8 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -2,6 +2,7 @@ import nose from nose.tools import assert_equals +from ckan.tests.helpers import assert_in from pylons import config import sqlalchemy.orm as orm import paste.fixture @@ -37,14 +38,16 @@ def setup_class(cls): 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, {'id': 'published'}, - {'id': u'characters', u'type': u'_text'}], + {'id': u'characters', u'type': u'_text'}, + {'id': 'random_letters', 'type': 'text[]'}], 'records': [{u'b\xfck': 'annakarenina', - 'author': 'tolstoy', - 'published': '2005-03-01', - 'nested': ['b', {'moo': 'moo'}], - u'characters': [u'Princess Anna', u'Sergius']}, + 'author': 'tolstoy', + 'published': '2005-03-01', + 'nested': ['b', {'moo': 'moo'}], + u'characters': [u'Princess Anna', u'Sergius'], + 'random_letters': ['a', 'e', 'x']}, {u'b\xfck': 'warandpeace', 'author': 'tolstoy', - 'nested': {'a': 'b'}}] + 'nested': {'a': 'b'}, 'random_letters': []}] } postparams = '%s=1' % json.dumps(cls.data) auth = {'Authorization': str(cls.sysadmin_user.apikey)} @@ -67,10 +70,13 @@ def test_dump_basic(self): res = self.app.get('/datastore/dump/{0}'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = u'_id,b\xfck,author,published,characters,nested' + + expected = ( + u'_id,b\xfck,author,published' + u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) - assert 'warandpeace' in content - assert "[u'Princess Anna', u'Sergius']" in content + assert_in('warandpeace', content) + assert_in('"[""Princess Anna"", ""Sergius""]"', content) # get with alias instead of id res = self.app.get('/datastore/dump/{0}'.format(str( @@ -86,6 +92,15 @@ def test_dump_limit(self): res = self.app.get('/datastore/dump/{0}?limit=1'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = u'_id,b\xfck,author,published,characters,nested' + + expected = (u'_id,b\xfck,author,published' + u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) - assert_equals(len(content), 148) + + expected_content = ( + u'_id,b\xfck,author,published,characters,random_letters,' + u'nested\r\n1,annakarenina,tolstoy,2005-03-01T00:00:00,' + u'"[""Princess Anna"", ""Sergius""]",' + u'"[""a"", ""e"", ""x""]","[""b"", ' + u'{""moo"": ""moo""}]"\r\n') + assert_equals(content, expected_content) From bf1a5e78e7a9bb2598790f66499107462a59e283 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 3 Apr 2017 15:24:18 +0300 Subject: [PATCH 35/76] updated _dump_nested --- ckanext/datastore/controller.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index d050fa96ec8..dc4190a07fb 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -11,15 +11,9 @@ from ckan.common import request -def _dump_nested(column, record): - name, ctype = column - value = record[name] +def _json_dump_nested(value): + is_nested = isinstance(value, (list, dict)) - is_nested = ( - ctype == 'json' or - ctype.startswith('_') or - ctype.endswith(']') - ) if is_nested: return json.dumps(value) return value @@ -54,13 +48,9 @@ def dump(self, resource_id): header = [x['id'] for x in result['fields']] wr.writerow(header) - columns = [ - (x['id'], x['type']) - for x in result['fields']] - for record in result['records']: wr.writerow([ - _dump_nested(column, record) - for column in columns]) + _json_dump_nested(record[column]) + for column in header]) return f.getvalue() From fde012d01f956439c296ba5df318e634b77b1860 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 4 Apr 2017 10:19:39 +0100 Subject: [PATCH 36/76] Update version for 2.5.5 release --- ckan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/__init__.py b/ckan/__init__.py index 67428c8c5d2..75bb565ac23 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.5.5b' +__version__ = '2.5.5' __description__ = 'CKAN Software' __long_description__ = \ From ef1f09a58dc03d13309baf2219692f72a4d8fc16 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 21 Jun 2017 11:21:50 +0100 Subject: [PATCH 37/76] Update version number for 2.5.6b --- ckan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/__init__.py b/ckan/__init__.py index 75bb565ac23..67428c8c5d2 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.5.5' +__version__ = '2.5.5b' __description__ = 'CKAN Software' __long_description__ = \ From 505f6b6d0397620671da5d7f20b3d5d91cadccc4 Mon Sep 17 00:00:00 2001 From: Yan Date: Sun, 26 Feb 2017 21:37:39 +0200 Subject: [PATCH 38/76] [#3457]Create new resource view if resource format changed --- ckan/logic/action/update.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 1ca9b9caa46..919ca23a6f4 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -124,6 +124,7 @@ def resource_update(context, data_dict): model = context['model'] user = context['user'] id = _get_or_bust(data_dict, "id") + old_resource = _get_action('resource_show')(context, {'id': id}) resource = model.Resource.get(id) context["resource"] = resource @@ -172,6 +173,19 @@ def resource_update(context, data_dict): resource = _get_action('resource_show')(context, {'id': id}) + if old_resource['format'] != resource['format']: + q = model.Session.query(model.ResourceView) \ + .filter(model.ResourceView.resource_id == resource['id']) + resources_view_id = q.all() + if resources_view_id: + for view_id in resources_view_id: + _get_action( + 'resource_view_delete')(context, {'id': view_id.id}) + _get_action('package_create_default_resource_views')( + {'model': context['model'], 'user': context['user'], + 'ignore_auth': True}, + {'package': updated_pkg_dict}) + for plugin in plugins.PluginImplementations(plugins.IResourceController): plugin.after_update(context, resource) From 1f0b514a8015c0ae9c0ff52ce006dc4390c4ee98 Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 22 May 2017 15:36:23 +0300 Subject: [PATCH 39/76] Added test and modification to logic --- ckan/logic/action/update.py | 7 -- ckan/tests/logic/action/test_update.py | 96 ++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 7 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 919ca23a6f4..6b9a4a55111 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -174,13 +174,6 @@ def resource_update(context, data_dict): resource = _get_action('resource_show')(context, {'id': id}) if old_resource['format'] != resource['format']: - q = model.Session.query(model.ResourceView) \ - .filter(model.ResourceView.resource_id == resource['id']) - resources_view_id = q.all() - if resources_view_id: - for view_id in resources_view_id: - _get_action( - 'resource_view_delete')(context, {'id': view_id.id}) _get_action('package_create_default_resource_views')( {'model': context['model'], 'user': context['user'], 'ignore_auth': True}, diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index c85a4e8910c..801ccbe2c92 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -794,6 +794,102 @@ def test_datastore_active_not_present_if_not_provided_and_not_datastore_plugin_e assert 'datastore_active' not in res_returned + def test_resource_format_update(self): + dataset = factories.Dataset() + + # Create resource without format + resource = factories.Resource(package=dataset, + url='http://localhost', + name='Test') + res_views = helpers.call_action( + 'resource_view_list', + id=resource['id']) + + assert_equals(len(res_views), 0) + + # Update resource with format + resource = helpers.call_action( + 'resource_update', + id=resource['id'], + format='CSV') + + # Format changed + assert_equals(resource['format'], 'CSV') + + res_views = helpers.call_action( + 'resource_view_list', + id=resource['id']) + + # View for resource is created + assert_equals(len(res_views), 1) + + second_resource = factories.Resource( + package=dataset, + url='http://localhost', + name='Test2', + format='CSV') + + res_views = helpers.call_action( + 'resource_view_list', + id=second_resource['id']) + + assert_equals(len(res_views), 1) + + second_resource = helpers.call_action( + 'resource_update', + id=second_resource['id'], + format='PNG') + + # Format changed + assert_equals(second_resource['format'], 'PNG') + + res_views = helpers.call_action( + 'resource_view_list', + id=second_resource['id']) + + assert_equals(len(res_views), 2) + + third_resource = factories.Resource( + package=dataset, + url='http://localhost', + name='Test2') + + res_views = helpers.call_action( + 'resource_view_list', + id=third_resource['id']) + + assert_equals(len(res_views), 0) + + third_resource = helpers.call_action( + 'resource_update', + id=third_resource['id'], + format='Test format') + + # Format added + assert_equals(third_resource['format'], 'Test format') + + res_views = helpers.call_action( + 'resource_view_list', + id=third_resource['id']) + + # No view created, cause no such format + assert_equals(len(res_views), 0) + + third_resource = helpers.call_action( + 'resource_update', + id=third_resource['id'], + format='CSV') + + # Format changed + assert_equals(third_resource['format'], 'CSV') + + res_views = helpers.call_action( + 'resource_view_list', + id=third_resource['id']) + + # View is created + assert_equals(len(res_views), 1) + class TestConfigOptionUpdate(object): From 2fb647570a8a40b6e7fd25e387bcff67f07a7951 Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 22 May 2017 17:49:31 +0300 Subject: [PATCH 40/76] Added logic fix and test fix # Conflicts: # ckan/tests/logic/action/test_update.py --- ckan/logic/action/update.py | 9 +++++---- ckan/tests/logic/action/test_update.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 6b9a4a55111..dfddf236224 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -124,10 +124,10 @@ def resource_update(context, data_dict): model = context['model'] user = context['user'] id = _get_or_bust(data_dict, "id") - old_resource = _get_action('resource_show')(context, {'id': id}) resource = model.Resource.get(id) context["resource"] = resource + old_resource_format = resource.format if not resource: log.error('Could not find resource ' + id) @@ -173,11 +173,12 @@ def resource_update(context, data_dict): resource = _get_action('resource_show')(context, {'id': id}) - if old_resource['format'] != resource['format']: - _get_action('package_create_default_resource_views')( + if old_resource_format != resource['format']: + _get_action('resource_create_default_resource_views')( {'model': context['model'], 'user': context['user'], 'ignore_auth': True}, - {'package': updated_pkg_dict}) + {'package': updated_pkg_dict, + 'resource': resource}) for plugin in plugins.PluginImplementations(plugins.IResourceController): plugin.after_update(context, resource) diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 801ccbe2c92..18bd5671fa4 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -631,8 +631,17 @@ def setup(self): import ckan.model as model model.repo.rebuild_db() + @classmethod + def setup_class(cls): + if not p.plugin_loaded('image_view'): + p.load('image_view') + if not p.plugin_loaded('recline_view'): + p.load('recline_view') + @classmethod def teardown_class(cls): + p.unload('image_view') + p.unload('recline_view') helpers.reset_db() def test_url_only(self): @@ -794,6 +803,8 @@ def test_datastore_active_not_present_if_not_provided_and_not_datastore_plugin_e assert 'datastore_active' not in res_returned + + @helpers.change_config('ckan.views.default_views', 'image_view recline_view') def test_resource_format_update(self): dataset = factories.Dataset() From 3f162b8c5a9b23c07102e2cf66901ea4e0976567 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 2 May 2017 13:09:29 +0300 Subject: [PATCH 41/76] Restrict access to `members` and `bulk_actions` After this fix only users with permission `bulk_update_public` will be able to visit `bulc_process` page and only those who have `group_edit_permissions' will be able to visit `members` page # Conflicts: # ckan/controllers/group.py --- ckan/controllers/group.py | 14 ++++++++++---- ckan/logic/auth/update.py | 30 ++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 8483ea07c0b..054c40e8394 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -405,6 +405,7 @@ def bulk_process(self, id): data_dict = {'id': id} try: + self._check_access('bulk_update_public', context, data_dict) # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict['include_datasets'] = False @@ -413,7 +414,7 @@ def bulk_process(self, id): except NotFound: abort(404, _('Group not found')) except NotAuthorized: - abort(401, _('Unauthorized to read group %s') % id) + abort(403, _('User %r not authorized to edit %s') % (c.user, id)) #use different form names so that ie7 can be detected form_names = set(["bulk_action.public", "bulk_action.delete", @@ -673,16 +674,21 @@ def members(self, id): 'user': c.user or c.author} try: + data_dict = {'id': id} + self._check_access('group_edit_permissions', context, data_dict) c.members = self._action('member_list')( context, {'id': id, 'object_type': 'user'} ) - data_dict = {'id': id} data_dict['include_datasets'] = False c.group_dict = self._action('group_show')(context, data_dict) - except NotAuthorized: - abort(401, _('Unauthorized to delete group %s') % '') except NotFound: abort(404, _('Group not found')) + except NotAuthorized: + abort( + 403, + _('User %r not authorized to edit members of %s') % ( + c.user, id)) + return self._render_template('group/members.html', group_type) def member_new(self, id): diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index a5f4a0e24f1..1d484616b28 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -173,14 +173,32 @@ def group_edit_permissions(context, data_dict): user = context['user'] group = logic_auth.get_group_object(context, data_dict) - authorized = authz.has_user_permission_for_group_or_org(group.id, - user, - 'update') + authorized = authz.has_user_permission_for_group_or_org( + group.id, user, 'update') if not authorized: - return {'success': False, - 'msg': _('User %s not authorized to edit permissions of group %s') % - (str(user), group.id)} + return { + 'success': False, + 'msg': _('User %s not authorized to' + ' edit permissions of group %s') % + (str(user), group.id)} + else: + return {'success': True} + + +def organization_edit_permissions(context, data_dict): + user = context['user'] + group = logic_auth.get_group_object(context, data_dict) + + authorized = authz.has_user_permission_for_group_or_org( + group.id, user, 'update') + + if not authorized: + return { + 'success': False, + 'msg': _('User %s not authorized to edit' + ' permissions of organization %s') % + (str(user), group.id)} else: return {'success': True} From 7261336a35ff89ea8ec25c96f035ea0a57f77a78 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 2 May 2017 14:27:25 +0300 Subject: [PATCH 42/76] test fixes --- ckan/controllers/group.py | 4 ++-- ckan/logic/auth/update.py | 17 ----------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 054c40e8394..f760926ac75 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -405,7 +405,7 @@ def bulk_process(self, id): data_dict = {'id': id} try: - self._check_access('bulk_update_public', context, data_dict) + self._check_access('bulk_update_public', context, {'org_id': id}) # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict['include_datasets'] = False @@ -675,7 +675,7 @@ def members(self, id): try: data_dict = {'id': id} - self._check_access('group_edit_permissions', context, data_dict) + check_access('group_edit_permissions', context, data_dict) c.members = self._action('member_list')( context, {'id': id, 'object_type': 'user'} ) diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index 1d484616b28..8ee8dab0857 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -186,23 +186,6 @@ def group_edit_permissions(context, data_dict): return {'success': True} -def organization_edit_permissions(context, data_dict): - user = context['user'] - group = logic_auth.get_group_object(context, data_dict) - - authorized = authz.has_user_permission_for_group_or_org( - group.id, user, 'update') - - if not authorized: - return { - 'success': False, - 'msg': _('User %s not authorized to edit' - ' permissions of organization %s') % - (str(user), group.id)} - else: - return {'success': True} - - @logic.auth_allow_anonymous_access def user_update(context, data_dict): user = context['user'] From 210054ad416a4e0d8ddce80c06731467f62adb55 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 2 May 2017 15:56:06 +0300 Subject: [PATCH 43/76] fixes in controller tests --- ckan/tests/controllers/test_group.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index a240cb00b3b..7fcdb5796f3 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -264,9 +264,12 @@ def test_membership_list(self): group = self._create_group(user_one['name'], other_users) + env = {'REMOTE_USER': user_one['name'].encode('ascii')} + member_list_url = url_for(controller='group', action='members', id=group['id']) - member_list_response = app.get(member_list_url) + member_list_response = app.get( + member_list_url, extra_environ=env) assert_true('2 members' in member_list_response) @@ -357,7 +360,7 @@ def test_remove_member(self): env = {'REMOTE_USER': user_one['name'].encode('ascii')} remove_response = app.post(remove_url, extra_environ=env, status=302) # redirected to member list after removal - remove_response = remove_response.follow() + remove_response = remove_response.follow(extra_environ=env) assert_true('Group member has been deleted.' in remove_response) assert_true('1 members' in remove_response) From 44a82d732e8be97ac2df03c11b2659ff1a4d6e30 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Wed, 7 Jun 2017 14:11:07 -0400 Subject: [PATCH 44/76] fix broken language toggle url when there is params in current url # Conflicts: # ckan/lib/helpers.py # ckan/templates/snippets/language_selector.html --- ckan/lib/helpers.py | 5 +++++ ckan/templates/snippets/language_selector.html | 3 +-- ckan/tests/controllers/test_package.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index cd48dc2465a..4d228c228a6 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -347,6 +347,11 @@ def full_current_url(): return (url_for(request.environ['CKAN_CURRENT_URL'], qualified=True)) +def current_url(): + ''' Returns current url unquoted''' + return urllib.unquote(request.environ['CKAN_CURRENT_URL']) + + def lang(): ''' Return the language code for the current locale eg `en` ''' return request.environ.get('CKAN_LANG') diff --git a/ckan/templates/snippets/language_selector.html b/ckan/templates/snippets/language_selector.html index dfe62c1686a..fa48b643ce2 100644 --- a/ckan/templates/snippets/language_selector.html +++ b/ckan/templates/snippets/language_selector.html @@ -1,10 +1,9 @@ -{% set current_url = request.environ.CKAN_CURRENT_URL %} {% set current_lang = request.environ.CKAN_LANG %}