From 1eb9d94077bd7d0dc586fefcb0d6b49042606b6c Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 19 Jun 2013 15:27:33 +0200 Subject: [PATCH 001/159] [#1011] Fix a 500 to a 404 Catch Genshi TemplateNotFound errors when rendering the package read page. This can happen if an IDatasetForm plugin returns a custom package read template, e.g. 'read.html', but the user has requested the dataset in RDF format and the plugin does not provide a corresponding 'read.rdf' template. (CKAN will take the path to read.html, replace the filename extension with rdf, try to render this non-existing template file, and crash.) Replace this crash with a proper 404. Add to the IDatasetForm docs, explaining how to provide RDF templates for datasets. Fixes #1011. --- ckan/controllers/package.py | 11 ++++++++++- ckan/plugins/interfaces.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 65541ee33bf..693ceb851a2 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -5,6 +5,7 @@ from pylons import config from genshi.template import MarkupTemplate from genshi.template.text import NewTextTemplate +import genshi.template.loader from paste.deploy.converters import asbool import ckan.logic as logic @@ -352,7 +353,15 @@ def read(self, id, format='html'): template = self._read_template(package_type) template = template[:template.index('.') + 1] + format - return render(template, loader_class=loader) + try: + return render(template, loader_class=loader) + except genshi.template.loader.TemplateNotFound: + msg = _("Viewing {package_type} datasets in {format} format is " + "not supported (template file {file} not found).".format( + package_type=package_type, format=format, file=template)) + abort(404, msg) + + assert False, "We should never get here" def history(self, id): package_type = self._get_package_type(id.split('@')[0]) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index a37d3c96ae8..826fec921cd 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -676,6 +676,15 @@ def read_template(self): The path should be relative to the plugin's templates dir, e.g. ``'package/read.html'``. + If the user requests the dataset in a format other than HTML + (CKAN supports returning datasets in RDF or N3 format by appending .rdf + or .n3 to the dataset read URL, see :doc:`linked-data-and-rdf`) then + CKAN will try to render + a template file with the same path as returned by this function, + but a different filename extension, e.g. ``'package/read.rdf'``. + If your extension doesn't have this RDF version of the template + file, the user will get a 404 error. + :rtype: string ''' From 25c80f16f57ecc34fda2d52bbd8d3a41e0b33d66 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sat, 6 Jul 2013 12:05:59 +0200 Subject: [PATCH 002/159] Add coverage reports with coveralls --- .coveragerc | 3 +++ .travis.yml | 4 ++++ bin/travis-build | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000000..28f34e39273 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = /ckan/migration/* +source = ckan, ckanext diff --git a/.travis.yml b/.travis.yml index af450298b6c..b4f63b6c669 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ python: env: - PGVERSION=9.1 - PGVERSION=8.4 +install: + - pip install coveralls --use-mirrors script: ./bin/travis-build notifications: irc: @@ -14,3 +16,5 @@ notifications: on_failure: change template: - "%{repository} %{branch} %{commit} %{build_url} %{author}: %{message}" +after_success: + - coveralls diff --git a/bin/travis-build b/bin/travis-build index df7c398d1b6..de7599cdddd 100755 --- a/bin/travis-build +++ b/bin/travis-build @@ -48,4 +48,4 @@ fi cat test-core.ini # And finally, run the tests -nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext +nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext --with-coverage --cover-package=ckanext --cover-package=ckan From 4d2d54a6e3622cdf4526ab59002e19114e7776c0 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 13:03:55 +0200 Subject: [PATCH 003/159] [#1011] Refactor an exception raise and catch Raise and catch a CKAN exception, instead of a Genshi one. --- ckan/controllers/package.py | 4 ++-- ckan/lib/base.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 693ceb851a2..d73ab4ee740 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -5,7 +5,6 @@ from pylons import config from genshi.template import MarkupTemplate from genshi.template.text import NewTextTemplate -import genshi.template.loader from paste.deploy.converters import asbool import ckan.logic as logic @@ -20,6 +19,7 @@ import ckan.lib.datapreview as datapreview import ckan.lib.plugins import ckan.plugins as p +import lib.render from ckan.common import OrderedDict, _, json, request, c, g, response from home import CACHE_PARAMETERS @@ -355,7 +355,7 @@ def read(self, id, format='html'): try: return render(template, loader_class=loader) - except genshi.template.loader.TemplateNotFound: + except lib.render.TemplateNotFound: msg = _("Viewing {package_type} datasets in {format} format is " "not supported (template file {file} not found).".format( package_type=package_type, format=format, file=template)) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 7227734ddf4..914b42741ea 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -103,11 +103,7 @@ def render_template(): # we remove it so any bad templates crash and burn del globs['url'] - try: - template_path, template_type = lib.render.template_info(template_name) - except lib.render.TemplateNotFound: - template_type = 'genshi' - template_path = '' + template_path, template_type = lib.render.template_info(template_name) # snippets should not pass the context # but allow for legacy genshi templates From 5b31702667badc49cc3b963506c3a0208ea24eda Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 13 Aug 2013 18:02:27 +0200 Subject: [PATCH 004/159] [#1183] Show very simple indicator for job status on resource page --- ckan/templates/package/resource_read.html | 1 + ckanext/datastore/logic/action.py | 54 +++++++++++++++++++++++ ckanext/datastore/logic/auth.py | 4 ++ ckanext/datastore/plugin.py | 5 ++- 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 46406746a6f..025816c8923 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -26,6 +26,7 @@ {% block resource_actions %}
    {% block resource_actions_inner %} +
  • {% snippet 'snippets/datapusher_status.html', resource=res %}
    • {% if h.check_access('package_update', {'id':pkg.id }) %}
  • {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
    • {% endif %} diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 4b79157355e..75ae3a163a8 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -429,6 +429,13 @@ def datastore_make_public(context, data_dict): db.make_public(context, data_dict) +def datapusher_enabled(context, data_dict): + """ Returns True if the DataPusher is configured """ + + datapusher_url = pylons.config.get('datapusher.url') + return True if datapusher_url else False + + def datapusher_submit(context, data_dict): ''' Submit a job to the datapusher. The datapusher is a service that imports tabular data into the datastore. @@ -550,6 +557,53 @@ def datapusher_hook(context, data_dict): p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) +def datapusher_status(context, data_dict): + """ Get the status of a datapusher job for a certain resource. + + :param resource_id: The resource id of the resource that you want the + datapusher status for. + :type resource_id: string + """ + + p.toolkit.check_access('datapusher_status', context, data_dict) + + if 'id' in data_dict: + data_dict['resource_id'] = data_dict['id'] + res_id = _get_or_bust(data_dict, 'resource_id') + + try: + task_id = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_id' + }) + except p.toolkit.ObjectNotFound: + return { + 'status': 'unknown' + } + + task_key = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_key' + }) + + datapusher_url = pylons.config.get('datapusher.url') + if not datapusher_url: + raise p.toolkit.ValidationError( + {'configuration': ['DataPusher not configured.']}) + + url = urlparse.urljoin(datapusher_url, 'job', task_id['value']) + + return { + 'status': task_id['state'], + 'job_id': task_id['value'], + 'job_url': url, + 'last_updated': task_id['last_updated'], + 'job_key': task_key['value'] + } + + def _resource_exists(context, data_dict): # Returns true if the resource exists in CKAN and in the datastore model = _get_or_bust(context, 'model') diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index 4d044755035..c94a413382f 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -38,5 +38,9 @@ def datapusher_submit(context, data_dict): return _datastore_auth(context, data_dict) +def datapusher_status(context, data_dict): + return _datastore_auth(context, data_dict) + + def datastore_change_permissions(context, data_dict): return _datastore_auth(context, data_dict) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 4477ad70526..232c3e25829 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -256,8 +256,10 @@ def get_actions(self): 'datastore_upsert': action.datastore_upsert, 'datastore_delete': action.datastore_delete, 'datastore_search': action.datastore_search, + 'datapusher_enabled': action.datapusher_enabled, 'datapusher_submit': action.datapusher_submit, 'datapusher_hook': action.datapusher_hook, + 'datapusher_status': action.datapusher_status, 'resource_show': self.resource_show_action, } if not self.legacy_mode: @@ -273,7 +275,8 @@ def get_auth_functions(self): 'datastore_delete': auth.datastore_delete, 'datastore_search': auth.datastore_search, 'datastore_change_permissions': auth.datastore_change_permissions, - 'datapusher_submit': auth.datapusher_submit} + 'datapusher_submit': auth.datapusher_submit, + 'datapusher_status': auth.datapusher_status} def before_map(self, m): m.connect('/datastore/dump/{resource_id}', From d691059da2d71306a266e5ad5dd616a19e8b6b85 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 13 Aug 2013 18:24:02 +0200 Subject: [PATCH 005/159] [#1183] Fix url --- ckanext/datastore/logic/action.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 75ae3a163a8..2993bd9b083 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -593,8 +593,7 @@ def datapusher_status(context, data_dict): raise p.toolkit.ValidationError( {'configuration': ['DataPusher not configured.']}) - url = urlparse.urljoin(datapusher_url, 'job', task_id['value']) - + url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value']) return { 'status': task_id['state'], 'job_id': task_id['value'], From 64ba2315b3f987215b12880041b7aa4272185424 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 13 Aug 2013 18:38:11 +0200 Subject: [PATCH 006/159] [#938] Fix datapusher test --- ckanext/datastore/tests/test_create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 589a6c26e3a..d7b893235ed 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -556,6 +556,7 @@ def test_create_ckan_resource_in_package(self): @httpretty.activate def test_providing_res_with_url_calls_datapusher_correctly(self): + pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' httpretty.HTTPretty.register_uri( httpretty.HTTPretty.POST, 'http://datapusher.ckan.org/job', From f367fd2ae3f00300895e352f2733b1e91489fc79 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 13 Aug 2013 23:36:44 +0200 Subject: [PATCH 007/159] [#1183] Add snippet for datapusher status indicator --- ckan/public/base/less/ckan.less | 1 + ckan/public/base/less/datapusher.less | 18 ++++++++++++++++++ .../templates/snippets/datapusher_status.html | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 ckan/public/base/less/datapusher.less create mode 100644 ckan/templates/snippets/datapusher_status.html diff --git a/ckan/public/base/less/ckan.less b/ckan/public/base/less/ckan.less index f7c108f8734..f842bd6dbae 100644 --- a/ckan/public/base/less/ckan.less +++ b/ckan/public/base/less/ckan.less @@ -20,6 +20,7 @@ @import "activity.less"; @import "dropdown.less"; @import "dashboard.less"; +@import "datapusher.less"; body { // Using the masthead/footer gradient prevents the color from changing diff --git a/ckan/public/base/less/datapusher.less b/ckan/public/base/less/datapusher.less new file mode 100644 index 00000000000..f147c0c3f6c --- /dev/null +++ b/ckan/public/base/less/datapusher.less @@ -0,0 +1,18 @@ +.datapusher-status-link:hover { + text-decoration: none; +} + +.datapusher-status { + &.status-unknown { + color: #bbb; + } + &.status-pending { + color: #FFCC00; + } + &.status-error { + color: red; + } + &.status-complete { + color: #009900; + } +} \ No newline at end of file diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html new file mode 100644 index 00000000000..bb95b08d809 --- /dev/null +++ b/ckan/templates/snippets/datapusher_status.html @@ -0,0 +1,19 @@ +{# Datapusher status indicator + +resource: the resource + +#} +{% if resource.datastore_active %} + {% set datapusher_enabled = h.get_action('datapusher_enabled') %} + {% if datapusher_enabled %} + {% set job = h.get_action('datapusher_status', data_dict={'resource_id': resource.id}) %} + {% set status = job.status %} + {% set datapusher_job_url = job.job_url %} + {% set title = _('Datapusher status: {status}').format(status=status) %} + {% if status == 'unknown' %} + + {% else %} + + {% endif %} + {% endif %} +{% endif %} From afb105999e4b6a49ea1f54bcb122c4278c9648ff Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 13 Aug 2013 23:45:09 +0200 Subject: [PATCH 008/159] [#1183] Clean up template code --- ckan/templates/snippets/datapusher_status.html | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html index bb95b08d809..d5ac0108520 100644 --- a/ckan/templates/snippets/datapusher_status.html +++ b/ckan/templates/snippets/datapusher_status.html @@ -7,13 +7,11 @@ {% set datapusher_enabled = h.get_action('datapusher_enabled') %} {% if datapusher_enabled %} {% set job = h.get_action('datapusher_status', data_dict={'resource_id': resource.id}) %} - {% set status = job.status %} - {% set datapusher_job_url = job.job_url %} - {% set title = _('Datapusher status: {status}').format(status=status) %} - {% if status == 'unknown' %} - + {% set title = _('Datapusher status: {status}.').format(status=job.status) %} + {% if job.status == 'unknown' %} + {% else %} - + {% endif %} {% endif %} {% endif %} From 77060c2ac1d24f13ce67d9ac9ea7294e92985f67 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 13 Aug 2013 23:47:43 +0200 Subject: [PATCH 009/159] [#1183] Only show for users who can edit resource --- ckan/templates/package/resource_read.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 025816c8923..d4433d4a4a8 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -26,8 +26,8 @@ {% block resource_actions %}
      {% block resource_actions_inner %} -
    • {% snippet 'snippets/datapusher_status.html', resource=res %}
      • {% if h.check_access('package_update', {'id':pkg.id }) %} +
    • {% snippet 'snippets/datapusher_status.html', resource=res %}
    • {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
      • {% endif %} {% if res.url %} From 78a289e119591e3d8c0972f0fab8447855b73f10 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Thu, 15 Aug 2013 20:32:34 +0200 Subject: [PATCH 010/159] [#1183] Don't use get_action because it will be deprecated in #1193 --- ckan/templates/snippets/datapusher_status.html | 5 ++--- ckanext/datastore/logic/action.py | 15 +++++---------- ckanext/datastore/plugin.py | 7 +++++++ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html index d5ac0108520..863faeee491 100644 --- a/ckan/templates/snippets/datapusher_status.html +++ b/ckan/templates/snippets/datapusher_status.html @@ -4,9 +4,8 @@ #} {% if resource.datastore_active %} - {% set datapusher_enabled = h.get_action('datapusher_enabled') %} - {% if datapusher_enabled %} - {% set job = h.get_action('datapusher_status', data_dict={'resource_id': resource.id}) %} + {% if h.datapusher_enabled() %} + {% set job = h.datapusher_status(resource.id) %} {% set title = _('Datapusher status: {status}.').format(status=job.status) %} {% if job.status == 'unknown' %} diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 2993bd9b083..8aa76493a6a 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -571,16 +571,11 @@ def datapusher_status(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - try: - task_id = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_id' - }) - except p.toolkit.ObjectNotFound: - return { - 'status': 'unknown' - } + task_id = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_id' + }) task_key = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 232c3e25829..b24dc30446b 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -5,6 +5,7 @@ import ckanext.datastore.logic.action as action import ckanext.datastore.logic.auth as auth import ckanext.datastore.db as db +import ckanext.datastore.helpers as helpers import ckan.logic as logic import ckan.model as model @@ -26,6 +27,7 @@ class DatastorePlugin(p.SingletonPlugin): p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.IRoutes, inherit=True) p.implements(p.IResourceController, inherit=True) + p.implements(p.ITemplateHelpers) legacy_mode = False resource_show_action = None @@ -293,3 +295,8 @@ def before_show(self, resource_dict): controller='ckanext.datastore.controller:DatastoreController', action='dump', resource_id=resource_dict['id']) return resource_dict + + def get_helpers(self): + return { + 'datapusher_status': helpers.datapusher_status, + 'datapusher_enabled': helpers.datapusher_enabled} From 8c85e3aeee07fa808cae3e0f532809acd804a780 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Thu, 15 Aug 2013 20:45:37 +0200 Subject: [PATCH 011/159] [#938] Use datapusher_enabled action instead of checking whether it is enabled ourselves --- ckanext/datastore/logic/action.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 8aa76493a6a..b57dd982b2b 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -460,12 +460,11 @@ def datapusher_submit(context, data_dict): p.toolkit.check_access('datapusher_submit', context, data_dict) - datapusher_url = pylons.config.get('datapusher.url') - - # no datapusher url means the datapusher should not be used - if not datapusher_url: + if not p.toolkit.get_action('datapusher_enabled')(context): return False + datapusher_url = pylons.config.get('datapusher.url') + callback_url = p.toolkit.url_for( controller='api', action='action', logic_function='datapusher_hook', ver=3, qualified=True) From deeb6efb7fec700516a97d5030a1ab0a69b2a8d1 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Thu, 15 Aug 2013 20:48:20 +0200 Subject: [PATCH 012/159] [#1183] Add file that I forgot --- ckanext/datastore/helpers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ckanext/datastore/helpers.py diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py new file mode 100644 index 00000000000..bf06b2e063a --- /dev/null +++ b/ckanext/datastore/helpers.py @@ -0,0 +1,15 @@ +import ckan.plugins.toolkit as toolkit + + +def datapusher_status(resource_id): + try: + return toolkit.get_action('datapusher_status')( + {}, {'resource_id': resource_id}) + except toolkit.ObjectNotFound: + return { + 'status': 'unknown' + } + + +def datapusher_enabled(): + return toolkit.get_action('datapusher_enabled')({}) From 236a8e927c78597d3e73e16dd0a2eba90d458d81 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Thu, 15 Aug 2013 20:48:20 +0200 Subject: [PATCH 013/159] [#1183] Add file that I forgot --- ckanext/datastore/helpers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ckanext/datastore/helpers.py diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py new file mode 100644 index 00000000000..bf06b2e063a --- /dev/null +++ b/ckanext/datastore/helpers.py @@ -0,0 +1,15 @@ +import ckan.plugins.toolkit as toolkit + + +def datapusher_status(resource_id): + try: + return toolkit.get_action('datapusher_status')( + {}, {'resource_id': resource_id}) + except toolkit.ObjectNotFound: + return { + 'status': 'unknown' + } + + +def datapusher_enabled(): + return toolkit.get_action('datapusher_enabled')({}) From 7d0b22b560a4d3b96d7e64d41e375f501acef73b Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Thu, 15 Aug 2013 21:17:45 +0200 Subject: [PATCH 014/159] [#1196] Move datapusher code into separate extension --- ckan/config/deployment.ini_tmpl | 1 + ckanext/datapusher/__init__.py | 0 ckanext/{datastore => datapusher}/helpers.py | 0 ckanext/datapusher/logic/__init__.py | 0 ckanext/datapusher/logic/action.py | 183 +++++++++++++++++++ ckanext/datapusher/logic/auth.py | 9 + ckanext/datapusher/plugin.py | 78 ++++++++ ckanext/datastore/logic/action.py | 168 ----------------- ckanext/datastore/logic/auth.py | 20 +- ckanext/datastore/plugin.py | 57 +----- ckanext/datastore/tests/test_create.py | 2 + setup.py | 1 + 12 files changed, 284 insertions(+), 235 deletions(-) create mode 100644 ckanext/datapusher/__init__.py rename ckanext/{datastore => datapusher}/helpers.py (100%) create mode 100644 ckanext/datapusher/logic/__init__.py create mode 100644 ckanext/datapusher/logic/action.py create mode 100644 ckanext/datapusher/logic/auth.py create mode 100644 ckanext/datapusher/plugin.py diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 97c4f9412ba..0729c606f3c 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -79,6 +79,7 @@ ckan.site_id = default ## Plugins Settings # Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher # Add ``pdf_preview`` to enable the resource preview for PDFs # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy diff --git a/ckanext/datapusher/__init__.py b/ckanext/datapusher/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datastore/helpers.py b/ckanext/datapusher/helpers.py similarity index 100% rename from ckanext/datastore/helpers.py rename to ckanext/datapusher/helpers.py diff --git a/ckanext/datapusher/logic/__init__.py b/ckanext/datapusher/logic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py new file mode 100644 index 00000000000..5957887d44e --- /dev/null +++ b/ckanext/datapusher/logic/action.py @@ -0,0 +1,183 @@ +import logging +import json +import urlparse +import datetime + +import pylons +import requests + +import ckan.lib.navl.dictization_functions +import ckan.logic as logic +import ckan.plugins as p + +log = logging.getLogger(__name__) +_get_or_bust = logic.get_or_bust +_validate = ckan.lib.navl.dictization_functions.validate + + +def datapusher_enabled(context, data_dict): + """ Returns True if the DataPusher is configured """ + + datapusher_url = pylons.config.get('datapusher.url') + return True if datapusher_url else False + + +def datapusher_submit(context, data_dict): + ''' Submit a job to the datapusher. The datapusher is a service that + imports tabular data into the datastore. + + :param resource_id: The resource id of the resource that the data + should be imported in. The resource's URL will be used to get the data. + :type resource_id: string + :param set_url_type: If set to true, the ``url_type`` of the resource will + be set to ``datastore`` and the resource URL will automatically point + to the :ref:`datastore dump ` URL. (optional, default: False) + :type set_url_type: boolean + + Returns ``True`` if the job has been submitted and ``False`` if the job + has not been submitted, i.e. when the datapusher is not configured. + + :rtype: boolean + ''' + + if 'id' in data_dict: + data_dict['resource_id'] = data_dict['id'] + res_id = _get_or_bust(data_dict, 'resource_id') + + p.toolkit.check_access('datapusher_submit', context, data_dict) + + if not p.toolkit.get_action('datapusher_enabled')(context): + return False + + datapusher_url = pylons.config.get('datapusher.url') + + callback_url = p.toolkit.url_for( + controller='api', action='action', logic_function='datapusher_hook', + ver=3, qualified=True) + + user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) + try: + r = requests.post( + urlparse.urljoin(datapusher_url, 'job'), + headers={ + 'Content-Type': 'application/json' + }, + data=json.dumps({ + 'api_key': user['apikey'], + 'job_type': 'push_to_datastore', + 'result_url': callback_url, + 'metadata': { + 'ckan_url': pylons.config['ckan.site_url'], + 'resource_id': res_id, + 'set_url_type': data_dict.get('set_url_type', False) + } + })) + r.raise_for_status() + except requests.exceptions.ConnectionError, e: + raise p.toolkit.ValidationError({'datapusher': { + 'message': 'Could not connect to DataPusher.', + 'details': str(e)}}) + except requests.exceptions.HTTPError, e: + m = 'An Error occurred while sending the job: {0}'.format(e.message) + try: + body = e.response.json() + except ValueError: + body = e.response.text + raise p.toolkit.ValidationError({'datapusher': { + 'message': m, + 'details': body, + 'status_code': r.status_code}}) + + empty_task = { + 'entity_id': res_id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'last_updated': str(datetime.datetime.now()), + 'state': 'pending' + } + + tasks = [] + for (k, v) in [('job_id', r.json()['job_id']), + ('job_key', r.json()['job_key'])]: + t = empty_task.copy() + t['key'] = k + t['value'] = v + tasks.append(t) + p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) + + return True + + +def datapusher_hook(context, data_dict): + """ Update datapusher task. This action is typically called by the + datapusher whenever the status of a job changes. + + Expects a job with ``status`` and ``metadata`` with a ``resource_id``. + """ + + # TODO: use a schema to validate + + p.toolkit.check_access('datapusher_submit', context, data_dict) + + res_id = data_dict['metadata']['resource_id'] + + task_id = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_id' + }) + + task_key = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_key' + }) + + tasks = [task_id, task_key] + + for task in tasks: + task['state'] = data_dict['status'] + task['last_updated'] = str(datetime.datetime.now()) + + p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) + + +def datapusher_status(context, data_dict): + """ Get the status of a datapusher job for a certain resource. + + :param resource_id: The resource id of the resource that you want the + datapusher status for. + :type resource_id: string + """ + + p.toolkit.check_access('datapusher_status', context, data_dict) + + if 'id' in data_dict: + data_dict['resource_id'] = data_dict['id'] + res_id = _get_or_bust(data_dict, 'resource_id') + + task_id = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_id' + }) + + task_key = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'job_key' + }) + + datapusher_url = pylons.config.get('datapusher.url') + if not datapusher_url: + raise p.toolkit.ValidationError( + {'configuration': ['DataPusher not configured.']}) + + url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value']) + return { + 'status': task_id['state'], + 'job_id': task_id['value'], + 'job_url': url, + 'last_updated': task_id['last_updated'], + 'job_key': task_key['value'] + } diff --git a/ckanext/datapusher/logic/auth.py b/ckanext/datapusher/logic/auth.py new file mode 100644 index 00000000000..55a7e832488 --- /dev/null +++ b/ckanext/datapusher/logic/auth.py @@ -0,0 +1,9 @@ +import ckanext.datastore.logic.auth as auth + + +def datapusher_submit(context, data_dict): + return auth.datastore_auth(context, data_dict) + + +def datapusher_status(context, data_dict): + return auth.datastore_auth(context, data_dict) diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py new file mode 100644 index 00000000000..17f9022fd22 --- /dev/null +++ b/ckanext/datapusher/plugin.py @@ -0,0 +1,78 @@ +import logging + +import ckan.plugins as p +import ckanext.datapusher.logic.action as action +import ckanext.datapusher.logic.auth as auth +import ckanext.datapusher.helpers as helpers +import ckan.logic as logic +import ckan.model as model + +log = logging.getLogger(__name__) +_get_or_bust = logic.get_or_bust + +DEFAULT_FORMATS = [] + + +class DatastoreException(Exception): + pass + + +class DatapusherPlugin(p.SingletonPlugin): + p.implements(p.IConfigurable, inherit=True) + p.implements(p.IActions) + p.implements(p.IAuthFunctions) + p.implements(p.IResourceUrlChange) + p.implements(p.IDomainObjectModification, inherit=True) + p.implements(p.IResourceController, inherit=True) + p.implements(p.ITemplateHelpers) + + legacy_mode = False + resource_show_action = None + + def configure(self, config): + self.config = config + + datapusher_formats = config.get('datapusher.formats', '').split() + self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS + + def notify(self, entity, operation=None): + if isinstance(entity, model.Resource): + if (operation == model.domain_object.DomainObjectOperation.new + or not operation): + # if operation is None, resource URL has been changed, as + # the notify function in IResourceUrlChange only takes + # 1 parameter + context = {'model': model, 'ignore_auth': True} + package = p.toolkit.get_action('package_show')(context, { + 'id': entity.get_package_id() + }) + if (not package['private'] and + entity.format in self.datapusher_formats): + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': entity.id + }) + + def get_actions(self): + return {'datapusher_enabled': action.datapusher_enabled, + 'datapusher_submit': action.datapusher_submit, + 'datapusher_hook': action.datapusher_hook, + 'datapusher_status': action.datapusher_status} + + def get_auth_functions(self): + return {'datapusher_submit': auth.datapusher_submit, + 'datapusher_status': auth.datapusher_status} + + def before_show(self, resource_dict): + ''' Modify the resource url of datastore resources so that + they link to the datastore dumps. + ''' + if resource_dict['url_type'] == 'datastore': + resource_dict['url'] = p.toolkit.url_for( + controller='ckanext.datastore.controller:DatastoreController', + action='dump', resource_id=resource_dict['id']) + return resource_dict + + def get_helpers(self): + return { + 'datapusher_status': helpers.datapusher_status, + 'datapusher_enabled': helpers.datapusher_enabled} diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index b57dd982b2b..9e93deed315 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -429,174 +429,6 @@ def datastore_make_public(context, data_dict): db.make_public(context, data_dict) -def datapusher_enabled(context, data_dict): - """ Returns True if the DataPusher is configured """ - - datapusher_url = pylons.config.get('datapusher.url') - return True if datapusher_url else False - - -def datapusher_submit(context, data_dict): - ''' Submit a job to the datapusher. The datapusher is a service that - imports tabular data into the datastore. - - :param resource_id: The resource id of the resource that the data - should be imported in. The resource's URL will be used to get the data. - :type resource_id: string - :param set_url_type: If set to true, the ``url_type`` of the resource will - be set to ``datastore`` and the resource URL will automatically point - to the :ref:`datastore dump ` URL. (optional, default: False) - :type set_url_type: boolean - - Returns ``True`` if the job has been submitted and ``False`` if the job - has not been submitted, i.e. when the datapusher is not configured. - - :rtype: boolean - ''' - - if 'id' in data_dict: - data_dict['resource_id'] = data_dict['id'] - res_id = _get_or_bust(data_dict, 'resource_id') - - p.toolkit.check_access('datapusher_submit', context, data_dict) - - if not p.toolkit.get_action('datapusher_enabled')(context): - return False - - datapusher_url = pylons.config.get('datapusher.url') - - callback_url = p.toolkit.url_for( - controller='api', action='action', logic_function='datapusher_hook', - ver=3, qualified=True) - - user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) - try: - r = requests.post( - urlparse.urljoin(datapusher_url, 'job'), - headers={ - 'Content-Type': 'application/json' - }, - data=json.dumps({ - 'api_key': user['apikey'], - 'job_type': 'push_to_datastore', - 'result_url': callback_url, - 'metadata': { - 'ckan_url': pylons.config['ckan.site_url'], - 'resource_id': res_id, - 'set_url_type': data_dict.get('set_url_type', False) - } - })) - r.raise_for_status() - except requests.exceptions.ConnectionError, e: - raise p.toolkit.ValidationError({'datapusher': { - 'message': 'Could not connect to DataPusher.', - 'details': str(e)}}) - except requests.exceptions.HTTPError, e: - m = 'An Error occurred while sending the job: {0}'.format(e.message) - try: - body = e.response.json() - except ValueError: - body = e.response.text - raise p.toolkit.ValidationError({'datapusher': { - 'message': m, - 'details': body, - 'status_code': r.status_code}}) - - empty_task = { - 'entity_id': res_id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - } - - tasks = [] - for (k, v) in [('job_id', r.json()['job_id']), - ('job_key', r.json()['job_key'])]: - t = empty_task.copy() - t['key'] = k - t['value'] = v - tasks.append(t) - p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) - - return True - - -def datapusher_hook(context, data_dict): - """ Update datapusher task. This action is typically called by the - datapusher whenever the status of a job changes. - - Expects a job with ``status`` and ``metadata`` with a ``resource_id``. - """ - - # TODO: use a schema to validate - - p.toolkit.check_access('datapusher_submit', context, data_dict) - - res_id = data_dict['metadata']['resource_id'] - - task_id = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_id' - }) - - task_key = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_key' - }) - - tasks = [task_id, task_key] - - for task in tasks: - task['state'] = data_dict['status'] - task['last_updated'] = str(datetime.datetime.now()) - - p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) - - -def datapusher_status(context, data_dict): - """ Get the status of a datapusher job for a certain resource. - - :param resource_id: The resource id of the resource that you want the - datapusher status for. - :type resource_id: string - """ - - p.toolkit.check_access('datapusher_status', context, data_dict) - - if 'id' in data_dict: - data_dict['resource_id'] = data_dict['id'] - res_id = _get_or_bust(data_dict, 'resource_id') - - task_id = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_id' - }) - - task_key = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_key' - }) - - datapusher_url = pylons.config.get('datapusher.url') - if not datapusher_url: - raise p.toolkit.ValidationError( - {'configuration': ['DataPusher not configured.']}) - - url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value']) - return { - 'status': task_id['state'], - 'job_id': task_id['value'], - 'job_url': url, - 'last_updated': task_id['last_updated'], - 'job_key': task_key['value'] - } - - def _resource_exists(context, data_dict): # Returns true if the resource exists in CKAN and in the datastore model = _get_or_bust(context, 'model') diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index c94a413382f..357ae97fd6b 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -1,7 +1,7 @@ import ckan.plugins as p -def _datastore_auth(context, data_dict, privilege='resource_update'): +def datastore_auth(context, data_dict, privilege='resource_update'): if not 'id' in data_dict: data_dict['id'] = data_dict.get('resource_id') user = context.get('user') @@ -19,28 +19,20 @@ def _datastore_auth(context, data_dict, privilege='resource_update'): def datastore_create(context, data_dict): - return _datastore_auth(context, data_dict) + return datastore_auth(context, data_dict) def datastore_upsert(context, data_dict): - return _datastore_auth(context, data_dict) + return datastore_auth(context, data_dict) def datastore_delete(context, data_dict): - return _datastore_auth(context, data_dict) + return datastore_auth(context, data_dict) def datastore_search(context, data_dict): - return _datastore_auth(context, data_dict, 'resource_show') - - -def datapusher_submit(context, data_dict): - return _datastore_auth(context, data_dict) - - -def datapusher_status(context, data_dict): - return _datastore_auth(context, data_dict) + return datastore_auth(context, data_dict, 'resource_show') def datastore_change_permissions(context, data_dict): - return _datastore_auth(context, data_dict) + return datastore_auth(context, data_dict) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index b24dc30446b..8eb753f8e46 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -5,7 +5,6 @@ import ckanext.datastore.logic.action as action import ckanext.datastore.logic.auth as auth import ckanext.datastore.db as db -import ckanext.datastore.helpers as helpers import ckan.logic as logic import ckan.model as model @@ -26,8 +25,6 @@ class DatastorePlugin(p.SingletonPlugin): p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.IRoutes, inherit=True) - p.implements(p.IResourceController, inherit=True) - p.implements(p.ITemplateHelpers) legacy_mode = False resource_show_action = None @@ -104,36 +101,11 @@ def new_resource_show(context, data_dict): self.resource_show_action = new_resource_show def notify(self, entity, operation=None): - ''' - if not isinstance(entity, model.Resource): - return - if operation: - if operation == model.domain_object.DomainObjectOperation.new: - self._create_datastorer_task(entity) - else: - # if operation is None, resource URL has been changed, as the - # notify function in IResourceUrlChange only takes 1 parameter - self._create_datastorer_task(entity) - ''' - context = {'model': model, 'ignore_auth': True} - if isinstance(entity, model.Resource): - if (operation == model.domain_object.DomainObjectOperation.new - or not operation): - # if operation is None, resource URL has been changed, as - # the notify function in IResourceUrlChange only takes - # 1 parameter - package = p.toolkit.get_action('package_show')(context, { - 'id': entity.get_package_id() - }) - if (not package['private'] and - entity.format in self.datapusher_formats): - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': entity.id - }) if not isinstance(entity, model.Package) or self.legacy_mode: return # if a resource is new, it cannot have a datastore resource, yet if operation == model.domain_object.DomainObjectOperation.changed: + context = {'model': model, 'ignore_auth': True} if entity.private: func = p.toolkit.get_action('datastore_make_private') else: @@ -160,7 +132,7 @@ def _check_urls_and_permissions(self): self._log_or_raise('CKAN and DataStore database ' 'cannot be the same.') - # in legacy mode, the read and write url are ths same (both write url) + # in legacy mode, the read and write url are the same (both write url) # consequently the same url check and and write privilege check # don't make sense if not self.legacy_mode: @@ -258,10 +230,6 @@ def get_actions(self): 'datastore_upsert': action.datastore_upsert, 'datastore_delete': action.datastore_delete, 'datastore_search': action.datastore_search, - 'datapusher_enabled': action.datapusher_enabled, - 'datapusher_submit': action.datapusher_submit, - 'datapusher_hook': action.datapusher_hook, - 'datapusher_status': action.datapusher_status, 'resource_show': self.resource_show_action, } if not self.legacy_mode: @@ -276,27 +244,10 @@ def get_auth_functions(self): 'datastore_upsert': auth.datastore_upsert, 'datastore_delete': auth.datastore_delete, 'datastore_search': auth.datastore_search, - 'datastore_change_permissions': auth.datastore_change_permissions, - 'datapusher_submit': auth.datapusher_submit, - 'datapusher_status': auth.datapusher_status} + 'datastore_change_permissions': auth.datastore_change_permissions} def before_map(self, m): m.connect('/datastore/dump/{resource_id}', controller='ckanext.datastore.controller:DatastoreController', action='dump') - return m - - def before_show(self, resource_dict): - ''' Modify the resource url of datastore resources so that - they link to the datastore dumps. - ''' - if resource_dict['url_type'] == 'datastore': - resource_dict['url'] = p.toolkit.url_for( - controller='ckanext.datastore.controller:DatastoreController', - action='dump', resource_id=resource_dict['id']) - return resource_dict - - def get_helpers(self): - return { - 'datapusher_status': helpers.datapusher_status, - 'datapusher_enabled': helpers.datapusher_enabled} + return m \ No newline at end of file diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index d7b893235ed..7a6933008e1 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -38,6 +38,7 @@ def setup_class(cls): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") p.load('datastore') + p.load('datapusher') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') @@ -49,6 +50,7 @@ def setup_class(cls): def teardown_class(cls): rebuild_all_dbs(cls.Session) p.unload('datastore') + p.unload('datapusher') def test_create_requires_auth(self): resource = model.Package.get('annakarenina').resources[0] diff --git a/setup.py b/setup.py index 276e03a5fdf..eb0d8058d6f 100644 --- a/setup.py +++ b/setup.py @@ -115,6 +115,7 @@ organizations=ckanext.organizations.forms:OrganizationForm organizations_dataset=ckanext.organizations.forms:OrganizationDatasetForm datastore=ckanext.datastore.plugin:DatastorePlugin + datapusher=ckanext.datapusher.plugin:DatapusherPlugin test_tag_vocab_plugin=ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin resource_proxy=ckanext.resourceproxy.plugin:ResourceProxy text_preview=ckanext.textpreview.plugin:TextPreview From 7c60ba441dcbdb309e82344a6a981b87243a20a3 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Sun, 4 Aug 2013 13:30:18 -0300 Subject: [PATCH 015/159] [#1163] Users can be deleted To do this, I've configured the User model to be stateful using vdm.sqlalchemy. Right now, there're two states: active and deleted. If a user is deleted, he can't login, and is unauthorized to do anything. She also doesn't appear in the user's list anymore, but you can still access her profile page, if you know her username. If she was logged in when her user was deleted, the next time she goes into CKAN, she'll be logged off. Unfortunately, there's not a useful message like "Your user has been deleted." Yet. There's no way to undelete a user, but it should be simply creating an action to set her state to active. --- ckan/config/routing.py | 1 + ckan/controllers/user.py | 15 ++ ckan/lib/authenticator.py | 22 +- ckan/lib/base.py | 5 +- ckan/lib/create_test_data.py | 3 +- ckan/logic/action/delete.py | 22 ++ ckan/logic/action/get.py | 8 +- ckan/logic/auth/delete.py | 4 + .../070_add_state_column_to_user_table.py | 17 ++ ckan/model/follower.py | 201 +++++++----------- ckan/model/user.py | 9 +- ckan/new_authz.py | 35 +-- ckan/templates/user/edit_user_form.html | 6 + ckan/templates/user/read_base.html | 4 + ckan/tests/functional/test_user.py | 19 ++ ckan/tests/lib/test_authenticator.py | 106 +++++++++ ckan/tests/lib/test_dictization.py | 1 + ckan/tests/logic/test_action.py | 38 ++++ ckan/tests/logic/test_auth.py | 27 ++- ckan/tests/models/test_follower.py | 124 +++++++++++ ckan/tests/models/test_user.py | 7 + 21 files changed, 527 insertions(+), 147 deletions(-) create mode 100644 ckan/migration/versions/070_add_state_column_to_user_table.py create mode 100644 ckan/tests/lib/test_authenticator.py create mode 100644 ckan/tests/models/test_follower.py diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 0fbb02bb886..8f78a6c099d 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -359,6 +359,7 @@ def make_map(): action='followers', ckan_icon='group') m.connect('user_edit', '/user/edit/{id:.*}', action='edit', ckan_icon='cog') + m.connect('user_delete', '/user/delete/{id}', action='delete') m.connect('/user/reset/{id:.*}', action='perform_reset') m.connect('register', '/user/register', action='register') m.connect('login', '/user/login', action='login') diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index ab162ef0659..e0b1ba6c762 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -178,6 +178,21 @@ def new(self, data=None, errors=None, error_summary=None): c.form = render(self.new_user_form, extra_vars=vars) return render('user/new.html') + def delete(self, id): + '''Delete user with id passed as parameter''' + context = {'model': model, + 'session': model.Session, + 'user': c.user} + data_dict = {'id': id} + + try: + get_action('user_delete')(context, data_dict) + user_index = h.url_for(controller='user', action='index') + h.redirect_to(user_index) + except NotAuthorized: + msg = _('Unauthorized to delete user with id "{user_id}".') + abort(401, msg.format(user_id=id)) + def _save_new(self, context): try: data_dict = logic.clean_dict(unflatten( diff --git a/ckan/lib/authenticator.py b/ckan/lib/authenticator.py index 6f061caad91..f64db4f5b8d 100644 --- a/ckan/lib/authenticator.py +++ b/ckan/lib/authenticator.py @@ -12,9 +12,9 @@ class OpenIDAuthenticator(object): def authenticate(self, environ, identity): if 'repoze.who.plugins.openid.userid' in identity: - openid = identity.get('repoze.who.plugins.openid.userid') + openid = identity['repoze.who.plugins.openid.userid'] user = User.by_openid(openid) - if user is None: + if user is None or user.is_deleted(): return None else: return user.name @@ -25,14 +25,20 @@ class UsernamePasswordAuthenticator(object): implements(IAuthenticator) def authenticate(self, environ, identity): - if not 'login' in identity or not 'password' in identity: + if not ('login' in identity and 'password' in identity): return None - user = User.by_name(identity.get('login')) + + login = identity['login'] + user = User.by_name(login) + if user is None: - log.debug('Login failed - username %r not found', identity.get('login')) - return None - if user.validate_password(identity.get('password')): + log.debug('Login failed - username %r not found', login) + elif user.is_deleted(): + log.debug('Login as %r failed - user is deleted', login) + elif not user.validate_password(identity['password']): + log.debug('Login as %r failed - password not valid', login) + else: return user.name - log.debug('Login as %r failed - password not valid', identity.get('login')) + return None diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 1298e0665c7..bfffcaf2a5d 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -290,8 +290,9 @@ def _identify_user_default(self): if c.user: c.user = c.user.decode('utf8') c.userobj = model.User.by_name(c.user) - if c.userobj is None: - # This occurs when you are logged in, clean db + if c.userobj is None or c.userobj.is_deleted(): + # This occurs when a user that was still logged in is deleted, + # or when you are logged in, clean db # and then restart (or when you change your username) # There is no user object, so even though repoze thinks you # are logged in and your cookie has ckan_display_name, we diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 3edf865c7c7..6c7a275c36f 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -519,8 +519,9 @@ def _create_user_without_commit(cls, name='', **user_dict): @classmethod def create_user(cls, name='', **kwargs): - cls._create_user_without_commit(name, **kwargs) + user = cls._create_user_without_commit(name, **kwargs) model.Session.commit() + return user @classmethod def flag_for_deletion(cls, pkg_names=[], tag_names=[], group_names=[], diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 933e7624b43..9191151aa01 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -18,6 +18,28 @@ _get_or_bust = ckan.logic.get_or_bust _get_action = ckan.logic.get_action +def user_delete(context, data_dict): + '''Delete a user. + + Only sysadmins can delete users. + + :param id: the id or usernamename of the user to delete + :type id: string + ''' + + _check_access('user_delete', context, data_dict) + + model = context['model'] + user_id = _get_or_bust(data_dict, 'id') + user = model.User.get(user_id) + + if user is None: + raise NotFound('User "{id}" was not found.'.format(id=user_id)) + + user.delete() + model.repo.commit() + + def package_delete(context, data_dict): '''Delete a dataset (package). diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index c0fe9639764..a6cec306c91 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -7,6 +7,7 @@ from pylons import config import sqlalchemy +import vdm.sqlalchemy import ckan.lib.dictization import ckan.logic as logic @@ -664,6 +665,9 @@ def user_list(context, data_dict): else_=model.User.fullname) ) + # Filter deleted users + query = query.filter(model.User.state != vdm.sqlalchemy.State.DELETED) + ## hack for pagination if context.get('return_query'): return query @@ -1173,7 +1177,9 @@ def user_autocomplete(context, data_dict): q = data_dict['q'] limit = data_dict.get('limit', 20) - query = model.User.search(q).limit(limit) + query = model.User.search(q) + query = query.filter(model.User.state != vdm.sqlalchemy.State.DELETED) + query = query.limit(limit) user_list = [] for user in query.all(): diff --git a/ckan/logic/auth/delete.py b/ckan/logic/auth/delete.py index 61b44697407..a2b121c11ad 100644 --- a/ckan/logic/auth/delete.py +++ b/ckan/logic/auth/delete.py @@ -4,6 +4,10 @@ from ckan.logic.auth import get_resource_object from ckan.lib.base import _ +def user_delete(context, data_dict): + # Only sysadmins are authorized to purge organizations. + return {'success': False} + def package_delete(context, data_dict): user = context['user'] package = get_package_object(context, data_dict) diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/070_add_state_column_to_user_table.py new file mode 100644 index 00000000000..1dfadbe7d32 --- /dev/null +++ b/ckan/migration/versions/070_add_state_column_to_user_table.py @@ -0,0 +1,17 @@ +import vdm.sqlalchemy + + +def upgrade(migrate_engine): + migrate_engine.execute( + ''' + ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' + ''' % vdm.sqlalchemy.State.ACTIVE + ) + + +def downgrade(migrate_engine): + migrate_engine.exeecute( + ''' + ALTER TABLE "user" DROP COLUMN "state" + ''' + ) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index 6e6096ed5cc..d14ffe667ae 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -1,30 +1,28 @@ -import sqlalchemy import meta import datetime -import domain_object +import sqlalchemy +import vdm.sqlalchemy -class UserFollowingUser(domain_object.DomainObject): - '''A many-many relationship between users. +import ckan.model +import domain_object - A relationship between one user (the follower) and another (the object), - that means that the follower is currently following the object. - ''' +class ModelFollowingModel(domain_object.DomainObject): def __init__(self, follower_id, object_id): self.follower_id = follower_id self.object_id = object_id self.datetime = datetime.datetime.now() @classmethod - def get(self, follower_id, object_id): - '''Return a UserFollowingUser object for the given follower_id and + def get(cls, follower_id, object_id): + '''Return a ModelFollowingModel object for the given follower_id and object_id, or None if no such follower exists. ''' - query = meta.Session.query(UserFollowingUser) - query = query.filter(UserFollowingUser.follower_id==follower_id) - query = query.filter(UserFollowingUser.object_id==object_id) - return query.first() + query = cls._get(follower_id, object_id) + following = cls._filter_following_objects(query) + if len(following) == 1: + return following[0] @classmethod def is_following(cls, follower_id, object_id): @@ -32,33 +30,76 @@ def is_following(cls, follower_id, object_id): otherwise. ''' - return UserFollowingUser.get(follower_id, object_id) is not None - + return cls.get(follower_id, object_id) is not None @classmethod def followee_count(cls, follower_id): - '''Return the number of users followed by a user.''' - return meta.Session.query(UserFollowingUser).filter( - UserFollowingUser.follower_id == follower_id).count() + '''Return the number of objects followed by the follower.''' + return cls._get_followees(follower_id).count() @classmethod def followee_list(cls, follower_id): - '''Return a list of users followed by a user.''' - return meta.Session.query(UserFollowingUser).filter( - UserFollowingUser.follower_id == follower_id).all() + '''Return a list of objects followed by the follower.''' + query = cls._get_followees(follower_id).all() + followees = cls._filter_following_objects(query) + return followees + + @classmethod + def follower_count(cls, object_id): + '''Return the number of followers of the object.''' + return cls._get_followers(object_id).count() + + @classmethod + def follower_list(cls, object_id): + '''Return a list of followers of the object.''' + query = cls._get_followers(object_id).all() + followers = cls._filter_following_objects(query) + return followers + + @classmethod + def _filter_following_objects(cls, query): + return [q[0] for q in query] + + @classmethod + def _get_followees(cls, follower_id): + return cls._get(follower_id) + + @classmethod + def _get_followers(cls, object_id): + return cls._get(None, object_id) + + @classmethod + def _get(cls, follower_id=None, object_id=None): + follower_alias = sqlalchemy.orm.aliased(cls._follower_class()) + object_alias = sqlalchemy.orm.aliased(cls._object_class()) + + follower_id = follower_id or cls.follower_id + object_id = object_id or cls.object_id + + query = meta.Session.query(cls, follower_alias, object_alias)\ + .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ + cls.follower_id == follower_alias.id,\ + cls.object_id == object_alias.id,\ + follower_alias.state != vdm.sqlalchemy.State.DELETED,\ + object_alias.state != vdm.sqlalchemy.State.DELETED,\ + object_alias.id == object_id)) + return query + +class UserFollowingUser(ModelFollowingModel): + '''A many-many relationship between users. + + A relationship between one user (the follower) and another (the object), + that means that the follower is currently following the object. + ''' @classmethod - def follower_count(cls, user_id): - '''Return the number of followers of a user.''' - return meta.Session.query(UserFollowingUser).filter( - UserFollowingUser.object_id == user_id).count() + def _follower_class(cls): + return ckan.model.User @classmethod - def follower_list(cls, user_id): - '''Return a list of followers of a user.''' - return meta.Session.query(UserFollowingUser).filter( - UserFollowingUser.object_id == user_id).all() + def _object_class(cls): + return ckan.model.User user_following_user_table = sqlalchemy.Table('user_following_user', @@ -76,62 +117,20 @@ def follower_list(cls, user_id): meta.mapper(UserFollowingUser, user_following_user_table) -class UserFollowingDataset(domain_object.DomainObject): +class UserFollowingDataset(ModelFollowingModel): '''A many-many relationship between users and datasets (packages). A relationship between a user (the follower) and a dataset (the object), that means that the user is currently following the dataset. ''' - def __init__(self, follower_id, object_id): - self.follower_id = follower_id - self.object_id = object_id - self.datetime = datetime.datetime.now() - - @classmethod - def get(self, follower_id, object_id): - '''Return a UserFollowingDataset object for the given follower_id and - object_id, or None if no such follower exists. - - ''' - query = meta.Session.query(UserFollowingDataset) - query = query.filter(UserFollowingDataset.follower_id==follower_id) - query = query.filter(UserFollowingDataset.object_id==object_id) - return query.first() - - @classmethod - def is_following(cls, follower_id, object_id): - '''Return True if follower_id is currently following object_id, False - otherwise. - - ''' - return UserFollowingDataset.get(follower_id, object_id) is not None - - - @classmethod - def followee_count(cls, follower_id): - '''Return the number of datasets followed by a user.''' - return meta.Session.query(UserFollowingDataset).filter( - UserFollowingDataset.follower_id == follower_id).count() - - @classmethod - def followee_list(cls, follower_id): - '''Return a list of datasets followed by a user.''' - return meta.Session.query(UserFollowingDataset).filter( - UserFollowingDataset.follower_id == follower_id).all() - - @classmethod - def follower_count(cls, dataset_id): - '''Return the number of followers of a dataset.''' - return meta.Session.query(UserFollowingDataset).filter( - UserFollowingDataset.object_id == dataset_id).count() + def _follower_class(cls): + return ckan.model.User @classmethod - def follower_list(cls, dataset_id): - '''Return a list of followers of a dataset.''' - return meta.Session.query(UserFollowingDataset).filter( - UserFollowingDataset.object_id == dataset_id).all() + def _object_class(cls): + return ckan.model.Package user_following_dataset_table = sqlalchemy.Table('user_following_dataset', @@ -150,60 +149,20 @@ def follower_list(cls, dataset_id): meta.mapper(UserFollowingDataset, user_following_dataset_table) -class UserFollowingGroup(domain_object.DomainObject): +class UserFollowingGroup(ModelFollowingModel): '''A many-many relationship between users and groups. A relationship between a user (the follower) and a group (the object), that means that the user is currently following the group. ''' - def __init__(self, follower_id, object_id): - self.follower_id = follower_id - self.object_id = object_id - self.datetime = datetime.datetime.now() - - @classmethod - def get(self, follower_id, object_id): - '''Return a UserFollowingGroup object for the given follower_id and - object_id, or None if no such relationship exists. - - ''' - query = meta.Session.query(UserFollowingGroup) - query = query.filter(UserFollowingGroup.follower_id == follower_id) - query = query.filter(UserFollowingGroup.object_id == object_id) - return query.first() - - @classmethod - def is_following(cls, follower_id, object_id): - '''Return True if follower_id is currently following object_id, False - otherwise. - - ''' - return UserFollowingGroup.get(follower_id, object_id) is not None - - @classmethod - def followee_count(cls, follower_id): - '''Return the number of groups followed by a user.''' - return meta.Session.query(UserFollowingGroup).filter( - UserFollowingGroup.follower_id == follower_id).count() - @classmethod - def followee_list(cls, follower_id): - '''Return a list of groups followed by a user.''' - return meta.Session.query(UserFollowingGroup).filter( - UserFollowingGroup.follower_id == follower_id).all() + def _follower_class(cls): + return ckan.model.User @classmethod - def follower_count(cls, object_id): - '''Return the number of users following a group.''' - return meta.Session.query(UserFollowingGroup).filter( - UserFollowingGroup.object_id == object_id).count() - - @classmethod - def follower_list(cls, object_id): - '''Return a list of the users following a group.''' - return meta.Session.query(UserFollowingGroup).filter( - UserFollowingGroup.object_id == object_id).all() + def _object_class(cls): + return ckan.model.Group user_following_group_table = sqlalchemy.Table('user_following_group', meta.metadata, diff --git a/ckan/model/user.py b/ckan/model/user.py index d3eb3d6f1c1..d36dd5343cc 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -6,6 +6,7 @@ from sqlalchemy.sql.expression import or_ from sqlalchemy.orm import synonym from sqlalchemy import types, Column, Table +import vdm.sqlalchemy import meta import types as _types @@ -28,8 +29,11 @@ Column('sysadmin', types.Boolean, default=False), ) +vdm.sqlalchemy.make_table_stateful(user_table) -class User(domain_object.DomainObject): + +class User(vdm.sqlalchemy.StatefulObjectMixin, + domain_object.DomainObject): VALID_NAME = re.compile(r"^[a-zA-Z0-9_\-]{3,255}$") DOUBLE_SLASH = re.compile(':\/([^/])') @@ -162,6 +166,9 @@ def number_administered_packages(self): q = q.filter_by(user=self, role=model.Role.ADMIN) return q.count() + def is_deleted(self): + return self.state == vdm.sqlalchemy.State.DELETED + def is_in_group(self, group): return group in self.get_group_ids() diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 01fb07927ac..bf1d0830d47 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -101,23 +101,30 @@ def clean_action_name(action_name): def is_sysadmin(username): - ''' returns True is username is a sysadmin ''' + ''' Returns True is username is a sysadmin ''' + user = _get_user(username) + return user and user.sysadmin + + +def is_deleted(username): + ''' Returns True if username is deleted ''' + user = _get_user(username) + return user and user.is_deleted() + + +def _get_user(username): + ''' Try to get the user from c, if possible, and fallback to using the DB ''' if not username: - return False - # see if we can authorise without touching the database + return None + # See if we can get the user without touching the DB try: if c.userobj and c.userobj.name == username: - if c.userobj.sysadmin: - return True - return False + return c.userobj except TypeError: # c is not available pass - # get user from the database - user = model.User.get(username) - if user and user.sysadmin: - return True - return False + # Get user from the DB + return model.User.get(username) def get_group_or_org_admin_ids(group_id): @@ -146,10 +153,14 @@ def is_authorized(action, context, data_dict=None): action = clean_action_name(action) auth_function = _AuthFunctions.get(action) if auth_function: + username = context.get('user') + # deleted users are always unauthorized + if is_deleted(username): + return {'success': False} # sysadmins can do anything unless the auth_sysadmins_check # decorator was used in which case they are treated like all other # users. - if is_sysadmin(context.get('user')): + elif is_sysadmin(username): if not getattr(auth_function, 'auth_sysadmins_check', False): return {'success': True} diff --git a/ckan/templates/user/edit_user_form.html b/ckan/templates/user/edit_user_form.html index 19b8755b2bd..ce31e34b583 100644 --- a/ckan/templates/user/edit_user_form.html +++ b/ckan/templates/user/edit_user_form.html @@ -33,6 +33,12 @@
      + {% block delete_button %} + {% if h.check_access('user_delete', {'id': data.id}) %} + {% set locale = h.dump_json({'content': _('Are you sure you want to delete this User?')}) %} + {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + {% endif %} + {% endblock %}
      diff --git a/ckan/templates/user/read_base.html b/ckan/templates/user/read_base.html index aa27a2b4119..f2b94e20782 100644 --- a/ckan/templates/user/read_base.html +++ b/ckan/templates/user/read_base.html @@ -76,6 +76,10 @@

      {{ user.display_name }}

      {{ _('Member Since') }}
      {{ h.render_datetime(user.created) }}
      +
      +
      {{ _('State') }}
      +
      {{ user.state }}
      +
      {% if c.is_myself %}
      {{ _('API Key') }} {{ _('Private') }}
      diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index 0ef528fd9ad..16795561f46 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -64,6 +64,25 @@ def test_user_read(self): 'rel="nofollow"') assert 'Edit Profile' not in main_res, main_res + def test_user_delete_redirects_to_user_index(self): + user = CreateTestData.create_user('a_user') + url = url_for(controller='user', action='delete', id=user.id) + extra_environ = {'REMOTE_USER': 'testsysadmin'} + + redirect_url = url_for(controller='user', action='index', + qualified=True) + res = self.app.get(url, status=302, extra_environ=extra_environ) + + assert user.is_deleted(), user + assert res.header('Location').startswith(redirect_url), res.header('Location') + + def test_user_delete_by_unauthorized_user(self): + user = model.User.by_name(u'annafan') + url = url_for(controller='user', action='delete', id=user.id) + extra_environ = {'REMOTE_USER': 'an_unauthorized_user'} + + self.app.get(url, status=401, extra_environ=extra_environ) + def test_user_read_without_id(self): offset = '/user/' res = self.app.get(offset, status=302) diff --git a/ckan/tests/lib/test_authenticator.py b/ckan/tests/lib/test_authenticator.py new file mode 100644 index 00000000000..c7c026112d2 --- /dev/null +++ b/ckan/tests/lib/test_authenticator.py @@ -0,0 +1,106 @@ +import ckan + +import ckan.lib.create_test_data as ctd +import ckan.lib.authenticator as authenticator + +CreateTestData = ctd.CreateTestData + + +class TestUsernamePasswordAuthenticator(object): + @classmethod + def setup_class(cls): + auth = authenticator.UsernamePasswordAuthenticator() + cls.authenticate = auth.authenticate + + @classmethod + def teardown(cls): + ckan.model.repo.rebuild_db() + + def test_authenticate_succeeds_if_login_and_password_are_correct(self): + environ = {} + password = 'somepass' + user = CreateTestData.create_user('a_user', **{'password': password}) + identity = {'login': user.name, 'password': password} + + username = self.authenticate(environ, identity) + assert username == user.name, username + + def test_authenticate_fails_if_user_is_deleted(self): + environ = {} + password = 'somepass' + user = CreateTestData.create_user('a_user', **{'password': password}) + identity = {'login': user.name, 'password': password} + user.delete() + + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_password_is_wrong(self): + environ = {} + user = CreateTestData.create_user('a_user') + identity = {'login': user.name, 'password': 'wrong-password'} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_received_no_login_or_pass(self): + environ = {} + identity = {} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_received_just_login(self): + environ = {} + identity = {'login': 'some-user'} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_received_just_password(self): + environ = {} + identity = {'password': 'some-password'} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_user_doesnt_exist(self): + environ = {} + identity = {'login': 'inexistent-user'} + assert self.authenticate(environ, identity) is None + + +class TestOpenIDAuthenticator(object): + @classmethod + def setup_class(cls): + auth = authenticator.OpenIDAuthenticator() + cls.authenticate = auth.authenticate + + @classmethod + def teardown(cls): + ckan.model.repo.rebuild_db() + + def test_authenticate_succeeds_if_openid_is_correct(self): + environ = {} + openid = 'some-openid-key' + user = CreateTestData.create_user('a_user', **{'openid': openid}) + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + username = self.authenticate(environ, identity) + assert username == user.name, username + + def test_authenticate_fails_if_openid_is_incorrect(self): + environ = {} + openid = 'wrong-openid-key' + user = CreateTestData.create_user('a_user') + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_user_is_deleted(self): + environ = {} + openid = 'some-openid-key' + user = CreateTestData.create_user('a_user', **{'openid': openid}) + user.delete() + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_user_have_no_openid(self): + environ = {} + identity = {} + assert self.authenticate(environ, identity) is None diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index fb8bb49e628..72ca6fa31e5 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -939,6 +939,7 @@ def test_16_group_dictized(self): 'users': [{'about': u'I love reading Annakarenina. My site: http://anna.com', 'display_name': u'annafan', 'capacity' : 'public', + 'state': 'active', 'sysadmin': False, 'email_hash': 'd41d8cd98f00b204e9800998ecf8427e', 'fullname': None, diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 73a3b838f0c..fecbdf96b53 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -7,6 +7,7 @@ from pylons import config import datetime +import vdm.sqlalchemy import ckan from ckan.lib.create_test_data import CreateTestData from ckan.lib.dictization.model_dictize import resource_dictize @@ -334,6 +335,11 @@ def test_42_create_resource_with_error(self): def test_04_user_list(self): + # Create deleted user to make sure he won't appear in the user_list + deleted_user = CreateTestData.create_user('deleted_user') + deleted_user.delete() + model.repo.commit() + postparams = '%s=1' % json.dumps({}) res = self.app.post('/api/action/user_list', params=postparams) res_obj = json.loads(res.body) @@ -555,6 +561,33 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) + def test_user_delete(self): + name = 'normal_user' + CreateTestData.create_user(name) + user = model.User.get(name) + user_dict = {'id': user.id} + postparams = '%s=1' % json.dumps(user_dict) + + res = self.app.post('/api/action/user_delete', params=postparams, + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}) + + res_obj = json.loads(res.body) + deleted_user = model.User.get(name) + assert res_obj['success'] is True + assert deleted_user.is_deleted(), deleted_user + + def test_user_delete_requires_data_dict_with_key_id(self): + user_dict = {'name': 'normal_user'} + postparams = '%s=1' % json.dumps(user_dict) + + res = self.app.post('/api/action/user_delete', params=postparams, + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + status=StatusCodes.STATUS_409_CONFLICT) + + res_obj = json.loads(res.body) + assert res_obj['success'] is False + assert res_obj['error']['id'] == ['Missing value'] + def test_13_group_list(self): postparams = '%s=1' % json.dumps({}) res = self.app.post('/api/action/group_list', params=postparams) @@ -633,6 +666,11 @@ def test_14_group_show(self): assert res_obj['success'] is False def test_16_user_autocomplete(self): + # Create deleted user to make sure he won't appear in the user_list + deleted_user = CreateTestData.create_user('joe') + deleted_user.delete() + model.repo.commit() + #Empty query postparams = '%s=1' % json.dumps({}) res = self.app.post( diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 3ea9f2db39f..0f0a63372b2 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -48,8 +48,33 @@ def create_user(self, name): self.apikeys[name] = str(json.loads(res.body)['result']['apikey']) -class TestAuthOrgs(TestAuth): +class TestAuthUsers(TestAuth): + def test_only_sysadmins_can_delete_users(self): + username = 'username' + user = {'id': username} + self.create_user(username) + + self._call_api('user_delete', user, username, 403) + self._call_api('user_delete', user, 'sysadmin', 200) + + def test_auth_deleted_users_are_always_unauthorized(self): + always_success = lambda x,y: {'success': True} + new_authz._AuthFunctions._build() + new_authz._AuthFunctions._functions['always_success'] = always_success + # We can't reuse the username with the other tests because we can't + # rebuild_db(), because in the setup_class we get the sysadmin. If we + # rebuild the DB, we would delete the sysadmin as well. + username = 'other_username' + self.create_user(username) + user = model.User.get(username) + user.delete() + + assert not new_authz.is_authorized_boolean('always_success', {'user': username}) + del new_authz._AuthFunctions._functions['always_success'] + + +class TestAuthOrgs(TestAuth): def test_01_create_users(self): # actual roles assigned later self.create_user('org_admin') diff --git a/ckan/tests/models/test_follower.py b/ckan/tests/models/test_follower.py new file mode 100644 index 00000000000..9f4eb51ea3f --- /dev/null +++ b/ckan/tests/models/test_follower.py @@ -0,0 +1,124 @@ +import ckan.model as model +import ckan.lib.create_test_data as ctd + +CreateTestData = ctd.CreateTestData + + +class FollowerClassesTests(object): + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def test_get(self): + following = self.FOLLOWER_CLASS.get(self.follower.id, self.followee.id) + assert following.follower_id == self.follower.id, following + assert following.object_id == self.followee.id, following + + def test_get_returns_none_if_couldnt_find_users(self): + following = self.FOLLOWER_CLASS.get('some-id', 'other-id') + assert following is None, following + + def test_is_following(self): + assert self.FOLLOWER_CLASS.is_following(self.follower.id, + self.followee.id) + + def test_is_following_returns_false_if_user_isnt_following(self): + assert not self.FOLLOWER_CLASS.is_following(self.followee.id, + self.follower.id) + + def test_followee_count(self): + count = self.FOLLOWER_CLASS.followee_count(self.follower.id) + assert count == 1, count + + def test_followee_list(self): + followees = self.FOLLOWER_CLASS.followee_list(self.follower.id) + object_ids = [f.object_id for f in followees] + assert object_ids == [self.followee.id], object_ids + + def test_follower_count(self): + count = self.FOLLOWER_CLASS.follower_count(self.followee.id) + assert count == 1, count + + def test_follower_list(self): + followers = self.FOLLOWER_CLASS.follower_list(self.followee.id) + follower_ids = [f.follower_id for f in followers] + assert follower_ids == [self.follower.id], follower_ids + + +class TestUserFollowingUser(FollowerClassesTests): + FOLLOWER_CLASS = model.UserFollowingUser + + @classmethod + def setup_class(cls): + model.repo.rebuild_db() + cls.follower = CreateTestData.create_user('follower') + cls.followee = CreateTestData.create_user('followee') + cls.FOLLOWER_CLASS(cls.follower.id, cls.followee.id).save() + cls._create_deleted_models() + + @classmethod + def _create_deleted_models(cls): + deleted_user = CreateTestData.create_user('deleted_user') + cls.FOLLOWER_CLASS(deleted_user.id, cls.followee.id).save() + cls.FOLLOWER_CLASS(cls.follower.id, deleted_user.id).save() + deleted_user.delete() + deleted_user.save() + + +class TestUserFollowingDataset(FollowerClassesTests): + FOLLOWER_CLASS = model.UserFollowingDataset + + @classmethod + def setup_class(cls): + model.repo.rebuild_db() + cls.follower = CreateTestData.create_user('follower') + cls.followee = cls._create_dataset('followee') + cls.FOLLOWER_CLASS(cls.follower.id, cls.followee.id).save() + cls._create_deleted_models() + + @classmethod + def _create_deleted_models(cls): + deleted_user = CreateTestData.create_user('deleted_user') + cls.FOLLOWER_CLASS(deleted_user.id, cls.followee.id).save() + deleted_user.delete() + deleted_user.save() + deleted_dataset = cls._create_dataset('deleted_dataset') + cls.FOLLOWER_CLASS(cls.follower.id, deleted_dataset.id).save() + deleted_dataset.delete() + deleted_dataset.save() + + @classmethod + def _create_dataset(self, name): + CreateTestData.create_arbitrary({'name': name}) + return model.Package.get(name) + + +class TestUserFollowingGroup(FollowerClassesTests): + FOLLOWER_CLASS = model.UserFollowingGroup + + @classmethod + def setup_class(cls): + model.repo.rebuild_db() + model.repo.new_revision() + cls.follower = CreateTestData.create_user('follower') + cls.followee = cls._create_group('followee') + cls.FOLLOWER_CLASS(cls.follower.id, cls.followee.id).save() + cls._create_deleted_models() + model.repo.commit_and_remove() + + @classmethod + def _create_deleted_models(cls): + deleted_user = CreateTestData.create_user('deleted_user') + cls.FOLLOWER_CLASS(deleted_user.id, cls.followee.id).save() + deleted_user.delete() + deleted_user.save() + deleted_group = cls._create_group('deleted_group') + cls.FOLLOWER_CLASS(cls.follower.id, deleted_group.id).save() + deleted_group.delete() + deleted_group.save() + + @classmethod + def _create_group(self, name): + group = model.Group(name) + group.save() + return group diff --git a/ckan/tests/models/test_user.py b/ckan/tests/models/test_user.py index b9ef7f0624f..c4b356ae854 100644 --- a/ckan/tests/models/test_user.py +++ b/ckan/tests/models/test_user.py @@ -56,6 +56,13 @@ def test_4_get_openid_missing_slash(self): assert out assert out.fullname == u'Sandra' + def test_is_deleted(self): + user = CreateTestData._create_user_without_commit('a_user') + assert not user.is_deleted(), user + user.delete() + assert user.is_deleted(), user + + def to_names(domain_obj_list): '''Takes a list of domain objects and returns a corresponding list of their names.''' From ad2de00a4e2c2772d1e59faee26ca818731168c9 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 9 Aug 2013 16:07:29 -0300 Subject: [PATCH 016/159] [#1163] Use ckan.model.State instead of vdm.sqlalchemy.State This makes us a bit less tied to vdm. --- ckan/logic/action/get.py | 5 ++--- .../migration/versions/070_add_state_column_to_user_table.py | 4 ++-- ckan/model/follower.py | 4 ++-- ckan/model/group.py | 4 ++-- ckan/model/user.py | 4 +++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a6cec306c91..9ede04d7a39 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -7,7 +7,6 @@ from pylons import config import sqlalchemy -import vdm.sqlalchemy import ckan.lib.dictization import ckan.logic as logic @@ -666,7 +665,7 @@ def user_list(context, data_dict): ) # Filter deleted users - query = query.filter(model.User.state != vdm.sqlalchemy.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) ## hack for pagination if context.get('return_query'): @@ -1178,7 +1177,7 @@ def user_autocomplete(context, data_dict): limit = data_dict.get('limit', 20) query = model.User.search(q) - query = query.filter(model.User.state != vdm.sqlalchemy.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) query = query.limit(limit) user_list = [] diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/070_add_state_column_to_user_table.py index 1dfadbe7d32..46c5c3d4cd5 100644 --- a/ckan/migration/versions/070_add_state_column_to_user_table.py +++ b/ckan/migration/versions/070_add_state_column_to_user_table.py @@ -1,11 +1,11 @@ -import vdm.sqlalchemy +import ckan.model def upgrade(migrate_engine): migrate_engine.execute( ''' ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % vdm.sqlalchemy.State.ACTIVE + ''' % ckan.model.State.ACTIVE ) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index d14ffe667ae..66ac1da435d 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -80,8 +80,8 @@ def _get(cls, follower_id=None, object_id=None): .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ cls.follower_id == follower_alias.id,\ cls.object_id == object_alias.id,\ - follower_alias.state != vdm.sqlalchemy.State.DELETED,\ - object_alias.state != vdm.sqlalchemy.State.DELETED,\ + follower_alias.state != ckan.model.State.DELETED,\ + object_alias.state != ckan.model.State.DELETED,\ object_alias.id == object_id)) return query diff --git a/ckan/model/group.py b/ckan/model/group.py index c16424affee..d5fadab6c2c 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -195,8 +195,8 @@ def packages(self, with_private=False, limit=None, query = meta.Session.query(_package.Package).\ filter( - or_(_package.Package.state == vdm.sqlalchemy.State.ACTIVE, - _package.Package.state == vdm.sqlalchemy.State.PENDING)). \ + or_(_package.Package.state == model.State.ACTIVE, + _package.Package.state == model.State.PENDING)). \ filter(group_table.c.id == self.id).\ filter(member_table.c.state == 'active') diff --git a/ckan/model/user.py b/ckan/model/user.py index d36dd5343cc..82e1acd99b8 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -167,7 +167,9 @@ def number_administered_packages(self): return q.count() def is_deleted(self): - return self.state == vdm.sqlalchemy.State.DELETED + # have to import here to avoid circular imports + import ckan.model as model + return self.state == model.State.DELETED def is_in_group(self, group): return group in self.get_group_ids() From d6fde48e5cd73a2d0754b3ceb9fe6319deedaa20 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 9 Aug 2013 18:40:36 -0300 Subject: [PATCH 017/159] [#1163] Use ckan.model.core.State instead of ckan.model.State With this change, we're able to avoid having to load ckan.model inside methods, to avoid circular dependencies. --- ckan/logic/action/get.py | 5 +++-- .../migration/versions/070_add_state_column_to_user_table.py | 4 ++-- ckan/model/follower.py | 5 +++-- ckan/model/group.py | 4 ++-- ckan/model/user.py | 5 ++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 9ede04d7a39..176096f4481 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -14,6 +14,7 @@ import ckan.logic.schema import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.navl.dictization_functions +import ckan.model.core as core import ckan.model.misc as misc import ckan.plugins as plugins import ckan.lib.search as search @@ -665,7 +666,7 @@ def user_list(context, data_dict): ) # Filter deleted users - query = query.filter(model.User.state != model.State.DELETED) + query = query.filter(model.User.state != core.State.DELETED) ## hack for pagination if context.get('return_query'): @@ -1177,7 +1178,7 @@ def user_autocomplete(context, data_dict): limit = data_dict.get('limit', 20) query = model.User.search(q) - query = query.filter(model.User.state != model.State.DELETED) + query = query.filter(model.User.state != core.State.DELETED) query = query.limit(limit) user_list = [] diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/070_add_state_column_to_user_table.py index 46c5c3d4cd5..367e3a48c32 100644 --- a/ckan/migration/versions/070_add_state_column_to_user_table.py +++ b/ckan/migration/versions/070_add_state_column_to_user_table.py @@ -1,11 +1,11 @@ -import ckan.model +import ckan.model.core def upgrade(migrate_engine): migrate_engine.execute( ''' ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % ckan.model.State.ACTIVE + ''' % ckan.model.core.State.ACTIVE ) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index 66ac1da435d..a8afd07706f 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -3,6 +3,7 @@ import sqlalchemy import vdm.sqlalchemy +import core import ckan.model import domain_object @@ -80,8 +81,8 @@ def _get(cls, follower_id=None, object_id=None): .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ cls.follower_id == follower_alias.id,\ cls.object_id == object_alias.id,\ - follower_alias.state != ckan.model.State.DELETED,\ - object_alias.state != ckan.model.State.DELETED,\ + follower_alias.state != core.State.DELETED,\ + object_alias.state != core.State.DELETED,\ object_alias.id == object_id)) return query diff --git a/ckan/model/group.py b/ckan/model/group.py index d5fadab6c2c..86963983e9a 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -195,8 +195,8 @@ def packages(self, with_private=False, limit=None, query = meta.Session.query(_package.Package).\ filter( - or_(_package.Package.state == model.State.ACTIVE, - _package.Package.state == model.State.PENDING)). \ + or_(_package.Package.state == core.State.ACTIVE, + _package.Package.state == core.State.PENDING)). \ filter(group_table.c.id == self.id).\ filter(member_table.c.state == 'active') diff --git a/ckan/model/user.py b/ckan/model/user.py index 82e1acd99b8..ce882ca2b8f 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -9,6 +9,7 @@ import vdm.sqlalchemy import meta +import core import types as _types import domain_object @@ -167,9 +168,7 @@ def number_administered_packages(self): return q.count() def is_deleted(self): - # have to import here to avoid circular imports - import ckan.model as model - return self.state == model.State.DELETED + return self.state == core.State.DELETED def is_in_group(self, group): return group in self.get_group_ids() From 89a38101afe7b1415d431cdffc013e45bf90bf67 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 12 Aug 2013 17:04:00 -0300 Subject: [PATCH 018/159] [#1178] Add PENDING state to User --- ckan/lib/authenticator.py | 6 +++--- ckan/lib/base.py | 2 +- ckan/model/user.py | 11 +++++++++++ ckan/new_authz.py | 27 ++++++++++++--------------- ckan/tests/lib/test_authenticator.py | 19 +++++++++++++++++++ ckan/tests/logic/test_auth.py | 18 +++++++++++++++++- ckan/tests/models/test_user.py | 26 ++++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 20 deletions(-) diff --git a/ckan/lib/authenticator.py b/ckan/lib/authenticator.py index f64db4f5b8d..cf6ed016694 100644 --- a/ckan/lib/authenticator.py +++ b/ckan/lib/authenticator.py @@ -14,7 +14,7 @@ def authenticate(self, environ, identity): if 'repoze.who.plugins.openid.userid' in identity: openid = identity['repoze.who.plugins.openid.userid'] user = User.by_openid(openid) - if user is None or user.is_deleted(): + if user is None or not user.is_active(): return None else: return user.name @@ -33,8 +33,8 @@ def authenticate(self, environ, identity): if user is None: log.debug('Login failed - username %r not found', login) - elif user.is_deleted(): - log.debug('Login as %r failed - user is deleted', login) + elif not user.is_active(): + log.debug('Login as %r failed - user isn\'t active', login) elif not user.validate_password(identity['password']): log.debug('Login as %r failed - password not valid', login) else: diff --git a/ckan/lib/base.py b/ckan/lib/base.py index bfffcaf2a5d..eabd9200b1d 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -290,7 +290,7 @@ def _identify_user_default(self): if c.user: c.user = c.user.decode('utf8') c.userobj = model.User.by_name(c.user) - if c.userobj is None or c.userobj.is_deleted(): + if c.userobj is None or not c.userobj.is_active(): # This occurs when a user that was still logged in is deleted, # or when you are logged in, clean db # and then restart (or when you change your username) diff --git a/ckan/model/user.py b/ckan/model/user.py index ce882ca2b8f..a6d6c44b612 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -167,9 +167,20 @@ def number_administered_packages(self): q = q.filter_by(user=self, role=model.Role.ADMIN) return q.count() + def activate(self): + ''' Activate the user ''' + self.state = core.State.ACTIVE + + def set_pending(self): + ''' Set the user as pending ''' + self.state = core.State.PENDING + def is_deleted(self): return self.state == core.State.DELETED + def is_pending(self): + return self.state == core.State.PENDING + def is_in_group(self, group): return group in self.get_group_ids() diff --git a/ckan/new_authz.py b/ckan/new_authz.py index bf1d0830d47..b98434a9b45 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -106,12 +106,6 @@ def is_sysadmin(username): return user and user.sysadmin -def is_deleted(username): - ''' Returns True if username is deleted ''' - user = _get_user(username) - return user and user.is_deleted() - - def _get_user(username): ''' Try to get the user from c, if possible, and fallback to using the DB ''' if not username: @@ -154,15 +148,18 @@ def is_authorized(action, context, data_dict=None): auth_function = _AuthFunctions.get(action) if auth_function: username = context.get('user') - # deleted users are always unauthorized - if is_deleted(username): - return {'success': False} - # sysadmins can do anything unless the auth_sysadmins_check - # decorator was used in which case they are treated like all other - # users. - elif is_sysadmin(username): - if not getattr(auth_function, 'auth_sysadmins_check', False): - return {'success': True} + user = _get_user(username) + + if user: + # inactive users are always unauthorized + if not user.is_active(): + return {'success': False} + # sysadmins can do anything unless the auth_sysadmins_check + # decorator was used in which case they are treated like all other + # users. + elif user.sysadmin: + if not getattr(auth_function, 'auth_sysadmins_check', False): + return {'success': True} return auth_function(context, data_dict) else: diff --git a/ckan/tests/lib/test_authenticator.py b/ckan/tests/lib/test_authenticator.py index c7c026112d2..8e7254d94c7 100644 --- a/ckan/tests/lib/test_authenticator.py +++ b/ckan/tests/lib/test_authenticator.py @@ -34,6 +34,15 @@ def test_authenticate_fails_if_user_is_deleted(self): assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_user_is_pending(self): + environ = {} + password = 'somepass' + user = CreateTestData.create_user('a_user', **{'password': password}) + identity = {'login': user.name, 'password': password} + user.set_pending() + + assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_password_is_wrong(self): environ = {} user = CreateTestData.create_user('a_user') @@ -100,6 +109,16 @@ def test_authenticate_fails_if_user_is_deleted(self): assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_user_is_deleted(self): + environ = {} + openid = 'some-openid-key' + user = CreateTestData.create_user('a_user', **{'openid': openid}) + user.set_pending() + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_user_have_no_openid(self): environ = {} identity = {} diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 0f0a63372b2..2b5204e878a 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -64,7 +64,7 @@ def test_auth_deleted_users_are_always_unauthorized(self): # We can't reuse the username with the other tests because we can't # rebuild_db(), because in the setup_class we get the sysadmin. If we # rebuild the DB, we would delete the sysadmin as well. - username = 'other_username' + username = 'deleted_user' self.create_user(username) user = model.User.get(username) user.delete() @@ -73,6 +73,22 @@ def test_auth_deleted_users_are_always_unauthorized(self): del new_authz._AuthFunctions._functions['always_success'] + def test_auth_pending_users_are_always_unauthorized(self): + always_success = lambda x,y: {'success': True} + new_authz._AuthFunctions._build() + new_authz._AuthFunctions._functions['always_success'] = always_success + # We can't reuse the username with the other tests because we can't + # rebuild_db(), because in the setup_class we get the sysadmin. If we + # rebuild the DB, we would delete the sysadmin as well. + username = 'pending_user' + self.create_user(username) + user = model.User.get(username) + user.state = model.State.PENDING + + assert not new_authz.is_authorized_boolean('always_success', {'user': username}) + + del new_authz._AuthFunctions._functions['always_success'] + class TestAuthOrgs(TestAuth): def test_01_create_users(self): diff --git a/ckan/tests/models/test_user.py b/ckan/tests/models/test_user.py index c4b356ae854..bd34641b936 100644 --- a/ckan/tests/models/test_user.py +++ b/ckan/tests/models/test_user.py @@ -58,10 +58,36 @@ def test_4_get_openid_missing_slash(self): def test_is_deleted(self): user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' assert not user.is_deleted(), user user.delete() assert user.is_deleted(), user + def test_user_is_active_by_default(self): + user = CreateTestData._create_user_without_commit('a_user') + assert user.is_active(), user + + def test_activate(self): + user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' + assert not user.is_active(), user + user.activate() + assert user.is_active(), user + + def test_activate(self): + user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' + assert not user.is_active(), user + user.activate() + assert user.is_active(), user + + def test_is_pending(self): + user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' + assert not user.is_pending(), user + user.set_pending() + assert user.is_pending(), user + def to_names(domain_obj_list): '''Takes a list of domain objects and returns a corresponding list From c0c6803b574d48016af6185fd0a2a71605567a77 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 14 Aug 2013 18:13:36 -0300 Subject: [PATCH 019/159] [#1178] The user is activated when it performs the reset password --- ckan/controllers/user.py | 1 + ckan/tests/functional/test_user.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index e0b1ba6c762..4e76b098d37 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -479,6 +479,7 @@ def perform_reset(self, id): new_password = self._get_form_password() user_dict['password'] = new_password user_dict['reset_key'] = c.reset_key + user_dict['state'] = model.State.ACTIVE user = get_action('user_update')(context, user_dict) h.flash_success(_("Your password has been reset.")) diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index 16795561f46..f0ff38cfdac 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -955,3 +955,21 @@ def test_perform_reset_user_password_link_user_incorrect(self): id='randomness', # i.e. incorrect key='randomness') res = self.app.get(offset, status=404) + + def test_perform_reset_activates_pending_user(self): + password = 'password' + params = { 'password1': password, 'password2': password } + user = CreateTestData.create_user(name='username', + email='user@email.com') + user.set_pending() + create_reset_key(user) + assert user.is_pending(), user.state + + offset = url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key) + res = self.app.post(offset, params=params, status=302) + + user = model.User.get(user.id) + assert user.is_active(), user From 74f649c9e3eb0690f5e48f939b4546b415b0777b Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 14 Aug 2013 21:42:22 -0300 Subject: [PATCH 020/159] [#1178] Admins can invite users The invited user starts in pending state, with the password reset key set. We still have to send an email to the user telling him/her to change the password and log in. I had to change authorization code to only automatically unauthorize deleted users, not pending. This was because the users needs to be able to perform the password reset when pending, to be able to become active. --- ckan/controllers/user.py | 2 +- ckan/logic/action/create.py | 37 ++++++++++++++++++++++ ckan/logic/auth/create.py | 2 ++ ckan/logic/schema.py | 1 + ckan/new_authz.py | 4 +-- ckan/tests/functional/test_user.py | 20 +++++++++++- ckan/tests/logic/test_action.py | 50 ++++++++++++++++++++++++++++++ ckan/tests/logic/test_auth.py | 22 ++++--------- dev-requirements.txt | 1 + 9 files changed, 119 insertions(+), 20 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 4e76b098d37..000f3db5a0c 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -447,7 +447,7 @@ def perform_reset(self, id): # FIXME We should reset the reset key when it is used to prevent # reuse of the url context = {'model': model, 'session': model.Session, - 'user': c.user, + 'user': c.user or id, 'keep_sensitive_data': True} data_dict = {'id': id} diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 1fc0fb6e058..346fd30e680 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1,6 +1,8 @@ '''API functions for adding data to CKAN.''' import logging +import random +import re from pylons import config import paste.deploy.converters @@ -15,6 +17,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions +import ckan.lib.navl.validators as validators from ckan.common import _ @@ -836,6 +839,40 @@ def user_create(context, data_dict): log.debug('Created user {name}'.format(name=user.name)) return user_dict +def user_invite(context, data_dict): + '''docstring''' + _check_access('user_invite', context, data_dict) + + user_invite_schema = { + 'email': [validators.not_empty, unicode] + } + _, errors = _validate(data_dict, user_invite_schema, context) + if errors: + raise ValidationError(errors) + + while True: + try: + import ckan.lib.mailer + name = _get_random_username_from_email(data_dict['email']) + password = str(random.SystemRandom().random()) + data_dict['name'] = name + data_dict['password'] = password + data_dict['state'] = ckan.model.State.PENDING + user_dict = _get_action('user_create')(context, data_dict) + user = ckan.model.User.get(user_dict['id']) + ckan.lib.mailer.create_reset_key(user) + return model_dictize.user_dictize(user, context) + except ValidationError as e: + if 'name' not in e.error_dict: + raise e + +def _get_random_username_from_email(email): + localpart = email.split('@')[0] + cleaned_localpart = re.sub(r'[^\w]', '', localpart) + random_number = random.SystemRandom().random() * 10000 + name = '%s-%d' % (cleaned_localpart, random_number) + return name + ## Modifications for rest api def package_create_rest(context, data_dict): diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index bf9c3d17ea3..fafbc5a2e4c 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -112,6 +112,8 @@ def user_create(context, data_dict=None): else: return {'success': True} +def user_invite(context, data_dict=None): + return {'success': False} def _check_group_auth(context, data_dict): # FIXME This code is shared amoung other logic.auth files and should be diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..a8cd4ce6779 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -395,6 +395,7 @@ def default_user_schema(): 'apikey': [ignore], 'reset_key': [ignore], 'activity_streams_email_notifications': [ignore_missing], + 'state': [ignore_missing], } return schema diff --git a/ckan/new_authz.py b/ckan/new_authz.py index b98434a9b45..62a4e88a978 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -151,8 +151,8 @@ def is_authorized(action, context, data_dict=None): user = _get_user(username) if user: - # inactive users are always unauthorized - if not user.is_active(): + # deleted users are always unauthorized + if user.is_deleted(): return {'success': False} # sysadmins can do anything unless the auth_sysadmins_check # decorator was used in which case they are treated like all other diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index f0ff38cfdac..fe30f53a505 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -959,7 +959,7 @@ def test_perform_reset_user_password_link_user_incorrect(self): def test_perform_reset_activates_pending_user(self): password = 'password' params = { 'password1': password, 'password2': password } - user = CreateTestData.create_user(name='username', + user = CreateTestData.create_user(name='pending_user', email='user@email.com') user.set_pending() create_reset_key(user) @@ -973,3 +973,21 @@ def test_perform_reset_activates_pending_user(self): user = model.User.get(user.id) assert user.is_active(), user + + def test_perform_reset_doesnt_activate_deleted_user(self): + password = 'password' + params = { 'password1': password, 'password2': password } + user = CreateTestData.create_user(name='deleted_user', + email='user@email.com') + user.delete() + create_reset_key(user) + assert user.is_deleted(), user.state + + offset = url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key) + res = self.app.post(offset, params=params, status=302) + + user = model.User.get(user.id) + assert user.is_deleted(), user diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index fecbdf96b53..7bf17bd094e 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -6,6 +6,7 @@ from nose.plugins.skip import SkipTest from pylons import config import datetime +import mock import vdm.sqlalchemy import ckan @@ -561,6 +562,55 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) + def test_user_invite(self): + email_username = 'invited_user$ckan' + email = '%s@email.com' % email_username + user_dict = {'email': email} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ) + + res_obj = json.loads(res.body) + user = model.User.get(res_obj['result']['id']) + expected_username = email_username.replace('$', '') + assert res_obj['success'] is True, res_obj + assert user.email == email, (user.email, email) + assert user.name.startswith(expected_username), (user.name, expected_username) + assert user.is_pending(), user + assert user.reset_key is not None, user + + def test_user_invite_without_email_raises_error(self): + user_dict = {} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ, + status=StatusCodes.STATUS_409_CONFLICT) + + res_obj = json.loads(res.body) + assert res_obj['success'] is False, res_obj + assert 'email' in res_obj['error'], res_obj + + @mock.patch('ckan.logic.action.create._get_random_username_from_email') + def test_user_invite_should_work_even_if_tried_username_already_exists(self, random_username_mock): + email = 'invited_user@email.com' + user_dict = {'email': email} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + usernames = ['first', 'first', 'second'] + random_username_mock.side_effect = lambda email: usernames.pop(0) + + for _ in range(2): + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ) + + res_obj = json.loads(res.body) + assert res_obj['success'] is True, res_obj + def test_user_delete(self): name = 'normal_user' CreateTestData.create_user(name) diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 2b5204e878a..13e7b38f2a5 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -49,6 +49,12 @@ def create_user(self, name): class TestAuthUsers(TestAuth): + def test_only_sysadmins_can_invite_users(self): + username = 'normal_user' + self.create_user(username) + + assert not new_authz.is_authorized_boolean('user_invite', {'user': username}) + def test_only_sysadmins_can_delete_users(self): username = 'username' user = {'id': username} @@ -73,22 +79,6 @@ def test_auth_deleted_users_are_always_unauthorized(self): del new_authz._AuthFunctions._functions['always_success'] - def test_auth_pending_users_are_always_unauthorized(self): - always_success = lambda x,y: {'success': True} - new_authz._AuthFunctions._build() - new_authz._AuthFunctions._functions['always_success'] = always_success - # We can't reuse the username with the other tests because we can't - # rebuild_db(), because in the setup_class we get the sysadmin. If we - # rebuild the DB, we would delete the sysadmin as well. - username = 'pending_user' - self.create_user(username) - user = model.User.get(username) - user.state = model.State.PENDING - - assert not new_authz.is_authorized_boolean('always_success', {'user': username}) - - del new_authz._AuthFunctions._functions['always_success'] - class TestAuthOrgs(TestAuth): def test_01_create_users(self): diff --git a/dev-requirements.txt b/dev-requirements.txt index a445712d239..a781573b0b8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,3 +7,4 @@ nose==1.3.0 pep8==1.4.6 Sphinx==1.2b1 polib==1.0.3 +mock==1.0.1 From 31523cd27a35958c2f0660a2762bcfd0274c8006 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 15 Aug 2013 11:49:03 -0300 Subject: [PATCH 021/159] [#1178] perform_reset uses the received id as the context's user When performing a password reset, the user is probably (always?) not logged in. So c.user is an empty string. So, the auth functions have no way to tell which user is trying to reset his/her password. This worked fine before, because everyone was able to reset the password. But now that we've got users in DELETED state, it's not the case anymore. --- ckan/controllers/user.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 000f3db5a0c..98033d9a0ab 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -447,17 +447,16 @@ def perform_reset(self, id): # FIXME We should reset the reset key when it is used to prevent # reuse of the url context = {'model': model, 'session': model.Session, - 'user': c.user or id, + 'user': id, 'keep_sensitive_data': True} - data_dict = {'id': id} - try: check_access('user_reset', context) except NotAuthorized: abort(401, _('Unauthorized to reset password.')) try: + data_dict = {'id': id} user_dict = get_action('user_show')(context, data_dict) # Be a little paranoid, and get rid of sensitive data that's From e5890740164c38ca0ebf10bf25a297b0b634e204 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 16 Aug 2013 10:39:25 -0300 Subject: [PATCH 022/159] [#1178] Send email to the invited user I removed the time.sleep(0.1) on TestMailer. Looking through the code, I couldn't find anywhere where a timer looked needed. And I ran these tests a hundred times without the timer to see if I could make them fail, but no. So, I guess they're not needed anymore. I also had to move the RESET_LINK_MESSAGE inside get_reset_link_body(). This was because, when importing ckan.lib.mailer in ckan.logic.action.create.py, I got: TypeError: No object (name: translator) has been registered for this thread This seems to be because we were using _() before pylons had a change to set up the translator. Moving it inside the method solves this. --- ckan/lib/mailer.py | 59 +++++++++++++++++++++++---------- ckan/logic/action/create.py | 18 +++++++--- ckan/tests/lib/test_mailer.py | 37 ++++++++++++++------- ckan/tests/logic/test_action.py | 28 ++++++++++++++-- 4 files changed, 105 insertions(+), 37 deletions(-) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index f0e1978ac2c..72ebe33e2e0 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -100,21 +100,37 @@ def mail_user(recipient, subject, body, headers={}): mail_recipient(recipient.display_name, recipient.email, subject, body, headers=headers) +def get_reset_link_body(user): + reset_link_message = _( + '''You have requested your password on %(site_title)s to be reset. -RESET_LINK_MESSAGE = _( -'''You have requested your password on %(site_title)s to be reset. + Please click the following link to confirm this request: -Please click the following link to confirm this request: + %(reset_link)s + ''') - %(reset_link)s -''') + d = { + 'reset_link': get_reset_link(user), + 'site_title': g.site_title + } + return reset_link_message % d -def make_key(): - return uuid.uuid4().hex[:10] +def get_invite_body(user): + invite_message = _( + '''You have been invited to %(site_title)s. A user has already been created to + you with the username %(user_name)s. You can change it later. -def create_reset_key(user): - user.reset_key = unicode(make_key()) - model.repo.commit_and_remove() + To accept this invite, please reset your password at: + + %(reset_link)s + ''') + + d = { + 'reset_link': get_reset_link(user), + 'site_title': g.site_title, + 'user_name': user.name, + } + return invite_message % d def get_reset_link(user): return urljoin(g.site_url, @@ -123,17 +139,24 @@ def get_reset_link(user): id=user.id, key=user.reset_key)) -def get_reset_link_body(user): - d = { - 'reset_link': get_reset_link(user), - 'site_title': g.site_title - } - return RESET_LINK_MESSAGE % d - def send_reset_link(user): create_reset_key(user) body = get_reset_link_body(user) - mail_user(user, _('Reset your password'), body) + subject = _('Reset your password') + mail_user(user, subject, body) + +def send_invite(user): + create_reset_key(user) + body = get_invite_body(user) + subject = _('Invite for {site_title}'.format(site_title=g.site_title)) + mail_user(user, subject, body) + +def create_reset_key(user): + user.reset_key = unicode(make_key()) + model.repo.commit_and_remove() + +def make_key(): + return uuid.uuid4().hex[:10] def verify_reset_link(user, key): if not key: diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 346fd30e680..edef0a95f30 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -18,6 +18,7 @@ import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions import ckan.lib.navl.validators as validators +import ckan.lib.mailer as mailer from ckan.common import _ @@ -840,7 +841,17 @@ def user_create(context, data_dict): return user_dict def user_invite(context, data_dict): - '''docstring''' + '''Invite a new user. + + You must be authorized to invite users. + + :param email: the email address for the new user + :type email: string + + :returns: the newly created yser + :rtype: dictionary + + ''' _check_access('user_invite', context, data_dict) user_invite_schema = { @@ -852,7 +863,6 @@ def user_invite(context, data_dict): while True: try: - import ckan.lib.mailer name = _get_random_username_from_email(data_dict['email']) password = str(random.SystemRandom().random()) data_dict['name'] = name @@ -860,7 +870,7 @@ def user_invite(context, data_dict): data_dict['state'] = ckan.model.State.PENDING user_dict = _get_action('user_create')(context, data_dict) user = ckan.model.User.get(user_dict['id']) - ckan.lib.mailer.create_reset_key(user) + mailer.send_invite(user) return model_dictize.user_dictize(user, context) except ValidationError as e: if 'name' not in e.error_dict: @@ -868,7 +878,7 @@ def user_invite(context, data_dict): def _get_random_username_from_email(email): localpart = email.split('@')[0] - cleaned_localpart = re.sub(r'[^\w]', '', localpart) + cleaned_localpart = re.sub(r'[^\w]', '-', localpart) random_number = random.SystemRandom().random() * 10000 name = '%s-%d' % (cleaned_localpart, random_number) return name diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index b95d4d290e3..22b3985930b 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -1,13 +1,12 @@ -import time from nose.tools import assert_equal, assert_raises from pylons import config from email.mime.text import MIMEText import hashlib -from ckan import model +import ckan.model as model +import ckan.lib.mailer as mailer from ckan.tests.pylons_controller import PylonsTestCase from ckan.tests.mock_mail_server import SmtpServerHarness -from ckan.lib.mailer import mail_recipient, mail_user, send_reset_link, add_msg_niceties, MailerException, get_reset_link_body, get_reset_link from ckan.lib.create_test_data import CreateTestData from ckan.lib.base import g @@ -35,7 +34,7 @@ def setup(self): def mime_encode(self, msg, recipient_name): sender_name = g.site_title sender_url = g.site_url - body = add_msg_niceties(recipient_name, msg, sender_name, sender_url) + body = mailer.add_msg_niceties(recipient_name, msg, sender_name, sender_url) encoded_body = MIMEText(body.encode('utf-8'), 'plain', 'utf-8').get_payload().strip() return encoded_body @@ -49,8 +48,7 @@ def test_mail_recipient(self): 'subject': 'Meeting', 'body': 'The meeting is cancelled.', 'headers': {'header1': 'value1'}} - mail_recipient(**test_email) - time.sleep(0.1) + mailer.mail_recipient(**test_email) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -74,8 +72,7 @@ def test_mail_user(self): 'subject': 'Meeting', 'body': 'The meeting is cancelled.', 'headers': {'header1': 'value1'}} - mail_user(**test_email) - time.sleep(0.1) + mailer.mail_user(**test_email) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -96,12 +93,11 @@ def test_mail_user_without_email(self): 'subject': 'Meeting', 'body': 'The meeting is cancelled.', 'headers': {'header1': 'value1'}} - assert_raises(MailerException, mail_user, **test_email) + assert_raises(mailer.MailerException, mailer.mail_user, **test_email) def test_send_reset_email(self): # send email - send_reset_link(model.User.by_name(u'bob')) - time.sleep(0.1) + mailer.send_reset_link(model.User.by_name(u'bob')) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -110,7 +106,24 @@ def test_send_reset_email(self): assert_equal(msg[1], config['smtp.mail_from']) assert_equal(msg[2], [model.User.by_name(u'bob').email]) assert 'Reset' in msg[3], msg[3] - test_msg = get_reset_link_body(model.User.by_name(u'bob')) + test_msg = mailer.get_reset_link_body(model.User.by_name(u'bob')) + expected_body = self.mime_encode(test_msg, + u'bob') + assert expected_body in msg[3], '%r not in %r' % (expected_body, msg[3]) + + # reset link tested in user functional test + + def test_send_invite_email(self): + # send email + mailer.send_invite(model.User.by_name(u'bob')) + + # check it went to the mock smtp server + msgs = self.get_smtp_messages() + assert_equal(len(msgs), 1) + msg = msgs[0] + assert_equal(msg[1], config['smtp.mail_from']) + assert_equal(msg[2], [model.User.by_name(u'bob').email]) + test_msg = mailer.get_invite_body(model.User.by_name(u'bob')) expected_body = self.mime_encode(test_msg, u'bob') assert expected_body in msg[3], '%r not in %r' % (expected_body, msg[3]) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 7bf17bd094e..b6361803f0e 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -562,7 +562,8 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) - def test_user_invite(self): + @mock.patch('ckan.lib.mailer.mail_user') + def test_user_invite(self, mail_user): email_username = 'invited_user$ckan' email = '%s@email.com' % email_username user_dict = {'email': email} @@ -574,14 +575,32 @@ def test_user_invite(self): res_obj = json.loads(res.body) user = model.User.get(res_obj['result']['id']) - expected_username = email_username.replace('$', '') + expected_username = email_username.replace('$', '-') assert res_obj['success'] is True, res_obj assert user.email == email, (user.email, email) assert user.name.startswith(expected_username), (user.name, expected_username) assert user.is_pending(), user assert user.reset_key is not None, user - def test_user_invite_without_email_raises_error(self): + @mock.patch('ckan.lib.mailer.send_invite') + def test_user_invite_sends_email(self, send_invite): + email_username = 'invited_user' + email = '%s@email.com' % email_username + user_dict = {'email': email} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ) + + res_obj = json.loads(res.body) + user = model.User.get(res_obj['result']['id']) + assert res_obj['success'] is True, res_obj + assert send_invite.called + assert send_invite.call_args[0][0].id == res_obj['result']['id'] + + @mock.patch('ckan.lib.mailer.mail_user') + def test_user_invite_without_email_raises_error(self, mail_user): user_dict = {} postparams = '%s=1' % json.dumps(user_dict) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} @@ -596,6 +615,8 @@ def test_user_invite_without_email_raises_error(self): @mock.patch('ckan.logic.action.create._get_random_username_from_email') def test_user_invite_should_work_even_if_tried_username_already_exists(self, random_username_mock): + patcher = mock.patch('ckan.lib.mailer.mail_user') + patcher.start() email = 'invited_user@email.com' user_dict = {'email': email} postparams = '%s=1' % json.dumps(user_dict) @@ -610,6 +631,7 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran res_obj = json.loads(res.body) assert res_obj['success'] is True, res_obj + patcher.stop() def test_user_delete(self): name = 'normal_user' From 85c50e18c43cccc69446aa977b082de0411a0560 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 20 Aug 2013 11:49:31 +0200 Subject: [PATCH 023/159] [#938] Validate datapusher_submit with a schema --- ckanext/datapusher/logic/action.py | 10 +++++++--- ckanext/datapusher/logic/schema.py | 25 +++++++++++++++++++++++++ ckanext/datapusher/plugin.py | 3 ++- ckanext/datastore/logic/action.py | 2 +- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 ckanext/datapusher/logic/schema.py diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 5957887d44e..47a255778f7 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -9,6 +9,7 @@ import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p +import ckanext.datapusher.logic.schema as dpschema log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust @@ -40,9 +41,12 @@ def datapusher_submit(context, data_dict): :rtype: boolean ''' - if 'id' in data_dict: - data_dict['resource_id'] = data_dict['id'] - res_id = _get_or_bust(data_dict, 'resource_id') + schema = context.get('schema', dpschema.datapusher_submit_schema()) + data_dict, errors = _validate(data_dict, schema, context) + if errors: + raise p.toolkit.ValidationError(errors) + + res_id = data_dict['resource_id'] p.toolkit.check_access('datapusher_submit', context, data_dict) diff --git a/ckanext/datapusher/logic/schema.py b/ckanext/datapusher/logic/schema.py new file mode 100644 index 00000000000..07e8a36aec8 --- /dev/null +++ b/ckanext/datapusher/logic/schema.py @@ -0,0 +1,25 @@ +import ckan.plugins as p +import ckanext.datastore.logic.schema as dsschema + +get_validator = p.toolkit.get_validator + +not_missing = get_validator('not_missing') +not_empty = get_validator('not_empty') +resource_id_exists = get_validator('resource_id_exists') +package_id_exists = get_validator('package_id_exists') +ignore_missing = get_validator('ignore_missing') +empty = get_validator('empty') +boolean_validator = get_validator('boolean_validator') +int_validator = get_validator('int_validator') +OneOf = get_validator('OneOf') + + +def datapusher_submit_schema(): + schema = { + 'resource_id': [not_missing, not_empty, unicode], + 'id': [ignore_missing], + 'set_url_type': [ignore_missing, boolean_validator], + '__junk': [empty], + '__before': [dsschema.rename('id', 'resource_id')] + } + return schema diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 17f9022fd22..650282ea9e1 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -47,7 +47,8 @@ def notify(self, entity, operation=None): 'id': entity.get_package_id() }) if (not package['private'] and - entity.format in self.datapusher_formats): + entity.format in self.datapusher_formats and + entity.url_type != 'datapusher'): p.toolkit.get_action('datapusher_submit')(context, { 'resource_id': entity.id }) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 9e93deed315..23dae4dc5b0 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -99,7 +99,7 @@ def datastore_create(context, data_dict): if has_url: p.toolkit.get_action('datapusher_submit')(context, { 'resource_id': res['id'], - 'set_url_to_dump': True + 'set_url_type': True }) # create empty resource else: From 09bde044490fe73c776064cad2fbb26e41652617 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 20 Aug 2013 16:02:02 +0200 Subject: [PATCH 024/159] [#1200] Imports and comments clean up --- ckanext/datapusher/logic/action.py | 16 ++++++++-------- ckanext/datastore/logic/action.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 47a255778f7..1fe1d5abb8b 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -17,7 +17,7 @@ def datapusher_enabled(context, data_dict): - """ Returns True if the DataPusher is configured """ + ''' Returns True if the DataPusher is configured ''' datapusher_url = pylons.config.get('datapusher.url') return True if datapusher_url else False @@ -30,15 +30,15 @@ def datapusher_submit(context, data_dict): :param resource_id: The resource id of the resource that the data should be imported in. The resource's URL will be used to get the data. :type resource_id: string - :param set_url_type: If set to true, the ``url_type`` of the resource will + :param set_url_type: If set to True, the ``url_type`` of the resource will be set to ``datastore`` and the resource URL will automatically point to the :ref:`datastore dump ` URL. (optional, default: False) - :type set_url_type: boolean + :type set_url_type: bool Returns ``True`` if the job has been submitted and ``False`` if the job has not been submitted, i.e. when the datapusher is not configured. - :rtype: boolean + :rtype: bool ''' schema = context.get('schema', dpschema.datapusher_submit_schema()) @@ -113,11 +113,11 @@ def datapusher_submit(context, data_dict): def datapusher_hook(context, data_dict): - """ Update datapusher task. This action is typically called by the + ''' Update datapusher task. This action is typically called by the datapusher whenever the status of a job changes. Expects a job with ``status`` and ``metadata`` with a ``resource_id``. - """ + ''' # TODO: use a schema to validate @@ -147,12 +147,12 @@ def datapusher_hook(context, data_dict): def datapusher_status(context, data_dict): - """ Get the status of a datapusher job for a certain resource. + ''' Get the status of a datapusher job for a certain resource. :param resource_id: The resource id of the resource that you want the datapusher status for. :type resource_id: string - """ + ''' p.toolkit.check_access('datapusher_status', context, data_dict) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 23dae4dc5b0..489616f49dc 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -1,10 +1,6 @@ import logging -import json -import urlparse -import datetime import pylons -import requests import sqlalchemy import ckan.lib.navl.dictization_functions @@ -101,11 +97,16 @@ def datastore_create(context, data_dict): 'resource_id': res['id'], 'set_url_type': True }) + # since we'll overwrite the datastore resource anyway, we + # don't need to create it here + return # create empty resource else: # no need to set the full url because it will be set in before_show res['url_type'] = 'datastore' p.toolkit.get_action('resource_update')(context, res) + else: + _check_read_only(context, data_dict) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] From d8a9f37bab6d1db3401117690200c9bf7476437a Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 20 Aug 2013 16:03:13 +0200 Subject: [PATCH 025/159] [#1200] Do not allow writing if url_type != datastore unless force=true is passed --- ckanext/datastore/logic/action.py | 35 ++++++++++++++++++++++---- ckanext/datastore/logic/schema.py | 3 +++ ckanext/datastore/tests/helpers.py | 11 ++++++++ ckanext/datastore/tests/test_create.py | 4 ++- ckanext/datastore/tests/test_delete.py | 4 ++- ckanext/datastore/tests/test_dump.py | 1 + ckanext/datastore/tests/test_search.py | 13 +++++++--- ckanext/datastore/tests/test_upsert.py | 8 +++++- 8 files changed, 68 insertions(+), 11 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 489616f49dc..ee2a44e9e4a 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -35,6 +35,8 @@ def datastore_create(context, data_dict): :param resource_id: resource id that the data is going to be stored against. :type resource_id: string + :param force: set to True to edit a read-only resource + :type force: bool (optional, default: False) :param resource: resource dictionary that is passed to :meth:`~ckan.logic.action.create.resource_create`. Use instead of ``resource_id`` (optional) @@ -154,6 +156,8 @@ def datastore_upsert(context, data_dict): :param resource_id: resource id that the data is going to be stored under. :type resource_id: string + :param force: set to True to edit a read-only resource + :type force: bool (optional, default: False) :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional) :type records: list of dictionaries :param method: the method to use to put the data into the datastore. @@ -174,6 +178,10 @@ def datastore_upsert(context, data_dict): if errors: raise p.toolkit.ValidationError(errors) + p.toolkit.check_access('datastore_upsert', context, data_dict) + + _check_read_only(context, data_dict) + data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] @@ -187,8 +195,6 @@ def datastore_upsert(context, data_dict): u'Resource "{0}" was not found.'.format(res_id) )) - p.toolkit.check_access('datastore_upsert', context, data_dict) - result = db.upsert(context, data_dict) result.pop('id', None) result.pop('connection_url') @@ -200,6 +206,8 @@ def datastore_delete(context, data_dict): :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string + :param force: set to True to edit a read-only resource + :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary @@ -218,6 +226,10 @@ def datastore_delete(context, data_dict): if errors: raise p.toolkit.ValidationError(errors) + p.toolkit.check_access('datastore_delete', context, data_dict) + + _check_read_only(context, data_dict) + data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] @@ -231,8 +243,6 @@ def datastore_delete(context, data_dict): u'Resource "{0}" was not found.'.format(res_id) )) - p.toolkit.check_access('datastore_delete', context, data_dict) - result = db.delete(context, data_dict) result.pop('id', None) result.pop('connection_url') @@ -431,7 +441,7 @@ def datastore_make_public(context, data_dict): def _resource_exists(context, data_dict): - # Returns true if the resource exists in CKAN and in the datastore + ''' Returns true if the resource exists in CKAN and in the datastore ''' model = _get_or_bust(context, 'model') res_id = _get_or_bust(data_dict, 'resource_id') if not model.Resource.get(res_id): @@ -441,3 +451,18 @@ def _resource_exists(context, data_dict): WHERE name = :id AND alias_of IS NULL''') results = db._get_engine(data_dict).execute(resources_sql, id=res_id) return results.rowcount > 0 + + +def _check_read_only(context, data_dict): + ''' Raises exception if the resource is read-only. + Make sure the resource id is in resource_id + ''' + if data_dict.get('force'): + return + res = p.toolkit.get_action('resource_show')( + context, {'id': data_dict['resource_id']}) + if res.get('url_type') != 'datastore': + raise p.toolkit.ValidationError({ + 'read-only': ['Cannot edit read-only resource. Either pass' + '"force=True" or change url-type to "datastore"'] + }) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 018eb249d79..6f0a74723ca 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -68,6 +68,7 @@ def json_validator(value, context): def datastore_create_schema(): schema = { 'resource_id': [ignore_missing, unicode, resource_id_exists], + 'force': [ignore_missing, boolean_validator], 'id': [ignore_missing], 'aliases': [ignore_missing, list_of_strings_or_string], 'fields': { @@ -85,6 +86,7 @@ def datastore_create_schema(): def datastore_upsert_schema(): schema = { 'resource_id': [not_missing, not_empty, unicode], + 'force': [ignore_missing, boolean_validator], 'id': [ignore_missing], 'method': [ignore_missing, unicode, OneOf( ['upsert', 'insert', 'update'])], @@ -97,6 +99,7 @@ def datastore_upsert_schema(): def datastore_delete_schema(): schema = { 'resource_id': [not_missing, not_empty, unicode], + 'force': [ignore_missing, boolean_validator], 'id': [ignore_missing], '__junk': [empty], '__before': [rename('id', 'resource_id')] diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py index cf83f57378a..3ee89cdda20 100644 --- a/ckanext/datastore/tests/helpers.py +++ b/ckanext/datastore/tests/helpers.py @@ -1,6 +1,8 @@ import ckan.model as model import ckan.lib.cli as cli +import ckan.plugins as p + def extract(d, keys): return dict((k, d[k]) for k in keys if k in d) @@ -29,3 +31,12 @@ def rebuild_all_dbs(Session): model.repo.tables_created_and_initialised = False clear_db(Session) model.repo.rebuild_db() + + +def set_url_type(resources, user): + context = {'user': user.name} + for resource in resources: + resource = p.toolkit.get_action('resource_show')( + context, {'id': resource.id}) + resource['url_type'] = 'datastore' + p.toolkit.get_action('resource_update')(context, resource) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 7a6933008e1..f95e8044951 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -17,7 +17,7 @@ import ckan.config.middleware as middleware import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type # avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 @@ -45,6 +45,8 @@ def setup_class(cls): engine = db._get_engine( {'connection_url': pylons.config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) @classmethod def teardown_class(cls): diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index ce1b02efdb1..bb7db39217a 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -11,7 +11,7 @@ import ckan.tests as tests import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type class TestDatastoreDelete(tests.WsgiAppCase): @@ -43,6 +43,8 @@ def setup_class(cls): engine = db._get_engine( {'connection_url': pylons.config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) @classmethod def teardown_class(cls): diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 041105af986..6061535f4ba 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -32,6 +32,7 @@ def setup_class(cls): resource = model.Package.get('annakarenina').resources[0] cls.data = { 'resource_id': resource.id, + 'force': True, 'aliases': 'books', 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index 21cb19de4e2..1b7c9486da7 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -30,6 +30,7 @@ def setup_class(cls): cls.resource = cls.dataset.resources[0] cls.data = { 'resource_id': cls.resource.id, + 'force': True, 'aliases': 'books3', 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, @@ -116,7 +117,7 @@ def test_search_private_dataset(self): context, {'name': 'privatedataset', 'private': True, - 'owner_org' : self.organization['id'], + 'owner_org': self.organization['id'], 'groups': [{ 'id': group.id }]}) @@ -128,6 +129,7 @@ def test_search_private_dataset(self): postparams = '%s=1' % json.dumps({ 'resource_id': resource['id'], + 'force': True }) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, @@ -425,6 +427,7 @@ def setup_class(cls): resource = model.Package.get('annakarenina').resources[0] cls.data = dict( resource_id=resource.id, + force=True, fields=[ {'id': 'id'}, {'id': 'date', 'type':'date'}, @@ -499,6 +502,7 @@ def setup_class(cls): resource = cls.dataset.resources[0] cls.data = { 'resource_id': resource.id, + 'force': True, 'aliases': 'books4', 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, @@ -517,7 +521,7 @@ def setup_class(cls): extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True - + # Make an organization, because private datasets must belong to one. cls.organization = tests.call_action_api( cls.app, 'organization_create', @@ -669,6 +673,7 @@ def test_new_datastore_table_from_private_resource(self): postparams = '%s=1' % json.dumps({ 'resource_id': resource['id'], + 'force': True }) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, @@ -708,7 +713,9 @@ def test_making_resource_private_makes_datastore_private(self): 'package_id': package['id']}) postparams = '%s=1' % json.dumps({ - 'resource_id': resource['id']}) + 'resource_id': resource['id'], + 'force': True + }) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, extra_environ=auth) diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index e27f8b9c523..9d65700198a 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -11,7 +11,7 @@ import ckan.tests as tests import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type class TestDatastoreUpsert(tests.WsgiAppCase): @@ -26,6 +26,8 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) resource = model.Package.get('annakarenina').resources[0] cls.data = { 'resource_id': resource.id, @@ -249,6 +251,8 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) resource = model.Package.get('annakarenina').resources[0] cls.data = { 'resource_id': resource.id, @@ -349,6 +353,8 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) resource = model.Package.get('annakarenina').resources[0] hhguide = u"hitchhiker's guide to the galaxy" cls.data = { From f2f9095926e143ef7a1fe46f613c88bbf1096478 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Tue, 20 Aug 2013 19:27:49 -0300 Subject: [PATCH 026/159] [#1178] Adds user to organization when inviting, and only org admins can invite --- ckan/logic/action/create.py | 22 +++++++++++---- ckan/logic/auth/create.py | 2 +- ckan/model/user.py | 13 ++++----- ckan/tests/lib/test_mailer.py | 5 +++- ckan/tests/logic/test_action.py | 47 ++++++++++++++++----------------- ckan/tests/logic/test_auth.py | 11 ++++---- 6 files changed, 58 insertions(+), 42 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index edef0a95f30..5e1e2a477e0 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -843,19 +843,25 @@ def user_create(context, data_dict): def user_invite(context, data_dict): '''Invite a new user. - You must be authorized to invite users. + You must be authorized to create organization members. - :param email: the email address for the new user + :param email: the email of the user to be invited to the organization :type email: string + :param organization_id: the id or name of the organization + :type organization_id: string + :param role: role of the user in the group. One of ``member``, ``editor``, + or ``admin`` + :type role: string :returns: the newly created yser :rtype: dictionary - ''' _check_access('user_invite', context, data_dict) user_invite_schema = { - 'email': [validators.not_empty, unicode] + 'email': [validators.not_empty, unicode], + 'organization_id': [validators.not_empty], + 'role': [validators.not_empty], } _, errors = _validate(data_dict, user_invite_schema, context) if errors: @@ -870,6 +876,12 @@ def user_invite(context, data_dict): data_dict['state'] = ckan.model.State.PENDING user_dict = _get_action('user_create')(context, data_dict) user = ckan.model.User.get(user_dict['id']) + member_dict = { + 'username': user.id, + 'id': data_dict['organization_id'], + 'role': data_dict['role'] + } + _get_action('organization_member_create')(context, member_dict) mailer.send_invite(user) return model_dictize.user_dictize(user, context) except ValidationError as e: @@ -1187,7 +1199,7 @@ def _group_or_org_member_create(context, data_dict, is_org=False): role = data_dict.get('role') group_id = data_dict.get('id') group = model.Group.get(group_id) - result = session.query(model.User).filter_by(name=username).first() + result = model.User.get(username) if result: user_id = result.id else: diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index fafbc5a2e4c..55ec2d7df6f 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -113,7 +113,7 @@ def user_create(context, data_dict=None): return {'success': True} def user_invite(context, data_dict=None): - return {'success': False} + return organization_member_create(context, data_dict) def _check_group_auth(context, data_dict): # FIXME This code is shared amoung other logic.auth files and should be diff --git a/ckan/model/user.py b/ckan/model/user.py index a6d6c44b612..4f87175edf4 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -181,20 +181,21 @@ def is_deleted(self): def is_pending(self): return self.state == core.State.PENDING - def is_in_group(self, group): - return group in self.get_group_ids() + def is_in_group(self, group_id): + return group_id in self.get_group_ids() - def is_in_groups(self, groupids): + def is_in_groups(self, group_ids): ''' Given a list of group ids, returns True if this user is in any of those groups ''' guser = set(self.get_group_ids()) - gids = set(groupids) + gids = set(group_ids) return len(guser.intersection(gids)) > 0 - def get_group_ids(self, group_type=None): + def get_group_ids(self, group_type=None, capacity=None): ''' Returns a list of group ids that the current user belongs to ''' - return [g.id for g in self.get_groups(group_type=group_type)] + return [g.id for g in + self.get_groups(group_type=group_type, capacity=capacity)] def get_groups(self, group_type=None, capacity=None): import ckan.model as model diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index 22b3985930b..83a35c3b92c 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -114,8 +114,10 @@ def test_send_reset_email(self): # reset link tested in user functional test def test_send_invite_email(self): + user = model.User.by_name(u'bob') + assert user.reset_key is None, user # send email - mailer.send_invite(model.User.by_name(u'bob')) + mailer.send_invite(user) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -127,5 +129,6 @@ def test_send_invite_email(self): expected_body = self.mime_encode(test_msg, u'bob') assert expected_body in msg[3], '%r not in %r' % (expected_body, msg[3]) + assert user.reset_key is not None, user # reset link tested in user functional test diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index b6361803f0e..f2ef0df9f7d 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -562,12 +562,18 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) - @mock.patch('ckan.lib.mailer.mail_user') - def test_user_invite(self, mail_user): + @mock.patch('ckan.lib.mailer.send_invite') + def test_user_invite(self, send_invite): email_username = 'invited_user$ckan' email = '%s@email.com' % email_username - user_dict = {'email': email} - postparams = '%s=1' % json.dumps(user_dict) + organization_name = 'an_organization' + CreateTestData.create_groups([{'name': organization_name}]) + role = 'member' + organization = model.Group.get(organization_name) + params = {'email': email, + 'organization_id': organization.id, + 'role': role} + postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/user_invite', params=postparams, @@ -575,27 +581,14 @@ def test_user_invite(self, mail_user): res_obj = json.loads(res.body) user = model.User.get(res_obj['result']['id']) - expected_username = email_username.replace('$', '-') assert res_obj['success'] is True, res_obj assert user.email == email, (user.email, email) - assert user.name.startswith(expected_username), (user.name, expected_username) assert user.is_pending(), user - assert user.reset_key is not None, user - - @mock.patch('ckan.lib.mailer.send_invite') - def test_user_invite_sends_email(self, send_invite): - email_username = 'invited_user' - email = '%s@email.com' % email_username - user_dict = {'email': email} - postparams = '%s=1' % json.dumps(user_dict) - extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} - - res = self.app.post('/api/action/user_invite', params=postparams, - extra_environ=extra_environ) - - res_obj = json.loads(res.body) - user = model.User.get(res_obj['result']['id']) - assert res_obj['success'] is True, res_obj + expected_username = email_username.replace('$', '-') + assert user.name.startswith(expected_username), (user.name, + expected_username) + group_ids = user.get_group_ids(capacity=role) + assert organization.id in group_ids, (group_ids, organization.id) assert send_invite.called assert send_invite.call_args[0][0].id == res_obj['result']['id'] @@ -618,8 +611,14 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran patcher = mock.patch('ckan.lib.mailer.mail_user') patcher.start() email = 'invited_user@email.com' - user_dict = {'email': email} - postparams = '%s=1' % json.dumps(user_dict) + organization_name = 'an_organization' + CreateTestData.create_groups([{'name': organization_name}]) + role = 'member' + organization = model.Group.get(organization_name) + params = {'email': email, + 'organization_id': organization.id, + 'role': role} + postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} usernames = ['first', 'first', 'second'] diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 13e7b38f2a5..9ae69f32081 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -1,3 +1,5 @@ +import mock + import ckan.tests as tests from ckan.logic import get_action import ckan.model as model @@ -49,11 +51,10 @@ def create_user(self, name): class TestAuthUsers(TestAuth): - def test_only_sysadmins_can_invite_users(self): - username = 'normal_user' - self.create_user(username) - - assert not new_authz.is_authorized_boolean('user_invite', {'user': username}) + @mock.patch('ckan.logic.auth.create.organization_member_create') + def test_invite_user_delegates_to_organization_member_create(self, organization_member_create): + new_authz.is_authorized_boolean('user_invite', {}) + organization_member_create.assert_called() def test_only_sysadmins_can_delete_users(self): username = 'username' From 34b4466358dca5f9a6f27dd693cc32e28a3aa644 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 10:56:41 +0200 Subject: [PATCH 027/159] [#938] Prepend config option with ckan. --- ckan/config/deployment.ini_tmpl | 4 ++-- ckanext/datapusher/logic/action.py | 4 ++-- doc/configuration.rst | 12 ++++++------ test-core.ini | 2 ++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 0729c606f3c..665cc8ec08e 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -138,8 +138,8 @@ ckan.feeds.author_link = # Make sure you have set up the DataStore -datapusher.formats = csv -datapusher.url = http://datapusher.ckan.org/ +ckan.datapusher.formats = csv +ckan.datapusher.url = http://datapusher.ckan.org/ ## Activity Streams Settings diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 1fe1d5abb8b..c7161a9ccc5 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -53,7 +53,7 @@ def datapusher_submit(context, data_dict): if not p.toolkit.get_action('datapusher_enabled')(context): return False - datapusher_url = pylons.config.get('datapusher.url') + datapusher_url = pylons.config.get('ckan.datapusher.url') callback_url = p.toolkit.url_for( controller='api', action='action', logic_function='datapusher_hook', @@ -172,7 +172,7 @@ def datapusher_status(context, data_dict): 'key': 'job_key' }) - datapusher_url = pylons.config.get('datapusher.url') + datapusher_url = pylons.config.get('ckan.datapusher.url') if not datapusher_url: raise p.toolkit.ValidationError( {'configuration': ['DataPusher not configured.']}) diff --git a/doc/configuration.rst b/doc/configuration.rst index 284dbbfde76..88794f2d05a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1065,23 +1065,23 @@ Only used with the Google storage backend. DataPusher Settings ------------------- -.. _datapusher.formats: +.. _ckan.datapusher.formats: -datapusher.formats +ckan.datapusher.formats ^^^^^^^^^^^^^^^^^^ Example:: - datapusher.formats = csv xls xlsx + ckan.datapusher.formats = csv xls xlsx .. todo:: Expand -.. _datapusher.url +.. _ckan.datapusher.url -datapusher.url +ckan.datapusher.url ^^^^^^^^^^^^^^ Example:: - datapusher.url = http://datapusher.ckan.org/ + ckan.datapusher.url = http://datapusher.ckan.org/ .. todo:: Expand diff --git a/test-core.ini b/test-core.ini index 21dc08cfae9..da5c1a0814a 100644 --- a/test-core.ini +++ b/test-core.ini @@ -25,6 +25,8 @@ sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_test ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_test ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_test +ckan.datapusher.url = http://datapusher.ckan.org/ + ## Solr support solr_url = http://127.0.0.1:8983/solr From 91f6a38ec63f6fe5d31598230eb32a346a39ca9f Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 10:57:36 +0200 Subject: [PATCH 028/159] [#1196] Move datapusher tests --- ckanext/datapusher/tests/__init__.py | 0 ckanext/datapusher/tests/test.py | 192 +++++++++++++++++++++++++ ckanext/datastore/tests/test_create.py | 140 ------------------ 3 files changed, 192 insertions(+), 140 deletions(-) create mode 100644 ckanext/datapusher/tests/__init__.py create mode 100644 ckanext/datapusher/tests/test.py diff --git a/ckanext/datapusher/tests/__init__.py b/ckanext/datapusher/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py new file mode 100644 index 00000000000..f867274d50d --- /dev/null +++ b/ckanext/datapusher/tests/test.py @@ -0,0 +1,192 @@ +import json +import httpretty +import nose +import sys +import datetime + +import pylons +from pylons import config +import sqlalchemy.orm as orm +import paste.fixture + +import ckan.plugins as p +import ckan.lib.create_test_data as ctd +import ckan.model as model +import ckan.tests as tests +import ckan.config.middleware as middleware + +import ckanext.datastore.db as db +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type + + +# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 +if sys.version_info < (2, 7, 0): + import socket + socket.setdefaulttimeout(1) + + +class TestDatastoreCreate(tests.WsgiAppCase): + sysadmin_user = None + normal_user = None + + @classmethod + def setup_class(cls): + + wsgiapp = middleware.make_app(config['global_conf'], **config) + cls.app = paste.fixture.TestApp(wsgiapp) + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") + p.load('datastore') + p.load('datapusher') + ctd.CreateTestData.create() + cls.sysadmin_user = model.User.get('testsysadmin') + cls.normal_user = model.User.get('annafan') + engine = db._get_engine( + {'connection_url': pylons.config['ckan.datastore.write_url']}) + cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) + + @classmethod + def teardown_class(cls): + rebuild_all_dbs(cls.Session) + p.unload('datastore') + p.unload('datapusher') + + def test_create_ckan_resource_in_package(self): + package = model.Package.get('annakarenina') + data = { + 'resource': {'package_id': package.id} + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=200) + res_dict = json.loads(res.body) + + assert 'resource_id' in res_dict['result'] + assert len(model.Package.get('annakarenina').resources) == 3 + + res = tests.call_action_api( + self.app, 'resource_show', id=res_dict['result']['resource_id']) + assert res['url'] == '/datastore/dump/' + res['id'], res + + @httpretty.activate + def test_providing_res_with_url_calls_datapusher_correctly(self): + pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' + httpretty.HTTPretty.register_uri( + httpretty.HTTPretty.POST, + 'http://datapusher.ckan.org/job', + content_type='application/json', + body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) + + package = model.Package.get('annakarenina') + + tests.call_action_api( + self.app, 'datastore_create', apikey=self.sysadmin_user.apikey, + resource=dict(package_id=package.id, url='demo.ckan.org')) + + assert len(package.resources) == 4, len(package.resources) + resource = package.resources[3] + data = json.loads(httpretty.last_request().body) + assert data['metadata']['resource_id'] == resource.id, data + assert data['result_url'].endswith('/action/datapusher_hook'), data + assert data['result_url'].startswith('http://'), data + + def test_cant_provide_resource_and_resource_id(self): + package = model.Package.get('annakarenina') + resource = package.resources[0] + data = { + 'resource_id': resource.id, + 'resource': {'package_id': package.id} + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + + assert res_dict['error']['__type'] == 'Validation Error' + + @httpretty.activate + def test_send_datapusher_creates_task(self): + httpretty.HTTPretty.register_uri( + httpretty.HTTPretty.POST, + 'http://datapusher.ckan.org/job', + content_type='application/json', + body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) + + package = model.Package.get('annakarenina') + resource = package.resources[0] + + context = { + 'ignore_auth': True, + 'user': self.sysadmin_user.name + } + + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': resource.id + }) + + task = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': resource.id, + 'task_type': 'datapusher', + 'key': 'job_id' + }) + + assert task['state'] == 'pending', task + + def test_datapusher_hook(self): + package = model.Package.get('annakarenina') + resource = package.resources[0] + + context = { + 'user': self.sysadmin_user.name + } + + p.toolkit.get_action('task_status_update')(context, { + 'entity_id': resource.id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'key': 'job_id', + 'value': 'my_id', + 'last_updated': str(datetime.datetime.now()), + 'state': 'pending' + }) + + p.toolkit.get_action('task_status_update')(context, { + 'entity_id': resource.id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'key': 'job_key', + 'value': 'my_key', + 'last_updated': str(datetime.datetime.now()), + 'state': 'pending' + }) + + data = { + 'status': 'success', + 'metadata': { + 'resource_id': resource.id + } + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datapusher_hook', params=postparams, + extra_environ=auth, status=200) + print res.body + res_dict = json.loads(res.body) + + assert res_dict['success'] is True + + task = tests.call_action_api( + self.app, 'task_status_show', entity_id=resource.id, + task_type='datapusher', key='job_id') + + assert task['state'] == 'success', task + + task = tests.call_action_api( + self.app, 'task_status_show', entity_id=resource.id, + task_type='datapusher', key='job_key') + + assert task['state'] == 'success', task \ No newline at end of file diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index f95e8044951..f2db232af8a 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -38,7 +38,6 @@ def setup_class(cls): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") p.load('datastore') - p.load('datapusher') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') @@ -52,7 +51,6 @@ def setup_class(cls): def teardown_class(cls): rebuild_all_dbs(cls.Session) p.unload('datastore') - p.unload('datapusher') def test_create_requires_auth(self): resource = model.Package.get('annakarenina').resources[0] @@ -540,144 +538,6 @@ def test_create_basic(self): assert res_dict['success'] is True, res_dict - def test_create_ckan_resource_in_package(self): - package = model.Package.get('annakarenina') - data = { - 'resource': {'package_id': package.id} - } - postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=200) - res_dict = json.loads(res.body) - - assert 'resource_id' in res_dict['result'] - assert len(model.Package.get('annakarenina').resources) == 3 - - res = tests.call_action_api( - self.app, 'resource_show', id=res_dict['result']['resource_id']) - assert res['url'] == '/datastore/dump/' + res['id'], res - - @httpretty.activate - def test_providing_res_with_url_calls_datapusher_correctly(self): - pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' - httpretty.HTTPretty.register_uri( - httpretty.HTTPretty.POST, - 'http://datapusher.ckan.org/job', - content_type='application/json', - body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) - - package = model.Package.get('annakarenina') - - tests.call_action_api( - self.app, 'datastore_create', apikey=self.sysadmin_user.apikey, - resource=dict(package_id=package.id, url='demo.ckan.org')) - - assert len(package.resources) == 4, len(package.resources) - resource = package.resources[3] - data = json.loads(httpretty.last_request().body) - assert data['metadata']['resource_id'] == resource.id, data - assert data['result_url'].endswith('/action/datapusher_hook'), data - assert data['result_url'].startswith('http://'), data - - def test_cant_provide_resource_and_resource_id(self): - package = model.Package.get('annakarenina') - resource = package.resources[0] - data = { - 'resource_id': resource.id, - 'resource': {'package_id': package.id} - } - postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) - res_dict = json.loads(res.body) - - assert res_dict['error']['__type'] == 'Validation Error' - - @httpretty.activate - def test_send_datapusher_creates_task(self): - httpretty.HTTPretty.register_uri( - httpretty.HTTPretty.POST, - 'http://datapusher.ckan.org/job', - content_type='application/json', - body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) - - package = model.Package.get('annakarenina') - resource = package.resources[0] - - context = { - 'ignore_auth': True, - 'user': self.sysadmin_user.name - } - - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': resource.id - }) - - task = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': resource.id, - 'task_type': 'datapusher', - 'key': 'job_id' - }) - - assert task['state'] == 'pending', task - - def test_datapusher_hook(self): - package = model.Package.get('annakarenina') - resource = package.resources[0] - - context = { - 'user': self.sysadmin_user.name - } - - p.toolkit.get_action('task_status_update')(context, { - 'entity_id': resource.id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'key': 'job_id', - 'value': 'my_id', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - }) - - p.toolkit.get_action('task_status_update')(context, { - 'entity_id': resource.id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'key': 'job_key', - 'value': 'my_key', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - }) - - data = { - 'status': 'success', - 'metadata': { - 'resource_id': resource.id - } - } - postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datapusher_hook', params=postparams, - extra_environ=auth, status=200) - print res.body - res_dict = json.loads(res.body) - - assert res_dict['success'] is True - - task = tests.call_action_api( - self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_id') - - assert task['state'] == 'success', task - - task = tests.call_action_api( - self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_key') - - assert task['state'] == 'success', task - def test_guess_types(self): resource = model.Package.get('annakarenina').resources[1] From ef7b3e94c3bc09d60086bbc4ae8b320ad8c61b94 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 11:13:21 +0200 Subject: [PATCH 029/159] [#1183] Remove datapusher_enabled action and move before_show to datstore. Instead of using the action or the template helper, check whether the plugin is enabled. Moved the action so that changing the url works even if the datapusher is disabled. --- ckan/templates/package/resource_read.html | 4 +++- .../templates/snippets/datapusher_status.html | 14 +++++------ ckanext/datapusher/helpers.py | 4 ---- ckanext/datapusher/logic/action.py | 10 -------- ckanext/datapusher/plugin.py | 24 +++++++------------ ckanext/datastore/logic/action.py | 17 +++++++++---- ckanext/datastore/plugin.py | 13 +++++++++- 7 files changed, 41 insertions(+), 45 deletions(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index d4433d4a4a8..f914780d179 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -27,7 +27,9 @@
        {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} -
      • {% snippet 'snippets/datapusher_status.html', resource=res %}
        • + {% if 'datapusher' in g.plugins %} +
      • {% snippet 'snippets/datapusher_status.html', resource=res %}
        • + {% endif %}
      • {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
        • {% endif %} {% if res.url %} diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html index 863faeee491..3aa8908aef8 100644 --- a/ckan/templates/snippets/datapusher_status.html +++ b/ckan/templates/snippets/datapusher_status.html @@ -4,13 +4,11 @@ #} {% if resource.datastore_active %} - {% if h.datapusher_enabled() %} - {% set job = h.datapusher_status(resource.id) %} - {% set title = _('Datapusher status: {status}.').format(status=job.status) %} - {% if job.status == 'unknown' %} - - {% else %} - - {% endif %} + {% set job = h.datapusher_status(resource.id) %} + {% set title = _('Datapusher status: {status}.').format(status=job.status) %} + {% if job.status == 'unknown' %} + + {% else %} + {% endif %} {% endif %} diff --git a/ckanext/datapusher/helpers.py b/ckanext/datapusher/helpers.py index bf06b2e063a..d5291425b28 100644 --- a/ckanext/datapusher/helpers.py +++ b/ckanext/datapusher/helpers.py @@ -9,7 +9,3 @@ def datapusher_status(resource_id): return { 'status': 'unknown' } - - -def datapusher_enabled(): - return toolkit.get_action('datapusher_enabled')({}) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index c7161a9ccc5..41d3e063284 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -16,13 +16,6 @@ _validate = ckan.lib.navl.dictization_functions.validate -def datapusher_enabled(context, data_dict): - ''' Returns True if the DataPusher is configured ''' - - datapusher_url = pylons.config.get('datapusher.url') - return True if datapusher_url else False - - def datapusher_submit(context, data_dict): ''' Submit a job to the datapusher. The datapusher is a service that imports tabular data into the datastore. @@ -50,9 +43,6 @@ def datapusher_submit(context, data_dict): p.toolkit.check_access('datapusher_submit', context, data_dict) - if not p.toolkit.get_action('datapusher_enabled')(context): - return False - datapusher_url = pylons.config.get('ckan.datapusher.url') callback_url = p.toolkit.url_for( diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 650282ea9e1..b20f464d789 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -23,7 +23,6 @@ class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IAuthFunctions) p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) - p.implements(p.IResourceController, inherit=True) p.implements(p.ITemplateHelpers) legacy_mode = False @@ -32,9 +31,14 @@ class DatapusherPlugin(p.SingletonPlugin): def configure(self, config): self.config = config - datapusher_formats = config.get('datapusher.formats', '').split() + datapusher_formats = config.get('ckan.datapusher.formats', '').split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS + datapusher_url = config.get('ckan.datapusher.url') + if not datapusher_url: + raise Exception( + 'Config option `ckan.datapusher.url` has to be set.') + def notify(self, entity, operation=None): if isinstance(entity, model.Resource): if (operation == model.domain_object.DomainObjectOperation.new @@ -54,8 +58,7 @@ def notify(self, entity, operation=None): }) def get_actions(self): - return {'datapusher_enabled': action.datapusher_enabled, - 'datapusher_submit': action.datapusher_submit, + return {'datapusher_submit': action.datapusher_submit, 'datapusher_hook': action.datapusher_hook, 'datapusher_status': action.datapusher_status} @@ -63,17 +66,6 @@ def get_auth_functions(self): return {'datapusher_submit': auth.datapusher_submit, 'datapusher_status': auth.datapusher_status} - def before_show(self, resource_dict): - ''' Modify the resource url of datastore resources so that - they link to the datastore dumps. - ''' - if resource_dict['url_type'] == 'datastore': - resource_dict['url'] = p.toolkit.url_for( - controller='ckanext.datastore.controller:DatastoreController', - action='dump', resource_id=resource_dict['id']) - return resource_dict - def get_helpers(self): return { - 'datapusher_status': helpers.datapusher_status, - 'datapusher_enabled': helpers.datapusher_enabled} + 'datapusher_status': helpers.datapusher_status} diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index ee2a44e9e4a..742462ba8ae 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -88,20 +88,27 @@ def datastore_create(context, data_dict): if 'resource' in data_dict: has_url = 'url' in data_dict['resource'] - data_dict['resource'].setdefault('url', '_tmp') + # A datastore only resource does not have a url in the db + data_dict['resource'].setdefault('url', '_datastore_only_resource') res = p.toolkit.get_action('resource_create')(context, data_dict['resource']) data_dict['resource_id'] = res['id'] # create resource from file if has_url: - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': res['id'], - 'set_url_type': True - }) + try: + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': res['id'], + 'set_url_type': True + }) + except KeyError: + raise p.toolkit.ValidationError({'resource': [ + 'The datapusher has to be enabled.']}) + # since we'll overwrite the datastore resource anyway, we # don't need to create it here return + # create empty resource else: # no need to set the full url because it will be set in before_show diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 8eb753f8e46..61cdd783d81 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -25,6 +25,7 @@ class DatastorePlugin(p.SingletonPlugin): p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.IRoutes, inherit=True) + p.implements(p.IResourceController, inherit=True) legacy_mode = False resource_show_action = None @@ -250,4 +251,14 @@ def before_map(self, m): m.connect('/datastore/dump/{resource_id}', controller='ckanext.datastore.controller:DatastoreController', action='dump') - return m \ No newline at end of file + return m + + def before_show(self, resource_dict): + ''' Modify the resource url of datastore resources so that + they link to the datastore dumps. + ''' + if resource_dict['url_type'] == 'datastore': + resource_dict['url'] = p.toolkit.url_for( + controller='ckanext.datastore.controller:DatastoreController', + action='dump', resource_id=resource_dict['id']) + return resource_dict From 4e16bc1abc7578ebcfd564482355f7a5d671a493 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 11:29:54 +0200 Subject: [PATCH 030/159] PEP8 --- ckanext/datapusher/tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index f867274d50d..f47f256cef6 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -189,4 +189,4 @@ def test_datapusher_hook(self): self.app, 'task_status_show', entity_id=resource.id, task_type='datapusher', key='job_key') - assert task['state'] == 'success', task \ No newline at end of file + assert task['state'] == 'success', task From 98508d1a67b0ec0a86a1786988f3195d6535f2be Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 26 Aug 2013 16:13:48 +0200 Subject: [PATCH 031/159] [#1011] Fix a broken import in the package controller An exception was not getting caught because of the way this import was done. --- ckan/controllers/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index c9940cd57ac..e23c6e39fa1 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -19,7 +19,7 @@ import ckan.lib.datapreview as datapreview import ckan.lib.plugins import ckan.plugins as p -import lib.render +import ckan.lib.render from ckan.common import OrderedDict, _, json, request, c, g, response from home import CACHE_PARAMETERS @@ -366,7 +366,7 @@ def read(self, id, format='html'): try: return render(template, loader_class=loader) - except lib.render.TemplateNotFound: + except ckan.lib.render.TemplateNotFound: msg = _("Viewing {package_type} datasets in {format} format is " "not supported (template file {file} not found).".format( package_type=package_type, format=format, file=template)) From 2d3bcdd7a04b27be135045cf388ef9c983a3c16a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 26 Aug 2013 16:14:47 +0200 Subject: [PATCH 032/159] [#1011] base.py: Explicitly re-raise TemplateNotFound Functions should not implicitly raise exceptions by failing to catch exceptions raised by functions that they call, they should explicitly catch and re-raise the exceptions instead. --- ckan/lib/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index a91a1170135..cf5cf031f76 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -104,7 +104,10 @@ def render_template(): # we remove it so any bad templates crash and burn del globs['url'] - template_path, template_type = render_.template_info(template_name) + try: + template_path, template_type = render_.template_info(template_name) + except render_.TemplateNotFound: + raise # snippets should not pass the context # but allow for legacy genshi templates @@ -202,6 +205,8 @@ def render_template(): raise ckan.exceptions.CkanUrlException( '\nAn Exception has been raised for template %s\n%s' % (template_name, e.message)) + except render_.TemplateNotFound: + raise class ValidationException(Exception): From c136aac9c65fcde7c03a2e1c7a9e88e4b5b2f595 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 26 Aug 2013 16:16:03 +0200 Subject: [PATCH 033/159] [#1011] Update some exception handling in template.py Catch the new ckan.lib.render.TemplateNotFound exception that render() now raises, not the Genshi exception it used to. --- ckan/controllers/template.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ckan/controllers/template.py b/ckan/controllers/template.py index 1ebc700345b..0755b1a2cf5 100644 --- a/ckan/controllers/template.py +++ b/ckan/controllers/template.py @@ -1,6 +1,5 @@ -from genshi.template.loader import TemplateNotFound - import ckan.lib.base as base +import ckan.lib.render class TemplateController(base.BaseController): @@ -29,11 +28,11 @@ def view(self, url): """ try: return base.render(url) - except TemplateNotFound: + except ckan.lib.render.TemplateNotFound: if url.endswith('.html'): base.abort(404) url += '.html' try: return base.render(url) - except TemplateNotFound: + except ckan.lib.render.TemplateNotFound: base.abort(404) From b75524ee9b74166428555c6c5f76db7f58c67f14 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 29 Aug 2013 21:33:52 -0300 Subject: [PATCH 034/159] [#1178] Invite user on add member to organization/group I've added an "Email" field to the "Add Member" to Organization/Group page. If you add an e-mail there, we'll create a new user and invite her (doesn't matter if there's already a user with the same email). I still need to change the template and add help texts. --- ckan/controllers/group.py | 13 +++++++++ ckan/logic/action/create.py | 14 +++++----- ckan/logic/auth/create.py | 3 ++- ckan/model/user.py | 4 +++ ckan/templates/group/member_new.html | 1 + ckan/templates/organization/member_new.html | 1 + ckan/tests/functional/test_group.py | 30 +++++++++++++++++++++ ckan/tests/logic/test_action.py | 4 +-- ckan/tests/logic/test_auth.py | 13 ++++++--- 9 files changed, 69 insertions(+), 14 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ccef46bfe78..c8775168c8f 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -612,6 +612,19 @@ def member_new(self, id): data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['id'] = id + + email = data_dict.get('email') + if email and email != '': + user_data_dict = { + 'email': email, + 'group_id': data_dict['id'], + 'role': data_dict['role'] + } + del data_dict['email'] + user_dict = self._action('user_invite')(context, + user_data_dict) + data_dict['username'] = user_dict['name'] + c.group_dict = self._action('group_member_create')(context, data_dict) self._redirect_to(controller='group', action='members', id=id) else: diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 5e1e2a477e0..7bc750fa10e 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -843,12 +843,12 @@ def user_create(context, data_dict): def user_invite(context, data_dict): '''Invite a new user. - You must be authorized to create organization members. + You must be authorized to create group members. - :param email: the email of the user to be invited to the organization + :param email: the email of the user to be invited to the group :type email: string - :param organization_id: the id or name of the organization - :type organization_id: string + :param group_id: the id or name of the group + :type group_id: string :param role: role of the user in the group. One of ``member``, ``editor``, or ``admin`` :type role: string @@ -860,7 +860,7 @@ def user_invite(context, data_dict): user_invite_schema = { 'email': [validators.not_empty, unicode], - 'organization_id': [validators.not_empty], + 'group_id': [validators.not_empty], 'role': [validators.not_empty], } _, errors = _validate(data_dict, user_invite_schema, context) @@ -878,10 +878,10 @@ def user_invite(context, data_dict): user = ckan.model.User.get(user_dict['id']) member_dict = { 'username': user.id, - 'id': data_dict['organization_id'], + 'id': data_dict['group_id'], 'role': data_dict['role'] } - _get_action('organization_member_create')(context, member_dict) + _get_action('group_member_create')(context, member_dict) mailer.send_invite(user) return model_dictize.user_dictize(user, context) except ValidationError as e: diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 55ec2d7df6f..25e9f18988c 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -113,7 +113,8 @@ def user_create(context, data_dict=None): return {'success': True} def user_invite(context, data_dict=None): - return organization_member_create(context, data_dict) + context['id'] = context.get('group_id') + return group_member_create(context, data_dict) def _check_group_auth(context, data_dict): # FIXME This code is shared amoung other logic.auth files and should be diff --git a/ckan/model/user.py b/ckan/model/user.py index 4f87175edf4..da92a57c11a 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -44,6 +44,10 @@ def by_openid(cls, openid): obj = meta.Session.query(cls).autoflush(False) return obj.filter_by(openid=openid).first() + @classmethod + def by_email(cls, email): + return meta.Session.query(cls).filter_by(email=email).all() + @classmethod def get(cls, user_reference): # double slashes in an openid often get turned into single slashes diff --git a/ckan/templates/group/member_new.html b/ckan/templates/group/member_new.html index 87c6dbb60c2..2385a1957b9 100644 --- a/ckan/templates/group/member_new.html +++ b/ckan/templates/group/member_new.html @@ -19,6 +19,7 @@

        {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} {% endif %} + {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
        diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index dd2c4e50bfa..74487100199 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -21,6 +21,7 @@

        {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} {% endif %} + {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
        diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index b07bbe7ea6f..c5dc63b1097 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -1,6 +1,7 @@ import re from nose.tools import assert_equal +import mock import ckan.tests.test_plugins as test_plugins import ckan.model as model @@ -660,3 +661,32 @@ def test_2_atom_feed(self): assert '' in res, res + +class TestMemberInvite(FunctionalTestCase): + @classmethod + def setup_class(self): + model.Session.remove() + model.repo.rebuild_db() + + def teardown(self): + model.repo.rebuild_db() + + @mock.patch('ckan.lib.mailer.mail_user') + def test_member_new_invites_user_if_received_email(self, mail_user): + user = CreateTestData.create_user('a_user', sysadmin=True) + group_name = 'a_group' + CreateTestData.create_groups([{'name': group_name}], user.name) + group = model.Group.get(group_name) + url = url_for(controller='group', action='member_new', id=group.id) + email = 'invited_user@mailinator.com' + role = 'member' + + params = {'email': email, 'role': role} + res = self.app.post(url, params, + extra_environ={'REMOTE_USER': str(user.name)}) + + users = model.User.by_email(email) + assert len(users) == 1, users + user = users[0] + assert user.email == email, user + assert group.id in user.get_group_ids(capacity=role) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index f2ef0df9f7d..8c1d3e0586d 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -571,7 +571,7 @@ def test_user_invite(self, send_invite): role = 'member' organization = model.Group.get(organization_name) params = {'email': email, - 'organization_id': organization.id, + 'group_id': organization.id, 'role': role} postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} @@ -616,7 +616,7 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran role = 'member' organization = model.Group.get(organization_name) params = {'email': email, - 'organization_id': organization.id, + 'group_id': organization.id, 'role': role} postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 9ae69f32081..ff8a18ccdf3 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -51,10 +51,15 @@ def create_user(self, name): class TestAuthUsers(TestAuth): - @mock.patch('ckan.logic.auth.create.organization_member_create') - def test_invite_user_delegates_to_organization_member_create(self, organization_member_create): - new_authz.is_authorized_boolean('user_invite', {}) - organization_member_create.assert_called() + @mock.patch('ckan.logic.auth.create.group_member_create') + def test_invite_user_prepares_context_and_delegates_to_group_member_create(self, group_member_create): + context = {'group_id': 42} + group_member_create_context = context + group_member_create_context['id'] = context['group_id'] + + new_authz.is_authorized_boolean('user_invite', context) + + group_member_create.assert_called(group_member_create_context, None) def test_only_sysadmins_can_delete_users(self): username = 'username' From 89db2ed23ca5cb0136a44d2829f2e713f2b2558d Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Sat, 31 Aug 2013 11:26:31 -0300 Subject: [PATCH 035/159] [#1178] Move migration from 070 to 071, as we have a new 070 --- ...umn_to_user_table.py => 071_add_state_column_to_user_table.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ckan/migration/versions/{070_add_state_column_to_user_table.py => 071_add_state_column_to_user_table.py} (100%) diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py similarity index 100% rename from ckan/migration/versions/070_add_state_column_to_user_table.py rename to ckan/migration/versions/071_add_state_column_to_user_table.py From 52c93774c83b13cbfa98107cba87aed6ad9d1b13 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Wed, 4 Sep 2013 17:00:05 +0100 Subject: [PATCH 036/159] [#1224] propogate the NotAuthorized exception extra messages to api --- ckan/controllers/api.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 22ea7d4f5c3..795488ee02a 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -113,8 +113,10 @@ def _finish_ok(self, response_data=None, return self._finish(status_int, response_data, content_type) - def _finish_not_authz(self): + def _finish_not_authz(self, extra_msg=None): response_data = _('Access denied') + if extra_msg: + response_data = '%s - %s' % (response_data, extra_msg) return self._finish(403, response_data, 'json') def _finish_not_found(self, extra_msg=None): @@ -193,10 +195,14 @@ def action(self, logic_function, ver=None): 'data': request_data} return_dict['success'] = False return self._finish(400, return_dict, content_type='json') - except NotAuthorized: + except NotAuthorized, e: return_dict['error'] = {'__type': 'Authorization Error', 'message': _('Access denied')} return_dict['success'] = False + + if e.extra_msg: + return_dict['error']['message'] += ': %s' % e.extra_msg + return self._finish(403, return_dict, content_type='json') except NotFound, e: return_dict['error'] = {'__type': 'Not Found Error', @@ -275,8 +281,9 @@ def list(self, ver=None, register=None, subregister=None, id=None): except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) def show(self, ver=None, register=None, subregister=None, id=None, id2=None): @@ -306,8 +313,9 @@ def show(self, ver=None, register=None, subregister=None, except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) def _represent_package(self, package): return package.as_dict(ref_package_by=self.ref_package_by, @@ -352,8 +360,9 @@ def create(self, ver=None, register=None, subregister=None, data_dict.get("id"))) return self._finish_ok(response_data, resource_location=location) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) @@ -408,8 +417,9 @@ def update(self, ver=None, register=None, subregister=None, try: response_data = action(context, data_dict) return self._finish_ok(response_data) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) @@ -456,8 +466,9 @@ def delete(self, ver=None, register=None, subregister=None, try: response_data = action(context, data_dict) return self._finish_ok(response_data) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) From 910ca29a902b265e5788864f533a70d8626cbcf1 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:14 +0530 Subject: [PATCH 037/159] New option to disable user creation via web --- ckan/config/deployment.ini_tmpl | 1 + ckan/logic/auth/create.py | 22 +++++++++++++++------- ckan/new_authz.py | 2 ++ ckan/templates/header.html | 4 +++- ckan/templates/user/login.html | 20 +++++++++++--------- test-core.ini | 3 ++- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 97c4f9412ba..6a0833d2d9f 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true ckan.auth.user_delete_groups = true ckan.auth.user_delete_organizations = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ## Search Settings diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index e96b789bcf5..35a9f39d876 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -103,14 +103,22 @@ def rating_create(context, data_dict): # No authz check in the logic function return {'success': True} -def user_create(context, data_dict=None): - user = context['user'] - if ('api_version' in context - and not new_authz.check_config_permission('create_user_via_api')): - return {'success': False, 'msg': _('User %s not authorized to create users') % user} - else: - return {'success': True} +def user_create(context, data_dict=None): + # create_user_via_api is deprecated + using_api = 'api_version' in context + create_user_via_api = new_authz.check_config_permission( + 'create_user_via_api') + create_user_via_web = new_authz.check_config_permission( + 'create_user_via_web') + + if using_api and not create_user_via_api: + return {'success': False, 'msg': _('User {user} not authorized to ' + 'create users via the API').format(user=context.get('user'))} + if not using_api and not create_user_via_web: + return {'success': False, 'msg': _('Not authorized to ' + 'create users')} + return {'success': True} def _check_group_auth(context, data_dict): diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 00a7dfe0e17..03ca209faf4 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -86,6 +86,7 @@ def _build(self): def clear_auth_functions_cache(): _AuthFunctions.clear() + CONFIG_PERMISSIONS.clear() def auth_functions_list(): @@ -319,6 +320,7 @@ def get_user_id_for_username(user_name, allow_none=False): 'user_delete_groups': True, 'user_delete_organizations': True, 'create_user_via_api': False, + 'create_user_via_web': True, } CONFIG_PERMISSIONS = {} diff --git a/ckan/templates/header.html b/ckan/templates/header.html index 5b4ebe225a7..268b370e968 100644 --- a/ckan/templates/header.html +++ b/ckan/templates/header.html @@ -49,7 +49,9 @@
          {% block header_account_notlogged %}
        • {% link_for _('Log in'), controller='user', action='login' %}
          • -
        • {% link_for _('Register'), controller='user', action='register', class_='sub' %}
          • + {% if h.check_access('user_create') %} +
        • {% link_for _('Register'), controller='user', action='register', class_='sub' %}
          • + {% endif %} {% endblock %}
        diff --git a/ckan/templates/user/login.html b/ckan/templates/user/login.html index c84e3522748..98c99510570 100644 --- a/ckan/templates/user/login.html +++ b/ckan/templates/user/login.html @@ -18,15 +18,17 @@

        {% block page_heading %}{{ _('Login') }}{% endblock %}< {% endblock %} {% block secondary_content %} -
        -

        {{ _('Need an Account?') }}

        -
        -

        {% trans %}Then sign right up, it only takes a minute.{% endtrans %}

        -

        - {{ _('Create an Account') }} -

        -
        -
        + {% if h.check_access('user_create') %} +
        +

        {{ _('Need an Account?') }}

        +
        +

        {% trans %}Then sign right up, it only takes a minute.{% endtrans %}

        +

        + {{ _('Create an Account') }} +

        +
        +
        + {% endif %}

        {{ _('Forgotten your details?') }}

        diff --git a/test-core.ini b/test-core.ini index 21dc08cfae9..08aba34ed24 100644 --- a/test-core.ini +++ b/test-core.ini @@ -31,6 +31,7 @@ solr_url = http://127.0.0.1:8983/solr ckan.auth.user_create_organizations = true ckan.auth.user_create_groups = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ckan.auth.create_dataset_if_not_in_organization = true ckan.auth.anon_create_dataset = false ckan.auth.user_delete_groups=true @@ -80,7 +81,7 @@ smtp.mail_from = info@test.ckan.net ckan.locale_default = en ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv -ckan.locales_filtered_out = +ckan.locales_filtered_out = ckan.datastore.enabled = 1 From 6bc0c3ee3b9ea158ad358c5697e26eedd372ba9f Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:50 +0530 Subject: [PATCH 038/159] Add documentation for new config option --- doc/configuration.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0efe9573226..853ffd23939 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -355,6 +355,20 @@ Default value: ``False`` Allow new user accounts to be created via the API. +.. _ckan.auth.create_user_via_web: + +ckan.auth.create_user_via_api +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.auth.create_user_via_web = True + +Default value: ``True`` + + +Allow new user accounts to be created via the Web. + .. end_config-authorization From c75e487ff99765418d5c82e9f683e2d694afc98d Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 16:46:10 +0530 Subject: [PATCH 039/159] Fix docs, remove deprecated comment --- ckan/logic/auth/create.py | 1 - doc/configuration.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 35a9f39d876..ec38483bd0d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): def user_create(context, data_dict=None): - # create_user_via_api is deprecated using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( 'create_user_via_api') diff --git a/doc/configuration.rst b/doc/configuration.rst index 853ffd23939..29ecbd310d8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -357,7 +357,7 @@ Allow new user accounts to be created via the API. .. _ckan.auth.create_user_via_web: -ckan.auth.create_user_via_api +ckan.auth.create_user_via_web ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example:: From 40397fe5c9bd0e2703f0d4e26227863c6cbc6b4a Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 9 Sep 2013 16:10:10 -0300 Subject: [PATCH 040/159] [#1178] Add help text to the add member forms --- ckan/public/base/less/forms.less | 30 ++++++++++++++ ckan/templates/group/member_new.html | 46 ++++++++++++++++----- ckan/templates/organization/member_new.html | 46 ++++++++++++++++----- 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..a641129f705 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,33 @@ textarea { .box-shadow(none); } } + +.add-member-form .control-label { + width: 100%; + text-align: left; +} + +.add-member-form .controls { + margin-left: auto; +} + +.add-member-or { + float: left; + line-height: 100px; + margin-left: 10px; + margin-right: 10px; + text-transform: uppercase; + color: grey; + font-weight: bold; +} + +.add-member-form .row-fluid .control-group { + float: left; + width: 45%; +} + +.add-member-form .row-fluid { + .select2-container, input { + width: 100% !important; + } +} diff --git a/ckan/templates/group/member_new.html b/ckan/templates/group/member_new.html index 2385a1957b9..efa3235461a 100644 --- a/ckan/templates/group/member_new.html +++ b/ckan/templates/group/member_new.html @@ -10,16 +10,42 @@

        {% block page_heading %}{{ _('Edit Member') if user else _('Add Member') }}{% endblock %}

        {% block form %} -
        - {% if user %} - - {% set format_attrs = {'disabled': true} %} - {{ form.input('username', label=_('User'), value=user.name, classes=['control-medium'], attrs=format_attrs) }} - {% else %} - {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} - {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} - {% endif %} - {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} + +
        +
        + + + {{ _('If you wish to add an existing user, search for their username below.') }} + +
        + {% if user %} + + + {% else %} + + {% endif %} +
        +
        +
        + {{ _('or') }} +
        +
        + + + {{ _('If you wish to invite a new user, enter their email address.') }} + +
        + +
        +
        +
        {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
        diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index 74487100199..b76de7663fa 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -12,16 +12,42 @@

        {% block page_heading %}{{ _('Edit Member') if user else _('Add Member') }}{% endblock %}

        {% block form %} - - {% if user %} - - {% set format_attrs = {'disabled': true} %} - {{ form.input('username', label=_('User'), value=user.name, classes=['control-medium'], attrs=format_attrs) }} - {% else %} - {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} - {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} - {% endif %} - {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} + +
        +
        + + + {{ _('If you wish to add an existing user, search for her username below.') }} + +
        + {% if user %} + + + {% else %} + + {% endif %} +
        +
        +
        + {{ _('or') }} +
        +
        + + + {{ _('If you wish to invite a new user, enter her email address.') }} + +
        + +
        +
        +
        {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
        From 2adda64767ea178e558ccd6844b84efe267a6dcf Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 9 Sep 2013 17:00:45 -0300 Subject: [PATCH 041/159] [#1178] Fix typo exeecute -> execute in migration --- ckan/migration/versions/071_add_state_column_to_user_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py index 367e3a48c32..791a046bad2 100644 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_add_state_column_to_user_table.py @@ -10,7 +10,7 @@ def upgrade(migrate_engine): def downgrade(migrate_engine): - migrate_engine.exeecute( + migrate_engine.execute( ''' ALTER TABLE "user" DROP COLUMN "state" ''' From e079385c05876834fad681261b0e68df6c9b9257 Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 10 Sep 2013 22:36:32 +0100 Subject: [PATCH 042/159] [1200] add defaults and make sure datapusher down does not cause errors --- ckan/model/meta.py | 2 +- ckan/plugins/interfaces.py | 1 - ckanext/datapusher/logic/action.py | 1 + ckanext/datapusher/plugin.py | 23 +++++++++++++++-------- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ckan/model/meta.py b/ckan/model/meta.py index 795094965c5..0103019bb3a 100644 --- a/ckan/model/meta.py +++ b/ckan/model/meta.py @@ -163,4 +163,4 @@ def engine_is_pg(sa_engine=None): # Returns true iff the engine is connected to a postgresql database. # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql # all Postgres driver names start with `postgresql` - return (sa_engine or engine).url.drivername.startswith('postgresql') + return (sa_engine or engine).url.drivername.startswith('postgres') diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index aa08fc64bab..d1372d2e895 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -443,7 +443,6 @@ def before_view(self, pkg_dict): class IResourceController(Interface): """ Hook into the resource controller. - (see IGroupController) """ def before_show(self, resource_dict): diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 41d3e063284..3201d35572a 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -50,6 +50,7 @@ def datapusher_submit(context, data_dict): ver=3, qualified=True) user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) + try: r = requests.post( urlparse.urljoin(datapusher_url, 'job'), diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index b20f464d789..04a27db435b 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust -DEFAULT_FORMATS = [] +DEFAULT_FORMATS = ['csv', 'xls', 'application/csv', 'application/vnd.ms-excel'] class DatastoreException(Exception): @@ -31,7 +31,7 @@ class DatapusherPlugin(p.SingletonPlugin): def configure(self, config): self.config = config - datapusher_formats = config.get('ckan.datapusher.formats', '').split() + datapusher_formats = config.get('ckan.datapusher.formats', '').lower().split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS datapusher_url = config.get('ckan.datapusher.url') @@ -46,16 +46,23 @@ def notify(self, entity, operation=None): # if operation is None, resource URL has been changed, as # the notify function in IResourceUrlChange only takes # 1 parameter - context = {'model': model, 'ignore_auth': True} + context = {'model': model, 'ignore_auth': True, 'defer_commit': True} package = p.toolkit.get_action('package_show')(context, { 'id': entity.get_package_id() }) - if (not package['private'] and - entity.format in self.datapusher_formats and + if (not package['private'] and entity.format and + entity.format.lower() in self.datapusher_formats and entity.url_type != 'datapusher'): - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': entity.id - }) + try: + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': entity.id + }) + except p.toolkit.ValidationError, e: + # If datapusher is offline want to catch error instead of + # raising otherwise resource save will fail with 500 + log.critical(e) + pass + def get_actions(self): return {'datapusher_submit': action.datapusher_submit, From b0f3e643b2640c2a717d56f802a5d61477fcf7f3 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 11 Sep 2013 12:32:26 +0100 Subject: [PATCH 043/159] [#1237] Improve facet.limit search option A negative value should be allowed if you don't want to limit the facet values, but the search params schema didn't allow it. Also mention in the docs that this can be set via configuration and that the default is 50. Added some tests. --- ckan/logic/action/get.py | 5 ++-- ckan/logic/schema.py | 2 +- ckan/tests/logic/test_action.py | 42 +++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index ab50329c608..a810cfa3cbe 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1289,8 +1289,9 @@ def package_search(context, data_dict): :param facet.mincount: the minimum counts for facet fields should be included in the results. :type facet.mincount: int - :param facet.limit: the maximum number of constraint counts that should be - returned for the facet fields. A negative value means unlimited + :param facet.limit: the maximum number of values the facet fields. A + negative value means unlimited. This can be set instance-wide with + the :ref:`search.facets.limit` config option. Default is 50. :type facet.limit: int :param facet.field: the fields to facet upon. Default empty. If empty, then the returned facet information is empty. diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..75d49808234 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -539,7 +539,7 @@ def default_package_search_schema(): 'qf': [ignore_missing, unicode], 'facet': [ignore_missing, unicode], 'facet.mincount': [ignore_missing, natural_number_validator], - 'facet.limit': [ignore_missing, natural_number_validator], + 'facet.limit': [ignore_missing, int_validator], 'facet.field': [ignore_missing, list_of_strings], 'extras': [ignore_missing] # Not used by Solr, but useful for extensions } diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 89e030beac8..8b4fe3dca14 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1377,6 +1377,48 @@ def test_1_basic(self): assert_equal(result['count'], 1) assert_equal(result['results'][0]['name'], 'annakarenina') + def test_1_facet_limit(self): + params = { + 'q':'*:*', + 'facet.field': ['groups', 'tags', 'res_format', 'license'], + 'rows': 20, + 'start': 0, + } + postparams = '%s=1' % json.dumps(params) + res = self.app.post('/api/action/package_search', params=postparams) + res = json.loads(res.body) + assert_equal(res['success'], True) + + assert_equal(len(res['result']['search_facets']['groups']['items']), 2) + + params = { + 'q':'*:*', + 'facet.field': ['groups', 'tags', 'res_format', 'license'], + 'facet.limit': 1, + 'rows': 20, + 'start': 0, + } + postparams = '%s=1' % json.dumps(params) + res = self.app.post('/api/action/package_search', params=postparams) + res = json.loads(res.body) + assert_equal(res['success'], True) + + assert_equal(len(res['result']['search_facets']['groups']['items']), 1) + + params = { + 'q':'*:*', + 'facet.field': ['groups', 'tags', 'res_format', 'license'], + 'facet.limit': -1, # No limit + 'rows': 20, + 'start': 0, + } + postparams = '%s=1' % json.dumps(params) + res = self.app.post('/api/action/package_search', params=postparams) + res = json.loads(res.body) + assert_equal(res['success'], True) + + assert_equal(len(res['result']['search_facets']['groups']['items']), 2) + def test_1_basic_no_params(self): postparams = '%s=1' % json.dumps({}) res = self.app.post('/api/action/package_search', params=postparams) From 2d76140db869595b83d2d501380bbe9f248192f5 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 11 Sep 2013 16:58:41 +0100 Subject: [PATCH 044/159] [#1178] Small CSS tweaks and fix for premature closing div --- ckan/public/base/less/forms.less | 8 ++++---- ckan/templates/organization/member_new.html | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index a641129f705..1263abdd219 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -682,11 +682,11 @@ textarea { .add-member-or { float: left; - line-height: 100px; - margin-left: 10px; - margin-right: 10px; + margin-top: 75px; + width: 7%; + text-align: center; text-transform: uppercase; - color: grey; + color: @grayLight; font-weight: bold; } diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index b76de7663fa..352cb9f3790 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -65,7 +65,6 @@

        {% endblock %} -
        {% endblock %} {% block secondary_content %} From 55ed1ef2b6d4eca22c324ddf627eb7fe1e0a4d24 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 12 Sep 2013 18:39:42 +0100 Subject: [PATCH 045/159] [#1241] Raise CkanVersionException properly --- ckan/plugins/toolkit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 4e4ac5da2d0..34e9d671cf1 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -348,7 +348,7 @@ def _requires_ckan_version(cls, min_version, max_version=None): else: error = 'Requires ckan version between %s and %s' % \ (min_version, max_version) - raise cls.CkanVersionException(error) + raise CkanVersionException(error) def __getattr__(self, name): ''' return the function/object requested ''' From 5813ae1f4b3dd0acde1e78a21b13cc533388e8e3 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 18 Sep 2013 14:27:42 +0100 Subject: [PATCH 046/159] [#1092] Updates helper text for orgs, groups and datasets --- ckan/templates/group/snippets/helper.html | 15 ++++++++------- ckan/templates/organization/snippets/helper.html | 16 +++++++--------- ckan/templates/package/base_form_page.html | 6 +++--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/ckan/templates/group/snippets/helper.html b/ckan/templates/group/snippets/helper.html index a73c9d6dde7..38585ab7eae 100644 --- a/ckan/templates/group/snippets/helper.html +++ b/ckan/templates/group/snippets/helper.html @@ -4,12 +4,13 @@

        {{ _('What are Groups?') }}

        - {% trans %} -

        Groups allow you to group together datasets under a community (for - example, Civil Liberty data) or topic (e.g. Transport, Health, - Environment) to make it easier for users to browse datasets by theme. - Datasets can be part of a group, but do not belong to the group for - editing or authorisation purposes.

        - {% endtrans %} +

        + {% trans %} + You can use CKAN Groups to create and manage collections of datasets. + This could be to catalogue datasets for a particular project or team, + or on a particular theme, or as a very simple way to help people find + and search your own published datasets. + {% endtrans %} +

        \ No newline at end of file diff --git a/ckan/templates/organization/snippets/helper.html b/ckan/templates/organization/snippets/helper.html index f0acbd46d73..6ec71c0d67e 100644 --- a/ckan/templates/organization/snippets/helper.html +++ b/ckan/templates/organization/snippets/helper.html @@ -4,14 +4,12 @@

        {{ _('What are Organizations?') }}

        - {% trans %} -

        Organizations act like publishing departments for datasets (for - example, the Department of Health). This means that datasets can be - published by and belong to a department instead of an individual - user.

        -

        Within organizations, admins can assign roles and authorisation its - members, giving individual users the right to publish datasets from - that particular organisation (e.g. Office of National Statistics).

        - {% endtrans %} +

        + {% trans %} + CKAN Organizations are used to create, manage and publish collections + of datasets. Users can have different roles within an Organization, + depending on their level of authorisation to create, edit and publish. + {% endtrans %} +

        \ No newline at end of file diff --git a/ckan/templates/package/base_form_page.html b/ckan/templates/package/base_form_page.html index 049d83b1ebb..a738a8716c8 100644 --- a/ckan/templates/package/base_form_page.html +++ b/ckan/templates/package/base_form_page.html @@ -15,9 +15,9 @@

        {{ _('What are dataset

        {% trans %} - Datasets are simply used to group related pieces of data. These - can then be found under a single url with a description and - licensing information. + A CKAN Dataset is a collection of data resources (such as files), + together with a description and other information, at a fixed URL. + Datasets are what users see when searching for data. {% endtrans %}

        From b8f449e83555ec850a1f0c4fc4bcbb7ded7b3d04 Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 20 Sep 2013 05:14:46 +0100 Subject: [PATCH 047/159] [#1200] make resource data page for reuploading and make error messages better --- ckan/config/routing.py | 2 +- ckan/logic/schema.py | 1 + ckan/templates/package/new_resource.html | 12 +- ckan/templates/package/resource_data.html | 45 ++++++++ ckan/templates/package/resource_read.html | 3 - ckanext/datapusher/logic/action.py | 127 +++++++++++++--------- ckanext/datapusher/plugin.py | 56 ++++++++++ ckanext/datastore/controller.py | 1 + 8 files changed, 189 insertions(+), 58 deletions(-) create mode 100644 ckan/templates/package/resource_data.html diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 0fbb02bb886..6b41d8a66f2 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -246,7 +246,7 @@ def make_map(): action='resource_read') m.connect('/dataset/{id}/resource_delete/{resource_id}', action='resource_delete') - m.connect('/dataset/{id}/resource_edit/{resource_id}', + m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}', action='resource_edit') m.connect('/dataset/{id}/resource/{resource_id}/download', action='resource_download') diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..8058e10aea0 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -85,6 +85,7 @@ def default_resource_schema(): 'cache_last_updated': [ignore_missing, isodate], 'webstore_last_updated': [ignore_missing, isodate], 'tracking_summary': [ignore_missing], + 'datastore_active': [ignore], '__extras': [ignore_missing, extras_unicode_convert, keep_extras], } diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 40d8ed97c9e..7eedbad761f 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -29,8 +29,16 @@

        {{ _('What\'s a resour {% endblock %} {% block primary_content %} - {% if pkg_dict and pkg_dict.state != 'draft' %} -
        + {% set res = c.resource %} + {% if pkg_dict and pkg_dict.state != 'draft' and res %} +
        +
          + {{ h.build_nav_icon('resource_edit', _('Edit Resource'), id=pkg_dict.id, resource_id=res.id) }} + {% if 'datapusher' in g.plugins %} + {{ h.build_nav_icon('resource_data', _('Resource Data'), id=pkg_dict.id, resource_id=res.id) }} + {% endif %} +
        +
        {% endif %} {{ super() }} {% endblock %} diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html new file mode 100644 index 00000000000..30c9fe59e98 --- /dev/null +++ b/ckan/templates/package/resource_data.html @@ -0,0 +1,45 @@ +{% extends "package/new_resource.html" %} + +{% set pkg_dict = c.pkg_dict %} +{% set res = c.resource %} + +{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %} + +{% block breadcrumb_content_selected %}{% endblock %} + +{% block breadcrumb_content %} + {{ super() }} +
      • {{ _('Edit') }}
        • +{% endblock %} + +{% block content_action %} + {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% endblock %} + +{# logged_in is defined in new_resource.html #} +{% block form %} + + {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg_dict.id, resource_id=res.id) %} + +
        +

        {{ _('Last Upload Status:') }} {% if status.status %} {{ status.status.capitalize() }} {% else %} {{ _('Not Uploaded Yet') }} {% endif %}

        + {% if status.error and status.error.message %} +

        {{ _('Upload Error:') }} {{ status.error.message }}

        + {% endif %} + {% if status.status %} +

        {{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }}

        + {% endif %} + + + {% if status.status and status.task_info %} +

        {{ _('Upload Log ') }}

        + {{ h.debug_inspect(status.task_info.logs) }} + {% endif %} + {{ status }} +
        + +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=pkg_dict.id, action='resource_edit' %} +{% endblock %} diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index f914780d179..46406746a6f 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -27,9 +27,6 @@
          {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} - {% if 'datapusher' in g.plugins %} -
        • {% snippet 'snippets/datapusher_status.html', resource=res %}
          • - {% endif %}
        • {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
          • {% endif %} {% if res.url %} diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 3201d35572a..653dd44e0b5 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -51,6 +51,29 @@ def datapusher_submit(context, data_dict): user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) + task = { + 'entity_id': res_id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'last_updated': str(datetime.datetime.now()), + 'state': 'submitting', + 'key': 'datapusher', + 'value': '{}', + 'error': '{}', + } + try: + task_id = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'datapusher' + })['id'] + task['id'] = task_id + except logic.NotFound: + pass + + result = p.toolkit.get_action('task_status_update')(context, task) + task_id = result['id'] + try: r = requests.post( urlparse.urljoin(datapusher_url, 'job'), @@ -69,36 +92,37 @@ def datapusher_submit(context, data_dict): })) r.raise_for_status() except requests.exceptions.ConnectionError, e: - raise p.toolkit.ValidationError({'datapusher': { - 'message': 'Could not connect to DataPusher.', - 'details': str(e)}}) + error = {'message': 'Could not connect to DataPusher.', + 'details': str(e)} + task['error'] = json.dumps(error) + task['state'] = 'error' + task['last_updated'] = str(datetime.datetime.now()), + p.toolkit.get_action('task_status_update')(context, task) + raise p.toolkit.ValidationError(error) + except requests.exceptions.HTTPError, e: m = 'An Error occurred while sending the job: {0}'.format(e.message) try: body = e.response.json() except ValueError: body = e.response.text - raise p.toolkit.ValidationError({'datapusher': { - 'message': m, - 'details': body, - 'status_code': r.status_code}}) - - empty_task = { - 'entity_id': res_id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - } - - tasks = [] - for (k, v) in [('job_id', r.json()['job_id']), - ('job_key', r.json()['job_key'])]: - t = empty_task.copy() - t['key'] = k - t['value'] = v - tasks.append(t) - p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) + error = {'message': m, + 'details': body, + 'status_code': r.status_code} + task['error'] = json.dumps(error) + task['state'] = 'error' + task['last_updated'] = str(datetime.datetime.now()), + p.toolkit.get_action('task_status_update')(context, task) + raise p.toolkit.ValidationError(error) + + value = json.dumps( + {'job_id': r.json()['job_id'], + 'job_key': r.json()['job_key']} + ) + task['value'] = value + task['state'] = 'pending' + task['last_updated'] = str(datetime.datetime.now()), + p.toolkit.get_action('task_status_update')(context, task) return True @@ -111,30 +135,20 @@ def datapusher_hook(context, data_dict): ''' # TODO: use a schema to validate - p.toolkit.check_access('datapusher_submit', context, data_dict) res_id = data_dict['metadata']['resource_id'] - task_id = p.toolkit.get_action('task_status_show')(context, { + task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'datapusher', - 'key': 'job_id' + 'key': 'datapusher' }) - task_key = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_key' - }) - - tasks = [task_id, task_key] - - for task in tasks: - task['state'] = data_dict['status'] - task['last_updated'] = str(datetime.datetime.now()) + task['state'] = data_dict['status'] + task['last_updated'] = str(datetime.datetime.now()) - p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) + p.toolkit.get_action('task_status_update')(context, task) def datapusher_status(context, data_dict): @@ -151,16 +165,10 @@ def datapusher_status(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - task_id = p.toolkit.get_action('task_status_show')(context, { + task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'datapusher', - 'key': 'job_id' - }) - - task_key = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_key' + 'key': 'datapusher' }) datapusher_url = pylons.config.get('ckan.datapusher.url') @@ -168,11 +176,26 @@ def datapusher_status(context, data_dict): raise p.toolkit.ValidationError( {'configuration': ['DataPusher not configured.']}) - url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value']) + value = json.loads(task['value']) + job_key = value.get('job_key') + job_id = value.get('job_id') + url = None + job_detail = None + if job_id: + url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id) + try: + r = requests.get(url, headers={'Content-Type': 'application/json', + 'Authorization': job_key}) + job_detail = r.json() + except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: + job_detail = {'error': 'cannot connect to datapusher'} + return { - 'status': task_id['state'], - 'job_id': task_id['value'], + 'status': task['state'], + 'job_id': job_id, 'job_url': url, - 'last_updated': task_id['last_updated'], - 'job_key': task_key['value'] + 'last_updated': task['last_updated'], + 'job_key': job_key, + 'task_info': job_detail, + 'error': json.loads(task['error']) } diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 04a27db435b..a545323a7dd 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -1,11 +1,14 @@ import logging import ckan.plugins as p +import ckan.lib.base as base +import ckan.lib.helpers as core_helpers import ckanext.datapusher.logic.action as action import ckanext.datapusher.logic.auth as auth import ckanext.datapusher.helpers as helpers import ckan.logic as logic import ckan.model as model +from ckan.common import c, _, request log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust @@ -16,6 +19,48 @@ class DatastoreException(Exception): pass +class ResourceDataController(base.BaseController): + + def resource_data(self, id, resource_id): + + if request.method == 'POST': + result = request.POST + try: + c.pkg_dict = p.toolkit.get_action('datapusher_submit')( + None, {'resource_id': resource_id} + ) + except logic.ValidationError: + pass + + base.redirect(core_helpers.url_for( + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data', + id=id, + resource_id=resource_id) + ) + + try: + c.pkg_dict = p.toolkit.get_action('package_show')( + None, {'id': id} + ) + c.resource = p.toolkit.get_action('resource_show')( + None, {'id': resource_id} + ) + except logic.NotFound: + base.abort(404, _('Resource not found')) + except logic.NotAuthorized: + base.abort(401, _('Unauthorized to edit this resource')) + + try: + datapusher_status = p.toolkit.get_action('datapusher_status')( + None, {'resource_id': resource_id} + ) + except logic.NotFound: + datapusher_status = {} + + return base.render('package/resource_data.html', + extra_vars={'status': datapusher_status}) + class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IConfigurable, inherit=True) @@ -24,6 +69,7 @@ class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.ITemplateHelpers) + p.implements(p.IRoutes, inherit=True) legacy_mode = False resource_show_action = None @@ -39,6 +85,11 @@ def configure(self, config): raise Exception( 'Config option `ckan.datapusher.url` has to be set.') + self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '') + if not datapusher_url: + raise Exception( + 'Config option `ckan.datapusher.secret_key` has to be set.') + def notify(self, entity, operation=None): if isinstance(entity, model.Resource): if (operation == model.domain_object.DomainObjectOperation.new @@ -63,6 +114,11 @@ def notify(self, entity, operation=None): log.critical(e) pass + def before_map(self, m): + m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}', + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data') + return m def get_actions(self): return {'datapusher_submit': action.datapusher_submit, diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 3bb5def4e73..f4a14a352c6 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -42,3 +42,4 @@ def dump(self, resource_id): for record in result['records']: wr.writerow([record[column] for column in header]) return f.getvalue() + From dcf8761053e91f635b1febd6e25e97722272cd38 Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 20 Sep 2013 18:25:05 +0100 Subject: [PATCH 048/159] [1200] pep8 and test failure fixups --- ckan/logic/action/get.py | 4 ++-- ckanext/datapusher/logic/action.py | 10 +++++----- ckanext/datapusher/plugin.py | 30 ++++++++++++++++-------------- ckanext/datapusher/tests/test.py | 20 +++++--------------- ckanext/datastore/controller.py | 1 - 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 7c9cdfddace..d02f6e1578b 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1834,11 +1834,11 @@ def task_status_show(context, data_dict): context['task_status'] = task_status + _check_access('task_status_show', context, data_dict) + if task_status is None: raise NotFound - _check_access('task_status_show', context, data_dict) - task_status_dict = model_dictize.task_status_dictize(task_status, context) return task_status_dict diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 653dd44e0b5..66c56d22c82 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -115,10 +115,9 @@ def datapusher_submit(context, data_dict): p.toolkit.get_action('task_status_update')(context, task) raise p.toolkit.ValidationError(error) - value = json.dumps( - {'job_id': r.json()['job_id'], - 'job_key': r.json()['job_key']} - ) + value = json.dumps({'job_id': r.json()['job_id'], + 'job_key': r.json()['job_key']}) + task['value'] = value task['state'] = 'pending' task['last_updated'] = str(datetime.datetime.now()), @@ -187,7 +186,8 @@ def datapusher_status(context, data_dict): r = requests.get(url, headers={'Content-Type': 'application/json', 'Authorization': job_key}) job_detail = r.json() - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: + except (requests.exceptions.ConnectionError, + requests.exceptions.HTTPError), e: job_detail = {'error': 'cannot connect to datapusher'} return { diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index a545323a7dd..2138497ad57 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -19,6 +19,7 @@ class DatastoreException(Exception): pass + class ResourceDataController(base.BaseController): def resource_data(self, id, resource_id): @@ -33,10 +34,10 @@ def resource_data(self, id, resource_id): pass base.redirect(core_helpers.url_for( - controller='ckanext.datapusher.plugin:ResourceDataController', - action='resource_data', - id=id, - resource_id=resource_id) + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data', + id=id, + resource_id=resource_id) ) try: @@ -59,7 +60,7 @@ def resource_data(self, id, resource_id): datapusher_status = {} return base.render('package/resource_data.html', - extra_vars={'status': datapusher_status}) + extra_vars={'status': datapusher_status}) class DatapusherPlugin(p.SingletonPlugin): @@ -77,15 +78,14 @@ class DatapusherPlugin(p.SingletonPlugin): def configure(self, config): self.config = config - datapusher_formats = config.get('ckan.datapusher.formats', '').lower().split() - self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS + datapusher_formats = config.get('ckan.datapusher.formats', '').lower() + self.datapusher_formats = datapusher_formats.split() or DEFAULT_FORMATS datapusher_url = config.get('ckan.datapusher.url') if not datapusher_url: raise Exception( 'Config option `ckan.datapusher.url` has to be set.') - self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '') if not datapusher_url: raise Exception( 'Config option `ckan.datapusher.secret_key` has to be set.') @@ -97,7 +97,8 @@ def notify(self, entity, operation=None): # if operation is None, resource URL has been changed, as # the notify function in IResourceUrlChange only takes # 1 parameter - context = {'model': model, 'ignore_auth': True, 'defer_commit': True} + context = {'model': model, 'ignore_auth': True, + 'defer_commit': True} package = p.toolkit.get_action('package_show')(context, { 'id': entity.get_package_id() }) @@ -109,15 +110,16 @@ def notify(self, entity, operation=None): 'resource_id': entity.id }) except p.toolkit.ValidationError, e: - # If datapusher is offline want to catch error instead of - # raising otherwise resource save will fail with 500 + # If datapusher is offline want to catch error instead + # of raising otherwise resource save will fail with 500 log.critical(e) pass def before_map(self, m): - m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}', - controller='ckanext.datapusher.plugin:ResourceDataController', - action='resource_data') + m.connect( + 'resource_data', '/dataset/{id}/resource_data/{resource_id}', + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data') return m def get_actions(self): diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index f47f256cef6..9e8e88a939c 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -131,7 +131,7 @@ def test_send_datapusher_creates_task(self): task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': resource.id, 'task_type': 'datapusher', - 'key': 'job_id' + 'key': 'datapusher' }) assert task['state'] == 'pending', task @@ -148,18 +148,8 @@ def test_datapusher_hook(self): 'entity_id': resource.id, 'entity_type': 'resource', 'task_type': 'datapusher', - 'key': 'job_id', - 'value': 'my_id', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - }) - - p.toolkit.get_action('task_status_update')(context, { - 'entity_id': resource.id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'key': 'job_key', - 'value': 'my_key', + 'key': 'datapusher', + 'value': '{"job_id": "my_id", "job_key":"my_key"}', 'last_updated': str(datetime.datetime.now()), 'state': 'pending' }) @@ -181,12 +171,12 @@ def test_datapusher_hook(self): task = tests.call_action_api( self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_id') + task_type='datapusher', key='datapusher') assert task['state'] == 'success', task task = tests.call_action_api( self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_key') + task_type='datapusher', key='datapusher') assert task['state'] == 'success', task diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index f4a14a352c6..3bb5def4e73 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -42,4 +42,3 @@ def dump(self, resource_id): for record in result['records']: wr.writerow([record[column] for column in header]) return f.getvalue() - From e39f6362e6c682703da7eb65f32ff7e7375e35ef Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 20 Sep 2013 21:21:23 +0100 Subject: [PATCH 049/159] [1200] fix task_staus in context --- ckanext/datapusher/tests/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index 9e8e88a939c..2ec2df42d21 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -128,6 +128,8 @@ def test_send_datapusher_creates_task(self): 'resource_id': resource.id }) + context.pop('task_status', None) + task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': resource.id, 'task_type': 'datapusher', From c8a483a77f82c4c8a545c569e87cd7843a3d92dc Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 30 Sep 2013 11:58:35 +0100 Subject: [PATCH 050/159] [#1221] use write url for search as this uses correct priviledges --- ckanext/datastore/logic/action.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 47a464518ce..d9d291b9df3 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -300,9 +300,7 @@ def datastore_search(context, data_dict): raise p.toolkit.ValidationError(errors) res_id = data_dict['resource_id'] - data_dict['connection_url'] = pylons.config.get( - 'ckan.datastore.read_url', - pylons.config['ckan.datastore.write_url']) + data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] resources_sql = sqlalchemy.text(u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') From c339162836695677f02e9090f2ce40f6a65ad194 Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 2 Oct 2013 16:59:33 +0200 Subject: [PATCH 051/159] [#1178] PEP8 (and correct a comment) --- ckan/logic/action/create.py | 3 +++ ckan/logic/action/delete.py | 1 + ckan/logic/auth/delete.py | 4 +++- ckan/model/follower.py | 13 +++++++------ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 371ca62e625..066ed7bcc87 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -841,6 +841,7 @@ def user_create(context, data_dict): log.debug('Created user {name}'.format(name=user.name)) return user_dict + def user_invite(context, data_dict): '''Invite a new user. @@ -889,6 +890,7 @@ def user_invite(context, data_dict): if 'name' not in e.error_dict: raise e + def _get_random_username_from_email(email): localpart = email.split('@')[0] cleaned_localpart = re.sub(r'[^\w]', '-', localpart) @@ -896,6 +898,7 @@ def _get_random_username_from_email(email): name = '%s-%d' % (cleaned_localpart, random_number) return name + ## Modifications for rest api def package_create_rest(context, data_dict): diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 9191151aa01..210f9d72387 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -18,6 +18,7 @@ _get_or_bust = ckan.logic.get_or_bust _get_action = ckan.logic.get_action + def user_delete(context, data_dict): '''Delete a user. diff --git a/ckan/logic/auth/delete.py b/ckan/logic/auth/delete.py index a2b121c11ad..08974144a71 100644 --- a/ckan/logic/auth/delete.py +++ b/ckan/logic/auth/delete.py @@ -4,10 +4,12 @@ from ckan.logic.auth import get_resource_object from ckan.lib.base import _ + def user_delete(context, data_dict): - # Only sysadmins are authorized to purge organizations. + # sysadmins only return {'success': False} + def package_delete(context, data_dict): user = context['user'] package = get_package_object(context, data_dict) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index a8afd07706f..eb54cd9ca67 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -1,7 +1,6 @@ import meta import datetime import sqlalchemy -import vdm.sqlalchemy import core import ckan.model @@ -78,15 +77,17 @@ def _get(cls, follower_id=None, object_id=None): object_id = object_id or cls.object_id query = meta.Session.query(cls, follower_alias, object_alias)\ - .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ - cls.follower_id == follower_alias.id,\ - cls.object_id == object_alias.id,\ - follower_alias.state != core.State.DELETED,\ - object_alias.state != core.State.DELETED,\ + .filter(sqlalchemy.and_( + follower_alias.id == follower_id, + cls.follower_id == follower_alias.id, + cls.object_id == object_alias.id, + follower_alias.state != core.State.DELETED, + object_alias.state != core.State.DELETED, object_alias.id == object_id)) return query + class UserFollowingUser(ModelFollowingModel): '''A many-many relationship between users. From 2e0266f12003244bc32795e82a9825574b93bf72 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 2 Oct 2013 16:04:32 +0100 Subject: [PATCH 052/159] [#1070] Makes recaptcha work with the jinja2 templates --- ckan/lib/app_globals.py | 2 +- ckan/public/base/less/forms.less | 5 +++++ ckan/templates/user/new_user_form.html | 5 +++++ ckan/templates/user/snippets/recaptcha.html | 12 ++++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 ckan/templates/user/snippets/recaptcha.html diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 0739274c671..1b95264ad1e 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -38,7 +38,7 @@ # has been setup in load_environment(): 'ckan.site_id': {}, 'ckan.recaptcha.publickey': {'name': 'recaptcha_publickey'}, - 'ckan.recaptcha.privatekey': {'name': 'recaptcha_publickey'}, + 'ckan.recaptcha.privatekey': {'name': 'recaptcha_privatekey'}, 'ckan.template_title_deliminater': {'default': '-'}, 'ckan.template_head_end': {}, 'ckan.template_footer_end': {}, diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..c9de5810ee4 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,8 @@ textarea { .box-shadow(none); } } + +#recaptcha_table { + table-layout: inherit; + line-height: 1; +} diff --git a/ckan/templates/user/new_user_form.html b/ckan/templates/user/new_user_form.html index 3879f2dde6c..57a0f1d85e0 100644 --- a/ckan/templates/user/new_user_form.html +++ b/ckan/templates/user/new_user_form.html @@ -7,6 +7,11 @@ {{ form.input("email", id="field-email", label=_("Email"), type="email", placeholder="joe@example.com", value=data.email, error=errors.email, classes=["control-medium"]) }} {{ form.input("password1", id="field-password", label=_("Password"), type="password", placeholder="••••••••", value=data.password1, error=errors.password1, classes=["control-medium"]) }} {{ form.input("password2", id="field-confirm-password", label=_("Confirm"), type="password", placeholder="••••••••", value=data.password2, error=errors.password1, classes=["control-medium"]) }} + + {% if g.recaptcha_publickey %} + {% snippet "user/snippets/recaptcha.html", public_key=g.recaptcha_publickey %} + {% endif %} +
          diff --git a/ckan/templates/user/snippets/recaptcha.html b/ckan/templates/user/snippets/recaptcha.html new file mode 100644 index 00000000000..994867dc97f --- /dev/null +++ b/ckan/templates/user/snippets/recaptcha.html @@ -0,0 +1,12 @@ +
          +
          + + + + +
          +
          From afc80ed59d6173a96980576fc4760702413ac8dc Mon Sep 17 00:00:00 2001 From: John Martin Date: Thu, 3 Oct 2013 10:49:20 +0100 Subject: [PATCH 053/159] [#1259] Fixes a JS error with autocomplete module and improves UX 1. Basically adds an extra check to make sure that 'abort()' is defined 2. Then changes the module to behave a little nicer when inbetween ajax requests --- .../base/javascript/modules/autocomplete.js | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js index 768a708c9c3..adea1628f49 100644 --- a/ckan/public/base/javascript/modules/autocomplete.js +++ b/ckan/public/base/javascript/modules/autocomplete.js @@ -86,6 +86,8 @@ this.ckan.module('autocomplete', function (jQuery, _) { $('.select2-choice', select2.container).on('click', function() { return false; }); + + this._select2 = select2; }, /* Looks up the completions for the current search term and passes them @@ -140,29 +142,30 @@ this.ckan.module('autocomplete', function (jQuery, _) { // old data. this._lastTerm = string; + // Kills previous timeout + clearTimeout(this._debounced); + + // OK, wipe the dropdown before we start ajaxing the completions + fn({results:[]}); + if (string) { - if (!this._debounced) { - // Set a timer to prevent the search lookup occurring too often. - this._debounced = setTimeout(function () { - var term = module._lastTerm; + // Set a timer to prevent the search lookup occurring too often. + this._debounced = setTimeout(function () { + var term = module._lastTerm; - delete module._debounced; + // Cancel the previous request if it hasn't yet completed. + if (module._last && typeof module._last.abort == 'function') { + module._last.abort(); + } - // Cancel the previous request if it hasn't yet completed. - if (module._last) { - module._last.abort(); - } + module._last = module.getCompletions(term, function (terms) { + fn(terms); + }); + }, this.options.interval); - module._last = module.getCompletions(term, function (terms) { - fn(module._lastResults = terms); - }); - }, this.options.interval); - } else { - // Re-use the last set of terms. - fn(this._lastResults || {results: []}); - } - } else { - fn({results: []}); + // This forces the ajax throbber to appear, because we've called the + // callback already and that hides the throbber + $('.select2-search input', this._select2.dropdown).addClass('select2-active'); } }, From 3d5fe71e8b17067f032784be75b932a352163fc6 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 7 Oct 2013 10:00:07 +0100 Subject: [PATCH 054/159] [1262] basic image upload functionality --- ckan/config/middleware.py | 6 ++ ckan/controllers/group.py | 28 ++++++++- ckan/lib/dictization/model_dictize.py | 21 +++++++ ckan/lib/helpers.py | 5 ++ ckan/lib/munge.py | 7 +++ ckan/lib/uploader.py | 63 +++++++++++++++++++ ckan/logic/schema.py | 1 + ckan/templates/group/snippets/group_form.html | 11 +++- ckan/templates/group/snippets/group_item.html | 2 +- .../snippets/organization_form.html | 10 ++- .../snippets/organization_item.html | 2 +- ckan/templates/snippets/group.html | 2 +- ckan/templates/snippets/group_item.html | 2 +- ckan/templates/snippets/organization.html | 2 +- .../templates/snippets/organization_item.html | 2 +- 15 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 ckan/lib/uploader.py diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 4df0aa84cb0..63f5568cd5f 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -147,6 +147,12 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): cache_max_age=static_max_age) static_parsers = [static_app, app] + storage_directory = config.get('ckan.storage_path') + if storage_directory: + storage_app = StaticURLParser(storage_directory, + cache_max_age=static_max_age) + static_parsers.insert(0, storage_app) + # Configurable extra static file paths extra_static_parsers = [] for public_path in config.get('extra_public_paths', '').split(','): diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ccef46bfe78..a11dd03cc40 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -1,14 +1,18 @@ import re +import os import logging import genshi +import cgi import datetime from urllib import urlencode from pylons.i18n import get_lang +from pylons import config import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.maintain as maintain +import ckan.lib.uploader as uploader import ckan.lib.navl.dictization_functions as dict_fns import ckan.logic as logic import ckan.lib.search as search @@ -421,6 +425,9 @@ def new(self, data=None, errors=None, error_summary=None): return self._save_new(context, group_type) data = data or {} + if not data.get('image_url', '').startswith('http'): + data.pop('image_url', None) + errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, @@ -445,10 +452,17 @@ def edit(self, id, data=None, errors=None, error_summary=None): return self._save_edit(id, context) try: + ## only needs removing when form repopulates + if data and not data.get('image_url', '').startswith('http'): + data.pop('image_url', None) old_data = self._action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') data = data or old_data + image_url = data.get('image_url') + if image_url and not image_url.startswith('http'): + data['upload_file'] = image_url + data['image_url'] = '' except NotFound: abort(404, _('Group not found')) except NotAuthorized: @@ -456,6 +470,7 @@ def edit(self, id, data=None, errors=None, error_summary=None): group = context.get("group") c.group = group + context.pop('for_edit') c.group_dict = self._action('group_show')(context, data_dict) try: @@ -483,12 +498,17 @@ def _get_group_type(self, id): def _save_new(self, context, group_type=None): try: + upload = uploader.Upload('group') + upload.register_request(request, 'image_url', + 'image_upload', 'clear_upload') + data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}] group = self._action('group_create')(context, data_dict) + upload.upload() # Redirect to the appropriate _read route for the type of group h.redirect_to(group['type'] + '_read', id=group['name']) @@ -514,13 +534,19 @@ def _force_reindex(self, grp): def _save_edit(self, id, context): try: + old_group = self._action('group_show')(context, {"id": id}) + old_image_url = old_group.get('image_url') + upload = uploader.Upload('group', old_image_url) + upload.register_request(request, 'image_url', + 'image_upload', 'clear_upload') + data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') data_dict['id'] = id context['allow_partial_update'] = True group = self._action('group_update')(context, data_dict) - + upload.upload() if id != group['name']: self._force_reindex(group) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index b3a83235cd6..98756abc2ec 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -10,6 +10,7 @@ import ckan.lib.dictization as d import ckan.new_authz as new_authz import ckan.lib.search as search +import ckan.lib.munge as munge ## package save @@ -41,6 +42,16 @@ def group_list_dictize(obj_list, context, group_dict['display_name'] = obj.display_name + image_url = group_dict.get('image_url') + group_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + group_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % group_dict.get('image_url') + ) + if obj.is_organization: group_dict['packages'] = query.facets['owner_org'].get(obj.id, 0) else: @@ -354,6 +365,16 @@ def group_dictize(group, context): for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) + if not context.get('for_edit'): + image_url = result_dict.get('image_url') + result_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + result_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % result_dict.get('image_url') + ) return result_dict def tag_list_dictize(tag_list, context): diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index c70d74d7dbc..fceab6f1b6a 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1663,6 +1663,10 @@ def new_activities(): action = logic.get_action('dashboard_new_activities_count') return action({}, {}) +def uploads_enabled(): + if config.get('ckan.storage_path'): + return True + return False # these are the functions that will end up in `h` template helpers __allowed_functions__ = [ @@ -1761,4 +1765,5 @@ def new_activities(): 'radio', 'submit', 'asbool', + 'uploads_enabled', ] diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index c943e1b113f..a47c8c02258 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -105,6 +105,13 @@ def munge_tag(tag): tag = _munge_to_length(tag, model.MIN_TAG_LENGTH, model.MAX_TAG_LENGTH) return tag +def munge_filename(filename): + filename = substitute_ascii_equivalents(filename) + filename = filename.lower().strip() + filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-') + filename = _munge_to_length(filename, 3, 100) + return filename + def _munge_to_length(string, min_length, max_length): '''Pad/truncates a string''' if len(string) < min_length: diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py new file mode 100644 index 00000000000..5fc144e3aef --- /dev/null +++ b/ckan/lib/uploader.py @@ -0,0 +1,63 @@ +import os +import cgi +import pylons +import datetime +import ckan.lib.munge as munge + +class Upload(object): + def __init__(self, object_type, old_filename=None): + path = pylons.config.get('ckan.storage_path', '/tmp') + self.storage_path = os.path.join(path, 'uploads', object_type) + try: + os.makedirs(self.storage_path) + except OSError, e: + pass + self.object_type = object_type + self.old_filename = old_filename + if old_filename: + self.old_filepath = os.path.join(self.storage_path, old_filename) + self.filename = None + self.filepath = None + + def register_request(self, request, url_field, file_field, clear_field): + + self.request = request + self.url = request.POST.get(url_field) + self.clear = request.POST.get(clear_field) + self.upload_field_storage = request.POST.pop(file_field, None) + + if isinstance(self.upload_field_storage, cgi.FieldStorage): + self.filename = self.upload_field_storage.filename + self.filename = str(datetime.datetime.utcnow()) + self.filename + self.filename = munge.munge_filename(self.filename) + self.filepath = os.path.join(self.storage_path, self.filename) + request.POST[url_field] = self.filename + self.upload_file = self.upload_field_storage.file + self.tmp_filepath = self.filepath + '~' + ### keep the file if there has been no change + if self.old_filename and not self.url and not self.clear: + request.POST[url_field] = self.old_filename + + + def upload(self): + + if self.filename: + output_file = open(self.tmp_filepath, 'wb') + self.upload_file.seek(0) + while True: + data = self.upload_file.read(2 ** 20) #mb chuncks + if not data: + break + output_file.write(data) + output_file.close() + os.rename(self.tmp_filepath, self.filepath) + self.clear = True + elif self.url: + self.clear = True + + if (self.clear and self.old_filename + and not self.old_filename.startswith('http')): + try: + os.remove(self.old_filepath) + except OSError, e: + pass diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..90465b7f7ec 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -262,6 +262,7 @@ def default_group_schema(): 'title': [ignore_missing, unicode], 'description': [ignore_missing, unicode], 'image_url': [ignore_missing, unicode], + 'image_display_url': [ignore_missing, unicode], 'type': [ignore_missing, unicode], 'state': [ignore_not_group_admin, ignore_missing], 'created': [ignore], diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 6d4d3216b26..1b9cac179c7 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} -
          + {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} @@ -21,6 +21,15 @@ {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + + {% if h.uploads_enabled() %} + {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} + {% set upload_file = data.get('upload_file', '') %} + {% if upload_file %} + {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + {% endblock %} {% block custom_fields %} diff --git a/ckan/templates/group/snippets/group_item.html b/ckan/templates/group/snippets/group_item.html index 9e32ce86e6a..34688b57c96 100644 --- a/ckan/templates/group/snippets/group_item.html +++ b/ckan/templates/group/snippets/group_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(group.type ~ '_read', action='read', id=group.name) %}
        • {% block image %} - {{ group.name }} + {{ group.name }} {% endblock %} {% block title %}

          {{ group.display_name }}

          diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 7b05e7d1542..7aee9ded32f 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} - + {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} @@ -21,6 +21,14 @@ {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {% if h.uploads_enabled() %} + {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} + {% set upload_file = data.get('upload_file', '') %} + {% if upload_file %} + {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + {% endblock %} {% block custom_fields %} diff --git a/ckan/templates/organization/snippets/organization_item.html b/ckan/templates/organization/snippets/organization_item.html index 818d5cc338a..8118fa9a1c6 100644 --- a/ckan/templates/organization/snippets/organization_item.html +++ b/ckan/templates/organization/snippets/organization_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
        • {% block image %} - {{ organization.name }} + {{ organization.name }} {% endblock %} {% block title %}

          {{ organization.display_name }}

          diff --git a/ckan/templates/snippets/group.html b/ckan/templates/snippets/group.html index db549fda75a..db758d9bc96 100644 --- a/ckan/templates/snippets/group.html +++ b/ckan/templates/snippets/group.html @@ -14,7 +14,7 @@
          - {{ group.name }} + {{ group.name }}

          {{ group.title or group.name }}

          diff --git a/ckan/templates/snippets/group_item.html b/ckan/templates/snippets/group_item.html index 74c68fbca28..60f43f63b79 100644 --- a/ckan/templates/snippets/group_item.html +++ b/ckan/templates/snippets/group_item.html @@ -5,7 +5,7 @@ {% set truncate_title = truncate_title or 0 %} {% set title = group.title or group.name %} - {{ group.name }} + {{ group.name }}

          {{ group.title or group.name }}

          {% if group.description %} diff --git a/ckan/templates/snippets/organization.html b/ckan/templates/snippets/organization.html index ef08a9ed440..e0f5417024a 100644 --- a/ckan/templates/snippets/organization.html +++ b/ckan/templates/snippets/organization.html @@ -23,7 +23,7 @@

          {{ _('Organization') }} {% block image %}
          - {{ organization.name }} + {{ organization.name }}
          {% endblock %} diff --git a/ckan/templates/snippets/organization_item.html b/ckan/templates/snippets/organization_item.html index 0213723defa..06579756f32 100644 --- a/ckan/templates/snippets/organization_item.html +++ b/ckan/templates/snippets/organization_item.html @@ -3,7 +3,7 @@ {% set url=h.url_for(controller='organization', action='read', id=organization.name) %} {% set truncate=truncate or 0 %} - {{ organization.name }} + {{ organization.name }}

          {{ organization.title or organization.name }}

          {% if organization.description %} From 4b2c46b59bd6cb5ed1235b60e1555d776c5456b2 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 7 Oct 2013 17:46:18 +0100 Subject: [PATCH 055/159] [#1265] Keep visibility value on form errors The wrong value was checked to choose the selected option. --- ckan/templates/package/snippets/package_basic_fields.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/snippets/package_basic_fields.html b/ckan/templates/package/snippets/package_basic_fields.html index 044b9ebe5bc..d1ac7e25577 100644 --- a/ckan/templates/package/snippets/package_basic_fields.html +++ b/ckan/templates/package/snippets/package_basic_fields.html @@ -86,8 +86,8 @@
          From dad667eb4233b00561167db2aff41092afecfd2d Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 7 Oct 2013 13:35:25 -0700 Subject: [PATCH 056/159] Correct encoding to make coveralls work --- ckan/config/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 49be9b726a6..409f77435a4 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -159,7 +159,7 @@ def find_controller(self, controller): ''' This code is based on Genshi code - Copyright © 2006-2012 Edgewall Software + Copyright © 2006-2012 Edgewall Software All rights reserved. Redistribution and use in source and binary forms, with or From a903ef999d8d67db48118ea8413595707c2e3080 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 7 Oct 2013 14:20:57 -0700 Subject: [PATCH 057/159] Add test coverage banner to readme --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 93d80e668cc..2eafd662e07 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,10 @@ CKAN: The Open Source Data Portal Software :target: http://travis-ci.org/okfn/ckan :alt: Build Status +.. image:: https://coveralls.io/repos/okfn/ckan/badge.png?branch=coveralls + :target: https://coveralls.io/r/okfn/ckan?branch=coveralls + :alt: Test coverage + **CKAN is the world’s leading open-source data portal platform**. CKAN makes it easy to publish, share and work with data. It's a data management system that provides a powerful platform for cataloging, storing and accessing From 5ad2c9bc8d651c31b302499780b6926b7a798ebc Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 7 Oct 2013 14:25:57 -0700 Subject: [PATCH 058/159] Don't cover tests files --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 28f34e39273..ed2c3b6bc6f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,3 @@ [run] -omit = /ckan/migration/* +omit = /ckan/migration/*, /ckan/tests/*, */tests/* source = ckan, ckanext From 66105d360e010114751ff0280b07196e6f73445a Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 8 Oct 2013 03:02:12 +0100 Subject: [PATCH 059/159] [#1262] fix how url field is handled --- ckan/config/middleware.py | 9 ++++++++- ckan/controllers/group.py | 7 ------- ckan/lib/dictization/model_dictize.py | 20 +++++++++---------- ckan/lib/uploader.py | 16 +++++++-------- ckan/templates/group/snippets/group_form.html | 7 +++---- ckan/templates/group/snippets/info.html | 2 +- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 63f5568cd5f..7699596138d 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -4,6 +4,7 @@ import logging import json import hashlib +import os import sqlalchemy as sa from beaker.middleware import CacheMiddleware, SessionMiddleware @@ -149,7 +150,13 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): storage_directory = config.get('ckan.storage_path') if storage_directory: - storage_app = StaticURLParser(storage_directory, + path = os.path.join(storage_directory, 'storage') + try: + os.makedirs(path) + except OSError, e: + pass + + storage_app = StaticURLParser(path, cache_max_age=static_max_age) static_parsers.insert(0, storage_app) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index a11dd03cc40..ed12edcb082 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -452,17 +452,10 @@ def edit(self, id, data=None, errors=None, error_summary=None): return self._save_edit(id, context) try: - ## only needs removing when form repopulates - if data and not data.get('image_url', '').startswith('http'): - data.pop('image_url', None) old_data = self._action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') data = data or old_data - image_url = data.get('image_url') - if image_url and not image_url.startswith('http'): - data['upload_file'] = image_url - data['image_url'] = '' except NotFound: abort(404, _('Group not found')) except NotAuthorized: diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 98756abc2ec..019e648140b 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -365,16 +365,16 @@ def group_dictize(group, context): for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) - if not context.get('for_edit'): - image_url = result_dict.get('image_url') - result_dict['image_display_url'] = image_url - if image_url and not image_url.startswith('http'): - #munge here should not have an effect only doing it incase - #of potential vulnerability of dodgy api input - image_url = munge.munge_filename(image_url) - result_dict['image_display_url'] = h.url_for_static( - 'uploads/group/%s' % result_dict.get('image_url') - ) + image_url = result_dict.get('image_url') + result_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + result_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % result_dict.get('image_url'), + qualified = True + ) return result_dict def tag_list_dictize(tag_list, context): diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 5fc144e3aef..bcfdd7e9ecc 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -7,7 +7,7 @@ class Upload(object): def __init__(self, object_type, old_filename=None): path = pylons.config.get('ckan.storage_path', '/tmp') - self.storage_path = os.path.join(path, 'uploads', object_type) + self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) try: os.makedirs(self.storage_path) except OSError, e: @@ -20,9 +20,7 @@ def __init__(self, object_type, old_filename=None): self.filepath = None def register_request(self, request, url_field, file_field, clear_field): - - self.request = request - self.url = request.POST.get(url_field) + self.url = request.POST.get(url_field, '') self.clear = request.POST.get(clear_field) self.upload_field_storage = request.POST.pop(file_field, None) @@ -35,12 +33,14 @@ def register_request(self, request, url_field, file_field, clear_field): self.upload_file = self.upload_field_storage.file self.tmp_filepath = self.filepath + '~' ### keep the file if there has been no change - if self.old_filename and not self.url and not self.clear: - request.POST[url_field] = self.old_filename + elif self.old_filename and not self.old_filename.startswith('http'): + if not self.clear: + request.POST[url_field] = self.old_filename + if self.clear and self.url == self.old_filename: + request.POST[url_field] = '' def upload(self): - if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) @@ -52,8 +52,6 @@ def upload(self): output_file.close() os.rename(self.tmp_filepath, self.filepath) self.clear = True - elif self.url: - self.clear = True if (self.clear and self.old_filename and not self.old_filename.startswith('http')): diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 1b9cac179c7..322f9ebff8a 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -19,14 +19,13 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if h.uploads_enabled() %} {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% set upload_file = data.get('upload_file', '') %} - {% if upload_file %} - {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% if data.image_url and not data.image_url.startswith('http') %} + {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} diff --git a/ckan/templates/group/snippets/info.html b/ckan/templates/group/snippets/info.html index b7c749ab147..dc86036bb4c 100644 --- a/ckan/templates/group/snippets/info.html +++ b/ckan/templates/group/snippets/info.html @@ -2,7 +2,7 @@
          - {{ group.name }} + {{ group.name }}

          {{ group.display_name }}

          From 1ab6ff449faef790a4f160d30aedd08a82b726fb Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 8 Oct 2013 13:06:11 +0100 Subject: [PATCH 060/159] [#1268] Add 'python setup.py develop' step to source upgrade instructions --- doc/upgrade-source.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/upgrade-source.rst b/doc/upgrade-source.rst index af70ac4ea05..539b96dcb62 100644 --- a/doc/upgrade-source.rst +++ b/doc/upgrade-source.rst @@ -37,6 +37,12 @@ CKAN release you're upgrading to: pip install --upgrade -r requirements.txt +#. Register any new or updated plugins: + + :: + + python setup.py develop + #. If you are upgrading to a new :ref:`major release ` you need to update your Solr schema symlink. From 23cfdbee2fcbc9267f66f06b6ecf9a879c92241f Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 8 Oct 2013 16:44:23 +0100 Subject: [PATCH 061/159] [#1269] First working version of the proper IA for resources --- ckan/config/routing.py | 7 ++-- ckan/templates/package/edit.html | 29 ++------------ ckan/templates/package/edit_base.html | 23 +++++++++++ ckan/templates/package/new_resource.html | 29 ++++---------- ckan/templates/package/read_base.html | 15 +------ ckan/templates/package/resource_edit.html | 25 ++---------- .../templates/package/resource_edit_base.html | 28 +++++++++++++ ckan/templates/package/snippets/info.html | 40 ++++++++----------- 8 files changed, 85 insertions(+), 111 deletions(-) create mode 100644 ckan/templates/package/edit_base.html create mode 100644 ckan/templates/package/resource_edit_base.html diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 0fbb02bb886..33175a35c12 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -222,7 +222,6 @@ def make_map(): ]))) m.connect('/dataset/{action}/{id}', requirements=dict(action='|'.join([ - 'edit', 'new_metadata', 'new_resource', 'history', @@ -234,6 +233,8 @@ def make_map(): 'delete', 'api_data', ]))) + m.connect('dataset_edit', '/dataset/edit/{id}', action='edit', + ckan_icon='edit') m.connect('dataset_followers', '/dataset/followers/{id}', action='followers', ckan_icon='group') m.connect('dataset_activity', '/dataset/activity/{id}', @@ -246,8 +247,8 @@ def make_map(): action='resource_read') m.connect('/dataset/{id}/resource_delete/{resource_id}', action='resource_delete') - m.connect('/dataset/{id}/resource_edit/{resource_id}', - action='resource_edit') + m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}', + action='resource_edit', ckan_icon='edit') m.connect('/dataset/{id}/resource/{resource_id}/download', action='resource_download') m.connect('/dataset/{id}/resource/{resource_id}/embed', diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html index 4f15985d033..dec84769ac4 100644 --- a/ckan/templates/package/edit.html +++ b/ckan/templates/package/edit.html @@ -1,28 +1,5 @@ -{% extends 'package/base_form_page.html' %} +{% extends 'package/edit_base.html' %} -{% set pkg = c.pkg_dict %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
        • {% link_for _('Edit'), controller='package', action='edit', id=pkg.name %}
          • -{% endblock %} - -{% block resources_module %} - {% snippet 'package/snippets/resources.html', pkg=pkg, action='resource_edit' %} -{% endblock %} - -{% block content_action %} - {% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %} +{% block primary_content_inner %} + {% block form %}{{ c.form | safe }}{% endblock %} {% endblock %} - -{% block secondary_content %} - {% snippet 'package/snippets/info.html', pkg=pkg, action='package_edit' %} -{% endblock %} - -{% block primary_content %} -
          - {{ super() }} -{% endblock %} - diff --git a/ckan/templates/package/edit_base.html b/ckan/templates/package/edit_base.html new file mode 100644 index 00000000000..df1d91dfb02 --- /dev/null +++ b/ckan/templates/package/edit_base.html @@ -0,0 +1,23 @@ +{% extends 'package/base.html' %} + +{% set pkg = c.pkg_dict %} + +{% block breadcrumb_content_selected %}{% endblock %} + +{% block breadcrumb_content %} + {{ super() }} +
        • {% link_for _('Edit'), controller='package', action='edit', id=pkg.name %}
          • +{% endblock %} + +{% block content_action %} + {% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_edit', _('Edit'), id=pkg.name) }} + {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %} +{% endblock %} diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 13cce195453..49d02387768 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -1,10 +1,6 @@ {% extends "package/base_form_page.html" %} -{% if c.userobj %} - {% set logged_in = true %} -{% else %} - {% set logged_in = false %} -{% endif %} +{% set logged_in = true if c.userobj else false %} {% block subtitle %}{{ _('Add data to the dataset') }}{% endblock %} @@ -17,26 +13,15 @@ {% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=g.ofs_impl and logged_in %}{% endblock %} {% block secondary_content %} - {% if pkg_dict and pkg_dict.state != 'draft' %} - {% snippet 'package/snippets/info.html', pkg=pkg_dict, action='resource_new' %} - {% else %} -
          -

          {{ _('What\'s a resource?') }}

          -
          -

          {{ _('A resource can be any file or link to a file containing useful data.') }}

          -
          -
          - {% endif %} +
          +

          {{ _('What\'s a resource?') }}

          +
          +

          {{ _('A resource can be any file or link to a file containing useful data.') }}

          +
          +
          {% endblock %} {% block scripts %} {{ super() }} {% resource 'vendor/fileupload' %} {% endblock %} - -{% block primary_content %} - {% if pkg_dict and pkg_dict.state != 'draft' %} -
          - {% endif %} - {{ super() }} -{% endblock %} diff --git a/ckan/templates/package/read_base.html b/ckan/templates/package/read_base.html index 9cf73a5c5a1..f0a58a21018 100644 --- a/ckan/templates/package/read_base.html +++ b/ckan/templates/package/read_base.html @@ -50,20 +50,7 @@ {% block secondary_help_content %}{% endblock %} {% block package_info %} -
          -
          -

          {{ pkg.title or pkg.name }}

          -
          -
          -
          {{ _('Followers') }}
          -
          {{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}
          -
          -
          - -
          -
          + {% snippet 'package/snippets/info.html', pkg=pkg %} {% endblock %} {% block package_organization %} diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index 52ac2ab4722..ab4b9aaa606 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -1,24 +1,5 @@ -{% extends "package/new_resource.html" %} +{% extends "package/resource_edit_base.html" %} -{% set pkg_dict = c.pkg_dict %} -{% set res = c.resource %} - -{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
        • {{ _('Edit') }}
          • -{% endblock %} - -{% block content_action %} - {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} -{% endblock %} - -{# logged_in is defined in new_resource.html #} -{% block form %}{{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }}{% endblock %} - -{% block secondary_content %} - {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=data.id, action='resource_edit' %} +{% block primary_content_inner %} + {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} {% endblock %} diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html new file mode 100644 index 00000000000..fba67b43c80 --- /dev/null +++ b/ckan/templates/package/resource_edit_base.html @@ -0,0 +1,28 @@ +{% extends "package/base.html" %} + +{% set logged_in = true if c.userobj else false %} +{% set pkg_dict = c.pkg_dict %} +{% set res = c.resource %} + +{% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg_dict) }}{% endblock %} + +{% block breadcrumb_content_selected %}{% endblock %} + +{% block breadcrumb_content %} + {{ super() }} +
        • {% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id %}
          • +
        • {{ _('Edit') }}
          • +{% endblock %} + +{% block content_action %} + {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('resource_edit', _('Edit'), id=pkg.name, resource_id=res.id) }} + {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %} +{% endblock %} diff --git a/ckan/templates/package/snippets/info.html b/ckan/templates/package/snippets/info.html index 49efa7b8fc7..b12ce3f6998 100644 --- a/ckan/templates/package/snippets/info.html +++ b/ckan/templates/package/snippets/info.html @@ -2,33 +2,25 @@ Displays a sidebard module with information for given package pkg - The package dict that owns the resources. -active - The active resource. -action - The action that this is coming from. Example: {% snippet "package/snippets/info.html", pkg=pkg %} #} -{% if pkg and h.check_access('package_update', {'id':pkg.id }) %} -
          -

          {{ _("Edit Dataset") }}

          -
            -
          • - {% link_for h.dataset_display_name(pkg)|truncate(30), controller='package', action='edit', id=pkg.name %} -
            • -
          - {% set resources = pkg.resources or [] %} -

          {{ _("Edit Resources") }}

          -
            - {% block package_resource_list %} - {% for resource in resources %} -
          • - {% link_for h.resource_display_name(resource)|truncate(30), controller='package', action='resource_edit', id=pkg.name, resource_id=resource.id, inner_span=true %} -
            • - {% endfor %} - {% endblock %} -
          • {{ _('Add New Resource') }}
            • -
          -
          -{% endif %} +
          +
          +

          {{ pkg.title or pkg.name }}

          +
          +
          +
          {{ _('Followers') }}
          +
          {{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}
          +
          +
          + {% if not hide_follow_button %} + + {% endif %} +
          +
          From 1b62423d9d013dbc4481bbddd5a1037b49e52529 Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 9 Oct 2013 03:12:00 +0100 Subject: [PATCH 062/159] [1262] use logic instead of requests --- ckan/controllers/group.py | 11 ----------- ckan/lib/uploader.py | 14 +++++++------- ckan/logic/action/create.py | 7 ++++++- ckan/logic/action/update.py | 7 +++++++ .../organization/snippets/organization_form.html | 7 +++---- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ed12edcb082..adbd3d7cacc 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -12,7 +12,6 @@ import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.maintain as maintain -import ckan.lib.uploader as uploader import ckan.lib.navl.dictization_functions as dict_fns import ckan.logic as logic import ckan.lib.search as search @@ -491,17 +490,12 @@ def _get_group_type(self, id): def _save_new(self, context, group_type=None): try: - upload = uploader.Upload('group') - upload.register_request(request, 'image_url', - 'image_upload', 'clear_upload') - data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}] group = self._action('group_create')(context, data_dict) - upload.upload() # Redirect to the appropriate _read route for the type of group h.redirect_to(group['type'] + '_read', id=group['name']) @@ -529,17 +523,12 @@ def _save_edit(self, id, context): try: old_group = self._action('group_show')(context, {"id": id}) old_image_url = old_group.get('image_url') - upload = uploader.Upload('group', old_image_url) - upload.register_request(request, 'image_url', - 'image_upload', 'clear_upload') - data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') data_dict['id'] = id context['allow_partial_update'] = True group = self._action('group_update')(context, data_dict) - upload.upload() if id != group['name']: self._force_reindex(group) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index bcfdd7e9ecc..9b7cea2922e 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -19,25 +19,25 @@ def __init__(self, object_type, old_filename=None): self.filename = None self.filepath = None - def register_request(self, request, url_field, file_field, clear_field): - self.url = request.POST.get(url_field, '') - self.clear = request.POST.get(clear_field) - self.upload_field_storage = request.POST.pop(file_field, None) + def update_data_dict(self, data_dict, url_field, file_field, clear_field): + self.url = data_dict.get(url_field, '') + self.clear = data_dict.pop(clear_field, None) + self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename self.filename = str(datetime.datetime.utcnow()) + self.filename self.filename = munge.munge_filename(self.filename) self.filepath = os.path.join(self.storage_path, self.filename) - request.POST[url_field] = self.filename + data_dict[url_field] = self.filename self.upload_file = self.upload_field_storage.file self.tmp_filepath = self.filepath + '~' ### keep the file if there has been no change elif self.old_filename and not self.old_filename.startswith('http'): if not self.clear: - request.POST[url_field] = self.old_filename + data_dict[url_field] = self.old_filename if self.clear and self.url == self.old_filename: - request.POST[url_field] = '' + data_dict[url_field] = '' def upload(self): diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index baac5a2154c..702a6028d99 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -15,6 +15,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions +import ckan.lib.uploader as uploader from ckan.common import _ @@ -446,6 +447,7 @@ def member_create(context, data_dict=None): if not obj: raise NotFound('%s was not found.' % obj_type.title()) + # User must be able to update the group to add a member to it _check_access('group_update', context, data_dict) @@ -475,7 +477,9 @@ def _group_or_org_create(context, data_dict, is_org=False): parent = context.get('parent', None) data_dict['is_organization'] = is_org - + upload = uploader.Upload('group') + upload.update_data_dict(data_dict, 'image_url', + 'image_upload', 'clear_upload') # get the schema group_plugin = lib_plugins.lookup_group_plugin( group_type=data_dict.get('type')) @@ -561,6 +565,7 @@ def _group_or_org_create(context, data_dict, is_org=False): logic.get_action('activity_create')(activity_create_context, activity_dict) + upload.upload() if not context.get('defer_commit'): model.repo.commit() context["group"] = group diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 3db2d0d19cf..f8aee9f62fb 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -19,6 +19,7 @@ import ckan.lib.plugins as lib_plugins import ckan.lib.email_notifications as email_notifications import ckan.lib.search as search +import ckan.lib.uploader as uploader from ckan.common import _, request @@ -424,6 +425,10 @@ def _group_or_org_update(context, data_dict, is_org=False): except AttributeError: schema = group_plugin.form_to_db_schema() + upload = uploader.Upload('group', group.image_url) + upload.update_data_dict(data_dict, 'image_url', + 'image_upload', 'clear_upload') + if is_org: _check_access('organization_update', context, data_dict) else: @@ -528,9 +533,11 @@ def _group_or_org_update(context, data_dict, is_org=False): # TODO: Also create an activity detail recording what exactly changed # in the group. + upload.upload() if not context.get('defer_commit'): model.repo.commit() + return model_dictize.group_dictize(group, context) def group_update(context, data_dict): diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 7aee9ded32f..0a2134e89f7 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -19,13 +19,12 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if h.uploads_enabled() %} {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% set upload_file = data.get('upload_file', '') %} - {% if upload_file %} - {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% if data.image_url and not data.image_url.startswith('http') %} + {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} From 8f8d2ab1a502100d7da4358ea2947fbf9a5c4adb Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 9 Oct 2013 11:28:26 +0100 Subject: [PATCH 063/159] 1200 make sure all users can submit job --- ckanext/datapusher/logic/action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 66c56d22c82..cfbd4620d09 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -71,6 +71,7 @@ def datapusher_submit(context, data_dict): except logic.NotFound: pass + context['ignore_auth'] = True result = p.toolkit.get_action('task_status_update')(context, task) task_id = result['id'] From 48041a4e40e837f3f84ea9bb4f7cd94918daa21a Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 9 Oct 2013 12:03:04 +0100 Subject: [PATCH 064/159] [#1269] Adds resource listing page within edit dataset section - Also tidys up the edit dataset templats - Adds a new state to the new resource action that uses a different template when dataset is no in draft mode (e.g. it's not part of the dataset wizard creation any more) --- ckan/config/routing.py | 2 ++ ckan/controllers/package.py | 30 ++++++++++++++++++- ckan/templates/package/edit.html | 2 ++ ckan/templates/package/edit_base.html | 4 +-- .../package/new_resource_not_draft.html | 8 +++++ ckan/templates/package/resource_edit.html | 8 +++-- .../templates/package/resource_edit_base.html | 28 +++++------------ ckan/templates/package/resources.html | 21 +++++++++++++ .../package/snippets/resource_item.html | 15 +++++++++- 9 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 ckan/templates/package/new_resource_not_draft.html create mode 100644 ckan/templates/package/resources.html diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 33175a35c12..9ef4e0895eb 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -241,6 +241,8 @@ def make_map(): action='activity', ckan_icon='time') m.connect('/dataset/activity/{id}/{offset}', action='activity') m.connect('/dataset/{id}.{format}', action='read') + m.connect('dataset_resources', '/dataset/resources/{id}', + action='resources', ckan_icon='reorder') m.connect('dataset_read', '/dataset/{id}', action='read', ckan_icon='sitemap') m.connect('/dataset/{id}/resource/{resource_id}', diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index d1852cd95b7..8468dfa3947 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -301,6 +301,31 @@ def _content_type_from_accept(self): ct, mu, ext = accept.parse_header(request.headers.get('Accept', '')) return ct, ext, (NewTextTemplate, MarkupTemplate)[mu] + def resources(self, id): + package_type = self._get_package_type(id.split('@')[0]) + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'for_view': True, + 'auth_user_obj': c.userobj} + data_dict = {'id': id} + + try: + check_access('package_update', context) + except NotAuthorized, e: + abort(401, _('User %r not authorized to edit %s') % (c.user, id)) + # check if package exists + try: + c.pkg_dict = get_action('package_show')(context, data_dict) + c.pkg = context['package'] + except NotFound: + abort(404, _('Dataset not found')) + except NotAuthorized: + abort(401, _('Unauthorized to read package %s') % id) + + self._setup_template_variables(context, {'id': id}, + package_type=package_type) + + return render('package/resources.html') + def read(self, id, format='html'): if not format == 'html': ctype, extension, loader = \ @@ -666,11 +691,14 @@ def new_resource(self, id, data=None, errors=None, error_summary=None): abort(404, _('The dataset {id} could not be found.').format(id=id)) # required for nav menu vars['pkg_dict'] = pkg_dict + template = 'package/new_resource_not_draft.html' if pkg_dict['state'] == 'draft': vars['stage'] = ['complete', 'active'] + template = 'package/new_resource.html' elif pkg_dict['state'] == 'draft-complete': vars['stage'] = ['complete', 'active', 'complete'] - return render('package/new_resource.html', extra_vars=vars) + template = 'package/new_resource.html' + return render(template, extra_vars=vars) def new_metadata(self, id, data=None, errors=None, error_summary=None): ''' FIXME: This is a temporary action to allow styling of the diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html index dec84769ac4..565623f7ebd 100644 --- a/ckan/templates/package/edit.html +++ b/ckan/templates/package/edit.html @@ -1,5 +1,7 @@ {% extends 'package/edit_base.html' %} +{% block subtitle %}{{ _('Edit') }} - {{ h.dataset_display_name(pkg) }}{% endblock %} + {% block primary_content_inner %} {% block form %}{{ c.form | safe }}{% endblock %} {% endblock %} diff --git a/ckan/templates/package/edit_base.html b/ckan/templates/package/edit_base.html index df1d91dfb02..e8b785712b3 100644 --- a/ckan/templates/package/edit_base.html +++ b/ckan/templates/package/edit_base.html @@ -14,8 +14,8 @@ {% endblock %} {% block content_primary_nav %} - {{ h.build_nav_icon('dataset_edit', _('Edit'), id=pkg.name) }} - {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} + {{ h.build_nav_icon('dataset_edit', _('Edit metadata'), id=pkg.name) }} + {{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }} {% endblock %} {% block secondary_content %} diff --git a/ckan/templates/package/new_resource_not_draft.html b/ckan/templates/package/new_resource_not_draft.html new file mode 100644 index 00000000000..a24f37ce459 --- /dev/null +++ b/ckan/templates/package/new_resource_not_draft.html @@ -0,0 +1,8 @@ +{% extends "package/resource_edit_base.html" %} + +{% block subtitle %}{{ _('Add resource') }} - {{ h.dataset_display_name(pkg) }}{% endblock %} +{% block form_title %}{{ _('Add resource') }}{% endblock %} + +{% block form %} + {% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=g.ofs_impl and logged_in %} +{% endblock %} diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index ab4b9aaa606..9756c8f9581 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -1,5 +1,9 @@ {% extends "package/resource_edit_base.html" %} -{% block primary_content_inner %} - {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} +{% set res = c.resource %} + +{% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg) }}{% endblock %} + +{% block form %} + {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} {% endblock %} diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html index fba67b43c80..7c3a8d012ec 100644 --- a/ckan/templates/package/resource_edit_base.html +++ b/ckan/templates/package/resource_edit_base.html @@ -1,28 +1,16 @@ -{% extends "package/base.html" %} +{% extends "package/edit_base.html" %} {% set logged_in = true if c.userobj else false %} -{% set pkg_dict = c.pkg_dict %} -{% set res = c.resource %} - -{% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg_dict) }}{% endblock %} {% block breadcrumb_content_selected %}{% endblock %} -{% block breadcrumb_content %} - {{ super() }} -
        • {% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id %}
          • -
        • {{ _('Edit') }}
          • -{% endblock %} - -{% block content_action %} - {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% block primary_content_inner %} + {% link_for _('View all resources'), controller='package', action='resources', id=pkg.name, class_='btn pull-right', icon='arrow-left' %} +

          {% block form_title %}{{ _('Edit resource') }}{% endblock %}

          + {% block form %}{% endblock %} {% endblock %} -{% block content_primary_nav %} - {{ h.build_nav_icon('resource_edit', _('Edit'), id=pkg.name, resource_id=res.id) }} - {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} -{% endblock %} - -{% block secondary_content %} - {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %} +{% block scripts %} + {{ super() }} + {% resource 'vendor/fileupload' %} {% endblock %} diff --git a/ckan/templates/package/resources.html b/ckan/templates/package/resources.html new file mode 100644 index 00000000000..e353d03f10b --- /dev/null +++ b/ckan/templates/package/resources.html @@ -0,0 +1,21 @@ +{% extends "package/edit_base.html" %} + +{% block subtitle %}{{ _('Resources') }} - {{ h.dataset_display_name(pkg) }}{% endblock %} + +{% block page_primary_action %} + {% link_for _('Add new resource'), controller='package', action='new_resource', id=c.pkg_dict.name, class_='btn btn-primary', icon='plus' %} +{% endblock %} + +{% block primary_content_inner %} + {% if pkg.resources %} +
            + {% for resource in pkg.resources %} + {% snippet 'package/snippets/resource_item.html', pkg=pkg, res=resource, url_is_edit=true %} + {% endfor %} +
          + {% else %} + {% trans url=h.url_for(controller='package', action='new_resource', id=pkg.name) %} +

          This dataset has no data, why not add some?

          + {% endtrans %} + {% endif %} +{% endblock %} diff --git a/ckan/templates/package/snippets/resource_item.html b/ckan/templates/package/snippets/resource_item.html index fccd94ff4d2..c5b22521d64 100644 --- a/ckan/templates/package/snippets/resource_item.html +++ b/ckan/templates/package/snippets/resource_item.html @@ -1,4 +1,7 @@ -{% set url = h.url_for(controller='package', action='resource_read', id=pkg.name, resource_id=res.id) %} +{% set can_edit = h.check_access('package_update', {'id':pkg.id }) %} +{% set url_action = 'resource_edit' if url_is_edit and can_edit else 'resource_read' %} +{% set url = h.url_for(controller='package', action=url_action, id=pkg.name, resource_id=res.id) %} +
        • {% block resource_item_title %} @@ -14,6 +17,7 @@ {% endif %}

          {% block resource_item_explore %} + {% if not url_is_edit %}
          @@ -44,8 +48,17 @@ {% endif %}
          • + {% if can_edit %} +
        • + + + {{ _('Edit') }} + +
          • + {% endif %} {% endblock %}
        + {% endif %} {% endblock %} From 485f681f6f5d5fe890dc339d57a8563844e2fde2 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Wed, 9 Oct 2013 14:02:39 +0100 Subject: [PATCH 065/159] [#1272] prevent keyerror when looking up 'url_type' in datastore before_show --- ckanext/datastore/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 4477ad70526..04f201f9eb0 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -285,7 +285,7 @@ def before_show(self, resource_dict): ''' Modify the resource url of datastore resources so that they link to the datastore dumps. ''' - if resource_dict['url_type'] == 'datastore': + if resource_dict.get('url_type') == 'datastore': resource_dict['url'] = p.toolkit.url_for( controller='ckanext.datastore.controller:DatastoreController', action='dump', resource_id=resource_dict['id']) From c4d673327c6b83e01ead0091246cecb83875160a Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 10 Oct 2013 20:40:49 +0100 Subject: [PATCH 066/159] [#1262] fix test failures --- ckan/lib/uploader.py | 12 +++++++----- ckan/tests/lib/test_dictization.py | 2 ++ ckan/tests/lib/test_dictization_schema.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 9b7cea2922e..eb0f28eec54 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -4,10 +4,12 @@ import datetime import ckan.lib.munge as munge + class Upload(object): def __init__(self, object_type, old_filename=None): path = pylons.config.get('ckan.storage_path', '/tmp') - self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) + self.storage_path = os.path.join(path, 'storage', + 'uploads', object_type) try: os.makedirs(self.storage_path) except OSError, e: @@ -22,7 +24,7 @@ def __init__(self, object_type, old_filename=None): def update_data_dict(self, data_dict, url_field, file_field, clear_field): self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) - self.upload_field_storage = data_dict.pop(file_field, None) + self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename @@ -39,13 +41,13 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): if self.clear and self.url == self.old_filename: data_dict[url_field] = '' - def upload(self): if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) while True: - data = self.upload_file.read(2 ** 20) #mb chuncks + # mb chuncks + data = self.upload_file.read(2 ** 20) if not data: break output_file.write(data) @@ -54,7 +56,7 @@ def upload(self): self.clear = True if (self.clear and self.old_filename - and not self.old_filename.startswith('http')): + and not self.old_filename.startswith('http')): try: os.remove(self.old_filepath) except OSError, e: diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index 57f29963910..dcf955d3cc5 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -923,6 +923,7 @@ def test_16_group_dictized(self): 'capacity' : 'public', 'display_name': u'simple', 'image_url': u'', + 'image_display_url': u'', 'name': u'simple', 'packages': 0, 'state': u'active', @@ -944,6 +945,7 @@ def test_16_group_dictized(self): 'name': u'help', 'display_name': u'help', 'image_url': u'', + 'image_display_url': u'', 'package_count': 2, 'is_organization': False, 'packages': [{'author': None, diff --git a/ckan/tests/lib/test_dictization_schema.py b/ckan/tests/lib/test_dictization_schema.py index 5a94d2e54e8..9507be2806f 100644 --- a/ckan/tests/lib/test_dictization_schema.py +++ b/ckan/tests/lib/test_dictization_schema.py @@ -160,6 +160,7 @@ def test_2_group_schema(self): 'is_organization': False, 'type': u'group', 'image_url': u'', + 'image_display_url': u'', 'packages': sorted([{'id': group_pack[0].id, 'name': group_pack[0].name, 'title': group_pack[0].title}, From fc1097763c8d9bdb3be42bbd43c037aab59f1b6d Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 10 Oct 2013 20:55:23 +0100 Subject: [PATCH 067/159] [#1267] fix url for by faking routes threadlocal --- ckan/lib/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 14c217ed096..5d13da6638b 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -10,6 +10,8 @@ import ckan.include.rcssmin as rcssmin import ckan.lib.fanstatic_resources as fanstatic_resources import sqlalchemy as sa +import urlparse +import routes import paste.script from paste.registry import Registry @@ -99,6 +101,12 @@ def _load_config(self): self.translator_obj = MockTranslator() self.registry.register(pylons.translator, self.translator_obj) + ## give routes enough information to run url_for + parsed = urlparse.urlparse(conf.get('ckan.site_url', 'http://0.0.0.0')) + request_config = routes.request_config() + request_config.host = parsed.netloc + parsed.path + request_config.protocol = parsed.scheme + def _setup_app(self): cmd = paste.script.appinstall.SetupCommand('setup-app') cmd.run([self.filename]) From 16924d937af4207a585fd34eaf2be083bcc2d6d2 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:23:41 -0300 Subject: [PATCH 068/159] [#1178] Remove backward migrations, as we don't support it --- .../versions/071_add_state_column_to_user_table.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py index 791a046bad2..c4f3f5013aa 100644 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_add_state_column_to_user_table.py @@ -7,11 +7,3 @@ def upgrade(migrate_engine): ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' ''' % ckan.model.core.State.ACTIVE ) - - -def downgrade(migrate_engine): - migrate_engine.execute( - ''' - ALTER TABLE "user" DROP COLUMN "state" - ''' - ) From 0bd163e75eacc65fee0546fa356f195f1e109713 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:40:46 -0300 Subject: [PATCH 069/159] [#1178] Fix repeated test name --- ckan/tests/lib/test_authenticator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/lib/test_authenticator.py b/ckan/tests/lib/test_authenticator.py index 8e7254d94c7..8b2c6139f0f 100644 --- a/ckan/tests/lib/test_authenticator.py +++ b/ckan/tests/lib/test_authenticator.py @@ -109,7 +109,7 @@ def test_authenticate_fails_if_user_is_deleted(self): assert self.authenticate(environ, identity) is None - def test_authenticate_fails_if_user_is_deleted(self): + def test_authenticate_fails_if_user_is_pending(self): environ = {} openid = 'some-openid-key' user = CreateTestData.create_user('a_user', **{'openid': openid}) From d073ce7f6b56361cf8c5452c2f969e4d45356437 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:54:24 -0300 Subject: [PATCH 070/159] [#1178] Use the data returned by validate, instead of the one passed as param --- ckan/logic/action/create.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 066ed7bcc87..f4ab9fa4ba1 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -865,23 +865,23 @@ def user_invite(context, data_dict): 'group_id': [validators.not_empty], 'role': [validators.not_empty], } - _, errors = _validate(data_dict, user_invite_schema, context) + data, errors = _validate(data_dict, user_invite_schema, context) if errors: raise ValidationError(errors) while True: try: - name = _get_random_username_from_email(data_dict['email']) + name = _get_random_username_from_email(data['email']) password = str(random.SystemRandom().random()) - data_dict['name'] = name - data_dict['password'] = password - data_dict['state'] = ckan.model.State.PENDING - user_dict = _get_action('user_create')(context, data_dict) + data['name'] = name + data['password'] = password + data['state'] = ckan.model.State.PENDING + user_dict = _get_action('user_create')(context, data) user = ckan.model.User.get(user_dict['id']) member_dict = { 'username': user.id, - 'id': data_dict['group_id'], - 'role': data_dict['role'] + 'id': data['group_id'], + 'role': data['role'] } _get_action('group_member_create')(context, member_dict) mailer.send_invite(user) From 1da817c40b59d7dfa866abd9e68ed844442f34d4 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:58:11 -0300 Subject: [PATCH 071/159] [#1178] Move user_invite_schema to ckan.logic.schema --- ckan/logic/action/create.py | 8 ++------ ckan/logic/schema.py | 8 ++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index f4ab9fa4ba1..e4f13b3baad 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -860,12 +860,8 @@ def user_invite(context, data_dict): ''' _check_access('user_invite', context, data_dict) - user_invite_schema = { - 'email': [validators.not_empty, unicode], - 'group_id': [validators.not_empty], - 'role': [validators.not_empty], - } - data, errors = _validate(data_dict, user_invite_schema, context) + schema = context.get('schema') or ckan.logic.schema.default_user_invite_schema() + data, errors = _validate(data_dict, schema, context) if errors: raise ValidationError(errors) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index a8cd4ce6779..60f36411c7b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -424,6 +424,14 @@ def default_update_user_schema(): return schema +def default_user_invite_schema(): + schema = { + 'email': [not_empty, unicode], + 'group_id': [not_empty], + 'role': [not_empty], + } + return schema + def default_task_status_schema(): schema = { 'id': [ignore], From c2e4f767eed6485383a7ef2efd40a8e990d0a623 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 22:59:53 -0300 Subject: [PATCH 072/159] [#1178] Refactor user name creation I've isolated the loop where it needs to be. This makes the code clearer, but makes it susceptible to errors. For example, after the random name was generated but before the user is created, another user with the same name might be created, breaking our code. This should be rare enough to bother us. --- ckan/logic/action/create.py | 43 +++++++++++++++------------------ ckan/tests/logic/test_action.py | 7 +++--- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index e4f13b3baad..9dca2940773 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -865,34 +865,31 @@ def user_invite(context, data_dict): if errors: raise ValidationError(errors) - while True: - try: - name = _get_random_username_from_email(data['email']) - password = str(random.SystemRandom().random()) - data['name'] = name - data['password'] = password - data['state'] = ckan.model.State.PENDING - user_dict = _get_action('user_create')(context, data) - user = ckan.model.User.get(user_dict['id']) - member_dict = { - 'username': user.id, - 'id': data['group_id'], - 'role': data['role'] - } - _get_action('group_member_create')(context, member_dict) - mailer.send_invite(user) - return model_dictize.user_dictize(user, context) - except ValidationError as e: - if 'name' not in e.error_dict: - raise e + name = _get_random_username_from_email(data['email']) + password = str(random.SystemRandom().random()) + data['name'] = name + data['password'] = password + data['state'] = ckan.model.State.PENDING + user_dict = _get_action('user_create')(context, data) + user = ckan.model.User.get(user_dict['id']) + member_dict = { + 'username': user.id, + 'id': data['group_id'], + 'role': data['role'] + } + _get_action('group_member_create')(context, member_dict) + mailer.send_invite(user) + return model_dictize.user_dictize(user, context) def _get_random_username_from_email(email): localpart = email.split('@')[0] cleaned_localpart = re.sub(r'[^\w]', '-', localpart) - random_number = random.SystemRandom().random() * 10000 - name = '%s-%d' % (cleaned_localpart, random_number) - return name + while True: + random_number = random.SystemRandom().random() * 10000 + name = '%s-%d' % (cleaned_localpart, random_number) + if not ckan.model.User.get(name): + return name ## Modifications for rest api diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 1fb96e83ca1..d5a043c0b9a 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -613,8 +613,8 @@ def test_user_invite_without_email_raises_error(self, mail_user): assert res_obj['success'] is False, res_obj assert 'email' in res_obj['error'], res_obj - @mock.patch('ckan.logic.action.create._get_random_username_from_email') - def test_user_invite_should_work_even_if_tried_username_already_exists(self, random_username_mock): + @mock.patch('random.SystemRandom') + def test_user_invite_should_work_even_if_tried_username_already_exists(self, system_random_mock): patcher = mock.patch('ckan.lib.mailer.mail_user') patcher.start() email = 'invited_user@email.com' @@ -628,8 +628,7 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} - usernames = ['first', 'first', 'second'] - random_username_mock.side_effect = lambda email: usernames.pop(0) + system_random_mock.return_value.random.side_effect = [1000, 1000, 2000, 3000] for _ in range(2): res = self.app.post('/api/action/user_invite', params=postparams, From dfd8ae9fb07ff551707bdc08c6e23bd424c7335c Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 23:07:25 -0300 Subject: [PATCH 073/159] [#1178] Simplify "if" expression, removing redundancy --- ckan/controllers/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index c8775168c8f..66bd533fd43 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -614,7 +614,7 @@ def member_new(self, id): data_dict['id'] = id email = data_dict.get('email') - if email and email != '': + if email: user_data_dict = { 'email': email, 'group_id': data_dict['id'], From bae4d47c3227bd5f7581ac5859edcb9badd027c0 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Sat, 12 Oct 2013 16:52:19 -0300 Subject: [PATCH 074/159] [#1178] Use model.State instead of core.State --- ckan/logic/action/get.py | 6 +++--- .../versions/071_add_state_column_to_user_table.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 57c62a2916c..0621f55cf9a 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -15,7 +15,7 @@ import ckan.logic.schema import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.navl.dictization_functions -import ckan.model.core as core +import ckan.model as model import ckan.model.misc as misc import ckan.plugins as plugins import ckan.lib.search as search @@ -690,7 +690,7 @@ def user_list(context, data_dict): ) # Filter deleted users - query = query.filter(model.User.state != core.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) ## hack for pagination if context.get('return_query'): @@ -1246,7 +1246,7 @@ def user_autocomplete(context, data_dict): limit = data_dict.get('limit', 20) query = model.User.search(q) - query = query.filter(model.User.state != core.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) query = query.limit(limit) user_list = [] diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py index c4f3f5013aa..828ebfb5f44 100644 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_add_state_column_to_user_table.py @@ -1,9 +1,9 @@ -import ckan.model.core +import ckan.model def upgrade(migrate_engine): migrate_engine.execute( ''' ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % ckan.model.core.State.ACTIVE + ''' % ckan.model.State.ACTIVE ) From 5023e0c3354ca03475b06ee11a3e47f867adedaf Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 10:51:01 +0530 Subject: [PATCH 075/159] Don't exempt sysadmins --- ckan/logic/auth/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index ec38483bd0d..ca3a2d70b6d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -104,6 +104,7 @@ def rating_create(context, data_dict): return {'success': True} +@logic.auth_sysadmins_check def user_create(context, data_dict=None): using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( From 878772cfc37932805cca58439150d702506fe043 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 12:22:29 +0530 Subject: [PATCH 076/159] [#1226] Add tests --- ckan/tests/functional/api/test_user.py | 190 ++++++++++++++++++++++++- ckan/tests/logic/test_auth.py | 3 +- 2 files changed, 189 insertions(+), 4 deletions(-) diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index 06dd35dd7fc..26565810c2f 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -1,20 +1,26 @@ +import paste +from pylons import config from nose.tools import assert_equal import ckan.logic as logic +import ckan.new_authz as new_authz from ckan import model from ckan.lib.create_test_data import CreateTestData from ckan.tests import TestController as ControllerTestCase +from ckan.tests.pylons_controller import PylonsTestCase from ckan.tests import url_for +import ckan.config.middleware +from ckan.common import json class TestUserApi(ControllerTestCase): @classmethod def setup_class(cls): CreateTestData.create() - + @classmethod def teardown_class(cls): model.repo.rebuild_db() - + def test_autocomplete(self): response = self.app.get( url=url_for(controller='api', action='user_autocomplete', ver=2), @@ -51,8 +57,186 @@ def test_autocomplete_limit(self): print response.json assert_equal(len(response.json), 1) + +class TestCreateUserApiDisabled(PylonsTestCase): + ''' + Tests for the creating user when create_user_via_api is disabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled_sysadmin(self): + params = { + 'name': 'testinganewusersysadmin', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + def test_user_create_api_disabled_anon(self): + params = { + 'name': 'testinganewuseranon', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + +class TestCreateUserApiEnabled(PylonsTestCase): + ''' + Tests for the creating user when create_user_via_api is enabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_api'] = True + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + PylonsTestCase.setup_class() + cls.sysadmin_user = model.User.get('testsysadmin') + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled_sysadmin(self): + params = { + 'name': 'testinganewusersysadmin', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': + str(self.sysadmin_user.apikey)}) + res_dict = res.json + assert res_dict['success'] is True + + def test_user_create_api_disabled_anon(self): + params = { + 'name': 'testinganewuseranon', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is True + + +class TestCreateUserWebDisabled(PylonsTestCase): + ''' + Tests for the creating user by create_user_via_web is disabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_web'] = False + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled(self): + params = { + 'name': 'testinganewuser', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + +class TestCreateUserWebEnabled(PylonsTestCase): + ''' + Tests for the creating user by create_user_via_web is enabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_web'] = True + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled(self): + params = { + 'name': 'testinganewuser', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + class TestUserActions(object): - + @classmethod def setup_class(cls): CreateTestData.create() diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 3ea9f2db39f..b73834d2657 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -11,8 +11,9 @@ 'user_create_organizations': False, 'user_delete_groups': False, 'user_delete_organizations': False, - 'create_user_via_api': False, 'create_unowned_dataset': False, + 'create_user_via_api': False, + 'create_user_via_web': True, } From 3273ebf7d0e06dbf3d854a25f701e651fe88bb0b Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 15:34:07 +0530 Subject: [PATCH 077/159] [#1226] Return the old behaviour for create_user --- ckan/logic/auth/create.py | 1 - ckan/tests/functional/api/test_user.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 13e02125855..53c8ae153ab 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): @logic.auth_allow_anonymous_access -@logic.auth_sysadmins_check def user_create(context, data_dict=None): using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index ba67b97455e..cdbe039d012 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -83,7 +83,7 @@ def teardown_class(cls): model.repo.rebuild_db() - def test_user_create_api_disabled_sysadmin(self): + def test_user_create_api_enabled_sysadmin(self): params = { 'name': 'testinganewusersysadmin', 'email': 'testinganewuser@ckan.org', @@ -93,7 +93,7 @@ def test_user_create_api_disabled_sysadmin(self): extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json - assert res_dict['success'] is False + assert res_dict['success'] is True def test_user_create_api_disabled_anon(self): params = { @@ -190,7 +190,6 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json assert res_dict['success'] is False @@ -229,7 +228,6 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json assert res_dict['success'] is False From 3218b581443ad07cd09a93d364bf5085fb0f7884 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 14 Oct 2013 17:31:04 +0100 Subject: [PATCH 078/159] [#1237] Fix docstring --- ckan/logic/action/get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a810cfa3cbe..384026a64e8 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1289,8 +1289,8 @@ def package_search(context, data_dict): :param facet.mincount: the minimum counts for facet fields should be included in the results. :type facet.mincount: int - :param facet.limit: the maximum number of values the facet fields. A - negative value means unlimited. This can be set instance-wide with + :param facet.limit: the maximum number of values the facet fields return. + A negative value means unlimited. This can be set instance-wide with the :ref:`search.facets.limit` config option. Default is 50. :type facet.limit: int :param facet.field: the fields to facet upon. Default empty. If empty, From bb6cb7d0ff417b72f002dd372bdfd09ba560a766 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 14 Oct 2013 21:46:39 +0100 Subject: [PATCH 079/159] [#1261] add filesize limit, make storage folder backwards compatable, document a bit more --- ckan/lib/helpers.py | 3 +- ckan/lib/uploader.py | 69 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index fceab6f1b6a..3aa73ad3a8b 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -37,6 +37,7 @@ import ckan.lib.maintain as maintain import ckan.lib.datapreview as datapreview import ckan.logic as logic +import ckan.lib.uploader as uploader from ckan.common import ( _, ungettext, g, c, request, session, json, OrderedDict @@ -1664,7 +1665,7 @@ def new_activities(): return action({}, {}) def uploads_enabled(): - if config.get('ckan.storage_path'): + if uploader.get_storage_path(): return True return False diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index eb0f28eec54..880488b8161 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -3,11 +3,51 @@ import pylons import datetime import ckan.lib.munge as munge +import logging +import ckan.logic as logic + +log = logging.getLogger(__name__) + +_storage_path = None + +def get_storage_path(): + '''Function to cache storage path''' + global _storage_path + + #None means it has not been set. False means not in config. + if _storage_path is None: + storage_path = pylons.config.get('ckan.storage_path') + ofs_impl = pylons.config.get('ofs.impl') + ofs_storage_dir = pylons.config.get('ofs.storage_dir') + if storage_path: + _storage_path = storage_path + elif ofs_impl == 'pairtree' and ofs_storage_dir: + log.warn('''Please use config option ckan.storage_path instaed of + ofs.storage_path''') + _storage_path = ofs_storage_dir + return _storage_path + elif ofs_impl: + log.critical('''We only support local file storage form version 2.2 of ckan + please specify ckan.storage_path in your config for your uploads''') + _storage_path = False + else: + log.critical('''Please specify a ckan.storage_path in your config for your uploads''') + _storage_path = False + + return _storage_path + class Upload(object): def __init__(self, object_type, old_filename=None): - path = pylons.config.get('ckan.storage_path', '/tmp') + ''' Setup upload by creating a subdirectory of the storage directory of name + object_type. old_filename is the name of the file in the url field last time''' + self.storage_path = None + self.filename = None + self.filepath = None + path = get_storage_path() + if not path: + return self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) try: @@ -18,12 +58,20 @@ def __init__(self, object_type, old_filename=None): self.old_filename = old_filename if old_filename: self.old_filepath = os.path.join(self.storage_path, old_filename) - self.filename = None - self.filepath = None def update_data_dict(self, data_dict, url_field, file_field, clear_field): + ''' Manipulate data from the data_dict. url_field is the name of the field + where the upload is going to be. file_field is name of the key where the + FieldStorage is kept (i.e the field where the file data actually is). clear_field + is the name of a boolean field which requests the upload to be deleted. This needs + to be called before it reaches any validators''' + + if not self.storage_path: + return + self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) + self.file_field = file_field self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): @@ -41,16 +89,27 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): if self.clear and self.url == self.old_filename: data_dict[url_field] = '' - def upload(self): + def upload(self, max_size=2): + ''' Actually upload the file. This should happen just before a commit but after + the data has been validated and flushed to the db. This is so we do not store anything + unless the request is actually good. Max_size is size in MB maximum of the file''' + if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) + current_size = 0 while True: - # mb chuncks + current_size = current_size + 1 + # MB chuncks data = self.upload_file.read(2 ** 20) if not data: break output_file.write(data) + if current_size > max_size: + os.remove(self.tmp_filepath) + raise logic.ValidationError( + {self.file_field: ['File upload too large']} + ) output_file.close() os.rename(self.tmp_filepath, self.filepath) self.clear = True From f5d168cd9018d68868644188dded4dd2b6b01f36 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 15 Oct 2013 13:35:08 +0100 Subject: [PATCH 080/159] [#1256] Don't load the synchronous_search plugin on cli It is already loaded on startup --- ckan/lib/cli.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 14c217ed096..c74c25e85e2 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -842,12 +842,10 @@ def show(self, dataset_ref): pprint.pprint(dataset.as_dict()) def delete(self, dataset_ref): - from ckan import plugins import ckan.model as model dataset = self._get_dataset(dataset_ref) old_state = dataset.state - plugins.load('synchronous_search') rev = model.repo.new_revision() dataset.delete() model.repo.commit_and_remove() @@ -855,12 +853,10 @@ def delete(self, dataset_ref): print '%s %s -> %s' % (dataset.name, old_state, dataset.state) def purge(self, dataset_ref): - from ckan import plugins import ckan.model as model dataset = self._get_dataset(dataset_ref) name = dataset.name - plugins.load('synchronous_search') rev = model.repo.new_revision() dataset.purge() model.repo.commit_and_remove() @@ -1286,8 +1282,6 @@ class CreateTestDataCommand(CkanCommand): def command(self): self._load_config() self._setup_app() - from ckan import plugins - plugins.load('synchronous_search') # so packages get indexed from create_test_data import CreateTestData if self.args: From 6431ba3dc6a117926101a97ae859132a906cf0f2 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 16:00:30 +0200 Subject: [PATCH 081/159] [#1178] Add auth_user_obj back to context in User controller methods. --- ckan/controllers/user.py | 48 ++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 98033d9a0ab..1c6fefff344 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -38,7 +38,8 @@ class UserController(base.BaseController): def __before__(self, action, **env): base.BaseController.__before__(self, action, **env) try: - context = {'model': model, 'user': c.user or c.author} + context = {'model': model, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: if c.action not in ('login', 'request_reset', 'perform_reset',): @@ -89,7 +90,8 @@ def index(self): c.q = request.params.get('q', '') c.order_by = request.params.get('order_by', 'name') - context = {'return_query': True} + context = {'return_query': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'q': c.q, 'order_by': c.order_by} @@ -111,7 +113,8 @@ def index(self): def read(self, id=None): context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj} try: @@ -140,7 +143,8 @@ def me(self, locale=None): id=user_ref) def register(self, data=None, errors=None, error_summary=None): - context = {'model': model, 'session': model.Session, 'user': c.user} + context = {'model': model, 'session': model.Session, 'user': c.user, + 'auth_user_obj': c.userobj} try: check_access('user_create', context) except NotAuthorized: @@ -154,6 +158,7 @@ def new(self, data=None, errors=None, error_summary=None): ''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, + 'auth_user_obj': c.userobj, 'schema': self._new_form_to_db_schema(), 'save': 'save' in request.params} @@ -182,7 +187,8 @@ def delete(self, id): '''Delete user with id passed as parameter''' context = {'model': model, 'session': model.Session, - 'user': c.user} + 'user': c.user, + 'auth_user_obj': c.userobj} data_dict = {'id': id} try: @@ -234,7 +240,7 @@ def edit(self, id=None, data=None, errors=None, error_summary=None): context = {'save': 'save' in request.params, 'schema': self._edit_form_to_db_schema(), 'model': model, 'session': model.Session, - 'user': c.user, + 'user': c.user, 'auth_user_obj': c.userobj } if id is None: if c.userobj: @@ -391,7 +397,8 @@ def logged_out_page(self): return render('user/logout.html') def request_reset(self): - context = {'model': model, 'session': model.Session, 'user': c.user} + context = {'model': model, 'session': model.Session, 'user': c.user, + 'auth_user_obj': c.userobj} data_dict = {'id': request.params.get('user')} try: check_access('request_reset', context) @@ -511,7 +518,8 @@ def _get_form_password(self): raise ValueError(_('You must provide a password')) def followers(self, id=None): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'id': id, 'user_obj': c.userobj} self._setup_template_variables(context, data_dict) f = get_action('user_follower_list') @@ -525,7 +533,8 @@ def activity(self, id, offset=0): '''Render this user's public activity stream page.''' context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj} try: check_access('user_show', context, data_dict) @@ -553,7 +562,8 @@ def display_name(followee): if (filter_type and filter_id): context = { 'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True } data_dict = {'id': filter_id} followee = None @@ -595,7 +605,8 @@ def display_name(followee): def dashboard(self, id=None, offset=0): context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj, 'offset': offset} self._setup_template_variables(context, data_dict) @@ -618,19 +629,22 @@ def dashboard(self, id=None, offset=0): return render('user/dashboard.html') def dashboard_datasets(self): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard_datasets.html') def dashboard_organizations(self): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard_organizations.html') def dashboard_groups(self): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard_groups.html') @@ -639,7 +653,8 @@ def follow(self, id): '''Start following this user.''' context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'id': id} try: get_action('follow_user')(context, data_dict) @@ -658,7 +673,8 @@ def unfollow(self, id): '''Stop following this user.''' context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'id': id} try: get_action('unfollow_user')(context, data_dict) From 3dee85226f1fa6a29a03d598c8c11be1c5dba4ca Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:10:09 +0100 Subject: [PATCH 082/159] [#1262] First working version of the JS module for image uploading --- .../base/javascript/modules/image-upload.js | 147 ++++++++++++++++++ ckan/public/base/javascript/resource.config | 1 + ckan/public/base/less/forms.less | 34 ++++ ckan/templates/group/snippets/group_form.html | 10 +- ckan/templates/macros/form.html | 27 ++++ .../snippets/organization_form.html | 9 +- 6 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 ckan/public/base/javascript/modules/image-upload.js diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js new file mode 100644 index 00000000000..69ca8f93c05 --- /dev/null +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -0,0 +1,147 @@ +/* Image Upload + * + * + * Options + * - id: what's the id of the context? + */ +this.ckan.module('image-upload', function($, _) { + return { + /* options object can be extended using data-module-* attributes */ + options: { + is_url: true, + has_image: false, + i18n: { + upload: _('From computer'), + url: _('From web'), + remove: _('Remove'), + label: _('Upload image'), + label_url: _('Image URL'), + remove_tooltip: _('Reset this') + }, + template: [ + '' + ].join("\n") + }, + + state: { + attached: 1, + blank: 2, + web: 3 + }, + + /* Initialises the module setting up elements and event listeners. + * + * Returns nothing. + */ + initialize: function () { + $.proxyAll(this, /_on/); + var options = this.options; + + // firstly setup the fields + this.input = $('input[name="image_upload"]', this.el); + this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); + this.field_image = this.input.parents('.control-group'); + + var checkbox = $('input[name="clear_upload"]', this.el); + if (checkbox.length > 0) { + options.has_image = true; + checkbox.parents('.control-group').remove(); + } + + this.field_clear = $('') + .appendTo(this.el); + + this.button_url = $(' '+this.i18n('url')+'') + .on('click', this._onFromWeb) + .insertAfter(this.input); + + this.button_upload = $(''+this.i18n('upload')+'') + .insertAfter(this.input); + + this.button_remove = $('') + .text(this.i18n('remove')) + .on('click', this._onRemove) + .insertAfter(this.button_upload); + + $('') + .prop('title', this.i18n('remove_tooltip')) + .on('click', this._onRemove) + .insertBefore($('input', this.field_url)); + + $('label[for="field-image-upload"]').text(this.i18n('label')); + + this.input + .on('mouseover', this._onInputMouseOver) + .on('mouseout', this._onInputMouseOut) + .on('change', this._onInputChange) + .css('width', this.button_upload.outerWidth()) + .hide(); + + this.fields = $('') + .add(this.button_remove) + .add(this.button_upload) + .add(this.button_url) + .add(this.input) + .add(this.field_url) + .add(this.field_image); + + if (options.is_url) { + this.changeState(this.state.web); + } else if (options.has_image) { + this.changeState(this.state.attached); + } else { + this.changeState(this.state.blank); + } + + }, + + changeState: function(state) { + this.fields.hide(); + if (state == this.state.blank) { + this.button_upload + .add(this.field_image) + .add(this.button_url) + .add(this.input) + .show(); + } else if (state == this.state.attached) { + this.button_remove + .add(this.field_image) + .show(); + } else if (state == this.state.web) { + this.field_url + .show(); + } + }, + + _onFromWeb: function() { + this.changeState(this.state.web); + $('input', this.field_url).focus(); + if (this.options.has_image) { + this.field_clear.val('true'); + } + }, + + _onRemove: function() { + this.changeState(this.state.blank); + $('input', this.field_url).val(''); + if (this.options.has_image) { + this.field_clear.val('true'); + } + }, + + _onInputChange: function() { + this.file_name = this.input.val(); + this.field_clear.val(''); + this.changeState(this.state.attached); + }, + + _onInputMouseOver: function() { + this.button_upload.addClass('hover'); + }, + + _onInputMouseOut: function() { + this.button_upload.removeClass('hover'); + } + + } +}); diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config index b1a7302d9e8..564da811a6e 100644 --- a/ckan/public/base/javascript/resource.config +++ b/ckan/public/base/javascript/resource.config @@ -38,6 +38,7 @@ ckan = modules/table-toggle-more.js modules/dataset-visibility.js modules/media-grid.js + modules/image-upload.js main = apply_html_class diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..2bfc1a2d492 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,37 @@ textarea { .box-shadow(none); } } + +.js .image-upload { + #field-image-upload { + cursor: pointer; + position: absolute; + z-index: 1; + .opacity(0); + } + .controls { + position: relative; + } + .btn { + position: relative; + top: 0; + margin-right: 10px; + &.hover { + color: @grayDark; + text-decoration: none; + background-position: 0 -15px; + .transition(background-position .1s linear); + } + } + .btn-remove-url { + position: absolute; + margin-right: 0; + top: 4px; + right: 5px; + padding: 0 4px; + .border-radius(100px); + .icon-remove { + margin-right: 0; + } + } +} diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 322f9ebff8a..cd19912d66b 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -19,15 +19,7 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - - - {% if h.uploads_enabled() %} - {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% if data.image_url and not data.image_url.startswith('http') %} - {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} - {% endif %} - {% endif %} + {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }} {% endblock %} diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 497dea6bf02..7dc044ccca2 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -395,3 +395,30 @@

        {% endmacro %} +{# +Builds a file upload for input + +Example + {% import 'macros/form.html' as form %} + {{ form.image_upload(data, errors, is_upload_enabled=true) }} + +#} +{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %} + {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} + {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} + {% set is_url = data.image_url and data.image_url.startswith('http') %} + + {% if is_upload_enabled %}
        {% endif %} + + {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + + {% if is_upload_enabled %} + {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {% if has_uploaded_data %} + {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + + {% if is_upload_enabled %}
        {% endif %} + +{% endmacro %} diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 0a2134e89f7..0b97b958d38 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -19,14 +19,7 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - - {% if h.uploads_enabled() %} - {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% if data.image_url and not data.image_url.startswith('http') %} - {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} - {% endif %} - {% endif %} + {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }} {% endblock %} From 0f74db8a964451e5cba1360b4f15ea3a84a56544 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:21:14 +0100 Subject: [PATCH 083/159] [#1262] Adds in doc blocks into the image_upload JS module --- .../base/javascript/modules/image-upload.js | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index 69ca8f93c05..dbb83ade15a 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -1,8 +1,5 @@ /* Image Upload * - * - * Options - * - id: what's the id of the context? */ this.ckan.module('image-upload', function($, _) { return { @@ -42,41 +39,49 @@ this.ckan.module('image-upload', function($, _) { this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); + // Is there a clear checkbox on the form already? var checkbox = $('input[name="clear_upload"]', this.el); if (checkbox.length > 0) { options.has_image = true; checkbox.parents('.control-group').remove(); } + // Adds the hidden clear input to the form this.field_clear = $('') .appendTo(this.el); + // Button to set the field to be a URL this.button_url = $(' '+this.i18n('url')+'') .on('click', this._onFromWeb) .insertAfter(this.input); + // Button to attach local file to the form this.button_upload = $(''+this.i18n('upload')+'') .insertAfter(this.input); + // Button to reset the form back to the first from when there is a image uploaded this.button_remove = $('') .text(this.i18n('remove')) .on('click', this._onRemove) .insertAfter(this.button_upload); + // Button for resetting the form when there is a URL set $('') .prop('title', this.i18n('remove_tooltip')) .on('click', this._onRemove) .insertBefore($('input', this.field_url)); + // Update the main label $('label[for="field-image-upload"]').text(this.i18n('label')); + // Setup the file input this.input .on('mouseover', this._onInputMouseOver) .on('mouseout', this._onInputMouseOut) .on('change', this._onInputChange) - .css('width', this.button_upload.outerWidth()) - .hide(); + .css('width', this.button_upload.outerWidth()); + // Fields storage. Used in this.changeState this.fields = $('') .add(this.button_remove) .add(this.button_upload) @@ -85,6 +90,7 @@ this.ckan.module('image-upload', function($, _) { .add(this.field_url) .add(this.field_image); + // Setup the initial state if (options.is_url) { this.changeState(this.state.web); } else if (options.has_image) { @@ -95,6 +101,16 @@ this.ckan.module('image-upload', function($, _) { }, + /* Method to change the display state of the image fields + * + * state - Pseudo constant for passing the state we should be in now + * + * Examples + * + * this.changeState(this.state.web); // Sets the state in URL mode + * + * Returns nothing. + */ changeState: function(state) { this.fields.hide(); if (state == this.state.blank) { @@ -113,6 +129,10 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for when someone sets the field to URL mode + * + * Returns nothing. + */ _onFromWeb: function() { this.changeState(this.state.web); $('input', this.field_url).focus(); @@ -121,6 +141,10 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for resetting the field back to the blank state + * + * Returns nothing. + */ _onRemove: function() { this.changeState(this.state.blank); $('input', this.field_url).val(''); @@ -129,16 +153,28 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for when someone chooses a file to upload + * + * Returns nothing. + */ _onInputChange: function() { this.file_name = this.input.val(); this.field_clear.val(''); this.changeState(this.state.attached); }, + /* Event listener for when a user mouseovers the hidden file input + * + * Returns nothing. + */ _onInputMouseOver: function() { this.button_upload.addClass('hover'); }, + /* Event listener for when a user mouseouts the hidden file input + * + * Returns nothing. + */ _onInputMouseOut: function() { this.button_upload.removeClass('hover'); } From 4ac411b61cfae7048cdc6cc5cace42aec20a13e9 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 16:28:12 +0200 Subject: [PATCH 084/159] [#1178] Fix typo in org member_new template --- ckan/templates/organization/member_new.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index 352cb9f3790..a121549dab4 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -19,7 +19,7 @@

        {{ _('Existing User') }} - {{ _('If you wish to add an existing user, search for her username below.') }} + {{ _('If you wish to add an existing user, search for their username below.') }}
        {% if user %} @@ -41,7 +41,7 @@

        {{ _('New User') }} - {{ _('If you wish to invite a new user, enter her email address.') }} + {{ _('If you wish to invite a new user, enter their email address.') }}
        From 7d15d591499d5722f60cc0ecfc09abf59dad177f Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:29:45 +0100 Subject: [PATCH 085/159] [#1262] Adds support for different field names for image upload --- ckan/public/base/javascript/modules/image-upload.js | 13 +++++++------ ckan/templates/macros/form.html | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index dbb83ade15a..96308ed5764 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -7,6 +7,9 @@ this.ckan.module('image-upload', function($, _) { options: { is_url: true, has_image: false, + field_upload: 'input[name="image_upload"]', + field_url: 'input[name="image_url"]', + field_clear: 'input[name="clear_upload"]', i18n: { upload: _('From computer'), url: _('From web'), @@ -35,12 +38,12 @@ this.ckan.module('image-upload', function($, _) { var options = this.options; // firstly setup the fields - this.input = $('input[name="image_upload"]', this.el); - this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); + this.input = $(options.field_upload, this.el); + this.field_url = $(options.field_url, this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); // Is there a clear checkbox on the form already? - var checkbox = $('input[name="clear_upload"]', this.el); + var checkbox = $(options.field_clear, this.el); if (checkbox.length > 0) { options.has_image = true; checkbox.parents('.control-group').remove(); @@ -148,9 +151,7 @@ this.ckan.module('image-upload', function($, _) { _onRemove: function() { this.changeState(this.state.blank); $('input', this.field_url).val(''); - if (this.options.has_image) { - this.field_clear.val('true'); - } + this.field_clear.val('true'); }, /* Event listener for when someone chooses a file to upload diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 7dc044ccca2..5a6c00a6cb1 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -403,19 +403,19 @@ {{ form.image_upload(data, errors, is_upload_enabled=true) }} #} -{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %} +{% macro image_upload(data, errors, field_url='image_url', field_upload='image_upload', field_clear='clear_upload', image_url=false, is_upload_enabled=false, placeholder=false) %} {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} {% set is_url = data.image_url and data.image_url.startswith('http') %} {% if is_upload_enabled %}
        {% endif %} - {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if is_upload_enabled %} - {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {{ input(field_upload, label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} {% if has_uploaded_data %} - {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {{ checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} From 0f8727a31f26069561352017064c7febd9b48b2a Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:44:51 +0100 Subject: [PATCH 086/159] [#1269] Tweaks to the templates to make them better --- ckan/templates/package/resource_edit.html | 2 +- .../templates/package/resource_edit_base.html | 24 ++++++++++++++++--- .../package/snippets/resource_info.html | 21 ++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 ckan/templates/package/snippets/resource_info.html diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index 9756c8f9581..3d761185230 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -5,5 +5,5 @@ {% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg) }}{% endblock %} {% block form %} - {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} + {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} {% endblock %} diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html index 7c3a8d012ec..e6c98ad7ca3 100644 --- a/ckan/templates/package/resource_edit_base.html +++ b/ckan/templates/package/resource_edit_base.html @@ -1,12 +1,30 @@ -{% extends "package/edit_base.html" %} +{% extends "package/base.html" %} {% set logged_in = true if c.userobj else false %} {% block breadcrumb_content_selected %}{% endblock %} +{% block breadcrumb_content %} + {{ super() }} +
      • {% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg.name, resource_id=res.id %}
        • +
      • Edit
        • +{% endblock %} + +{% block content_action %} + {% link_for _('All resources'), controller='package', action='resources', id=pkg.name, class_='btn', icon='arrow-left' %} + {% link_for _('View resource'), controller='package', action='resource_read', id=pkg.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('resource_edit', _('Edit resource'), id=pkg.name, resource_id=res.id) }} +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/resource_info.html', res=res %} +{% endblock %} + {% block primary_content_inner %} - {% link_for _('View all resources'), controller='package', action='resources', id=pkg.name, class_='btn pull-right', icon='arrow-left' %} -

        {% block form_title %}{{ _('Edit resource') }}{% endblock %}

        +

        {% block form_title %}{{ _('Edit resource') }}{% endblock %}

        {% block form %}{% endblock %} {% endblock %} diff --git a/ckan/templates/package/snippets/resource_info.html b/ckan/templates/package/snippets/resource_info.html new file mode 100644 index 00000000000..d8ada30d83a --- /dev/null +++ b/ckan/templates/package/snippets/resource_info.html @@ -0,0 +1,21 @@ +{# +Displays a sidebard module with information for given resource + +res - The respource dict + +Example: + + {% snippet "package/snippets/resrouce_info.html", res=res %} + +#} +
        +
        +

        {{ res.name or res.id }}

        +
        +
        +
        {{ _('Format') }}
        +
        {{ res.format }}
        +
        +
        +
        +
        From 322149c6934a574589f37ffe8317c444ad0885ff Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 16:51:07 +0200 Subject: [PATCH 087/159] [#1178] Constrain number of attempts to generate a random user name in user_invite --- ckan/logic/action/create.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 9dca2940773..693845398d0 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -860,7 +860,8 @@ def user_invite(context, data_dict): ''' _check_access('user_invite', context, data_dict) - schema = context.get('schema') or ckan.logic.schema.default_user_invite_schema() + schema = context.get('schema', + ckan.logic.schema.default_user_invite_schema()) data, errors = _validate(data_dict, schema, context) if errors: raise ValidationError(errors) @@ -885,7 +886,12 @@ def user_invite(context, data_dict): def _get_random_username_from_email(email): localpart = email.split('@')[0] cleaned_localpart = re.sub(r'[^\w]', '-', localpart) - while True: + + # if we can't create a unique user name within this many attempts + # then something else is probably wrong and we should give up + max_name_creation_attempts = 100 + + for i in range(max_name_creation_attempts): random_number = random.SystemRandom().random() * 10000 name = '%s-%d' % (cleaned_localpart, random_number) if not ckan.model.User.get(name): From a480ace5ae73532627ad9b04cc3416f855373fc6 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 17:09:29 +0200 Subject: [PATCH 088/159] [#1178] handle edge case where _get_random_username_from_email fails by returning the cleaned local part of the email address. This is expected to fail validation in user_create. --- ckan/logic/action/create.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 693845398d0..c7fb98be30a 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -897,6 +897,8 @@ def _get_random_username_from_email(email): if not ckan.model.User.get(name): return name + return cleaned_localpart + ## Modifications for rest api From 971026bf286aee490e8080bab715870d65b84cf7 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 16:22:49 +0100 Subject: [PATCH 089/159] [#1263] Remove the 'clear all' from the faceter on the left nav --- ckan/public/base/less/iehacks.less | 5 ----- ckan/public/base/less/module.less | 11 ----------- ckan/templates/development/snippets/facet.html | 2 +- ckan/templates/development/snippets/module.html | 2 +- ckan/templates/snippets/facet_list.html | 1 - 5 files changed, 2 insertions(+), 19 deletions(-) diff --git a/ckan/public/base/less/iehacks.less b/ckan/public/base/less/iehacks.less index 232ee72ee9c..7d9bc5cc274 100644 --- a/ckan/public/base/less/iehacks.less +++ b/ckan/public/base/less/iehacks.less @@ -200,11 +200,6 @@ .media-content { position: relative; } - .action { - position: absolute; - top: 9px; - right: 10px; - } .media-image img { float: left; } diff --git a/ckan/public/base/less/module.less b/ckan/public/base/less/module.less index a575abcbda4..aa964cf5ea3 100644 --- a/ckan/public/base/less/module.less +++ b/ckan/public/base/less/module.less @@ -13,17 +13,6 @@ border-bottom: 1px solid @moduleHeadingBorderColor; } -.module-heading .action { - float: right; - color: @moduleHeadingActionTextColor; - font-size: 12px; - line-height: @baseLineHeight; - text-decoration: underline; - &:hover { - color: @layoutTextColor; - } -} - .module-content { padding: 0 @gutterX; margin: 20px 0; diff --git a/ckan/templates/development/snippets/facet.html b/ckan/templates/development/snippets/facet.html index ac9b9df1a0c..7280bf9a3d2 100644 --- a/ckan/templates/development/snippets/facet.html +++ b/ckan/templates/development/snippets/facet.html @@ -1,6 +1,6 @@
        {% with items=(("First", true), ("Second", false), ("Third", true), ("Fourth", false), ("Last", false)) %} -

        Facet List Clear All

        +

        Facet List