From 9579744fdc6c0665d5e92e7106daffedf537dff6 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 23 Sep 2019 19:41:16 +0300 Subject: [PATCH 1/4] DatapushserController -> DatapusherBlueprint --- ckanext/datapusher/blueprint.py | 71 +++++++ ckanext/datapusher/plugin.py | 178 ++++++++---------- .../datapusher/resource_data.html | 2 +- .../package/resource_edit_base.html | 2 +- .../templates/datapusher/resource_data.html | 2 +- .../templates/package/resource_edit_base.html | 2 +- ckanext/stats/blueprint.py | 1 - 7 files changed, 152 insertions(+), 106 deletions(-) create mode 100644 ckanext/datapusher/blueprint.py diff --git a/ckanext/datapusher/blueprint.py b/ckanext/datapusher/blueprint.py new file mode 100644 index 00000000000..668cce2d3a7 --- /dev/null +++ b/ckanext/datapusher/blueprint.py @@ -0,0 +1,71 @@ +# encoding: utf-8 + +from flask import Blueprint +from flask.views import MethodView + +import ckan.plugins.toolkit as toolkit +import ckan.logic as logic +import ckan.lib.helpers as core_helpers +import ckan.lib.base as base + +from ckan.common import _ + +datapusher = Blueprint(u'datapusher', __name__) + + +class ResourceDataView(MethodView): + + def post(self, id, resource_id): + try: + toolkit.get_action(u'datapusher_submit')( + None, { + u'resource_id': resource_id + } + ) + except logic.ValidationError: + pass + + return core_helpers.redirect_to( + u'datapusher.resource_data', id=id, resource_id=resource_id + ) + + def get(self, id, resource_id): + try: + pkg_dict = toolkit.get_action(u'package_show')(None, {u'id': id}) + resource = toolkit.get_action(u'resource_show' + )(None, { + u'id': resource_id + }) + + # backward compatibility with old templates + toolkit.c.pkg_dict = pkg_dict + toolkit.c.resource = resource + + except (logic.NotFound, logic.NotAuthorized): + base.abort(404, _(u'Resource not found')) + + try: + datapusher_status = toolkit.get_action(u'datapusher_status')( + None, { + u'resource_id': resource_id + } + ) + except logic.NotFound: + datapusher_status = {} + except logic.NotAuthorized: + base.abort(403, _(u'Not authorized to see this page')) + + return base.render( + u'datapusher/resource_data.html', + extra_vars={ + u'status': datapusher_status, + u'pkg_dict': pkg_dict, + u'resource': resource, + } + ) + + +datapusher.add_url_rule( + u'/dataset//resource_data/', + view_func=ResourceDataView.as_view(str(u'resource_data')) +) diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index d435d52e61c..a4c55524a08 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -2,26 +2,28 @@ import logging -import ckan.plugins as p -import ckan.lib.base as base -import ckan.lib.helpers as core_helpers -import ckanext.datapusher.logic.action as action -import ckanext.datapusher.logic.auth as auth -import ckanext.datapusher.helpers as helpers import ckan.logic as logic import ckan.model as model +import ckan.plugins as p import ckan.plugins.toolkit as toolkit - -from ckan.common import _ +import ckanext.datapusher.blueprint as blueprint +import ckanext.datapusher.helpers as helpers +import ckanext.datapusher.logic.action as action +import ckanext.datapusher.logic.auth as auth log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust DEFAULT_FORMATS = [ - 'csv', 'xls', 'xlsx', 'tsv', 'application/csv', - 'application/vnd.ms-excel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'ods', 'application/vnd.oasis.opendocument.spreadsheet', + u'csv', + u'xls', + u'xlsx', + u'tsv', + u'application/csv', + u'application/vnd.ms-excel', + u'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + u'ods', + u'application/vnd.oasis.opendocument.spreadsheet', ] @@ -29,52 +31,6 @@ class DatastoreException(Exception): pass -class ResourceDataController(base.BaseController): - - def resource_data(self, id, resource_id): - - if toolkit.request.method == 'POST': - try: - toolkit.c.pkg_dict = p.toolkit.get_action('datapusher_submit')( - None, {'resource_id': resource_id} - ) - except logic.ValidationError: - pass - - core_helpers.redirect_to( - controller='ckanext.datapusher.plugin:ResourceDataController', - action='resource_data', - id=id, - resource_id=resource_id - ) - - try: - toolkit.c.pkg_dict = p.toolkit.get_action('package_show')( - None, {'id': id} - ) - toolkit.c.resource = p.toolkit.get_action('resource_show')( - None, {'id': resource_id} - ) - except (logic.NotFound, logic.NotAuthorized): - base.abort(404, _('Resource not found')) - - try: - datapusher_status = p.toolkit.get_action('datapusher_status')( - None, {'resource_id': resource_id} - ) - except logic.NotFound: - datapusher_status = {} - except logic.NotAuthorized: - base.abort(403, _('Not authorized to see this page')) - - return base.render('datapusher/resource_data.html', - extra_vars={ - 'status': datapusher_status, - 'pkg_dict': toolkit.c.pkg_dict, - 'resource': toolkit.c.resource, - }) - - class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IConfigurer, inherit=True) p.implements(p.IConfigurable, inherit=True) @@ -83,88 +39,108 @@ class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.ITemplateHelpers) - p.implements(p.IRoutes, inherit=True) + p.implements(p.IBlueprint) legacy_mode = False resource_show_action = None def update_config(self, config): - templates_base = config.get('ckan.base_templates_folder') - p.toolkit.add_template_directory(config, templates_base) + templates_base = config.get(u'ckan.base_templates_folder') + toolkit.add_template_directory(config, templates_base) def configure(self, config): self.config = config - datapusher_formats = config.get('ckan.datapusher.formats', '').lower() + datapusher_formats = config.get(u'ckan.datapusher.formats', + u'').lower() self.datapusher_formats = datapusher_formats.split() or DEFAULT_FORMATS - for config_option in ('ckan.site_url', 'ckan.datapusher.url',): + for config_option in ( + u'ckan.site_url', + u'ckan.datapusher.url', + ): if not config.get(config_option): raise Exception( - 'Config option `{0}` must be set to use the DataPusher.' - .format(config_option)) + u'Config option `{0}` must be set to use the DataPusher.'. + format(config_option) + ) def notify(self, entity, operation=None): if isinstance(entity, model.Resource): - if (operation == model.domain_object.DomainObjectOperation.new or - not operation): + if ( + operation == model.domain_object.DomainObjectOperation.new + or not operation + ): # if operation is None, resource URL has been changed, as # the notify function in IResourceUrlChange only takes # 1 parameter - context = {'model': model, 'ignore_auth': True, - 'defer_commit': True} - if (entity.format and - entity.format.lower() in self.datapusher_formats and - entity.url_type != 'datapusher'): + context = { + u'model': model, + u'ignore_auth': True, + u'defer_commit': True + } + if ( + entity.format + and entity.format.lower() in self.datapusher_formats + and entity.url_type != u'datapusher' + ): try: - task = p.toolkit.get_action('task_status_show')( + task = toolkit.get_action(u'task_status_show')( context, { - 'entity_id': entity.id, - 'task_type': 'datapusher', - 'key': 'datapusher'} + u'entity_id': entity.id, + u'task_type': u'datapusher', + u'key': u'datapusher' + } ) - if task.get('state') == 'pending': + if task.get(u'state') == u'pending': # There already is a pending DataPusher submission, # skip this one ... log.debug( - 'Skipping DataPusher submission for ' - 'resource {0}'.format(entity.id)) + u'Skipping DataPusher submission for ' + u'resource {0}'.format(entity.id) + ) return - except p.toolkit.ObjectNotFound: + except toolkit.ObjectNotFound: pass try: - log.debug('Submitting resource {0}'.format(entity.id) + - ' to DataPusher') - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': entity.id - }) - except p.toolkit.ValidationError as e: + log.debug( + u'Submitting resource {0}'.format(entity.id) + + u' to DataPusher' + ) + toolkit.get_action(u'datapusher_submit')( + context, { + u'resource_id': entity.id + } + ) + except toolkit.ValidationError as e: # If datapusher is offline want to catch error instead # of raising otherwise resource save will fail with 500 log.critical(e) pass - def before_map(self, m): - m.connect( - 'resource_data', '/dataset/{id}/resource_data/{resource_id}', - controller='ckanext.datapusher.plugin:ResourceDataController', - action='resource_data', ckan_icon='cloud-upload') - return m - def get_actions(self): - return {'datapusher_submit': action.datapusher_submit, - 'datapusher_hook': action.datapusher_hook, - 'datapusher_status': action.datapusher_status} + return { + u'datapusher_submit': action.datapusher_submit, + u'datapusher_hook': action.datapusher_hook, + u'datapusher_status': action.datapusher_status + } def get_auth_functions(self): - return {'datapusher_submit': auth.datapusher_submit, - 'datapusher_status': auth.datapusher_status} + return { + u'datapusher_submit': auth.datapusher_submit, + u'datapusher_status': auth.datapusher_status + } def get_helpers(self): return { - 'datapusher_status': helpers.datapusher_status, - 'datapusher_status_description': - helpers.datapusher_status_description, + u'datapusher_status': helpers.datapusher_status, + u'datapusher_status_description': helpers. + datapusher_status_description, } + + # IBlueprint + + def get_blueprint(self): + return blueprint.datapusher diff --git a/ckanext/datapusher/templates-bs2/datapusher/resource_data.html b/ckanext/datapusher/templates-bs2/datapusher/resource_data.html index 2b89234f791..fa181ceaf47 100644 --- a/ckanext/datapusher/templates-bs2/datapusher/resource_data.html +++ b/ckanext/datapusher/templates-bs2/datapusher/resource_data.html @@ -4,7 +4,7 @@ {% block primary_content_inner %} - {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg.name, resource_id=res.id) %} + {% set action = h.url_for('datapusher.resource_data', id=pkg.name, resource_id=res.id) %} {% set show_table = true %}
diff --git a/ckanext/datapusher/templates-bs2/package/resource_edit_base.html b/ckanext/datapusher/templates-bs2/package/resource_edit_base.html index 736c434c0b8..38072aeeaf7 100644 --- a/ckanext/datapusher/templates-bs2/package/resource_edit_base.html +++ b/ckanext/datapusher/templates-bs2/package/resource_edit_base.html @@ -2,5 +2,5 @@ {% block inner_primary_nav %} {{ super() }} - {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} + {{ h.build_nav_icon('datapusher.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} {% endblock %} diff --git a/ckanext/datapusher/templates/datapusher/resource_data.html b/ckanext/datapusher/templates/datapusher/resource_data.html index 79727927628..84c7df7b291 100644 --- a/ckanext/datapusher/templates/datapusher/resource_data.html +++ b/ckanext/datapusher/templates/datapusher/resource_data.html @@ -4,7 +4,7 @@ {% block primary_content_inner %} - {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg.name, resource_id=res.id) %} + {% set action = h.url_for('datapusher.resource_data', id=pkg.name, resource_id=res.id) %} {% set show_table = true %} diff --git a/ckanext/datapusher/templates/package/resource_edit_base.html b/ckanext/datapusher/templates/package/resource_edit_base.html index 736c434c0b8..38072aeeaf7 100644 --- a/ckanext/datapusher/templates/package/resource_edit_base.html +++ b/ckanext/datapusher/templates/package/resource_edit_base.html @@ -2,5 +2,5 @@ {% block inner_primary_nav %} {{ super() }} - {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} + {{ h.build_nav_icon('datapusher.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} {% endblock %} diff --git a/ckanext/stats/blueprint.py b/ckanext/stats/blueprint.py index 1c5c73795b1..201a5464b8c 100644 --- a/ckanext/stats/blueprint.py +++ b/ckanext/stats/blueprint.py @@ -3,7 +3,6 @@ from flask import Blueprint from ckan.plugins.toolkit import c, render -from ckan.lib.base import BaseController import ckanext.stats.stats as stats_lib import ckan.lib.helpers as h From cec4815c244bf977aadb027b1e48417072c722fc Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 24 Sep 2019 09:29:35 +0300 Subject: [PATCH 2/4] Update tests --- ckanext/datapusher/tests/test_controller.py | 6 +++--- ckanext/datastore/tests/test_dictionary.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ckanext/datapusher/tests/test_controller.py b/ckanext/datapusher/tests/test_controller.py index 118ff7a086e..7d91401988f 100644 --- a/ckanext/datapusher/tests/test_controller.py +++ b/ckanext/datapusher/tests/test_controller.py @@ -3,11 +3,11 @@ import nose import ckan.tests.legacy as tests -from ckan.tests.helpers import FunctionalTestBase +from ckanext.datastore.tests.helpers import DatastoreFunctionalTestBase import ckan.tests.factories as factories -class TestController(FunctionalTestBase): +class TestController(DatastoreFunctionalTestBase): sysadmin_user = None normal_user = None @@ -15,10 +15,10 @@ class TestController(FunctionalTestBase): @classmethod def setup_class(cls): - cls.app = cls._get_test_app() if not tests.is_datastore_supported(): raise nose.SkipTest(u'Datastore not supported') super(TestController, cls).setup_class() + cls.app = cls._get_test_app() def test_resource_data(self): user = factories.User() diff --git a/ckanext/datastore/tests/test_dictionary.py b/ckanext/datastore/tests/test_dictionary.py index bcc0b6daf39..7f6661e59c0 100644 --- a/ckanext/datastore/tests/test_dictionary.py +++ b/ckanext/datastore/tests/test_dictionary.py @@ -6,6 +6,7 @@ class TestDatastoreDictionary(DatastoreFunctionalTestBase): + _load_plugins = (u'datastore', u'datapusher') @classmethod def setup_class(cls): From 08c7a9eb0306e108be1f80d57e781acc9f949fce Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Sat, 28 Sep 2019 19:27:33 +0100 Subject: [PATCH 3/4] Updated Sysadmin and UserCmd help strings Added name and fullname to the help strings for both the Sysadmin and UserCmd commands --- ckan/lib/cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 2218f46f6d1..1793e56ecbb 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -689,8 +689,10 @@ class Sysadmin(CkanCommand): (prompts for password and email if not supplied). Field can be: apikey - password email + fullname + name (this will be the username) + password sysadmin remove USERNAME - removes user from sysadmins ''' @@ -776,8 +778,10 @@ class UserCmd(CkanCommand): - add a user (prompts for email and password if not supplied). Field can be: apikey - password email + fullname + name (this will be the username) + password user setpass USERNAME - set user password (prompts) user remove USERNAME - removes user from users user search QUERY - searches for a user name From 9ea12fdbb7cfbce04191725bbf35eac987d4b6d4 Mon Sep 17 00:00:00 2001 From: Francesco Frassinelli Date: Tue, 1 Oct 2019 15:44:02 +0200 Subject: [PATCH 4/4] Fix PostGIS base image tag --- contrib/docker/postgresql/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 82d96fd9b5e..30d95abc546 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -1,5 +1,5 @@ #FROM postgres:9.6 -FROM mdillon/postgis +FROM mdillon/postgis:11 MAINTAINER Open Knowledge # Allow connections; we don't map out any ports so only linked docker containers can connect