From 80b9c6e86a7c22b5efa56741a3ab80a4041892ba Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 2 Jan 2017 15:55:25 -0500 Subject: [PATCH 01/36] [#3384] factor load_config() out of CkanCommand class --- ckan/lib/cli.py | 108 +++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 3cc819fed70..ed1c3c6a829 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -133,6 +133,61 @@ def ungettext(self, singular, plural, n): return singular +def _get_config(config=None): + from paste.deploy import appconfig + + if config: + filename = os.path.abspath(config) + config_source = '-c parameter' + elif os.environ.get('CKAN_INI'): + filename = os.environ.get('CKAN_INI') + config_source = '$CKAN_INI' + else: + filename = os.path.join(os.getcwd(), 'development.ini') + config_source = 'default value' + + if not os.path.exists(filename): + msg = 'Config file not found: %s' % filename + msg += '\n(Given by: %s)' % config_source + exit(msg) + + fileConfig(filename) + return appconfig('config:' + filename) + + +def load_config(config, load_site_user=True): + conf = _get_config(config) + assert 'ckan' not in dir() # otherwise loggers would be disabled + # We have now loaded the config. Now we can import ckan for the + # first time. + from ckan.config.environment import load_environment + load_environment(conf.global_conf, conf.local_conf) + + registry = Registry() + registry.prepare() + import pylons + registry.register(pylons.translator, MockTranslator()) + + if model.user_table.exists() and load_site_user: + # If the DB has already been initialized, create and register + # a pylons context object, and add the site user to it, so the + # auth works as in a normal web request + c = pylons.util.AttribSafeContextObj() + + registry.register(pylons.c, c) + + site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {}) + + pylons.c.user = site_user['name'] + pylons.c.userobj = model.User.get(site_user['name']) + + ## give routes enough information to run url_for + parsed = urlparse.urlparse(conf.get('ckan.site_url', 'http://0.0.0.0')) + request_config = routes.request_config() + request_config.host = parsed.netloc + parsed.path + request_config.protocol = parsed.scheme + + class CkanCommand(paste.script.command.Command): '''Base class for classes that implement CKAN paster commands to inherit.''' parser = paste.script.command.Command.standard_parser(verbose=True) @@ -145,59 +200,8 @@ class CkanCommand(paste.script.command.Command): default_verbosity = 1 group_name = 'ckan' - def _get_config(self): - from paste.deploy import appconfig - - if self.options.config: - self.filename = os.path.abspath(self.options.config) - config_source = '-c parameter' - elif os.environ.get('CKAN_INI'): - self.filename = os.environ.get('CKAN_INI') - config_source = '$CKAN_INI' - else: - self.filename = os.path.join(os.getcwd(), 'development.ini') - config_source = 'default value' - - if not os.path.exists(self.filename): - msg = 'Config file not found: %s' % self.filename - msg += '\n(Given by: %s)' % config_source - raise self.BadCommand(msg) - - fileConfig(self.filename) - return appconfig('config:' + self.filename) - def _load_config(self, load_site_user=True): - conf = self._get_config() - assert 'ckan' not in dir() # otherwise loggers would be disabled - # We have now loaded the config. Now we can import ckan for the - # first time. - from ckan.config.environment import load_environment - load_environment(conf.global_conf, conf.local_conf) - - self.registry = Registry() - self.registry.prepare() - import pylons - self.translator_obj = MockTranslator() - self.registry.register(pylons.translator, self.translator_obj) - - if model.user_table.exists() and load_site_user: - # If the DB has already been initialized, create and register - # a pylons context object, and add the site user to it, so the - # auth works as in a normal web request - c = pylons.util.AttribSafeContextObj() - - self.registry.register(pylons.c, c) - - self.site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {}) - - pylons.c.user = self.site_user['name'] - pylons.c.userobj = model.User.get(self.site_user['name']) - - ## give routes enough information to run url_for - parsed = urlparse.urlparse(conf.get('ckan.site_url', 'http://0.0.0.0')) - request_config = routes.request_config() - request_config.host = parsed.netloc + parsed.path - request_config.protocol = parsed.scheme + load_config(self.options.config, load_site_user) def _setup_app(self): cmd = paste.script.appinstall.SetupCommand('setup-app') From d0f99dbeb9c8d240bb406e57148fbc03e845f56b Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 2 Jan 2017 15:58:42 -0500 Subject: [PATCH 02/36] [#3384] docopt and simple function for datastore cli --- ckanext/datastore/commands.py | 57 +++++++++++++++-------------------- requirements.in | 1 + requirements.txt | 1 + setup.py | 2 +- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index 5aa88f23a23..d6a626e0a74 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -1,19 +1,26 @@ # encoding: utf-8 +u"""Perform commands to set up the datastore + +Usage: + paster [options] datastore set-permissions + +Emit an SQL script that will set the permissions for the +datastore users as configured in your configuration file. + +Options: + -c --config=CONFIG CKAN configuration file + --plugin=ckan paster plugin (when used outside of ckan directory) +""" -from __future__ import print_function -import argparse import os import sys -import ckan.lib.cli as cli +from ckan.lib import cli +from docopt import docopt -def _abort(message): - print(message, file=sys.stderr) - sys.exit(1) - -def _set_permissions(args): +def _set_permissions(): write_url = cli.parse_db_config('ckan.datastore.write_url') read_url = cli.parse_db_config('ckan.datastore.read_url') db_url = cli.parse_db_config('sqlalchemy.url') @@ -22,7 +29,7 @@ def _set_permissions(args): # This obviously doesn't check they're the same database (the hosts/ports # could be different), but it's better than nothing, I guess. if write_url['db_name'] != read_url['db_name']: - _abort("The datastore write_url and read_url must refer to the same " + exit("The datastore write_url and read_url must refer to the same " "database!") context = { @@ -46,30 +53,14 @@ def _permissions_sql(context): return template.format(**context) -parser = argparse.ArgumentParser( - prog='paster datastore', - description='Perform commands to set up the datastore', - epilog='Make sure that the datastore URLs are set properly before you run ' - 'these commands!') -subparsers = parser.add_subparsers(title='commands') - -parser_set_perms = subparsers.add_parser( - 'set-permissions', - description='Set the permissions on the datastore.', - help='This command will help ensure that the permissions for the ' - 'datastore users as configured in your configuration file are ' - 'correct at the database. It will emit an SQL script that ' - 'you can use to set these permissions.', - epilog='"The ships hung in the sky in much the same way that bricks ' - 'don\'t."') -parser_set_perms.set_defaults(func=_set_permissions) - +def datastore_command(command): + opts = docopt(__doc__) -class SetupDatastoreCommand(cli.CkanCommand): - summary = parser.description + cli.load_config(opts['--config']) - def command(self): - self._load_config() + if opts['set-permissions']: + _set_permissions() + exit(0) # avoid paster error - args = parser.parse_args(self.args) - args.func(args) +# for paster's command index +datastore_command.summary = __doc__.split(u'\n')[0] diff --git a/requirements.in b/requirements.in index c7f7eafde37..ebb861936b3 100644 --- a/requirements.in +++ b/requirements.in @@ -3,6 +3,7 @@ Babel==2.3.4 Beaker==1.8.1 # Needs to be pinned to a more up to date version than the Pylons one bleach==1.5.0 +docopt==0.6.2 fanstatic==0.12 Flask==0.11.1 Jinja2==2.8 diff --git a/requirements.txt b/requirements.txt index 73cb5e79f4a..b49a3f2ccf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ Beaker==1.8.1 # via pylons bleach==1.5.0 click==6.6 # via rq decorator==4.0.6 # via pylons, sqlalchemy-migrate +docopt==0.6.2 fanstatic==0.12 Flask==0.11.1 FormEncode==1.3.0 # via pylons diff --git a/setup.py b/setup.py index 99c24f5ff33..7b05d5d97f5 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'trans = ckan.lib.cli:TranslationsCommand', 'minify = ckan.lib.cli:MinifyCommand', 'less = ckan.lib.cli:LessCommand', - 'datastore = ckanext.datastore.commands:SetupDatastoreCommand', + 'datastore = ckanext.datastore.commands:datastore_command', 'datapusher = ckanext.datapusher.cli:DatapusherCommand', 'front-end-build = ckan.lib.cli:FrontEndBuildCommand', 'views = ckan.lib.cli:ViewsCommand', From c2950187501f802b254eb60370c9d3fdfa9de9e3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 2 Jan 2017 16:35:21 -0500 Subject: [PATCH 03/36] [#3384] pep8 --- ckanext/datastore/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index d6a626e0a74..f10c9a0649e 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -30,7 +30,7 @@ def _set_permissions(): # could be different), but it's better than nothing, I guess. if write_url['db_name'] != read_url['db_name']: exit("The datastore write_url and read_url must refer to the same " - "database!") + "database!") context = { 'maindb': db_url['db_name'], @@ -62,5 +62,6 @@ def datastore_command(command): _set_permissions() exit(0) # avoid paster error + # for paster's command index datastore_command.summary = __doc__.split(u'\n')[0] From e94354a55a513c72e70db7d6162b241ee74139ed Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 2 Jan 2017 17:10:56 -0500 Subject: [PATCH 04/36] [#3384] use datastore.helpers.identifier to format sql safely --- ckanext/datastore/commands.py | 60 ++++++++++++++------------- ckanext/datastore/set_permissions.sql | 24 +++++------ 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index f10c9a0649e..d6d0963a8b6 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -4,10 +4,11 @@ Usage: paster [options] datastore set-permissions -Emit an SQL script that will set the permissions for the -datastore users as configured in your configuration file. +set-permissions: Emit an SQL script that will set the permissions +for the datastore users as configured in your configuration file. Options: + -h --help This help text -c --config=CONFIG CKAN configuration file --plugin=ckan paster plugin (when used outside of ckan directory) """ @@ -16,11 +17,26 @@ import sys from ckan.lib import cli +from ckanext.datastore.helpers import identifier from docopt import docopt -def _set_permissions(): +def datastore_command(command): + opts = docopt(__doc__) + + cli.load_config(opts['--config']) + + if opts['set-permissions']: + set_permissions() + exit(0) # avoid paster error + + +# for paster's command index +datastore_command.summary = __doc__.split(u'\n')[0] + + +def set_permissions(): write_url = cli.parse_db_config('ckan.datastore.write_url') read_url = cli.parse_db_config('ckan.datastore.read_url') db_url = cli.parse_db_config('sqlalchemy.url') @@ -32,36 +48,24 @@ def _set_permissions(): exit("The datastore write_url and read_url must refer to the same " "database!") - context = { - 'maindb': db_url['db_name'], - 'datastoredb': write_url['db_name'], - 'mainuser': db_url['db_user'], - 'writeuser': write_url['db_user'], - 'readuser': read_url['db_user'], - } - - sql = _permissions_sql(context) + sql = permissions_sql( + maindb=db_url['db_name'], + datastoredb=write_url['db_name'], + mainuser=db_url['db_user'], + writeuser=write_url['db_user'], + readuser=read_url['db_user']) print(sql) -def _permissions_sql(context): +def permissions_sql(maindb, datastoredb, mainuser, writeuser, readuser): template_filename = os.path.join(os.path.dirname(__file__), 'set_permissions.sql') with open(template_filename) as fp: template = fp.read() - return template.format(**context) - - -def datastore_command(command): - opts = docopt(__doc__) - - cli.load_config(opts['--config']) - - if opts['set-permissions']: - _set_permissions() - exit(0) # avoid paster error - - -# for paster's command index -datastore_command.summary = __doc__.split(u'\n')[0] + return template.format( + maindb=identifier(maindb), + datastoredb=identifier(datastoredb), + mainuser=identifier(mainuser), + writeuser=identifier(writeuser), + readuser=identifier(readuser)) diff --git a/ckanext/datastore/set_permissions.sql b/ckanext/datastore/set_permissions.sql index 33b3bb4007b..79f0cba0c3a 100644 --- a/ckanext/datastore/set_permissions.sql +++ b/ckanext/datastore/set_permissions.sql @@ -29,25 +29,25 @@ over SSH: REVOKE CREATE ON SCHEMA public FROM PUBLIC; REVOKE USAGE ON SCHEMA public FROM PUBLIC; -GRANT CREATE ON SCHEMA public TO "{mainuser}"; -GRANT USAGE ON SCHEMA public TO "{mainuser}"; +GRANT CREATE ON SCHEMA public TO {mainuser}; +GRANT USAGE ON SCHEMA public TO {mainuser}; -GRANT CREATE ON SCHEMA public TO "{writeuser}"; -GRANT USAGE ON SCHEMA public TO "{writeuser}"; +GRANT CREATE ON SCHEMA public TO {writeuser}; +GRANT USAGE ON SCHEMA public TO {writeuser}; -- take connect permissions from main db -REVOKE CONNECT ON DATABASE "{maindb}" FROM "{readuser}"; +REVOKE CONNECT ON DATABASE {maindb} FROM {readuser}; -- grant select permissions for read-only user -GRANT CONNECT ON DATABASE "{datastoredb}" TO "{readuser}"; -GRANT USAGE ON SCHEMA public TO "{readuser}"; +GRANT CONNECT ON DATABASE {datastoredb} TO {readuser}; +GRANT USAGE ON SCHEMA public TO {readuser}; -- grant access to current tables and views to read-only user -GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{readuser}"; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO {readuser}; -- grant access to new tables and views by default -ALTER DEFAULT PRIVILEGES FOR USER "{writeuser}" IN SCHEMA public - GRANT SELECT ON TABLES TO "{readuser}"; +ALTER DEFAULT PRIVILEGES FOR USER {writeuser} IN SCHEMA public + GRANT SELECT ON TABLES TO {readuser}; CREATE OR REPLACE VIEW "_table_metadata" AS SELECT DISTINCT @@ -67,5 +67,5 @@ CREATE OR REPLACE VIEW "_table_metadata" AS OR dependee.relname IN (SELECT viewname FROM pg_catalog.pg_views)) AND dependee.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname='public') ORDER BY dependee.oid DESC; -ALTER VIEW "_table_metadata" OWNER TO "{writeuser}"; -GRANT SELECT ON "_table_metadata" TO "{readuser}"; +ALTER VIEW "_table_metadata" OWNER TO {writeuser}; +GRANT SELECT ON "_table_metadata" TO {readuser}; From fe1c98364d153935dd96baf8b7ec3a9405eef805 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sat, 7 Jan 2017 15:01:54 -0500 Subject: [PATCH 05/36] [#3384] use click instead of docopt --- ckanext/datastore/commands.py | 43 +++++++++++++++++++++++------------ requirements.in | 1 - requirements.txt | 1 - 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index d6d0963a8b6..10e73cec9a5 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -16,30 +16,45 @@ import os import sys -from ckan.lib import cli +from ckan.lib.cli import load_config, parse_db_config from ckanext.datastore.helpers import identifier -from docopt import docopt +import click def datastore_command(command): - opts = docopt(__doc__) - - cli.load_config(opts['--config']) - - if opts['set-permissions']: - set_permissions() + 'a small adapter for paster -> click' + cli() exit(0) # avoid paster error # for paster's command index datastore_command.summary = __doc__.split(u'\n')[0] - - -def set_permissions(): - write_url = cli.parse_db_config('ckan.datastore.write_url') - read_url = cli.parse_db_config('ckan.datastore.read_url') - db_url = cli.parse_db_config('sqlalchemy.url') +datastore_command.group_name = 'ckan' + + +@click.group('paster') +@click.help_option('-h', '--help') +@click.option( + '--plugin', + metavar='ckan', + help='paster plugin (when run outside ckan directory)') +@click.argument('datastore', metavar='datastore') +def cli(plugin, datastore): + pass + + +@cli.command( + 'set-permissions', + help='Emit an SQL script that will set the permissions for the ' + 'datastore users as configured in your configuration file.') +@click.option('-c', '--config', default='development.ini') +def set_permissions(config): + load_config(config) + + write_url = parse_db_config('ckan.datastore.write_url') + read_url = parse_db_config('ckan.datastore.read_url') + db_url = parse_db_config('sqlalchemy.url') # Basic validation that read and write URLs reference the same database. # This obviously doesn't check they're the same database (the hosts/ports diff --git a/requirements.in b/requirements.in index ebb861936b3..c7f7eafde37 100644 --- a/requirements.in +++ b/requirements.in @@ -3,7 +3,6 @@ Babel==2.3.4 Beaker==1.8.1 # Needs to be pinned to a more up to date version than the Pylons one bleach==1.5.0 -docopt==0.6.2 fanstatic==0.12 Flask==0.11.1 Jinja2==2.8 diff --git a/requirements.txt b/requirements.txt index b49a3f2ccf8..73cb5e79f4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ Beaker==1.8.1 # via pylons bleach==1.5.0 click==6.6 # via rq decorator==4.0.6 # via pylons, sqlalchemy-migrate -docopt==0.6.2 fanstatic==0.12 Flask==0.11.1 FormEncode==1.3.0 # via pylons From 584b22bbec94e2b8f1a094022689555a3dad960b Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 8 Jan 2017 11:54:42 -0500 Subject: [PATCH 06/36] [#3384] factor out paster_click_group wrapper --- ckan/lib/cli.py | 38 +++++++++++++++++++++++++++ ckanext/datastore/commands.py | 48 +++++++++-------------------------- setup.py | 2 +- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index ed1c3c6a829..29379cead2a 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -20,6 +20,7 @@ import paste.script from paste.registry import Registry from paste.script.util.logging_config import fileConfig +import click import ckan.logic as logic import ckan.model as model @@ -188,6 +189,43 @@ def load_config(config, load_site_user=True): request_config.protocol = parsed.scheme +def paster_click_group(command, summary): + '''Return a paster command click.Group for paster subcommands + + :param command: the paster command linked to this function from + setup.py, used in help text (e.g. "datastore") + :param summary: summary text used in paster's help/command listings + (e.g. "Perform commands to set up the datastore") + ''' + class PasterClickGroup(click.Group): + '''A click.Group that may be called like a paster command''' + def __call__(self, ignored_command): + super(PasterClickGroup, self).__call__() + + @click.group('paster', cls=PasterClickGroup) + @click.help_option('-h', '--help') + @click.option( + '--plugin', + metavar='ckan', + help='paster plugin (when run outside ckan directory)') + @click.argument('command', metavar=command) + def cli(plugin, command): + pass + + cli.summary = summary + cli.group_name = u'ckan' + return cli + + +# common definition for paster ... --config +click_config_option = click.option( + '-c', + '--config', + default=None, + metavar='CONFIG', + help=u'Config file to use (default: development.ini)') + + class CkanCommand(paste.script.command.Command): '''Base class for classes that implement CKAN paster commands to inherit.''' parser = paste.script.command.Command.standard_parser(verbose=True) diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index 10e73cec9a5..6bc3e3389c9 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -1,54 +1,30 @@ # encoding: utf-8 -u"""Perform commands to set up the datastore - -Usage: - paster [options] datastore set-permissions - -set-permissions: Emit an SQL script that will set the permissions -for the datastore users as configured in your configuration file. - -Options: - -h --help This help text - -c --config=CONFIG CKAN configuration file - --plugin=ckan paster plugin (when used outside of ckan directory) -""" import os import sys -from ckan.lib.cli import load_config, parse_db_config +from ckan.lib.cli import ( + load_config, + parse_db_config, + paster_click_group, + click_config_option, +) from ckanext.datastore.helpers import identifier import click -def datastore_command(command): - 'a small adapter for paster -> click' - cli() - exit(0) # avoid paster error - - -# for paster's command index -datastore_command.summary = __doc__.split(u'\n')[0] -datastore_command.group_name = 'ckan' +datastore_group = paster_click_group( + command=u'datastore', + summary=u'Perform commands to set up the datastore') -@click.group('paster') -@click.help_option('-h', '--help') -@click.option( - '--plugin', - metavar='ckan', - help='paster plugin (when run outside ckan directory)') -@click.argument('datastore', metavar='datastore') -def cli(plugin, datastore): - pass - - -@cli.command( +@datastore_group.command( 'set-permissions', help='Emit an SQL script that will set the permissions for the ' 'datastore users as configured in your configuration file.') -@click.option('-c', '--config', default='development.ini') +@click.help_option('-h', '--help') +@click_config_option def set_permissions(config): load_config(config) diff --git a/setup.py b/setup.py index 7b05d5d97f5..b5d19b859f9 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'trans = ckan.lib.cli:TranslationsCommand', 'minify = ckan.lib.cli:MinifyCommand', 'less = ckan.lib.cli:LessCommand', - 'datastore = ckanext.datastore.commands:datastore_command', + 'datastore = ckanext.datastore.commands:datastore_group', 'datapusher = ckanext.datapusher.cli:DatapusherCommand', 'front-end-build = ckan.lib.cli:FrontEndBuildCommand', 'views = ckan.lib.cli:ViewsCommand', From a840886bbaf2cbd1900ba30b1d9e086fabd39eb3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 8 Jan 2017 13:01:25 -0500 Subject: [PATCH 07/36] [#3384] literal string style --- ckan/tests/test_coding_standards.py | 1 - ckanext/datastore/commands.py | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index d01b61b3059..4aba46377a1 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -584,7 +584,6 @@ def find_unprefixed_string_literals(filename): u'ckanext/datapusher/tests/test_action.py', u'ckanext/datapusher/tests/test_default_views.py', u'ckanext/datapusher/tests/test_interfaces.py', - u'ckanext/datastore/commands.py', u'ckanext/datastore/controller.py', u'ckanext/datastore/db.py', u'ckanext/datastore/helpers.py', diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index 6bc3e3389c9..80384b232a4 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -20,24 +20,24 @@ @datastore_group.command( - 'set-permissions', - help='Emit an SQL script that will set the permissions for the ' - 'datastore users as configured in your configuration file.') -@click.help_option('-h', '--help') + u'set-permissions', + help=u'Emit an SQL script that will set the permissions for the ' + u'datastore users as configured in your configuration file.') +@click.help_option(u'-h', u'--help') @click_config_option def set_permissions(config): load_config(config) - write_url = parse_db_config('ckan.datastore.write_url') - read_url = parse_db_config('ckan.datastore.read_url') - db_url = parse_db_config('sqlalchemy.url') + write_url = parse_db_config(u'ckan.datastore.write_url') + read_url = parse_db_config(u'ckan.datastore.read_url') + db_url = parse_db_config(u'sqlalchemy.url') # Basic validation that read and write URLs reference the same database. # This obviously doesn't check they're the same database (the hosts/ports # could be different), but it's better than nothing, I guess. if write_url['db_name'] != read_url['db_name']: - exit("The datastore write_url and read_url must refer to the same " - "database!") + exit(u"The datastore write_url and read_url must refer to the same " + u"database!") sql = permissions_sql( maindb=db_url['db_name'], @@ -51,7 +51,7 @@ def set_permissions(config): def permissions_sql(maindb, datastoredb, mainuser, writeuser, readuser): template_filename = os.path.join(os.path.dirname(__file__), - 'set_permissions.sql') + u'set_permissions.sql') with open(template_filename) as fp: template = fp.read() return template.format( From 0d37eff0bc4c32974064f6bbb4cb5b5d181c8a9f Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 8 Jan 2017 17:34:28 -0500 Subject: [PATCH 08/36] [#3384] fix prog_name, help_option_names --- ckan/lib/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 29379cead2a..5fcac1c5dca 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -200,16 +200,16 @@ def paster_click_group(command, summary): class PasterClickGroup(click.Group): '''A click.Group that may be called like a paster command''' def __call__(self, ignored_command): - super(PasterClickGroup, self).__call__() + super(PasterClickGroup, self).__call__( + prog_name=u'paster ' + command, + help_option_names=[u'-h', u'--help']) @click.group('paster', cls=PasterClickGroup) - @click.help_option('-h', '--help') @click.option( '--plugin', metavar='ckan', help='paster plugin (when run outside ckan directory)') - @click.argument('command', metavar=command) - def cli(plugin, command): + def cli(plugin): pass cli.summary = summary From 1103a45f46ef55a77fdf8f967004ffaea65fb811 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 12 Jan 2017 16:22:34 -0500 Subject: [PATCH 09/36] [#3384] fix for paster_click_group --- ckan/lib/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 5fcac1c5dca..95c7e2ae082 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -204,12 +204,17 @@ def __call__(self, ignored_command): prog_name=u'paster ' + command, help_option_names=[u'-h', u'--help']) - @click.group('paster', cls=PasterClickGroup) + class HiddenClickArgument(click.Argument): + def get_usage_pieces(self, ctx): + return [] + + @click.group(cls=PasterClickGroup) @click.option( '--plugin', metavar='ckan', help='paster plugin (when run outside ckan directory)') - def cli(plugin): + @click.argument('command', cls=HiddenClickArgument) + def cli(plugin, command): pass cli.summary = summary From c7bd2ab4aa7f11655838fd7946706085c7d1ac02 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 3 Feb 2017 15:00:00 -0500 Subject: [PATCH 10/36] [#3384] paster->click adapter fix for -c before subcommand --- ckan/lib/cli.py | 22 +++++++++++----------- ckanext/datastore/commands.py | 6 +++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 95c7e2ae082..09108376483 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -189,7 +189,7 @@ def load_config(config, load_site_user=True): request_config.protocol = parsed.scheme -def paster_click_group(command, summary): +def paster_click_group(summary): '''Return a paster command click.Group for paster subcommands :param command: the paster command linked to this function from @@ -200,22 +200,22 @@ def paster_click_group(command, summary): class PasterClickGroup(click.Group): '''A click.Group that may be called like a paster command''' def __call__(self, ignored_command): - super(PasterClickGroup, self).__call__( - prog_name=u'paster ' + command, - help_option_names=[u'-h', u'--help']) - - class HiddenClickArgument(click.Argument): - def get_usage_pieces(self, ctx): - return [] + sys.argv.remove(ignored_command) + return super(PasterClickGroup, self).__call__( + prog_name=u'paster ' + ignored_command, + help_option_names=[u'-h', u'--help'], + obj={}) @click.group(cls=PasterClickGroup) @click.option( '--plugin', metavar='ckan', help='paster plugin (when run outside ckan directory)') - @click.argument('command', cls=HiddenClickArgument) - def cli(plugin, command): - pass + @click_config_option + @click.pass_context + def cli(ctx, plugin, config): + ctx.obj['config'] = config + cli.summary = summary cli.group_name = u'ckan' diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py index 80384b232a4..8b6a31d28b3 100644 --- a/ckanext/datastore/commands.py +++ b/ckanext/datastore/commands.py @@ -15,7 +15,6 @@ datastore_group = paster_click_group( - command=u'datastore', summary=u'Perform commands to set up the datastore') @@ -25,8 +24,9 @@ u'datastore users as configured in your configuration file.') @click.help_option(u'-h', u'--help') @click_config_option -def set_permissions(config): - load_config(config) +@click.pass_context +def set_permissions(ctx, config): + load_config(config or ctx.obj['config']) write_url = parse_db_config(u'ckan.datastore.write_url') read_url = parse_db_config(u'ckan.datastore.read_url') From 487d5fa26643823272b6671cb3aef05fe2f7185d Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Mon, 20 Feb 2017 13:15:39 +0200 Subject: [PATCH 11/36] [#3071] Regenerating API Key fails for users missing required columns (ex: email) --- ckan/lib/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 3cc819fed70..c833029d414 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -684,9 +684,14 @@ def add(self): print 'User "%s" not found' % username makeuser = raw_input('Create new user: %s? [y/n]' % username) if makeuser == 'y': + useremail = raw_input('Please input %s email: ' % username) + if not useremail: + print 'Need email to create new user' + return password = UserCmd.password_prompt() print('Creating %s user' % username) user = model.User(name=unicode(username), + email=unicode(useremail), password=password) else: print 'Exiting ...' From e60c204ca9961ef0ecefd03db2996be9134fe218 Mon Sep 17 00:00:00 2001 From: Gleb Date: Wed, 22 Feb 2017 14:47:48 +0200 Subject: [PATCH 12/36] #3028 / UI labels for the last_modified and revision_timestamp fields --- ckan/templates/package/resource_read.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index e305ada5516..be0bba84aa6 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -151,8 +151,12 @@

{{ _('Additional Information') }}

- {{ _('Last updated') }} - {{ h.render_datetime(res.last_modified) or h.render_datetime(res.revision_timestamp) or h.render_datetime(res.created) or _('unknown') }} + {{ _('Data last updated') }} + {{ h.render_datetime(res.last_modified) or h.render_datetime(res.created) or _('unknown') }} + + + {{ _('Metadata last updated') }} + {{ h.render_datetime(res.revision_timestamp) or h.render_datetime(res.created) or _('unknown') }} {{ _('Created') }} From 927d729b084424458db3a342fc4b296ddcca5914 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Mar 2017 14:48:38 +0200 Subject: [PATCH 13/36] nested columns serialized with json instead of python str representation --- ckanext/datastore/controller.py | 28 ++++++++++++++++++----- ckanext/datastore/tests/test_dump.py | 34 ++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 2d11e3d5adf..adf04405b11 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -1,9 +1,6 @@ # encoding: utf-8 -import StringIO -import md5 - -import pylons +import json from ckan.plugins.toolkit import ( Invalid, @@ -34,6 +31,20 @@ PAGINATE_BY = 10000 +def _dump_nested(column, record): + name, ctype = column + value = record[name] + + is_nested = ( + ctype == 'json' or + ctype.startswith('_') or + ctype.endswith(']') + ) + if is_nested: + return json.dumps(value) + return value + + class DatastoreController(BaseController): def dump(self, resource_id): try: @@ -72,7 +83,9 @@ def result_page(offset, limit): abort(404, _('DataStore resource not found')) result = result_page(offset, limit) - columns = [x['id'] for x in result['fields']] + columns = [ + (x['id'], x['type']) + for x in result['fields']] with start_writer(result['fields']) as wr: while True: @@ -80,7 +93,10 @@ def result_page(offset, limit): break for record in result['records']: - wr.writerow([record[column] for column in columns]) + + wr.writerow([ + _dump_nested(column, record) + for column in columns]) if len(result['records']) < PAGINATE_BY: break diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 8b3abbd58ba..45a1aaa1ceb 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -13,7 +13,7 @@ import paste.fixture import sqlalchemy.orm as orm from ckan.common import config -from nose.tools import assert_equals +from nose.tools import assert_equals, assert_in class TestDatastoreDump(object): @@ -50,6 +50,10 @@ def setup_class(cls): { 'id': u'characters', u'type': u'_text' + }, + { + 'id': 'random_letters', + 'type': 'text[]' } ], 'records': [ @@ -64,12 +68,18 @@ def setup_class(cls): u'characters': [ u'Princess Anna', u'Sergius' + ], + 'random_letters': [ + 'a', 'e', 'x' ] }, { u'b\xfck': 'warandpeace', 'author': 'tolstoy', - 'nested': {'a': 'b'} + 'nested': {'a': 'b'}, + 'random_letters': [ + + ] } ] } @@ -94,10 +104,12 @@ def test_dump_basic(self): res = self.app.get('/datastore/dump/{0}'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = u'_id,b\xfck,author,published,characters,nested' + expected = ( + u'_id,b\xfck,author,published' + u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) - assert 'warandpeace' in content - assert "[u'Princess Anna', u'Sergius']" in content + assert_in('warandpeace', content) + assert_in('"[""Princess Anna"", ""Sergius""]"', content) # get with alias instead of id res = self.app.get('/datastore/dump/{0}'.format(str( @@ -113,6 +125,14 @@ def test_dump_limit(self): res = self.app.get('/datastore/dump/{0}?limit=1'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = u'_id,b\xfck,author,published,characters,nested' + expected = (u'_id,b\xfck,author,published' + u',characters,random_letters,nested') assert_equals(content[:len(expected)], expected) - assert_equals(len(content), 148) + + expected_content = ( + u'_id,b\xfck,author,published,characters,random_letters,' + u'nested\r\n1,annakarenina,tolstoy,2005-03-01T00:00:00,' + u'"[""Princess Anna"", ""Sergius""]",' + u'"[""a"", ""e"", ""x""]","[""b"", ' + u'{""moo"": ""moo""}]"\r\n') + assert_equals(content, expected_content) From fe968a99dbac23506aad76d857135e6a7b9d5083 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Mar 2017 17:57:44 +0200 Subject: [PATCH 14/36] move dump function to writers --- ckanext/datastore/controller.py | 23 +--------- ckanext/datastore/tests/test_dump.py | 64 ++++++++++++++++++++++++++-- ckanext/datastore/writer.py | 50 +++++++++++++++++++--- 3 files changed, 106 insertions(+), 31 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index adf04405b11..a9ebfb70a03 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -31,20 +31,6 @@ PAGINATE_BY = 10000 -def _dump_nested(column, record): - name, ctype = column - value = record[name] - - is_nested = ( - ctype == 'json' or - ctype.startswith('_') or - ctype.endswith(']') - ) - if is_nested: - return json.dumps(value) - return value - - class DatastoreController(BaseController): def dump(self, resource_id): try: @@ -83,9 +69,7 @@ def result_page(offset, limit): abort(404, _('DataStore resource not found')) result = result_page(offset, limit) - columns = [ - (x['id'], x['type']) - for x in result['fields']] + columns = [x['id'] for x in result['fields']] with start_writer(result['fields']) as wr: while True: @@ -93,10 +77,7 @@ def result_page(offset, limit): break for record in result['records']: - - wr.writerow([ - _dump_nested(column, record) - for column in columns]) + wr.writerow([record[column] for column in columns]) if len(result['records']) < PAGINATE_BY: break diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 45a1aaa1ceb..c460db873de 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -125,9 +125,6 @@ def test_dump_limit(self): res = self.app.get('/datastore/dump/{0}?limit=1'.format(str( self.data['resource_id'])), extra_environ=auth) content = res.body.decode('utf-8') - expected = (u'_id,b\xfck,author,published' - u',characters,random_letters,nested') - assert_equals(content[:len(expected)], expected) expected_content = ( u'_id,b\xfck,author,published,characters,random_letters,' @@ -136,3 +133,64 @@ def test_dump_limit(self): u'"[""a"", ""e"", ""x""]","[""b"", ' u'{""moo"": ""moo""}]"\r\n') assert_equals(content, expected_content) + + def test_dump_tsv(self): + auth = {'Authorization': str(self.normal_user.apikey)} + res = self.app.get('/datastore/dump/{0}?limit=1&format=tsv'.format(str( + self.data['resource_id'])), extra_environ=auth) + content = res.body.decode('utf-8') + + expected_content = ( + u'_id\tb\xfck\tauthor\tpublished\tcharacters\trandom_letters\t' + u'nested\r\n1\tannakarenina\ttolstoy\t2005-03-01T00:00:00\t' + u'"[""Princess Anna"", ""Sergius""]"\t' + u'"[""a"", ""e"", ""x""]"\t"[""b"", ' + u'{""moo"": ""moo""}]"\r\n') + assert_equals(content, expected_content) + + def test_dump_json(self): + auth = {'Authorization': str(self.normal_user.apikey)} + res = self.app.get('/datastore/dump/{0}?limit=1&format=json'.format( + str(self.data['resource_id'])), extra_environ=auth) + content = res.body.decode('utf-8') + expected_content = ( + u'{\n "fields": [{"type":"int4","id":"_id"},{"type":"text",' + u'"id":"b\xfck"},{"type":"text","id":"author"},{"type":"timestamp"' + u',"id":"published"},{"type":"_text","id":"characters"},' + u'{"type":"_text","id":"random_letters"},{"type":"json",' + u'"id":"nested"}],\n "records": [\n ' + u'[1,"annakarenina","tolstoy","2005-03-01T00:00:00",' + u'["Princess Anna","Sergius"],["a","e","x"],["b",' + u'{"moo":"moo"}]]\n]}\n') + assert_equals(content, expected_content) + + def test_dump_xml(self): + auth = {'Authorization': str(self.normal_user.apikey)} + res = self.app.get('/datastore/dump/{0}?limit=1&format=xml'.format(str( + self.data['resource_id'])), extra_environ=auth) + content = res.body.decode('utf-8') + expected_content = ( + u'\n' + r'' + u'annakarenina' + u'tolstoy' + u'2005-03-01T00:00:00' + u'' + u'Princess Anna' + u'Sergius' + u'' + u'' + u'a' + u'e' + u'x' + u'' + u'' + u'b' + u'' + u'moo' + u'' + u'' + u'\n' + u'\n' + ) + assert_equals(content, expected_content) diff --git a/ckanext/datastore/writer.py b/ckanext/datastore/writer.py index 62dc0b66619..9883c5e5dca 100644 --- a/ckanext/datastore/writer.py +++ b/ckanext/datastore/writer.py @@ -10,6 +10,14 @@ UTF8_BOM = u'\uFEFF'.encode(u'utf-8') +def _json_dump_nested(value): + is_nested = isinstance(value, (list, dict)) + + if is_nested: + return json.dumps(value) + return value + + @contextmanager def csv_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 CSV data to response @@ -31,7 +39,7 @@ def csv_writer(response, fields, name=None, bom=False): response.headers['Content-disposition'] = ( b'attachment; filename="{name}.csv"'.format( name=encode_rfc2231(name))) - wr = unicodecsv.writer(response, encoding=u'utf-8') + wr = CSVWriter(response, fields, encoding=u'utf-8') if bom: response.write(UTF8_BOM) wr.writerow(f['id'] for f in fields) @@ -60,14 +68,25 @@ def tsv_writer(response, fields, name=None, bom=False): response.headers['Content-disposition'] = ( b'attachment; filename="{name}.tsv"'.format( name=encode_rfc2231(name))) - wr = unicodecsv.writer( - response, encoding=u'utf-8', dialect=unicodecsv.excel_tab) + wr = CSVWriter( + response, fields, encoding=u'utf-8', dialect=unicodecsv.excel_tab, + ) if bom: response.write(UTF8_BOM) wr.writerow(f['id'] for f in fields) yield wr +class CSVWriter(object): + def __init__(self, response, columns, *args, **kwargs): + self._wr = unicodecsv.writer(response, *args, **kwargs) + self.columns = columns + + def writerow(self, row): + return self._wr.writerow([ + _json_dump_nested(val) for val in row]) + + @contextmanager def json_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 JSON data to response @@ -148,6 +167,9 @@ def xml_writer(response, fields, name=None, bom=False): class XMLWriter(object): + _key_attr = 'key' + _value_tag = 'value' + def __init__(self, response, columns): self.response = response self.id_col = columns[0] == u'_id' @@ -155,15 +177,29 @@ def __init__(self, response, columns): columns = columns[1:] self.columns = columns + def _insert_node(self, root, k, v, key_attr=None): + element = SubElement(root, k) + if v is None: + element.attrib[u'xsi:nil'] = u'true' + elif not isinstance(v, (list, dict)): + element.text = unicode(v) + else: + if isinstance(v, list): + it = enumerate(v) + else: + it = v.items() + for key, value in it: + self._insert_node(element, self._value_tag, value, key) + + if key_attr is not None: + element.attrib[self._key_attr] = unicode(key_attr) + def writerow(self, row): root = Element(u'row') if self.id_col: root.attrib[u'_id'] = unicode(row[0]) row = row[1:] for k, v in zip(self.columns, row): - if v is None: - SubElement(root, k).attrib[u'xsi:nil'] = u'true' - continue - SubElement(root, k).text = unicode(v) + self._insert_node(root, k, v) ElementTree(root).write(self.response, encoding=u'utf-8') self.response.write(b'\n') From ea7b59055606dc1d269e6a858d4193755c30fa61 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 16 Mar 2017 18:31:35 +0200 Subject: [PATCH 15/36] unicode literals --- ckanext/datastore/writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/datastore/writer.py b/ckanext/datastore/writer.py index 9883c5e5dca..5cf75e4efeb 100644 --- a/ckanext/datastore/writer.py +++ b/ckanext/datastore/writer.py @@ -167,8 +167,8 @@ def xml_writer(response, fields, name=None, bom=False): class XMLWriter(object): - _key_attr = 'key' - _value_tag = 'value' + _key_attr = u'key' + _value_tag = u'value' def __init__(self, response, columns): self.response = response From bf0f683d48a31e3b82d4cc2fb35c36f4ce3849d7 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Mon, 20 Mar 2017 14:44:28 -0400 Subject: [PATCH 16/36] support chained action in plugins --- ckan/logic/__init__.py | 34 ++++++-- ckan/plugins/interfaces.py | 5 ++ ckan/plugins/toolkit.py | 3 + .../datastore/tests/test_chained_action.py | 81 +++++++++++++++++++ setup.py | 1 + 5 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 ckanext/datastore/tests/test_chained_action.py diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 300429fc35c..f0833f316b1 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -4,6 +4,7 @@ import logging import re import sys +from collections import defaultdict import formencode.validators @@ -312,6 +313,15 @@ def clear_actions_cache(): _actions.clear() +def chained_action(func): + func.chained_action = True + return func + + +def _is_chained_action(func): + return getattr(func, 'chained_action', False) + + def get_action(action): '''Return the named :py:mod:`ckan.logic.action` function. @@ -394,20 +404,32 @@ def get_action(action): # Then overwrite them with any specific ones in the plugins: resolved_action_plugins = {} fetched_actions = {} + chained_actions = defaultdict(list) for plugin in p.PluginImplementations(p.IActions): for name, auth_function in plugin.get_actions().items(): - if name in resolved_action_plugins: + if _is_chained_action(auth_function): + chained_actions[name].append(auth_function) + elif name in resolved_action_plugins: raise NameConflict( 'The action %r is already implemented in %r' % ( name, resolved_action_plugins[name] ) ) - resolved_action_plugins[name] = plugin.name - # Extensions are exempted from the auth audit for now - # This needs to be resolved later - auth_function.auth_audit_exempt = True - fetched_actions[name] = auth_function + else: + resolved_action_plugins[name] = plugin.name + # Extensions are exempted from the auth audit for now + # This needs to be resolved later + auth_function.auth_audit_exempt = True + fetched_actions[name] = auth_function + for name, func_list in chained_actions.iteritems(): + if name not in fetched_actions: + raise NotFound('The action %r is not found for chained action' % ( + name)) + for func in reversed(func_list): + prev_func = fetched_actions[name] + fetched_actions[name] = functools.partial(func, prev_func) + # Use the updated ones in preference to the originals. _actions.update(fetched_actions) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index 83742f639bc..517b3f2e371 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -892,6 +892,11 @@ def get_actions(self): By decorating a function with the `ckan.logic.side_effect_free` decorator, the associated action will be made available by a GET request (as well as the usual POST request) through the action API. + + By decrorating a function with the 'ckan.plugins.toolkit.chained_action, + the action will be chained to another function defined in plugins with a + "first plugin wins" pattern, which means the first plugin declaring a + chained action should be called first. ''' diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index ac475ed5811..2fc2e5fc087 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -44,6 +44,8 @@ class _Toolkit(object): 'literal', # get logic action function 'get_action', + # decorator for chained action + 'chained_action', # get navl schema converter 'get_converter', # get navl schema validator @@ -227,6 +229,7 @@ def _initialize(self): t['literal'] = webhelpers.html.tags.literal t['get_action'] = logic.get_action + t['chained_action'] = logic.chained_action t['get_converter'] = logic.get_validator # For backwards compatibility t['get_validator'] = logic.get_validator t['check_access'] = logic.check_access diff --git a/ckanext/datastore/tests/test_chained_action.py b/ckanext/datastore/tests/test_chained_action.py new file mode 100644 index 00000000000..872fad963d7 --- /dev/null +++ b/ckanext/datastore/tests/test_chained_action.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +import nose + +import ckan.plugins as p +import ckan.tests.helpers as helpers +import ckan.tests.factories as factories + +assert_equals = nose.tools.assert_equals +assert_raises = nose.tools.assert_raises + + +deleted_count = 0 + + +@p.toolkit.chained_action +def datastore_delete(up_func, context, data_dict): + global deleted_count + res = helpers.call_action(u"datastore_search", + resource_id=data_dict[u'resource_id'], + filters=data_dict[u'filters'], + limit=10,) + result = up_func(context, data_dict) + deleted_count = res.get(u'total', 0) + + return result + + +class ChainedDataStorePlugin(p.SingletonPlugin): + p.implements(p.IActions) + + def get_actions(self): + return ({u'datastore_delete': datastore_delete}) + + +class TestChainedAction(object): + @classmethod + def setup_class(cls): + p.load(u'datastore') + p.load(u'chained_datastore_plugin') + + @classmethod + def teardown_class(cls): + p.unload(u'chained_datastore_plugin') + p.unload(u'datastore') + + def setup(self): + helpers.reset_db() + + def test_datastore_delete_filters(self): + records = [ + {u'age': 20}, {u'age': 30}, {u'age': 40} + ] + resource = self._create_datastore_resource(records) + filters = {u'age': 30} + + helpers.call_action(u'datastore_delete', + resource_id=resource[u'id'], + force=True, + filters=filters) + + result = helpers.call_action(u'datastore_search', + resource_id=resource[u'id']) + + new_records_ages = [r[u'age'] for r in result[u'records']] + new_records_ages.sort() + assert_equals(new_records_ages, [20, 40]) + assert_equals(deleted_count, 1) + + def _create_datastore_resource(self, records): + dataset = factories.Dataset() + resource = factories.Resource(package=dataset) + + data = { + u'resource_id': resource[u'id'], + u'force': True, + u'records': records + } + + helpers.call_action(u'datastore_create', **data) + + return resource diff --git a/setup.py b/setup.py index 624bc601901..c79e9e1b72e 100644 --- a/setup.py +++ b/setup.py @@ -163,6 +163,7 @@ 'test_resource_preview = tests.legacy.ckantestplugins:MockResourcePreviewExtension', 'test_json_resource_preview = tests.legacy.ckantestplugins:JsonMockResourcePreviewExtension', 'sample_datastore_plugin = ckanext.datastore.tests.sample_datastore_plugin:SampleDataStorePlugin', + 'chained_datastore_plugin = ckanext.datastore.tests.test_chained_action:ChainedDataStorePlugin', 'test_datastore_view = ckan.tests.lib.test_datapreview:MockDatastoreBasedResourceView', 'test_datapusher_plugin = ckanext.datapusher.tests.test_interfaces:FakeDataPusherPlugin', 'test_routing_plugin = ckan.tests.config.test_middleware:MockRoutingPlugin', From b78c85cddc392e5fcac8295d2f91d9f2d3061e0c Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Mon, 27 Mar 2017 08:19:28 -0400 Subject: [PATCH 17/36] change test name to ExampleDataStoreDeletedWithCountPlugin --- ckanext/datastore/tests/test_chained_action.py | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/tests/test_chained_action.py b/ckanext/datastore/tests/test_chained_action.py index 872fad963d7..caca8181ba0 100644 --- a/ckanext/datastore/tests/test_chained_action.py +++ b/ckanext/datastore/tests/test_chained_action.py @@ -13,7 +13,7 @@ @p.toolkit.chained_action -def datastore_delete(up_func, context, data_dict): +def ExampleDataStoreDeletedWithCountPlugin(up_func, context, data_dict): global deleted_count res = helpers.call_action(u"datastore_search", resource_id=data_dict[u'resource_id'], @@ -36,11 +36,11 @@ class TestChainedAction(object): @classmethod def setup_class(cls): p.load(u'datastore') - p.load(u'chained_datastore_plugin') + p.load(u'example_datastore_deleted_with_count_plugin') @classmethod def teardown_class(cls): - p.unload(u'chained_datastore_plugin') + p.unload(u'example_datastore_deleted_with_count_plugin') p.unload(u'datastore') def setup(self): diff --git a/setup.py b/setup.py index c79e9e1b72e..71bf8702b89 100644 --- a/setup.py +++ b/setup.py @@ -163,7 +163,7 @@ 'test_resource_preview = tests.legacy.ckantestplugins:MockResourcePreviewExtension', 'test_json_resource_preview = tests.legacy.ckantestplugins:JsonMockResourcePreviewExtension', 'sample_datastore_plugin = ckanext.datastore.tests.sample_datastore_plugin:SampleDataStorePlugin', - 'chained_datastore_plugin = ckanext.datastore.tests.test_chained_action:ChainedDataStorePlugin', + 'example_datastore_deleted_with_count_plugin = ckanext.datastore.tests.test_chained_action:ExampleDataStoreDeletedWithCountPlugin', 'test_datastore_view = ckan.tests.lib.test_datapreview:MockDatastoreBasedResourceView', 'test_datapusher_plugin = ckanext.datapusher.tests.test_interfaces:FakeDataPusherPlugin', 'test_routing_plugin = ckan.tests.config.test_middleware:MockRoutingPlugin', From 225003f339e1fed7e1514a5572d1b4ee33d49af1 Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Mon, 27 Mar 2017 15:23:34 +0300 Subject: [PATCH 18/36] Use def user_add for paster user add and sysadmin add --- ckan/lib/cli.py | 100 +++++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index c833029d414..93394b411e3 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -85,6 +85,54 @@ def parse_db_config(config_key='sqlalchemy.url'): return db_details +def user_add(args): + '''Add new user if we use paster sysadmin add + or paster user add + ''' + if len(args) < 2: + error('Need name and email of the user.') + username = args[0] + + # parse args into data_dict + data_dict = {'name': username} + for arg in args[1:]: + try: + field, value = arg.split('=', 1) + data_dict[field] = value + except ValueError: + raise ValueError( + 'Could not parse arg: %r (expected "