Merge branch 'master' into fix/replace-paster-make-config

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
           CKAN_POSTGRES_PWD: pass
           PGPASSWORD: ckan
           NODE_TESTS_CONTAINER: 2
-          NOSETEST_COMMON_OPTIONS: -v --ckan --reset-db --with-pylons=test-core-circle-ci.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext --with-xunit --xunit-file=/root/junit/junit.xml ckan ckanext
+          PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4  --test-group-random-seed 1
       - image: postgres:10
         environment:
           POSTGRES_USER: ckan
@@ -65,26 +65,18 @@ jobs:
       - run: |
           mkdir -p ~/junit
           case $CIRCLE_NODE_INDEX in
-            0) nosetests $NOSETEST_COMMON_OPTIONS --segments 0123
+            0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1
             ;;
-            1) nosetests $NOSETEST_COMMON_OPTIONS --segments 4567
+            1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2
             ;;
-            2) nosetests $NOSETEST_COMMON_OPTIONS --segments 89ab
+            2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3
             ;;
-            3) nosetests $NOSETEST_COMMON_OPTIONS --segments cdef
+            3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4
             ;;
           esac
       - store_test_results:
           path: ~/junit
 
-      # test_revision_legacy_code is run separately because it mucks up the model for some other tests when nose starts up
-      - run:
-          command: |
-            case $CIRCLE_NODE_INDEX in
-              3) nosetests -v --ckan --with-pylons=test-core-circle-ci.ini --nologcapture test_revision_legacy_code.py
-              ;;
-            esac
-
       # Tests Frontend, only in one container
       - run:
           command: |

ckan/authz.py

@@ -3,7 +3,7 @@
 import functools
 import sys
 
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
 from logging import getLogger
 
 import six
@@ -13,7 +13,7 @@
 
 import ckan.plugins as p
 import ckan.model as model
-from ckan.common import OrderedDict, _, c
+from ckan.common import _, c
 
 import ckan.lib.maintain as maintain
 

ckan/cli/__init__.py

@@ -22,10 +22,13 @@ def error_shout(exception):
 )
 
 
-def load_config(config=None):
+def load_config(ini_path=None):
     from paste.deploy import appconfig
-    if config:
-        filename = os.path.abspath(config)
+
+    if ini_path:
+        if ini_path.startswith(u'~'):
+            ini_path = os.path.expanduser(ini_path)
+        filename = os.path.abspath(ini_path)
         config_source = u'-c parameter'
     elif os.environ.get(u'CKAN_INI'):
         filename = os.environ.get(u'CKAN_INI')

ckan/cli/cli.py

@@ -2,7 +2,9 @@
 
 import logging
 import sys
+from collections import defaultdict
 
+import ckan.plugins as p
 import click
 from ckan.cli import config_tool
 from ckan.cli import (
@@ -32,14 +34,41 @@
 log = logging.getLogger(__name__)
 
 
+class CustomGroup(click.Group):
+    def get_command(self, ctx, name):
+        cmd = super(CustomGroup, self).get_command(ctx, name)
+        if not cmd:
+            ctx.invoke(self)
+            cmd = super(CustomGroup, self).get_command(ctx, name)
+        return cmd
+
+    def format_commands(self, ctx, formatter):
+        super(CustomGroup, self).format_commands(ctx, formatter)
+        ctx.invoke(self)
+
+        ext_commands = defaultdict(list)
+        for subcommand in self.list_commands(ctx):
+            cmd = self.get_command(ctx, subcommand)
+            if cmd is None or not hasattr(cmd, u'_ckanext'):
+                continue
+
+            help = cmd.short_help or u''
+            ext_commands[cmd._ckanext].append((subcommand, help))
+        if ext_commands:
+            with formatter.section(u'Plugins'):
+                for ext, rows in ext_commands.items():
+                    with formatter.section(ext):
+                        formatter.write_dl(rows)
+
+
 class CkanCommand(object):
 
     def __init__(self, conf=None):
         self.config = load_config(conf)
         self.app = make_app(self.config.global_conf, **self.config.local_conf)
 
 
-@click.group()
+@click.group(cls=CustomGroup)
 @click.help_option(u'-h', u'--help')
 @click_config_option
 @click.pass_context
@@ -49,6 +78,10 @@ def ckan(ctx, config, *args, **kwargs):
     if all(arg in sys.argv for arg in ['generate', 'config']):
         return
     ctx.obj = CkanCommand(config)
+    for plugin in p.PluginImplementations(p.IClick):
+        for cmd in plugin.get_commands():
+            cmd._ckanext = plugin.name
+            ckan.add_command(cmd)
 
 
 ckan.add_command(jobs.jobs)

ckan/cli/dataset.py

@@ -53,7 +53,6 @@ def delete(package):
     dataset = _get_dataset(package)
     old_state = dataset.state
 
-    model.repo.new_revision()
     dataset.delete()
     model.repo.commit_and_remove()
     dataset = _get_dataset(package)

ckan/cli/datastore.py

@@ -6,7 +6,7 @@
 
 import click
 
-from ckan.cli import error_shout
+from ckan.model import parse_db_config
 from ckan.common import config
 
 import ckanext.datastore as datastore_module
@@ -30,9 +30,9 @@ def set_permissions():
     u'''Emit an SQL script that will set the permissions for the datastore
     users as configured in your configuration file.'''
 
-    write_url = parse_db_config(u'ckan.datastore.write_url')
-    read_url = parse_db_config(u'ckan.datastore.read_url')
-    db_url = parse_db_config(u'sqlalchemy.url')
+    write_url = _parse_db_config(u'ckan.datastore.write_url')
+    read_url = _parse_db_config(u'ckan.datastore.read_url')
+    db_url = _parse_db_config(u'sqlalchemy.url')
 
     # Basic validation that read and write URLs reference the same database.
     # This obviously doesn't check they're the same database (the hosts/ports
@@ -102,25 +102,13 @@ def dump(ctx, resource_id, output_file, format, offset, limit, bom):
         )
 
 
-def parse_db_config(config_key=u'sqlalchemy.url'):
-    u''' Takes a config key for a database connection url and parses it into
-    a dictionary. Expects a url like:
-
-    'postgres://tester:pass@localhost/ckantest3'
-    '''
-    url = config[config_key]
-    regex = [
-        u'^\\s*(?P<db_type>\\w*)', u'://', u'(?P<db_user>[^:]*)', u':?',
-        u'(?P<db_pass>[^@]*)', u'@', u'(?P<db_host>[^/:]*)', u':?',
-        u'(?P<db_port>[^/]*)', u'/', u'(?P<db_name>[\\w.-]*)'
-    ]
-    db_details_match = re.match(u''.join(regex), url)
-    if not db_details_match:
+def _parse_db_config(config_key=u'sqlalchemy.url'):
+    db_config = parse_db_config(config_key)
+    if not db_config:
         click.secho(
-            u'Could not extract db details from url: %r' % url,
+            u'Could not extract db details from url: %r' % config[config_key],
             fg=u'red',
             bold=True
         )
         raise click.Abort()
-    db_details = db_details_match.groupdict()
-    return db_details
+    return db_config

ckan/common.py

@@ -28,11 +28,6 @@
 
 current_app = flask.current_app
 
-try:
-    from collections import OrderedDict  # from python 2.7
-except ImportError:
-    from sqlalchemy.util import OrderedDict
-
 
 def is_flask_request():
     u'''
@@ -71,10 +66,7 @@ def streaming_response(
 
 
 def ugettext(*args, **kwargs):
-    if is_flask_request():
-        return flask_ugettext(*args, **kwargs)
-    else:
-        return pylons_ugettext(*args, **kwargs)
+    return flask_ugettext(*args, **kwargs)
 
 
 _ = ugettext
@@ -124,34 +116,40 @@ def clear(self):
             flask.current_app.config.clear()
         except RuntimeError:
             pass
-        try:
-            pylons.config.clear()
-            # Pylons set this default itself
-            pylons.config[u'lang'] = None
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                pylons.config.clear()
+                # Pylons set this default itself
+                pylons.config[u'lang'] = None
+            except TypeError:
+                pass
 
     def __setitem__(self, key, value):
         self.store[key] = value
         try:
             flask.current_app.config[key] = value
         except RuntimeError:
             pass
-        try:
-            pylons.config[key] = value
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                pylons.config[key] = value
+            except TypeError:
+                pass
 
     def __delitem__(self, key):
         del self.store[key]
         try:
             del flask.current_app.config[key]
         except RuntimeError:
             pass
-        try:
-            del pylons.config[key]
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                del pylons.config[key]
+            except TypeError:
+                pass
 
 
 def _get_request():