Merge branch 'master' into remove-vdm

.gitignore

@@ -23,6 +23,7 @@ tmp/*
 solr_runtime/*
 fl_notes.txt
 *.ini
+!ckan/migration/alembic.ini
 .noseids
 *~
 .idea

.travis.yml

@@ -8,7 +8,7 @@ flake8-steps: &flake8-steps
   before_script:
     - flake8 --version
     # stop the build if there are Python syntax errors or undefined names
-    - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ./ckan/include/rjsmin.py
+    - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude ./ckan/include/rjsmin.py
     # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
     - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
   script:

CHANGELOG.rst

@@ -10,26 +10,29 @@ Changelog
 v.2.9.0 TBA
 ==================
 
- * This version requires script 'migrate_package_activity.py' to be run
-   *before* CKAN is upgraded to this version (or higher). The idea is you do
-   this special migration while CKAN is running, because the script takes a
-   while to run. It adds in the Activity Stream detail, visible only to admins
-   by default. You will not be able to run ``paster db upgrade`` until
-   'migrate_package_activity.py' is done.
-   Download and run migrate_package_activity.py like this:
+ * When upgrading from previous CKAN versions, the Activity Stream needs a
+   migrate_package_activity.py running for displaying the history of dataset
+   changes. This can be performed while CKAN is running or stopped (whereas the
+   standard `paster db upgrade` migrations need CKAN to be stopped). Ideally it
+   is run before CKAN is upgraded, but it can be run afterwards. If running
+   previous versions or this version of CKAN, download and run
+   migrate_package_activity.py like this:
 
      cd /usr/lib/ckan/default/src/ckan/
      wget https://raw.githubusercontent.com/ckan/ckan/3484_revision_ui_removal2/ckan/migration/migrate_package_activity.py
      wget https://raw.githubusercontent.com/ckan/ckan/3484_revision_ui_removal2/ckan/migration/revision_legacy_code.py
      python migrate_package_activity.py -c /etc/ckan/production.ini
 
-   More info: https://github.com/ckan/ckan/wiki/Migrate-package-activity
+   Future versions of CKAN are likely to need a slightly different procedure.
+   Full info about this migration is found here:
+   https://github.com/ckan/ckan/wiki/Migrate-package-activity
 
  * A full history of dataset changes is now displayed in the Activity Stream to
-   admins, and optionally to the public. Open data CKANs upgrading are
-   encouraged to make this history open to the public, by setting this to
-   production.ini: ``ckan.auth.public_activity_stream_detail = true``
-   (#3972)
+   admins, and optionally to the public. By default this is enabled for new
+   installs, but disabled for sites which upgrade (just in case the history is
+   sensitive). When upgrading, open data CKANs are encouraged to make this
+   history open to the public, by setting this in production.ini:
+   ``ckan.auth.public_activity_stream_detail = true`` (#3972)
 
 Minor changes:
 

ckan/authz.py

@@ -102,13 +102,19 @@ def _build(self):
                     fetched_auth_functions[name] = auth_function
 
         for name, func_list in chained_auth_functions.iteritems():
-            if name not in fetched_auth_functions:
+            if (name not in fetched_auth_functions and
+                    name not in self._functions):
                 raise Exception('The auth %r is not found for chained auth' % (
                     name))
             # create the chain of functions in the correct order
             for func in reversed(func_list):
-                prev_func = fetched_auth_functions[name]
-                fetched_auth_functions[name] = functools.partial(func, prev_func)
+                if name in fetched_auth_functions:
+                    prev_func = fetched_auth_functions[name]
+                else:
+                    # fallback to chaining off the builtin auth function
+                    prev_func = self._functions[name]
+                fetched_auth_functions[name] = (
+                    functools.partial(func, prev_func))
 
         # Use the updated ones in preference to the originals.
         self._functions.update(fetched_auth_functions)
@@ -200,7 +206,7 @@ def is_authorized(action, context, data_dict=None):
             return {
                 'success': False,
                 'msg': 'Action {0} requires an authenticated user'.format(
-                    auth_function.__name__)
+                    action)
             }
 
         return auth_function(context, data_dict)

ckan/cli/cli.py

@@ -3,9 +3,14 @@
 import logging
 
 import click
+from ckan.cli import config_tool
+from ckan.cli import (
+    click_config_option, db, load_config, search_index, server,
+    translation,
+)
 
-from ckan.cli import click_config_option, db, load_config, search_index, server
 from ckan.config.middleware import make_app
+from ckan.cli import seed
 
 log = logging.getLogger(__name__)
 
@@ -25,6 +30,9 @@ def ckan(ctx, config, *args, **kwargs):
     ctx.obj = CkanCommand(config)
 
 
+ckan.add_command(config_tool.config_tool)
 ckan.add_command(server.run)
+ckan.add_command(seed.seed)
 ckan.add_command(db.db)
 ckan.add_command(search_index.search_index)
+ckan.add_command(translation.translation)

ckan/cli/config_tool.py

@@ -0,0 +1,77 @@
+# encoding: utf-8
+
+import logging
+
+import click
+
+from ckan.cli import error_shout
+import ckan.lib.config_tool as ct
+
+log = logging.getLogger(__name__)
+
+
+class ConfigOption(click.ParamType):
+    name = u'config-option'
+
+    def convert(self, value, param, ctx):
+        if u'=' not in value:
+            self.fail(
+                u'An option does not have an equals sign. '
+                u'It should be \'key=value\'. If there are spaces '
+                u'you\'ll need to quote the option.\n'
+            )
+        return value
+
+
+@click.command(
+    name=u'config-tool',
+    short_help=u'Tool for editing options in a CKAN config file.'
+)
+@click.option(
+    u'--section',
+    u'-s',
+    default=u'app:main',
+    help=u'Section of the config file'
+)
+@click.option(
+    u'--edit',
+    u'-e',
+    is_flag=True,
+    help=u'Checks the option already exists in the config file.'
+)
+@click.option(
+    u'--file',
+    u'-f',
+    u'merge_filepath',
+    help=u'Supply an options file to merge in.'
+)
+@click.argument(u'config_filepath', type=click.Path(exists=True))
+@click.argument(u'options', nargs=-1, type=ConfigOption())
+def config_tool(config_filepath, options, section, edit, merge_filepath):
+    u'''Tool for editing options in a CKAN config file
+
+    paster config-tool <default.ini> <key>=<value> [<key>=<value> ...]
+
+    paster config-tool <default.ini> -f <custom_options.ini>
+
+    Examples:
+
+      paster config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC'
+
+      paster config-tool default.ini -s server:main -e port=8080
+
+      paster config-tool default.ini -f custom_options.ini
+    '''
+
+    if merge_filepath:
+        ct.config_edit_using_merge_file(
+            config_filepath, merge_filepath
+        )
+    if not (options or merge_filepath):
+        return error_shout(u'No options provided')
+    try:
+        ct.config_edit_using_option_strings(
+            config_filepath, options, section, edit=edit
+        )
+    except ct.ConfigToolError as e:
+        error_shout(e)

ckan/cli/db.py

@@ -1,7 +1,8 @@
 # encoding: utf-8
 
+import os
 import logging
-
+import ckan.migration as migration_repo
 import click
 
 from ckan.cli import error_shout
@@ -44,8 +45,8 @@ def cleandb():
 
 
 @db.command(u'upgrade', short_help=u'Upgrade the database')
-@click.option(u'-v', u'--version', help=u'Migration version')
-def updatedb(version=None):
+@click.option(u'-v', u'--version', help=u'Migration version', default=u'head')
+def updatedb(version):
     u'''Upgrading the database'''
     try:
         import ckan.model as model
@@ -56,17 +57,49 @@ def updatedb(version=None):
         click.secho(u'Upgrading DB: SUCCESS', fg=u'green', bold=True)
 
 
-@db.command(u'version', short_help=u'Returns current version of data schema')
-def version():
-    u'''Return current version'''
-    log.info(u"Returning current DB version")
+@db.command(u'downgrade', short_help=u'Downgrade the database')
+@click.option(u'-v', u'--version', help=u'Migration version', default=u'base')
+def downgradedb(version):
+    u'''Downgrading the database'''
     try:
-        from ckan.model import Session
-        ver = Session.execute(u'select version from '
-                              u'migrate_version;').fetchall()
-        click.secho(
-            u"Latest data schema version: {0}".format(ver[0][0]),
-            bold=True
-        )
+        import ckan.model as model
+        model.repo.downgrade_db(version)
     except Exception as e:
         error_shout(e)
+    else:
+        click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True)
+
+
+@db.command(u'version', short_help=u'Returns current version of data schema')
+@click.option(u'--hash', is_flag=True)
+def version(hash):
+    u'''Return current version'''
+    log.info(u"Returning current DB version")
+    import ckan.model as model
+    model.repo.setup_migration_version_control()
+    current = model.repo.current_version()
+    if not hash:
+        current = _version_hash_to_ordinal(current)
+    click.secho(
+        u'Current DB version: {}'.format(current),
+        fg=u'green', bold=True
+    )
+
+
+def _version_hash_to_ordinal(version):
+    if u'base' == version:
+        return 0
+    versions_dir = os.path.join(
+        os.path.dirname(migration_repo.__file__), u'versions'
+    )
+    versions = sorted(os.listdir(versions_dir))
+
+    # latest version looks like `123abc (head)`
+    if version.endswith(u'(head)'):
+        return int(versions[-1].split(u'_')[0])
+    for name in versions:
+        if version in name:
+            return int(name.split(u'_')[0])
+    error_shout(u'Version `{}` was not found in {}'.format(
+        version, versions_dir
+    ))

ckan/cli/search_index.py

@@ -93,9 +93,9 @@ def start(ids):
     def chunks(l, n):
         u""" Yield n successive chunks from l."""
         newn = int(len(l) / n)
-        for i in range(0, n-1):
-            yield l[i*newn:i*newn+newn]
-        yield l[n*newn-newn:]
+        for i in range(0, n - 1):
+            yield l[i * newn:i * newn + newn]
+        yield l[n * newn - newn:]
 
     processes = []
     with flask_app.test_request_context():

ckan/cli/seed.py

@@ -0,0 +1,85 @@
+# encoding: utf-8
+
+import logging
+
+import click
+
+from ckan.lib.create_test_data import CreateTestData
+
+log = logging.getLogger(__name__)
+
+
+@click.group(short_help=u'Create test data in the database.')
+def seed():
+    u'''Create test data in the database.
+
+    Tests can also delete the created objects easily with the delete() method.
+    '''
+    pass
+
+
+@seed.command(short_help=u'Annakarenina and warandpeace.')
+@click.pass_context
+def basic(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_basic_test_data()
+
+
+@seed.command(short_help=u'Realistic data to test search.')
+@click.pass_context
+def search(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_search_test_data()
+
+
+@seed.command(short_help=u'Government style data.')
+@click.pass_context
+def gov(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_gov_test_data()
+
+
+@seed.command(short_help=u'Package relationships data.')
+@click.pass_context
+def family(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_family_test_data()
+
+
+@seed.command(short_help=u'Create a user "tester" with api key "tester".')
+@click.pass_context
+def user(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_test_user()
+    click.echo(
+        u'Created user {0} with password {0} and apikey {0}'.format(u'tester')
+    )
+
+
+@seed.command(short_help=u'Test translations of terms.')
+@click.pass_context
+def translations(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_translations_test_data()
+
+
+@seed.command(short_help=u'Some test vocabularies.')
+@click.pass_context
+def vocabs(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_vocabs_test_data()
+
+
+@seed.command(short_help=u'Hierarchy of groups.')
+@click.pass_context
+def hierarchy(ctx):
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        CreateTestData.create_group_hierarchy_test_data()