conflict with master

.gitignore

@@ -23,6 +23,7 @@ tmp/*
 solr_runtime/*
 fl_notes.txt
 *.ini
+!ckan/migration/alembic.ini
 .noseids
 *~
 .idea

.travis.yml

@@ -8,7 +8,7 @@ flake8-steps: &flake8-steps
   before_script:
     - flake8 --version
     # stop the build if there are Python syntax errors or undefined names
-    - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ./ckan/include/rjsmin.py
+    - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics --exclude ./ckan/include/rjsmin.py
     # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
     - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
   script:

LICENSE.txt

@@ -2,7 +2,7 @@ License
 +++++++
 
 CKAN - Data Catalogue Software
-Copyright (c) 2006-2018 Open Knowledge International and contributors
+Copyright (c) 2006-2018 Open Knowledge Foundation and contributors
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as

README.rst

@@ -94,7 +94,7 @@ ckan-dev mailing list or on Gitter.
 Copying and License
 -------------------
 
-This material is copyright (c) 2006-2018 Open Knowledge International and contributors.
+This material is copyright (c) 2006-2018 Open Knowledge Foundation and contributors.
 
 It is open and licensed under the GNU Affero General Public License (AGPL) v3.0
 whose full text may be found at:

ckan/cli/cli.py

@@ -5,9 +5,12 @@
 import click
 from ckan.cli import config_tool
 from ckan.cli import (
+    datapusher,
     click_config_option, db, load_config, search_index, server,
     asset,
+    datastore,
     translation,
+    dataset,
 )
 
 from ckan.config.middleware import make_app
@@ -35,6 +38,9 @@ def ckan(ctx, config, *args, **kwargs):
 ckan.add_command(server.run)
 ckan.add_command(seed.seed)
 ckan.add_command(db.db)
+ckan.add_command(datapusher.datapusher)
 ckan.add_command(search_index.search_index)
 ckan.add_command(asset.asset)
+ckan.add_command(datastore.datastore)
 ckan.add_command(translation.translation)
+ckan.add_command(dataset.dataset)

ckan/cli/datapusher.py

@@ -0,0 +1,101 @@
+# encoding: utf-8
+
+from __future__ import print_function
+
+import logging
+
+import click
+
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+import ckanext.datastore.backend as datastore_backend
+from ckan.cli import error_shout
+
+log = logging.getLogger(__name__)
+
+question = (
+    u"Data in any datastore resource that isn't in their source files "
+    u"(e.g. data added using the datastore API) will be permanently "
+    u"lost. Are you sure you want to proceed?"
+)
+requires_confirmation = click.option(
+    u'--yes', u'-y', is_flag=True, help=u'Always answer yes to questions'
+)
+
+
+def confirm(yes):
+    if yes:
+        return
+    click.confirm(question, abort=True)
+
+
+@click.group()
+def datapusher():
+    u'''Perform commands in the datapusher.
+    '''
+
+
+@datapusher.command()
+@requires_confirmation
+def resubmit(yes):
+    u'''Resubmit udated datastore resources.
+    '''
+    confirm(yes)
+
+    resource_ids = datastore_backend.get_all_resources_ids_in_datastore()
+    _submit(resource_ids)
+
+
+@datapusher.command()
+@click.argument(u'package', required=False)
+@requires_confirmation
+def submit(package, yes):
+    u'''Submits resources from package.
+
+    If no package ID/name specified, submits all resources from all
+    packages.
+    '''
+    confirm(yes)
+
+    if not package:
+        ids = tk.get_action(u'package_list')({
+            u'model': model,
+            u'ignore_auth': True
+        }, {})
+    else:
+        ids = [package]
+
+    for id in ids:
+        package_show = tk.get_action(u'package_show')
+        try:
+            pkg = package_show({
+                u'model': model,
+                u'ignore_auth': True
+            }, {u'id': id})
+        except Exception as e:
+            error_shout(e)
+            error_shout(u"Package '{}' was not found".format(package))
+            raise click.Abort()
+        if not pkg[u'resources']:
+            continue
+        resource_ids = [r[u'id'] for r in pkg[u'resources']]
+        _submit(resource_ids)
+
+
+def _submit(resources):
+    click.echo(u'Submitting {} datastore resources'.format(len(resources)))
+    user = tk.get_action(u'get_site_user')({
+        u'model': model,
+        u'ignore_auth': True
+    }, {})
+    datapusher_submit = tk.get_action(u'datapusher_submit')
+    for id in resources:
+        click.echo(u'Submitting {}...'.format(id), nl=False)
+        data_dict = {
+            u'resource_id': id,
+            u'ignore_hash': True,
+        }
+        if datapusher_submit({u'user': user[u'name']}, data_dict):
+            click.echo(u'OK')
+        else:
+            click.echo(u'Fail')

ckan/cli/dataset.py

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+import logging
+import pprint
+
+import click
+from six import text_type
+
+import ckan.logic as logic
+import ckan.model as model
+
+log = logging.getLogger(__name__)
+
+
+@click.group()
+def dataset():
+    u'''Manage datasets
+    '''
+
+
+@dataset.command()
+@click.argument(u'package')
+def show(package):
+    u'''Shows dataset properties.
+    '''
+    dataset = _get_dataset(package)
+    click.echo(pprint.pformat(dataset.as_dict()))
+
+
+@dataset.command()
+def list():
+    u'''Lists datasets.
+    '''
+    click.echo(u'Datasets:')
+    datasets = model.Session.query(model.Package)
+    click.echo(u'count = %i' % datasets.count())
+    for dataset in datasets:
+        state = (
+            u'(%s)' % dataset.state
+        ) if dataset.state != u'active' else u''
+
+        click.echo(
+            u'%s %s %s' %
+            (click.style(dataset.id, bold=True), dataset.name, state)
+        )
+
+
+@dataset.command()
+@click.argument(u'package')
+def delete(package):
+    u'''Changes dataset state to 'deleted'.
+    '''
+    dataset = _get_dataset(package)
+    old_state = dataset.state
+
+    model.repo.new_revision()
+    dataset.delete()
+    model.repo.commit_and_remove()
+    dataset = _get_dataset(package)
+    click.echo(
+        u'%s %s -> %s' % (
+            dataset.name, click.style(old_state, fg=u'red'),
+            click.style(dataset.state, fg=u'green')
+        )
+    )
+
+
+@dataset.command()
+@click.argument(u'package')
+def purge(package):
+    u'''Removes dataset from db entirely.
+    '''
+    dataset = _get_dataset(package)
+    name = dataset.name
+
+    site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {})
+    context = {u'user': site_user[u'name']}
+    logic.get_action(u'dataset_purge')(context, {u'id': package})
+    click.echo(u'%s purged' % name)
+
+
+def _get_dataset(package):
+    dataset = model.Package.get(text_type(package))
+    assert dataset, u'Could not find dataset matching reference: {}'.format(
+        package
+    )
+    return dataset

ckan/cli/datastore.py

@@ -0,0 +1,126 @@
+# encoding: utf-8
+
+import logging
+import os
+import re
+
+import click
+
+from ckan.cli import error_shout
+from ckan.common import config
+
+import ckanext.datastore as datastore_module
+from ckanext.datastore.backend.postgres import identifier
+from ckanext.datastore.controller import DUMP_FORMATS, dump_to
+
+log = logging.getLogger(__name__)
+
+
+@click.group()
+def datastore():
+    u'''Perform commands to set up the datastore.
+    '''
+
+
+@datastore.command(
+    u'set-permissions',
+    short_help=u'Generate SQL for permission configuration.'
+)
+def set_permissions():
+    u'''Emit an SQL script that will set the permissions for the datastore
+    users as configured in your configuration file.'''
+
+    write_url = parse_db_config(u'ckan.datastore.write_url')
+    read_url = parse_db_config(u'ckan.datastore.read_url')
+    db_url = parse_db_config(u'sqlalchemy.url')
+
+    # Basic validation that read and write URLs reference the same database.
+    # This obviously doesn't check they're the same database (the hosts/ports
+    # could be different), but it's better than nothing, I guess.
+
+    if write_url[u'db_name'] != read_url[u'db_name']:
+        click.secho(
+            u'The datastore write_url and read_url must refer to the same '
+            u'database!',
+            fg=u'red',
+            bold=True
+        )
+        raise click.Abort()
+
+    sql = permissions_sql(
+        maindb=db_url[u'db_name'],
+        datastoredb=write_url[u'db_name'],
+        mainuser=db_url[u'db_user'],
+        writeuser=write_url[u'db_user'],
+        readuser=read_url[u'db_user']
+    )
+
+    click.echo(sql)
+
+
+def permissions_sql(maindb, datastoredb, mainuser, writeuser, readuser):
+    template_filename = os.path.join(
+        os.path.dirname(datastore_module.__file__), u'set_permissions.sql'
+    )
+    with open(template_filename) as fp:
+        template = fp.read()
+    return template.format(
+        maindb=identifier(maindb),
+        datastoredb=identifier(datastoredb),
+        mainuser=identifier(mainuser),
+        writeuser=identifier(writeuser),
+        readuser=identifier(readuser)
+    )
+
+
+@datastore.command()
+@click.argument(u'resource-id', nargs=1)
+@click.argument(
+    u'output-file',
+    type=click.File(u'wb'),
+    default=click.get_binary_stream(u'stdout')
+)
+@click.option(u'--format', default=u'csv', type=click.Choice(DUMP_FORMATS))
+@click.option(u'--offset', type=click.IntRange(0, None), default=0)
+@click.option(u'--limit', type=click.IntRange(0))
+@click.option(u'--bom', is_flag=True)  # FIXME: options based on format
+@click.pass_context
+def dump(ctx, resource_id, output_file, format, offset, limit, bom):
+    u'''Dump a datastore resource.
+    '''
+    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    with flask_app.test_request_context():
+        dump_to(
+            resource_id,
+            output_file,
+            fmt=format,
+            offset=offset,
+            limit=limit,
+            options={u'bom': bom},
+            sort=u'_id',
+            search_params={}
+        )
+
+
+def parse_db_config(config_key=u'sqlalchemy.url'):
+    u''' Takes a config key for a database connection url and parses it into
+    a dictionary. Expects a url like:
+
+    'postgres://tester:pass@localhost/ckantest3'
+    '''
+    url = config[config_key]
+    regex = [
+        u'^\\s*(?P<db_type>\\w*)', u'://', u'(?P<db_user>[^:]*)', u':?',
+        u'(?P<db_pass>[^@]*)', u'@', u'(?P<db_host>[^/:]*)', u':?',
+        u'(?P<db_port>[^/]*)', u'/', u'(?P<db_name>[\\w.-]*)'
+    ]
+    db_details_match = re.match(u''.join(regex), url)
+    if not db_details_match:
+        click.secho(
+            u'Could not extract db details from url: %r' % url,
+            fg=u'red',
+            bold=True
+        )
+        raise click.Abort()
+    db_details = db_details_match.groupdict()
+    return db_details