Merge branch 'master' into remove-vdm

LICENSE.txt

@@ -2,7 +2,7 @@ License
 +++++++
 
 CKAN - Data Catalogue Software
-Copyright (c) 2006-2018 Open Knowledge International and contributors
+Copyright (c) 2006-2018 Open Knowledge Foundation and contributors
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as

README.rst

@@ -94,7 +94,7 @@ ckan-dev mailing list or on Gitter.
 Copying and License
 -------------------
 
-This material is copyright (c) 2006-2018 Open Knowledge International and contributors.
+This material is copyright (c) 2006-2018 Open Knowledge Foundation and contributors.
 
 It is open and licensed under the GNU Affero General Public License (AGPL) v3.0
 whose full text may be found at:

ckan/cli/asset.py

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+import logging
+
+import click
+from webassets import script
+from webassets.exceptions import BundleError
+
+from ckan.lib import webassets_tools
+from ckan.cli import error_shout
+
+log = logging.getLogger(__name__)
+
+
+@click.group(name=u'asset', short_help=u'WebAssets commands')
+def asset():
+    pass
+
+
+@asset.command(u'build', short_help=u'Builds all bundles.')
+def build():
+    u'''Builds bundles, regardless of whether they are changed or not.'''
+    script.main(['build'], webassets_tools.env)
+    click.secho(u'Compile assets: SUCCESS', fg=u'green', bold=True)
+
+
+@asset.command(u'watch', short_help=u'Watch changes in source files.')
+def watch():
+    u'''Start a daemon which monitors source files, and rebuilds bundles.
+
+    This can be useful during development, if building is not
+    instantaneous, and you are losing valuable time waiting for the
+    build to finish while trying to access your site.
+
+    '''
+    script.main(['watch'], webassets_tools.env)
+
+
+@asset.command(u'clean', short_help=u'Clear cache.')
+def clean():
+    u'''Will clear out the cache, which after a while can grow quite large.'''
+    try:
+        script.main(['clean'], webassets_tools.env)
+    except BundleError as e:
+        return error_shout(e)
+    click.secho(u'Clear cache: SUCCESS', fg=u'green', bold=True)

ckan/cli/cli.py

@@ -5,8 +5,12 @@
 import click
 from ckan.cli import config_tool
 from ckan.cli import (
+    datapusher,
     click_config_option, db, load_config, search_index, server,
+    asset,
+    datastore,
     translation,
+    dataset,
 )
 
 from ckan.config.middleware import make_app
@@ -34,5 +38,9 @@ def ckan(ctx, config, *args, **kwargs):
 ckan.add_command(server.run)
 ckan.add_command(seed.seed)
 ckan.add_command(db.db)
+ckan.add_command(datapusher.datapusher)
 ckan.add_command(search_index.search_index)
+ckan.add_command(asset.asset)
+ckan.add_command(datastore.datastore)
 ckan.add_command(translation.translation)
+ckan.add_command(dataset.dataset)

ckan/cli/datapusher.py

@@ -0,0 +1,101 @@
+# encoding: utf-8
+
+from __future__ import print_function
+
+import logging
+
+import click
+
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+import ckanext.datastore.backend as datastore_backend
+from ckan.cli import error_shout
+
+log = logging.getLogger(__name__)
+
+question = (
+    u"Data in any datastore resource that isn't in their source files "
+    u"(e.g. data added using the datastore API) will be permanently "
+    u"lost. Are you sure you want to proceed?"
+)
+requires_confirmation = click.option(
+    u'--yes', u'-y', is_flag=True, help=u'Always answer yes to questions'
+)
+
+
+def confirm(yes):
+    if yes:
+        return
+    click.confirm(question, abort=True)
+
+
+@click.group()
+def datapusher():
+    u'''Perform commands in the datapusher.
+    '''
+
+
+@datapusher.command()
+@requires_confirmation
+def resubmit(yes):
+    u'''Resubmit udated datastore resources.
+    '''
+    confirm(yes)
+
+    resource_ids = datastore_backend.get_all_resources_ids_in_datastore()
+    _submit(resource_ids)
+
+
+@datapusher.command()
+@click.argument(u'package', required=False)
+@requires_confirmation
+def submit(package, yes):
+    u'''Submits resources from package.
+
+    If no package ID/name specified, submits all resources from all
+    packages.
+    '''
+    confirm(yes)
+
+    if not package:
+        ids = tk.get_action(u'package_list')({
+            u'model': model,
+            u'ignore_auth': True
+        }, {})
+    else:
+        ids = [package]
+
+    for id in ids:
+        package_show = tk.get_action(u'package_show')
+        try:
+            pkg = package_show({
+                u'model': model,
+                u'ignore_auth': True
+            }, {u'id': id})
+        except Exception as e:
+            error_shout(e)
+            error_shout(u"Package '{}' was not found".format(package))
+            raise click.Abort()
+        if not pkg[u'resources']:
+            continue
+        resource_ids = [r[u'id'] for r in pkg[u'resources']]
+        _submit(resource_ids)
+
+
+def _submit(resources):
+    click.echo(u'Submitting {} datastore resources'.format(len(resources)))
+    user = tk.get_action(u'get_site_user')({
+        u'model': model,
+        u'ignore_auth': True
+    }, {})
+    datapusher_submit = tk.get_action(u'datapusher_submit')
+    for id in resources:
+        click.echo(u'Submitting {}...'.format(id), nl=False)
+        data_dict = {
+            u'resource_id': id,
+            u'ignore_hash': True,
+        }
+        if datapusher_submit({u'user': user[u'name']}, data_dict):
+            click.echo(u'OK')
+        else:
+            click.echo(u'Fail')

ckan/cli/dataset.py

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+import logging
+import pprint
+
+import click
+from six import text_type
+
+import ckan.logic as logic
+import ckan.model as model
+
+log = logging.getLogger(__name__)
+
+
+@click.group()
+def dataset():
+    u'''Manage datasets
+    '''
+
+
+@dataset.command()
+@click.argument(u'package')
+def show(package):
+    u'''Shows dataset properties.
+    '''
+    dataset = _get_dataset(package)
+    click.echo(pprint.pformat(dataset.as_dict()))
+
+
+@dataset.command()
+def list():
+    u'''Lists datasets.
+    '''
+    click.echo(u'Datasets:')
+    datasets = model.Session.query(model.Package)
+    click.echo(u'count = %i' % datasets.count())
+    for dataset in datasets:
+        state = (
+            u'(%s)' % dataset.state
+        ) if dataset.state != u'active' else u''
+
+        click.echo(
+            u'%s %s %s' %
+            (click.style(dataset.id, bold=True), dataset.name, state)
+        )
+
+
+@dataset.command()
+@click.argument(u'package')
+def delete(package):
+    u'''Changes dataset state to 'deleted'.
+    '''
+    dataset = _get_dataset(package)
+    old_state = dataset.state
+
+    model.repo.new_revision()
+    dataset.delete()
+    model.repo.commit_and_remove()
+    dataset = _get_dataset(package)
+    click.echo(
+        u'%s %s -> %s' % (
+            dataset.name, click.style(old_state, fg=u'red'),
+            click.style(dataset.state, fg=u'green')
+        )
+    )
+
+
+@dataset.command()
+@click.argument(u'package')
+def purge(package):
+    u'''Removes dataset from db entirely.
+    '''
+    dataset = _get_dataset(package)
+    name = dataset.name
+
+    site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {})
+    context = {u'user': site_user[u'name']}
+    logic.get_action(u'dataset_purge')(context, {u'id': package})
+    click.echo(u'%s purged' % name)
+
+
+def _get_dataset(package):
+    dataset = model.Package.get(text_type(package))
+    assert dataset, u'Could not find dataset matching reference: {}'.format(
+        package
+    )
+    return dataset