Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into webassets
- Loading branch information
Showing
749 changed files
with
5,742 additions
and
129,890 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,6 +23,7 @@ tmp/* | |
solr_runtime/* | ||
fl_notes.txt | ||
*.ini | ||
!ckan/migration/alembic.ini | ||
.noseids | ||
*~ | ||
.idea | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
# encoding: utf-8 | ||
|
||
from __future__ import print_function | ||
|
||
import logging | ||
|
||
import click | ||
|
||
import ckan.model as model | ||
import ckan.plugins.toolkit as tk | ||
import ckanext.datastore.backend as datastore_backend | ||
from ckan.cli import error_shout | ||
|
||
log = logging.getLogger(__name__) | ||
|
||
question = ( | ||
u"Data in any datastore resource that isn't in their source files " | ||
u"(e.g. data added using the datastore API) will be permanently " | ||
u"lost. Are you sure you want to proceed?" | ||
) | ||
requires_confirmation = click.option( | ||
u'--yes', u'-y', is_flag=True, help=u'Always answer yes to questions' | ||
) | ||
|
||
|
||
def confirm(yes): | ||
if yes: | ||
return | ||
click.confirm(question, abort=True) | ||
|
||
|
||
@click.group() | ||
def datapusher(): | ||
u'''Perform commands in the datapusher. | ||
''' | ||
|
||
|
||
@datapusher.command() | ||
@requires_confirmation | ||
def resubmit(yes): | ||
u'''Resubmit udated datastore resources. | ||
''' | ||
confirm(yes) | ||
|
||
resource_ids = datastore_backend.get_all_resources_ids_in_datastore() | ||
_submit(resource_ids) | ||
|
||
|
||
@datapusher.command() | ||
@click.argument(u'package', required=False) | ||
@requires_confirmation | ||
def submit(package, yes): | ||
u'''Submits resources from package. | ||
If no package ID/name specified, submits all resources from all | ||
packages. | ||
''' | ||
confirm(yes) | ||
|
||
if not package: | ||
ids = tk.get_action(u'package_list')({ | ||
u'model': model, | ||
u'ignore_auth': True | ||
}, {}) | ||
else: | ||
ids = [package] | ||
|
||
for id in ids: | ||
package_show = tk.get_action(u'package_show') | ||
try: | ||
pkg = package_show({ | ||
u'model': model, | ||
u'ignore_auth': True | ||
}, {u'id': id}) | ||
except Exception as e: | ||
error_shout(e) | ||
error_shout(u"Package '{}' was not found".format(package)) | ||
raise click.Abort() | ||
if not pkg[u'resources']: | ||
continue | ||
resource_ids = [r[u'id'] for r in pkg[u'resources']] | ||
_submit(resource_ids) | ||
|
||
|
||
def _submit(resources): | ||
click.echo(u'Submitting {} datastore resources'.format(len(resources))) | ||
user = tk.get_action(u'get_site_user')({ | ||
u'model': model, | ||
u'ignore_auth': True | ||
}, {}) | ||
datapusher_submit = tk.get_action(u'datapusher_submit') | ||
for id in resources: | ||
click.echo(u'Submitting {}...'.format(id), nl=False) | ||
data_dict = { | ||
u'resource_id': id, | ||
u'ignore_hash': True, | ||
} | ||
if datapusher_submit({u'user': user[u'name']}, data_dict): | ||
click.echo(u'OK') | ||
else: | ||
click.echo(u'Fail') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
# encoding: utf-8 | ||
|
||
import logging | ||
import pprint | ||
|
||
import click | ||
from six import text_type | ||
|
||
import ckan.logic as logic | ||
import ckan.model as model | ||
|
||
log = logging.getLogger(__name__) | ||
|
||
|
||
@click.group() | ||
def dataset(): | ||
u'''Manage datasets | ||
''' | ||
|
||
|
||
@dataset.command() | ||
@click.argument(u'package') | ||
def show(package): | ||
u'''Shows dataset properties. | ||
''' | ||
dataset = _get_dataset(package) | ||
click.echo(pprint.pformat(dataset.as_dict())) | ||
|
||
|
||
@dataset.command() | ||
def list(): | ||
u'''Lists datasets. | ||
''' | ||
click.echo(u'Datasets:') | ||
datasets = model.Session.query(model.Package) | ||
click.echo(u'count = %i' % datasets.count()) | ||
for dataset in datasets: | ||
state = ( | ||
u'(%s)' % dataset.state | ||
) if dataset.state != u'active' else u'' | ||
|
||
click.echo( | ||
u'%s %s %s' % | ||
(click.style(dataset.id, bold=True), dataset.name, state) | ||
) | ||
|
||
|
||
@dataset.command() | ||
@click.argument(u'package') | ||
def delete(package): | ||
u'''Changes dataset state to 'deleted'. | ||
''' | ||
dataset = _get_dataset(package) | ||
old_state = dataset.state | ||
|
||
model.repo.new_revision() | ||
dataset.delete() | ||
model.repo.commit_and_remove() | ||
dataset = _get_dataset(package) | ||
click.echo( | ||
u'%s %s -> %s' % ( | ||
dataset.name, click.style(old_state, fg=u'red'), | ||
click.style(dataset.state, fg=u'green') | ||
) | ||
) | ||
|
||
|
||
@dataset.command() | ||
@click.argument(u'package') | ||
def purge(package): | ||
u'''Removes dataset from db entirely. | ||
''' | ||
dataset = _get_dataset(package) | ||
name = dataset.name | ||
|
||
site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {}) | ||
context = {u'user': site_user[u'name']} | ||
logic.get_action(u'dataset_purge')(context, {u'id': package}) | ||
click.echo(u'%s purged' % name) | ||
|
||
|
||
def _get_dataset(package): | ||
dataset = model.Package.get(text_type(package)) | ||
assert dataset, u'Could not find dataset matching reference: {}'.format( | ||
package | ||
) | ||
return dataset |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,126 @@ | ||
# encoding: utf-8 | ||
|
||
import logging | ||
import os | ||
import re | ||
|
||
import click | ||
|
||
from ckan.cli import error_shout | ||
from ckan.common import config | ||
|
||
import ckanext.datastore as datastore_module | ||
from ckanext.datastore.backend.postgres import identifier | ||
from ckanext.datastore.controller import DUMP_FORMATS, dump_to | ||
|
||
log = logging.getLogger(__name__) | ||
|
||
|
||
@click.group() | ||
def datastore(): | ||
u'''Perform commands to set up the datastore. | ||
''' | ||
|
||
|
||
@datastore.command( | ||
u'set-permissions', | ||
short_help=u'Generate SQL for permission configuration.' | ||
) | ||
def set_permissions(): | ||
u'''Emit an SQL script that will set the permissions for the datastore | ||
users as configured in your configuration file.''' | ||
|
||
write_url = parse_db_config(u'ckan.datastore.write_url') | ||
read_url = parse_db_config(u'ckan.datastore.read_url') | ||
db_url = parse_db_config(u'sqlalchemy.url') | ||
|
||
# Basic validation that read and write URLs reference the same database. | ||
# This obviously doesn't check they're the same database (the hosts/ports | ||
# could be different), but it's better than nothing, I guess. | ||
|
||
if write_url[u'db_name'] != read_url[u'db_name']: | ||
click.secho( | ||
u'The datastore write_url and read_url must refer to the same ' | ||
u'database!', | ||
fg=u'red', | ||
bold=True | ||
) | ||
raise click.Abort() | ||
|
||
sql = permissions_sql( | ||
maindb=db_url[u'db_name'], | ||
datastoredb=write_url[u'db_name'], | ||
mainuser=db_url[u'db_user'], | ||
writeuser=write_url[u'db_user'], | ||
readuser=read_url[u'db_user'] | ||
) | ||
|
||
click.echo(sql) | ||
|
||
|
||
def permissions_sql(maindb, datastoredb, mainuser, writeuser, readuser): | ||
template_filename = os.path.join( | ||
os.path.dirname(datastore_module.__file__), u'set_permissions.sql' | ||
) | ||
with open(template_filename) as fp: | ||
template = fp.read() | ||
return template.format( | ||
maindb=identifier(maindb), | ||
datastoredb=identifier(datastoredb), | ||
mainuser=identifier(mainuser), | ||
writeuser=identifier(writeuser), | ||
readuser=identifier(readuser) | ||
) | ||
|
||
|
||
@datastore.command() | ||
@click.argument(u'resource-id', nargs=1) | ||
@click.argument( | ||
u'output-file', | ||
type=click.File(u'wb'), | ||
default=click.get_binary_stream(u'stdout') | ||
) | ||
@click.option(u'--format', default=u'csv', type=click.Choice(DUMP_FORMATS)) | ||
@click.option(u'--offset', type=click.IntRange(0, None), default=0) | ||
@click.option(u'--limit', type=click.IntRange(0)) | ||
@click.option(u'--bom', is_flag=True) # FIXME: options based on format | ||
@click.pass_context | ||
def dump(ctx, resource_id, output_file, format, offset, limit, bom): | ||
u'''Dump a datastore resource. | ||
''' | ||
flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app | ||
with flask_app.test_request_context(): | ||
dump_to( | ||
resource_id, | ||
output_file, | ||
fmt=format, | ||
offset=offset, | ||
limit=limit, | ||
options={u'bom': bom}, | ||
sort=u'_id', | ||
search_params={} | ||
) | ||
|
||
|
||
def parse_db_config(config_key=u'sqlalchemy.url'): | ||
u''' Takes a config key for a database connection url and parses it into | ||
a dictionary. Expects a url like: | ||
'postgres://tester:pass@localhost/ckantest3' | ||
''' | ||
url = config[config_key] | ||
regex = [ | ||
u'^\\s*(?P<db_type>\\w*)', u'://', u'(?P<db_user>[^:]*)', u':?', | ||
u'(?P<db_pass>[^@]*)', u'@', u'(?P<db_host>[^/:]*)', u':?', | ||
u'(?P<db_port>[^/]*)', u'/', u'(?P<db_name>[\\w.-]*)' | ||
] | ||
db_details_match = re.match(u''.join(regex), url) | ||
if not db_details_match: | ||
click.secho( | ||
u'Could not extract db details from url: %r' % url, | ||
fg=u'red', | ||
bold=True | ||
) | ||
raise click.Abort() | ||
db_details = db_details_match.groupdict() | ||
return db_details |
Oops, something went wrong.