Merge remote-tracking branch 'origin/master' into pytest

.circleci/config.yml

@@ -42,7 +42,7 @@ jobs:
 
           # Python Dependencies
           pip install -r requirement-setuptools.txt
-          pip install -r requirements.txt
+          pip install -r requirements-py2.txt
           pip install -r dev-requirements.txt
           python setup.py develop
 

Dockerfile

@@ -50,7 +50,7 @@ RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
 ADD . $CKAN_VENV/src/ckan/
 RUN ckan-pip install -U pip && \
     ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \
-    ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt && \
+    ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements-py2.txt && \
     ckan-pip install -e $CKAN_VENV/src/ckan/ && \
     ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
     cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \

bin/travis-install-dependencies

@@ -30,7 +30,7 @@ sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh
 
 export PIP_USE_MIRRORS=true
 pip install -r requirement-setuptools.txt --allow-all-external
-pip install -r requirements.txt --allow-all-external
+pip install -r requirements-py2.txt --allow-all-external
 pip install -r dev-requirements.txt --allow-all-external
 
 python setup.py develop

ckan/cli/cli.py

@@ -5,12 +5,16 @@
 import click
 from ckan.cli import config_tool
 from ckan.cli import (
+    jobs,
     datapusher,
+    front_end_build,
     click_config_option, db, load_config, search_index, server,
+    profile,
     asset,
     datastore,
     translation,
     dataset,
+    views,
     plugin_info,
     notify,
     tracking,
@@ -40,8 +44,11 @@ def ckan(ctx, config, *args, **kwargs):
     ctx.obj = CkanCommand(config)
 
 
+ckan.add_command(jobs.jobs)
 ckan.add_command(config_tool.config_tool)
+ckan.add_command(front_end_build.front_end_build)
 ckan.add_command(server.run)
+ckan.add_command(profile.profile)
 ckan.add_command(seed.seed)
 ckan.add_command(db.db)
 ckan.add_command(datapusher.datapusher)
@@ -50,6 +57,7 @@ def ckan(ctx, config, *args, **kwargs):
 ckan.add_command(datastore.datastore)
 ckan.add_command(translation.translation)
 ckan.add_command(dataset.dataset)
+ckan.add_command(views.views)
 ckan.add_command(plugin_info.plugin_info)
 ckan.add_command(notify.notify)
 ckan.add_command(tracking.tracking)

ckan/cli/front_end_build.py

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+import os
+
+import click
+
+from ckan.cli import minify, less, translation
+import ckan.plugins.toolkit as toolkit
+
+
+@click.group(
+    name=u"front-end-build",
+    short_help=u"Creates and minifies css and JavaScript files.",
+    invoke_without_command=True,
+)
+@click.pass_context
+def front_end_build(ctx):
+    if ctx.invoked_subcommand is None:
+        ctx.invoke(build)
+
+
+@front_end_build.command(short_help=u"Compile css and js.",)
+@click.pass_context
+def build(ctx):
+    ctx.invoke(less.less)
+    ctx.invoke(translation.js)
+
+    # minification
+    public = toolkit.config.get(u"ckan.base_public_folder")
+    root = os.path.join(os.path.dirname(__file__), u"..", public, u"base")
+    root = os.path.abspath(root)
+    ckanext = os.path.join(os.path.dirname(__file__), u"..", u"..", u"ckanext")
+    ckanext = os.path.abspath(ckanext)
+    cmd = ctx.invoke(minify.minify, path=(root, ckanext))

ckan/cli/generate.py

@@ -4,20 +4,29 @@
 import sys
 import click
 from ckan.cli import error_shout
-from cookiecutter.main import cookiecutter
 
 
-@click.group(name=u'generate',
-             short_help=u"Generate empty extension files to expand CKAN.")
+@click.group(
+    name=u'generate',
+    short_help=u"Generate empty extension files to expand CKAN.",
+    invoke_without_command=True,
+)
 def generate():
-    pass
+    try:
+        from cookiecutter.main import cookiecutter
+    except ImportError:
+        error_shout(u"`cookiecutter` library is missing from import path.")
+        error_shout(u"Make sure you have dev-dependencies installed:")
+        error_shout(u"\tpip install -r dev-requirements.txt")
+        raise click.Abort()
 
 
 @generate.command(name=u'extension', short_help=u"Create empty extension.")
 @click.option(u'-o', u'--output-dir', help=u"Location to put the generated "
                                            u"template.",
               default=u'.')
 def extension(output_dir):
+    from cookiecutter.main import cookiecutter
     cur_loc = os.path.dirname(os.path.abspath(__file__))
     os.chdir(cur_loc)
     os.chdir(u'../../contrib/cookiecutter/ckan_extension/')

ckan/cli/jobs.py

@@ -0,0 +1,131 @@
+# encoding: utf-8
+
+import click
+
+import ckan.lib.jobs as bg_jobs
+import ckan.logic as logic
+import ckan.plugins as p
+from ckan.cli import error_shout
+
+
+@click.group(name=u"jobs", short_help=u"Manage background jobs.")
+def jobs():
+    pass
+
+
+@jobs.command(short_help=u"Start a worker.",)
+@click.option(u"--burst", is_flag=True, help=u"Start worker in burst mode.")
+@click.argument(u"queues", nargs=-1)
+def worker(burst, queues):
+    """Start a worker that fetches jobs from queues and executes them. If
+    no queue names are given then the worker listens to the default
+    queue, this is equivalent to
+
+        paster jobs worker default
+
+    If queue names are given then the worker listens to those queues
+    and only those:
+
+        paster jobs worker my-custom-queue
+
+    Hence, if you want the worker to listen to the default queue and
+    some others then you must list the default queue explicitly:
+
+        paster jobs worker default my-custom-queue
+
+    If the `--burst` option is given then the worker will exit as soon
+    as all its queues are empty.
+    """
+    bg_jobs.Worker(queues).work(burst=burst)
+
+
+@jobs.command(name=u"list", short_help=u"List jobs.")
+@click.argument(u"queues", nargs=-1)
+def list_jobs(queues):
+    """List currently enqueued jobs from the given queues. If no queue
+    names are given then the jobs from all queues are listed.
+    """
+    data_dict = {
+        u"queues": list(queues),
+    }
+    jobs = p.toolkit.get_action(u"job_list")({u"ignore_auth": True}, data_dict)
+    if not jobs:
+        return click.secho(u"There are no pending jobs.", fg=u"green")
+    for job in jobs:
+        if job[u"title"] is None:
+            job[u"title"] = u""
+        else:
+            job[u"title"] = u'"{}"'.format(job[u"title"])
+        click.secho(u"{created} {id} {queue} {title}".format(**job))
+
+
+@jobs.command(short_help=u"Show details about a specific job.")
+@click.argument(u"id")
+def show(id):
+    try:
+        job = p.toolkit.get_action(u"job_show")(
+            {u"ignore_auth": True}, {u"id": id}
+        )
+    except logic.NotFound:
+        return error_shout(u'There is no job with ID "{}"'.format(id))
+    click.secho(u"ID:      {}".format(job[u"id"]))
+    if job[u"title"] is None:
+        title = u"None"
+    else:
+        title = u'"{}"'.format(job[u"title"])
+    click.secho(u"Title:   {}".format(title))
+    click.secho(u"Created: {}".format(job[u"created"]))
+    click.secho(u"Queue:   {}".format(job[u"queue"]))
+
+
+@jobs.command(short_help=u"Cancel a specific job.")
+@click.argument(u"id")
+def cancel(id):
+    """Cancel a specific job. Jobs can only be canceled while they are
+    enqueued. Once a worker has started executing a job it cannot be
+    aborted anymore.
+
+    """
+    try:
+        p.toolkit.get_action(u"job_cancel")(
+            {u"ignore_auth": True}, {u"id": id}
+        )
+    except logic.NotFound:
+        return error_shout(u'There is no job with ID "{}"'.format(id))
+
+    click.secho(u"Cancelled job {}".format(id), fg=u"green")
+
+
+@jobs.command(short_help=u"Cancel all jobs.")
+@click.argument(u"queues", nargs=-1)
+def clear(queues):
+    """Cancel all jobs on the given queues. If no queue names are given
+    then ALL queues are cleared.
+
+    """
+    data_dict = {
+        u"queues": list(queues),
+    }
+    queues = p.toolkit.get_action(u"job_clear")(
+        {u"ignore_auth": True}, data_dict
+    )
+    queues = (u'"{}"'.format(q) for q in queues)
+    click.secho(u"Cleared queue(s) {}".format(u", ".join(queues)), fg=u"green")
+
+
+@jobs.command(short_help=u"Enqueue a test job.")
+@click.argument(u"queues", nargs=-1)
+def test(queues):
+    """Enqueue a test job. If no queue names are given then the job is
+    added to the default queue. If queue names are given then a
+    separate test job is added to each of the queues.
+
+    """
+    for queue in queues or [bg_jobs.DEFAULT_QUEUE_NAME]:
+        job = bg_jobs.enqueue(
+            bg_jobs.test_job, [u"A test job"], title=u"A test job", queue=queue
+        )
+        click.secho(
+            u'Added test job {} to queue "{}"'.format(job.id, queue),
+            fg=u"green",
+        )

ckan/cli/profile.py

@@ -0,0 +1,68 @@
+# encoding: utf-8
+
+import re
+import traceback
+
+import click
+
+from ckan.cli import error_shout
+
+
+@click.group(
+    short_help=u"Code speed profiler.", invoke_without_command=True,
+)
+@click.pass_context
+def profile(ctx):
+    """Provide a ckan url and it will make the request and record how
+    long each function call took in a file that can be read by
+    pstats.Stats (command-line) or runsnakerun (gui).
+
+    Usage:
+       profile URL [username]
+
+    e.g. profile /data/search
+
+    The result is saved in profile.data.search
+    To view the profile in runsnakerun:
+       runsnakerun ckan.data.search.profile
+
+    You may need to install python module: cProfile
+
+    """
+    if ctx.invoked_subcommand is None:
+        ctx.invoke(profile)
+
+
+@profile.command(short_help=u"Code speed profiler.",)
+@click.argument(u"url")
+@click.argument(u"user", required=False, default=u"visitor")
+def profile(url, user):
+    import cProfile
+    from ckan.tests.helpers import _get_test_app
+
+    app = _get_test_app()
+
+    def profile_url(url):
+        try:
+            res = app.get(
+                url, status=[200], extra_environ={u"REMOTE_USER": str(user)}
+            )
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            error_shout(traceback.format_exc())
+
+    output_filename = u"ckan%s.profile" % re.sub(
+        u"[/?]", u".", url.replace(u"/", u".")
+    )
+    profile_command = u"profile_url('%s')" % url
+    cProfile.runctx(
+        profile_command, globals(), locals(), filename=output_filename
+    )
+    import pstats
+
+    stats = pstats.Stats(output_filename)
+    stats.sort_stats(u"cumulative")
+    stats.print_stats(0.1)  # show only top 10% of lines
+    click.secho(u"Only top 10% of lines shown")
+    click.secho(u"Written profile to: %s" % output_filename, fg=u"green")