Permalink
Browse files

First crack at containers documentation and related usability improve…

…ments.

- Implement ``planemo lint --biocontainer`` to mirror ``planemo lint --conda_requirements``. Just checks that a tool is ready to go with best practice namespaces for BioContainers tools.
- Implement test case for new linting flag.
- Add new section of documentation to advanced tool topics for containers. Work through examples of both an existing working container (seqtk) and one that must be built on the fly. Demonstrate ``planemo mull``.
- Hack around a bug in Conda 4.2 that makes it so ``planemo mull`` doesn't work out of the box on Mac OS X.
- Add more options and more documentation to the ``planemo mull`` command.
  • Loading branch information...
jmchilton committed Feb 27, 2017
1 parent 9af67bf commit 0a1abfea53523f746ad48c8a4ba1df63da3f7765
@@ -1,3 +1,5 @@
.. _dependencies_and_conda:

Dependencies and Conda
===========================================

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -86,7 +86,7 @@

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ['_build', '_*']

# The reST default role (used for this markup: `text`) to use for all
# documents.
@@ -68,6 +68,14 @@ planemo.commands.cmd_clone module
:undoc-members:
:show-inheritance:

planemo.commands.cmd_conda_build module
---------------------------------------

.. automodule:: planemo.commands.cmd_conda_build
:members:
:undoc-members:
:show-inheritance:

planemo.commands.cmd_conda_env module
-------------------------------------

@@ -4,6 +4,14 @@ planemo.linters package
Submodules
----------

planemo.linters.biocontainer_registered module
----------------------------------------------

.. automodule:: planemo.linters.biocontainer_registered
:members:
:undoc-members:
:show-inheritance:

planemo.linters.conda_requirements module
-----------------------------------------

@@ -14,6 +14,7 @@ available - check out tutorial_ if you have never developed a Galaxy tool.
.. include:: _writing_tool_metadata.rst
.. include:: _writing_clusters.rst
.. include:: _writing_dependencies_conda.rst
.. include:: _writing_dependencies_docker.rst

.. _Galaxy: http://galaxyproject.org/
.. _Docker: https://www.docker.com/
@@ -64,6 +64,7 @@ def cli(ctx, path, **kwds):
if container.type == "docker":
identifier = container.identifier

# Refactor mulled container resolver to be able to do this.
if kwds["from_tag"]:
identifier = "-t %s" % identifier

@@ -33,6 +33,12 @@
default=False,
help="Check tool requirements for availability in best practice Conda channels.",
)
@click.option(
"--biocontainer",
is_flag=True,
default=False,
help="Check best practice BioContainer namespaces for a container definition applicable for this tool.",
)
# @click.option(
# "--verify",
# is_flag=True,
@@ -13,6 +13,8 @@
@click.command('mull')
@options.optional_tools_arg(multiple=True)
@options.recursive_option()
@options.mulled_options()
@options.conda_ensure_channels_option()
@command_function
def cli(ctx, paths, **kwds):
"""Build containers for specified tools.
@@ -30,4 +32,5 @@ def cli(ctx, paths, **kwds):
for conda_targets in collect_conda_target_lists(ctx, paths):
mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
mull_target_kwds = build_mull_target_kwds(ctx, **kwds)
mull_targets(mulled_targets, command="build", **mull_target_kwds)
command = kwds["mulled_command"]
mull_targets(mulled_targets, command=command, **mull_target_kwds)
@@ -15,6 +15,7 @@
urlopen,
)

import planemo.linters.biocontainer_registered
import planemo.linters.conda_requirements
import planemo.linters.doi
import planemo.linters.urls
@@ -60,6 +61,9 @@ def _lint_extra_modules(**kwds):
if kwds.get("conda_requirements", False):
linters.append(planemo.linters.conda_requirements)

if kwds.get("biocontainer", False):
linters.append(planemo.linters.biocontainer_registered)

return linters


@@ -0,0 +1,60 @@
"""Ensure best-practice biocontainer registered for this tool."""

from galaxy.tools.deps.mulled.util import (
build_target,
image_name,
mulled_tags_for,
split_tag,
)

from planemo.conda import tool_source_conda_targets

MESSAGE_WARN_NO_REQUIREMENTS = "No valid package requirement tags found to infer BioContainer from."
MESSAGE_WARN_NO_CONTAINER = "Failed to find a BioContainer registered for these requirements."
MESSAGE_INFO_FOUND_BIOCONTAINER = "BioContainer best-practice container found [%s]."


def lint_biocontainer_registered(tool_source, lint_ctx):
conda_targets = tool_source_conda_targets(tool_source)
if not conda_targets:
lint_ctx.warn(MESSAGE_WARN_NO_REQUIREMENTS)
return

mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
name = mulled_container_name("biocontainers", mulled_targets)
if name:
lint_ctx.info(MESSAGE_INFO_FOUND_BIOCONTAINER % name)
else:
lint_ctx.warn(MESSAGE_WARN_NO_CONTAINER)


# TODO: Refactor following method into mulled util and then refactor same code out of mulled
# container resolver.
def mulled_container_name(namespace, targets):
name = None

if len(targets) == 1:
target = targets[0]
target_version = target.version
tags = mulled_tags_for(namespace, target.package_name)

if not tags:
return None

if target_version:
for tag in tags:
version, build = split_tag(tag)
if version == target_version:
name = "%s:%s--%s" % (target.package_name, version, build)
break
else:
version, build = split_tag(tags[0])
name = "%s:%s--%s" % (target.package_name, version, build)
else:
base_image_name = image_name(targets)
tags = mulled_tags_for(namespace, base_image_name)
if tags:
name = "%s:%s" % (base_image_name, tags[0])

if name:
return "quay.io/%s/%s" % (namespace, name)
@@ -12,7 +12,7 @@
InvolucroContext,
)

from planemo.io import shell
from planemo.io import IS_OS_X, shell


def build_involucro_context(ctx, **kwds):
@@ -35,13 +35,26 @@ def build_mull_target_kwds(ctx, **kwds):
"""Adapt Planemo's CLI and workspace configuration to galaxy-lib's mulled_build options."""
involucro_context = build_involucro_context(ctx, **kwds)
channels = kwds.get("conda_ensure_channels", ",".join(DEFAULT_CHANNELS))

return {
namespace = kwds.get("mulled_namespace", "biocontainers")
target_kwds = {
'involucro_context': involucro_context,
'channels': channels.split(","),
'namespace': namespace,
'verbose': ctx.verbose,
}

conda_version = kwds.get("mulled_conda_version", None)
if conda_version is not None:
target_kwds["conda_version"] = conda_version
else:
# Hack to workaround a bug with osxfs + Conda 4.2 - remove
# when container gets upgraded to 4.3
if IS_OS_X:
target_kwds["conda_version"] = "4.3"
return target_kwds


__all__ = (
"build_involucro_context",
"build_mull_target_kwds",
)
@@ -1538,3 +1538,44 @@ def tool_init_example_command_option(help=EXAMPLE_COMMAND_HELP):
prompt=False,
help=help,
)


def mulled_conda_option():
return planemo_option(
"--mulled_conda_version",
type=click.STRING,
default=None,
help=("Install a specific version of Conda before running the command, by "
"default the version that comes with the continuumio miniconda3 image "
"will be used under Linux and under Mac OS X Conda will be upgraded to "
"to work around a bug in 4.2.")
)


def mulled_namespace_option():
return planemo_option(
"--mulled_namespace",
type=click.STRING,
default="biocontainers",
help=("Build a mulled image with the specified namespace - defaults to "
"biocontainers. Galaxy currently only recognizes images with the "
"namespace biocontainers.")
)


def mulled_action_option():
return planemo_option(
"--mulled_command",
type=click.STRING,
default="build",
help=("Mulled action to perform for targets - this defaults to 'build'. "
"Set this to build-and-test to also test the resulting container.")
)


def mulled_options():
return _compose(
mulled_conda_option(),
mulled_namespace_option(),
mulled_action_option(),
)
@@ -1,7 +1,7 @@
<tool id="bwa_and_samtools" name="BWA and samtools" version="0.1.0">
<requirements>
<requirement type="package" version="0.7.12">bwa</requirement>
<requirement type="package" version="1.2">samtools</requirement>
<requirement type="package" version="0.7.15">bwa</requirement>
<requirement type="package" version="1.3.1">samtools</requirement>
</requirements>
<stdio>
<exit_code range="2:" />
@@ -23,12 +23,12 @@
<test>
<output name="output_1">
<assert_contents>
<has_text text="Version: 0.7.12-r1039" />
<has_text text="Version: 0.7.15-r1140" />
</assert_contents>
</output>
<output name="output_2">
<assert_contents>
<has_text text="Version: 1.2" />
<has_text text="Version: 1.3.1" />
</assert_contents>
</output>
</test>
@@ -13,6 +13,6 @@ virtualenv
lxml
gxformat2>=0.1.1
ephemeris>=0.2.0
galaxy-lib>=17.5.5
galaxy-lib>=17.5.6
html5lib>=0.9999999,!=0.99999999,!=0.999999999,!=1.0b10,!=1.0b09 ; python_version == '2.7'
cwltool==1.0.20160726135535 ; python_version == '2.7'
@@ -91,6 +91,5 @@ def test_lint_dependencies_okay(self):
self._check_exit_code(lint_cmd)
lint_cmd = ["lint", "--conda_requirements", seqtk_seq_v6]
self._check_exit_code(lint_cmd)
# Doesn't work, it seems to be missing?
# lint_cmd = ["lint", "--biocontainer_registered", bwa_no_reqs]
# self._check_exit_code(lint_cmd)
lint_cmd = ["lint", "--biocontainer", seqtk_seq_v6]
self._check_exit_code(lint_cmd)

0 comments on commit 0a1abfe

Please sign in to comment.