Permalink
Browse files

Mulled support.

Implement option to build mulled containers for tools:

```
    planemo mull [/path/to/tools]*
```

Implement option to force tool serving and execution to occur in mulled containers.

```
    planemo test --mulled_containers
```

This includes fixes for support or tools in Docker, something that has been around since the early days but seemingly broken the whole time.

xref #583
  • Loading branch information...
jmchilton committed Oct 4, 2016
1 parent 2e4e5fc commit 08cef54b79f2d4721f344933da3effee4f481777
Showing with 143 additions and 4 deletions.
  1. +33 −0 planemo/commands/cmd_mull.py
  2. +15 −0 planemo/conda.py
  3. +37 −3 planemo/galaxy/config.py
  4. +46 −0 planemo/mulled.py
  5. +11 −0 planemo/options.py
  6. +1 −1 requirements.txt
@@ -0,0 +1,33 @@
"""Module describing the planemo ``mull`` command."""
import click

from galaxy.tools.deps.mulled.mulled_build import mull_targets
from galaxy.tools.deps.mulled.util import build_target

from planemo import options
from planemo.cli import command_function
from planemo.conda import collect_conda_target_lists
from planemo.mulled import build_mull_target_kwds


@click.command('mull')
@options.optional_tools_arg(multiple=True)
@options.recursive_option()
@command_function
def cli(ctx, paths, **kwds):
"""Build containers for specified tools.
Supplied tools will be inspected for referenced requirement packages. For
each combination of requirements a "mulled" container will be built. Galaxy
can automatically discover this container and subsequently use it to run
or test the tool.
For this to work, the tool's requirements will need to be present in a known
Conda channel such as bioconda (https://github.com/bioconda/bioconda-recipes).
This can be verified by running ``planemo lint --conda_requirements`` on the
target tool(s).
"""
for conda_targets in collect_conda_target_lists(ctx, paths):
mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
mull_target_kwds = build_mull_target_kwds(ctx, **kwds)
mull_targets(mulled_targets, command="build", **mull_target_kwds)
@@ -44,6 +44,20 @@ def collect_conda_targets(ctx, paths, found_tool_callback=None, conda_context=No
return conda_targets


def collect_conda_target_lists(ctx, paths, found_tool_callback=None):
"""Load CondaTarget lists from supplied artifact sources.
If a tool contains more than one requirement, the requirements will all
appear together as one list element of the output list.
"""
conda_target_lists = set([])
for (tool_path, tool_source) in yield_tool_sources_on_paths(ctx, paths):
if found_tool_callback:
found_tool_callback(tool_path)
conda_target_lists.add(frozenset(tool_source_conda_targets(tool_source)))
return conda_target_lists


def tool_source_conda_targets(tool_source):
"""Load CondaTarget object from supplied abstract tool source."""
requirements, _ = tool_source.parse_requirements_and_containers()
@@ -53,5 +67,6 @@ def tool_source_conda_targets(tool_source):
__all__ = [
"build_conda_context",
"collect_conda_targets",
"collect_conda_target_lists",
"tool_source_conda_targets",
]
@@ -22,6 +22,7 @@

from planemo import git
from planemo.conda import build_conda_context
from planemo.config import OptionSource
from planemo.docker import docker_host_args
from planemo.io import (
communicate,
@@ -32,6 +33,7 @@
warn,
write_file,
)
from planemo.mulled import build_involucro_context
from planemo.shed import tool_shed_url

from .api import (
@@ -118,13 +120,20 @@
</handlers>
<destinations default="planemo_dest">
<destination id="planemo_dest" runner="planemo_runner">
<param id="docker_enable">${docker_enable}</param>
<param id="require_container">${require_container}</param>
<param id="docker_enabled">${docker_enable}</param>
<param id="docker_sudo">${docker_sudo}</param>
<param id="docker_sudo_cmd">${docker_sudo_cmd}</param>
<param id="docker_cmd">${docker_cmd}</param>
<param id="docker_host">${docker_host}</param>
${docker_host_param}
</destination>
<destination id="upload_dest" runner="planemo_runner">
<param id="docker_enable">false</param>
</destination>
</destinations>
<tools>
<tool id="upload1" destination="upload_dest" />
</tools>
</job_conf>
"""

@@ -211,6 +220,7 @@ def config_join(*args):
)
port = _get_port(kwds)
properties = _shared_galaxy_properties(kwds)
_handle_container_resolution(ctx, kwds, properties)
master_api_key = _get_master_api_key(kwds)

template_args = dict(
@@ -276,6 +286,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
)
galaxy_root = _check_galaxy(ctx, **kwds)
install_galaxy = galaxy_root is None

# Duplicate block in docker variant above.
if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
if ctx.get_option_source("docker") != OptionSource.cli:
kwds["docker"] = True
else:
raise Exception("Specified no docker and mulled containers together.")

with _config_directory(ctx, **kwds) as config_directory:
def config_join(*args):
return os.path.join(config_directory, *args)
@@ -374,6 +392,7 @@ def config_join(*args):
migrated_tools_config=empty_tool_conf,
test_data_dir=test_data_dir, # TODO: make gx respect this
))
_handle_container_resolution(ctx, kwds, properties)
if not for_tests:
properties["database_connection"] = _database_connection(database_location, **kwds)

@@ -1106,13 +1125,20 @@ def _handle_job_config_file(config_directory, server_name, kwds):
config_directory,
"job_conf.xml",
)
docker_enable = str(kwds.get("docker", False))
docker_host = str(kwds.get("docker_host", docker_util.DEFAULT_HOST))
docker_host_param = ""
if docker_host:
docker_host_param = """<param id="docker_host">%s</param>""" % docker_host

conf_contents = Template(template_str).safe_substitute({
"server_name": server_name,
"docker_enable": str(kwds.get("docker", False)),
"require_container": docker_enable,
"docker_sudo": str(kwds.get("docker_sudo", False)),
"docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
"docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
"docker_host": str(kwds.get("docker_host", docker_util.DEFAULT_HOST)),
"docker_host": docker_host_param,
})
write_file(job_config_file, conf_contents)
kwds["job_config_file"] = job_config_file
@@ -1184,6 +1210,14 @@ def add_attribute(key, value):
kwds["dependency_resolvers_config_file"] = resolvers_conf


def _handle_container_resolution(ctx, kwds, galaxy_properties):
if kwds.get("mulled_containers", False):
galaxy_properties["enable_beta_mulled_containers"] = "True"
involucro_context = build_involucro_context(ctx, **kwds)
galaxy_properties["involucro_auto_init"] = "False" # Use planemo's
galaxy_properties["involucro_path"] = involucro_context.involucro_bin


def _handle_job_metrics(config_directory, kwds):
metrics_conf = os.path.join(config_directory, "job_metrics_conf.xml")
open(metrics_conf, "w").write(EMPTY_JOB_METRICS_TEMPLATE)
@@ -0,0 +1,46 @@
"""Planemo specific utilities for dealing with mulled containers.
The extend Galaxy/galaxy-lib's features with planemo specific idioms.
"""
from __future__ import absolute_import

import os

from galaxy.tools.deps.mulled.mulled_build import (
DEFAULT_CHANNELS,
ensure_installed,
InvolucroContext,
)

from planemo.io import shell


def build_involucro_context(ctx, **kwds):
"""Build a galaxy-lib CondaContext tailored to planemo use.
Using planemo's common command-line/global config options.
"""
involucro_path_default = os.path.join(ctx.workspace, "involucro")
involucro_path = kwds.get("involucro_path", involucro_path_default)
use_planemo_shell = kwds.get("use_planemo_shell_exec", True)
shell_exec = shell if use_planemo_shell else None
involucro_context = InvolucroContext(involucro_bin=involucro_path,
shell_exec=shell_exec)
if not ensure_installed(involucro_context, True):
raise Exception("Failed to install involucro for Planemo.")
return involucro_context


def build_mull_target_kwds(ctx, **kwds):
"""Adapt Planemo's CLI and workspace configuration to galaxy-lib's mulled_build options."""
involucro_context = build_involucro_context(ctx, **kwds)
channels = kwds.get("conda_ensure_channels", ",".join(DEFAULT_CHANNELS))

return {
'involucro_context': involucro_context,
'channels': channels.split(","),
}

__all__ = [
"build_involucro_context",
]
@@ -333,6 +333,14 @@ def job_config_option():
)


def mulled_containers_option():
return planemo_option(
"--mulled_containers",
is_flag=True,
help="Test tools against mulled containers (forces --docker).",
)


def install_galaxy_option():
return planemo_option(
"--install_galaxy",
@@ -430,6 +438,8 @@ def conda_debug_option():

def conda_ensure_channels_option():
return planemo_option(
"conda_ensure_channels",
"--conda_channels",
"--conda_ensure_channels",
type=str,
use_global_config=True,
@@ -878,6 +888,7 @@ def galaxy_target_options():
no_cleanup_option(),
galaxy_email_option(),
galaxy_docker_options(),
mulled_containers_option(),
# Profile options...
job_config_option(),
tool_dependency_dir_option(),
@@ -10,6 +10,6 @@ virtualenv
lxml
gxformat2>=0.1.1
ephemeris>=0.2.0
galaxy-lib>=16.10.3
galaxy-lib>=16.10.5
html5lib>=0.9999999,!=0.99999999,!=0.999999999,!=1.0b10,!=1.0b09 ; python_version == '2.7'
cwltool==1.0.20160726135535 ; python_version == '2.7'

0 comments on commit 08cef54

Please sign in to comment.