Permalink
Browse files

Implement conda support.

This PR adds conda support to Planemo and via shared libraries to Galaxy in the form of new planemo commands providing a Galaxy tool-centric interface to conda for development and a Galaxy dependency resolver. Once the corresponding Galaxy PR is merged, the following command::

    planemo t --conda_dependency_resolution --conda_auto_install --conda_auto_init bwa_and_samtools.xml

Will test the supplied tool with only conda dependency resolution available. On startup, if Galaxy cannot find conda it will install it and for every dependency encountered it will attempt to find and install it on-demand.

Planemo Conda Commands:

 - conda_init: Install a conda runtime.
 - conda_install: Takes in a tool or set of tools and installs their requirements using conda in such a way that they can be recovered and reused by dependency resolution.
 - conda_env: Build isolated environment for a tool or set of tools.

Conda CLI Options:

Common options for conda commands in planemo.

  --conda_prefix: Location of conda runtime (defaults to ~/miniconda2).
  --conda_exec: Location of conda executble (defaults to <conda_prefix/bin/conda)
  --conda_debug: Flag, when enabled conda will execute with the --debug flag.
  --conda_ensure_channels: Ensure channels are available (defaults to "r,bioconda")

This PR implement a conda dependency resolver for Galaxy. This has a variety of options including:

 - copy_dependencies: This will copy dependencies into the working directory instead linking them - by passing --copy to conda create. Defaults to false.
 - auto_init: This will install conda if not avaiable on the system. Defaults to false.
 - auto_install: This will attempt to install packages for if they are not already installed.
 - Options mirroring the above for planemo - prefix, exec, debug, ensur_channels.

All these dependency resolution option can be set as attributes on the dependency resolver element in dependency_resolvers_conf.xml.sample, or in galaxy.ini with the prefix conda_.

Trying it out:

    $ planemo conda_init  # install conda
    $ # setup a couple tools for conda testing
    $ planemo project_init --templates conda_test
    $ cd conda_test
    $ # look at tool requirements and install packages
    $ planemo conda_install bwa.xml
    $ # Load up the tool's environment to explore interactively
    $ . <(planemo conda_env bwa.xml)
    (bwa)$ which bwa
    /home/john/miniconda2/envs/jobdepsdpzBA...186c6a5504767f9e7/bin/bwa
    $ conda_env_deactivate
    $ # Run the tool in galaxy using conda dependency resolution
    $ # All the tool and test case do is verify the dependency and version
    $ planemo t --conda_dependency_resolution bwa.xml
    $ # The galaxy dependency resolver can even install dependencies on demand
    $ planemo t --conda_dependency_resolution --conda_auto_install bwa_and_samtools.xml
  • Loading branch information...
jmchilton committed Dec 15, 2015
1 parent 4bccd2c commit f99f6c13b6ba7056c4ac133155d5c54049c4907b
@@ -4,6 +4,7 @@

from planemo.cli import pass_context
from planemo import options
from planemo.io import ps1_for_path

from galaxy.tools.loader import load_tool
from galaxy.tools.deps.requirements import parse_requirements_from_xml
@@ -15,11 +16,7 @@
@click.command('brew_env')
@options.optional_tools_arg()
@options.brew_option()
@click.option(
"--skip_install",
is_flag=True,
help="Skip installation - only source requirements already available."
)
@options.skip_install_option()
@click.option(
"--shell",
is_flag=True
@@ -67,10 +64,9 @@ def cli(ctx, path, brew=None, skip_install=False, shell=None):
# TODO: Would be cool if this wasn't a bunch of random hackery.
launch_shell = os.environ.get("SHELL")
if "bash" in launch_shell:
file_name = os.path.basename(path)
base_name = os.path.splitext(file_name)[0]
ps1 = ps1_for_path(path)
launch_shell = '(source ~/.bashrc; env PS1="%s" %s --norc)' % (
"(%s)${PS1}" % base_name,
ps1,
launch_shell,
)
lines.extend([launch_shell])
@@ -0,0 +1,68 @@
from __future__ import print_function
import click

from planemo.cli import pass_context
from planemo import options

from planemo.io import ps1_for_path
from planemo.io import error

from planemo.conda import build_conda_context, collect_conda_targets

from galaxy.tools.deps import conda_util


SOURCE_COMMAND = """
PRE_CONDA_PS1=$PS1
source %s %s
if [[ -n $BASH_VERSION ]]; then
hash -r
elif [[ -n $ZSH_VERSION ]]; then
rehash
else
echo 'Only bash and zsh are supported'
return 1
fi
PS1="%s"
echo 'Deactivate environment with conda_env_deactivate'
alias conda_env_deactivate="source %s; %s"
"""


@click.command('conda_env')
@options.optional_tools_arg()
@options.conda_target_options()
# @options.skip_install_option() # TODO
@pass_context
def cli(ctx, path, **kwds):
"""Source output to activate a conda environment for this tool.
% . <(planemo conda_env bowtie2.xml)
% which bowtie2
TODO_PLACE_PATH_HERE
"""
conda_context = build_conda_context(use_planemo_shell_exec=False, **kwds)
conda_targets = collect_conda_targets(
path, conda_context=conda_context
)
installed_conda_targets = conda_util.filter_installed_targets(
conda_targets, conda_context=conda_context
)
env_name, exit_code = conda_util.build_isolated_environment(
installed_conda_targets, conda_context=conda_context
)
if exit_code:
error("Failed to build environmnt for request.")
return 1

ps1 = ps1_for_path(path, base="PRE_CONDA_PS1")
remove_env = "%s env remove -y --name '%s'" % (
conda_context.conda_exec, env_name
)
deactivate = conda_context.deactivate
activate = conda_context.activate
command = SOURCE_COMMAND % (
activate, env_name, ps1,
deactivate, remove_env
)
print(command)
@@ -0,0 +1,26 @@
import click

from planemo.cli import pass_context
from planemo.io import shell
from planemo import options
from planemo.conda import build_conda_context

from galaxy.tools.deps import conda_util


@click.command('conda_init')
@options.conda_target_options()
@pass_context
def cli(ctx, **kwds):
"""Download and install conda.
This will download conda for managing dependencies for your platform
using the appropriate Miniconda installer.
By running this command, you are agreeing to the terms of the conda
license a 3-clause BSD 3 license. Please review full license at
http://docs.continuum.io/anaconda/eula.
"""
conda_context = build_conda_context(**kwds)
return conda_util.install_conda(conda_context=conda_context,
shell_exec=shell)
@@ -0,0 +1,27 @@
import click

from planemo.cli import pass_context
from planemo.io import coalesce_return_codes
from planemo import options

from planemo.conda import build_conda_context, collect_conda_targets

from galaxy.tools.deps import conda_util


@click.command('conda_install')
@options.optional_tools_arg()
@options.conda_target_options()
@pass_context
def cli(ctx, path, **kwds):
"""Install conda packages for tool requirements.
"""
conda_context = build_conda_context(**kwds)
return_codes = []
for conda_target in collect_conda_targets(path):
ctx.log("Install conda target %s" % conda_target)
return_code = conda_util.install_conda_target(
conda_target, conda_context=conda_context
)
return_codes.append(return_code)
return coalesce_return_codes(return_codes, assert_at_least_one=True)
@@ -0,0 +1,32 @@
""" Planemo specific utilities for dealing with conda, extending Galaxy's
features with planemo specific idioms.
"""

from galaxy.tools.deps import conda_util
from planemo.io import shell

from galaxy.tools.deps.requirements import parse_requirements_from_xml
from galaxy.tools.loader_directory import load_tool_elements_from_path


def build_conda_context(**kwds):
""" Build a Galaxy CondaContext tailored to planemo use
and common command-line arguments.
"""
conda_prefix = kwds.get("conda_prefix", None)
use_planemo_shell = kwds.get("use_planemo_shell_exec", True)
ensure_channels = kwds.get("conda_ensure_channels", "")
shell_exec = shell if use_planemo_shell else None
return conda_util.CondaContext(conda_prefix=conda_prefix,
ensure_channels=ensure_channels,
shell_exec=shell_exec)


def collect_conda_targets(path, found_tool_callback=None, conda_context=None):
conda_targets = []
for (tool_path, tool_xml) in load_tool_elements_from_path(path):
if found_tool_callback:
found_tool_callback(tool_path)
requirements, containers = parse_requirements_from_xml(tool_xml)
conda_targets.extend(conda_util.requirements_to_conda_targets(requirements))
return conda_targets
@@ -61,6 +61,13 @@
</job_metrics>
"""

# TODO: fill in properties to match CLI args.
CONDA_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
<conda ${attributes} />
<conda versionless="true" ${attributes} />
</dependency_resolvers>
"""


BREW_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
<homebrew />
@@ -84,6 +91,7 @@
STOCK_DEPENDENCY_RESOLUTION_STRATEGIES = {
"brew_dependency_resolution": BREW_DEPENDENCY_RESOLUTION_CONF,
"shed_dependency_resolution": SHED_DEPENDENCY_RESOLUTION_CONF,
"conda_dependency_resolution": CONDA_DEPENDENCY_RESOLUTION_CONF,
}

EMPTY_TOOL_CONF_TEMPLATE = """<toolbox></toolbox>"""
@@ -649,6 +657,7 @@ def _handle_dependency_resolution(config_directory, kwds):
"brew_dependency_resolution",
"dependency_resolvers_config_file",
"shed_dependency_resolution",
"conda_dependency_resolution",
]

selected_strategies = 0
@@ -660,13 +669,36 @@ def _handle_dependency_resolution(config_directory, kwds):
message = "At most one option from [%s] may be specified"
raise click.UsageError(message % resolutions_strategies)

dependency_attribute_kwds = {
'conda_prefix': None,
'conda_exec': None,
'conda_debug': False,
'conda_copy_dependencies': False,
'conda_auto_init': False,
'conda_auto_install': False,
'conda_ensure_channels': '',
}

attributes = []
for key, default_value in dependency_attribute_kwds.iteritems():
value = kwds.get(key, default_value)
if value != default_value:
# Strip leading prefix (conda_) off attributes
attribute_key = "_".join(key.split("_")[1:])
attributes.append('%s="%s"' % (attribute_key, value))

attribute_str = " ".join(attributes)

for key in STOCK_DEPENDENCY_RESOLUTION_STRATEGIES:
if kwds.get(key):
resolvers_conf = os.path.join(
config_directory,
"resolvers_conf.xml"
)
conf_contents = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
template_str = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
conf_contents = Template(template_str).safe_substitute({
'attributes': attribute_str
})
open(resolvers_conf, "w").write(conf_contents)
kwds["dependency_resolvers_config_file"] = resolvers_conf

@@ -101,6 +101,16 @@ def temp_directory(prefix="planemo_tmp_"):
shutil.rmtree(temp_dir)


def ps1_for_path(path, base="PS1"):
""" Used by environment commands to build a PS1 shell
variables for tool or directory of tools.
"""
file_name = os.path.basename(path)
base_name = os.path.splitext(file_name)[0]
ps1 = "(%s)${%s}" % (base_name, base)
return ps1


def kill_pid_file(pid_file):
if not os.path.exists(pid_file):
return
Oops, something went wrong.

0 comments on commit f99f6c1

Please sign in to comment.