Implement conda support.

This PR adds conda support to Planemo and via shared libraries to Galaxy in the form of new planemo commands providing a Galaxy tool-centric interface to conda for development and a Galaxy dependency resolver. Once the corresponding Galaxy PR is merged, the following command::

    planemo t --conda_dependency_resolution --conda_auto_install --conda_auto_init bwa_and_samtools.xml

Will test the supplied tool with only conda dependency resolution available. On startup, if Galaxy cannot find conda it will install it and for every dependency encountered it will attempt to find and install it on-demand.

Planemo Conda Commands:

 - conda_init: Install a conda runtime.
 - conda_install: Takes in a tool or set of tools and installs their requirements using conda in such a way that they can be recovered and reused by dependency resolution.
 - conda_env: Build isolated environment for a tool or set of tools.

Conda CLI Options:

Common options for conda commands in planemo.

  --conda_prefix: Location of conda runtime (defaults to ~/miniconda2).
  --conda_exec: Location of conda executble (defaults to <conda_prefix/bin/conda)
  --conda_debug: Flag, when enabled conda will execute with the --debug flag.
  --conda_ensure_channels: Ensure channels are available (defaults to "r,bioconda")

This PR implement a conda dependency resolver for Galaxy. This has a variety of options including:

 - copy_dependencies: This will copy dependencies into the working directory instead linking them - by passing --copy to conda create. Defaults to false.
 - auto_init: This will install conda if not avaiable on the system. Defaults to false.
 - auto_install: This will attempt to install packages for if they are not already installed.
 - Options mirroring the above for planemo - prefix, exec, debug, ensur_channels.

All these dependency resolution option can be set as attributes on the dependency resolver element in dependency_resolvers_conf.xml.sample, or in galaxy.ini with the prefix conda_.

Trying it out:

    $ planemo conda_init  # install conda
    $ # setup a couple tools for conda testing
    $ planemo project_init --templates conda_test
    $ cd conda_test
    $ # look at tool requirements and install packages
    $ planemo conda_install bwa.xml
    $ # Load up the tool's environment to explore interactively
    $ . <(planemo conda_env bwa.xml)
    (bwa)$ which bwa
    /home/john/miniconda2/envs/jobdepsdpzBA...186c6a5504767f9e7/bin/bwa
    $ conda_env_deactivate
    $ # Run the tool in galaxy using conda dependency resolution
    $ # All the tool and test case do is verify the dependency and version
    $ planemo t --conda_dependency_resolution bwa.xml
    $ # The galaxy dependency resolver can even install dependencies on demand
    $ planemo t --conda_dependency_resolution --conda_auto_install bwa_and_samtools.xml

planemo/commands/cmd_brew_env.py

@@ -4,6 +4,7 @@
 
 from planemo.cli import pass_context
 from planemo import options
+from planemo.io import ps1_for_path
 
 from galaxy.tools.loader import load_tool
 from galaxy.tools.deps.requirements import parse_requirements_from_xml
@@ -15,11 +16,7 @@
 @click.command('brew_env')
 @options.optional_tools_arg()
 @options.brew_option()
-@click.option(
-    "--skip_install",
-    is_flag=True,
-    help="Skip installation - only source requirements already available."
-)
+@options.skip_install_option()
 @click.option(
     "--shell",
     is_flag=True
@@ -67,10 +64,9 @@ def cli(ctx, path, brew=None, skip_install=False, shell=None):
         # TODO: Would be cool if this wasn't a bunch of random hackery.
         launch_shell = os.environ.get("SHELL")
         if "bash" in launch_shell:
-            file_name = os.path.basename(path)
-            base_name = os.path.splitext(file_name)[0]
+            ps1 = ps1_for_path(path)
             launch_shell = '(source ~/.bashrc; env PS1="%s" %s --norc)' % (
-                "(%s)${PS1}" % base_name,
+                ps1,
                 launch_shell,
             )
         lines.extend([launch_shell])

planemo/commands/cmd_conda_env.py

@@ -0,0 +1,68 @@
+from __future__ import print_function
+import click
+
+from planemo.cli import pass_context
+from planemo import options
+
+from planemo.io import ps1_for_path
+from planemo.io import error
+
+from planemo.conda import build_conda_context, collect_conda_targets
+
+from galaxy.tools.deps import conda_util
+
+
+SOURCE_COMMAND = """
+PRE_CONDA_PS1=$PS1
+source %s %s
+if [[ -n $BASH_VERSION ]]; then
+    hash -r
+elif [[ -n $ZSH_VERSION ]]; then
+    rehash
+else
+echo 'Only bash and zsh are supported'
+    return 1
+fi
+PS1="%s"
+echo 'Deactivate environment with conda_env_deactivate'
+alias conda_env_deactivate="source %s; %s"
+"""
+
+
+@click.command('conda_env')
+@options.optional_tools_arg()
+@options.conda_target_options()
+# @options.skip_install_option()  # TODO
+@pass_context
+def cli(ctx, path, **kwds):
+    """Source output to activate a conda environment for this tool.
+
+        % . <(planemo conda_env bowtie2.xml)
+        % which bowtie2
+        TODO_PLACE_PATH_HERE
+    """
+    conda_context = build_conda_context(use_planemo_shell_exec=False, **kwds)
+    conda_targets = collect_conda_targets(
+        path, conda_context=conda_context
+    )
+    installed_conda_targets = conda_util.filter_installed_targets(
+        conda_targets, conda_context=conda_context
+    )
+    env_name, exit_code = conda_util.build_isolated_environment(
+        installed_conda_targets, conda_context=conda_context
+    )
+    if exit_code:
+        error("Failed to build environmnt for request.")
+        return 1
+
+    ps1 = ps1_for_path(path, base="PRE_CONDA_PS1")
+    remove_env = "%s env remove -y --name '%s'" % (
+        conda_context.conda_exec, env_name
+    )
+    deactivate = conda_context.deactivate
+    activate = conda_context.activate
+    command = SOURCE_COMMAND % (
+        activate, env_name, ps1,
+        deactivate, remove_env
+    )
+    print(command)

planemo/commands/cmd_conda_init.py

@@ -0,0 +1,26 @@
+import click
+
+from planemo.cli import pass_context
+from planemo.io import shell
+from planemo import options
+from planemo.conda import build_conda_context
+
+from galaxy.tools.deps import conda_util
+
+
+@click.command('conda_init')
+@options.conda_options()
+@pass_context
+def cli(ctx, **kwds):
+    """Download and install conda.
+
+    This will download conda for managing dependencies for your platform
+    using the appropriate Miniconda installer.
+
+    By running this command, you are agreeing to the terms of the conda
+    license a 3-clause BSD 3 license. Please review full license at
+    http://docs.continuum.io/anaconda/eula.
+    """
+    conda_context = build_conda_context(**kwds)
+    return conda_util.install_conda(conda_context=conda_context,
+                                    shell_exec=shell)

planemo/commands/cmd_conda_install.py

@@ -0,0 +1,27 @@
+import click
+
+from planemo.cli import pass_context
+from planemo.io import coalesce_return_codes
+from planemo import options
+
+from planemo.conda import build_conda_context, collect_conda_targets
+
+from galaxy.tools.deps import conda_util
+
+
+@click.command('conda_install')
+@options.optional_tools_arg()
+@options.conda_target_options()
+@pass_context
+def cli(ctx, path, **kwds):
+    """Install conda packages for tool requirements.
+    """
+    conda_context = build_conda_context(**kwds)
+    return_codes = []
+    for conda_target in collect_conda_targets(path):
+        ctx.log("Install conda target %s" % conda_target)
+        return_code = conda_util.install_conda_target(
+            conda_target, conda_context=conda_context
+        )
+        return_codes.append(return_code)
+    return coalesce_return_codes(return_codes, assert_at_least_one=True)

planemo/conda.py

@@ -0,0 +1,32 @@
+""" Planemo specific utilities for dealing with conda, extending Galaxy's
+features with planemo specific idioms.
+"""
+
+from galaxy.tools.deps import conda_util
+from planemo.io import shell
+
+from galaxy.tools.deps.requirements import parse_requirements_from_xml
+from galaxy.tools.loader_directory import load_tool_elements_from_path
+
+
+def build_conda_context(**kwds):
+    """ Build a Galaxy CondaContext tailored to planemo use
+    and common command-line arguments.
+    """
+    conda_prefix = kwds.get("conda_prefix", None)
+    use_planemo_shell = kwds.get("use_planemo_shell_exec", True)
+    ensure_channels = kwds.get("conda_ensure_channels", "")
+    shell_exec = shell if use_planemo_shell else None
+    return conda_util.CondaContext(conda_prefix=conda_prefix,
+                                   ensure_channels=ensure_channels,
+                                   shell_exec=shell_exec)
+
+
+def collect_conda_targets(path, found_tool_callback=None, conda_context=None):
+    conda_targets = []
+    for (tool_path, tool_xml) in load_tool_elements_from_path(path):
+        if found_tool_callback:
+            found_tool_callback(tool_path)
+        requirements, containers = parse_requirements_from_xml(tool_xml)
+        conda_targets.extend(conda_util.requirements_to_conda_targets(requirements))
+    return conda_targets

planemo/galaxy_config.py

@@ -61,6 +61,13 @@
 </job_metrics>
 """
 
+# TODO: fill in properties to match CLI args.
+CONDA_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
+  <conda ${attributes} />
+  <conda versionless="true" ${attributes} />
+</dependency_resolvers>
+"""
+
 
 BREW_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
   <homebrew />
@@ -84,6 +91,7 @@
 STOCK_DEPENDENCY_RESOLUTION_STRATEGIES = {
     "brew_dependency_resolution": BREW_DEPENDENCY_RESOLUTION_CONF,
     "shed_dependency_resolution": SHED_DEPENDENCY_RESOLUTION_CONF,
+    "conda_dependency_resolution": CONDA_DEPENDENCY_RESOLUTION_CONF,
 }
 
 EMPTY_TOOL_CONF_TEMPLATE = """<toolbox></toolbox>"""
@@ -633,6 +641,7 @@ def _handle_dependency_resolution(config_directory, kwds):
         "brew_dependency_resolution",
         "dependency_resolvers_config_file",
         "shed_dependency_resolution",
+        "conda_dependency_resolution",
     ]
 
     selected_strategies = 0
@@ -644,13 +653,36 @@ def _handle_dependency_resolution(config_directory, kwds):
         message = "At most one option from [%s] may be specified"
         raise click.UsageError(message % resolutions_strategies)
 
+    dependency_attribute_kwds = {
+        'conda_prefix': None,
+        'conda_exec': None,
+        'conda_debug': False,
+        'conda_copy_dependencies': False,
+        'conda_auto_init': False,
+        'conda_auto_install': False,
+        'conda_ensure_channels': '',
+    }
+
+    attributes = []
+    for key, default_value in dependency_attribute_kwds.iteritems():
+        value = kwds.get(key, default_value)
+        if value != default_value:
+            # Strip leading prefix (conda_) off attributes
+            attribute_key = "_".join(key.split("_")[1:])
+            attributes.append('%s="%s"' % (attribute_key, value))
+
+    attribute_str = " ".join(attributes)
+
     for key in STOCK_DEPENDENCY_RESOLUTION_STRATEGIES:
         if kwds.get(key):
             resolvers_conf = os.path.join(
                 config_directory,
                 "resolvers_conf.xml"
             )
-            conf_contents = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
+            template_str = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
+            conf_contents = Template(template_str).safe_substitute({
+                'attributes': attribute_str
+            })
             open(resolvers_conf, "w").write(conf_contents)
             kwds["dependency_resolvers_config_file"] = resolvers_conf
 

planemo/io.py

@@ -101,6 +101,16 @@ def temp_directory(prefix="planemo_tmp_"):
         shutil.rmtree(temp_dir)
 
 
+def ps1_for_path(path, base="PS1"):
+    """ Used by environment commands to build a PS1 shell
+    variables for tool or directory of tools.
+    """
+    file_name = os.path.basename(path)
+    base_name = os.path.splitext(file_name)[0]
+    ps1 = "(%s)${%s}" % (base_name, base)
+    return ps1
+
+
 def kill_pid_file(pid_file):
     if not os.path.exists(pid_file):
         return