diff --git a/planemo/commands/cmd_brew_env.py b/planemo/commands/cmd_brew_env.py index 2dfbcf884..f34d6e26b 100644 --- a/planemo/commands/cmd_brew_env.py +++ b/planemo/commands/cmd_brew_env.py @@ -4,6 +4,7 @@ from planemo.cli import pass_context from planemo import options +from planemo.io import ps1_for_path from galaxy.tools.loader import load_tool from galaxy.tools.deps.requirements import parse_requirements_from_xml @@ -15,11 +16,7 @@ @click.command('brew_env') @options.optional_tools_arg() @options.brew_option() -@click.option( - "--skip_install", - is_flag=True, - help="Skip installation - only source requirements already available." -) +@options.skip_install_option() @click.option( "--shell", is_flag=True @@ -67,10 +64,9 @@ def cli(ctx, path, brew=None, skip_install=False, shell=None): # TODO: Would be cool if this wasn't a bunch of random hackery. launch_shell = os.environ.get("SHELL") if "bash" in launch_shell: - file_name = os.path.basename(path) - base_name = os.path.splitext(file_name)[0] + ps1 = ps1_for_path(path) launch_shell = '(source ~/.bashrc; env PS1="%s" %s --norc)' % ( - "(%s)${PS1}" % base_name, + ps1, launch_shell, ) lines.extend([launch_shell]) diff --git a/planemo/commands/cmd_conda_env.py b/planemo/commands/cmd_conda_env.py new file mode 100644 index 000000000..ec6efde1f --- /dev/null +++ b/planemo/commands/cmd_conda_env.py @@ -0,0 +1,68 @@ +from __future__ import print_function +import click + +from planemo.cli import pass_context +from planemo import options + +from planemo.io import ps1_for_path +from planemo.io import error + +from planemo.conda import build_conda_context, collect_conda_targets + +from galaxy.tools.deps import conda_util + + +SOURCE_COMMAND = """ +PRE_CONDA_PS1=$PS1 +source %s %s +if [[ -n $BASH_VERSION ]]; then + hash -r +elif [[ -n $ZSH_VERSION ]]; then + rehash +else +echo 'Only bash and zsh are supported' + return 1 +fi +PS1="%s" +echo 'Deactivate environment with conda_env_deactivate' +alias conda_env_deactivate="source %s; %s" +""" + + +@click.command('conda_env') +@options.optional_tools_arg() +@options.conda_target_options() +# @options.skip_install_option() # TODO +@pass_context +def cli(ctx, path, **kwds): + """Source output to activate a conda environment for this tool. + + % . <(planemo conda_env bowtie2.xml) + % which bowtie2 + TODO_PLACE_PATH_HERE + """ + conda_context = build_conda_context(use_planemo_shell_exec=False, **kwds) + conda_targets = collect_conda_targets( + path, conda_context=conda_context + ) + installed_conda_targets = conda_util.filter_installed_targets( + conda_targets, conda_context=conda_context + ) + env_name, exit_code = conda_util.build_isolated_environment( + installed_conda_targets, conda_context=conda_context + ) + if exit_code: + error("Failed to build environmnt for request.") + return 1 + + ps1 = ps1_for_path(path, base="PRE_CONDA_PS1") + remove_env = "%s env remove -y --name '%s'" % ( + conda_context.conda_exec, env_name + ) + deactivate = conda_context.deactivate + activate = conda_context.activate + command = SOURCE_COMMAND % ( + activate, env_name, ps1, + deactivate, remove_env + ) + print(command) diff --git a/planemo/commands/cmd_conda_init.py b/planemo/commands/cmd_conda_init.py new file mode 100644 index 000000000..a67d892e8 --- /dev/null +++ b/planemo/commands/cmd_conda_init.py @@ -0,0 +1,26 @@ +import click + +from planemo.cli import pass_context +from planemo.io import shell +from planemo import options +from planemo.conda import build_conda_context + +from galaxy.tools.deps import conda_util + + +@click.command('conda_init') +@options.conda_options() +@pass_context +def cli(ctx, **kwds): + """Download and install conda. + + This will download conda for managing dependencies for your platform + using the appropriate Miniconda installer. + + By running this command, you are agreeing to the terms of the conda + license a 3-clause BSD 3 license. Please review full license at + http://docs.continuum.io/anaconda/eula. + """ + conda_context = build_conda_context(**kwds) + return conda_util.install_conda(conda_context=conda_context, + shell_exec=shell) diff --git a/planemo/commands/cmd_conda_install.py b/planemo/commands/cmd_conda_install.py new file mode 100644 index 000000000..b805c1095 --- /dev/null +++ b/planemo/commands/cmd_conda_install.py @@ -0,0 +1,27 @@ +import click + +from planemo.cli import pass_context +from planemo.io import coalesce_return_codes +from planemo import options + +from planemo.conda import build_conda_context, collect_conda_targets + +from galaxy.tools.deps import conda_util + + +@click.command('conda_install') +@options.optional_tools_arg() +@options.conda_target_options() +@pass_context +def cli(ctx, path, **kwds): + """Install conda packages for tool requirements. + """ + conda_context = build_conda_context(**kwds) + return_codes = [] + for conda_target in collect_conda_targets(path): + ctx.log("Install conda target %s" % conda_target) + return_code = conda_util.install_conda_target( + conda_target, conda_context=conda_context + ) + return_codes.append(return_code) + return coalesce_return_codes(return_codes, assert_at_least_one=True) diff --git a/planemo/conda.py b/planemo/conda.py new file mode 100644 index 000000000..bcb3282c4 --- /dev/null +++ b/planemo/conda.py @@ -0,0 +1,32 @@ +""" Planemo specific utilities for dealing with conda, extending Galaxy's +features with planemo specific idioms. +""" + +from galaxy.tools.deps import conda_util +from planemo.io import shell + +from galaxy.tools.deps.requirements import parse_requirements_from_xml +from galaxy.tools.loader_directory import load_tool_elements_from_path + + +def build_conda_context(**kwds): + """ Build a Galaxy CondaContext tailored to planemo use + and common command-line arguments. + """ + conda_prefix = kwds.get("conda_prefix", None) + use_planemo_shell = kwds.get("use_planemo_shell_exec", True) + ensure_channels = kwds.get("conda_ensure_channels", "") + shell_exec = shell if use_planemo_shell else None + return conda_util.CondaContext(conda_prefix=conda_prefix, + ensure_channels=ensure_channels, + shell_exec=shell_exec) + + +def collect_conda_targets(path, found_tool_callback=None, conda_context=None): + conda_targets = [] + for (tool_path, tool_xml) in load_tool_elements_from_path(path): + if found_tool_callback: + found_tool_callback(tool_path) + requirements, containers = parse_requirements_from_xml(tool_xml) + conda_targets.extend(conda_util.requirements_to_conda_targets(requirements)) + return conda_targets diff --git a/planemo/galaxy_config.py b/planemo/galaxy_config.py index cfb06a293..e17c35f7c 100644 --- a/planemo/galaxy_config.py +++ b/planemo/galaxy_config.py @@ -61,6 +61,13 @@ """ +# TODO: fill in properties to match CLI args. +CONDA_DEPENDENCY_RESOLUTION_CONF = """ + + + +""" + BREW_DEPENDENCY_RESOLUTION_CONF = """ @@ -84,6 +91,7 @@ STOCK_DEPENDENCY_RESOLUTION_STRATEGIES = { "brew_dependency_resolution": BREW_DEPENDENCY_RESOLUTION_CONF, "shed_dependency_resolution": SHED_DEPENDENCY_RESOLUTION_CONF, + "conda_dependency_resolution": CONDA_DEPENDENCY_RESOLUTION_CONF, } EMPTY_TOOL_CONF_TEMPLATE = """""" @@ -633,6 +641,7 @@ def _handle_dependency_resolution(config_directory, kwds): "brew_dependency_resolution", "dependency_resolvers_config_file", "shed_dependency_resolution", + "conda_dependency_resolution", ] selected_strategies = 0 @@ -644,13 +653,36 @@ def _handle_dependency_resolution(config_directory, kwds): message = "At most one option from [%s] may be specified" raise click.UsageError(message % resolutions_strategies) + dependency_attribute_kwds = { + 'conda_prefix': None, + 'conda_exec': None, + 'conda_debug': False, + 'conda_copy_dependencies': False, + 'conda_auto_init': False, + 'conda_auto_install': False, + 'conda_ensure_channels': '', + } + + attributes = [] + for key, default_value in dependency_attribute_kwds.iteritems(): + value = kwds.get(key, default_value) + if value != default_value: + # Strip leading prefix (conda_) off attributes + attribute_key = "_".join(key.split("_")[1:]) + attributes.append('%s="%s"' % (attribute_key, value)) + + attribute_str = " ".join(attributes) + for key in STOCK_DEPENDENCY_RESOLUTION_STRATEGIES: if kwds.get(key): resolvers_conf = os.path.join( config_directory, "resolvers_conf.xml" ) - conf_contents = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key] + template_str = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key] + conf_contents = Template(template_str).safe_substitute({ + 'attributes': attribute_str + }) open(resolvers_conf, "w").write(conf_contents) kwds["dependency_resolvers_config_file"] = resolvers_conf diff --git a/planemo/io.py b/planemo/io.py index 886c71256..1904a02e2 100644 --- a/planemo/io.py +++ b/planemo/io.py @@ -101,6 +101,16 @@ def temp_directory(prefix="planemo_tmp_"): shutil.rmtree(temp_dir) +def ps1_for_path(path, base="PS1"): + """ Used by environment commands to build a PS1 shell + variables for tool or directory of tools. + """ + file_name = os.path.basename(path) + base_name = os.path.splitext(file_name)[0] + ps1 = "(%s)${%s}" % (base_name, base) + return ps1 + + def kill_pid_file(pid_file): if not os.path.exists(pid_file): return diff --git a/planemo/options.py b/planemo/options.py index c189c6827..c16943b0e 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -126,6 +126,14 @@ def brew_dependency_resolution(): ) +def conda_dependency_resolution(): + return click.option( + "--conda_dependency_resolution", + is_flag=True, + help="Configure Galaxy to use only conda for dependency resolution.", + ) + + def shed_dependency_resolution(): return click.option( "--shed_dependency_resolution", @@ -180,6 +188,14 @@ def no_cache_galaxy_option(): ) +def skip_install_option(): + return click.option( + "--skip_install", + is_flag=True, + help="Skip installation - only source requirements already available." + ) + + def brew_option(): return click.option( "--brew", @@ -188,6 +204,67 @@ def brew_option(): ) +def conda_prefix_option(): + return click.option( + "--conda_prefix", + type=click.Path(file_okay=False, dir_okay=True), + help="Conda prefix to use for conda dependency commands." + ) + + +def conda_exec_option(): + return click.option( + "--conda_exec", + type=click.Path(exists=True, file_okay=True, dir_okay=False), + help="Location of conda executable." + ) + + +def conda_debug_option(): + return click.option( + "--conda_debug", + is_flag=True, + help="Enable more verbose conda logging." + ) + + +def conda_ensure_channels_option(): + return click.option( + "--conda_ensure_channels", + type=str, + help=("Ensure conda is configured with specified comma separated " + "list of channels."), + default="r,bioconda" + ) + + +def conda_copy_dependencies_option(): + return click.option( + "--conda_copy_dependencies", + is_flag=True, + help=("Conda dependency resolution for Galaxy will copy dependencies " + "instead of attempting to link them.") + ) + + +def conda_auto_install_option(): + return click.option( + "--conda_auto_install", + is_flag=True, + help=("Conda dependency resolution for Galaxy will auto install " + "will attempt to install requested but missing packages.") + ) + + +def conda_auto_init_option(): + return click.option( + "--conda_auto_init", + is_flag=True, + help=("Conda dependency resolution for Galaxy will auto install " + "conda itself using miniconda if not availabe on conda_prefix.") + ) + + def required_tool_arg(): """ Decorate click method as requiring the path to a single tool. """ @@ -507,6 +584,15 @@ def shed_target_options(): ) +def conda_target_options(): + return _compose( + conda_prefix_option(), + conda_exec_option(), + conda_debug_option(), + conda_ensure_channels_option(), + ) + + def galaxy_run_options(): return _compose( galaxy_target_options(), @@ -523,6 +609,11 @@ def galaxy_config_options(): tool_dependency_dir_option(), brew_dependency_resolution(), shed_dependency_resolution(), + conda_target_options(), + conda_dependency_resolution(), + conda_copy_dependencies_option(), + conda_auto_install_option(), + conda_auto_init_option(), ) diff --git a/planemo_ext/galaxy/tools/deps/__init__.py b/planemo_ext/galaxy/tools/deps/__init__.py index 2bf6f6eac..05217de07 100644 --- a/planemo_ext/galaxy/tools/deps/__init__.py +++ b/planemo_ext/galaxy/tools/deps/__init__.py @@ -10,8 +10,21 @@ from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver +from .resolvers.conda import CondaDependencyResolver from galaxy.util import plugin_config +# TODO: Load these from the plugins. Would require a two step initialization of +# DependencyManager - where the plugins are loaded first and then the config +# is parsed and sent through. +EXTRA_CONFIG_KWDS = { + 'conda_prefix': None, + 'conda_exec': None, + 'conda_debug': None, + 'conda_channels': 'r,bioconda', + 'conda_auto_install': False, + 'conda_auto_init': False, +} + def build_dependency_manager( config ): if getattr( config, "use_tool_dependencies", False ): @@ -19,6 +32,8 @@ def build_dependency_manager( config ): 'default_base_path': config.tool_dependency_dir, 'conf_file': config.dependency_resolvers_config_file, } + for key, default_value in EXTRA_CONFIG_KWDS.iteritems(): + dependency_manager_kwds[key] = getattr(config, key, default_value) dependency_manager = DependencyManager( **dependency_manager_kwds ) else: dependency_manager = NullDependencyManager() @@ -49,7 +64,7 @@ class DependencyManager( object ): and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, default_base_path, conf_file=None ): + def __init__( self, default_base_path, conf_file=None, **extra_config ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -58,6 +73,7 @@ def __init__( self, default_base_path, conf_file=None ): log.warn( "Path '%s' does not exist, ignoring", default_base_path ) if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) + self.extra_config = extra_config self.default_base_path = os.path.abspath( default_base_path ) self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) @@ -106,6 +122,8 @@ def __default_dependency_resolvers( self ): ToolShedPackageDependencyResolver(self), GalaxyPackageDependencyResolver(self), GalaxyPackageDependencyResolver(self, versionless=True), + CondaDependencyResolver(self), + CondaDependencyResolver(self, versionless=True), ] def __parse_resolver_conf_xml(self, plugin_source): diff --git a/planemo_ext/galaxy/tools/deps/conda_util.py b/planemo_ext/galaxy/tools/deps/conda_util.py new file mode 100644 index 000000000..942e7f82c --- /dev/null +++ b/planemo_ext/galaxy/tools/deps/conda_util.py @@ -0,0 +1,305 @@ +import functools +import hashlib +import os.path +import re +import shutil +from sys import platform as _platform +import tempfile + +import six +import yaml + +from ..deps import commands + +# Not sure there are security concerns, lets just fail fast if we are going +# break shell commands we are building. +SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']") + +IS_OS_X = _platform == "darwin" + +# BSD 3-clause +CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula" + + +def conda_link(): + if IS_OS_X: + url = "https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh" + else: + url = "https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh" + return url + + +def find_conda_prefix(conda_prefix=None): + """ If supplied conda_prefix is not set, default to the default location + for Miniconda installs. + """ + if conda_prefix is None: + return os.path.join(os.path.expanduser("~"), "miniconda2") + return conda_prefix + + +class CondaContext(object): + + def __init__(self, conda_prefix=None, conda_exec=None, + shell_exec=None, debug=False, ensure_channels=''): + if conda_prefix is None: + conda_prefix = find_conda_prefix(conda_prefix) + self.conda_prefix = conda_prefix + if conda_exec is None: + conda_exec = self._bin("conda") + self.conda_exec = conda_exec + self.debug = debug + self.shell_exec = shell_exec or commands.shell + if ensure_channels: + if not isinstance(ensure_channels, list): + ensure_channels = [c for c in ensure_channels.split(",") if c] + + changed = False + conda_conf = self.load_condarc() + if "channels" not in conda_conf: + conda_conf["channels"] = [] + channels = conda_conf["channels"] + for channel in ensure_channels: + if channel not in channels: + changed = True + channels.append(channel) + + if changed: + self.save_condarc(conda_conf) + + def load_condarc(self): + condarc = self.condarc + if os.path.exists(condarc): + with open(condarc, "r") as f: + return yaml.safe_load(f) + else: + return {"channels": ["defaults"]} + + def save_condarc(self, conf): + condarc = self.condarc + with open(condarc, "w") as f: + return yaml.safe_dump(conf, f) + + @property + def condarc(self): + return os.path.join(os.path.expanduser("~"), ".condarc") + + def command(self, operation, args): + if isinstance(args, list): + args = " ".join(args) + conda_prefix = self.conda_exec + if self.debug: + conda_prefix = "%s --debug" + return "%s %s %s" % (conda_prefix, operation, args) + + def exec_command(self, operation, args): + command = self.command(operation, args) + self.shell_exec(command) + + def exec_create(self, args): + create_base_args = [ + "-y" + ] + create_base_args.extend(args) + return self.exec_command("create", create_base_args) + + def exec_install(self, args): + install_base_args = [ + "-y" + ] + install_base_args.extend(args) + return self.exec_command("install", install_base_args) + + def export_list(self, name, path): + return self.exec_command("list", [ + "--name", name, + "--export", ">", path + ]) + + def env_path(self, env_name): + return os.path.join(self.conda_prefix, "envs", env_name) + + def has_env(self, env_name): + env_path = self.env_path(env_name) + return os.path.isdir(env_path) + + @property + def deactivate(self): + return self._bin("deactivate") + + @property + def activate(self): + return self._bin("activate") + + def _bin(self, name): + return os.path.join(self.conda_prefix, "bin", name) + + +@six.python_2_unicode_compatible +class CondaTarget(object): + + def __init__(self, package, version=None, channel=None): + if SHELL_UNSAFE_PATTERN.search(package) is not None: + raise ValueError("Invalid package [%s] encountered." % package) + self.package = package + if version and SHELL_UNSAFE_PATTERN.search(version) is not None: + raise ValueError("Invalid version [%s] encountered." % version) + self.version = version + if channel and SHELL_UNSAFE_PATTERN.search(channel) is not None: + raise ValueError("Invalid version [%s] encountered." % channel) + self.channel = channel + + def __str__(self): + attributes = "package=%s" % self.package + if self.version is not None: + attributes = "%s,version=%s" % (self.package, self.version) + else: + attributes = "%s,unversioned" % self.package + + if self.channel: + attributes = "%s,channel=%s" % self.channel + + return "CondaTarget[%s]" % attributes + + @property + def package_specifier(self): + """ Return a package specifier as consumed by conda install/create. + """ + if self.version: + return "%s=%s" % (self.package, self.version) + else: + return self.package + + @property + def install_environment(self): + """ The dependency resolution and installation frameworks will + expect each target to be installed it its own environment with + a fixed and predictable name given package and version. + """ + if self.version: + return "__package__%s@__version__%s" % (self.package, self.version) + else: + return "__package__%s@__unversion__" % (self.package) + + +def hash_conda_packages(conda_packages, conda_target=None): + """ Produce a unique hash on supplied packages. + TODO: Ideally we would do this in such a way that preserved environments. + """ + h = hashlib.new('sha256') + for conda_package in conda_packages: + h.update(conda_package.install_environment) + return h.hexdigest() + + +def install_conda(conda_context=None): + conda_context = _ensure_conda_context(conda_context) + download_cmd = " ".join(commands.download_command(conda_link(), quote_url=True)) + download_cmd = "%s > /tmp/conda.bash" % download_cmd + install_cmd = "bash /tmp/conda.bash -b -p '%s'" % conda_context.conda_prefix + full_command = "%s; %s" % (download_cmd, install_cmd) + return conda_context.shell_exec(full_command) + + +def install_conda_target(conda_target, conda_context=None): + """ Install specified target into a its own environment. + """ + conda_context = _ensure_conda_context(conda_context) + create_args = [ + "--name", conda_target.install_environment, # enviornment for package + conda_target.package_specifier, + ] + conda_context.exec_create(create_args) + + +def is_conda_target_installed(conda_target, conda_context=None): + conda_context = _ensure_conda_context(conda_context) + return conda_context.has_env(conda_target.install_environment) + + +def filter_installed_targets(conda_targets, conda_context=None): + conda_context = _ensure_conda_context(conda_context) + installed = functools.partial(is_conda_target_installed, + conda_context=conda_context) + return filter(installed, conda_targets) + + +def build_isolated_environment( + conda_packages, + path=None, + copy=False, + conda_context=None, +): + """ Build a new environment (or reuse an existing one from hashes) + for specified conda packages. + """ + if not isinstance(conda_packages, list): + conda_packages = [conda_packages] + + # Lots we could do in here, hashing, checking revisions, etc... + conda_context = _ensure_conda_context(conda_context) + try: + hash = hash_conda_packages(conda_packages) + tempdir = tempfile.mkdtemp(prefix="jobdeps", suffix=hash) + tempdir_name = os.path.basename(tempdir) + + export_paths = [] + for conda_package in conda_packages: + name = conda_package.install_environment + export_path = os.path.join(tempdir, name) + conda_context.export_list( + name, + export_path + ) + export_paths.append(export_path) + create_args = ["--unknown", "--offline"] + if path is None: + create_args.extend(["--name", tempdir_name]) + else: + create_args.extend(["--prefix", path]) + + if copy: + create_args.append("--copy") + for export_path in export_paths: + create_args.extend([ + "--file", export_path, ">", "/dev/null" + ]) + + if path is not None and os.path.exists(path): + exit_code = conda_context.exec_install(create_args) + else: + exit_code = conda_context.exec_create(create_args) + + return (path or tempdir_name, exit_code) + finally: + shutil.rmtree(tempdir) + + +def requirement_to_conda_targets(requirement, conda_context=None): + conda_target = None + if requirement.type == "package": + conda_target = CondaTarget(requirement.name, + version=requirement.version) + return conda_target + + +def requirements_to_conda_targets(requirements, conda_context=None): + r_to_ct = functools.partial(requirement_to_conda_targets, + conda_context=conda_context) + conda_targets = map(r_to_ct, requirements) + return [c for c in conda_targets if c is not None] + + +def _ensure_conda_context(conda_context): + if conda_context is None: + conda_context = CondaContext() + return conda_context + + +__all__ = [ + 'CondaContext', + 'CondaTarget', + 'install_conda', + 'install_conda_target', + 'requirements_to_conda_targets', +] diff --git a/planemo_ext/galaxy/tools/deps/resolvers/__init__.py b/planemo_ext/galaxy/tools/deps/resolvers/__init__.py index 765e9d4da..2f5e1e6da 100644 --- a/planemo_ext/galaxy/tools/deps/resolvers/__init__.py +++ b/planemo_ext/galaxy/tools/deps/resolvers/__init__.py @@ -15,6 +15,18 @@ def resolve( self, name, version, type, **kwds ): version for instance if the request version is 'default'.) """ + def _get_config_option(self, key, dependency_resolver, default=None, prefix=None, **kwds): + """ Look in resolver-specific settings for option and then fallback to + global settings. + """ + global_key = "%s_%s" % (prefix, key) + if key in kwds: + return kwds.get(key) + elif global_key in dependency_resolver.extra_config: + return dependency_resolver.extra_config.get(global_key) + else: + return default + class Dependency( object ): __metaclass__ = ABCMeta diff --git a/planemo_ext/galaxy/tools/deps/resolvers/conda.py b/planemo_ext/galaxy/tools/deps/resolvers/conda.py new file mode 100644 index 000000000..2df46e51b --- /dev/null +++ b/planemo_ext/galaxy/tools/deps/resolvers/conda.py @@ -0,0 +1,128 @@ +""" +This is still an experimental module and there will almost certainly be backward +incompatible changes coming. +""" + + +import os + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from ..conda_util import ( + CondaContext, + CondaTarget, + install_conda, + is_conda_target_installed, + install_conda_target, + build_isolated_environment, +) + +DEFAULT_ENSURE_CHANNELS = "r,bioconda" + +import logging +log = logging.getLogger(__name__) + + +class CondaDependencyResolver(DependencyResolver): + resolver_type = "conda" + + def __init__(self, dependency_manager, **kwds): + self.versionless = _string_as_bool(kwds.get('versionless', 'false')) + + def get_option(name): + return self._get_config_option(name, dependency_manager, prefix="conda", **kwds) + + # Conda context options (these define the environment) + conda_prefix = get_option("prefix") + conda_exec = get_option("exec") + debug = _string_as_bool(get_option("debug")) + ensure_channels = get_option("ensure_channels") + if ensure_channels is None: + ensure_channels = DEFAULT_ENSURE_CHANNELS + + conda_context = CondaContext( + conda_prefix=conda_prefix, + conda_exec=conda_exec, + debug=debug, + ensure_channels=ensure_channels, + ) + + # Conda operations options (these define how resolution will occur) + auto_init = _string_as_bool(get_option("auto_init")) + auto_install = _string_as_bool(get_option("auto_install")) + copy_dependencies = _string_as_bool(get_option("copy_dependencies")) + + if auto_init and not os.path.exists(conda_context.conda_prefix): + install_conda(conda_context) + + self.conda_context = conda_context + self.auto_install = auto_install + self.copy_dependencies = copy_dependencies + + def resolve(self, name, version, type, **kwds): + # Check for conda just not being there, this way we can enable + # conda by default and just do nothing in not configured. + if not os.path.isdir(self.conda_context.conda_prefix): + return INDETERMINATE_DEPENDENCY + + if type != "package": + return INDETERMINATE_DEPENDENCY + + job_directory = kwds.get("job_directory", None) + if job_directory is None: + log.warn("Conda dependency resolver not sent job directory.") + return INDETERMINATE_DEPENDENCY + + if self.versionless: + version = None + + conda_target = CondaTarget(name, version=version) + is_installed = is_conda_target_installed( + conda_target, conda_context=self.conda_context + ) + if not is_installed and self.auto_install: + install_conda_target(conda_target) + + # Recheck if installed + is_installed = is_conda_target_installed( + conda_target, conda_context=self.conda_context + ) + + if not is_installed: + return INDETERMINATE_DEPENDENCY + + # Have installed conda_target and job_directory to send it too. + conda_environment = os.path.join(job_directory, "conda-env") + env_path, exit_code = build_isolated_environment( + conda_target, + path=conda_environment, + copy=self.copy_dependencies, + conda_context=self.conda_context, + ) + if not exit_code: + return CondaDepenency( + self.conda_context.activate, + conda_environment + ) + else: + raise Exception("Conda dependency seemingly installed but failed to build job environment.") + + +class CondaDepenency(): + + def __init__(self, activate, environment_path): + self.activate = activate + self.environment_path = environment_path + + def shell_commands(self, requirement): + return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || source %s '%s'""" % ( + self.environment_path, + self.activate, + self.environment_path + ) + + +def _string_as_bool( value ): + return str( value ).lower() == "true" + + +__all__ = ['CondaDependencyResolver'] diff --git a/project_templates/conda_testing/bwa.xml b/project_templates/conda_testing/bwa.xml new file mode 100644 index 000000000..e5dcd1a69 --- /dev/null +++ b/project_templates/conda_testing/bwa.xml @@ -0,0 +1,28 @@ + + + bwa + + + + + $output_1 2>&1 + ]]> + + + + + + + + + + + + + + + + diff --git a/project_templates/conda_testing/bwa_and_samtools.xml b/project_templates/conda_testing/bwa_and_samtools.xml new file mode 100644 index 000000000..f7bb752ff --- /dev/null +++ b/project_templates/conda_testing/bwa_and_samtools.xml @@ -0,0 +1,36 @@ + + + bwa + samtools + + + + + $output_1 2>&1 ; + samtools > $output_2 2>&1 + ]]> + + + + + + + + + + + + + + + + + + + + + + diff --git a/update_galaxy_utils.sh b/update_galaxy_utils.sh new file mode 100755 index 000000000..ed52af637 --- /dev/null +++ b/update_galaxy_utils.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +usage() { +cat << EOF +Usage: ${0##*/} [-i] /path/to/galaxy... +Sync Planemo shared modules to those same modules in Galaxy directory (or vice versa if -i). + +EOF +} + +if [ $# -lt 1 ]; then + usage + exit 1 +fi + +invert=0 +OPTIND=1 +while getopts ":i" opt; do + case "$opt" in + h) + usage + exit 0 + ;; + i) + invert=1 + ;; + '?') + usage >&2 + exit 1 + ;; + esac +done +shift "$((OPTIND-1))" # Shift off the options and optional --. + +PLANEMO_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +GALAXY_DIRECTORY=$1 +GALAXY_LIB_DIR=$GALAXY_DIRECTORY/lib/galaxy + +if [ "$invert" -ne "1" ]; +then + + rm -rf $GALAXY_LIB_DIR/objectstore + cp -r $PLANEMO_DIRECTORY/planemo_ext/galaxy/objectstore $GALAXY_LIB_DIR + + rm -rf $GALAXY_LIB_DIR/tools/deps + cp -r $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/deps $GALAXY_LIB_DIR/tools + + rm -rf $GALAXY_LIB_DIR/jobs/metrics + cp -r $PLANEMO_DIRECTORY/planemo_ext/galaxy/jobs/metrics $GALAXY_LIB_DIR/jobs + + rm -rf $GALAXY_LIB_DIR/tools/linters + cp -r $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/linters $GALAXY_LIB_DIR/tools + + cp $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/lint.py $GALAXY_LIB_DIR/tools/lint.py + cp $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/loader.py $GALAXY_LIB_DIR/tools/loader.py + cp $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/loader_directory.py $GALAXY_LIB_DIR/tools/loader_directory.py + + cp $PLANEMO_DIRECTORY/planemo_ext/galaxy/util/plugin_config.py $GALAXY_LIB_DIR/util + cp $PLANEMO_DIRECTORY/planemo_ext/galaxy/util/xml_macros.py $GALAXY_LIB_DIR/util + +else + + rm -rf $PLANEMO_DIRECTORY/planemo_ext/galaxy/objectstore + cp -r $GALAXY_LIB_DIR/objectstore $PLANEMO_DIRECTORY/planemo_ext/galaxy + + rm -rf $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/deps + cp -r $GALAXY_LIB_DIR/tools/deps $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools + + rm -rf $PLANEMO_DIRECTORY/planemo_ext/galaxy/jobs/metrics + cp -r $GALAXY_LIB_DIR/jobs/metrics $PLANEMO_DIRECTORY/planemo_ext/galaxy/jobs + + rm -rf $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools/linters + cp -r $GALAXY_LIB_DIR/tools/linters $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools + + cp $GALAXY_LIB_DIR/tools/lint.py $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools + cp $GALAXY_LIB_DIR/tools/loader.py $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools + cp $GALAXY_LIB_DIR/tools/loader_directory.py $PLANEMO_DIRECTORY/planemo_ext/galaxy/tools + cp $GALAXY_LIB_DIR/util/plugin_config.py $PLANEMO_DIRECTORY/planemo_ext/galaxy/util/ + cp $GALAXY_LIB_DIR/util/xml_macros.py $PLANEMO_DIRECTORY/planemo_ext/galaxy/util/ + +fi