diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 0512b7700220..117b5b49b876 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -14,6 +14,7 @@
plugin_config
)
+from .requirements import ToolRequirement
from .resolvers import NullDependency
from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
@@ -107,38 +108,76 @@ def dependency_shell_commands( self, requirements, **kwds ):
def requirements_to_dependencies(self, requirements, **kwds):
"""
Takes a list of requirements and returns a dictionary
- with requirements as key and dependencies as value.
+ with requirements as key and dependencies as value caching
+ these on the tool instance if supplied.
"""
- requirement_to_dependency = OrderedDict()
- for requirement in requirements:
- if requirement.type in [ 'package', 'set_environment' ]:
- dependency = self.find_dep( name=requirement.name,
- version=requirement.version,
- type=requirement.type,
- **kwds )
- log.debug(dependency.resolver_msg)
- if dependency.dependency_type:
- requirement_to_dependency[requirement] = dependency
+ requirement_to_dependency = self._requirements_to_dependencies_dict(requirements, **kwds)
+
if 'tool_instance' in kwds:
kwds['tool_instance'].dependencies = [dep.to_dict() for dep in requirement_to_dependency.values()]
- return requirement_to_dependency
- def uses_tool_shed_dependencies(self):
- return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) )
+ return requirement_to_dependency
- def find_dep( self, name, version=None, type='package', **kwds ):
- log.debug('Find dependency %s version %s' % (name, version))
+ def _requirements_to_dependencies_dict(self, requirements, **kwds):
+ """Build simple requirements to dependencies dict for resolution."""
+ requirement_to_dependency = OrderedDict()
index = kwds.get('index', None)
require_exact = kwds.get('exact', False)
+
+ resolvable_requirements = []
+ for requirement in requirements:
+ if requirement.type in ['package', 'set_environment']:
+ resolvable_requirements.append(requirement)
+ else:
+ log.debug("Unresolvable requirement type [%s] found, will be ignored." % requirement.type)
+
for i, resolver in enumerate(self.dependency_resolvers):
if index is not None and i != index:
continue
- dependency = resolver.resolve( name, version, type, **kwds )
- if require_exact and not dependency.exact:
- continue
- if not isinstance(dependency, NullDependency):
- return dependency
- return NullDependency(version=version, name=name)
+
+ if len(requirement_to_dependency) == len(resolvable_requirements):
+ # Shortcut - resolution complete.
+ break
+
+ # Check requirements all at once
+ all_unmet = len(requirement_to_dependency) == 0
+ if all_unmet and hasattr(resolver, "resolve_all"):
+ dependencies = resolver.resolve_all(resolvable_requirements, **kwds)
+ if dependencies:
+ assert len(dependencies) == len(resolvable_requirements)
+ for requirement, dependency in zip(resolvable_requirements, dependencies):
+ requirement_to_dependency[requirement] = dependency
+
+ # Shortcut - resolution complete.
+ break
+
+ # Check individual requirements
+ for requirement in resolvable_requirements:
+ if requirement in requirement_to_dependency:
+ continue
+
+ if requirement.type in ['package', 'set_environment']:
+ dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds )
+ if require_exact and not dependency.exact:
+ continue
+
+ log.debug(dependency.resolver_msg)
+ if not isinstance(dependency, NullDependency):
+ requirement_to_dependency[requirement] = dependency
+
+ return requirement_to_dependency
+
+ def uses_tool_shed_dependencies(self):
+ return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) )
+
+ def find_dep( self, name, version=None, type='package', **kwds ):
+ log.debug('Find dependency %s version %s' % (name, version))
+ requirement = ToolRequirement(name=name, version=version, type=type)
+ dep_dict = self._requirements_to_dependencies_dict([requirement], **kwds)
+ if len(dep_dict) > 0:
+ return dep_dict.values()[0]
+ else:
+ return NullDependency(name=name, version=version)
def __build_dependency_resolvers( self, conf_file ):
if not conf_file:
diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py
index 496828f99afa..6632bd0fdd6c 100644
--- a/lib/galaxy/tools/deps/conda_util.py
+++ b/lib/galaxy/tools/deps/conda_util.py
@@ -364,6 +364,17 @@ def install_conda(conda_context=None):
os.remove(script_path)
+def install_conda_targets(conda_targets, env_name, conda_context=None):
+ conda_context = _ensure_conda_context(conda_context)
+ conda_context.ensure_channels_configured()
+ create_args = [
+ "--name", env_name, # enviornment for package
+ ]
+ for conda_target in conda_targets:
+ create_args.append(conda_target.package_specifier)
+ return conda_context.exec_create(create_args)
+
+
def install_conda_target(conda_target, conda_context=None):
""" Install specified target into a its own environment.
"""
@@ -376,10 +387,14 @@ def install_conda_target(conda_target, conda_context=None):
return conda_context.exec_create(create_args)
-def cleanup_failed_install(conda_target, conda_context=None):
+def cleanup_failed_install_of_environment(env, conda_context=None):
conda_context = _ensure_conda_context(conda_context)
- if conda_context.has_env(conda_target.install_environment):
- conda_context.exec_remove([conda_target.install_environment])
+ if conda_context.has_env(env):
+ conda_context.exec_remove([env])
+
+
+def cleanup_failed_install(conda_target, conda_context=None):
+ cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
def best_search_result(conda_target, conda_context=None, channels_override=None):
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index f384d1028e48..7ab8918ea484 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -11,10 +11,13 @@
from ..conda_util import (
build_isolated_environment,
cleanup_failed_install,
+ cleanup_failed_install_of_environment,
CondaContext,
CondaTarget,
+ hash_conda_packages,
install_conda,
install_conda_target,
+ install_conda_targets,
installed_conda_targets,
is_conda_target_installed,
USE_PATH_EXEC_DEFAULT,
@@ -102,6 +105,71 @@ def get_option(name):
def clean(self, **kwds):
return self.conda_context.exec_clean()
+ def install_all(self, conda_targets):
+ env = self.merged_environment_name(conda_targets)
+ return_code = install_conda_targets(conda_targets, env, conda_context=self.conda_context)
+ if return_code != 0:
+ is_installed = False
+ else:
+ # Recheck if installed
+ is_installed = self.conda_context.has_env(env)
+
+ if not is_installed:
+ log.debug("Removing failed conda install of {}".format(str(conda_targets)))
+ cleanup_failed_install_of_environment(env, conda_context=self.conda_context)
+
+ return is_installed
+
+ def resolve_all(self, requirements, **kwds):
+ if len(requirements) == 0:
+ return False
+
+ if not os.path.isdir(self.conda_context.conda_prefix):
+ return False
+
+ for requirement in requirements:
+ if requirement.type != "package":
+ return False
+
+ conda_targets = []
+ for requirement in requirements:
+ version = requirement.version
+ if self.versionless:
+ version = None
+
+ conda_targets.append(CondaTarget(requirement.name, version=version))
+
+ preserve_python_environment = kwds.get("preserve_python_environment", False)
+
+ env = self.merged_environment_name(conda_targets)
+ dependencies = []
+
+ is_installed = self.conda_context.has_env(env)
+ if not is_installed and (self.auto_install or kwds.get('install', False)):
+ is_installed = self.install_all(conda_targets)
+
+ if is_installed:
+ for requirement in requirements:
+ dependency = MergedCondaDependency(
+ self.conda_context,
+ self.conda_context.env_path(env),
+ exact=self.versionless or requirement.version is None,
+ name=requirement.name,
+ version=requirement.version,
+ preserve_python_environment=preserve_python_environment,
+ )
+ dependencies.append(dependency)
+
+ return dependencies
+
+ def merged_environment_name(self, conda_targets):
+ if len(conda_targets) > 1:
+ # For continuity with mulled containers this is kind of nice.
+ return "mulled-v1-%s" % hash_conda_packages(conda_targets)
+ else:
+ assert len(conda_targets) == 1
+ return conda_targets[0].install_environment
+
def resolve(self, name, version, type, **kwds):
# Check for conda just not being there, this way we can enable
# conda by default and just do nothing in not configured.
@@ -123,7 +191,7 @@ def resolve(self, name, version, type, **kwds):
preserve_python_environment = kwds.get("preserve_python_environment", False)
job_directory = kwds.get("job_directory", None)
- if not is_installed and self.auto_install and job_directory:
+ if not is_installed and (self.auto_install or kwds.get('install', False)):
is_installed = self.install_dependency(name=name, version=version, type=type)
if not is_installed:
@@ -193,6 +261,48 @@ def prefix(self):
return self.conda_context.conda_prefix
+class MergedCondaDependency(Dependency):
+ dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
+ dependency_type = 'conda'
+
+ def __init__(self, conda_context, environment_path, exact, name=None, version=None, preserve_python_environment=False):
+ self.activate = conda_context.activate
+ self.conda_context = conda_context
+ self.environment_path = environment_path
+ self._exact = exact
+ self._name = name
+ self._version = version
+ self.cache_path = None
+ self._preserve_python_environment = preserve_python_environment
+
+ @property
+ def exact(self):
+ return self._exact
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def version(self):
+ return self._version
+
+ def shell_commands(self, requirement):
+ if self._preserve_python_environment:
+ # On explicit testing the only such requirement I am aware of is samtools - and it seems to work
+ # fine with just appending the PATH as done below. Other tools may require additional
+ # variables in the future.
+ return """export PATH=$PATH:'%s/bin' """ % (
+ self.environment_path,
+ )
+ else:
+ return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
+ self.environment_path,
+ self.activate,
+ self.environment_path
+ )
+
+
class CondaDependency(Dependency):
dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
dependency_type = 'conda'
diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py
index 8f23335d7dfc..d37fed4fc8fe 100644
--- a/lib/galaxy/tools/deps/views.py
+++ b/lib/galaxy/tools/deps/views.py
@@ -45,6 +45,9 @@ def manager_dependency(self, **kwds):
def resolver_dependency(self, index, **kwds):
return self._dependency(**kwds)
+ def install_dependencies(self, requirements):
+ return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True})
+
def install_dependency(self, index=None, **payload):
"""
Installs dependency using highest priority resolver that supports dependency installation
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index b76dd9a3f406..d640bdae10d9 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -141,11 +141,11 @@ def install_dependencies(self, trans, id, **kwds):
force_rebuild: If true and chache dir exists, attempts to delete cache dir
"""
tool = self._get_tool(id)
- [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements]
+ tool._view.install_dependencies(tool.requirements)
if kwds.get('build_dependency_cache'):
tool.build_dependency_cache(**kwds)
# TODO: rework resolver install system to log and report what has been done.
- # _view.install_dependency should return a dict with stdout, stderr and success status
+ # _view.install_dependencies should return a dict with stdout, stderr and success status
return tool.tool_requirements_status
@expose_api
diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 7da8c6e40e7f..eaaa0469d21d 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -905,15 +905,15 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, to
if 'tools' in metadata and install_resolver_dependencies:
self.update_tool_shed_repository_status( tool_shed_repository,
self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
- requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
- [self._view.install_dependency(id=None, **req) for req in requirements]
- if self.app.config.use_cached_dependency_manager:
- cached_requirements = []
- for tool_d in metadata['tools']:
- tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
- if tool and tool.requirements not in cached_requirements:
- cached_requirements.append(tool.requirements)
+ installed_requirements = []
+ for tool_d in metadata['tools']:
+ tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
+ if tool and tool.requirements not in installed_requirements:
+ self._view.install_dependencies(tool.requirements)
+ installed_requirements.append(tool.requirements)
+ if self.app.config.use_cached_dependency_manager:
tool.build_dependency_cache()
+
if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" )
# Install tool dependencies.
diff --git a/test/functional/tools/mulled_example_conflict.xml b/test/functional/tools/mulled_example_conflict.xml
new file mode 100644
index 000000000000..4df76c2fab17
--- /dev/null
+++ b/test/functional/tools/mulled_example_conflict.xml
@@ -0,0 +1,17 @@
+
+
+ &1 | grep -q structural
+ ]]>
+
+ lumpy-sv
+ numpy
+
+
+
+
+
+
+
+
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index 58ecb3b6f5aa..ab7ed8220548 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -123,6 +123,7 @@
+