Merge pull request #3391 from jmchilton/resolve_all_at_once

Resolve Conda Dependencies All at Once

lib/galaxy/tools/deps/__init__.py

@@ -14,6 +14,7 @@
     plugin_config
 )
 
+from .requirements import ToolRequirement
 from .resolvers import NullDependency
 from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
 from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
@@ -107,38 +108,76 @@ def dependency_shell_commands( self, requirements, **kwds ):
     def requirements_to_dependencies(self, requirements, **kwds):
         """
         Takes a list of requirements and returns a dictionary
-        with requirements as key and dependencies as value.
+        with requirements as key and dependencies as value caching
+        these on the tool instance if supplied.
         """
-        requirement_to_dependency = OrderedDict()
-        for requirement in requirements:
-            if requirement.type in [ 'package', 'set_environment' ]:
-                dependency = self.find_dep( name=requirement.name,
-                                            version=requirement.version,
-                                            type=requirement.type,
-                                            **kwds )
-                log.debug(dependency.resolver_msg)
-                if dependency.dependency_type:
-                    requirement_to_dependency[requirement] = dependency
+        requirement_to_dependency = self._requirements_to_dependencies_dict(requirements, **kwds)
+
         if 'tool_instance' in kwds:
             kwds['tool_instance'].dependencies = [dep.to_dict() for dep in requirement_to_dependency.values()]
-        return requirement_to_dependency
 
-    def uses_tool_shed_dependencies(self):
-        return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) )
+        return requirement_to_dependency
 
-    def find_dep( self, name, version=None, type='package', **kwds ):
-        log.debug('Find dependency %s version %s' % (name, version))
+    def _requirements_to_dependencies_dict(self, requirements, **kwds):
+        """Build simple requirements to dependencies dict for resolution."""
+        requirement_to_dependency = OrderedDict()
         index = kwds.get('index', None)
         require_exact = kwds.get('exact', False)
+
+        resolvable_requirements = []
+        for requirement in requirements:
+            if requirement.type in ['package', 'set_environment']:
+                resolvable_requirements.append(requirement)
+            else:
+                log.debug("Unresolvable requirement type [%s] found, will be ignored." % requirement.type)
+
         for i, resolver in enumerate(self.dependency_resolvers):
             if index is not None and i != index:
                 continue
-            dependency = resolver.resolve( name, version, type, **kwds )
-            if require_exact and not dependency.exact:
-                continue
-            if not isinstance(dependency, NullDependency):
-                return dependency
-        return NullDependency(version=version, name=name)
+
+            if len(requirement_to_dependency) == len(resolvable_requirements):
+                # Shortcut - resolution complete.
+                break
+
+            # Check requirements all at once
+            all_unmet = len(requirement_to_dependency) == 0
+            if all_unmet and hasattr(resolver, "resolve_all"):
+                dependencies = resolver.resolve_all(resolvable_requirements, **kwds)
+                if dependencies:
+                    assert len(dependencies) == len(resolvable_requirements)
+                    for requirement, dependency in zip(resolvable_requirements, dependencies):
+                        requirement_to_dependency[requirement] = dependency
+
+                    # Shortcut - resolution complete.
+                    break
+
+            # Check individual requirements
+            for requirement in resolvable_requirements:
+                if requirement in requirement_to_dependency:
+                    continue
+
+                if requirement.type in ['package', 'set_environment']:
+                    dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds )
+                    if require_exact and not dependency.exact:
+                        continue
+
+                    log.debug(dependency.resolver_msg)
+                    if not isinstance(dependency, NullDependency):
+                        requirement_to_dependency[requirement] = dependency
+
+        return requirement_to_dependency
+
+    def uses_tool_shed_dependencies(self):
+        return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) )
+
+    def find_dep( self, name, version=None, type='package', **kwds ):
+        log.debug('Find dependency %s version %s' % (name, version))
+        requirement = ToolRequirement(name=name, version=version, type=type)
+        dep_dict = self._requirements_to_dependencies_dict([requirement], **kwds)
+        if len(dep_dict) > 0:
+            return dep_dict.values()[0]
+        else:
+            return NullDependency(name=name, version=version)
 
     def __build_dependency_resolvers( self, conf_file ):
         if not conf_file:

lib/galaxy/tools/deps/conda_util.py

@@ -364,6 +364,17 @@ def install_conda(conda_context=None):
             os.remove(script_path)
 
 
+def install_conda_targets(conda_targets, env_name, conda_context=None):
+    conda_context = _ensure_conda_context(conda_context)
+    conda_context.ensure_channels_configured()
+    create_args = [
+        "--name", env_name,  # enviornment for package
+    ]
+    for conda_target in conda_targets:
+        create_args.append(conda_target.package_specifier)
+    return conda_context.exec_create(create_args)
+
+
 def install_conda_target(conda_target, conda_context=None):
     """ Install specified target into a its own environment.
     """
@@ -376,10 +387,14 @@ def install_conda_target(conda_target, conda_context=None):
     return conda_context.exec_create(create_args)
 
 
-def cleanup_failed_install(conda_target, conda_context=None):
+def cleanup_failed_install_of_environment(env, conda_context=None):
     conda_context = _ensure_conda_context(conda_context)
-    if conda_context.has_env(conda_target.install_environment):
-        conda_context.exec_remove([conda_target.install_environment])
+    if conda_context.has_env(env):
+        conda_context.exec_remove([env])
+
+
+def cleanup_failed_install(conda_target, conda_context=None):
+    cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
 
 
 def best_search_result(conda_target, conda_context=None, channels_override=None):

lib/galaxy/tools/deps/resolvers/conda.py

@@ -11,10 +11,13 @@
 from ..conda_util import (
     build_isolated_environment,
     cleanup_failed_install,
+    cleanup_failed_install_of_environment,
     CondaContext,
     CondaTarget,
+    hash_conda_packages,
     install_conda,
     install_conda_target,
+    install_conda_targets,
     installed_conda_targets,
     is_conda_target_installed,
     USE_PATH_EXEC_DEFAULT,
@@ -102,6 +105,71 @@ def get_option(name):
     def clean(self, **kwds):
         return self.conda_context.exec_clean()
 
+    def install_all(self, conda_targets):
+        env = self.merged_environment_name(conda_targets)
+        return_code = install_conda_targets(conda_targets, env, conda_context=self.conda_context)
+        if return_code != 0:
+            is_installed = False
+        else:
+            # Recheck if installed
+            is_installed = self.conda_context.has_env(env)
+
+        if not is_installed:
+            log.debug("Removing failed conda install of {}".format(str(conda_targets)))
+            cleanup_failed_install_of_environment(env, conda_context=self.conda_context)
+
+        return is_installed
+
+    def resolve_all(self, requirements, **kwds):
+        if len(requirements) == 0:
+            return False
+
+        if not os.path.isdir(self.conda_context.conda_prefix):
+            return False
+
+        for requirement in requirements:
+            if requirement.type != "package":
+                return False
+
+        conda_targets = []
+        for requirement in requirements:
+            version = requirement.version
+            if self.versionless:
+                version = None
+
+            conda_targets.append(CondaTarget(requirement.name, version=version))
+
+        preserve_python_environment = kwds.get("preserve_python_environment", False)
+
+        env = self.merged_environment_name(conda_targets)
+        dependencies = []
+
+        is_installed = self.conda_context.has_env(env)
+        if not is_installed and (self.auto_install or kwds.get('install', False)):
+            is_installed = self.install_all(conda_targets)
+
+        if is_installed:
+            for requirement in requirements:
+                dependency = MergedCondaDependency(
+                    self.conda_context,
+                    self.conda_context.env_path(env),
+                    exact=self.versionless or requirement.version is None,
+                    name=requirement.name,
+                    version=requirement.version,
+                    preserve_python_environment=preserve_python_environment,
+                )
+                dependencies.append(dependency)
+
+        return dependencies
+
+    def merged_environment_name(self, conda_targets):
+        if len(conda_targets) > 1:
+            # For continuity with mulled containers this is kind of nice.
+            return "mulled-v1-%s" % hash_conda_packages(conda_targets)
+        else:
+            assert len(conda_targets) == 1
+            return conda_targets[0].install_environment
+
     def resolve(self, name, version, type, **kwds):
         # Check for conda just not being there, this way we can enable
         # conda by default and just do nothing in not configured.
@@ -123,7 +191,7 @@ def resolve(self, name, version, type, **kwds):
         preserve_python_environment = kwds.get("preserve_python_environment", False)
 
         job_directory = kwds.get("job_directory", None)
-        if not is_installed and self.auto_install and job_directory:
+        if not is_installed and (self.auto_install or kwds.get('install', False)):
             is_installed = self.install_dependency(name=name, version=version, type=type)
 
         if not is_installed:
@@ -193,6 +261,48 @@ def prefix(self):
         return self.conda_context.conda_prefix
 
 
+class MergedCondaDependency(Dependency):
+    dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
+    dependency_type = 'conda'
+
+    def __init__(self, conda_context, environment_path, exact, name=None, version=None, preserve_python_environment=False):
+        self.activate = conda_context.activate
+        self.conda_context = conda_context
+        self.environment_path = environment_path
+        self._exact = exact
+        self._name = name
+        self._version = version
+        self.cache_path = None
+        self._preserve_python_environment = preserve_python_environment
+
+    @property
+    def exact(self):
+        return self._exact
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def version(self):
+        return self._version
+
+    def shell_commands(self, requirement):
+        if self._preserve_python_environment:
+            # On explicit testing the only such requirement I am aware of is samtools - and it seems to work
+            # fine with just appending the PATH as done below. Other tools may require additional
+            # variables in the future.
+            return """export PATH=$PATH:'%s/bin' """ % (
+                self.environment_path,
+            )
+        else:
+            return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
+                self.environment_path,
+                self.activate,
+                self.environment_path
+            )
+
+
 class CondaDependency(Dependency):
     dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
     dependency_type = 'conda'

lib/galaxy/tools/deps/views.py

@@ -45,6 +45,9 @@ def manager_dependency(self, **kwds):
     def resolver_dependency(self, index, **kwds):
         return self._dependency(**kwds)
 
+    def install_dependencies(self, requirements):
+        return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True})
+
     def install_dependency(self, index=None, **payload):
         """
         Installs dependency using highest priority resolver that supports dependency installation

lib/galaxy/webapps/galaxy/api/tools.py

@@ -141,11 +141,11 @@ def install_dependencies(self, trans, id, **kwds):
             force_rebuild:           If true and chache dir exists, attempts to delete cache dir
         """
         tool = self._get_tool(id)
-        [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements]
+        tool._view.install_dependencies(tool.requirements)
         if kwds.get('build_dependency_cache'):
             tool.build_dependency_cache(**kwds)
         # TODO: rework resolver install system to log and report what has been done.
-        # _view.install_dependency should return a dict with stdout, stderr and success status
+        # _view.install_dependencies should return a dict with stdout, stderr and success status
         return tool.tool_requirements_status
 
     @expose_api

lib/tool_shed/galaxy_install/install_manager.py

@@ -905,15 +905,15 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, to
             if 'tools' in metadata and install_resolver_dependencies:
                 self.update_tool_shed_repository_status( tool_shed_repository,
                                                          self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
-                requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
-                [self._view.install_dependency(id=None, **req) for req in requirements]
-                if self.app.config.use_cached_dependency_manager:
-                    cached_requirements = []
-                    for tool_d in metadata['tools']:
-                        tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
-                        if tool and tool.requirements not in cached_requirements:
-                            cached_requirements.append(tool.requirements)
+                installed_requirements = []
+                for tool_d in metadata['tools']:
+                    tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
+                    if tool and tool.requirements not in installed_requirements:
+                        self._view.install_dependencies(tool.requirements)
+                        installed_requirements.append(tool.requirements)
+                        if self.app.config.use_cached_dependency_manager:
                             tool.build_dependency_cache()
+
             if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
                 work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" )
                 # Install tool dependencies.

test/functional/tools/mulled_example_conflict.xml

@@ -0,0 +1,17 @@
+<tool id="mulled_example_conflict" name="mulled_example_conflict" version="0.1.0">
+    <!-- Tool can't be resolved with older style Conda resolution - all requirements
+         must be installed together to ensure there is no conflict. -->
+    <command detect_errors="exit_code"><![CDATA[
+        lumpy 2>&1 | grep -q structural
+    ]]></command>
+    <requirements>
+        <requirement type="package" version="0.2.12">lumpy-sv</requirement>
+        <requirement type="package" version="1.11.2">numpy</requirement>
+    </requirements>
+    <inputs>
+        <param name="input1" type="text" value="The value" />
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -123,6 +123,7 @@
   </section>
 
   <tool file="mulled_example_multi_1.xml" />
+  <tool file="mulled_example_conflict.xml" />
 
   <tool file="simple_constructs.yml" />