Merge pull request #1345 from jmchilton/conda_0

Implement a conda dependency resolver.

config/dependency_resolvers_conf.xml.sample

@@ -7,6 +7,9 @@
      in Galaxy's config/galaxy.ini -->
   <galaxy_packages />
   <galaxy_packages versionless="true" />
+  <conda />
+  <conda versionless="true" />
+
 <!-- Example configuration of modules dependency resolver, uses Environment Modules -->
 <!--
   <modules modulecmd="/opt/Modules/3.2.9/bin/modulecmd" />

config/galaxy.ini.sample

@@ -172,6 +172,33 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # from the Tool Shed will fail.
 #tool_dependency_dir = None
 
+# The dependency resolves config file specifies an ordering and options for how 
+# Galaxy resolves tool dependencies (requirement tags in Tool XML). The default
+# ordering is to the use the tool shed for tools installed that way, use local
+# Galaxy packages, and then use conda if available.
+# See https://github.com/galaxyproject/galaxy/blob/dev/doc/source/admin/dependency_resolvers.rst
+# for more information on these options.
+#dependency_resolvers_config_file = config/dependency_resolvers_conf.xml
+
+# Following conda dependeny resolution options are experimental as of early
+# 2016. These will change the defaults for each conda resolver, but multiple
+# resolvers can be configured independently and these options overridden in
+# dependency_resolvers_config_file.
+#conda_prefix = <tool_dependency_dir>/_conda
+# Override the conda executable to use, it will default to the one on the
+# PATH (if available) and then to <conda_prefix>/bin/conda
+#conda_exec =
+# Pass debug flag to conda commands.
+#conda_debug = False
+# conda channels to enable by default (http://conda.pydata.org/docs/custom-channels.html)
+#conda_ensure_channels = r,bioconda
+# Set to True to instruct Galaxy to look for and install missing tool
+# dependencies before each job runs.
+#conda_auto_install = False
+# Set to True to instruct Galaxy to install conda from the web automatically
+# if it cannot find a local copy and conda_exec is not configured.
+#conda_auto_init = False
+
 # File containing the Galaxy Tool Sheds that should be made available to
 # install from in the admin interface (.sample used if default does not exist).
 #tool_sheds_config_file = config/tool_sheds_conf.xml

doc/source/admin/dependency_resolvers.rst

@@ -38,6 +38,8 @@ The default configuration of dependency resolvers is equivalent to the following
   <!-- then look for env.sh files profile according to the "galaxy packages" schema -->
     <galaxy_packages />
     <galaxy_packages versionless="true" />
+    <conda />
+    <conda versionless="true" />
   </dependency_resolvers>
 
 This default dependency resolver configuration contains three items. First, the *tool shed dependency resolver* is used,
@@ -144,5 +146,49 @@ is added to the script that is run to run the tool. E.g. ``modulecmd sh load bed
 used, they'll be used in the ``load`` command e.g. ``modulecmd sh load bwa/0.7.10.039ea20639``.
 
 
+Homebrew Dependency Resolver
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+This dependency resolver uses homebrew packages to resolve requirements.
 
+
+Brew Tool Shed Package Resolver
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This dependency resolver would resolve tool shed packages that had been
+auto converted to the tool shed. It is highly experimental, undocumented,
+and will almost certainy be removed from the code base.
+
+
+Conda Dependency Resolver
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The conda XML tag can be used to configure a conda dependency resolver.
+This resolver can be configured with the following options.
+
+prefix
+    The conda_prefix used to locate dependencies in (default: ``<tool_dependency_dir>/_conda``).
+
+exec
+    The conda executable to use, it will default to the one on the
+    PATH (if available) and then to ``<conda_prefix>/bin/conda``.
+
+versionless
+    whether to resolve tools using a version string or not (default: *false*)
+
+debug
+    Pass debug flag to conda commands (default: false).
+
+ensure_channels
+    conda channels to enable by default. See
+    http://conda.pydata.org/docs/custom-channels.html for more
+    information about channels. (default: r,bioconda).
+
+auto_install
+    Set to True to instruct Galaxy to look for and install missing tool
+    dependencies before each job runs. (default: False)
+
+auto_init
+    Set to True to instruct Galaxy to install conda from the web
+    automatically if it cannot find a local copy and conda_exec is not
+    configured.

lib/galaxy/jobs/__init__.py

@@ -863,7 +863,7 @@ def get_special( ):
         # Ensure galaxy_lib_dir is set in case there are any later chdirs
         self.galaxy_lib_dir
         # Shell fragment to inject dependencies
-        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
+        self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory)
         # We need command_line persisted to the db in order for Galaxy to re-queue the job
         # if the server was stopped and restarted before the job finished
         job.command_line = unicodify(self.command_line)
@@ -1751,7 +1751,7 @@ def prepare( self, compute_environment=None ):
         # Ensure galaxy_lib_dir is set in case there are any later chdirs
         self.galaxy_lib_dir
         # Shell fragment to inject dependencies
-        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
+        self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory)
         # We need command_line persisted to the db in order for Galaxy to re-queue the job
         # if the server was stopped and restarted before the job finished
         task.command_line = self.command_line

lib/galaxy/jobs/runners/__init__.py

@@ -264,7 +264,7 @@ def _handle_metadata_externally( self, job_wrapper, resolve_requirements=False )
                                                                             kwds={ 'overwrite' : False } )
             external_metadata_script = "%s %s %s" % (lib_adjust, venv, external_metadata_script)
             if resolve_requirements:
-                dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands()
+                dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands(job_directory=job_wrapper.working_directory)
                 if dependency_shell_commands:
                     if isinstance( dependency_shell_commands, list ):
                         dependency_shell_commands = "&&".join( dependency_shell_commands )

lib/galaxy/tools/__init__.py

@@ -1807,12 +1807,13 @@ def handle_job_failure_exception( self, e ):
             message = e.message
         return message
 
-    def build_dependency_shell_commands( self ):
+    def build_dependency_shell_commands( self, job_directory=None ):
         """Return a list of commands to be run to populate the current environment to include this tools requirements."""
         return self.app.toolbox.dependency_manager.dependency_shell_commands(
             self.requirements,
             installed_tool_dependencies=self.installed_tool_dependencies,
             tool_dir=self.tool_dir,
+            job_directory=job_directory,
         )
 
     @property

lib/galaxy/tools/deps/__init__.py

@@ -10,15 +10,37 @@
 from .resolvers import INDETERMINATE_DEPENDENCY
 from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
 from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
+from .resolvers.conda import CondaDependencyResolver
 from galaxy.util import plugin_config
 
+# TODO: Load these from the plugins. Would require a two step initialization of
+# DependencyManager - where the plugins are loaded first and then the config
+# is parsed and sent through.
+EXTRA_CONFIG_KWDS = {
+    'conda_prefix': None,
+    'conda_exec': None,
+    'conda_debug': None,
+    'conda_channels': 'r,bioconda',
+    'conda_auto_install': False,
+    'conda_auto_init': False,
+}
+
+CONFIG_VAL_NOT_FOUND = object()
+
 
 def build_dependency_manager( config ):
     if getattr( config, "use_tool_dependencies", False ):
         dependency_manager_kwds = {
             'default_base_path': config.tool_dependency_dir,
             'conf_file': config.dependency_resolvers_config_file,
         }
+        for key, default_value in EXTRA_CONFIG_KWDS.items():
+            value = getattr(config, key, CONFIG_VAL_NOT_FOUND)
+            if value is CONFIG_VAL_NOT_FOUND and hasattr(config, "config_dict"):
+                value = config.config_dict.get(key, CONFIG_VAL_NOT_FOUND)
+            if value is CONFIG_VAL_NOT_FOUND:
+                value = default_value
+            dependency_manager_kwds[key] = value
         dependency_manager = DependencyManager( **dependency_manager_kwds )
     else:
         dependency_manager = NullDependencyManager()
@@ -49,7 +71,7 @@ class DependencyManager( object ):
     and should each contain a file 'env.sh' which can be sourced to make the
     dependency available in the current shell environment.
     """
-    def __init__( self, default_base_path, conf_file=None ):
+    def __init__( self, default_base_path, conf_file=None, **extra_config ):
         """
         Create a new dependency manager looking for packages under the paths listed
         in `base_paths`.  The default base path is app.config.tool_dependency_dir.
@@ -58,6 +80,7 @@ def __init__( self, default_base_path, conf_file=None ):
             log.warn( "Path '%s' does not exist, ignoring", default_base_path )
         if not os.path.isdir( default_base_path ):
             log.warn( "Path '%s' is not directory, ignoring", default_base_path )
+        self.extra_config = extra_config
         self.default_base_path = os.path.abspath( default_base_path )
         self.resolver_classes = self.__resolvers_dict()
         self.dependency_resolvers = self.__build_dependency_resolvers( conf_file )
@@ -106,6 +129,8 @@ def __default_dependency_resolvers( self ):
             ToolShedPackageDependencyResolver(self),
             GalaxyPackageDependencyResolver(self),
             GalaxyPackageDependencyResolver(self, versionless=True),
+            CondaDependencyResolver(self),
+            CondaDependencyResolver(self, versionless=True),
         ]
 
     def __parse_resolver_conf_xml(self, plugin_source):