Merge branch 'dev' into ui_pref_routing

config/dependency_resolvers_conf.xml.sample

@@ -1,28 +1,32 @@
 <dependency_resolvers>
-<!-- the default configuration, first look for dependencies installed from the toolshed -->
+  <!-- the default configuration, first look for dependencies installed from the toolshed -->
   <tool_shed_packages />
-<!-- then look for env.sh files in directories according to the "galaxy packages" schema.
-     These resolvers can take a base_path attribute to specify where to look for
-     package definitions, but by default look in the directory specified by tool_dependency_dir
-     in Galaxy's config/galaxy.ini -->
+  <!-- then look for env.sh files in directories according to the "galaxy packages" schema.
+       These resolvers can take a base_path attribute to specify where to look for
+       package definitions, but by default look in the directory specified by tool_dependency_dir
+       in Galaxy's config/galaxy.ini -->
   <galaxy_packages />
-  <galaxy_packages versionless="true" />
+  <!-- check whether the correct version has been installed via conda -->
   <conda />
+  <!-- look for a "default" symlink pointing to a directory containing an
+       env.sh file for the package in the "galaxy packages" schema -->
+  <galaxy_packages versionless="true" />
+  <!-- look for any version of the dependency installed via conda -->
   <conda versionless="true" />
 
-<!-- Example configuration of modules dependency resolver, uses Environment Modules -->
-<!--
+  <!-- Example configuration of modules dependency resolver, uses Environment Modules -->
+  <!--
   <modules modulecmd="/opt/Modules/3.2.9/bin/modulecmd" />
   <modules modulecmd="/opt/Modules/3.2.9/bin/modulecmd" versionless="true" default_indicator="default" />
-Attributes are:
-* modulecmd - path to modulecmd
-* versionless - default: false - whether to resolve tools using a version number or not
-* find_by - directory or avail - use the DirectoryModuleChecker or AvailModuleChecker
-* prefetch - default: true - in the AvailModuleChecker prefetch module info with 'module avail'
-* default_indicator - default: '(default)' - what indicate to the AvailModuleChecker that a module is the default version
--->
-<!-- other resolvers
+  Attributes are:
+  * modulecmd - path to modulecmd
+  * versionless - default: false - whether to resolve tools using a version number or not
+  * find_by - directory or avail - use the DirectoryModuleChecker or AvailModuleChecker
+  * prefetch - default: true - in the AvailModuleChecker prefetch module info with 'module avail'
+  * default_indicator - default: '(default)' - what indicate to the AvailModuleChecker that a module is the default version
+  -->
+  <!-- other resolvers
   <tool_shed_tap />
   <homebrew />
--->
-</dependency_resolvers>
+  -->
+</dependency_resolvers>

doc/source/admin/dependency_resolvers.rst

@@ -6,10 +6,12 @@ Dependency Resolvers in Galaxy
 
 There are two parts to building a link between Galaxy and command line bioinformatics tools: the tool XML that
 specifies a mapping between the Galaxy web user interface and the tool command line and tool dependencies that specify
-how to source the actual packages that implement the tool’s commands. The final script that Galaxy submits to run a
-job uses includes commands, such as changes to the ``PATH`` environment variable, that are generated by *dependency
-resolvers*. There is a default dependency resolver configuration but administrators can provide their own configuration
-using the ``dependency_resolvers_conf.xml`` configuration file in the Galaxy ``config/`` directory.
+how to source the actual packages that implement the tool’s commands. The final script that Galaxy submits to run a job
+uses includes commands, such as changes to the ``PATH`` environment variable, that are generated by *dependency
+resolvers*.  These same dependency resolvers are used by the Galaxy administrative UI to display whether an installed
+tool's dependencies have been installed on the Galaxy server, and to show how they will be resolved at job runtime.
+There is a default dependency resolver configuration but administrators can provide their own configuration using the
+``dependency_resolvers_conf.xml`` configuration file in the Galaxy ``config/`` directory.
 
 The binding between tool XML and the tools they need to run is specified in the tool XML using ``<requirement>``
 tags, for example
@@ -34,30 +36,31 @@ The default configuration of dependency resolvers is equivalent to the following
 .. code-block:: xml
 
   <dependency_resolvers>
-    <!-- the default configuration, first look for legacy dependencies installed from the toolshed -->
     <tool_shed_packages />
-    <!-- then look for env.sh files profile according to the "galaxy packages" schema -->
     <galaxy_packages />
-    <galaxy_packages versionless="true" />
-    <!-- finally look for Conda dependencies. -->
     <conda />
+    <galaxy_packages versionless="true" />
     <conda versionless="true" />
   </dependency_resolvers>
 
-This default dependency resolver configuration contains five items. First, the *tool shed dependency resolver* is used,
-then the *Galaxy packages dependency resolver* is used (initially looking for packages by name and version string and then looking for the package just by name), and finally it checks *Conda* for a versioned or unversioned match.
-The default configuration thus prefers packages installed from the Galaxy Tool Shed using legacy ``tool_dependencies.xml``
-files, before trying to find a "Galaxy package" satisfying the specific version the dependency requires before
-falling back to looking for a Galaxy package with merely the correct name, and then looking for Conda recipes with
-matching name and version, and finally just for a Conda package with the correct name. If any of the dependency
-resolvers succeeds a dependency resolution object is returned and no more resolvers are called. This dependency
-resolution object provides shell commands to prepend to the shell script that runs the tool.
+This default dependency resolver configuration contains five items:
+
+1. First, the *Tool Shed dependency resolver* is used, which resolves packages installed from the Galaxy Tool Shed
+   using legacy ``tool_dependencies.xml`` files,
+2. then the *Galaxy packages dependency resolver* is checked for a package matching the requirement name and version,
+3. then the *Conda dependency resolver* is checked for a package matching the requirement name and version. If no
+   versioned match can be found, it then moves on to searching for unversioned matches, that is,
+4. the *Galaxy packages dependency resolver* is checked for a package matching the required name only, and
+5. finally the *Conda dependency resolver* is checked for a  package matching the required name only.
+
+If any of the dependency resolvers succeeds a dependency resolution object is returned and no more resolvers are
+called. This dependency resolution object provides shell commands to prepend to the shell script that runs the tool.
 
 This order can be thought of as a descending order of deliberation. Tool Shed dependencies must be declared next to the
 tool by the tool author and must be selected for installation at tool installation time - this requires specific actions
-by both the tool author and the deployer who installed the tools. The dependency is therefore expected to highly craft
-to the individual tool. If Galaxy packages have been setup, the deployer of a Galaxy tool has purposely crafted tool
-dependency statements for a specific installation - this is slightly less deliberate than tool shed packages but
+by both the tool author and the deployer who installed the tools. The dependency is therefore expected to highly
+crafted to the individual tool. If Galaxy packages have been setup, the deployer of a Galaxy tool has purposely crafted
+tool dependency statements for a specific installation - this is slightly less deliberate than tool shed packages but
 such requirements are less likely to be incidentally resolved than Conda packages. Conda recipes are neither tied to
 tools or a specific installation and are maintained in Conda channels such as Bioconda.
 

lib/galaxy/tools/deps/__init__.py

@@ -198,8 +198,8 @@ def __default_dependency_resolvers( self ):
         return [
             ToolShedPackageDependencyResolver(self),
             GalaxyPackageDependencyResolver(self),
-            GalaxyPackageDependencyResolver(self, versionless=True),
             CondaDependencyResolver(self),
+            GalaxyPackageDependencyResolver(self, versionless=True),
             CondaDependencyResolver(self, versionless=True),
         ]
 

lib/galaxy/tools/deps/conda_util.py

@@ -237,6 +237,7 @@ def exec_create(self, args):
         return self.exec_command("create", create_base_args)
 
     def exec_remove(self, args):
+        """Remove a conda environment using conda env remove -y --name `args`."""
         remove_base_args = [
             "remove",
             "-y",

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -19,14 +19,15 @@ class DependencyResolver(Dictifiable, object):
     """Abstract description of a technique for resolving container images for tool execution."""
 
     # Keys for dictification.
-    dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies']
+    dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies', 'can_uninstall_dependencies']
     # A "simple" dependency is one that does not depend on the the tool
     # resolving the dependency. Classic tool shed dependencies are non-simple
     # because the repository install context is used in dependency resolution
     # so the same requirement tags in different tools will have very different
     # resolution.
     disabled = False
     resolves_simple_dependencies = True
+    can_uninstall_dependencies = False
     config_options = {}
 
     @abstractmethod

lib/galaxy/tools/deps/resolvers/conda.py

@@ -59,6 +59,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
     _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
 
     def __init__(self, dependency_manager, **kwds):
+        self.can_uninstall_dependencies = True
         self._setup_mapping(dependency_manager, **kwds)
         self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
         self.dependency_manager = dependency_manager
@@ -116,6 +117,25 @@ def get_option(name):
     def clean(self, **kwds):
         return self.conda_context.exec_clean()
 
+    def uninstall(self, requirements):
+        """Uninstall requirements installed by install_all or multiple install statements."""
+        all_resolved = [r for r in self.resolve_all(requirements) if r.dependency_type]
+        if not all_resolved:
+            all_resolved = [self.resolve(requirement) for requirement in requirements]
+            all_resolved = [r for r in all_resolved if r.dependency_type]
+        if not all_resolved:
+            return None
+        environments = set([os.path.basename(dependency.environment_path) for dependency in all_resolved])
+        return_codes = [self.conda_context.exec_remove([env]) for env in environments]
+        final_return_code = 0
+        for env, return_code in zip(environments, return_codes):
+            if return_code == 0:
+                log.debug("Conda environment '%s' sucessfully removed." % env)
+            else:
+                log.debug("Conda environment '%s' could not be removed." % env)
+                final_return_code = return_code
+        return final_return_code
+
     def install_all(self, conda_targets):
         env = self.merged_environment_name(conda_targets)
         return_code = install_conda_targets(conda_targets, env, conda_context=self.conda_context)

lib/galaxy/tools/deps/views.py

@@ -55,6 +55,22 @@ def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=Non
         dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()}
         return dependencies_per_tool
 
+    def uninstall_dependencies(self, index=None, **payload):
+        """Attempt to uninstall requirements. Returns 0 if successfull, else None."""
+        requirements = payload.get('requirements')
+        if not requirements:
+            return None
+        if index:
+            resolver = self._dependency_resolvers[index]
+            if resolver.can_uninstall_dependencies:
+                return resolver.uninstall(requirements)
+        else:
+            for index in self.uninstallable_resolvers:
+                return_code = self._dependency_resolvers[index].uninstall(requirements)
+                if return_code == 0:
+                    return return_code
+        return None
+
     def install_dependencies(self, requirements):
         return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True})
 
@@ -134,10 +150,17 @@ def _dependency_resolvers(self):
     @property
     def installable_resolvers(self):
         """
-        List index for all active resolvers that have the 'install_dependency' attribute
+        List index for all active resolvers that have the 'install_dependency' attribute.
         """
         return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ]
 
+    @property
+    def uninstallable_resolvers(self):
+        """
+        List index for all active resolvers that can uninstall dependencies that have been installed through this resolver.
+        """
+        return [index for index, resolver in enumerate(self._dependency_resolvers) if resolver.can_uninstall_dependencies and not resolver.disabled]
+
     def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None):
         dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies)
         # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool.

lib/galaxy/web/base/controllers/admin.py

@@ -1122,15 +1122,20 @@ def job_info( self, trans, jobid=None ):
 
     @web.expose
     @web.require_admin
-    def manage_tool_dependencies( self, trans, install_dependencies=False, build_cache=False, install_for_tools=[], viewkey='View tool-centric dependencies'):
+    def manage_tool_dependencies( self, trans, install_dependencies=False, uninstall_dependencies=False, selected_tool_ids=None, viewkey='View tool-centric dependencies'):
+        if not selected_tool_ids:
+            selected_tool_ids = []
         tools_by_id = trans.app.toolbox.tools_by_id
-        if install_dependencies:
-            # install the dependencies for the tools in the install_for_tools list
-            if not isinstance(install_for_tools, list):
-                install_for_tools = [install_for_tools]
-            requirements = set([tools_by_id[tid].tool_requirements for tid in install_for_tools])
-            [trans.app.toolbox.tools_by_id[install_for_tools[0]]._view.install_dependencies(r) for r in requirements]
         view = six.next(six.itervalues(trans.app.toolbox.tools_by_id))._view
+        if selected_tool_ids:
+            # install the dependencies for the tools in the selected_tool_ids list
+            if not isinstance(selected_tool_ids, list):
+                selected_tool_ids = [selected_tool_ids]
+            requirements = set([tools_by_id[tid].tool_requirements for tid in selected_tool_ids])
+            if install_dependencies:
+                [view.install_dependencies(r) for r in requirements]
+            elif uninstall_dependencies:
+                [view.uninstall_dependencies(index=None, requirements=r) for r in requirements]
         tool_ids_by_requirements = {}
         for tid, tool in trans.app.toolbox.tools_by_id.items():
             if tool.tool_requirements not in tool_ids_by_requirements:

lib/galaxy/webapps/galaxy/api/tools.py

@@ -135,12 +135,16 @@ def requirements(self, trans, id, **kwds):
     @web.require_admin
     def install_dependencies(self, trans, id, **kwds):
         """
-        POST /api/tools/{tool_id}/install_dependencies
+        POST /api/tools/{tool_id}/dependencies
+
+        This endpoint is also available through POST /api/tools/{tool_id}/install_dependencies,
+        but will be deprecated in the future.
+
         Attempts to install requirements via the dependency resolver
 
         parameters:
             build_dependency_cache:  If true, attempts to cache dependencies for this tool
-            force_rebuild:           If true and chache dir exists, attempts to delete cache dir
+            force_rebuild:           If true and cache dir exists, attempts to delete cache dir
         """
         tool = self._get_tool(id)
         tool._view.install_dependencies(tool.requirements)
@@ -150,6 +154,19 @@ def install_dependencies(self, trans, id, **kwds):
         # _view.install_dependencies should return a dict with stdout, stderr and success status
         return tool.tool_requirements_status
 
+    @expose_api
+    @web.require_admin
+    def uninstall_dependencies(self, trans, id, **kwds):
+        """
+        DELETE /api/tools/{tool_id}/dependencies
+        Attempts to uninstall requirements via the dependency resolver
+
+        """
+        tool = self._get_tool(id)
+        tool._view.uninstall_dependencies(index=None, requirements=tool.requirements)
+        # TODO: rework resolver install system to log and report what has been done.
+        return tool.tool_requirements_status
+
     @expose_api
     @web.require_admin
     def build_dependency_cache(self, trans, id, **kwds):

lib/galaxy/webapps/galaxy/buildapp.py

@@ -270,6 +270,8 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.connect( '/api/tools/{id:.+?}/download', action='download', controller="tools" )
     webapp.mapper.connect( '/api/tools/{id:.+?}/requirements', action='requirements', controller="tools")
     webapp.mapper.connect( '/api/tools/{id:.+?}/install_dependencies', action='install_dependencies', controller="tools", conditions=dict( method=[ "POST" ] ))
+    webapp.mapper.connect( '/api/tools/{id:.+?}/dependencies', action='install_dependencies', controller="tools", conditions=dict( method=[ "POST" ] ))
+    webapp.mapper.connect( '/api/tools/{id:.+?}/dependencies', action='uninstall_dependencies', controller="tools", conditions=dict( method=[ "DELETE" ] ))
     webapp.mapper.connect( '/api/tools/{id:.+?}/build_dependency_cache', action='build_dependency_cache', controller="tools", conditions=dict( method=[ "POST" ] ))
     webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" )
     webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' )

templates/webapps/galaxy/admin/manage_dependencies.mako

@@ -24,7 +24,7 @@
 
 <%def name="render_tool_centric_table( tools, requirements_status)">
     <tr>
-        <th bgcolor="#D8D8D8">Install</th>
+        <th bgcolor="#D8D8D8">Select</th>
         <th bgcolor="#D8D8D8">Name</th>
         <th bgcolor="#D8D8D8">ID</th>
         <th bgcolor="#D8D8D8">Requirement</th>
@@ -42,7 +42,7 @@
                 <tr class="tr">
             %endif
             <td>
-                <input type="checkbox" name="install_for_tools" value="${tool.id}"/>
+                <input type="checkbox" name="selected_tool_ids" value="${tool.id}"/>
             </td>
             <td>${ tool.name | h }</td>
             <td>${ tool.id | h }</td>
@@ -55,7 +55,7 @@
 
 <%def name="render_dependencies_details( tools, requirements_status, tool_ids_by_requirements)">
     <tr>
-        <th bgcolor="#D8D8D8">Install</th>
+        <th bgcolor="#D8D8D8">Select</th>
         <th bgcolor="#D8D8D8">Used by</th>
         <th bgcolor="#D8D8D8">Environment Path</th>
         <th bgcolor="#D8D8D8">Requirement</th>
@@ -74,7 +74,7 @@
                 <tr class="tr">
             %endif
             <td>
-                <input type="checkbox" name="install_for_tools" value="${tool_ids[0]}"/>
+                <input type="checkbox" name="selected_tool_ids" value="${tool_ids[0]}"/>
             </td>
             <td>${ ", ".join([tools[tid].name for tid in tool_ids]) | h }</td>
             ${render_tool_dependencies( r_status, ctr=ctr, show_environment_path=True, ncols_extra=3) }
@@ -111,4 +111,5 @@
         </div>
     </div>
     <input type="submit" name="install_dependencies" value="Install checked dependencies using Conda"/>
+    <input type="submit" name="uninstall_dependencies" value="Uninstall checked dependencies using Conda"/>
 </form>