Replace ShedToolLineage with enhanced StockLineage

This means we don't rely on the Tool Shed metadata anymore and instead
deduce the tool lineage via the tool_id, which contains the version.
Version comparison then happens via LooseVersions.
This works for toolshed and non-toolshed tools, restoring proper
grouping in the tool panel and version switches, even for tool shed
tools whose lineage is broken.

lib/galaxy/config.py

@@ -925,11 +925,9 @@ def _configure_toolbox( self ):
         from galaxy import tools
         from galaxy.managers.citations import CitationsManager
         from galaxy.tools.deps import containers
-        from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
         import galaxy.tools.search
 
         self.citations_manager = CitationsManager( self )
-        self.tool_version_cache = ToolVersionCache(self)
 
         self._toolbox_lock = threading.RLock()
         # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.

lib/galaxy/model/tool_shed_install/__init__.py

@@ -589,51 +589,33 @@ def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repositor
         self.tool_id = tool_id
         self.tool_shed_repository = tool_shed_repository
 
+    @property
+    def version_ids(self):
+        return self.get_version_ids()
+
     def get_previous_version( self, app ):
-        parent_id = app.tool_version_cache.tool_id_to_parent_id.get(self.id, None)
-        if parent_id:
-            return app.tool_version_cache.tool_version_by_id[parent_id]
-        else:
-            return None
+        version_ids = self.version_ids
+        index = version_ids.index(self.tool_id)
+        if index > 0:
+            tool = self.app.toolbox._tools_by_id.get(version_ids[index - 1])
+            if tool:
+                return tool.tool_version
+        return None
 
     def get_next_version( self, app ):
-        child_id = app.tool_version_cache.parent_id_to_tool_id.get(self.id, None)
-        if child_id:
-            return app.tool_version_cache.tool_version_by_id[child_id]
-        else:
-            return None
-
-    def get_versions( self, app ):
-        tool_versions = []
-
-        # Prepend ancestors.
-        def __ancestors( app, tool_version ):
-            # Should we handle multiple parents at each level?
-            previous_version = tool_version.get_previous_version( app )
-            if previous_version:
-                if previous_version not in tool_versions:
-                    tool_versions.insert( 0, previous_version )
-                    __ancestors( app, previous_version )
-
-        # Append descendants.
-        def __descendants( app, tool_version ):
-            # Should we handle multiple child siblings at each level?
-            next_version = tool_version.get_next_version( app )
-            if next_version:
-                if next_version not in tool_versions:
-                    tool_versions.append( next_version )
-                    __descendants( app, next_version )
-
-        __ancestors( app, self )
-        if self not in tool_versions:
-            tool_versions.append( self )
-        __descendants( app, self )
-        return tool_versions
+        version_ids = self.version_ids
+        index = version_ids.index(self.tool_id)
+        if len(version_ids) > index + 1:
+            tool = self.app.toolbox._tools_by_id.get(version_ids[index + 1])
+            if tool:
+                return tool.tool_version
+        return None
 
     def get_version_ids( self, app, reverse=False ):
-        version_ids = [ tool_version.tool_id for tool_version in self.get_versions( app ) ]
+        lineage = app.toolbox._lineage_map.get(self.tool_id)
+        version_ids = lineage.get_versions()
         if reverse:
-            version_ids.reverse()
+            version_ids = reversed(version_ids)
         return version_ids
 
     def to_dict( self, view='element' ):

lib/galaxy/queue_worker.py

@@ -102,10 +102,8 @@ def _get_new_toolbox(app):
     """
     from galaxy import tools
     from galaxy.tools.special_tools import load_lib_tools
-    from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
     if hasattr(app, 'tool_shed_repository_cache'):
                 app.tool_shed_repository_cache.rebuild()
-    app.tool_version_cache = ToolVersionCache(app)  # Load new tools into version cache
     tool_configs = app.config.tool_configs
     if app.config.migrated_tools_config not in tool_configs:
         tool_configs.append(app.config.migrated_tools_config)
@@ -122,7 +120,6 @@ def _get_new_toolbox(app):
 def reload_data_managers(app, **kwargs):
     reload_timer = util.ExecutionTimer()
     from galaxy.tools.data_manager.manager import DataManagers
-    from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
     log.debug("Executing data managers reload on '%s'", app.config.server_name)
     if hasattr(app, 'tool_shed_repository_cache'):
         app.tool_shed_repository_cache.rebuild()
@@ -131,7 +128,6 @@ def reload_data_managers(app, **kwargs):
     reload_count = app.data_managers._reload_count
     app.data_managers = DataManagers(app)
     app.data_managers._reload_count = reload_count + 1
-    app.tool_version_cache = ToolVersionCache(app)
     if hasattr(app, 'tool_cache'):
         app.tool_cache.reset_status()
     log.debug("Data managers reloaded %s", reload_timer)

lib/galaxy/tools/__init__.py

@@ -442,7 +442,7 @@ def sa_session( self ):
     @property
     def tool_version( self ):
         """Return a ToolVersion if one exists for our id"""
-        return self.app.tool_version_cache.tool_version_by_tool_id.get(self.id)
+        return self.app.toolbox._lineage_map.get(self.id)
 
     @property
     def tool_versions( self ):
@@ -1832,11 +1832,7 @@ def to_json( self, trans, kwd={}, job=None, workflow_building_mode=False ):
             tool_help = unicodify( tool_help, 'utf-8' )
 
         # create tool versions
-        tool_versions = []
-        tools = self.app.toolbox.get_loaded_tools_by_lineage( self.id )
-        for t in tools:
-            if t.version not in tool_versions:
-                tool_versions.append( t.version )
+        tool_versions = list(self.tool_version.tool_versions)
 
         # update tool model
         tool_model.update({

lib/galaxy/tools/toolbox/base.py

@@ -427,8 +427,6 @@ def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=Fa
             # exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions
             rval = []
             tool_lineage = self._lineage_map.get( tool_id )
-            if not tool_lineage:
-                tool_lineage = self._lineage_map.get_versionless( tool_id )
             if tool_lineage:
                 lineage_tool_versions = tool_lineage.get_versions( )
                 for lineage_tool_version in lineage_tool_versions:
@@ -568,7 +566,7 @@ def _load_tool_tag_set( self, item, panel_dict, integrated_panel_dict, tool_path
                     tool.guid = guid
                     tool.version = item.elem.find( "version" ).text
                 # Make sure tools have a tool_version object.
-                tool_lineage = self._lineage_map.register( tool, from_toolshed=guid )
+                tool_lineage = self._lineage_map.register( tool )
                 tool.lineage = tool_lineage
                 if item.has_elem:
                     self._tool_tag_manager.handle_tags( tool.id, item.elem )
@@ -952,7 +950,7 @@ def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ):
             if not hasattr( tool, "lineage" ):
                 return None
             tool_lineage = tool.lineage
-        lineage_tool_versions = tool_lineage.get_versions( reverse=True )
+        lineage_tool_versions = reversed(tool_lineage.get_versions())
         for lineage_tool_version in lineage_tool_versions:
             lineage_tool = self._tool_from_lineage_version( lineage_tool_version )
             if lineage_tool:
@@ -967,14 +965,7 @@ def _newer_tool( self, tool1, tool2 ):
         """
         if not hasattr( tool1, "lineage" ):
             return True
-        lineage_tool_versions = tool1.lineage.get_versions()
-        for lineage_tool_version in lineage_tool_versions:
-            lineage_tool = self._tool_from_lineage_version( lineage_tool_version )
-            if lineage_tool is tool1:
-                return False
-            if lineage_tool is tool2:
-                return True
-        return True
+        return tool1.version_object > tool2.version_object
 
     def _tool_from_lineage_version( self, lineage_tool_version ):
         if lineage_tool_version.id_based:

lib/galaxy/tools/toolbox/lineages/__init__.py

@@ -1,6 +1,5 @@
 from .factory import LineageMap
 from .interface import ToolLineage
-from .tool_shed import ToolVersionCache
 
 
-__all__ = ("LineageMap", "ToolLineage", "ToolVersionCache")
+__all__ = ("LineageMap", "ToolLineage")

lib/galaxy/tools/toolbox/lineages/factory.py

@@ -1,15 +1,6 @@
-from .stock import StockLineage
-from .tool_shed import ToolShedLineage
-
+from galaxy.util.tool_version import remove_version_from_guid
 
-def remove_version_from_guid( guid ):
-    """
-    Removes version from toolshed-derived tool_id(=guid).
-    """
-    if "/repos/" not in guid:
-        return None
-    last_slash = guid.rfind('/')
-    return guid[:last_slash]
+from .stock import StockLineage
 
 
 class LineageMap(object):
@@ -20,28 +11,39 @@ def __init__(self, app):
         self.lineage_map = {}
         self.app = app
 
-    def register(self, tool, from_toolshed=False):
+    def register(self, tool):
         tool_id = tool.id
-        versionless_tool_id = remove_version_from_guid( tool_id )
-        if from_toolshed:
-            lineage = ToolShedLineage.from_tool(self.app, tool)
-        else:
+        versionless_tool_id = remove_version_from_guid( tool_id)
+        lineage = self.lineage_map.get(versionless_tool_id)
+        if not lineage:
             lineage = StockLineage.from_tool( tool )
         if versionless_tool_id and versionless_tool_id not in self.lineage_map:
             self.lineage_map[versionless_tool_id] = lineage
         if tool_id not in self.lineage_map:
             self.lineage_map[tool_id] = lineage
+        lineage.register_version(tool.version)
         return self.lineage_map[tool_id]
 
     def get(self, tool_id):
+        """
+        Get lineage for `tool_id`.
+
+        By preference the lineage for a version-agnostic tool_id is returned.
+        Falls back to fetching the lineage only when this fails.
+        This happens when the tool_id does not contain a version.
+        """
+        lineage = self._get_versionless(tool_id)
+        if lineage:
+            return lineage
         if tool_id not in self.lineage_map:
-            lineage = ToolShedLineage.from_tool_id( self.app, tool_id )
+            tool = self.app.toolbox._tools_by_id.get(tool_id)
+            if tool:
+                lineage = StockLineage.from_tool( tool )
             if lineage:
                 self.lineage_map[tool_id] = lineage
+        return self.lineage_map.get(tool_id)
 
-        return self.lineage_map.get(tool_id, None)
-
-    def get_versionless(self, tool_id):
+    def _get_versionless(self, tool_id):
         versionless_tool_id = remove_version_from_guid(tool_id)
         return self.lineage_map.get(versionless_tool_id, None)
 

lib/galaxy/tools/toolbox/lineages/interface.py

@@ -12,7 +12,7 @@ class ToolLineage(object):
     """
 
     @abstractmethod
-    def get_versions( self, reverse=False ):
+    def get_versions( self ):
         """ Return an ordered list of lineages (ToolLineageVersion) in this
         chain, from oldest to newest.
         """

lib/galaxy/tools/toolbox/lineages/stock.py

@@ -2,6 +2,8 @@
 
 from distutils.version import LooseVersion
 
+from galaxy.util.tool_version import remove_version_from_guid
+
 from .interface import ToolLineage
 from .interface import ToolLineageVersion
 
@@ -17,6 +19,12 @@ def __init__(self, tool_id, **kwds):
         self.tool_id = tool_id
         self.tool_versions = set()
 
+    @property
+    def tool_ids(self):
+        versionless_tool_id = remove_version_from_guid(self.tool_id)
+        tool_id = versionless_tool_id or self.tool_id
+        return ["%s/%s" % (tool_id, version) for version in self.tool_versions]
+
     @staticmethod
     def from_tool( tool ):
         tool_id = tool.id
@@ -32,11 +40,14 @@ def register_version( self, tool_version ):
         assert tool_version is not None
         self.tool_versions.add( tool_version )
 
-    def get_versions( self, reverse=False ):
-        versions = [ ToolLineageVersion( self.tool_id, v ) for v in self.tool_versions ]
+    def get_versions( self ):
+        versions = [ ToolLineageVersion( tool_id, tool_version ) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ]
         # Sort using LooseVersion which defines an appropriate __cmp__
         # method for comparing tool versions.
-        return sorted( versions, key=_to_loose_version, reverse=reverse )
+        return sorted( versions, key=_to_loose_version )
+
+    def get_version_ids(self):
+        return self.tool_ids
 
     def to_dict(self):
         return dict(