Merge ToolConfWatchers and separate from Toolbox

.ci/flake8_lint_include_list.txt

@@ -185,6 +185,7 @@ lib/galaxy/webapps/galaxy/controllers/requests.py
 lib/galaxy/webapps/galaxy/controllers/search.py
 lib/galaxy/webapps/galaxy/controllers/tool_runner.py
 lib/galaxy/webapps/galaxy/controllers/userskeys.py
+lib/galaxy/webapps/galaxy/config_watchers.py
 lib/galaxy/webapps/galaxy/__init__.py
 lib/galaxy/webapps/__init__.py
 lib/galaxy/webapps/reports/config.py

lib/galaxy/app.py

@@ -17,10 +17,12 @@
 from galaxy.visualization.plugins.registry import VisualizationsRegistry
 from galaxy.tools.special_tools import load_lib_tools
 from galaxy.tours import ToursRegistry
+from galaxy.webapps.galaxy.config_watchers import ConfigWatchers
 from galaxy.webhooks import WebhooksRegistry
 from galaxy.sample_tracking import external_service_types
 from galaxy.openid.providers import OpenIDProviders
 from galaxy.tools.data_manager.manager import DataManagers
+from galaxy.tools.toolbox.cache import ToolCache
 from galaxy.jobs import metrics as job_metrics
 from galaxy.web.proxy import ProxyManager
 from galaxy.web.stack import application_stack_instance
@@ -96,6 +98,10 @@ def __init__( self, **kwargs ):
         # Initialize the job management configuration
         self.job_config = jobs.JobConfiguration(self)
 
+        # Setup a Tool Cache
+        self.tool_cache = ToolCache()
+        # Watch various config files for immediate reload
+        self.watchers = ConfigWatchers(self)
         self._configure_toolbox()
 
         # Load Data Manager

lib/galaxy/config.py

@@ -908,11 +908,9 @@ def _configure_toolbox( self ):
         from galaxy import tools
         from galaxy.managers.citations import CitationsManager
         from galaxy.tools.deps import containers
-        from galaxy.tools.toolbox.cache import ToolCache
         from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
 
         self.citations_manager = CitationsManager( self )
-        self.tool_cache = ToolCache()
         self.tool_version_cache = ToolVersionCache(self)
 
         self._toolbox_lock = threading.RLock()

lib/galaxy/queue_worker.py

@@ -106,7 +106,7 @@ def _get_new_toolbox(app):
     if app.config.migrated_tools_config not in tool_configs:
         tool_configs.append(app.config.migrated_tools_config)
     start = time.time()
-    new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app, app.toolbox._tool_conf_watcher)
+    new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app)
     new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
     app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True)
     load_lib_tools(new_toolbox)
@@ -124,7 +124,7 @@ def reload_data_managers(app, **kwargs):
     app._configure_tool_data_tables(from_shed_config=False)
     reload_tool_data_tables(app)
     reload_count = app.data_managers._reload_count
-    app.data_managers = DataManagers(app, conf_watchers=app.data_managers.conf_watchers)
+    app.data_managers = DataManagers(app)
     app.data_managers._reload_count = reload_count + 1
 
 

lib/galaxy/tools/__init__.py

@@ -29,10 +29,7 @@
 )
 from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
 from galaxy.managers import histories
-from galaxy.queue_worker import (
-    reload_toolbox,
-    send_control_task
-)
+from galaxy.queue_worker import send_control_task
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.data_source import DataSourceToolAction
@@ -193,12 +190,8 @@ def __init__( self, config_filenames, tool_root_dir, app, tool_conf_watcher=None
             config_filenames=config_filenames,
             tool_root_dir=tool_root_dir,
             app=app,
-            tool_conf_watcher=tool_conf_watcher
         )
 
-    def handle_reload_toolbox(self):
-        reload_toolbox(self.app)
-
     def handle_panel_update(self, section_dict):
         """
         Sends a panel update to all threads/processes.

lib/galaxy/tools/data_manager/manager.py

@@ -7,14 +7,9 @@
 
 from galaxy import util
 from galaxy.queue_worker import (
-    reload_data_managers,
     send_control_task
 )
 from galaxy.tools.data import TabularToolDataTable
-from galaxy.tools.toolbox.watcher import (
-    get_tool_conf_watcher,
-    get_tool_data_dir_watcher
-)
 from galaxy.util.odict import odict
 from galaxy.util.template import fill_template
 from tool_shed.util import (
@@ -30,7 +25,7 @@
 
 
 class DataManagers( object ):
-    def __init__( self, app, xml_filename=None, conf_watchers=None ):
+    def __init__( self, app, xml_filename=None):
         self.app = app
         self.data_managers = odict()
         self.managed_data_tables = odict()
@@ -43,31 +38,6 @@ def __init__( self, app, xml_filename=None, conf_watchers=None ):
             self.load_from_xml( filename )
         if self.app.config.shed_data_manager_config_file:
             self.load_from_xml( self.app.config.shed_data_manager_config_file, store_tool_path=False, replace_existing=True )
-        if conf_watchers:
-            self.conf_watchers = conf_watchers
-        else:
-            self.conf_watchers = self.get_conf_watchers()
-
-    def get_conf_watchers(self):
-        """
-        Sets up monitoring of data manager related config files.
-        These are the data_manager_conf.xml and shed_data_manager_conf.xml files
-        as well as any loc file in the tool-data directory.
-        """
-        conf_watchers = []
-        conf_watchers.extend([(get_tool_conf_watcher(lambda: reload_data_managers(self.app)), filename) for filename in util.listify(self.filename) if filename])
-        if self.app.config.shed_data_manager_config_file:
-            conf_watchers.append((get_tool_conf_watcher(lambda: reload_data_managers(self.app)), self.app.config.shed_data_manager_config_file))
-        [watcher.watch_file(filename) for watcher, filename in conf_watchers]
-        tool_data_watcher = get_tool_data_dir_watcher(self.app.tool_data_tables, self.app.config)
-        tool_data_watcher.watch_directory(self.app.config.tool_data_path)
-        if self.app.config.shed_tool_data_path:
-            tool_data_watcher.watch_directory(self.app.config.shed_tool_data_path)
-        conf_watchers.append((tool_data_watcher, self.app.tool_data_tables.tool_data_path))
-        return [watcher for watcher, filename in conf_watchers]
-
-    def shutdown(self):
-        [watcher.shutdown() for watcher in self.conf_watchers]
 
     def load_from_xml( self, xml_filename, store_tool_path=True, replace_existing=False ):
         try:
@@ -166,46 +136,58 @@ def load_from_element( self, elem, tool_path ):
         self.version = elem.get( 'version', self.version )
         tool_shed_repository_id = None
         tool_guid = None
+
         if path is None:
             tool_elem = elem.find( 'tool' )
             assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) )
             path = tool_elem.get( "file", None )
             tool_guid = tool_elem.get( "guid", None )
             # need to determine repository info so that dependencies will work correctly
-            tool_shed_url = tool_elem.find( 'tool_shed' ).text
-            # Handle protocol changes.
-            tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
-            # The protocol is not stored in the database.
-            tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
-            repository_name = tool_elem.find( 'repository_name' ).text
-            repository_owner = tool_elem.find( 'repository_owner' ).text
-            installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
-            self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
-                                                        name=repository_name,
-                                                        owner=repository_owner,
-                                                        installed_changeset_revision=installed_changeset_revision )
-            tool_shed_repository = \
-                repository_util.get_installed_repository( self.data_managers.app,
-                                                          tool_shed=tool_shed,
-                                                          name=repository_name,
-                                                          owner=repository_owner,
-                                                          installed_changeset_revision=installed_changeset_revision )
-            if tool_shed_repository is None:
-                log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
-                # we'll set tool_path manually here from shed_conf_file
-                tool_shed_repository_id = None
-                try:
-                    tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
-                except Exception as e:
-                    log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
+            if tool_guid in self.data_managers.app.tool_cache._tool_paths_by_id:
+                path = self.data_managers.app.tool_cache._tool_paths_by_id[tool_guid]
+                tool = self.data_managers.app.tool_cache.get_tool(path)
+                tool_shed_repository = tool.tool_shed_repository
+                self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed_repository.tool_shed,
+                                                           name=tool_shed_repository.name,
+                                                           owner=tool_shed_repository.owner,
+                                                           installed_changeset_revision=tool_shed_repository.installed_changeset_revision)
+                tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
+                tool_path = ""
             else:
-                tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
-            # use shed_conf_file to determine tool_path
-            shed_conf_file = elem.get( "shed_conf_file", None )
-            if shed_conf_file:
-                shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
-                if shed_conf:
-                    tool_path = shed_conf.get( "tool_path", tool_path )
+                tool_shed_url = tool_elem.find( 'tool_shed' ).text
+                # Handle protocol changes.
+                tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
+                # The protocol is not stored in the database.
+                tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
+                repository_name = tool_elem.find( 'repository_name' ).text
+                repository_owner = tool_elem.find( 'repository_owner' ).text
+                installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
+                self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
+                                                            name=repository_name,
+                                                            owner=repository_owner,
+                                                            installed_changeset_revision=installed_changeset_revision )
+                tool_shed_repository = \
+                    repository_util.get_installed_repository( self.data_managers.app,
+                                                              tool_shed=tool_shed,
+                                                              name=repository_name,
+                                                              owner=repository_owner,
+                                                              installed_changeset_revision=installed_changeset_revision )
+                if tool_shed_repository is None:
+                    log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
+                    # we'll set tool_path manually here from shed_conf_file
+                    tool_shed_repository_id = None
+                    try:
+                        tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
+                    except Exception as e:
+                        log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
+                else:
+                    tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
+                # use shed_conf_file to determine tool_path
+                shed_conf_file = elem.get( "shed_conf_file", None )
+                if shed_conf_file:
+                    shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
+                    if shed_conf:
+                        tool_path = shed_conf.get( "tool_path", tool_path )
         assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) )
         self.load_tool( os.path.join( tool_path, path ),
                         guid=tool_guid,
@@ -287,7 +269,8 @@ def load_tool( self, tool_filename, guid=None, data_manager_id=None, tool_shed_r
         tool = toolbox.load_hidden_tool( tool_filename,
                                          guid=guid,
                                          data_manager_id=data_manager_id,
-                                         repository_id=tool_shed_repository_id )
+                                         repository_id=tool_shed_repository_id,
+                                         use_cached=True )
         self.data_managers.app.toolbox.data_manager_tools[ tool.id ] = tool
         self.tool = tool
         return tool