Allow adding new data tables without restart

lib/galaxy/config.py

@@ -1019,7 +1019,7 @@ def _configure_tool_shed_registry(self):
 
         # Set up the tool sheds registry
         if os.path.isfile(self.config.tool_sheds_config_file):
-            self.tool_shed_registry = tool_shed.tool_shed_registry.Registry(self.config.root, self.config.tool_sheds_config_file)
+            self.tool_shed_registry = tool_shed.tool_shed_registry.Registry(self.config.tool_sheds_config_file)
         else:
             self.tool_shed_registry = None
 

lib/galaxy/dependencies/__init__.py

@@ -115,6 +115,11 @@ def check_azure_storage(self):
     def check_kamaki(self):
         return 'pithos' in self.object_stores
 
+    def check_watchdog(self):
+        install_set = {'auto', 'True', 'true', 'polling'}
+        return (self.config['watch_tools'] in install_set or
+                self.config['watch_tool_data_dir'] in install_set)
+
 
 def optional(config_file):
     rval = []

lib/galaxy/dependencies/conditional-requirements.txt

@@ -19,3 +19,5 @@ chronos-python==0.38.0
 
 # Synnefo / Pithos+ object store client
 kamaki
+
+watchdog

lib/galaxy/dependencies/dev-requirements.txt

@@ -4,6 +4,7 @@ NoseHTML
 twill==0.9.1
 mock
 selenium
+watchdog
 
 # For relase process
 pygithub3

lib/galaxy/queue_worker.py

@@ -130,6 +130,8 @@ def reload_data_managers(app, **kwargs):
     app.data_managers._reload_count = reload_count + 1
     if hasattr(app, 'tool_cache'):
         app.tool_cache.reset_status()
+    if hasattr(app, 'watchers'):
+        app.watchers.update_watch_data_table_paths()
     log.debug("Data managers reloaded %s", reload_timer)
 
 

lib/galaxy/tools/parameters/dynamic_options.py

@@ -493,29 +493,16 @@ def load_from_parameter(from_parameter, transform_lines=None):
         self.missing_index_file = None
         dataset_file = elem.get('from_dataset', None)
         from_parameter = elem.get('from_parameter', None)
-        tool_data_table_name = elem.get('from_data_table', None)
+        self.tool_data_table_name = elem.get('from_data_table', None)
         # Options are defined from a data table loaded by the app
-        self.tool_data_table = None
-        self.missing_tool_data_table_name = None
-        if tool_data_table_name:
-            app = tool_param.tool.app
-            if tool_data_table_name in app.tool_data_tables:
-                self.tool_data_table = app.tool_data_tables[tool_data_table_name]
-                # Column definitions are optional, but if provided override those from the table
-                if elem.find("column") is not None:
-                    self.parse_column_definitions(elem)
-                else:
-                    self.columns = self.tool_data_table.columns
-                # Set self.missing_index_file if the index file to
-                # which the tool_data_table refers does not exist.
-                if self.tool_data_table.missing_index_file:
-                    self.missing_index_file = self.tool_data_table.missing_index_file
-            else:
-                self.missing_tool_data_table_name = tool_data_table_name
-                log.warning("Data table named '%s' is required by tool but not configured" % tool_data_table_name)
+        self._tool_data_table = None
+        self.elem = elem
+        self.column_elem = elem.find("column")
+        self.tool_data_table  # Need to touch tool data table once to populate self.columns
+
         # Options are defined by parsing tabular text data from a data file
         # on disk, a dataset, or the value of another parameter
-        elif data_file is not None or dataset_file is not None or from_parameter is not None:
+        if not self.tool_data_table_name and (data_file is not None or dataset_file is not None or from_parameter is not None):
             self.parse_column_definitions(elem)
             if data_file is not None:
                 data_file = data_file.strip()
@@ -545,6 +532,30 @@ def load_from_parameter(from_parameter, transform_lines=None):
         if self.dataset_ref_name:
             tool_param.data_ref = self.dataset_ref_name
 
+    @property
+    def tool_data_table(self):
+        if self.tool_data_table_name:
+            tool_data_table = self.tool_param.tool.app.tool_data_tables.get(self.tool_data_table_name, None)
+            if tool_data_table:
+                # Column definitions are optional, but if provided override those from the table
+                if self.column_elem is not None:
+                    self.parse_column_definitions(self.elem)
+                else:
+                    self.columns = tool_data_table.columns
+                # Set self.missing_index_file if the index file to
+                # which the tool_data_table refers does not exist.
+                if tool_data_table.missing_index_file:
+                    self.missing_index_file = tool_data_table.missing_index_file
+            return tool_data_table
+        return None
+
+    @property
+    def missing_tool_data_table_name(self):
+        if not self.tool_data_table:
+            log.warning("Data table named '%s' is required by tool but not configured" % self.tool_data_table_name)
+            return self.tool_data_table_name
+        return None
+
     def parse_column_definitions(self, elem):
         for column_elem in elem.findall('column'):
             name = column_elem.get('name', None)

lib/galaxy/tools/toolbox/base.py

@@ -741,7 +741,7 @@ def load_tool(self, config_file, guid=None, repository_id=None, use_cached=False
         tool = None
         if use_cached:
             tool = self.load_tool_from_cache(config_file)
-        if not tool:
+        if not tool or guid and guid != tool.guid:
             tool = self.create_tool(config_file=config_file, repository_id=repository_id, guid=guid, **kwds)
             if tool.tool_shed_repository or not guid:
                 self.add_tool_to_cache(tool, config_file)

lib/galaxy/webapps/galaxy/config_watchers.py

@@ -32,6 +32,12 @@ def start(self):
         [self.data_manager_config_watcher.watch_file(config) for config in self.data_manager_configs]
         [self.tool_data_watcher.watch_directory(tool_data_path) for tool_data_path in self.tool_data_paths]
 
+    def update_watch_data_table_paths(self):
+        if hasattr(self.tool_data_watcher, 'monitored_dirs'):
+            for tool_data_table_path in self.tool_data_paths:
+                if tool_data_table_path not in self.tool_data_watcher.monitored_dirs:
+                    self.tool_data_watcher.watch_directory(tool_data_table_path)
+
     @property
     def data_manager_configs(self):
         data_manager_configs = []

lib/tool_shed/tool_shed_registry.py

@@ -10,10 +10,10 @@
 
 class Registry(object):
 
-    def __init__(self, root_dir=None, config=None):
+    def __init__(self, config=None):
         self.tool_sheds = odict()
         self.tool_sheds_auth = odict()
-        if root_dir and config:
+        if config:
             # Parse tool_sheds_conf.xml
             tree, error_message = xml_util.parse_xml(config)
             if tree is None:

lib/tool_shed/tools/tool_validator.py

@@ -71,8 +71,6 @@ def check_tool_input_params(self, repo_dir, tool_config_name, tool, sample_files
                                                                                   persist=False)
                             if error:
                                 invalid_files_and_errors_tups.append(('tool_data_table_conf.xml.sample', correction_msg))
-                            else:
-                                options.missing_tool_data_table_name = None
                         else:
                             correction_msg = "This file requires an entry in the tool_data_table_conf.xml file.  "
                             correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository "
@@ -89,7 +87,6 @@ def check_tool_input_params(self, repo_dir, tool_config_name, tool, sample_files
                             sample_file_name = basic_util.strip_path(sample_file)
                             if sample_file_name == '%s.sample' % index_file_name:
                                 options.index_file = index_file_name
-                                options.missing_index_file = None
                                 if options.tool_data_table:
                                     options.tool_data_table.missing_index_file = None
                                 sample_found = True

test/integration/test_data_manager_table_reload.py

@@ -0,0 +1,134 @@
+import os
+import random
+import shutil
+import string
+import tempfile
+
+from base import integration_util
+from base.populators import DatasetPopulator
+from nose.plugins.skip import SkipTest
+
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+TOOL_SHEDS_CONF = os.path.join(SCRIPT_DIRECTORY, "tool_sheds_conf.xml")
+
+SHED_TOOL_CONF = string.Template("""<?xml version="1.0"?>
+<toolbox tool_path="$shed_tools_path">
+</toolbox>""")
+
+SHED_DATA_MANAGER_CONF = """<?xml version="1.0"?>
+<data_managers>
+</data_managers>"""
+
+SHED_DATA_TABLES = """<?xml version="1.0"?>
+<tables>
+</tables>"""
+
+CREATE_DBKEY_PAYLOAD = {'tool_shed_url': 'https://toolshed.g2.bx.psu.edu',
+                        'name': 'data_manager_fetch_genome_dbkeys_all_fasta',
+                        'owner': 'devteam',
+                        'changeset_revision': 'b1bc53e9bbc5'}
+SAM_FASTA_PAYLOAD = {'tool_shed_url': 'https://toolshed.g2.bx.psu.edu',
+                     'name': 'data_manager_sam_fasta_index_builder',
+                     'owner': 'devteam',
+                     'changeset_revision': '1865e693d8b2'}
+FETCH_TOOL_ID = 'toolshed.g2.bx.psu.edu/repos/devteam/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_fetch_genome_all_fasta_dbkey/0.0.2'
+FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT = {"dbkey_source|dbkey_source_selector": "new",
+                                       "dbkey_source|dbkey": "NC_001617.1",
+                                       "dbkey_source|dbkey_name": "NC_001617.1",
+                                       "sequence_name": "NC_001617.1",
+                                       "sequence_id": "NC_001617.1",
+                                       "reference_source|reference_source_selector": "ncbi",
+                                       "reference_source|requested_identifier": "NC_001617.1",
+                                       "sorting|sort_selector": "as_is"}
+SAM_FASTA_ID = "toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_index_builder/sam_fasta_index_builder/0.0.2"
+SAM_FASTA_INPUT = {"all_fasta_source": "NC_001617.1", "sequence_name": "", "sequence_id": ""}
+
+
+class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    """Test data manager installation and table reload through the API"""
+
+    framework_tool_and_types = True
+
+    def setUp(self):
+        super(DataManagerIntegrationTestCase, self).setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        try:
+            import watchdog  # noqa: F401
+        except ImportError:
+            raise SkipTest("watchdog library is not available")
+        cls.username = cls.get_secure_ascii_digits()
+        cls.conda_tmp_prefix = tempfile.mkdtemp()
+        cls.shed_tools_dir = tempfile.mkdtemp()
+        cls.shed_tool_data_dir = tempfile.mkdtemp()
+        config["conda_auto_init"] = True
+        config["conda_auto_install"] = True
+        config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda')
+        config["tool_sheds_config_file"] = TOOL_SHEDS_CONF
+        config["tool_config_file"] = os.path.join(cls.shed_tools_dir, 'shed_tool_conf.xml')
+        config["shed_data_manager_config_file"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_manager_config_file')
+        config["shed_tool_data_table_config"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_table_conf.xml')
+        config["shed_tool_data_path"] = cls.shed_tool_data_dir
+        config["tool_data_path"] = cls.shed_tool_data_dir
+        config["watch_tool_data_dir"] = True
+        config["admin_users"] = "%s@galaxy.org" % cls.username
+        with open(config["tool_config_file"], 'w') as tool_conf_file:
+            tool_conf_file.write(SHED_TOOL_CONF.substitute(shed_tools_path=cls.shed_tools_dir))
+        with open(config["shed_data_manager_config_file"], 'w') as shed_data_config:
+            shed_data_config.write(SHED_DATA_MANAGER_CONF)
+        with open(config["shed_tool_data_table_config"], 'w') as shed_data_table_config:
+            shed_data_table_config.write(SHED_DATA_TABLES)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Shutdown Galaxy server and cleanup temp directory."""
+        for dir in [cls.conda_tmp_prefix, cls.shed_tool_data_dir, cls.shed_tools_dir]:
+            shutil.rmtree(dir)
+        cls._test_driver.tear_down()
+        cls._app_available = False
+
+    def test_data_manager_installation_table_reload(self):
+        """
+        Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager.
+        """
+        create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=CREATE_DBKEY_PAYLOAD, admin=True)
+        self._assert_status_code_is(create_response, 200)
+        create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=SAM_FASTA_PAYLOAD, admin=True)
+        self._assert_status_code_is(create_response, 200)
+
+        with self._different_user(email="%s@galaxy.org" % self.username):
+            with self.dataset_populator.test_history() as history_id:
+                run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,
+                                                               inputs=FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response)
+                run_response = self.dataset_populator.run_tool(tool_id=SAM_FASTA_ID,
+                                                               inputs=SAM_FASTA_INPUT,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response)
+
+    def create_local_user(self):
+        """Creates a local user and returns the user id."""
+        password = self.get_secure_ascii_digits()
+        payload = {'username': self.username,
+                   'password': password,
+                   'email': "%s@galaxy.org" % self.username}
+        create_response = self._post('/users', data=payload, admin=True)
+        self._assert_status_code_is(create_response, 200)
+        response = create_response.json()
+        return response['id']
+
+    def create_api_key_for_user(self, user_id):
+        create_response = self._post("/users/%s/api_key" % user_id, data={}, admin=True)
+        self._assert_status_code_is(create_response, 200)
+        return create_response.json()
+
+    @classmethod
+    def get_secure_ascii_digits(cls, n=12):
+        return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12))

test/integration/tool_sheds_conf.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<tool_sheds>
+    <tool_shed name="Galaxy Main Tool Shed" url="https://toolshed.g2.bx.psu.edu/"/>
+</tool_sheds>