Merge pull request #1296 from mvdbeek/multiple_tool_data_paths

Enable providing multiple `--tool_data_table` options

planemo/galaxy/config.py

@@ -16,6 +16,10 @@
     mkdtemp,
     NamedTemporaryFile,
 )
+from typing import (
+    List,
+    Optional,
+)
 
 from galaxy.containers.docker_model import DockerVolume
 from galaxy.tool_util.deps import docker_util
@@ -294,7 +298,7 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
     """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
 
     test_data_dir = _find_test_data(runnables, **kwds)
-    tool_data_table = _find_tool_data_table(runnables, test_data_dir=test_data_dir, **kwds)
+    tool_data_tables = _find_tool_data_table(runnables, test_data_dir=test_data_dir, **kwds)
     data_manager_config_paths = [r.data_manager_conf_path for r in runnables if r.data_manager_conf_path]
     galaxy_root = _find_galaxy_root(ctx, **kwds)
     install_galaxy = kwds.get("install_galaxy", False)
@@ -412,7 +416,7 @@ def config_join(*args):
                 debug="${debug}",
                 watch_tools="auto",
                 default_job_shell="/bin/bash",  # For conda dependency resolution
-                tool_data_table_config_path=tool_data_table,
+                tool_data_table_config_path=",".join(tool_data_tables) if tool_data_tables else None,
                 data_manager_config_file=",".join(data_manager_config_paths)
                 or None,  # without 'or None' may raise IOError in galaxy (see #946)
                 integrated_tool_panel_config=("${temp_directory}/" "integrated_tool_panel_conf.xml"),
@@ -1125,24 +1129,27 @@ def _find_test_data(runnables, **kwds):
     return None
 
 
-def _find_tool_data_table(runnables, test_data_dir, **kwds):
+def _find_tool_data_table(runnables, test_data_dir, **kwds) -> Optional[List[str]]:
     tool_data_search_path = "."
     runnables = [r for r in runnables if r.has_tools]
     if len(runnables) > 0:
         tool_data_search_path = runnables[0].tool_data_search_path
 
     tool_data_table = kwds.get("tool_data_table", None)
     if tool_data_table:
-        return os.path.abspath(tool_data_table)
+        return [os.path.abspath(table_path) for table_path in tool_data_table]
     else:
         extra_paths = [test_data_dir] if test_data_dir else []
-        return _search_tool_path_for(
+        tool_data_table = _search_tool_path_for(
             tool_data_search_path,
             "tool_data_table_conf.xml.test",
             extra_paths,
         ) or _search_tool_path_for(  # if all else fails just use sample
             tool_data_search_path, "tool_data_table_conf.xml.sample"
         )
+        if tool_data_table:
+            return [tool_data_table]
+    return None
 
 
 def _search_tool_path_for(path, target, extra_paths=None):
@@ -1348,6 +1355,7 @@ def _handle_refgenie_config(config_directory, galaxy_root, kwds):
 
 def _handle_kwd_overrides(properties, kwds):
     kwds_gx_properties = [
+        "tool_data_path",
         "job_config_file",
         "job_metrics_config_file",
         "dependency_resolvers_config_file",

planemo/options.py

@@ -144,6 +144,7 @@ def tool_data_table_option():
         "--tool_data_table",
         type=click.Path(exists=True, file_okay=True, resolve_path=True),
         help="tool_data_table_conf.xml file to for specified tool(s).",
+        multiple=True,
     )
 
 
@@ -401,6 +402,16 @@ def job_config_option():
     )
 
 
+def tool_data_path_option():
+    return planemo_option(
+        "--tool_data_path",
+        type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+        help="Directory where data used by tools is located. Required if tests are run in docker and should make use of external reference data.",
+        default=None,
+        use_global_config=True,
+    )
+
+
 def mulled_containers_option():
     return planemo_option(
         "mulled_containers",
@@ -1183,6 +1194,7 @@ def galaxy_target_options():
         # Profile options...
         job_config_option(),
         tool_dependency_dir_option(),
+        tool_data_path_option(),
     )
 
 

tests/test_cmd_serve.py

@@ -170,6 +170,27 @@ def test_serve_workflow(self):
         workflow_id = workflow["id"]
         assert workflow_id
 
+    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
+    def test_serve_multiple_tool_data_tables(self):
+        tool_data_table_path_1 = self._tool_data_table("planemo1")
+        tool_data_table_path_2 = self._tool_data_table("planemo2")
+        extra_args = [
+            "--daemon",
+            "--skip_client_build",
+            "--pid_file",
+            self._pid_file,
+            "--tool_data_table",
+            tool_data_table_path_1,
+            "--tool_data_table",
+            tool_data_table_path_2,
+        ]
+        self._launch_thread_and_wait(self._run, extra_args)
+        admin_gi = api.gi(self._port)
+        table_contents = admin_gi.tool_data.show_data_table("__dbkeys__")
+        assert any("planemo1" in field[0] for field in table_contents["fields"]), table_contents
+        assert any("planemo2" in field[0] for field in table_contents["fields"]), table_contents
+
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
     @skip_if_environ("PLANEMO_SKIP_SHED_TESTS")
     @mark.tests_galaxy_branch
@@ -221,5 +242,23 @@ def _test_serve_profile(self, *db_options):
         with cli_daemon_galaxy(self._runner, self._pid_file, self._port, serve_cmd):
             assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 1
 
+    def _tool_data_table(self, dbkey):
+        with tempfile.NamedTemporaryFile(
+            "w", suffix=".xml.test", delete=False
+        ) as tool_data_table, tempfile.NamedTemporaryFile("w", suffix="bla.loc", delete=False) as loc_file:
+            tool_data_table.write(
+                f"""<tables>
+    <table name="__dbkeys__" comment_char="#">
+        <columns>value, name, len_path</columns>
+        <file path="{loc_file.name}" />
+    </table>
+</tables>"""
+            )
+            loc_file.write(f"{dbkey}\t{dbkey}\t{dbkey}.len")
+            tool_data_table.flush()
+            loc_file.flush()
+        self._cleanup_hooks.extend([lambda: os.remove(loc_file.name), lambda: os.remove(tool_data_table.name)])
+        return tool_data_table.name
+
     def _run_shed(self, serve_args=[]):
         return self._run(serve_args=serve_args, serve_cmd="shed_serve")