Implement shed_test command.

The test command tests local copies of files, shed_test resolves repositories, find the latest installed revisions on the shed, installs these revisions in a fresh Galaxy clone, and tests them with ``./run_tests.sh -installed``.

Requires galaxyproject/galaxy#267.

Closes #176.

planemo/commands/cmd_shed_test.py

@@ -0,0 +1,51 @@
+"""
+"""
+import socket
+import sys
+
+import click
+
+from planemo.cli import pass_context
+from planemo import options
+from planemo import galaxy_serve
+from planemo import shed
+from planemo import galaxy_test
+
+
+@click.command("shed_test")
+@options.shed_read_options()
+@options.galaxy_target_options()
+@options.test_options()
+@click.option(
+    "--skip_dependencies",
+    is_flag=True,
+    help="Do not install shed dependencies as part of repository installation."
+)
+@pass_context
+def cli(ctx, paths, **kwds):
+    """ Serve a transient Galaxy instance after installing repositories
+    from a remote Tool Shed.
+    """
+    galaxy_test.process_test_options(ctx, kwds)
+    install_args_list = shed.install_arg_lists(ctx, paths, **kwds)
+    port = get_free_port()
+    kwds["port"] = port
+    return_code = 1
+    with galaxy_serve.shed_serve(ctx, install_args_list, **kwds) as config:
+        config.kill()
+        return_code = galaxy_test.run_in_config(
+            ctx,
+            config,
+            installed=True,
+            **kwds
+        )
+    if return_code:
+        sys.exit(return_code)
+
+
+def get_free_port():
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.bind(('localhost', 0))
+    port = sock.getsockname()[1]
+    sock.close()
+    return port

planemo/commands/cmd_test.py

@@ -1,4 +1,3 @@
-import os
 import sys
 
 import click
@@ -7,57 +6,14 @@
 from planemo import options
 from planemo import galaxy_config
 
-from planemo.galaxy_test import run_in_config
-
-OUTPUT_DFEAULTS = {
-    "output": "tool_test_output.html",
-    "output_json": "tool_test_output.json",
-    "output_xunit": None,
-}
+from planemo.galaxy_test import (
+    run_in_config,
+    process_test_options,
+)
 
 
 @click.command('test')
 @options.optional_tools_arg(multiple=True)
-@click.option(
-    "--test_output",
-    type=click.Path(file_okay=True, resolve_path=True),
-    help=("Output test report (HTML - for humans) defaults to "
-          "tool_test_output.html."),
-    default=None,
-)
-@click.option(
-    "--test_output_xunit",
-    type=click.Path(file_okay=True, resolve_path=True),
-    help="Output test report (xUnit style - for computers).",
-    default=None,
-)
-@click.option(
-    "--test_output_json",
-    type=click.Path(file_okay=True, resolve_path=True),
-    help=("Output test report (planemo json) defaults to "
-          "tool_test_output.json."),
-    default=None,
-)
-@click.option(
-    "--job_output_files",
-    type=click.Path(file_okay=False, resolve_path=True),
-    help="Write job outputs to specified directory.",
-    default=None,
-)
-@click.option(
-    "--update_test_data",
-    is_flag=True,
-    help="Update test-data directory with job outputs (normally written to "
-         "directory --job_output_files if specified.)"
-)
-@click.option(
-    "--summary",
-    type=click.Choice(['none', 'minimal', 'compact']),
-    default="minimal",
-    help=("Summary style printed to planemo's standard output (see output "
-          "reports for more complete summary). Set to 'none' to disable "
-          "completely.")
-)
 @click.option(
     "--failed",
     is_flag=True,
@@ -69,7 +25,7 @@
 )
 @options.galaxy_target_options()
 @options.galaxy_config_options()
-@options.shed_dependency_resolution()
+@options.test_options()
 @pass_context
 def cli(ctx, paths, **kwds):
     """Run the tests in the specified tool tests in a Galaxy instance.
@@ -99,27 +55,10 @@ def cli(ctx, paths, **kwds):
     against that same Galaxy root - but this may not be bullet proof yet so
     please careful and do not try this against production Galaxy instances.
     """
-    for name, default in OUTPUT_DFEAULTS.items():
-        _populate_default_output(ctx, name, kwds, default)
+    process_test_options(ctx, kwds)
 
     kwds["for_tests"] = True
     with galaxy_config.galaxy_config(ctx, paths, **kwds) as config:
         return_value = run_in_config(ctx, config, **kwds)
         if return_value:
             sys.exit(return_value)
-
-
-def _populate_default_output(ctx, type, kwds, default):
-    kwd_key = "test_%s" % type
-    kwd_value = kwds.get(kwd_key, None)
-    if kwd_value is None:
-        global_config = ctx.global_config
-        global_config_key = "default_test_%s" % type
-        if global_config_key in global_config:
-            default_value = global_config[global_config_key]
-        else:
-            default_value = default
-
-        if default_value:
-            default_value = os.path.abspath(default_value)
-        kwds[kwd_key] = default_value

planemo/galaxy_config.py

@@ -124,13 +124,13 @@ def config_join(*args):
         _handle_job_metrics(config_directory, kwds)
         tool_definition = _tool_conf_entry_for(tool_paths)
         empty_tool_conf = config_join("empty_tool_conf.xml")
-        shed_tool_conf = config_join("shed_tool_conf.xml")
+        shed_tool_conf = _shed_tool_conf(install_galaxy, config_directory)
         tool_conf = config_join("tool_conf.xml")
         database_location = config_join("galaxy.sqlite")
         shed_tools_path = config_join("shed_tools")
         preseeded_database = True
         master_api_key = kwds.get("master_api_key", "test_key")
-        dependency_dir = os.path.join("config_directory", "deps")
+        dependency_dir = os.path.join(config_directory, "deps")
 
         try:
             _download_database_template(
@@ -211,6 +211,7 @@ def config_join(*args):
         if install_galaxy:
             _build_eggs_cache(ctx, env, kwds)
         _build_test_env(properties, env)
+        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf
 
         # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
         # work?
@@ -440,6 +441,14 @@ def _tool_conf_entry_for(tool_paths):
     return tool_definitions
 
 
+def _shed_tool_conf(install_galaxy, config_directory):
+    if install_galaxy:
+        config_dir = os.path.join(config_directory, "galaxy-dev", "config")
+    else:
+        config_dir = config_directory
+    return os.path.join(config_dir, "shed_tool_conf.xml")
+
+
 def _install_galaxy_if_needed(ctx, config_directory, kwds):
     installed = False
     if kwds.get("install_galaxy", None):
@@ -512,7 +521,6 @@ def _build_test_env(properties, env):
     # https://bitbucket.org/galaxy/galaxy-central/commits/d7dd1f9
     test_property_variants = {
         'GALAXY_TEST_MIGRATED_TOOL_CONF': 'migrated_tools_config',
-        'GALAXY_TEST_SHED_TOOL_CONF': 'migrated_tools_config',  # Hack
         'GALAXY_TEST_TOOL_CONF': 'tool_config_file',
         'GALAXY_TEST_FILE_DIR': 'test_data_dir',
         'GALAXY_TOOL_DEPENDENCY_DIR': 'tool_dependency_dir',

planemo/galaxy_test/__init__.py

@@ -1,3 +1,4 @@
 from .actions import run_in_config
+from .options import process_test_options
 
-__all__ = ["run_in_config"]
+__all__ = ["run_in_config", "process_test_options"]

planemo/galaxy_test/actions.py

@@ -48,7 +48,8 @@ def run_in_config(ctx, config, **kwds):
         html_report_file,
         xunit_report_file,
         structured_report_file,
-        failed=kwds["failed"],
+        failed=kwds.get("failed", False),
+        installed=kwds.get("installed", False),
     ).build()
     cmd = "; ".join([
         cd_to_galaxy_command,

planemo/galaxy_test/options.py

@@ -0,0 +1,28 @@
+import os
+
+OUTPUT_DFEAULTS = {
+    "output": "tool_test_output.html",
+    "output_json": "tool_test_output.json",
+    "output_xunit": None,
+}
+
+
+def process_test_options(ctx, kwds):
+    for name, default in OUTPUT_DFEAULTS.items():
+        _populate_default_output(ctx, name, kwds, default)
+
+
+def _populate_default_output(ctx, type, kwds, default):
+    kwd_key = "test_%s" % type
+    kwd_value = kwds.get(kwd_key, None)
+    if kwd_value is None:
+        global_config = ctx.global_config
+        global_config_key = "default_test_%s" % type
+        if global_config_key in global_config:
+            default_value = global_config[global_config_key]
+        else:
+            default_value = default
+
+        if default_value:
+            default_value = os.path.abspath(default_value)
+        kwds[kwd_key] = default_value

planemo/galaxy_test/structures.py

@@ -20,11 +20,13 @@ def __init__(
         xunit_report_file,
         structured_report_file,
         failed=False,
+        installed=False,
     ):
         self.html_report_file = html_report_file
         self.xunit_report_file = xunit_report_file
         self.structured_report_file = structured_report_file
         self.failed = failed
+        self.installed = installed
 
     def build(self):
         xunit_report_file = self.xunit_report_file
@@ -38,11 +40,14 @@ def build(self):
             sd_arg = "--structured_data_report_file %s" % sd_report_file
         else:
             sd_arg = ""
-        tests = "functional.test_toolbox"
-        if self.failed:
-            sd = StructuredData(self.structured_report_file)
-            failed_ids = sd.failed_ids
-            tests = " ".join(failed_ids)
+        if self.installed:
+            tests = "-installed"
+        else:
+            tests = "functional.test_toolbox"
+            if self.failed:
+                sd = StructuredData(self.structured_report_file)
+                failed_ids = sd.failed_ids
+                tests = " ".join(failed_ids)
         return RUN_TESTS_CMD % (html_report_file, xunit_arg, sd_arg, tests)
 
 

planemo/options.py

@@ -461,6 +461,51 @@ def recursive_option(help="Recursively perform command for subdirectories."):
     )
 
 
+def test_options():
+    return _compose(
+        click.option(
+            "--update_test_data",
+            is_flag=True,
+            help="Update test-data directory with job outputs (normally"
+                 " written to directory --job_output_files if specified.)"
+        ),
+        click.option(
+            "--test_output",
+            type=click.Path(file_okay=True, resolve_path=True),
+            help=("Output test report (HTML - for humans) defaults to "
+                  "tool_test_output.html."),
+            default=None,
+        ),
+        click.option(
+            "--test_output_xunit",
+            type=click.Path(file_okay=True, resolve_path=True),
+            help="Output test report (xUnit style - for computers).",
+            default=None,
+        ),
+        click.option(
+            "--test_output_json",
+            type=click.Path(file_okay=True, resolve_path=True),
+            help=("Output test report (planemo json) defaults to "
+                  "tool_test_output.json."),
+            default=None,
+        ),
+        click.option(
+            "--job_output_files",
+            type=click.Path(file_okay=False, resolve_path=True),
+            help="Write job outputs to specified directory.",
+            default=None,
+        ),
+        click.option(
+            "--summary",
+            type=click.Choice(['none', 'minimal', 'compact']),
+            default="minimal",
+            help=("Summary style printed to planemo's standard output (see "
+                  "output reports for more complete summary). Set to 'none' "
+                  "to disable completely.")
+        )
+    )
+
+
 def _compose(*functions):
     def compose2(f, g):
         return lambda x: f(g(x))

tests/test_shed_test.py

@@ -0,0 +1,20 @@
+import os
+
+from .test_utils import (
+    CliTestCase,
+    skip_if_environ,
+    TEST_REPOS_DIR,
+)
+
+
+class ShedTestTestCase(CliTestCase):
+
+    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    def test_serve(self):
+        fastqc_path = os.path.join(TEST_REPOS_DIR, "fastqc")
+        test_cmd = [
+            "shed_test",
+            "--install_galaxy",
+            fastqc_path,
+        ]
+        self._check_exit_code(test_cmd)