Merge pull request #54 from galaxyproject/more_test_output_config

More test output configuration updates.

docs/commands/test.rst

@@ -15,15 +15,25 @@ This section is auto-generated from the help text for the planemo command
 Run the tests in the specified tool tests in a Galaxy instance.
 
 All referenced tools (by default all the tools in the current working
-directory) will be tested and the resulted disposited in path specified
-with ``--test_output`` (defaults to tool_test_output.html).
+directory) will be tested and the results quickly summarized.
 
 To run these tests planemo needs a Galaxy instance to utilize, planemo
 will search parent directories to see if any is a Galaxy instance
 - but one can pick the Galaxy instance to use with the --galaxy_root
 option or force planemo to download a disposable instance with the
 ``--install_galaxy`` flag.
 
+In additon to to quick summary printed to the console - various detailed
+output summaries can be configured. ``tool_test_output.html`` (settable
+via ``--test_output``) will contain a human consumable HTML report
+describing the test run. A JSON file (settable via ``--test_output_json``
+and defaulting to ``tool_test_output.json``) will also be created. These
+files can can be disabled by passing in empty arguments or globally by
+setting the values ``default_test_output`` and/or
+``default_test_output_json`` in ``~/.planemo.yml`` to ``null``. For
+continuous integration testing a xUnit-style report can be confiured using
+the ``--test_output_xunit``.
+
 planemo uses temporarily generated config files and environment variables
 to attempt to shield this execution of Galaxy from manually launched runs
 against that same Galaxy root - but this may not be bullet proof yet so
@@ -32,10 +42,12 @@ please careful and do not try this against production Galaxy instances.
 **Options**::
 
 
-      --test_output PATH              Output test report (HTML - for humans).
+      --test_output PATH              Output test report (HTML - for humans)
+                                      defaults to tool_test_output.html.
       --test_output_xunit PATH        Output test report (xUnit style - for
                                       computers).
-      --test_output_json PATH         Output test report (planemo json).
+      --test_output_json PATH         Output test report (planemo json) defaults
+                                      to tool_test_output.json.
       --job_output_files DIRECTORY    Write job outputs to specified directory.
       --update_test_data              Update test-data directory with job outputs
                                       (normally written to directory

planemo/commands/cmd_test.py

@@ -33,14 +33,21 @@
     " functional.test_toolbox"
 )
 
+OUTPUT_DFEAULTS = {
+    "output": "tool_test_output.html",
+    "output_json": "tool_test_output.json",
+    "output_xunit": None,
+}
+
 
 @click.command('test')
 @options.optional_tools_arg()
 @click.option(
     "--test_output",
     type=click.Path(file_okay=True, resolve_path=True),
-    help="Output test report (HTML - for humans).",
-    default="tool_test_output.html",
+    help=("Output test report (HTML - for humans) defaults to "
+          "tool_test_output.html."),
+    default=None,
 )
 @click.option(
     "--test_output_xunit",
@@ -51,8 +58,9 @@
 @click.option(
     "--test_output_json",
     type=click.Path(file_okay=True, resolve_path=True),
-    help="Output test report (planemo json).",
-    default="tool_test_output.json",
+    help=("Output test report (planemo json) defaults to "
+          "tool_test_output.json."),
+    default=None,
 )
 @click.option(
     "--job_output_files",
@@ -87,20 +95,33 @@ def cli(ctx, path, **kwds):
     """Run the tests in the specified tool tests in a Galaxy instance.
 
     All referenced tools (by default all the tools in the current working
-    directory) will be tested and the resulted disposited in path specified
-    with ``--test_output`` (defaults to tool_test_output.html).
+    directory) will be tested and the results quickly summarized.
 
     To run these tests planemo needs a Galaxy instance to utilize, planemo
     will search parent directories to see if any is a Galaxy instance
     - but one can pick the Galaxy instance to use with the --galaxy_root
     option or force planemo to download a disposable instance with the
     ``--install_galaxy`` flag.
 
+    In additon to to quick summary printed to the console - various detailed
+    output summaries can be configured. ``tool_test_output.html`` (settable
+    via ``--test_output``) will contain a human consumable HTML report
+    describing the test run. A JSON file (settable via ``--test_output_json``
+    and defaulting to ``tool_test_output.json``) will also be created. These
+    files can can be disabled by passing in empty arguments or globally by
+    setting the values ``default_test_output`` and/or
+    ``default_test_output_json`` in ``~/.planemo.yml`` to ``null``. For
+    continuous integration testing a xUnit-style report can be confiured using
+    the ``--test_output_xunit``.
+
     planemo uses temporarily generated config files and environment variables
     to attempt to shield this execution of Galaxy from manually launched runs
     against that same Galaxy root - but this may not be bullet proof yet so
     please careful and do not try this against production Galaxy instances.
     """
+    for name, default in OUTPUT_DFEAULTS.items():
+        __populate_default_output(ctx, name, kwds, default)
+
     kwds["for_tests"] = True
     with galaxy_config.galaxy_config(ctx, path, **kwds) as config:
         config_directory = config.config_directory
@@ -169,6 +190,22 @@ def cli(ctx, path, **kwds):
             sys.exit(1)
 
 
+def __populate_default_output(ctx, type, kwds, default):
+    kwd_key = "test_%s" % type
+    kwd_value = kwds.get(kwd_key, None)
+    if kwd_value is None:
+        global_config = ctx.global_config
+        global_config_key = "default_test_%s" % type
+        if global_config_key in global_config:
+            default_value = global_config[global_config_key]
+        else:
+            default_value = default
+
+        if default_value:
+            default_value = os.path.abspath(default_value)
+        kwds[kwd_key] = default_value
+
+
 def __handle_summary(
     test_results,
     **kwds