From e5aa94e165522d75057bac7e0bd6034bb11f8a97 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 11 May 2016 09:07:06 -0400 Subject: [PATCH] Have `planemo test` verify output reports are actually updated by Galaxy. I'm fairly confident at this point that this is the cause of reported discrepancies between planemo's test output and output reports. Likely... - Fixes #381 - Fixes #306 --- planemo/galaxy/test/actions.py | 58 ++++++++++++++++++++++++++++------ tests/test_galaxy_test.py | 22 +++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/planemo/galaxy/test/actions.py b/planemo/galaxy/test/actions.py index 0b7a167f4..2a0891a46 100644 --- a/planemo/galaxy/test/actions.py +++ b/planemo/galaxy/test/actions.py @@ -20,6 +20,9 @@ NO_JSON_REPORT_MESSAGE = ("Cannot locate JSON report [%s] for tests - " "required to build planemo report and summarize " "tests.") +REPORT_NOT_CHANGED = ("Galaxy failed to update test report [%s] for tests - " + "required to build planemo report and summarize " + "tests.") NO_TESTS_MESSAGE = "No tests were executed - see Galaxy output for details." ALL_TESTS_PASSED_MESSAGE = "All %d test(s) executed passed." PROBLEM_COUNT_MESSAGE = ("There were problems with %d test(s) - out of %d " @@ -43,7 +46,9 @@ def run_in_config(ctx, config, run=run_galaxy_command, **kwds): job_output_files = os.path.join(config_directory, "jobfiles") xunit_report_file = _xunit_state(kwds, config) + xunit_report_file_tracker = _FileChangeTracker(xunit_report_file) structured_report_file = _structured_report_file(kwds, config) + structured_report_file_tracker = _FileChangeTracker(structured_report_file) info("Testing using galaxy_root %s", config.galaxy_root) # TODO: Allow running dockerized Galaxy here instead. @@ -85,15 +90,7 @@ def run_in_config(ctx, config, run=run_galaxy_command, **kwds): update_cp_args = (job_output_files, config.test_data_dir) shell('cp -r "%s"/* "%s"' % update_cp_args) - if not os.path.exists(xunit_report_file): - message = NO_XUNIT_REPORT_MESSAGE % xunit_report_file - error(message) - raise Exception(message) - - if not os.path.exists(structured_report_file): - message = NO_JSON_REPORT_MESSAGE % structured_report_file - error(message) - raise Exception(message) + _check_test_outputs(xunit_report_file_tracker, structured_report_file_tracker) test_results = test_structures.GalaxyTestResults( structured_report_file, @@ -248,6 +245,31 @@ def _print_command_line(structured_data, test_id): click.echo("| command: %s" % command) +def _check_test_outputs( + xunit_report_file_tracker, + structured_report_file_tracker +): + if not os.path.exists(xunit_report_file_tracker.path): + message = NO_XUNIT_REPORT_MESSAGE % xunit_report_file_tracker.path + error(message) + raise Exception(message) + + if not os.path.exists(structured_report_file_tracker.path): + message = NO_JSON_REPORT_MESSAGE % structured_report_file_tracker.path + error(message) + raise Exception(message) + + if not xunit_report_file_tracker.changed(): + message = REPORT_NOT_CHANGED % xunit_report_file_tracker.path + error(message) + raise Exception(message) + + if not structured_report_file_tracker.changed(): + message = REPORT_NOT_CHANGED % structured_report_file_tracker.path + error(message) + raise Exception(message) + + def _xunit_state(kwds, config): # This has been supported in Galaxy for well over a year, just going to assume # it from here on out. @@ -278,6 +300,24 @@ def _structured_report_file(kwds, config): return structured_report_file +class _FileChangeTracker(object): + + def __init__(self, path): + modification_time = None + if os.path.exists(path): + modification_time = os.path.getmtime(path) + + self.path = path + self.modification_time = modification_time + + def changed(self): + if self.modification_time: + new_modification_time = os.path.getmtime(self.path) + return self.modification_time != new_modification_time + else: + return os.path.exists(self.path) + + __all__ = [ "run_in_config", "handle_reports", diff --git a/tests/test_galaxy_test.py b/tests/test_galaxy_test.py index b3fced919..9817f9213 100644 --- a/tests/test_galaxy_test.py +++ b/tests/test_galaxy_test.py @@ -63,6 +63,28 @@ def mock_galaxy_run(ctx_, command, env, action): with self.assertRaises(Exception): self._do_run(mock_galaxy_run) + def test_failed_to_update_xml(self): + """Test an exception is thrown if XUnit report isn't updated.""" + self._copy_good_artifacts(["xml", "html", "json"]) + + def mock_galaxy_run(ctx_, command, env, action): + self._copy_good_artifacts(["json", "html"]) + return 0 + + with self.assertRaises(Exception): + self._do_run(mock_galaxy_run) + + def test_failed_to_update_json(self): + """Test an exception is thrown if XUnit report isn't updated.""" + self._copy_good_artifacts(["xml", "html", "json"]) + + def mock_galaxy_run(ctx_, command, env, action): + self._copy_good_artifacts(["xml", "html"]) + return 0 + + with self.assertRaises(Exception): + self._do_run(mock_galaxy_run) + def _copy_good_artifacts(self, extensions): for extension in extensions: source = os.path.join(TEST_DATA_DIR, "tt_success.%s" % extension)