diff --git a/.ci/jenkins/selenium/run_tests.sh b/.ci/jenkins/selenium/run_tests.sh index 6786d51c714d..4be5c13ed707 100755 --- a/.ci/jenkins/selenium/run_tests.sh +++ b/.ci/jenkins/selenium/run_tests.sh @@ -97,6 +97,12 @@ done; export GALAXY_TEST_SELENIUM_REMOTE=1 export GALAXY_TEST_SELENIUM_REMOTE_PORT="${SELENIUM_PORT}" +# Retry all failed Selenium tests a second time to deal +# with transiently failing tests. Failure information for +# first tests is still populated in database/test_errors +# and available at the top of the Jenkins test report. +export GALAXY_TEST_SELENIUM_RETRIES=1 + # Access Galaxy on localhost via port $GALAXY_PORT export GALAXY_TEST_PORT="${GALAXY_PORT}" diff --git a/run_tests.sh b/run_tests.sh index 6cf42689987f..4e38521a180b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -112,6 +112,20 @@ address Docker exposes localhost on to its child containers. This trick doesn't work on Mac OS X and so GALAXY_TEST_HOST will need to be crafted carefully ahead of time. +For Selenium test cases a stack trace is usually insufficient to diagnose +problems. For this reason, GALAXY_TEST_ERRORS_DIRECTORY is populated with +a new directory of information for each failing test case. This information +includes a screenshot, a stack trace, and the DOM of the currently rendered +Galaxy instance. The new directories are created with names that include +information about the failed test method name and the timestamp. By default, +GALAXY_TEST_ERRORS_DIRECTORY will be set to database/errors. + +The Selenium tests seem to be subject to transient failures at a higher +rate than the rest of the tests in Galaxy. Though this is unfortunate, +they have more moving pieces so this is perhaps not surprising. One can +set the GALAXY_TEST_SELENIUM_RETRIES to a number greater than 0 to +automatically retry every failed test case the specified number of times. + External Tests: A small subset of tests can be run against an existing Galaxy diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index d8d710dde20b..e1009773df08 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -6,6 +6,7 @@ import json import os import time +import traceback from functools import wraps @@ -49,6 +50,8 @@ GALAXY_TEST_SELENIUM_REMOTE_HOST = os.environ.get("GALAXY_TEST_SELENIUM_REMOTE_HOST", DEFAULT_SELENIUM_REMOTE_HOST) GALAXY_TEST_SELENIUM_HEADLESS = os.environ.get("GALAXY_TEST_SELENIUM_HEADLESS", DEFAULT_SELENIUM_HEADLESS) GALAXY_TEST_EXTERNAL_FROM_SELENIUM = os.environ.get("GALAXY_TEST_EXTERNAL_FROM_SELENIUM", None) +# Auto-retry selenium tests this many times. +GALAXY_TEST_SELENIUM_RETRIES = int(os.environ.get("GALAXY_TEST_SELENIUM_RETRIES", "0")) # Test case data DEFAULT_PASSWORD = '123456' @@ -66,32 +69,38 @@ def selenium_test(f): @wraps(f) def func_wrapper(self, *args, **kwds): - try: - return f(self, *args, **kwds) - except Exception: - if GALAXY_TEST_ERRORS_DIRECTORY and GALAXY_TEST_ERRORS_DIRECTORY != "0": - if not os.path.exists(GALAXY_TEST_ERRORS_DIRECTORY): - os.makedirs(GALAXY_TEST_ERRORS_DIRECTORY) - result_name = f.__name__ + datetime.datetime.now().strftime("%Y%m%d%H%M%s") - target_directory = os.path.join(GALAXY_TEST_ERRORS_DIRECTORY, result_name) - - def write_file(name, content): - with open(os.path.join(target_directory, name), "wb") as buf: - buf.write(content.encode("utf-8")) - - os.makedirs(target_directory) - self.driver.save_screenshot(os.path.join(target_directory, "last.png")) - write_file("page_source.txt", self.driver.page_source) - write_file("DOM.txt", self.driver.execute_script("return document.documentElement.outerHTML")) - iframes = self.driver.find_elements_by_css_selector("iframe") - for iframe in iframes: - pass - # TODO: Dump content out for debugging in the future. - # iframe_id = iframe.get_attribute("id") - # if iframe_id: - # write_file("iframe_%s" % iframe_id, "My content") - - raise + retry_attempts = 0 + while True: + try: + return f(self, *args, **kwds) + except Exception: + if GALAXY_TEST_ERRORS_DIRECTORY and GALAXY_TEST_ERRORS_DIRECTORY != "0": + if not os.path.exists(GALAXY_TEST_ERRORS_DIRECTORY): + os.makedirs(GALAXY_TEST_ERRORS_DIRECTORY) + result_name = f.__name__ + datetime.datetime.now().strftime("%Y%m%d%H%M%s") + target_directory = os.path.join(GALAXY_TEST_ERRORS_DIRECTORY, result_name) + + def write_file(name, content): + with open(os.path.join(target_directory, name), "wb") as buf: + buf.write(content.encode("utf-8")) + + os.makedirs(target_directory) + self.driver.save_screenshot(os.path.join(target_directory, "last.png")) + write_file("page_source.txt", self.driver.page_source) + write_file("DOM.txt", self.driver.execute_script("return document.documentElement.outerHTML")) + write_file("stacktrace.txt", traceback.format_exc()) + iframes = self.driver.find_elements_by_css_selector("iframe") + for iframe in iframes: + pass + # TODO: Dump content out for debugging in the future. + # iframe_id = iframe.get_attribute("id") + # if iframe_id: + # write_file("iframe_%s" % iframe_id, "My content") + + if retry_attempts < GALAXY_TEST_SELENIUM_RETRIES: + retry_attempts += 1 + else: + raise return func_wrapper