Add option to not restart on unexpected results.

wptrunner/testrunner.py

@@ -168,7 +168,7 @@ class TestRunnerManager(threading.Thread):
 
     def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
                  executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
-                 pause_on_unexpected=False, debug_info=None):
+                 pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None):
         """Thread that owns a single TestRunner process and any processes required
         by the TestRunner (e.g. the Firefox binary).
 
@@ -207,6 +207,7 @@ def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser
 
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
+        self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
 
         self.manager_number = next_manager_number()
@@ -525,7 +526,8 @@ def test_ended(self, test, results):
         self.test = None
 
         restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
-                               subtest_unexpected or is_unexpected)
+                               subtest_unexpected or
+                               (is_unexpected and self.restart_on_unexpected))
 
         if (self.pause_after_test or
             (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
@@ -592,6 +594,7 @@ def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
                  executor_cls, executor_kwargs,
                  pause_after_test=False,
                  pause_on_unexpected=False,
+                 restart_on_unexpected=True,
                  debug_info=None):
         """Main thread object that owns all the TestManager threads."""
         self.suite_name = suite_name
@@ -604,6 +607,7 @@ def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
         self.executor_kwargs = executor_kwargs
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
+        self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
 
         self.pool = set()
@@ -642,6 +646,7 @@ def run(self, test_type, tests):
                                             self.stop_flag,
                                             self.pause_after_test,
                                             self.pause_on_unexpected,
+                                            self.restart_on_unexpected,
                                             self.debug_info)
                 manager.start()
                 self.pool.add(manager)

wptrunner/wptcommandline.py

@@ -120,6 +120,9 @@ def create_parser(product_choices=None):
 
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
                                  help="Halt the test runner when an unexpected result is encountered")
+    debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected",
+                                 default=True, action="store_false",
+                                 help="Don't restart on an unexpected result")
 
     debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
                                  help="Path or url to symbols file used to analyse crash minidumps.")

wptrunner/wptrunner.py

@@ -197,6 +197,7 @@ def run_tests(config, test_paths, product, **kwargs):
                                       executor_kwargs,
                                       kwargs["pause_after_test"],
                                       kwargs["pause_on_unexpected"],
+                                      kwargs["restart_on_unexpected"],
                                       kwargs["debug_info"]) as manager_group:
                         try:
                             manager_group.run(test_type, test_loader.tests)