WebDriver: properly recover w3c tests after a webdriver server crash

https://bugs.webkit.org/show_bug.cgi?id=182242

Reviewed by Carlos Alberto Lopez Perez.

When a test makes the webdriver server crash, all other subsequent tests fail because they still try to send
messages to the server, gettin connection refused errors all the time. Selenium tests handle this correctly by
relaunching the server after every test failure, because other failures, even when not crashing the server,
might leave it in an bad state. WPT runner does the same for test files, it uses a subprocess to run the tests
and when any subtest fails, a new subsprocess is used for the following test file. We could do the same.

* Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
(WebDriverTestRunnerW3C.run): Restart the executor if any subtest failed.
* Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py:
(WebDriverW3CExecutor.__init__): Save timeout and expectations and do not import pytest.
(WebDriverW3CExecutor.setup): Create a subprocess to run the tests.
(WebDriverW3CExecutor.teardown): Send a message to the subprocess to terminate.
(WebDriverW3CExecutor._runner): Run the tests using pytest runner.
(WebDriverW3CExecutor.run): Send a message to the subprocess to run the given test and return the results message.

Canonical link: https://commits.webkit.org/198041@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@227732 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog

@@ -1,3 +1,25 @@
+2018-01-29  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: properly recover w3c tests after a webdriver server crash
+        https://bugs.webkit.org/show_bug.cgi?id=182242
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        When a test makes the webdriver server crash, all other subsequent tests fail because they still try to send
+        messages to the server, gettin connection refused errors all the time. Selenium tests handle this correctly by
+        relaunching the server after every test failure, because other failures, even when not crashing the server,
+        might leave it in an bad state. WPT runner does the same for test files, it uses a subprocess to run the tests
+        and when any subtest fails, a new subsprocess is used for the following test file. We could do the same.
+
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+        (WebDriverTestRunnerW3C.run): Restart the executor if any subtest failed.
+        * Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py:
+        (WebDriverW3CExecutor.__init__): Save timeout and expectations and do not import pytest.
+        (WebDriverW3CExecutor.setup): Create a subprocess to run the tests.
+        (WebDriverW3CExecutor.teardown): Send a message to the subprocess to terminate.
+        (WebDriverW3CExecutor._runner): Run the tests using pytest runner.
+        (WebDriverW3CExecutor.run): Send a message to the subprocess to run the given test and return the results message.
+
 2018-01-29  Per Arne Vollan  <pvollan@apple.com>
 
         Layout Test fast/events/beforeunload-dom-manipulation-crash.html is crashing

Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py

@@ -86,21 +86,25 @@ def _subtest_name(self, subtest):
     def run(self, tests=[]):
         self._server.start()
 
-        executor = WebDriverW3CExecutor(self._driver, self._server, self._display_driver)
+        executor = WebDriverW3CExecutor(self._driver, self._server, self._display_driver, self._port.get_option('timeout'), self._expectations)
         executor.setup()
-        timeout = self._port.get_option('timeout')
+        need_restart = False
         try:
             for test in tests:
                 test_name = os.path.relpath(test, self._tests_dir)
-                harness_result, test_results = executor.run(test, timeout, self._expectations)
+                harness_result, test_results = executor.run(test)
                 result = WebDriverTestResult(test_name, *harness_result)
                 if harness_result[0] == 'OK':
                     for subtest, status, message, backtrace in test_results:
                         result.add_subtest_results(self._subtest_name(subtest), status, message, backtrace)
+                        need_restart = need_restart or status in ('FAIL', 'ERROR', 'XFAIL', 'TIMEOUT')
                 else:
-                    # FIXME: handle other results.
-                    pass
+                    need_restart = True
                 self._results.append(result)
+
+                if need_restart:
+                    executor.teardown()
+                    executor.setup()
         finally:
             executor.teardown()
             self._server.stop()

Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py

@@ -25,6 +25,7 @@
 import json
 import sys
 
+from multiprocessing import Process, Queue
 from webkitpy.common.system.filesystem import FileSystem
 from webkitpy.common.webkit_finder import WebKitFinder
 import webkitpy.thirdparty.autoinstalled.mozlog
@@ -129,27 +130,54 @@ class WebKitDriverProtocol(WebDriverProtocol):
 class WebDriverW3CExecutor(WdspecExecutor):
     protocol_cls = WebKitDriverProtocol
 
-    def __init__(self, driver, server, display_driver):
+    def __init__(self, driver, server, display_driver, timeout, expectations):
         WebKitDriverServer.test_env = display_driver._setup_environ_for_test()
         WebKitDriverServer.test_env.update(driver.browser_env())
         server_config = {'host': server.host(), 'ports': {'http': [str(server.port())]}}
         WdspecExecutor.__init__(self, driver.browser_name(), server_config, driver.binary_path(), None, capabilities=driver.capabilities())
 
-        if pytest_runner is None:
-            do_delayed_imports()
+        self._timeout = timeout
+        self._expectations = expectations
+        self._test_queue = Queue()
+        self._result_queue = Queue()
 
     def setup(self):
         self.runner = TestRunner()
         self.protocol.setup(self.runner)
+        args = (self._test_queue,
+                self._result_queue,
+                self.protocol.session_config['host'],
+                str(self.protocol.session_config['port']),
+                json.dumps(self.protocol.session_config['capabilities']),
+                json.dumps(self.server_config),
+                self._timeout,
+                self._expectations)
+        self._process = Process(target=WebDriverW3CExecutor._runner, args=args)
+        self._process.start()
 
     def teardown(self):
         self.protocol.teardown()
+        self._test_queue.put('TEARDOWN')
+        self._process = None
+
+    @staticmethod
+    def _runner(test_queue, result_queue, host, port, capabilities, server_config, timeout, expectations):
+        if pytest_runner is None:
+            do_delayed_imports()
 
-    def run(self, test, timeout, expectations):
-        env = {'WD_HOST': self.protocol.session_config['host'],
-               'WD_PORT': str(self.protocol.session_config['port']),
-               'WD_CAPABILITIES': json.dumps(self.protocol.session_config['capabilities']),
-               'WD_SERVER_CONFIG': json.dumps(self.server_config)}
-        env.update(WebKitDriverServer.test_env)
-        args = ['--strict', '-p', 'no:mozlog']
-        return pytest_runner.run(test, args, timeout, env, expectations)
+        while True:
+            test = test_queue.get()
+            if test == 'TEARDOWN':
+                break
+
+            env = {'WD_HOST': host,
+                   'WD_PORT': port,
+                   'WD_CAPABILITIES': capabilities,
+                   'WD_SERVER_CONFIG': server_config}
+            env.update(WebKitDriverServer.test_env)
+            args = ['--strict', '-p', 'no:mozlog']
+            result_queue.put(pytest_runner.run(test, args, timeout, env, expectations))
+
+    def run(self, test):
+        self._test_queue.put(test)
+        return self._result_queue.get()