Merge r225262 - WebDriver: add an option to dump test results to a js…

…on file

https://bugs.webkit.org/show_bug.cgi?id=180082

Reviewed by Brian Burg.

Add --json-output command line option to run-webdriver-tests to dump test results to a json file in a format
compatible with the W3C report. WebDriverTestResult now represents a test file and contains a list of
subtests, instead of having one WebDriverTestResult per subtest. This way we can store also the harness result
and dump the results to different formats.

* Scripts/run-webdriver-tests:
* Scripts/webkitpy/webdriver_tests/webdriver_test_result.py:
(WebDriverTestResult.__init__):
(WebDriverTestResult):
(WebDriverTestResult.add_subtest_results):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
(WebDriverTestRunner.print_results):
(WebDriverTestRunner):
(WebDriverTestRunner.dump_results_to_json_file):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
(WebDriverTestRunnerW3C.run):

Tools/ChangeLog

@@ -1,3 +1,27 @@
+2017-11-28  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: add an option to dump test results to a json file
+        https://bugs.webkit.org/show_bug.cgi?id=180082
+
+        Reviewed by Brian Burg.
+
+        Add --json-output command line option to run-webdriver-tests to dump test results to a json file in a format
+        compatible with the W3C report. WebDriverTestResult now represents a test file and contains a list of
+        subtests, instead of having one WebDriverTestResult per subtest. This way we can store also the harness result
+        and dump the results to different formats.
+
+        * Scripts/run-webdriver-tests:
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_result.py:
+        (WebDriverTestResult.__init__):
+        (WebDriverTestResult):
+        (WebDriverTestResult.add_subtest_results):
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
+        (WebDriverTestRunner.print_results):
+        (WebDriverTestRunner):
+        (WebDriverTestRunner.dump_results_to_json_file):
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+        (WebDriverTestRunnerW3C.run):
+
 2017-11-14  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         Move JSONValues to WTF and convert uses of InspectorValues.h to JSONValues.h

Tools/Scripts/run-webdriver-tests

@@ -47,6 +47,8 @@ option_parser.add_option('--debug', action='store_const', const='Debug', dest="c
                          help='Set the configuration to Debug')
 option_parser.add_option('--timeout', action='store', type='int', dest='timeout', default=10,
                          help='Time in seconds until a test times out (use 0 to disable)')
+option_parser.add_option('--json-output', action='store', metavar="FILE",
+                         help='Write results to JSON file at the given path')
 option_parser.add_option('--display-server', choices=['xvfb', 'xorg', 'weston', 'wayland'], default='xvfb',
                          help='"xvfb": Use a virtualized X11 server. "xorg": Use the current X11 session. '
                               '"weston": Use a virtualized Weston server. "wayland": Use the current wayland session.')
@@ -66,5 +68,8 @@ runner = WebDriverTestRunner(port)
 retval = runner.run(args)
 runner.print_results()
 
+if options.json_output is not None:
+    runner.dump_results_to_json_file(options.json_output)
+
 sys.exit(retval)
 

Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_result.py

@@ -25,11 +25,14 @@
 
 class WebDriverTestResult(object):
 
-    def __init__(self, test_prefix, test, status, message, backtrace=None):
-        self.test = os.path.join(test_prefix, test)
+    def __init__(self, test, status, message=None):
+        self.test = test
         self.status = status
         self.message = message
-        self.backtrace = backtrace
+        self.subtest_results = []
+
+    def add_subtest_results(self, subtest, status, message, backtrace):
+        self.subtest_results.append((subtest, status, message, backtrace))
 
     def __repr__(self):
         return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.test, self.status)

Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py

@@ -20,7 +20,9 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import json
 import logging
+import os
 
 from webkitpy.webdriver_tests.webdriver_test_runner_w3c import WebDriverTestRunnerW3C
 
@@ -54,11 +56,16 @@ def print_results(self):
         passed_count = 0
         failures_count = 0
         for result in self._runner.results():
-            if result.status == 'PASS':
-                passed_count += 1
-            elif result.status == 'FAIL':
-                results.setdefault(result.status, []).append(result.test)
-                failures_count += 1
+            if result.status == 'OK':
+                for subtest, status, _, _ in result.subtest_results:
+                    if status == 'PASS':
+                        passed_count += 1
+                    elif status == 'FAIL':
+                        results.setdefault(status, []).append(os.path.join(os.path.dirname(result.test), subtest))
+                        failures_count += 1
+            else:
+                # FIXME: handle other results.
+                pass
 
         _log.info('')
 
@@ -73,3 +80,27 @@ def print_results(self):
             _log.info('Unexpected failures (%d)' % len(failed))
             for test in failed:
                 _log.info('  %s' % test)
+
+    def dump_results_to_json_file(self, output_path):
+        json_results = {}
+        json_results['results'] = []
+        for result in self._runner.results():
+            results = {}
+            results['test'] = result.test
+            results['status'] = result.status
+            results['message'] = result.message
+            results['subtests'] = []
+            for name, status, message, _ in result.subtest_results:
+                subtest = {}
+                subtest['name'] = name
+                subtest['status'] = status
+                subtest['message'] = message
+                results['subtests'].append(subtest)
+            json_results['results'].append(results)
+
+        directory = os.path.dirname(output_path)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        with open(output_path, 'wb') as fp:
+            json.dump(json_results, fp)

Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py

@@ -96,10 +96,14 @@ def run(self, tests=[]):
             for test in tests:
                 test_name = os.path.relpath(test, self._tests_dir())
                 harness_result, test_results = executor.run(test)
-                if harness_result[0] != 'OK':
-                    _log.error("Failed to run test %s: %s" % (test_name, harness_result[1]))
+                result = WebDriverTestResult(test_name, *harness_result)
+                if harness_result[0] == 'OK':
+                    for test_result in test_results:
+                        result.add_subtest_results(*test_result)
                 else:
-                    self._add_results(os.path.dirname(test_name), test_results)
+                    # FIXME: handle other results.
+                    pass
+                self._results.append(result)
         finally:
             executor.teardown()
             self._server.stop()