Merge r225390 - WebDriver: add support for importing and running sele…

…nium tests

https://bugs.webkit.org/show_bug.cgi?id=180145

Reviewed by Brian Burg.

Tools:

We currently import and run W3C tests, which are the best ones to ensure our implementation is spec
compliant. However, the selenium API is what user will actually use in the end, so it's important to ensure that
we don't break the selenium support. This patch adds webdriver_test_runner_selenium.py and
webdriver_selenium_executor.py to run selenium tests. The script import-w3c-webdriver-tests has been renamed as
import-webdriver-tests and it can now import selenium tests too.

* Scripts/import-w3c-webdriver-tests: Removed.
* Scripts/import-webdriver-tests: Added.
(Importer):
(Importer.__init__):
(Importer.download_tests):
(Importer.import_tests):
(Importer.import_tests.should_skip_file):
* Scripts/webkitpy/webdriver_tests/webdriver_driver.py:
(WebDriver.browser_path):
(WebDriver):
(WebDriver.browser_args):
(WebDriver.selenium_name):
* Scripts/webkitpy/webdriver_tests/webdriver_driver_gtk.py:
(WebDriverGtk.browser_path):
(WebDriverGtk):
(WebDriverGtk.browser_args):
(WebDriverGtk.capabilities):
(WebDriverGtk.selenium_name):
* Scripts/webkitpy/webdriver_tests/webdriver_driver_wpe.py:
(WebDriverWPE.browser_args):
(WebDriverWPE.capabilities):
* Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py: Added.
(_ensure_directory_in_path):
(CollectRecorder):
(CollectRecorder.__init__):
(CollectRecorder.pytest_collectreport):
(WebDriverSeleniumExecutor):
(WebDriverSeleniumExecutor.__init__):
(WebDriverSeleniumExecutor.collect):
(WebDriverSeleniumExecutor.run):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
(WebDriverTestRunner):
(WebDriverTestRunner.__init__):
(WebDriverTestRunner.run):
(WebDriverTestRunner.print_results):
(WebDriverTestRunner.dump_results_to_json_file):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py: Added.
(WebDriverTestRunnerSelenium):
(WebDriverTestRunnerSelenium.__init__):
(WebDriverTestRunnerSelenium._tests_dir):
(WebDriverTestRunnerSelenium.collect_tests):
(WebDriverTestRunnerSelenium.run):
(WebDriverTestRunnerSelenium.results):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
(WebDriverTestRunnerW3C.__init__):
(WebDriverTestRunnerW3C.run):
* Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py:

WebDriverTests:

* imported/selenium/importer.json: Added.
* imported/selenium/common/: Added.
* imported/selenium/py/: Added.

Tools/ChangeLog

@@ -1,3 +1,64 @@
+2017-11-30  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: add support for importing and running selenium tests
+        https://bugs.webkit.org/show_bug.cgi?id=180145
+
+        Reviewed by Brian Burg.
+
+        We currently import and run W3C tests, which are the best ones to ensure our implementation is spec
+        compliant. However, the selenium API is what user will actually use in the end, so it's important to ensure that
+        we don't break the selenium support. This patch adds webdriver_test_runner_selenium.py and
+        webdriver_selenium_executor.py to run selenium tests. The script import-w3c-webdriver-tests has been renamed as
+        import-webdriver-tests and it can now import selenium tests too.
+
+        * Scripts/import-w3c-webdriver-tests: Removed.
+        * Scripts/import-webdriver-tests: Added.
+        (Importer):
+        (Importer.__init__):
+        (Importer.download_tests):
+        (Importer.import_tests):
+        (Importer.import_tests.should_skip_file):
+        * Scripts/webkitpy/webdriver_tests/webdriver_driver.py:
+        (WebDriver.browser_path):
+        (WebDriver):
+        (WebDriver.browser_args):
+        (WebDriver.selenium_name):
+        * Scripts/webkitpy/webdriver_tests/webdriver_driver_gtk.py:
+        (WebDriverGtk.browser_path):
+        (WebDriverGtk):
+        (WebDriverGtk.browser_args):
+        (WebDriverGtk.capabilities):
+        (WebDriverGtk.selenium_name):
+        * Scripts/webkitpy/webdriver_tests/webdriver_driver_wpe.py:
+        (WebDriverWPE.browser_args):
+        (WebDriverWPE.capabilities):
+        * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py: Added.
+        (_ensure_directory_in_path):
+        (CollectRecorder):
+        (CollectRecorder.__init__):
+        (CollectRecorder.pytest_collectreport):
+        (WebDriverSeleniumExecutor):
+        (WebDriverSeleniumExecutor.__init__):
+        (WebDriverSeleniumExecutor.collect):
+        (WebDriverSeleniumExecutor.run):
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
+        (WebDriverTestRunner):
+        (WebDriverTestRunner.__init__):
+        (WebDriverTestRunner.run):
+        (WebDriverTestRunner.print_results):
+        (WebDriverTestRunner.dump_results_to_json_file):
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py: Added.
+        (WebDriverTestRunnerSelenium):
+        (WebDriverTestRunnerSelenium.__init__):
+        (WebDriverTestRunnerSelenium._tests_dir):
+        (WebDriverTestRunnerSelenium.collect_tests):
+        (WebDriverTestRunnerSelenium.run):
+        (WebDriverTestRunnerSelenium.results):
+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+        (WebDriverTestRunnerW3C.__init__):
+        (WebDriverTestRunnerW3C.run):
+        * Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py:
+
 2017-11-28  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         WebDriver: add an option to dump test results to a json file

Tools/Scripts/import-webdriver-tests

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2017 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer.
+# 2. Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials
+#    provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+import json
+import logging
+import optparse
+import os
+import sys
+
+from webkitpy.common.host import Host
+from webkitpy.common.system.filesystem import FileSystem
+from webkitpy.common.system.logutils import configure_logging
+from webkitpy.common.webkit_finder import WebKitFinder
+from webkitpy.common.checkout.scm.git import Git
+
+_log = logging.getLogger(__name__)
+
+host = Host()
+host.initialize_scm()
+webkit_finder = WebKitFinder(host.filesystem)
+
+class Importer(object):
+
+    def __init__(self, config_path, download_path, destination_path):
+        self.config = json.loads(host.filesystem.read_text_file(config_path))
+        self.download_path = download_path
+        self.destination_path = destination_path
+
+    def download_tests(self):
+        host.filesystem.maybe_make_directory(host.filesystem.dirname(self.download_path))
+        needs_clone = not host.filesystem.exists(self.download_path)
+        if needs_clone:
+            _log.info('Cloning %s into %s...' % (self.config['repository'], self.download_path))
+            Git.clone(self.config['repository'], self.download_path, host.executive)
+        git = Git(self.download_path, None, executive=host.executive, filesystem=host.filesystem)
+        if not needs_clone:
+            _log.info('Fetching %s...' % self.config['repository'])
+            git.fetch()
+
+        _log.info('Checking out revision %s' % self.config['revision'])
+        git.checkout(self.config['revision'], False)
+
+    def import_tests(self):
+        def should_skip_file(source_path, full_paths_to_skip):
+            for path in full_paths_to_skip:
+                if source_path.startswith(path):
+                    return True
+            return False
+
+        _log.info('Importing tests')
+        full_paths_to_skip = [host.filesystem.join(self.download_path, path) for path in self.config['paths_to_skip']]
+        for path in self.config['paths_to_import']:
+            source_directory = host.filesystem.join(self.download_path, path)
+            for source_path in host.filesystem.files_under(source_directory):
+                if should_skip_file(source_path, full_paths_to_skip):
+                    continue
+                destination_path = host.filesystem.join(self.destination_path, host.filesystem.relpath(source_path, self.download_path))
+                host.filesystem.maybe_make_directory(host.filesystem.dirname(destination_path))
+                host.filesystem.copyfile(source_path, destination_path)
+
+
+option_parser = optparse.OptionParser(usage='usage: %prog [options]')
+option_parser.add_option('--w3c', action='store_true', dest='w3c',
+                         help='Import W3C tests')
+option_parser.add_option('--selenium', action='store_true', dest='selenium',
+                         help='Import Selenium tests')
+options, _ = option_parser.parse_args()
+
+# Not giving any option means import all.
+if not options.w3c and not options.selenium:
+    options.w3c = options.selenium = True
+
+configure_logging()
+
+importers = []
+
+if options.w3c:
+    importer_config_path = webkit_finder.path_from_webkit_base('WebDriverTests', 'imported', 'w3c', 'importer.json')
+    download_path = webkit_finder.path_from_webkit_base('WebKitBuild', 'w3c-tests', 'web-platform-tests')
+    import_path = webkit_finder.path_from_webkit_base('WebDriverTests', 'imported', 'w3c')
+    importers.append(Importer(importer_config_path, download_path, import_path))
+
+if options.selenium:
+    importer_config_path = webkit_finder.path_from_webkit_base('WebDriverTests', 'imported', 'selenium', 'importer.json')
+    download_path = webkit_finder.path_from_webkit_base('WebKitBuild', 'selenium-tests', 'selenium')
+    import_path = webkit_finder.path_from_webkit_base('WebDriverTests', 'imported', 'selenium')
+    importers.append(Importer(importer_config_path, download_path, import_path))
+
+for importer in importers:
+    importer.download_tests()
+    importer.import_tests()

Tools/Scripts/webkitpy/webdriver_tests/webdriver_driver.py

@@ -34,9 +34,19 @@ def browser_name(self):
     def capabilities(self):
         raise NotImplementedError
 
+    def browser_path(self):
+        return None
+
+    def browser_args(self):
+        return []
+
     def browser_env(self):
         return {}
 
+    def selenium_name(self):
+        """Return the name of the driver used by Selenium, passed to pytest using --driver command line option.
+          If this is not implemented, Selenium tests will not be run."""
+        return None
 
 _drivers = {}
 

Tools/Scripts/webkitpy/webdriver_tests/webdriver_driver_gtk.py

@@ -34,10 +34,19 @@ def binary_path(self):
     def browser_name(self):
         return 'MiniBrowser'
 
+    def browser_path(self):
+        return self._port._build_path('bin', 'MiniBrowser')
+
+    def browser_args(self):
+        return ['--automation', '--javascript-can-open-windows-automatically=true', '--enable-xss-auditor=false']
+
     def capabilities(self):
         return {'webkitgtk:browserOptions': {
-            'binary': self._port._build_path('bin', 'MiniBrowser'),
-            'args': ['--automation', '--javascript-can-open-windows-automatically=true', '--enable-xss-auditor=false']}}
+            'binary': self.browser_path(),
+            'args': self.browser_args()}}
+
+    def selenium_name(self):
+        return 'WebKitGTK'
 
 
 register_driver('gtk', WebDriverGtk)

Tools/Scripts/webkitpy/webdriver_tests/webdriver_driver_wpe.py

@@ -35,10 +35,13 @@ def binary_path(self):
     def browser_name(self):
         return 'dyz'
 
+    def browser_args(self):
+        return ['--automation']
+
     def capabilities(self):
         return {'wpe:browserOptions': {
             'binary': self.browser_name(),
-            'args': ['--automation']}}
+            'args': self.browser_args()}}
 
     def browser_env(self):
         env = {}