Merge r227127 - WebDriver: ignore the driver in selenium test names w…

…hen getting expectations https://bugs.webkit.org/show_bug.cgi?id=181738 Reviewed by Carlos Alberto Lopez Perez. Tools: In selenium tests, the driver is added as a parameter to every test which results in tests names like foo[DriverName] or foo[DriverName-param2] in case of tests using more parameters. We don't want to include the driver name in the test expectations file, so we need to remove it when querying the expectations. * Scripts/webkitpy/webdriver_tests/pytest_runner.py: (TestExpectationsMarker.__init__): Save the param to ignore. (TestExpectationsMarker._item_name): Return the name of the test without the para to ignore. (TestExpectationsMarker.pytest_collection_modifyitems): Use _item_name(). (run): Pass param to ignore to TestExpectationsMarker(). * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py: (WebDriverSeleniumExecutor.__init__): Save the driver name. (WebDriverSeleniumExecutor.run): Pass the driver name as param to ignore. WebDriverTests: Add some expectations for selenium tests. * TestExpectations.json:
WebKit · Jan 24, 2018 · b2ca7a5 · b2ca7a5
1 parent 8b2d6c4
commit b2ca7a5
Show file tree

Hide file tree

Showing 5 changed files with 160 additions and 6 deletions.
diff --git a/Tools/ChangeLog b/Tools/ChangeLog
@@ -1,3 +1,23 @@
+2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: ignore the driver in selenium test names when getting expectations
+        https://bugs.webkit.org/show_bug.cgi?id=181738
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        In selenium tests, the driver is added as a parameter to every test which results in tests names like
+        foo[DriverName] or foo[DriverName-param2] in case of tests using more parameters. We don't want to include the
+        driver name in the test expectations file, so we need to remove it when querying the expectations.
+
+        * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
+        (TestExpectationsMarker.__init__): Save the param to ignore.
+        (TestExpectationsMarker._item_name): Return the name of the test without the para to ignore.
+        (TestExpectationsMarker.pytest_collection_modifyitems): Use _item_name().
+        (run): Pass param to ignore to TestExpectationsMarker().
+        * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
+        (WebDriverSeleniumExecutor.__init__): Save the driver name.
+        (WebDriverSeleniumExecutor.run): Pass the driver name as param to ignore.
+
 2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         WebDriver: add support for test expectations

diff --git a/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py b/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py
@@ -130,14 +130,29 @@ def record(self, test, status, message=None, stack=None):
 
 class TestExpectationsMarker(object):
 
-    def __init__(self, expectations):
+    def __init__(self, expectations, ignore_param):
         self._expectations = expectations
+        self._ignore_param = ignore_param
         self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
 
+    def _item_name(self, item):
+        if self._ignore_param is None:
+            return item.name
+
+        single_param = '[%s]' % self._ignore_param
+        if item.name.endswith(single_param):
+            return item.name[:-len(single_param)]
+
+        param = '[%s-' % self._ignore_param
+        if param in item.name:
+            return item.name.replace('%s-' % self._ignore_param, '')
+
+        return item.name
+
     def pytest_collection_modifyitems(self, session, config, items):
         for item in items:
             test = os.path.relpath(str(item.fspath), self._base_dir)
-            expected = self._expectations.get_expectation(test, item.name)[0]
+            expected = self._expectations.get_expectation(test, self._item_name(item))[0]
             if expected == 'FAIL':
                 item.add_marker(pytest.mark.xfail)
             elif expected == 'TIMEOUT':
@@ -161,10 +176,10 @@ def collect(directory, args):
     return collect_recorder.tests
 
 
-def run(path, args, timeout, env, expectations):
+def run(path, args, timeout, env, expectations, ignore_param=None):
     harness_recorder = HarnessResultRecorder()
     subtests_recorder = SubtestResultRecorder()
-    expectations_marker = TestExpectationsMarker(expectations)
+    expectations_marker = TestExpectationsMarker(expectations, ignore_param)
     _environ = dict(os.environ)
     os.environ.clear()
     os.environ.update(env)

diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py
@@ -41,10 +41,11 @@ def do_delayed_imports():
 class WebDriverSeleniumExecutor(object):
 
     def __init__(self, driver, display_driver):
+        self._driver_name = driver.selenium_name()
         self._env = display_driver._setup_environ_for_test()
         self._env.update(driver.browser_env())
 
-        self._args = ['--driver=%s' % driver.selenium_name(), '--driver-binary=%s' % driver.binary_path().encode()]
+        self._args = ['--driver=%s' % self._driver_name, '--driver-binary=%s' % driver.binary_path().encode()]
         browser_path = driver.browser_path().encode()
         if browser_path:
             self._args.extend(['--browser-binary=%s' % browser_path])
@@ -59,4 +60,4 @@ def collect(self, directory):
         return pytest_runner.collect(directory, self._args)
 
     def run(self, test, timeout, expectations):
-        return pytest_runner.run(test, self._args, timeout, self._env, expectations)
+        return pytest_runner.run(test, self._args, timeout, self._env, expectations, self._driver_name)
diff --git a/WebDriverTests/ChangeLog b/WebDriverTests/ChangeLog
@@ -1,3 +1,14 @@
+2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: ignore the driver in selenium test names when getting expectations
+        https://bugs.webkit.org/show_bug.cgi?id=181738
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        Add some expectations for selenium tests.
+
+        * TestExpectations.json:
+
 2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         WebDriver: add support for test expectations

diff --git a/WebDriverTests/TestExpectations.json b/WebDriverTests/TestExpectations.json
@@ -1,4 +1,111 @@
 {
+    "imported/selenium/py/test/selenium/webdriver/common/alerts_tests.py": {
+        "subtests": {
+            "testShouldAllowUsersToAcceptAnAlertInAFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181727"}}
+            },
+            "testShouldAllowUsersToAcceptAnAlertInANestedFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181727"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/api_example_tests.py": {
+        "subtests": {
+            "testChangeWindowSize": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181728"}}
+            },
+            "testGetLogTypes": {
+                "expected": {"all": {"status": ["SKIP"]}}
+            },
+            "testGetLog": {
+                "expected": {"all": {"status": ["SKIP"]}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/appcache_tests.py": {
+        "expected": {"all": {"status": ["SKIP"]}}
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/click_scrolling_tests.py": {
+        "subtests": {
+            "testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/174902"}}
+            },
+            "testShouldNotScrollIfAlreadyScrolledAndElementIsInView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181729"}}
+            },
+            "testShouldScrollOverflowElementsIfClickPointIsOutOfViewButElementIsInView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181730"}}
+            },
+            "testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldNotScrollWhenGettingElementSize": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldBeAbleToClickElementInATallFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/correct_event_firing_tests.py": {
+        "subtests": {
+            "testShouldEmitOnChangeEventsWhenChangingTheStateOfACheckbox": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181732"}}
+            },
+            "testClearingAnElementShouldCauseTheOnChangeHandlerToFire": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181732"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/form_handling_tests.py": {
+        "subtests": {
+            "testShouldThrowAnExceptionWhenSelectingAnUnselectableElement": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181733"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/interactions_tests.py": {
+        "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/position_and_size_tests.py": {
+        "subtests": {
+            "testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181734"}}
+            },
+            "testShouldGetCoordinatesOfAnElementInAFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181735"}}
+            },
+            "testShouldGetCoordinatesOfAnElementInANestedFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181735"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/rendered_webelement_tests.py": {
+        "subtests": {
+            "testShouldPickUpStyleOfAnElement": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181736"}}
+            },
+            "testShouldAllowInheritedStylesToBeUsed": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181736"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/text_handling_tests.py": {
+        "subtests": {
+            "testReadALargeAmountOfData": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181737"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/w3c_interaction_tests.py": {
+        "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+    },
     "imported/w3c/webdriver/tests/actions/key.py": {
         "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
     },