[py] Introduce marks for xfailing tests against specific drivers

conftest.py

@@ -21,6 +21,7 @@
 import urllib
 
 import pytest
+from _pytest.skipping import MarkEvaluator
 
 from selenium import webdriver
 from selenium.webdriver import DesiredCapabilities
@@ -62,10 +63,16 @@ def driver(request):
     except AttributeError:
         raise Exception('This test requires a --driver to be specified.')
 
-    skip = request.node.get_marker('ignore_{0}'.format(driver_class.lower()))
-    if skip is not None:
-        reason = skip.kwargs.get('reason') or skip.name
-        pytest.skip(reason)
+    # conditionally mark tests as expected to fail based on driver
+    request.node._evalxfail = request.node._evalxfail or MarkEvaluator(
+        request.node, 'xfail_{0}'.format(driver_class.lower()))
+
+    # skip driver instantiation if xfail(run=False)
+    if not request.config.getoption('runxfail'):
+        if request.node._evalxfail.istrue():
+            if request.node._evalxfail.get('run') is False:
+                yield
+                return
 
     if driver_class == 'BlackBerry':
         kwargs.update({'device_password': 'password'})

py/test/selenium/webdriver/common/api_example_tests.py

@@ -32,7 +32,11 @@
 
 import pytest
 
-from selenium.common.exceptions import NoSuchElementException, NoSuchWindowException
+from selenium.common.exceptions import (
+    NoSuchElementException,
+    NoSuchWindowException,
+    TimeoutException,
+    WebDriverException)
 from selenium.webdriver.support.wait import WebDriverWait
 
 
@@ -121,9 +125,11 @@ def testFindElementByTagNameWithinElement(self, driver, pages):
         elems = div.find_elements_by_tag_name("p")
         assert len(elems) == 1
 
+    @pytest.mark.xfail_marionette(
+        reason="W3C implementations can't switch to a window by name",
+        raises=TimeoutException,
+        run=False)
     def testSwitchToWindow(self, driver, pages):
-        if driver.w3c:
-            pytest.xfail("W3C implementations can't switch to a window by name")
         title_1 = "XHTML Test Page"
         title_2 = "We Arrive Here"
         switch_to_window_timeout = 5
@@ -221,9 +227,9 @@ def testIsElementDisplayed(self, driver, pages):
         assert visible
         assert not not_visible
 
+    @pytest.mark.xfail_phantomjs(
+        reason='https://github.com/detro/ghostdriver/issues/466')
     def testMoveWindowPosition(self, driver, pages):
-        if driver.capabilities['browserName'] == 'phantomjs':
-            pytest.xfail("phantomjs driver does not support moving the window position")
         pages.load("blank.html")
         loc = driver.get_window_position()
         # note can't test 0,0 since some OS's dont allow that location
@@ -252,12 +258,14 @@ def testChangeWindowSize(self, driver, pages):
         assert size['width'] == newSize[0]
         assert size['height'] == newSize[1]
 
-    @pytest.mark.ignore_marionette
+    @pytest.mark.xfail_marionette(
+        raises=WebDriverException)
     def testGetLogTypes(self, driver, pages):
         pages.load("blank.html")
         assert isinstance(driver.log_types, list)
 
-    @pytest.mark.ignore_marionette
+    @pytest.mark.xfail_marionette(
+        raises=WebDriverException)
     def testGetLog(self, driver, pages):
         pages.load("blank.html")
         for log_type in driver.log_types:

py/test/selenium/webdriver/common/appcache_tests.py

@@ -17,17 +17,16 @@
 
 import pytest
 
+from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.common.html5.application_cache import ApplicationCache
 
 
 class TestAppCache(object):
 
-    @pytest.mark.ignore_firefox
-    @pytest.mark.ignore_marionette
-    @pytest.mark.ignore_chrome
+    @pytest.mark.xfail_chrome
+    @pytest.mark.xfail_marionette(raises=WebDriverException)
+    @pytest.mark.xfail_phantomjs(raises=WebDriverException)
     def testWeCanGetTheStatusOfTheAppCache(self, driver, pages):
-        if driver.capabilities['browserName'] == 'phantomjs':
-            pytest.xfail("phantomjs driver does not implement appcache")
         pages.load('html5Page')
         driver.implicitly_wait(2)
         app_cache = driver.application_cache

py/test/selenium/webdriver/common/click_scrolling_tests.py

@@ -17,8 +17,11 @@
 
 import pytest
 
+from selenium.common.exceptions import (
+    ElementNotVisibleException,
+    MoveTargetOutOfBoundsException,
+    WebDriverException)
 from selenium.webdriver.common.by import By
-from selenium.common.exceptions import MoveTargetOutOfBoundsException
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
@@ -53,19 +56,20 @@ def testShouldScrollToClickOnAnElementHiddenByOverflow(self, driver, pages):
         except MoveTargetOutOfBoundsException as e:
             AssertionError("Should not be out of bounds: %s" % e.msg)
 
+    @pytest.mark.xfail_marionette(
+        reason='https://github.com/w3c/webdriver/issues/408')
     def testShouldBeAbleToClickOnAnElementHiddenByOverflow(self, driver, pages):
-        if driver.capabilities['browserName'] == 'firefox' and driver.w3c == True:
-            pytest.xfail("Marionette and W3C: https://github.com/w3c/webdriver/issues/408")
         pages.load("scroll.html")
 
         link = driver.find_element(By.ID, "line8")
         # This used to throw a MoveTargetOutOfBoundsException - we don't expect it to
         link.click()
         assert "line8" == driver.find_element(By.ID, "clicked").text
 
+    @pytest.mark.xfail_chrome(
+        reason='https://bugs.chromium.org/p/chromedriver/issues/detail?id=1536',
+        raises=WebDriverException)
     def testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow(self, driver, pages):
-        if driver.capabilities['browserName'] == 'chrome':
-            pytest.xfail("Chrome Issue: https://bugs.chromium.org/p/chromedriver/issues/detail?id=1536")
         pages.load("scrolling_tests/page_with_double_overflow_auto.html")
 
         driver.find_element(By.ID, "link").click()
@@ -85,9 +89,9 @@ def testShouldNotScrollOverflowElementsWhichAreVisible(self, driver, pages):
         yOffset = driver.execute_script("return arguments[0].scrollTop", list)
         assert 0 == yOffset, "Should not have scrolled"
 
+    @pytest.mark.xfail_chrome(
+        reason='https://bugs.chromium.org/p/chromedriver/issues/detail?id=1542')
     def testShouldNotScrollIfAlreadyScrolledAndElementIsInView(self, driver, pages):
-        if driver.capabilities['browserName'] == 'chrome':
-            pytest.xfail("Chrome Issue: https://bugs.chromium.org/p/chromedriver/issues/detail?id=1542")
         pages.load("scroll3.html")
         driver.find_element(By.ID, "button1").click()
         scrollTop = self.getScrollTop(driver)
@@ -99,16 +103,17 @@ def testShouldBeAbleToClickRadioButtonScrolledIntoView(self, driver, pages):
         driver.find_element(By.ID, "radio").click()
         # If we don't throw, we're good
 
+    @pytest.mark.xfail_marionette(
+        reason='https://github.com/w3c/webdriver/issues/408',
+        raises=ElementNotVisibleException)
     def testShouldScrollOverflowElementsIfClickPointIsOutOfViewButElementIsInView(self, driver, pages):
-        if driver.capabilities['browserName'] == 'firefox':
-            pytest.skip("Scrolling to containing frames isnt spec'ed https://github.com/w3c/webdriver/issues/408")
         pages.load("scroll5.html")
         driver.find_element(By.ID, "inner").click()
         assert "clicked" == driver.find_element(By.ID, "clicked").text
 
+    @pytest.mark.xfail_marionette(
+        reason='https://github.com/w3c/webdriver/issues/408')
     def testShouldBeAbleToClickElementInAFrameThatIsOutOfView(self, driver, pages):
-        if driver.capabilities['browserName'] == 'firefox':
-            pytest.skip("Scrolling to containing frames isnt spec'ed https://github.com/w3c/webdriver/issues/408")
         pages.load("scrolling_tests/page_with_frame_out_of_view.html")
         driver.switch_to.frame(driver.find_element_by_name("frame"))
         element = driver.find_element(By.NAME, "checkbox")
@@ -161,9 +166,9 @@ def testShouldNotScrollWhenGettingElementSize(self, driver, pages):
     def getScrollTop(self, driver):
         return driver.execute_script("return document.body.scrollTop")
 
+    @pytest.mark.xfail_marionette(
+        reason='https://github.com/w3c/webdriver/issues/408')
     def testShouldBeAbleToClickElementInATallFrame(self, driver, pages):
-        if driver.capabilities['browserName'] == 'firefox':
-            pytest.skip("Scrolling to containing frames isnt spec'ed https://github.com/w3c/webdriver/issues/408")
         pages.load("scrolling_tests/page_with_tall_frame.html")
         driver.switch_to.frame(driver.find_element_by_name("tall_frame"))
         element = driver.find_element(By.NAME, "checkbox")

py/test/selenium/webdriver/common/cookie_tests.py

@@ -47,9 +47,8 @@ def testAddCookie(self, cookie, driver):
         returned = driver.execute_script('return document.cookie')
         assert cookie['name'] in returned
 
+    @pytest.mark.xfail_ie
     def testAddingACookieThatExpiredInThePast(self, cookie, driver):
-        if driver.name == 'internet explorer':
-            pytest.skip("Issue needs investigating")
         expired = cookie.copy()
         expired['expiry'] = calendar.timegm(time.gmtime()) - 1
         driver.add_cookie(expired)

py/test/selenium/webdriver/common/driver_element_finding_tests.py

@@ -19,8 +19,11 @@
 
 from selenium.webdriver.common.by import By
 from selenium.common.exceptions import (
+    InvalidElementStateException,
     InvalidSelectorException,
-    NoSuchElementException)
+    NoSuchElementException,
+    NoSuchWindowException,
+    WebDriverException)
 
 
 class TestDriverElementFinding(object):
@@ -63,13 +66,13 @@ def test_Should_Not_Be_Able_To_Locate_By_Id_Multiple_Elements_That_Do_Not_Exist(
         elements = driver.find_elements(By.ID, "non_Existent_Button")
         assert len(elements) == 0
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchWindowException)
     def test_Finding_ASingle_Element_By_Empty_Id_Should_Throw(self, driver, pages):
         pages.load("formPage.html")
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.ID, "")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchElementException)
     def test_Finding_Multiple_Elements_By_Empty_Id_Should_Return_Empty_List(self, driver, pages):
         pages.load("formPage.html")
         elements = driver.find_elements(By.ID, "")
@@ -114,13 +117,13 @@ def test_Should_Not_Be_Able_To_Locate_By_Name_Multiple_Elements_That_Do_Not_Exis
         elements = driver.find_elements(By.NAME, "non_Existent_Button")
         assert len(elements) == 0
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchWindowException)
     def test_Finding_ASingle_Element_By_Empty_Name_Should_Throw(self, driver, pages):
         pages.load("formPage.html")
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.NAME, "")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchElementException)
     def test_Finding_Multiple_Elements_By_Empty_Name_Should_Return_Empty_List(self, driver, pages):
         pages.load("formPage.html")
         elements = driver.find_elements(By.NAME, "")
@@ -160,18 +163,18 @@ def test_Should_Not_Be_Able_To_Locate_By_Tag_Name_Multiple_Elements_That_Do_Not_
         elements = driver.find_elements(By.TAG_NAME, "non_Existent_Button")
         assert len(elements) == 0
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_chrome(
+        reason='https://bugs.chromium.org/p/chromedriver/issues/detail?id=1541')
+    @pytest.mark.xfail_phantomjs
     def test_Finding_ASingle_Element_By_Empty_Tag_Name_Should_Throw(self, driver, pages):
-        if driver.capabilities['browserName'] == 'chrome':
-            pytest.xfail("Chrome issue: https://bugs.chromium.org/p/chromedriver/issues/detail?id=1541")
         pages.load("formPage.html")
         with pytest.raises(InvalidSelectorException):
             driver.find_element(By.TAG_NAME, "")
 
-    @pytest.mark.ignore_phantomjs
-    def test_Finding_Multiple_Elements_By_Empty_Tag_Name_Should_Return_Empty_List(self, driver, pages):
-        if driver.capabilities['browserName'] == 'chrome':
-            pytest.xfail("Chrome Issue: https://bugs.chromium.org/p/chromedriver/issues/detail?id=1540")
+    @pytest.mark.xfail_chrome(
+        reason='https://bugs.chromium.org/p/chromedriver/issues/detail?id=1541')
+    @pytest.mark.xfail_phantomjs
+    def test_Finding_Multiple_Elements_By_Empty_Tag_Name_Should_Throw(self, driver, pages):
         pages.load("formPage.html")
         with pytest.raises(InvalidSelectorException):
             driver.find_elements(By.TAG_NAME, "")
@@ -231,31 +234,31 @@ def test_Should_Not_Find_Element_By_Class_When_The_Name_Queried_Is_Shorter_Than_
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.CLASS_NAME, "name_B")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchWindowException)
     def test_Finding_ASingle_Element_By_Empty_Class_Name_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.CLASS_NAME, "")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchElementException)
     def test_Finding_Multiple_Elements_By_Empty_Class_Name_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
             driver.find_elements(By.CLASS_NAME, "")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=WebDriverException)
     def test_Finding_ASingle_Element_By_Compound_Class_Name_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.CLASS_NAME, "a b")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=InvalidElementStateException)
     def test_Finding_ASingle_Element_By_Invalid_Class_Name_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.CLASS_NAME, "!@#$%^&*")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=InvalidElementStateException)
     def test_Finding_Multiple_Elements_By_Invalid_Class_Name_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
@@ -303,10 +306,11 @@ def test_Finding_ALink_By_Xpath_Using_Contains_Keyword_Should_Work(self, driver,
         element = driver.find_element(By.XPATH, "//a[contains(.,'hello world')]")
         assert "hello world" in element.text
 
-    @pytest.mark.ignore_firefox
-    @pytest.mark.ignore_marionette
-    @pytest.mark.ignore_phantomjs
-    @pytest.mark.ignore_chrome
+    @pytest.mark.xfail_chrome(raises=InvalidSelectorException)
+    @pytest.mark.xfail_firefox(raises=InvalidSelectorException)
+    @pytest.mark.xfail_marionette(raises=WebDriverException)
+    @pytest.mark.xfail_phantomjs(raises=InvalidSelectorException)
+    @pytest.mark.xfail_safari(raises=NoSuchElementException)
     def test_Should_Be_Able_To_Find_Element_By_XPath_With_Namespace(self, driver, pages):
         pages.load("svgPage.html")
         element = driver.find_element(By.XPATH, "//svg:svg//svg:text")
@@ -421,7 +425,7 @@ def test_Should_Not_Find_Elements_By_Css_Selector_When_There_Is_No_Such_Element(
         elements = driver.find_elements(By.CSS_SELECTOR, ".there-is-no-such-class")
         assert len(elements) == 0
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=NoSuchWindowException)
     def test_Finding_ASingle_Element_By_Empty_Css_Selector_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
@@ -432,13 +436,13 @@ def test_Finding_Multiple_Elements_By_Empty_Css_Selector_Should_Throw(self, driv
         with pytest.raises(NoSuchElementException):
             driver.find_elements(By.CSS_SELECTOR, "")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=InvalidElementStateException)
     def test_Finding_ASingle_Element_By_Invalid_Css_Selector_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):
             driver.find_element(By.CSS_SELECTOR, "//a/b/c[@id='1']")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(raises=InvalidElementStateException)
     def test_Finding_Multiple_Elements_By_Invalid_Css_Selector_Should_Throw(self, driver, pages):
         pages.load("xhtmlTest.html")
         with pytest.raises(NoSuchElementException):

py/test/selenium/webdriver/common/element_attribute_tests.py

@@ -91,9 +91,9 @@ def testShouldIndicateWhenATextAreaIsDisabled(self, driver, pages):
         textArea = driver.find_element_by_xpath("//textarea[@id='notWorkingArea']")
         assert not textArea.is_enabled()
 
+    @pytest.mark.xfail_marionette(
+        reason='https://bugzilla.mozilla.org/show_bug.cgi?id=1309234')
     def testShouldThrowExceptionIfSendingKeysToElementDisabledUsingRandomDisabledStrings(self, driver, pages):
-        if driver.capabilities['browserName'] == 'firefox' and driver.w3c == True:
-            pytest.xfail("Marionette Issue: https://bugzilla.mozilla.org/show_bug.cgi?id=1309234")
         pages.load("formPage.html")
         disabledTextElement1 = driver.find_element_by_id("disabledTextElement1")
         with pytest.raises(WebDriverException):
@@ -231,7 +231,7 @@ def testShouldReturnNullForNonPresentBooleanAttributes(self, driver, pages):
         element1 = driver.find_element_by_id("working")
         assert element1.get_attribute("required") is None
 
-    @pytest.mark.ignore_ie
+    @pytest.mark.xfail_ie
     def testShouldReturnTrueForPresentBooleanAttributes(self, driver, pages):
         pages.load("booleanAttributes.html")
         element1 = driver.find_element_by_id("emailRequired")

py/test/selenium/webdriver/common/executing_async_javascript_tests.py

@@ -94,14 +94,14 @@ def testShouldBeAbleToReturnArraysOfWebElementsFromAsyncScripts(self, driver, pa
         assert "body" == list_[0].tag_name
         # assert list_[0] == list_[1]
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(run=False)
     def testShouldTimeoutIfScriptDoesNotInvokeCallback(self, driver, pages):
         pages.load("ajaxy_page.html")
         with pytest.raises(TimeoutException):
             # Script is expected to be async and explicitly callback, so this should timeout.
             driver.execute_async_script("return 1 + 2;")
 
-    @pytest.mark.ignore_phantomjs
+    @pytest.mark.xfail_phantomjs(run=False)
     def testShouldTimeoutIfScriptDoesNotInvokeCallbackWithAZeroTimeout(self, driver, pages):
         pages.load("ajaxy_page.html")
         with pytest.raises(TimeoutException):