Merge pull request #637 from OndraM/feature/fix-saucelabs

Improve test stability on SauceLabs

Author: OndraM

.travis.yml

@@ -49,7 +49,7 @@ matrix:
         jwt:
           secure: HPq5xFhosa1eSGnaRdJzeyEuaE0mhRlG1gf3G7+dKS0VniF30husSyrxZhbGCCKBGxmIySoAQzd43BCwL69EkUEVKDN87Cpid1Ce9KrSfU3cnN8XIb+4QINyy7x1a47RUAfaaOEx53TrW0ShalvjD+ZwDE8LrgagSox6KQ+nQLE=
     - php: 7.2
-      env: SAUCELABS=1 BROWSER_NAME="chrome" VERSION="latest" PLATFORM="Windows 10"
+      env: SAUCELABS=1 BROWSER_NAME="chrome" VERSION="74.0" PLATFORM="Windows 10"
       before_script:
         - php -S 127.0.0.1:8000 -t tests/functional/web/ &>>./logs/php-server.log &
         - until $(echo | nc localhost 8000); do sleep 1; echo waiting for PHP server on port 8000...; done; echo "PHP server started"
@@ -58,7 +58,7 @@ matrix:
         jwt:
           secure: HPq5xFhosa1eSGnaRdJzeyEuaE0mhRlG1gf3G7+dKS0VniF30husSyrxZhbGCCKBGxmIySoAQzd43BCwL69EkUEVKDN87Cpid1Ce9KrSfU3cnN8XIb+4QINyy7x1a47RUAfaaOEx53TrW0ShalvjD+ZwDE8LrgagSox6KQ+nQLE=
     - php: 7.2
-      env: SAUCELABS=1 BROWSER_NAME="MicrosoftEdge" VERSION="15.15063" PLATFORM="Windows 10"
+      env: SAUCELABS=1 BROWSER_NAME="MicrosoftEdge" VERSION="16.16299" PLATFORM="Windows 10"
       before_script:
         - php -S 127.0.0.1:8000 -t tests/functional/web/ &>>./logs/php-server.log &
         - until $(echo | nc localhost 8000); do sleep 1; echo waiting for PHP server on port 8000...; done; echo "PHP server started"

tests/functional/RemoteWebDriverTest.php

@@ -145,6 +145,7 @@ public function testShouldCloseWindow()
 
     /**
      * @covers ::executeScript
+     * @group exclude-saucelabs
      */
     public function testShouldExecuteScriptAndDoNotBlockExecution()
     {
@@ -153,17 +154,18 @@ public function testShouldExecuteScriptAndDoNotBlockExecution()
         $element = $this->driver->findElement(WebDriverBy::id('id_test'));
         $this->assertSame('Test by ID', $element->getText());
 
+        $start = microtime(true);
         $this->driver->executeScript('
             setTimeout(
-                function(){document.getElementById("id_test").innerHTML = "Text changed by script"},
-                500
+                function(){document.getElementById("id_test").innerHTML = "Text changed by script";},
+                250
             )');
+        $end = microtime(true);
 
-        // Make sure the script don't block the test execution
-        $this->assertSame('Test by ID', $element->getText());
+        $this->assertLessThan(250, $end - $start, 'executeScript() should not block execution');
 
-        // If we wait, the script should be executed
-        usleep(1000000); // wait 1000 ms
+        // If we wait, the script should be executed and its value changed
+        usleep(300000); // wait 300 ms
         $this->assertSame('Text changed by script', $element->getText());
     }
 
@@ -180,6 +182,7 @@ public function testShouldExecuteAsyncScriptAndWaitUntilItIsFinished()
         $element = $this->driver->findElement(WebDriverBy::id('id_test'));
         $this->assertSame('Test by ID', $element->getText());
 
+        $start = microtime(true);
         $this->driver->executeAsyncScript(
             'var callback = arguments[arguments.length - 1];
             setTimeout(
@@ -190,6 +193,13 @@ function(){
                 250
             );'
         );
+        $end = microtime(true);
+
+        $this->assertGreaterThan(
+            0.250,
+            $end - $start,
+            'executeAsyncScript() should block execution until callback() is called'
+        );
 
         // The result must be immediately available, as the executeAsyncScript should block the execution until the
         // callback is called.
@@ -204,7 +214,7 @@ public function testShouldTakeScreenshot()
         if (!extension_loaded('gd')) {
             $this->markTestSkipped('GD extension must be enabled');
         }
-        if ($this->desiredCapabilities->getBrowserName() == WebDriverBrowserType::HTMLUNIT) {
+        if ($this->desiredCapabilities->getBrowserName() === WebDriverBrowserType::HTMLUNIT) {
             $this->markTestSkipped('Screenshots are not supported by HtmlUnit browser');
         }
 
@@ -227,7 +237,7 @@ public function testShouldSaveScreenshotToFile()
         if (!extension_loaded('gd')) {
             $this->markTestSkipped('GD extension must be enabled');
         }
-        if ($this->desiredCapabilities->getBrowserName() == WebDriverBrowserType::HTMLUNIT) {
+        if ($this->desiredCapabilities->getBrowserName() === WebDriverBrowserType::HTMLUNIT) {
             $this->markTestSkipped('Screenshots are not supported by HtmlUnit browser');
         }
 

tests/functional/WebDriverActionsTest.php

@@ -122,10 +122,7 @@ public function testShouldDoubleClickOnElement()
             ->doubleClick($element)
             ->perform();
 
-        $this->assertSame(
-            ['mouseover item-3', 'mousedown item-3', 'mouseup item-3', 'click item-3', 'dblclick item-3'],
-            $this->retrieveLoggedEvents()
-        );
+        $this->assertContains('dblclick item-3', $this->retrieveLoggedEvents());
     }
 
     /**

tests/functional/WebDriverTestCase.php

@@ -44,7 +44,7 @@ protected function setUp()
     {
         $this->desiredCapabilities = new DesiredCapabilities();
 
-        if ($this->isSauceLabsBuild()) {
+        if (static::isSauceLabsBuild()) {
             $this->setUpSauceLabs();
         } else {
             if (getenv('BROWSER_NAME')) {
@@ -87,7 +87,7 @@ protected function tearDown()
     /**
      * @return bool
      */
-    public function isSauceLabsBuild()
+    public static function isSauceLabsBuild()
     {
         return getenv('SAUCELABS') ? true : false;
     }

tests/functional/WebDriverTimeoutsTest.php

@@ -42,7 +42,7 @@ public function testShouldGetDelayedElementWithImplicitWait()
     {
         $this->driver->get($this->getTestPageUrl('delayed_element.html'));
 
-        $this->driver->manage()->timeouts()->implicitlyWait(1);
+        $this->driver->manage()->timeouts()->implicitlyWait(2);
         $element = $this->driver->findElement(WebDriverBy::id('delayed'));
 
         $this->assertInstanceOf(RemoteWebElement::class, $element);
@@ -54,7 +54,7 @@ public function testShouldGetDelayedElementWithImplicitWait()
      */
     public function testShouldFailIfPageIsLoadingLongerThanPageLoadTimeout()
     {
-        if ($this->desiredCapabilities->getBrowserName() == WebDriverBrowserType::HTMLUNIT) {
+        if ($this->desiredCapabilities->getBrowserName() === WebDriverBrowserType::HTMLUNIT) {
             $this->markTestSkipped('Not supported by HtmlUnit browser');
         }
 

tests/functional/web/delayed_element.html

@@ -12,7 +12,7 @@
     setTimeout(function () {
         var wrapper = document.getElementById("wrapper");
         wrapper.innerHTML = '<div id="delayed">Element appearing after 500ms</div>';
-    }, 500);
+    }, 1500);
 </script>
 
 </body>