Hardening against intermittent failures

hal · Mar 14, 2024 · bdae1c1 · bdae1c1
1 parent 7dcacb4
commit bdae1c1
Show file tree

Hide file tree

Showing 18 changed files with 170 additions and 51 deletions.
diff --git a/common/src/main/java/org/jboss/hal/testsuite/Console.java b/common/src/main/java/org/jboss/hal/testsuite/Console.java
@@ -41,6 +41,7 @@
 import org.jboss.hal.testsuite.util.Library;
 import org.openqa.selenium.By;
 import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.StaleElementReferenceException;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 
@@ -127,11 +128,18 @@ public void verify(PlaceRequest placeRequest) {
     /** Waits until all notifications are gone. */
     public void waitNoNotification() {
         List<WebElement> dismissibleNotifications = By.cssSelector(DOT + alertDismissable).findElements(browser);
-        for (WebElement notification : dismissibleNotifications) {
-            WebElement button = notification.findElement(By.cssSelector("button.close"));
-            if (button != null) {
-                button.click();
+        for (int remainingExpected = dismissibleNotifications.size(); !dismissibleNotifications.isEmpty()
+                && remainingExpected > 0; remainingExpected--) {
+
+            try {
+                WebElement button = dismissibleNotifications.get(0).findElement(By.cssSelector("button.close"));
+                if (button != null) {
+                    button.click();
+                }
+            } catch (StaleElementReferenceException ex) {
+                // Ignore the exception, the notification may be gone before clicking
             }
+            dismissibleNotifications = By.cssSelector(DOT + alertDismissable).findElements(browser);
         }
         waitModel().until().element(By.cssSelector(DOT + toastNotificationsListPf + ":empty")).is().present();
     }
@@ -152,7 +160,7 @@ public boolean verifyNoError() {
     }
 
     private void verifyNotification(String css) {
-        waitModel().until() // use waitModel() since it might take some time until the notification is visible
+        waitModel().until() // use waitModel() since the notification might take some time until the notification is visible
                 .element(By.cssSelector(DOT + toastNotificationsListPf + " ." + css))
                 .is().visible();
     }

diff --git a/common/src/main/java/org/jboss/hal/testsuite/CrudOperations.java b/common/src/main/java/org/jboss/hal/testsuite/CrudOperations.java
@@ -23,11 +23,14 @@
 import org.jboss.hal.testsuite.creaper.ManagementClientProvider;
 import org.jboss.hal.testsuite.creaper.ResourceVerifier;
 import org.jboss.hal.testsuite.fragment.AddResourceDialogFragment;
+import org.jboss.hal.testsuite.fragment.EmptyState;
 import org.jboss.hal.testsuite.fragment.FormFragment;
 import org.jboss.hal.testsuite.fragment.TableFragment;
+import org.openqa.selenium.TimeoutException;
 import org.wildfly.extras.creaper.core.online.OnlineManagementClient;
 import org.wildfly.extras.creaper.core.online.operations.Address;
 
+import static org.jboss.arquillian.graphene.Graphene.waitGui;
 import static org.jboss.hal.dmr.ModelDescriptionConstants.NAME;
 
 /** Methods useful to test and verify CRUD operations in application views. */
@@ -76,7 +79,20 @@ public void createSingleton(Address address, FormFragment form, Consumer<FormFra
     /** Adds a singleton resource using the main action of an empty state and the specified initial form values. */
     public void createSingleton(Address address, FormFragment form, Consumer<FormFragment> initialValues,
             VerifyChanges verifyChanges) throws Exception {
-        form.emptyState().mainAction();
+
+        try {
+            EmptyState emptyState = form.emptyState();
+            waitGui().until().element(emptyState.getRoot()).is().visible();
+            emptyState.mainAction();
+        } catch (TimeoutException ex) {
+            System.out.println("[DEBUG] Form text: '" + form.getRoot().getText() + "'");
+            // try again to get rid of intermittent failures
+            EmptyState emptyState = form.emptyState();
+            System.out.println("[DEBUG] Form emptyState text: '" + emptyState.getRoot().getText() + "'");
+            waitGui().until().element(emptyState.getRoot()).is().visible();
+            emptyState.mainAction();
+        }
+
         if (initialValues != null) {
             AddResourceDialogFragment dialog = console.addResourceDialog();
             initialValues.accept(dialog.getForm()); // use the form of add resource dialog!

diff --git a/common/src/main/java/org/jboss/hal/testsuite/Random.java b/common/src/main/java/org/jboss/hal/testsuite/Random.java
@@ -51,9 +51,9 @@ public static String jndiName(String name) {
         return JNDI_PREFIX + name;
     }
 
-    /** Returns a random integer between 0 and 99 */
+    /** Returns a random integer between 1 and 99 */
     public static int number() {
-        return RandomUtils.nextInt(0, 100);
+        return RandomUtils.nextInt(1, 100);
     }
 
     /** Returns a random double between 0.001 and 99.999 */

diff --git a/common/src/main/java/org/jboss/hal/testsuite/fragment/EmptyState.java b/common/src/main/java/org/jboss/hal/testsuite/fragment/EmptyState.java
@@ -16,9 +16,11 @@
 package org.jboss.hal.testsuite.fragment;
 
 import org.jboss.arquillian.graphene.fragment.Root;
+import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.support.FindBy;
 
+import static org.jboss.arquillian.graphene.Graphene.waitGui;
 import static org.jboss.hal.resources.CSS.blankSlatePfMainAction;
 import static org.jboss.hal.resources.CSS.btnPrimary;
 
@@ -30,6 +32,7 @@ public class EmptyState {
 
     /** Clicks on the main action */
     public void mainAction() {
+        waitGui().until().element(root, By.cssSelector("." + blankSlatePfMainAction + " button." + btnPrimary)).is().visible();
         primaryButton.click();
     }
 

diff --git a/common/src/main/java/org/jboss/hal/testsuite/fragment/FormFragment.java b/common/src/main/java/org/jboss/hal/testsuite/fragment/FormFragment.java
@@ -26,6 +26,7 @@
 import org.openqa.selenium.By;
 import org.openqa.selenium.ElementClickInterceptedException;
 import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.Keys;
 import org.openqa.selenium.TimeoutException;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebDriverException;
@@ -35,6 +36,7 @@
 
 import static org.jboss.arquillian.graphene.Graphene.createPageFragment;
 import static org.jboss.arquillian.graphene.Graphene.waitGui;
+import static org.jboss.arquillian.graphene.Graphene.waitModel;
 import static org.jboss.hal.resources.CSS.*;
 import static org.jboss.hal.resources.UIConstants.MASK_CHARACTER;
 
@@ -148,8 +150,18 @@ public void trySave() {
      * {@link #trySave()} instead.
      */
     public void save() {
-        console.scrollIntoView(saveButton).click();
-        waitGui().until().element(readOnlySection).is().visible();
+        try {
+            console.scrollIntoView(saveButton).click();
+        } catch (ElementClickInterceptedException ex) {
+            if (ex.getMessage().contains(autocompleteSuggestions)) {
+                // get rid of any open auto suggestions popup, which is interfering with the save button
+                saveButton.sendKeys(Keys.TAB);
+                waitGui().until().element(browser, ByJQuery
+                        .selector(DOT + autocompleteSuggestions + "[style*='display: none']")).is().present();
+                console.scrollIntoView(saveButton).click();
+            }
+        }
+        waitModel().until().element(readOnlySection).is().visible();
     }
 
     /**
@@ -168,6 +180,15 @@ public void text(String name, String value) {
         waitGui().until().element(inputElement).value().equalTo("");
         inputElement.sendKeys(value);
         waitGui().until().element(inputElement).value().equalTo(value);
+
+        // get rid of any open auto suggestions popup, which might interfere with save or cancel buttons
+        if (!ByJQuery.selector(DOT + autocompleteSuggestions + "[style*='display: block']")
+                .findElements(browser).isEmpty()) {
+
+            inputElement.sendKeys(Keys.TAB);
+            waitGui().until().element(browser, ByJQuery
+                    .selector(DOT + autocompleteSuggestions + "[style*='display: none']")).is().present();
+        }
     }
 
     public void textByLabel(String labelContent, String value) {

diff --git a/common/src/main/java/org/jboss/hal/testsuite/fragment/TableFragment.java b/common/src/main/java/org/jboss/hal/testsuite/fragment/TableFragment.java
@@ -28,6 +28,7 @@
 import org.openqa.selenium.support.FindBy;
 
 import static org.jboss.arquillian.graphene.Graphene.waitGui;
+import static org.jboss.arquillian.graphene.Graphene.waitModel;
 import static org.jboss.hal.resources.CSS.columnAction;
 import static org.jboss.hal.resources.CSS.halTableButtons;
 import static org.jboss.hal.testsuite.Selectors.contains;
@@ -91,6 +92,7 @@ public WebElement button(String text) {
      */
     public void select(String value) {
         By selector = ByJQuery.selector("td" + contains(value));
+        waitModel().until().element(root, selector).is().present();
         goToPageWithElement(selector);
         WebElement rowElement = root.findElement(selector);
 
@@ -120,6 +122,7 @@ public void select(String value) {
     /** Clicks on the &lt;action&gt; column in the row which contains "&lt;value&gt;". */
     public void action(String value, String action) {
         By selector = ByJQuery.selector("td" + contains(value) + " ~ td button." + columnAction + contains(action));
+        waitModel().until().element(root, selector).is().present();
         goToPageWithElement(selector);
         root.findElement(selector).click();
     }

diff --git a/common/src/main/java/org/jboss/hal/testsuite/fragment/VerticalNavigationFragment.java b/common/src/main/java/org/jboss/hal/testsuite/fragment/VerticalNavigationFragment.java
@@ -38,7 +38,6 @@ public void selectPrimary(String id) {
     public void selectSecondary(String primaryId, String secondaryId) {
         WebElement primaryMenuItem = root.findElement(By.cssSelector("#" + primaryId + " > a"));
         new Actions(browser).moveToElement(primaryMenuItem).perform();
-        primaryMenuItem.click();
         waitGui().until().element(By.id(primaryId + "-secondary")).is().visible();
         selectItem(secondaryId);
     }

diff --git a/common/src/main/java/org/jboss/hal/testsuite/page/BasePage.java b/common/src/main/java/org/jboss/hal/testsuite/page/BasePage.java
@@ -56,5 +56,10 @@ public void navigate(Map<String, String> params) {
     public void navigateAgain(String name, String value) {
         browser.navigate().refresh();
         navigate(name, value);
+        if (!console.verifyNoError()) {
+            // try again, the resource may be unavailable
+            browser.navigate().refresh();
+            navigate(name, value);
+        }
     }
 }
diff --git a/...ytron/src/test/java/org/jboss/hal/testsuite/test/configuration/elytron/JDBCRealmTest.java b/...ytron/src/test/java/org/jboss/hal/testsuite/test/configuration/elytron/JDBCRealmTest.java
@@ -97,6 +97,7 @@
 import static org.jboss.hal.testsuite.fixtures.ElytronFixtures.SQL_UPDATE2;
 import static org.jboss.hal.testsuite.fixtures.ElytronFixtures.datasourceAddress;
 import static org.jboss.hal.testsuite.fixtures.ElytronFixtures.jdbcRealmAddress;
+
 @RunWith(Arquillian.class)
 public class JDBCRealmTest {
 
@@ -628,9 +629,10 @@ public void principalQueryModularCryptMapperUpdate() throws Exception {
         TableFragment table = page.getPrincipalQueryTable();
         FormFragment form = page.getPrincipalQueryMcryptForm();
         table.bind(form);
+        jdbcTable.filter(JDBC_RLM_UPDATE);
         jdbcTable.action(JDBC_RLM_UPDATE, PQ_LABEL);
         waitGui().until().element(table.getRoot()).is().visible();
-        table.select(SQL_MCM_UPD);
+        table.filterAndSelect(SQL_MCM_UPD);
         page.getPrincipalQueryTabs().select(MCRYPT_MAPPER_TAB);
         long number = 23;
         ModelNode mapper = new ModelNode();

diff --git a/.../test/java/org/jboss/hal/testsuite/test/configuration/undertow/SessionOperationsTest.java b/.../test/java/org/jboss/hal/testsuite/test/configuration/undertow/SessionOperationsTest.java
@@ -42,6 +42,8 @@
 import org.wildfly.extras.creaper.core.online.operations.Operations;
 import org.wildfly.extras.creaper.core.online.operations.Values;
 
+import static org.jboss.arquillian.graphene.Graphene.waitGui;
+
 @RunWith(Arquillian.class)
 public class SessionOperationsTest {
 
@@ -147,6 +149,7 @@ private void verifyAttributeIsContained(String key, String value, List<WebElemen
 
     private void reloadSessions() {
         page.getSessionsTable().button("Reload").click();
+        waitGui().until().element(page.getSessionsTable().getRoot(), By.cssSelector("tr.odd")).is().visible();
     }
 
     @Test
@@ -212,7 +215,13 @@ public void verifyCreationTime() throws IOException {
         String creationTime = selectedRowColumns.get(1).getText();
         Calendar creationTimeCalendar = Calendar.getInstance();
         creationTimeCalendar.setTimeInMillis(creationTimeInMillis);
-        Assert.assertEquals("Creation time should be matching", DATE_FORMAT.format(creationTimeCalendar.getTime()), creationTime);
+        try {
+            Assert.assertEquals("Creation time should be matching", DATE_FORMAT.format(creationTimeCalendar.getTime()), creationTime);
+        } catch (AssertionError e) {
+            // get rid of errors caused by delay when at the end of minute
+            creationTimeCalendar.add(Calendar.SECOND, 1);
+            Assert.assertEquals("Creation time should be matching", DATE_FORMAT.format(creationTimeCalendar.getTime()), creationTime);
+        }
         invalidateSession(sessionId);
     }
 
@@ -223,16 +232,21 @@ public void verifyLastAccessedTime() throws IOException, InterruptedException {
         reloadSessions();
         String sessionId = getSessionsFromModel().get(0);
         page.getSessionsTable().select(sessionId);
-        long start = System.currentTimeMillis();
-        long end = start + TimeUnit.MINUTES.toMillis(1);
-        TimeUnit.MINUTES.sleep(1);
+        TimeUnit.SECONDS.sleep(5);
+        long accessTime = System.currentTimeMillis();
         deploymentBrowser.findElement(By.cssSelector("input[value=\"Increment\"")).click();
         reloadSessions();
         List<WebElement> selectedRowColumns = page.getSessionsTable().getRoot().findElements(By.cssSelector("tr.selected > td"));
         String lastAccessedTime = selectedRowColumns.get(2).getText();
         Calendar lastAccessedTimeCalendar = Calendar.getInstance();
-        lastAccessedTimeCalendar.setTimeInMillis(end);
-        Assert.assertEquals("Last accessed time should be matching", DATE_FORMAT.format(lastAccessedTimeCalendar.getTime()), lastAccessedTime);
+        lastAccessedTimeCalendar.setTimeInMillis(accessTime);
+        try {
+            Assert.assertEquals("Last accessed time should be matching", DATE_FORMAT.format(lastAccessedTimeCalendar.getTime()), lastAccessedTime);
+        } catch (AssertionError e) {
+            // get rid of errors caused by delay when at the end of minute
+            lastAccessedTimeCalendar.add(Calendar.SECOND, 1);
+            Assert.assertEquals("Last accessed time should be matching", DATE_FORMAT.format(lastAccessedTimeCalendar.getTime()), lastAccessedTime);
+        }
         invalidateSession(sessionId);
     }
 }
diff --git a/...estsuite/test/configuration/undertow/application/security/domain/SingleSignOnAddTest.java b/...estsuite/test/configuration/undertow/application/security/domain/SingleSignOnAddTest.java
@@ -50,13 +50,13 @@ public class SingleSignOnAddTest {
     @BeforeClass
     public static void setUp() throws IOException {
         operations.add(httpAuthenticationFactoryAddress(HTTP_AUTH_CREATE),
-                Values.of(HTTP_SERVER_MECH_FACTORY, "global").and(SECURITY_DOMAIN, "ApplicationDomain"));
+                Values.of(HTTP_SERVER_MECH_FACTORY, "global").and(SECURITY_DOMAIN, "ApplicationDomain")).assertSuccess();
         operations.add(applicationSecurityDomain(APPLICATION_SECURITY_DOMAIN_TO_BE_TESTED2),
-            Values.of(ApplicationSecurityDomainFixtures.HTTP_AUTHENTICATION_FACTORY, HTTP_AUTH_CREATE));
+            Values.of(ApplicationSecurityDomainFixtures.HTTP_AUTHENTICATION_FACTORY, HTTP_AUTH_CREATE)).assertSuccess();
         operations.add(ElytronFixtures.keyStoreAddress(KEY_STORE_TO_BE_ADDED),
             Values.of(TYPE, ElytronFixtures.JKS)
                 .and(ElytronFixtures.CREDENTIAL_REFERENCE, new ModelNodeGenerator.ModelNodePropertiesBuilder()
-                    .addProperty(ElytronFixtures.CREDENTIAL_REFERENCE_CLEAR_TEXT, Random.name()).build()));
+                    .addProperty(ElytronFixtures.CREDENTIAL_REFERENCE_CLEAR_TEXT, Random.name()).build())).assertSuccess();
     }
 
     @AfterClass

diff --git a/...l/testsuite/test/configuration/undertow/server/listener/AJPListenerConfigurationTest.java b/...l/testsuite/test/configuration/undertow/server/listener/AJPListenerConfigurationTest.java
@@ -69,14 +69,14 @@ public class AJPListenerConfigurationTest {
 
     @BeforeClass
     public static void setUp() throws IOException, CommandFailedException {
-        operations.add(IOFixtures.bufferPoolAddress(BUFFER_POOL_TO_BE_EDITED));
-        operations.add(IOFixtures.workerAddress(WORKER_TO_BE_EDITED));
-        operations.add(serverAddress(UNDERTOW_SERVER_TO_BE_TESTED));
+        operations.add(IOFixtures.bufferPoolAddress(BUFFER_POOL_TO_BE_EDITED)).assertSuccess();
+        operations.add(IOFixtures.workerAddress(WORKER_TO_BE_EDITED)).assertSuccess();
+        operations.add(serverAddress(UNDERTOW_SERVER_TO_BE_TESTED)).assertSuccess();
         client.apply(new AddLocalSocketBinding(SOCKET_BINDING));
         client.apply(new AddLocalSocketBinding(SOCKET_BINDING_TO_BE_EDITED));
         client.apply(new AddLocalSocketBinding(SOCKET_REDIRECT_TO_BE_EDITED));
         operations.add(serverAddress(UNDERTOW_SERVER_TO_BE_TESTED).and("ajp-listener", AJP_LISTENER_TO_BE_EDITED),
-            Values.of("socket-binding", SOCKET_BINDING.toLowerCase() + "ref"));
+            Values.of("socket-binding", SOCKET_BINDING.toLowerCase() + "ref")).assertSuccess();
     }
 
     @AfterClass

diff --git a/...va/org/jboss/hal/testsuite/test/configuration/undertow/servlet/container/CrawlerTest.java b/...va/org/jboss/hal/testsuite/test/configuration/undertow/servlet/container/CrawlerTest.java
@@ -17,11 +17,14 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.openqa.selenium.By;
+import org.openqa.selenium.TimeoutException;
 import org.wildfly.extras.creaper.core.online.OnlineManagementClient;
 import org.wildfly.extras.creaper.core.online.operations.Address;
 import org.wildfly.extras.creaper.core.online.operations.OperationException;
 import org.wildfly.extras.creaper.core.online.operations.Operations;
 
+import static org.jboss.arquillian.graphene.Graphene.waitGui;
 import static org.jboss.hal.dmr.ModelDescriptionConstants.NAME;
 
 @RunWith(Arquillian.class)
@@ -57,11 +60,11 @@ private static Address crawlerAddress(String servletContainer) {
 
     @BeforeClass
     public static void setUp() throws IOException {
-        operations.add(UndertowFixtures.servletContainerAddress(SERVLET_CONTAINER_EDIT));
-        operations.add(SERVLET_CONTAINER_EDIT_CRAWLER_ADDRESS);
-        operations.add(UndertowFixtures.servletContainerAddress(SERVLET_CONTAINER_CRAWLER_CREATE));
-        operations.add(UndertowFixtures.servletContainerAddress(SERVLET_CONTAINER_CRAWLER_REMOVE));
-        operations.add(crawlerAddress(SERVLET_CONTAINER_CRAWLER_REMOVE));
+        operations.add(UndertowFixtures.servletContainerAddress(SERVLET_CONTAINER_EDIT)).assertSuccess();
+        operations.add(SERVLET_CONTAINER_EDIT_CRAWLER_ADDRESS).assertSuccess();
+        operations.add(UndertowFixtures.servletContainerAddress(SERVLET_CONTAINER_CRAWLER_CREATE)).assertSuccess();
+        operations.add(UndertowFixtures.servletContainerAddress(SERVLET_CONTAINER_CRAWLER_REMOVE)).assertSuccess();
+        operations.add(crawlerAddress(SERVLET_CONTAINER_CRAWLER_REMOVE)).assertSuccess();
     }
 
     @AfterClass
@@ -74,6 +77,12 @@ public static void tearDown() throws IOException, OperationException {
     @Test
     public void create() throws Exception {
         navigateToCrawlerForm(SERVLET_CONTAINER_CRAWLER_CREATE);
+        try {
+            waitGui().until().element(By.id(Ids.build(Ids.UNDERTOW_SERVLET_CONTAINER_CRAWLER, "form-empty"))).is().visible();
+        } catch (TimeoutException ex) {
+            // ignore the intermittent exception and try again
+            navigateToCrawlerForm(SERVLET_CONTAINER_CRAWLER_CREATE);
+        }
         crudOperations.createSingleton(crawlerAddress(SERVLET_CONTAINER_CRAWLER_CREATE), page.getCrawlerForm());
     }