ISPN-13640 ResilienceMaxRetryIT is failing randomly

server/testdriver/core/src/main/java/org/infinispan/server/test/api/HotRodTestClientDriver.java

@@ -70,27 +70,13 @@ public <K, V> RemoteCache<K, V> get() {
    }
 
    /**
-    * Created a cache with the name of the method where this method is being called from.
-    * If the cache already exists, retrieves the existing cache
-    *
-    * @return {@link RemoteCache}, the cache
-    */
-   public <K, V> RemoteCache<K, V> create() {
-      return create(-1);
-   }
-
-   /**
-    * Create a cache adding in the initial server list the server address given by the index
-    * @param index the server index, -1 for all
+    * Create a cache adding in the initial server list the server address given by the index.
+    * If empty, it will add all servers by default OR those that were configured in the server list
+    * @param index the server index
     * @return {@link RemoteCache}, the cache
     */
-   public <K, V> RemoteCache<K, V> create(int index) {
-      RemoteCacheManager remoteCacheManager;
-      if (index >= 0) {
-         remoteCacheManager = createRemoteCacheManager(index);
-      } else {
-         remoteCacheManager = createRemoteCacheManager();
-      }
+   public <K, V> RemoteCache<K, V> create(int... index) {
+      RemoteCacheManager remoteCacheManager = createRemoteCacheManager(index);
       String name = testClient.getMethodName(qualifier);
       if (serverConfiguration != null) {
          return remoteCacheManager.administration().withFlags(flags).getOrCreateCache(name, serverConfiguration);
@@ -101,11 +87,7 @@ public <K, V> RemoteCache<K, V> create(int index) {
       }
    }
 
-   public RemoteCacheManager createRemoteCacheManager() {
-      return testClient.registerResource(testServer.newHotRodClient(clientConfiguration, port));
-   }
-
-   public RemoteCacheManager createRemoteCacheManager(int index) {
+   public RemoteCacheManager createRemoteCacheManager(int... index) {
       return testClient.registerResource(testServer.newHotRodClient(clientConfiguration, port, index));
    }
 

server/testdriver/core/src/main/java/org/infinispan/server/test/core/TestServer.java

@@ -58,25 +58,26 @@ public RemoteCacheManager newHotRodClient(ConfigurationBuilder builder) {
    }
 
    /**
+    * Create a client adding in the initial server list the server address given by the index.
+    * If empty, it will add all servers by default OR those that were configured in the server list
+    * @param index the server index
     * @return a client configured against the Hot Rod endpoint exposed by the server
     */
-   public RemoteCacheManager newHotRodClient(ConfigurationBuilder builder, int port) {
-      if (builder.servers().isEmpty()) {
-         for (int i = 0; i < getDriver().getConfiguration().numServers(); i++) {
+   public RemoteCacheManager newHotRodClient(ConfigurationBuilder builder, int port, int... index) {
+      if (index.length == 0) {
+         if (builder.servers().isEmpty()) {
+            for (int i = 0; i < getDriver().getConfiguration().numServers(); i++) {
+               configureHotRodClient(builder, port, i);
+            }
+         }
+      } else {
+         for (int i : index) {
             configureHotRodClient(builder, port, i);
          }
       }
       return getDriver().createRemoteCacheManager(builder);
    }
 
-   /**
-    * @return a client configured against the Hot Server index
-    */
-   public RemoteCacheManager newHotRodClient(ConfigurationBuilder builder, int port, int index) {
-      configureHotRodClient(builder, port, index);
-      return getDriver().createRemoteCacheManager(builder);
-   }
-
    private void configureHotRodClient(ConfigurationBuilder builder, int port, int i) {
       if (getDriver().getConfiguration().runMode() == ServerRunMode.CONTAINER) {
          ContainerInfinispanServerDriver containerDriver = (ContainerInfinispanServerDriver) serverDriver;

server/tests/src/test/java/org/infinispan/server/resilience/ResilienceMaxRetryIT.java

@@ -34,34 +34,31 @@ public class ResilienceMaxRetryIT {
 
    @Test
    public void testMaxRetries0() {
-      // 1 -> configure max-retries="0" and initial_server_list=localhost:port1
-      // 2 -> start 3 servers, localhost:port1, localhost:port2 and localhost:port3
-      ConfigurationBuilder builder = new ConfigurationBuilder();
-      builder.maxRetries(0);
-      RemoteCache<String, String> cache = SERVER_TEST.hotrod().withClientConfiguration(builder).withCacheMode(CacheMode.REPL_SYNC).create(0);
+      // 1 -> configure max-retries="0" and initial_server_list=server0
+      // 2 -> start 3 servers
+      RemoteCache<String, String> cache = createCache(0);
 
       // 3 -> start the client and do some operations
       for (int i = 0; i < 100; i++) {
          String random = UUID.randomUUID().toString();
          cache.put(random, random);
       }
 
-      // 4 -> kill localhost:port1
+      // 4 -> kill server0
       InetAddress killedServer = SERVERS.getServerDriver().getServerAddress(0);
       SERVERS.getServerDriver().kill(0);
 
-      // 5 -> do more operations, check that it's only connected to port2 and port3
-      // thinking one failure would be enough, but the topology is actually updated only when the client connects to one of the live servers
-      // you could have the first 2 connections to port1 and they'd both fail
-      // more than 2 is highly unlikely, but it might happen because the probability is not 0
+      // 5 -> do more operations, check that it's only connected to server1 and server2
+      // the topology is updated when the cache is created
+      // in this case, once the server hits the server0 that was killed, it shouldn't hit it more
+      boolean failedBefore = false;
       for (int i = 0; i < 100; i++) {
          String random = UUID.randomUUID().toString();
          try {
             cache.put(random, random);
          } catch (TransportException e) {
-            // if you do 10 operations after you kill port1, the first 0 or 1 or 2 operations might fail.
-            // But if the 3rd operation is successful, then operations 4-10 should be successful as well
-            assert i == 0 || i == 1 || i == 2;
+            assert !failedBefore;
+            failedBefore = true;
             // assert that the failed server is the one that we killed
             assert e.getMessage().contains(killedServer.toString());
          }
@@ -76,7 +73,7 @@ public void testMaxRetries0() {
          }
       }
 
-      // 6 -> kill port2 and port3, start port1
+      // 6 -> kill server1 and server2, start server0
       SERVERS.getServerDriver().kill(1);
       SERVERS.getServerDriver().kill(2);
       SERVERS.getServerDriver().restart(0);
@@ -108,4 +105,12 @@ public void testMaxRetries0() {
       String random = UUID.randomUUID().toString();
       cache.put(random, random);
    }
+
+   private RemoteCache<String, String> createCache(int... index) {
+      ConfigurationBuilder builder = new ConfigurationBuilder();
+      builder.maxRetries(0);
+      builder.connectionTimeout(5_000);
+      RemoteCache<String, String> cache = SERVER_TEST.hotrod().withClientConfiguration(builder).withCacheMode(CacheMode.REPL_SYNC).create(index);
+      return cache;
+   }
 }