ISPN-1091 HotRod clients can failover upon receiving SuspectException

client/hotrod-client/src/main/java/org/infinispan/client/hotrod/exceptions/RemoteNodeSuspecException.java

@@ -0,0 +1,39 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2011 Red Hat Inc. and/or its affiliates and other
+ * contributors as indicated by the @author tags. All rights reserved.
+ * See the copyright.txt in the distribution for a full listing of
+ * individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.infinispan.client.hotrod.exceptions;
+
+/**
+ * When a remote node is suspected and evicted from the cluster while an
+ * operation is ongoing, the Hot Rod client emits this exception.
+ *
+ * @author Galder Zamarreño
+ * @since 4.2
+ */
+public class RemoteNodeSuspecException extends HotRodClientException {
+
+   public RemoteNodeSuspecException(String msgFromServer, long messageId, short status) {
+      super(msgFromServer, messageId, status);
+   }
+
+}

client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/HotRodOperation.java

@@ -26,6 +26,7 @@
 import org.infinispan.client.hotrod.Flag;
 import org.infinispan.client.hotrod.exceptions.HotRodClientException;
 import org.infinispan.client.hotrod.exceptions.InvalidResponseException;
+import org.infinispan.client.hotrod.exceptions.RemoteNodeSuspecException;
 import org.infinispan.client.hotrod.impl.protocol.HotRodConstants;
 import org.infinispan.client.hotrod.impl.transport.Transport;
 import org.infinispan.util.logging.Log;
@@ -147,6 +148,12 @@ protected void checkForErrorsInResponseStatus(short status, long messageId, Tran
             String msgFromServer = transport.readString();
             if (status == HotRodConstants.COMMAND_TIMEOUT_STATUS && isTrace) {
                log.trace("Server-side timeout performing operation: %s", msgFromServer);
+            } if (msgFromServer.contains("SuspectException")) {
+               if (isTrace)
+                  log.trace("A remote node was suspected while executing messageId=%d. " +
+                     "Check if retry possible. Message from server: %s", messageId, msgFromServer);
+               // TODO: This will be better handled with its own status id in version 2 of protocol
+               throw new RemoteNodeSuspecException(msgFromServer, messageId, status);
             } else {
                log.warn("Error received from the server: %s", msgFromServer);
             }

client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/RetryOnFailureOperation.java

@@ -24,6 +24,8 @@
 
 import net.jcip.annotations.Immutable;
 import org.infinispan.client.hotrod.Flag;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
+import org.infinispan.client.hotrod.exceptions.RemoteNodeSuspecException;
 import org.infinispan.client.hotrod.exceptions.TransportException;
 import org.infinispan.client.hotrod.impl.transport.Transport;
 import org.infinispan.client.hotrod.impl.transport.TransportFactory;
@@ -60,6 +62,8 @@ public Object execute() {
             return executeOperation(transport);
          } catch (TransportException te) {
             logErrorAndThrowExceptionIfNeeded(retryCount, te);
+         } catch (RemoteNodeSuspecException e) {
+            logErrorAndThrowExceptionIfNeeded(retryCount, e);
          } finally {
             releaseTransport(transport);
          }
@@ -75,13 +79,13 @@ protected boolean shouldRetry(int retryCount) {
       return retryCount < transportFactory.getTransportCount();
    }
 
-   protected void logErrorAndThrowExceptionIfNeeded(int i, TransportException te) {
-      String message = "Transport exception. Retry " + i + " out of " + transportFactory.getTransportCount();
+   protected void logErrorAndThrowExceptionIfNeeded(int i, HotRodClientException te) {
+      String message = "Exception encountered. Retry %d out of %d";
       if (i == transportFactory.getTransportCount() - 1 || transportFactory.getTransportCount() < 0) {
          log.warn(message, te);
          throw te;
       } else {
-         log.trace(message + ":" + te);
+         log.trace(message, i, transportFactory.getTransportCount(), te);
       }
    }
 

client/hotrod-client/src/test/java/org/infinispan/client/hotrod/retry/ServerFailureRetryTest.java

@@ -0,0 +1,70 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2011 Red Hat Inc. and/or its affiliates and other
+ * contributors as indicated by the @author tags. All rights reserved.
+ * See the copyright.txt in the distribution for a full listing of
+ * individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.infinispan.client.hotrod.retry;
+
+import org.infinispan.config.Configuration;
+import org.infinispan.notifications.Listener;
+import org.infinispan.notifications.cachelistener.annotation.CacheEntryCreated;
+import org.infinispan.notifications.cachelistener.event.CacheEntryEvent;
+import org.infinispan.remoting.transport.jgroups.SuspectException;
+import org.testng.annotations.Test;
+
+import java.lang.reflect.Method;
+
+import static org.infinispan.test.TestingUtil.k;
+import static org.infinispan.test.TestingUtil.v;
+
+/**
+ * Test different server error situations and check how clients behave under
+ * those circumstances. Also verify whether failover is happening accordingly.
+ *
+ * @author Galder Zamarreño
+ * @since 4.2
+ */
+@Test(groups = "functional", testName = "client.hotrod.retry.ServerFailureRetryTest")
+public class ServerFailureRetryTest extends AbstractRetryTest {
+
+   @Override
+   protected Configuration getCacheConfig() {
+      return getDefaultClusteredConfig(Configuration.CacheMode.REPL_SYNC);
+   }
+
+   public void testRetryWithSuspectException(Method m) {
+      ErrorInducingListener listener = new ErrorInducingListener();
+      manager(0).getCache().addListener(listener);
+      remoteCache.put(k(m), v(m));
+   }
+
+   @Listener
+   public static class ErrorInducingListener {
+      boolean induceError = true;
+
+      @CacheEntryCreated
+      public void entryCreated(CacheEntryEvent event) throws Exception {
+         if (!event.isPre() && event.isOriginLocal() && induceError) {
+            throw new SuspectException("Simulated suspicion");
+         }
+      }
+   }
+}

core/src/main/java/org/infinispan/notifications/AbstractListenerImpl.java

@@ -199,7 +199,10 @@ public void run() {
                }
                catch (InvocationTargetException exception) {
                   Throwable cause = getRealException(exception);
-                  throw new CacheException("Caught exception invoking method " + method + " on listener instance " + target, cause);
+                  throw new CacheException(String.format(
+                     "Caught exception [%s] while invoking method [%s] on listener instance: %s"
+                     , cause.getClass().getName(), method, target
+                  ), cause);
                }
                catch (IllegalAccessException exception) {
                   getLog().warn("Unable to invoke method " + method + " on Object instance " + target + " - removing this target object from list of listeners!", exception);