From 3460e3c9a97650a5c4d895f99de44647a3a18f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Sun, 15 May 2011 17:49:39 +0200 Subject: [PATCH] ISPN-1091 HotRod clients can failover upon receiving SuspectException --- .../exceptions/RemoteNodeSuspecException.java | 39 +++++++++++ .../impl/operations/HotRodOperation.java | 7 ++ .../operations/RetryOnFailureOperation.java | 10 ++- .../hotrod/retry/ServerFailureRetryTest.java | 70 +++++++++++++++++++ .../notifications/AbstractListenerImpl.java | 5 +- 5 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 client/hotrod-client/src/main/java/org/infinispan/client/hotrod/exceptions/RemoteNodeSuspecException.java create mode 100644 client/hotrod-client/src/test/java/org/infinispan/client/hotrod/retry/ServerFailureRetryTest.java diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/exceptions/RemoteNodeSuspecException.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/exceptions/RemoteNodeSuspecException.java new file mode 100644 index 000000000000..b0c77b49608f --- /dev/null +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/exceptions/RemoteNodeSuspecException.java @@ -0,0 +1,39 @@ +/* + * JBoss, Home of Professional Open Source + * Copyright 2011 Red Hat Inc. and/or its affiliates and other + * contributors as indicated by the @author tags. All rights reserved. + * See the copyright.txt in the distribution for a full listing of + * individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ + +package org.infinispan.client.hotrod.exceptions; + +/** + * When a remote node is suspected and evicted from the cluster while an + * operation is ongoing, the Hot Rod client emits this exception. + * + * @author Galder Zamarreño + * @since 4.2 + */ +public class RemoteNodeSuspecException extends HotRodClientException { + + public RemoteNodeSuspecException(String msgFromServer, long messageId, short status) { + super(msgFromServer, messageId, status); + } + +} diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/HotRodOperation.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/HotRodOperation.java index f1152174d55e..c3f625f95251 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/HotRodOperation.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/HotRodOperation.java @@ -26,6 +26,7 @@ import org.infinispan.client.hotrod.Flag; import org.infinispan.client.hotrod.exceptions.HotRodClientException; import org.infinispan.client.hotrod.exceptions.InvalidResponseException; +import org.infinispan.client.hotrod.exceptions.RemoteNodeSuspecException; import org.infinispan.client.hotrod.impl.protocol.HotRodConstants; import org.infinispan.client.hotrod.impl.transport.Transport; import org.infinispan.util.logging.Log; @@ -147,6 +148,12 @@ protected void checkForErrorsInResponseStatus(short status, long messageId, Tran String msgFromServer = transport.readString(); if (status == HotRodConstants.COMMAND_TIMEOUT_STATUS && isTrace) { log.trace("Server-side timeout performing operation: %s", msgFromServer); + } if (msgFromServer.contains("SuspectException")) { + if (isTrace) + log.trace("A remote node was suspected while executing messageId=%d. " + + "Check if retry possible. Message from server: %s", messageId, msgFromServer); + // TODO: This will be better handled with its own status id in version 2 of protocol + throw new RemoteNodeSuspecException(msgFromServer, messageId, status); } else { log.warn("Error received from the server: %s", msgFromServer); } diff --git a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/RetryOnFailureOperation.java b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/RetryOnFailureOperation.java index 797036e2f410..0554e0524d5a 100644 --- a/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/RetryOnFailureOperation.java +++ b/client/hotrod-client/src/main/java/org/infinispan/client/hotrod/impl/operations/RetryOnFailureOperation.java @@ -24,6 +24,8 @@ import net.jcip.annotations.Immutable; import org.infinispan.client.hotrod.Flag; +import org.infinispan.client.hotrod.exceptions.HotRodClientException; +import org.infinispan.client.hotrod.exceptions.RemoteNodeSuspecException; import org.infinispan.client.hotrod.exceptions.TransportException; import org.infinispan.client.hotrod.impl.transport.Transport; import org.infinispan.client.hotrod.impl.transport.TransportFactory; @@ -60,6 +62,8 @@ public Object execute() { return executeOperation(transport); } catch (TransportException te) { logErrorAndThrowExceptionIfNeeded(retryCount, te); + } catch (RemoteNodeSuspecException e) { + logErrorAndThrowExceptionIfNeeded(retryCount, e); } finally { releaseTransport(transport); } @@ -75,13 +79,13 @@ protected boolean shouldRetry(int retryCount) { return retryCount < transportFactory.getTransportCount(); } - protected void logErrorAndThrowExceptionIfNeeded(int i, TransportException te) { - String message = "Transport exception. Retry " + i + " out of " + transportFactory.getTransportCount(); + protected void logErrorAndThrowExceptionIfNeeded(int i, HotRodClientException te) { + String message = "Exception encountered. Retry %d out of %d"; if (i == transportFactory.getTransportCount() - 1 || transportFactory.getTransportCount() < 0) { log.warn(message, te); throw te; } else { - log.trace(message + ":" + te); + log.trace(message, i, transportFactory.getTransportCount(), te); } } diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/retry/ServerFailureRetryTest.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/retry/ServerFailureRetryTest.java new file mode 100644 index 000000000000..db8c9136652b --- /dev/null +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/retry/ServerFailureRetryTest.java @@ -0,0 +1,70 @@ +/* + * JBoss, Home of Professional Open Source + * Copyright 2011 Red Hat Inc. and/or its affiliates and other + * contributors as indicated by the @author tags. All rights reserved. + * See the copyright.txt in the distribution for a full listing of + * individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ + +package org.infinispan.client.hotrod.retry; + +import org.infinispan.config.Configuration; +import org.infinispan.notifications.Listener; +import org.infinispan.notifications.cachelistener.annotation.CacheEntryCreated; +import org.infinispan.notifications.cachelistener.event.CacheEntryEvent; +import org.infinispan.remoting.transport.jgroups.SuspectException; +import org.testng.annotations.Test; + +import java.lang.reflect.Method; + +import static org.infinispan.test.TestingUtil.k; +import static org.infinispan.test.TestingUtil.v; + +/** + * Test different server error situations and check how clients behave under + * those circumstances. Also verify whether failover is happening accordingly. + * + * @author Galder Zamarreño + * @since 4.2 + */ +@Test(groups = "functional", testName = "client.hotrod.retry.ServerFailureRetryTest") +public class ServerFailureRetryTest extends AbstractRetryTest { + + @Override + protected Configuration getCacheConfig() { + return getDefaultClusteredConfig(Configuration.CacheMode.REPL_SYNC); + } + + public void testRetryWithSuspectException(Method m) { + ErrorInducingListener listener = new ErrorInducingListener(); + manager(0).getCache().addListener(listener); + remoteCache.put(k(m), v(m)); + } + + @Listener + public static class ErrorInducingListener { + boolean induceError = true; + + @CacheEntryCreated + public void entryCreated(CacheEntryEvent event) throws Exception { + if (!event.isPre() && event.isOriginLocal() && induceError) { + throw new SuspectException("Simulated suspicion"); + } + } + } +} diff --git a/core/src/main/java/org/infinispan/notifications/AbstractListenerImpl.java b/core/src/main/java/org/infinispan/notifications/AbstractListenerImpl.java index 2b7d0abd0825..0c1e074ead98 100644 --- a/core/src/main/java/org/infinispan/notifications/AbstractListenerImpl.java +++ b/core/src/main/java/org/infinispan/notifications/AbstractListenerImpl.java @@ -199,7 +199,10 @@ public void run() { } catch (InvocationTargetException exception) { Throwable cause = getRealException(exception); - throw new CacheException("Caught exception invoking method " + method + " on listener instance " + target, cause); + throw new CacheException(String.format( + "Caught exception [%s] while invoking method [%s] on listener instance: %s" + , cause.getClass().getName(), method, target + ), cause); } catch (IllegalAccessException exception) { getLog().warn("Unable to invoke method " + method + " on Object instance " + target + " - removing this target object from list of listeners!", exception);