diff --git a/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/annotation/MessageProcessor.java b/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/annotation/MessageProcessor.java index 8318c223..d7b340ba 100644 --- a/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/annotation/MessageProcessor.java +++ b/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/annotation/MessageProcessor.java @@ -102,16 +102,16 @@ public T newInstance() throws DempsyException { * Invokes the activation method of the passed instance. */ @Override - public void activate(final T instance, final Object key, final byte[] activationData) throws IllegalArgumentException, DempsyException { - wrap(() -> activationMethod.invoke(instance, key, activationData)); + public void activate(final T instance, final Object key) throws IllegalArgumentException, DempsyException { + wrap(() -> activationMethod.invoke(instance, key)); } /** * Invokes the passivation method of the passed instance. Will return the object's passivation data, null if there is none. */ @Override - public byte[] passivate(final T instance) throws IllegalArgumentException, DempsyException { - return wrap(() -> (byte[]) passivationMethod.invoke(instance)); + public void passivate(final T instance) throws IllegalArgumentException, DempsyException { + wrap(() -> (byte[]) passivationMethod.invoke(instance)); } /** @@ -318,7 +318,6 @@ private Method introspectClone() throws IllegalStateException { protected class MethodHandle { private final Method method; private int keyPosition = -1; - private int binayPosition = -1; private int totalArguments = 0; public MethodHandle(final Method method) { @@ -332,30 +331,26 @@ public MethodHandle(final Method method, final Class keyClass) { this.totalArguments = parameterTypes.length; for (int i = 0; i < parameterTypes.length; i++) { final Class parameter = parameterTypes[i]; - if (parameter.isArray() && parameter.getComponentType().isAssignableFrom(byte.class)) { - this.binayPosition = i; - } else if (keyClass != null && parameter.isAssignableFrom(keyClass)) { + if (keyClass != null && parameter.isAssignableFrom(keyClass)) { this.keyPosition = i; } } } } - public Object invoke(final Object instance, final Object key, final byte[] data) + public Object invoke(final Object instance, final Object key) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { if (this.method != null) { final Object[] parameters = new Object[this.totalArguments]; if (this.keyPosition > -1) parameters[this.keyPosition] = key; - if (this.binayPosition > -1) - parameters[this.binayPosition] = data; return this.method.invoke(instance, parameters); } return null; } public Object invoke(final Object instance) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { - return this.invoke(instance, null, null); + return this.invoke(instance, null); } public Method getMethod() { diff --git a/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/MessageProcessor.java b/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/MessageProcessor.java index 864d1167..1075cef4 100644 --- a/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/MessageProcessor.java +++ b/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/MessageProcessor.java @@ -7,8 +7,8 @@ import java.util.function.Supplier; import net.dempsy.config.ClusterId; -import net.dempsy.messages.KeyedMessageWithType; import net.dempsy.messages.KeyedMessage; +import net.dempsy.messages.KeyedMessageWithType; import net.dempsy.messages.MessageProcessorLifecycle; public class MessageProcessor implements MessageProcessorLifecycle { @@ -40,13 +40,13 @@ public Mp newInstance() { } @Override - public void activate(final Mp instance, final Object key, final byte[] activationData) throws IllegalArgumentException { - instance.activate(activationData, key); + public void activate(final Mp instance, final Object key) throws IllegalArgumentException { + instance.activate(key); } @Override - public byte[] passivate(final Mp instance) throws IllegalArgumentException { - return instance.passivate(); + public void passivate(final Mp instance) throws IllegalArgumentException { + instance.passivate(); } @Override diff --git a/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/Mp.java b/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/Mp.java index 14bab8fb..5d35504a 100644 --- a/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/Mp.java +++ b/dempsy-framework.api/src/main/java/net/dempsy/lifecycle/simple/Mp.java @@ -14,9 +14,7 @@ public default KeyedMessageWithType[] output() { return null; } - public default byte[] passivate() { - return null; - } + public default void passivate() {} - public default void activate(final byte[] data, final Object key) {} + public default void activate(final Object key) {} } diff --git a/dempsy-framework.api/src/main/java/net/dempsy/messages/MessageProcessorLifecycle.java b/dempsy-framework.api/src/main/java/net/dempsy/messages/MessageProcessorLifecycle.java index 5b84b4bb..408c6307 100644 --- a/dempsy-framework.api/src/main/java/net/dempsy/messages/MessageProcessorLifecycle.java +++ b/dempsy-framework.api/src/main/java/net/dempsy/messages/MessageProcessorLifecycle.java @@ -17,7 +17,7 @@ public interface MessageProcessorLifecycle { /** * Invokes the activation method of the passed instance. */ - public void activate(T instance, Object key, byte[] activationData) throws IllegalArgumentException, DempsyException; + public void activate(T instance, Object key) throws IllegalArgumentException, DempsyException; /** * Invokes the passivation method of the passed instance. Will return the object's passivation data, @@ -27,7 +27,7 @@ public interface MessageProcessorLifecycle { * @throws IllegalAccessException * @throws IllegalArgumentException */ - public byte[] passivate(T instance) throws IllegalArgumentException, DempsyException; + public void passivate(T instance) throws IllegalArgumentException, DempsyException; /** * Invokes the appropriate message handler of the passed instance. Caller is responsible for not passing diff --git a/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/MessageProcessorTest.java b/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/MessageProcessorTest.java index da4a1601..3f9a5664 100644 --- a/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/MessageProcessorTest.java +++ b/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/MessageProcessorTest.java @@ -16,9 +16,7 @@ package net.dempsy.lifecycle.annotations; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.junit.Test; @@ -40,12 +38,11 @@ public void testMethodHandleWithParameters() throws Throwable { helper.validate(); final TestMp mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final String ret = new String(helper.passivate(mp)); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertEquals("passivate", ret); } @Test @@ -54,12 +51,11 @@ public void testMethodHandleWithNoParameters() throws Throwable { helper.validate(); final TestMpEmptyActivate mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertNull(ret); } @Test @@ -68,12 +64,11 @@ public void testMethodHandleWithOnlyKey() throws Throwable { helper.validate(); final TestMpOnlyKey mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final String ret = new String(helper.passivate(mp)); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertEquals("passivate", ret); } @Test @@ -82,12 +77,11 @@ public void testMethodHandleExtraParameters() throws Throwable { helper.validate(); final TestMpExtraParameters mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final String ret = new String(helper.passivate(mp)); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertEquals("passivate", ret); } @Test @@ -97,12 +91,11 @@ public void testMethodHandleExtraParametersOrderChanged() throws Throwable { helper.validate(); final TestMpExtraParametersChangedOrder mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertNull(ret); } @Test @@ -111,12 +104,11 @@ public void testMethodHandleNoActivation() throws Throwable { helper.validate(); final TestMpNoActivation mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertFalse(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertFalse(mp.ispassivateCalled()); - assertNull(ret); } @Test @@ -124,12 +116,11 @@ public void testMethodHandleNoKey() throws Throwable { final MessageProcessor helper = new MessageProcessor(new TestMpNoKey()); final TestMpNoKey mp = helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertFalse(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertFalse(mp.ispassivateCalled()); - assertNull(ret); } } diff --git a/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/TestInvocation.java b/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/TestInvocation.java index 6ea68a84..6baaa1ec 100644 --- a/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/TestInvocation.java +++ b/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/annotations/TestInvocation.java @@ -138,7 +138,6 @@ public int hashCode() { @Mp public static class InvocationTestMp implements Cloneable { public boolean isActivated; - public String activationValue; public boolean isPassivated; public String lastStringHandlerValue; public Number lastNumberHandlerValue; @@ -150,15 +149,13 @@ public InvocationTestMp clone() throws CloneNotSupportedException { } @Activation - public void activate(final byte[] data) { + public void activate() { isActivated = true; - activationValue = new String(data); } @Passivation - public byte[] passivate() { + public void passivate() { isPassivated = true; - return activationValue.getBytes(); } @MessageHandler @@ -246,14 +243,12 @@ public void testLifeCycleMethods() { assertNotSame("instantiation failed; returned prototype", prototype, instance); assertFalse("instance activated before activation method called", instance.isActivated); - invoker.activate(instance, null, "ABC".getBytes()); + invoker.activate(instance, null); assertTrue("instance was not activated", instance.isActivated); - assertEquals("ABC", instance.activationValue); assertFalse("instance passivated before passivation method called", instance.isPassivated); - final byte[] data = invoker.passivate(instance); + invoker.passivate(instance); assertTrue("instance was not passivated", instance.isPassivated); - assertEquals("ABC", new String(data)); } @Test(expected = IllegalStateException.class) diff --git a/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/simple/SimpleMessageProcessorTest.java b/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/simple/SimpleMessageProcessorTest.java index 19aea32c..15427c26 100644 --- a/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/simple/SimpleMessageProcessorTest.java +++ b/dempsy-framework.api/src/test/java/net/dempsy/lifecycle/simple/SimpleMessageProcessorTest.java @@ -16,16 +16,14 @@ package net.dempsy.lifecycle.simple; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.junit.Test; import net.dempsy.lifecycle.annotation.MessageKey; -import net.dempsy.messages.KeyedMessageWithType; import net.dempsy.messages.KeyedMessage; +import net.dempsy.messages.KeyedMessageWithType; public class SimpleMessageProcessorTest { @Test @@ -33,12 +31,11 @@ public void testMethodHandleWithParameters() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMp()); final TestMp mp = (TestMp) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final String ret = new String(helper.passivate(mp)); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertEquals("passivate", ret); } @Test @@ -46,12 +43,11 @@ public void testMethodHandleWithNoParameters() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMpEmptyActivate()); final TestMpEmptyActivate mp = (TestMpEmptyActivate) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertNull(ret); } @Test @@ -59,12 +55,11 @@ public void testMethodHandleWithOnlyKey() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMpOnlyKey()); final TestMpOnlyKey mp = (TestMpOnlyKey) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final String ret = new String(helper.passivate(mp)); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertEquals("passivate", ret); } @Test @@ -72,12 +67,11 @@ public void testMethodHandleExtraParameters() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMpExtraParameters()); final TestMpExtraParameters mp = (TestMpExtraParameters) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final String ret = new String(helper.passivate(mp)); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertEquals("passivate", ret); } @Test @@ -85,12 +79,11 @@ public void testMethodHandleExtraParametersOrderChanged() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMpExtraParametersChangedOrder()); final TestMpExtraParametersChangedOrder mp = (TestMpExtraParametersChangedOrder) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertTrue(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertTrue(mp.ispassivateCalled()); - assertNull(ret); } @Test @@ -98,12 +91,11 @@ public void testMethodHandleNoActivation() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMpNoActivation()); final TestMpNoActivation mp = (TestMpNoActivation) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertFalse(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertFalse(mp.ispassivateCalled()); - assertNull(ret); } @Test @@ -111,12 +103,11 @@ public void testMethodHandleNoKey() throws Throwable { final MessageProcessor helper = new MessageProcessor(() -> new TestMpNoKey()); final TestMpNoKey mp = (TestMpNoKey) helper.newInstance(); assertFalse(mp.isActivated()); - helper.activate(mp, "activate", null); + helper.activate(mp, "activate"); assertFalse(mp.isActivated()); assertFalse(mp.ispassivateCalled()); - final Object ret = helper.passivate(mp); + helper.passivate(mp); assertFalse(mp.ispassivateCalled()); - assertNull(ret); } private class TestMp implements Mp { @@ -129,14 +120,13 @@ public KeyedMessageWithType[] handle(final KeyedMessage val) { } @Override - public void activate(final byte[] data, final Object key) { + public void activate(final Object key) { this.activated = true; } @Override - public byte[] passivate() { + public void passivate() { passivateCalled = true; - return "passivate".getBytes(); } public boolean isActivated() { @@ -158,14 +148,13 @@ public KeyedMessageWithType[] handle(final KeyedMessage val) { } @Override - public void activate(final byte[] data, final Object key) { + public void activate(final Object key) { this.activated = true; } @Override - public byte[] passivate() { + public void passivate() { passivateCalled = true; - return null; } public boolean isActivated() { @@ -187,14 +176,13 @@ public KeyedMessageWithType[] handle(final KeyedMessage val) { } @Override - public void activate(final byte[] data, final Object key) { + public void activate(final Object key) { this.activated = true; } @Override - public byte[] passivate() { + public void passivate() { passivateCalled = true; - return "passivate".getBytes(); } public boolean isActivated() { @@ -216,14 +204,13 @@ public KeyedMessageWithType[] handle(final KeyedMessage val) { } @Override - public void activate(final byte[] data, final Object key) { + public void activate(final Object key) { this.activated = true; } @Override - public byte[] passivate() { + public void passivate() { passivateCalled = true; - return "passivate".getBytes(); } public boolean isActivated() { @@ -245,14 +232,13 @@ public KeyedMessageWithType[] handle(final KeyedMessage val) { } @Override - public void activate(final byte[] data, final Object key) { + public void activate(final Object key) { this.activated = true; } @Override - public byte[] passivate() { + public void passivate() { passivateCalled = true; - return null; } public boolean isActivated() { diff --git a/dempsy-framework.core/src/main/java/net/dempsy/Infrastructure.java b/dempsy-framework.core/src/main/java/net/dempsy/Infrastructure.java index cb8ba588..084bb8f3 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/Infrastructure.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/Infrastructure.java @@ -6,9 +6,10 @@ import net.dempsy.config.ClusterId; import net.dempsy.monitoring.ClusterStatsCollector; import net.dempsy.monitoring.NodeStatsCollector; +import net.dempsy.threading.ThreadingModel; import net.dempsy.util.executor.AutoDisposeSingleThreadScheduler; -public interface Infrastructure { +public interface Infrastructure extends AutoCloseable { ClusterInfoSession getCollaborator(); AutoDisposeSingleThreadScheduler getScheduler(); @@ -23,6 +24,11 @@ public interface Infrastructure { String getNodeId(); + ThreadingModel getThreadingModel(); + + @Override + void close(); + public default String getConfigValue(final Class clazz, final String key, final String defaultValue) { final Map conf = getConfiguration(); final String entireKey = clazz.getPackage().getName() + "." + key; @@ -56,4 +62,5 @@ public static String nodes(final String application) { public static String clusters(final String application) { return root(application) + "/clusters"; } + } diff --git a/dempsy-framework.core/src/main/java/net/dempsy/KeyspaceChangeListener.java b/dempsy-framework.core/src/main/java/net/dempsy/KeyspaceChangeListener.java index 61198ee9..8d669a67 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/KeyspaceChangeListener.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/KeyspaceChangeListener.java @@ -1,12 +1,10 @@ package net.dempsy; -import net.dempsy.router.RoutingStrategy.Inbound; - /** * Since the responsibility for the portion of the keyspace that this node is responsible for * is determined by the Inbound strategy, when that responsibility changes, Dempsy itself * needs to be notified. */ public interface KeyspaceChangeListener { - public void keyspaceChanged(boolean less, boolean more, Inbound inbound); + public void keyspaceChanged(boolean less, boolean more); } diff --git a/dempsy-framework.core/src/main/java/net/dempsy/ServiceManager.java b/dempsy-framework.core/src/main/java/net/dempsy/ServiceManager.java index efe1c288..86dd580f 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/ServiceManager.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/ServiceManager.java @@ -59,6 +59,10 @@ public void stop() { LOGGER.warn("Failed to shut down an instance of " + clazz.getSimpleName(), ret); } }); + + if (infra != null) { + infra.close(); + } } @Override diff --git a/dempsy-framework.core/src/main/java/net/dempsy/threading/DefaultThreadingModel.java b/dempsy-framework.core/src/main/java/net/dempsy/threading/DefaultThreadingModel.java index 891d8652..735c25b3 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/threading/DefaultThreadingModel.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/threading/DefaultThreadingModel.java @@ -1,12 +1,13 @@ package net.dempsy.threading; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; @@ -15,17 +16,29 @@ public class DefaultThreadingModel implements ThreadingModel { private static final int minNumThreads = 4; + public static final String CONFIG_KEY_MAX_PENDING = "max_pending"; + public static final String DEFAULT_MAX_PENDING = "100000"; + + public static final String CONFIG_KEY_CORES_FACTOR = "cores_factor"; + public static final String DEFAULT_CORES_FACTOR = "1.0"; + + public static final String CONFIG_KEY_ADDITIONAL_THREADS = "additional_threads"; + public static final String DEFAULT_ADDITIONAL_THREADS = "1"; + + public static final String CONFIG_KEY_HARD_SHUTDOWN = "hard_shutdown"; + public static final String DEFAULT_HARD_SHUTDOWN = "true"; + private ScheduledExecutorService schedule = null; - private ThreadPoolExecutor executor = null; + private ExecutorService executor = null; private AtomicLong numLimited = null; private long maxNumWaitingLimitedTasks; private int threadPoolSize; - private double m = 1.25; - private int additionalThreads = 2; + private double m = Double.parseDouble(DEFAULT_CORES_FACTOR); + private int additionalThreads = Integer.parseInt(DEFAULT_ADDITIONAL_THREADS); private final Supplier nameSupplier; - private boolean hardShutdown = false; + private boolean hardShutdown = Boolean.parseBoolean(DEFAULT_ADDITIONAL_THREADS); public DefaultThreadingModel(final Supplier nameSupplier, final int threadPoolSize, final int maxNumWaitingLimitedTasks) { this.nameSupplier = nameSupplier; @@ -34,7 +47,7 @@ public DefaultThreadingModel(final Supplier nameSupplier, final int thre } public DefaultThreadingModel(final Supplier nameSupplier) { - this(nameSupplier, -1, -1); + this(nameSupplier, -1, Integer.parseInt(DEFAULT_MAX_PENDING)); } public DefaultThreadingModel(final String threadNameBase) { @@ -84,6 +97,9 @@ public DefaultThreadingModel setAdditionalThreads(final int additionalThreads) { return this; } + /** + * When closing this ThreadingModel, use a hard shutdown (shutdownNow) on the executors. + */ public DefaultThreadingModel setHardShutdown(final boolean hardShutdown) { this.hardShutdown = hardShutdown; return this; @@ -98,14 +114,24 @@ public DefaultThreadingModel start() { // then use the other constructor threadPoolSize = Math.max(cpuBasedThreadCount, minNumThreads); } - executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(threadPoolSize, + executor = Executors.newFixedThreadPool(threadPoolSize, r -> new Thread(r, nameSupplier.get())); schedule = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, nameSupplier.get() + "-Scheduled")); numLimited = new AtomicLong(0); - if (maxNumWaitingLimitedTasks < 0) - maxNumWaitingLimitedTasks = 20 * threadPoolSize; + return this; + } + private static String getConfigValue(final Map conf, final String key, final String defaultValue) { + final String entireKey = DefaultThreadingModel.class.getPackage().getName() + "." + key; + return conf.containsKey(entireKey) ? conf.get(entireKey) : defaultValue; + } + + public DefaultThreadingModel configure(final Map configuration) { + setMaxNumberOfQueuedLimitedTasks(Integer.parseInt(getConfigValue(configuration, CONFIG_KEY_MAX_PENDING, DEFAULT_MAX_PENDING))); + setHardShutdown(Boolean.parseBoolean(getConfigValue(configuration, CONFIG_KEY_HARD_SHUTDOWN, DEFAULT_HARD_SHUTDOWN))); + setCoresFactor(Double.parseDouble(getConfigValue(configuration, CONFIG_KEY_CORES_FACTOR, DEFAULT_CORES_FACTOR))); + setAdditionalThreads(Integer.parseInt(getConfigValue(configuration, CONFIG_KEY_ADDITIONAL_THREADS, DEFAULT_ADDITIONAL_THREADS))); return this; } @@ -140,11 +166,6 @@ public void close() { } } - @Override - public int getNumberPending() { - return executor.getQueue().size(); - } - @Override public int getNumberLimitedPending() { return numLimited.intValue(); @@ -163,27 +184,27 @@ public Future submit(final Callable r) { @Override public Future submitLimited(final Rejectable r, final boolean count) { - final Callable task = new Callable() { - Rejectable o = r; - @Override - public V call() throws Exception { + if (maxNumWaitingLimitedTasks > 0) { // maxNumWaitingLimitedTasks <= 0 means unlimited + final Callable task = () -> { if (!count) - return o.call(); + return r.call(); final long num = numLimited.decrementAndGet(); if (num <= maxNumWaitingLimitedTasks) - return o.call(); - o.rejected(); + return r.call(); + r.rejected(); return null; - } - }; + }; - if (count) - numLimited.incrementAndGet(); + if (count) + numLimited.incrementAndGet(); + + final Future ret = executor.submit(task); + return ret; + } else + return executor.submit(() -> r.call()); - final Future ret = executor.submit(task); - return ret; } @Override diff --git a/dempsy-framework.core/src/main/java/net/dempsy/threading/ThreadingModel.java b/dempsy-framework.core/src/main/java/net/dempsy/threading/ThreadingModel.java index 764c1485..e41b4985 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/threading/ThreadingModel.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/threading/ThreadingModel.java @@ -56,11 +56,6 @@ public default Thread newThread(final Runnable runnable, final String name) { return new Thread(runnable, name); } - /** - * How many pending tasks are there. - */ - public int getNumberPending(); - /** * How many pending limited tasks are there */ diff --git a/dempsy-framework.core/src/main/java/net/dempsy/transport/Listener.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/Listener.java index 6a23de9c..31c08e6b 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/transport/Listener.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/Listener.java @@ -16,6 +16,8 @@ package net.dempsy.transport; +import java.util.function.Supplier; + /** *

* This is the core abstraction for receiving messages. The client side of a transport implementation (called an "Adaptor") needs to be wired to a MessageTransportListener @@ -34,6 +36,10 @@ public interface Listener extends AutoCloseable { */ public boolean onMessage(T message) throws MessageTransportException; + public default boolean onMessage(final Supplier supplier) { + return onMessage(supplier.get()); + } + @Override public default void close() {} diff --git a/dempsy-framework.core/src/main/java/net/dempsy/transport/Receiver.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/Receiver.java index e3453ff6..77604215 100644 --- a/dempsy-framework.core/src/main/java/net/dempsy/transport/Receiver.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/Receiver.java @@ -16,7 +16,7 @@ package net.dempsy.transport; -import net.dempsy.threading.ThreadingModel; +import net.dempsy.Infrastructure; public interface Receiver extends AutoCloseable { /** @@ -27,7 +27,7 @@ public interface Receiver extends AutoCloseable { /** * A receiver is started with a Listener and a threading model. */ - public void start(Listener listener, ThreadingModel threadingModel) throws MessageTransportException; + public void start(Listener listener, Infrastructure threadingModel) throws MessageTransportException; /** * What is a unique Id for the transport that this {@link Receiver} is associated with. This information is used diff --git a/dempsy-framework.core/src/main/java/net/dempsy/transport/RoutedMessage.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/RoutedMessage.java new file mode 100644 index 00000000..a00f4765 --- /dev/null +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/RoutedMessage.java @@ -0,0 +1,23 @@ +package net.dempsy.transport; + +import java.io.Serializable; + +public class RoutedMessage implements Serializable { + private static final long serialVersionUID = 1L; + public final int[] containers; + public final Object key; + public final Object message; + + @SuppressWarnings("unused") + private RoutedMessage() { + containers = null; + key = null; + message = null; + } + + public RoutedMessage(final int[] containers, final Object key, final Object message) { + this.containers = containers; + this.key = key; + this.message = message; + } +} \ No newline at end of file diff --git a/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/AbstractTcpReceiver.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/AbstractTcpReceiver.java new file mode 100644 index 00000000..f33dd1b7 --- /dev/null +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/AbstractTcpReceiver.java @@ -0,0 +1,58 @@ +package net.dempsy.transport.tcp; + +import java.net.InetAddress; +import java.util.function.Supplier; + +import net.dempsy.serialization.Serializer; +import net.dempsy.transport.Receiver; + +public abstract class AbstractTcpReceiver> implements Receiver { + public final static int DEFAULT_MAX_MESSAGE_SIZE_BYTES = 1024 * 1024; + protected final Serializer serializer; + + protected int internalPort; + protected boolean useLocalHost = false; + protected TcpAddressResolver resolver = a -> a; + protected final String serId; + protected int maxMessageSize = DEFAULT_MAX_MESSAGE_SIZE_BYTES; + protected Supplier addrSupplier = null; + + public AbstractTcpReceiver(final Serializer serializer, final int port) { + this.internalPort = port; + this.serializer = serializer; + this.serId = serializer.getClass().getPackage().getName(); + } + + public AbstractTcpReceiver(final Serializer serializer) { + this(serializer, -1); + } + + @SuppressWarnings("unchecked") + public T setUseLocalHost(final boolean useLocalHost) { + this.useLocalHost = useLocalHost; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T setAddressSupplier(final Supplier addrSupplier) { + this.addrSupplier = addrSupplier; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T setResolver(final TcpAddressResolver resolver) { + this.resolver = resolver; + return (T) this; + } + + public AbstractTcpReceiver setMaxMessageSize(final int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + return this; + } + + @Override + public abstract TcpAddress getAddress(); + + public abstract AbstractTcpReceiver setNumHandlers(int numHandlerThreads); + +} diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpAddress.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpAddress.java similarity index 83% rename from dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpAddress.java rename to dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpAddress.java index 5e39ef36..d3686d44 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpAddress.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpAddress.java @@ -4,7 +4,7 @@ import net.dempsy.transport.NodeAddress; -public class TcpAddress implements NodeAddress { +public abstract class TcpAddress implements NodeAddress { private static final long serialVersionUID = 1L; @@ -12,20 +12,22 @@ public class TcpAddress implements NodeAddress { public final InetAddress inetAddress; public final int port; public final String serializerId; + public final int recvBufferSize; - @SuppressWarnings("unused") - private TcpAddress() { + protected TcpAddress() { guid = null; inetAddress = null; port = -1; serializerId = null; + recvBufferSize = -1; } - public TcpAddress(final InetAddress inetAddress, final int port, final String serializerId) { + public TcpAddress(final InetAddress inetAddress, final int port, final String serializerId, final int recvBufferSize) { this.inetAddress = inetAddress; this.port = port; this.guid = inetAddress.getHostAddress() + ":" + port; this.serializerId = serializerId; + this.recvBufferSize = recvBufferSize; } @Override diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpAddressResolver.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpAddressResolver.java similarity index 69% rename from dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpAddressResolver.java rename to dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpAddressResolver.java index 97c21732..96c799ec 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpAddressResolver.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpAddressResolver.java @@ -8,7 +8,7 @@ * map internal ports bound to, to external ports. */ @FunctionalInterface -public interface TcpAddressResolver { +public interface TcpAddressResolver { - public TcpAddress getExternalAddresses(final TcpAddress addr) throws DempsyException; + public T getExternalAddresses(final T addr) throws DempsyException; } diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpUtils.java b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpUtils.java similarity index 59% rename from dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpUtils.java rename to dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpUtils.java index 23851614..26da1a93 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/TcpUtils.java +++ b/dempsy-framework.core/src/main/java/net/dempsy/transport/tcp/TcpUtils.java @@ -4,24 +4,33 @@ import java.net.InetAddress; import java.net.NetworkInterface; import java.net.SocketException; -import java.util.Collections; +import java.util.ArrayList; import java.util.Enumeration; +import java.util.List; import net.dempsy.DempsyException; public class TcpUtils { public static InetAddress getFirstNonLocalhostInetAddress() throws SocketException { + final List addrs = getAllInetAddress(); + return addrs.stream() + .filter(i -> !i.isLoopbackAddress() && i instanceof Inet4Address) + .findFirst() + .orElseThrow(() -> new DempsyException("There are no non-local network interfaces among " + addrs)); + } + + public static List getAllInetAddress() throws SocketException { + final List ret = new ArrayList<>(); final Enumeration netInterfaces = NetworkInterface.getNetworkInterfaces(); while (netInterfaces.hasMoreElements()) { final NetworkInterface networkInterface = netInterfaces.nextElement(); for (final Enumeration loopInetAddress = networkInterface.getInetAddresses(); loopInetAddress.hasMoreElements();) { final InetAddress tempInetAddress = loopInetAddress.nextElement(); - if (!tempInetAddress.isLoopbackAddress() && tempInetAddress instanceof Inet4Address) - return tempInetAddress; + ret.add(tempInetAddress); } } - throw new DempsyException("There are no non-local network interfaces among " + Collections.list(netInterfaces)); + return ret; } } diff --git a/dempsy-framework.impl/pom.xml b/dempsy-framework.impl/pom.xml index 4f787866..72f99e2c 100644 --- a/dempsy-framework.impl/pom.xml +++ b/dempsy-framework.impl/pom.xml @@ -16,10 +16,6 @@ net.dempsy dempsy-framework.core - - io.netty - netty-all - io.dropwizard.metrics diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/NodeManager.java b/dempsy-framework.impl/src/main/java/net/dempsy/NodeManager.java index 8306d8c0..adc224f1 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/NodeManager.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/NodeManager.java @@ -154,8 +154,7 @@ else if (firstAdaptorClusterName.get() == null) LOGGER.warn("The node at " + nodeId + " contains no message processors but has a Reciever set. The receiver will never be started."); threading = tr.track(new DefaultThreadingModel("NodeThreadPool-" + nodeId + "-")) - .setCoresFactor(1.0).setAdditionalThreads(1).setHardShutdown(true) - .setMaxNumberOfQueuedLimitedTasks(10000).start(); + .configure(node.getConfiguration()).start(); final NodeReceiver nodeReciever = receiver == null ? null : tr .track(new NodeReceiver(containers.stream().map(pc -> pc.container).collect(Collectors.toList()), threading, nodeStatsCollector)); @@ -191,7 +190,7 @@ public boolean execute() { if (LOGGER.isDebugEnabled()) LOGGER.info(logmessage, e); else - LOGGER.info(logmessage); + LOGGER.info(logmessage, e); } return false; } @@ -216,25 +215,27 @@ public String toString() { this.rsManager = tr.start(new RoutingStrategyManager(), this); // create the router but don't start it yet. - this.router = new Router(rsManager, nodeAddress, nodeReciever, tManager, nodeStatsCollector); + this.router = new Router(rsManager, nodeAddress, nodeId, nodeReciever, tManager, nodeStatsCollector); // set up containers containers.forEach(pc -> pc.container.setDispatcher(router)); - // start containers - containers.forEach(pc -> tr.start(pc.container, this)); - - // set up adaptors - adaptors.values().forEach(a -> a.setDispatcher(router)); - // IB routing strategy final int numContainers = containers.size(); for (int i = 0; i < numContainers; i++) { final PerContainer c = containers.get(i); c.inboundStrategy.setContainerDetails(c.clusterDefinition.getClusterId(), new ContainerAddress(nodeAddress, i), c.container); - tr.start(c.inboundStrategy, this); } + // start containers after setting inbound + containers.forEach(pc -> tr.start(pc.container.setInbound(pc.inboundStrategy), this)); + + // set up adaptors + adaptors.values().forEach(a -> a.setDispatcher(router)); + + // start IB routing strategy + containers.forEach(pc -> tr.start(pc.inboundStrategy, this)); + // start router tr.start(this.router, this); @@ -253,7 +254,7 @@ public boolean execute() { startAdaptorAfterRouterIsRunning.process(); if (receiver != null) - tr.track(receiver).start(nodeReciever, threading); + tr.track(receiver).start(nodeReciever, this); tr.track(session); // close this session when we're done. // ===================================== @@ -279,7 +280,7 @@ public boolean isReady() { } @Override - public void close() throws Exception { + public void close() { stop(); } @@ -351,6 +352,11 @@ public String getNodeId() { return nodeId; } + @Override + public ThreadingModel getThreadingModel() { + return threading; + } + // Testing accessors // ============================================================================== diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/NodeReceiver.java b/dempsy-framework.impl/src/main/java/net/dempsy/NodeReceiver.java index e99b3fbb..4e55d1dc 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/NodeReceiver.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/NodeReceiver.java @@ -2,14 +2,15 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Supplier; -import net.dempsy.Router.RoutedMessage; import net.dempsy.container.Container; import net.dempsy.messages.KeyedMessage; import net.dempsy.monitoring.NodeStatsCollector; import net.dempsy.threading.ThreadingModel; import net.dempsy.transport.Listener; import net.dempsy.transport.MessageTransportException; +import net.dempsy.transport.RoutedMessage; public class NodeReceiver implements Listener { @@ -25,18 +26,35 @@ public NodeReceiver(final List nodeContainers, final ThreadingModel t @Override public boolean onMessage(final RoutedMessage message) throws MessageTransportException { - // TODO: consider if blocking should be configurable by cluster? node? etc. statsCollector.messageReceived(message); feedbackLoop(message); return true; } + @Override + public boolean onMessage(final Supplier supplier) { + statsCollector.messageReceived(supplier); + threadModel.submitLimited(new ThreadingModel.Rejectable() { + + @Override + public Object call() throws Exception { + doIt(supplier.get()); + return null; + } + + @Override + public void rejected() { + statsCollector.messageDiscarded(supplier); + } + }, true); + return true; + } + private void doIt(final RoutedMessage message) { Arrays.stream(message.containers).forEach(container -> containers[container].dispatch(new KeyedMessage(message.key, message.message), true)); } public void feedbackLoop(final RoutedMessage message) { - // onMessage(message); threadModel.submitLimited(new ThreadingModel.Rejectable() { @Override @@ -49,7 +67,6 @@ public Object call() throws Exception { public void rejected() { statsCollector.messageDiscarded(message); } - }, false); + }, true); } - } \ No newline at end of file diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/Router.java b/dempsy-framework.impl/src/main/java/net/dempsy/Router.java index 69367617..98bb379b 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/Router.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/Router.java @@ -1,6 +1,5 @@ package net.dempsy; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -8,9 +7,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,6 +24,7 @@ import net.dempsy.router.RoutingStrategy.ContainerAddress; import net.dempsy.router.RoutingStrategyManager; import net.dempsy.transport.NodeAddress; +import net.dempsy.transport.RoutedMessage; import net.dempsy.transport.Sender; import net.dempsy.transport.SenderFactory; import net.dempsy.transport.TransportManager; @@ -41,6 +41,7 @@ public class Router extends Dispatcher implements Service { private final AtomicReference outbounds = new AtomicReference<>(null); private final NodeAddress thisNode; + private final String thisNodeId; private final TransportManager tmanager; private final NodeReceiver nodeReciever; private final AtomicBoolean isReady = new AtomicBoolean(false); @@ -49,38 +50,39 @@ public class Router extends Dispatcher implements Service { private static class ApplicationState { private final Map outboundByClusterName; private final Map> clusterNameByMessageType; - private final Map senderByNode; private final Map current; + private final TransportManager tManager; + private final NodeAddress thisNode; - ApplicationState(final Map outboundByClusterName, - final Map> cnByType, final Map senderByNode, - final Map current) { + private final ConcurrentHashMap senders = new ConcurrentHashMap<>(); + + ApplicationState(final Map outboundByClusterName, final Map> cnByType, + final Map current, final TransportManager tManager, final NodeAddress thisNode) { this.outboundByClusterName = outboundByClusterName; this.clusterNameByMessageType = new HashMap<>(); cnByType.entrySet().forEach(e -> clusterNameByMessageType.put(e.getKey(), new ArrayList(e.getValue()))); - this.senderByNode = senderByNode; this.current = current; + this.tManager = tManager; + this.thisNode = thisNode; } - ApplicationState() { + ApplicationState(final TransportManager tManager, final NodeAddress thisNode) { outboundByClusterName = new HashMap<>(); clusterNameByMessageType = new HashMap<>(); - senderByNode = new HashMap<>(); current = new HashMap<>(); + this.tManager = tManager; + this.thisNode = thisNode; } public static class Update { final Set toAdd; final Set toDelete; final Set leaveAlone; - final NodeAddress thisNode; - public Update(final Set leaveAlone, final Set toAdd, final Set toDelete, - final NodeAddress thisNode) { + public Update(final Set leaveAlone, final Set toAdd, final Set toDelete) { this.toAdd = toAdd; this.toDelete = toDelete; this.leaveAlone = leaveAlone; - this.thisNode = thisNode; } public boolean change() { @@ -88,7 +90,7 @@ public boolean change() { } } - public Update update(final Set newState, final NodeAddress thisNode) { + public Update update(final Set newState, final NodeAddress thisNodeX, final String thisNodeId) { final Set toAdd = new HashSet<>(); final Set toDelete = new HashSet<>(); final Set knownAddr = new HashSet<>(current.keySet()); @@ -98,7 +100,7 @@ public Update update(final Set newState, final NodeAddress this for (final NodeInformation cur : newState) { final NodeInformation known = current.get(cur.nodeAddress); - if (cur.nodeAddress.equals(thisNode)) + if (cur.nodeAddress.equals(thisNodeX)) oursOnTheList = cur.nodeAddress; if (known == null) // then we don't know about this one yet. // we need to add this one @@ -116,15 +118,15 @@ public Update update(final Set newState, final NodeAddress this } } - if (oursOnTheList == null && thisNode != null) // we don't seem to have our own address registered with the collaborator. - // this condition is actually okay since the Router is started before the - // node is registered with the collaborator. - LOGGER.trace("Router at {} doesn't seem to have its own address registered with the collaborator yet", thisNode); + if (oursOnTheList == null && thisNodeX != null) // we don't seem to have our own address registered with the collaborator. + // this condition is actually okay since the Router is started before the + // node is registered with the collaborator. + LOGGER.trace("Router at {} doesn't seem to have its own address registered with the collaborator yet", thisNodeId); // dump the remaining knownAddrs on the toDelete list toDelete.addAll(knownAddr); - return new Update(leaveAlone, toAdd, toDelete, thisNode); + return new Update(leaveAlone, toAdd, toDelete); } public ApplicationState apply(final Update update, final TransportManager tmanager, final NodeStatsCollector statsCollector, @@ -132,33 +134,21 @@ public ApplicationState apply(final Update update, final TransportManager tmanag // apply toDelete first. final Set toDelete = update.toDelete; - for (final NodeAddress cur : toDelete) { - senderByNode.get(current.get(cur).nodeAddress).stop(); + if (toDelete.size() > 0) { // just clear all senders. + senders.clear(); } final Map newCurrent = new HashMap<>(); - final Map newSenders = new HashMap<>(); // the one's to carry over. final Set leaveAlone = update.leaveAlone; for (final NodeInformation cur : leaveAlone) { - // if this is our node then we don't want to add a sender. - // update.thisNode can be null but cur.nodeAddress cannot - if (!cur.nodeAddress.equals(update.thisNode)) { - final Sender sender = senderByNode.get(cur.nodeAddress); - newSenders.put(cur.nodeAddress, sender); - } newCurrent.put(cur.nodeAddress, cur); } // add new senders final Set toAdd = update.toAdd; for (final NodeInformation cur : toAdd) { - if (!cur.nodeAddress.equals(update.thisNode)) { - final SenderFactory sf = tmanager.getAssociatedInstance(cur.transportTypeId); - final Sender sender = sf.getSender(cur.nodeAddress); - newSenders.put(cur.nodeAddress, sender); - } newCurrent.put(cur.nodeAddress, cur); } @@ -196,15 +186,39 @@ public ApplicationState apply(final Update update, final TransportManager tmanag }); } - return new ApplicationState(newOutboundByClusterName, cnByType, newSenders, newCurrent); + return new ApplicationState(newOutboundByClusterName, cnByType, newCurrent, tmanager, thisNode); } + public void stop() { + outboundByClusterName.values().forEach(r -> { + try { + r.release(); + } catch (final RuntimeException rte) { + LOGGER.warn("Problem while shutting down an outbound router", rte); + } + }); + senders.clear(); + } + + public Sender getSender(final NodeAddress na) { + final Sender ret = senders.get(na); + if (ret == null) { + if (na.equals(thisNode)) + return null; + return senders.computeIfAbsent(na, n -> { + final SenderFactory sf = tManager.getAssociatedInstance(na.getClass().getPackage().getName()); + return sf.getSender(n); + }); + } + return ret; + } } - public Router(final RoutingStrategyManager manager, final NodeAddress thisNode, final NodeReceiver nodeReciever, + public Router(final RoutingStrategyManager manager, final NodeAddress thisNode, final String thisNodeId, final NodeReceiver nodeReciever, final TransportManager tmanager, final NodeStatsCollector statsCollector) { this.manager = manager; this.thisNode = thisNode; + this.thisNodeId = thisNodeId; this.tmanager = tmanager; this.nodeReciever = nodeReciever; this.statsCollector = statsCollector; @@ -212,8 +226,8 @@ public Router(final RoutingStrategyManager manager, final NodeAddress thisNode, @Override public void dispatch(final KeyedMessageWithType message) { - if (LOGGER.isTraceEnabled()) - LOGGER.trace("Dispatching " + SafeString.objectDescription(message.message) + "."); + // if (LOGGER.isTraceEnabled()) + // LOGGER.trace("Dispatching " + SafeString.objectDescription(message.message) + "."); boolean messageSentSomewhere = false; try { @@ -222,8 +236,10 @@ public void dispatch(final KeyedMessageWithType message) { // if we're in the midst of an update then we really want to wait for the new state. while (tmp == null) { - if (!isRunning.get()) - throw new DempsyException("Router dispatch called while stopped."); + if (!isRunning.get()) { + LOGGER.debug("Router dispatch called while stopped."); + return; + } if (!isReady.get()) // however, if we never were ready then we're not in the midst // of an update. @@ -261,7 +277,7 @@ public void dispatch(final KeyedMessageWithType message) { final ContainerAddress ca = ob.selectDestinationForMessage(message); // it's possible 'ca' is null when we don't know where to send the message. if (ca == null) - LOGGER.info("No way to send the message {} to the cluster {} for the time being", message.message, clusterName); + LOGGER.debug("No way to send the message {} to the cluster {} for the time being", message.message, clusterName); else { // When the message will be sent to 2 different clusters, but both clusters // are hosted in the same node, then we send 1 message to 1 ContainerAddress @@ -287,10 +303,13 @@ public void dispatch(final KeyedMessageWithType message) { nodeReciever.feedbackLoop(toSend); messageSentSomewhere = true; } else { - final Sender sf = cur.senderByNode.get(curNode); - if (sf == null) - LOGGER.error("Couldn't send message to " + curNode + " because there's no " + SenderFactory.class.getSimpleName()); - else { + final Sender sf = cur.getSender(curNode); + if (sf == null) { + // router update is probably behind the routing strategy update + if (isRunning.get()) + LOGGER.error("Couldn't send message to " + curNode + " from " + thisNodeId + " because there's no " + + Sender.class.getSimpleName()); + } else { // TODO: more error handling sf.send(toSend); messageSentSomewhere = true; @@ -348,12 +367,13 @@ public boolean execute() { } // check to see if there's new nodes. - final ApplicationState.Update ud = outbounds.get().update(alreadySeen, thisNode); + final ApplicationState.Update ud = outbounds.get().update(alreadySeen, thisNode, thisNodeId); if (!ud.change()) { isReady.set(true); return true; // nothing to update. - } + } else if (LOGGER.isTraceEnabled()) + LOGGER.trace("Updating for " + thisNodeId); // otherwise we will be making changes so remove the current ApplicationState final ApplicationState obs = outbounds.getAndSet(null); // this can cause instability. @@ -386,7 +406,7 @@ public String toString() { }; - outbounds.set(new ApplicationState()); + outbounds.set(new ApplicationState(tmanager, thisNode)); isRunning.set(true); checkup.process(); @@ -403,19 +423,6 @@ public boolean isReady() { if (!tmanager.isReady()) return false; - // go through all of the outbounds and make sure we know about all of the nodes reachable with the exception - // of thisNode. - final Set nodes = obs.outboundByClusterName.values().stream() - .map(r -> r.allDesintations().stream().map(ca -> ca.node)) - .flatMap(i -> i) - .filter(na -> !na.equals(thisNode)) // make sure it's not thisNode - .collect(Collectors.toSet()); - - // is there an outbound for each node? - for (final NodeAddress c : nodes) { - if (!obs.senderByNode.containsKey(c)) - return false; - } return true; } else return false; } @@ -424,47 +431,9 @@ public boolean isReady() { public void stop() { synchronized (isRunning) { isRunning.set(false); - stopEm(outbounds.getAndSet(null)); - } - } - - public static class RoutedMessage implements Serializable { - private static final long serialVersionUID = 1L; - public final int[] containers; - public final Object key; - public final Object message; - - @SuppressWarnings("unused") - private RoutedMessage() { - containers = null; - key = null; - message = null; - } - - public RoutedMessage(final int[] containers, final Object key, final Object message) { - this.containers = containers; - this.key = key; - this.message = message; - } - } - - private static void stopEm(final ApplicationState obs) { - if (obs != null) { - obs.outboundByClusterName.values().forEach(r -> { - try { - r.release(); - } catch (final RuntimeException rte) { - LOGGER.warn("Problem while shutting down an outbound router", rte); - } - }); - - obs.senderByNode.values().forEach(sf -> { - try { - sf.stop(); - } catch (final RuntimeException rte) { - LOGGER.warn("Problem while shutting down an SenderFactory", rte); - } - }); + final ApplicationState cur = outbounds.getAndSet(null); + if (cur != null) + cur.stop(); } } @@ -494,8 +463,8 @@ Collection allReachable(final String cluterName) { return ob.allDesintations(); } - NodeAddress thisNode() { - return thisNode; + String thisNodeId() { + return thisNodeId; } NodeStatsCollector getNodeStatCollector() { diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/container/Container.java b/dempsy-framework.impl/src/main/java/net/dempsy/container/Container.java index 077f3e7c..ccae64c6 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/container/Container.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/container/Container.java @@ -49,7 +49,11 @@ public abstract class Container implements Service, KeyspaceChangeListener { protected final Logger LOGGER; + private static final AtomicLong containerNumSequence = new AtomicLong(0L); + protected long containerNum = containerNumSequence.getAndIncrement(); + protected Dispatcher dispatcher; + protected Inbound inbound; // The ClusterId is set for the sake of error messages. protected ClusterId clusterId; @@ -57,6 +61,7 @@ public abstract class Container implements Service, KeyspaceChangeListener { protected long evictionCycleTime = -1; protected TimeUnit evictionTimeUnit = null; protected final AtomicBoolean isRunning = new AtomicBoolean(false); + protected boolean isRunningLazy = false; protected MessageProcessorLifecycle prototype; protected ClusterStatsCollector statCollector; @@ -82,6 +87,11 @@ public Dispatcher getDispatcher() { return dispatcher; } + public Container setInbound(final Inbound inbound) { + this.inbound = inbound; + return this; + } + @SuppressWarnings("unchecked") public Container setMessageProcessor(final MessageProcessorLifecycle prototype) { if (prototype == null) @@ -140,10 +150,12 @@ public void stop() { setOutputCycleConcurrency(-1); isRunning.set(false); + isRunningLazy = false; } @Override public void start(final Infrastructure infra) { + isRunningLazy = true; isRunning.set(true); statCollector = infra.getClusterStatsCollector(clusterId); @@ -273,28 +285,33 @@ public void run() { keyspaceChangeSwitch.workerRunning(); if (shrink) { - // First do the contract by evicting all - doevict(new EvictCheck() { - // we shouldEvict if the message key no longer belongs as - // part of this container. - // strategyInbound can't be null if we're here since this was invoked - // indirectly from it. So here we don't need to check for null. - @Override - public boolean shouldEvict(final Object key, final Object instance) { - return !inbound.doesMessageKeyBelongToNode(key); - } - - // In this case, it's evictable. - @Override - public boolean isGenerallyEvitable() { - return true; - } - - @Override - public boolean shouldStopEvicting() { - return keyspaceChangeSwitch.wasPreempted(); - } - }); + LOGGER.trace("Evicting Mps due to keyspace shrinkage."); + try { + // First do the contract by evicting all + doevict(new EvictCheck() { + // we shouldEvict if the message key no longer belongs as + // part of this container. + // strategyInbound can't be null if we're here since this was invoked + // indirectly from it. So here we don't need to check for null. + @Override + public boolean shouldEvict(final Object key, final Object instance) { + return !inbound.doesMessageKeyBelongToNode(key); + } + + // In this case, it's evictable. + @Override + public boolean isGenerallyEvitable() { + return true; + } + + @Override + public boolean shouldStopEvicting() { + return keyspaceChangeSwitch.wasPreempted(); + } + }); + } catch (final RuntimeException rte) { + LOGGER.error("Failed on eviction", rte); + } } if (grow) { @@ -312,8 +329,10 @@ public boolean shouldStopEvicting() { } private final KeyspaceChanger changer = new KeyspaceChanger(); + private static AtomicLong keyspaceChangeThreadNum = new AtomicLong(0L); - public void keyspaceChanged(final boolean less, final boolean more, final Inbound inbound) { + @Override + public void keyspaceChanged(final boolean less, final boolean more) { if (less) { // we need to run a special eviction pass. @@ -332,7 +351,7 @@ public void keyspaceChanged(final boolean less, final boolean more, final Inboun if (less) changer.shrink = true; - final Thread t = new Thread(changer, "Keyspace Change Thread"); + final Thread t = new Thread(changer, clusterId.toString() + "-Keyspace Change Thread-" + keyspaceChangeThreadNum.getAndIncrement()); t.setDaemon(true); t.start(); diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/container/altnonlocking/NonLockingAltContainer.java b/dempsy-framework.impl/src/main/java/net/dempsy/container/altnonlocking/NonLockingAltContainer.java index f2004b1a..b6c3d3ea 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/container/altnonlocking/NonLockingAltContainer.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/container/altnonlocking/NonLockingAltContainer.java @@ -118,35 +118,11 @@ public int getMessageWorkingCount() { // Internals // ---------------------------------------------------------------------------- - private static class CountedLinkedList { - private final LinkedList list = new LinkedList<>(); - private int size = 0; - - public T removeFirst() { - size--; - return list.removeFirst(); - } - - public boolean add(final T v) { - final boolean ret = list.add(v); - if (ret) - size++; - return ret; - } - - public T pushPop(final T toPush) { - if (size == 0) - return toPush; - list.add(toPush); - return list.removeFirst(); - } - } - private static class WorkingQueueHolder { - final AtomicReference> queue; + final AtomicReference> queue; WorkingQueueHolder(final boolean locked) { - queue = locked ? new AtomicReference<>(null) : new AtomicReference<>(new CountedLinkedList<>()); + queue = locked ? new AtomicReference<>(null) : new AtomicReference<>(new LinkedList<>()); } } @@ -201,22 +177,41 @@ private T waitFor(final Supplier condition) { throw new DempsyException("Not running."); } - private CountedLinkedList getQueue(final WorkingQueueHolder wp) { + private LinkedList getQueue(final WorkingQueueHolder wp) { return waitFor(() -> wp.queue.getAndSet(null)); } + private static T pushPop(final LinkedList q, final T toPush) { + if (q.size() == 0) + return toPush; + q.add(toPush); + return q.removeFirst(); + } + // this is called directly from tests but shouldn't be accessed otherwise. @Override public void dispatch(final KeyedMessage message, final boolean block) throws IllegalArgumentException, ContainerException { + if (!isRunningLazy) { + LOGGER.debug("Dispacth called on stopped container"); + statCollector.messageFailed(false); + } + if (message == null) return; // No. We didn't process the null message - if (message == null || message.message == null) + if (message.message == null) throw new IllegalArgumentException("the container for " + clusterId + " attempted to dispatch null message."); if (message.key == null) throw new ContainerException("Message " + objectDescription(message.message) + " contains no key."); + if (!inbound.doesMessageKeyBelongToNode(message.key)) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Message with key " + SafeString.objectDescription(message.key) + " sent to wrong container. "); + statCollector.messageFailed(false); + return; + } + numBeingWorked.incrementAndGet(); boolean instanceDone = false; @@ -236,8 +231,8 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill // if mailbox is null then I got it. if (mailbox == null) { final WorkingQueueHolder box = mref.ref; // can't be null if I got the mailbox - final CountedLinkedList q = getQueue(box); // spin until I get the queue - KeyedMessage toProcess = q.pushPop(message); + final LinkedList q = getQueue(box); // spin until I get the queue + KeyedMessage toProcess = pushPop(q, message); box.queue.lazySet(q); // put the queue back while (toProcess != null) { @@ -245,8 +240,8 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill numBeingWorked.getAndDecrement(); // get the next message - final CountedLinkedList queue = getQueue(box); - if (queue.size == 0) + final LinkedList queue = getQueue(box); + if (queue.size() == 0) // we need to leave the queue out break; @@ -259,7 +254,7 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill } else { // we didn't get exclusive access so let's see if we can add the message to the mailbox // make one try at putting the message in the mailbox. - final CountedLinkedList q = mailbox.queue.getAndSet(null); + final LinkedList q = mailbox.queue.getAndSet(null); if (q != null) { // I got it! q.add(message); @@ -292,12 +287,20 @@ protected void doevict(final EvictCheck check) { keys.addAll(instances.keySet()); while (keys.size() > 0 && instances.size() > 0 && isRunning.get() && !check.shouldStopEvicting()) { + + // store off anything that passes for later removal. This is to avoid a + // ConcurrentModificationException. + final Set keysProcessed = new HashSet(); + for (final Object key : keys) { final InstanceWrapper wrapper = instances.get(key); if (wrapper != null) { // if the MP still exists final WorkingQueueHolder mailbox = setIfAbsent(wrapper.mailbox, () -> mref.set(new WorkingQueueHolder(true))); if (mailbox == null) { // this means I got it. + + keysProcessed.add(key); // mark this to remove it from the set of keys later since we've dealing with it + // it was created locked so no one else will be able to drop messages in the mailbox. final Object instance = wrapper.instance; boolean evictMe; @@ -328,6 +331,7 @@ protected void doevict(final EvictCheck check) { } // end - I got the lock. Otherwise it's too busy to evict. } // end if mp exists. Otherwise the mp is already gone. } + keys.removeAll(keysProcessed); // remove the keys we already checked } } } @@ -489,15 +493,13 @@ InstanceWrapper getInstanceForKey(final Object key) throws ContainerException { ". The value returned from the clone call appears to be null."); // activate - final byte[] data = null; if (LOGGER.isTraceEnabled()) LOGGER.trace("the container for " + clusterId + " is activating instance " + String.valueOf(instance) - + " with " + ((data != null) ? data.length : 0) + " bytes of data" + " via " + SafeString.valueOf(prototype)); boolean activateSuccessful = false; try { - prototype.activate(instance, key, data); + prototype.activate(instance, key); activateSuccessful = true; } catch (final IllegalArgumentException e) { throw new ContainerException( diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/container/locking/LockingContainer.java b/dempsy-framework.impl/src/main/java/net/dempsy/container/locking/LockingContainer.java index 372cf659..5c85df86 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/container/locking/LockingContainer.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/container/locking/LockingContainer.java @@ -204,9 +204,21 @@ protected Object getInstance() { // this is called directly from tests but shouldn't be accessed otherwise. @Override public void dispatch(final KeyedMessage message, final boolean block) throws IllegalArgumentException, ContainerException { + if (!isRunningLazy) { + LOGGER.debug("Dispacth called on stopped container"); + statCollector.messageFailed(false); + } + if (message == null) return; // No. We didn't process the null message + if (!inbound.doesMessageKeyBelongToNode(message.key)) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Message with key " + SafeString.objectDescription(message.key) + " sent to wrong container. "); + statCollector.messageFailed(false); + return; + } + boolean evictedAndBlocking; if (message == null || message.message == null) @@ -282,7 +294,7 @@ protected void doevict(final EvictCheck check) { while (instancesToEvict.size() > 0 && instances.size() > 0 && isRunning.get() && !check.shouldStopEvicting()) { // store off anything that passes for later removal. This is to avoid a // ConcurrentModificationException. - final Set keysToRemove = new HashSet(); + final Set keysProcessed = new HashSet(); for (final Map.Entry entry : instancesToEvict.entrySet()) { if (check.shouldStopEvicting()) @@ -298,7 +310,7 @@ protected void doevict(final EvictCheck check) { // since we got here we're done with this instance, // so add it's key to the list of keys we plan don't // need to return to. - keysToRemove.add(key); + keysProcessed.add(key); boolean removeInstance = false; @@ -322,6 +334,8 @@ protected void doevict(final EvictCheck check) { // even if passivate throws an exception, if the eviction check returned 'true' then // we need to remove the instance. if (removeInstance) { + if (LOGGER.isTraceEnabled()) + LOGGER.trace("Evicting Mp with key " + SafeString.objectDescription(key) + " from " + clusterId.toString()); instances.remove(key); statCollector.messageProcessorDeleted(key); } @@ -334,7 +348,7 @@ protected void doevict(final EvictCheck check) { } // now clean up everything we managed to get hold of - for (final Object key : keysToRemove) + for (final Object key : keysProcessed) instancesToEvict.remove(key); } @@ -499,15 +513,13 @@ InstanceWrapper getInstanceForKey(final Object key) throws ContainerException { ". The value returned from the clone call appears to be null."); // activate - final byte[] data = null; if (LOGGER.isTraceEnabled()) LOGGER.trace("the container for " + clusterId + " is activating instance " + String.valueOf(instance) - + " with " + ((data != null) ? data.length : 0) + " bytes of data" + " via " + SafeString.valueOf(prototype)); boolean activateSuccessful = false; try { - prototype.activate(instance, key, data); + prototype.activate(instance, key); activateSuccessful = true; } catch (final IllegalArgumentException e) { throw new ContainerException( @@ -592,7 +604,8 @@ private void invokeOperation(final Object instance, final Operation op, final Ke try { dispatcher.dispatch(result); } catch (final Exception de) { - LOGGER.warn("Failed on subsequent dispatch of " + result + ": " + de.getLocalizedMessage()); + if (isRunning.get()) + LOGGER.warn("Failed on subsequent dispatch of " + result + ": " + de.getLocalizedMessage()); } } } diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/container/nonlocking/NonLockingContainer.java b/dempsy-framework.impl/src/main/java/net/dempsy/container/nonlocking/NonLockingContainer.java index a0e6be67..ea40537e 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/container/nonlocking/NonLockingContainer.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/container/nonlocking/NonLockingContainer.java @@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import java.util.function.Supplier; import org.slf4j.Logger; @@ -63,29 +62,8 @@ public class NonLockingContainer extends Container { private final AtomicBoolean isReady = new AtomicBoolean(false); private final AtomicInteger numBeingWorked = new AtomicInteger(0); - private static class CountedLinkedList { - private final LinkedList list = new LinkedList<>(); - private int size = 0; - - public void forEach(final Consumer c) { - list.forEach(c); - } - - public T removeFirst() { - size--; - return list.removeFirst(); - } - - public boolean add(final T v) { - final boolean ret = list.add(v); - if (ret) - size++; - return ret; - } - } - private static class WorkingQueueHolder { - CountedLinkedList queue = null; + LinkedList queue = null; } protected static class WorkingPlaceholder { @@ -166,15 +144,13 @@ private Object createAndActivate(final Object key) throws ContainerException { ". The value returned from the clone call appears to be null."); // activate - final byte[] data = null; if (LOGGER.isTraceEnabled()) LOGGER.trace("the container for " + clusterId + " is activating instance " + String.valueOf(instance) - + " with " + ((data != null) ? data.length : 0) + " bytes of data" + " via " + SafeString.valueOf(prototype)); boolean activateSuccessful = false; try { - prototype.activate(instance, key, data); + prototype.activate(instance, key); activateSuccessful = true; } catch (final IllegalArgumentException e) { throw new ContainerException( @@ -242,6 +218,11 @@ public final X set(final X ref) { @Override public void dispatch(final KeyedMessage message, final boolean block) throws IllegalArgumentException, ContainerException { + if (!isRunningLazy) { + LOGGER.debug("Dispacth called on stopped container"); + statCollector.messageFailed(false); + } + if (message == null) return; // No. We didn't process the null message @@ -251,6 +232,13 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill if (message.key == null) throw new ContainerException("Message " + objectDescription(message.message) + " contains no key."); + if (!inbound.doesMessageKeyBelongToNode(message.key)) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Message with key " + SafeString.objectDescription(message.key) + " sent to wrong container. "); + statCollector.messageFailed(false); + return; + } + final Object key = message.key; boolean keepTrying = true; @@ -298,7 +286,7 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill if (instance != null) { // work off the queue. final WorkingQueueHolder mailbox = getQueue(wp); // spin until I have it. - if (mailbox.queue != null && mailbox.queue.size > 0) { // if there are messages in the queue + if (mailbox.queue != null && mailbox.queue.size() > 0) { // if there are messages in the queue curMessage = mailbox.queue.removeFirst(); // take a message off the queue // curMessage CAN'T be NULL!!!! @@ -320,7 +308,7 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill } } finally { if (working.remove(key) == null) - System.out.println("WTF?"); + LOGGER.error("IMPOSSIBLE! Null key removed from working set.", new RuntimeException()); } } else { // ... we didn't get the lock if (!block) { // blocking means no collisions allowed. @@ -338,7 +326,7 @@ public void dispatch(final KeyedMessage message, final boolean block) throws Ill // drop a message in the mailbox queue and mark it as being worked. numBeingWorked.incrementAndGet(); if (mailbox.queue == null) - mailbox.queue = new CountedLinkedList<>(); + mailbox.queue = new LinkedList<>(); mailbox.queue.add(message); } finally { // put it back - releasing the lock @@ -366,6 +354,11 @@ public void doevict(final EvictCheck check) { keys.addAll(instances.keySet()); while (keys.size() > 0 && instances.size() > 0 && isRunning.get() && !check.shouldStopEvicting()) { + + // store off anything that passes for later removal. This is to avoid a + // ConcurrentModificationException. + final Set keysProcessed = new HashSet(); + for (final Object key : keys) { final WorkingPlaceholder wp = new WorkingPlaceholder(); @@ -380,6 +373,8 @@ public void doevict(final EvictCheck check) { final Object instance = instances.get(key); if (instance != null) { + keysProcessed.add(key); // track this key to remove it from the keys set later. + boolean evictMe; try { evictMe = check.shouldEvict(key, instance); @@ -410,6 +405,8 @@ public void doevict(final EvictCheck check) { } } } + + keys.removeAll(keysProcessed); // remove the keys we already checked } } } diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/ManagedInbound.java b/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/ManagedInbound.java index f0f8968f..f485e8f7 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/ManagedInbound.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/ManagedInbound.java @@ -29,7 +29,7 @@ public void start(final Infrastructure infra) { utils = new Utils(infra, clusterId, address); // subscriber first because it registers as a node. If there's no nodes // there's nothing for the leader to do. - subscriber = new Subscriber(utils, infra, isRunning, listener, this); + subscriber = new Subscriber(utils, infra, isRunning, listener); subscriber.process(); leader = new Leader(utils, infra, isRunning); leader.process(); diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/Subscriber.java b/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/Subscriber.java index a998c468..4a957955 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/Subscriber.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/router/managed/Subscriber.java @@ -17,7 +17,6 @@ import net.dempsy.cluster.ClusterInfoSession; import net.dempsy.cluster.DirMode; import net.dempsy.router.RoutingStrategy.ContainerAddress; -import net.dempsy.router.RoutingStrategy.Inbound; import net.dempsy.router.managed.Utils.ShardAssignment; import net.dempsy.utils.PersistentTask; @@ -28,18 +27,15 @@ public class Subscriber extends PersistentTask { private final ContainerAddress thisNode; private final AtomicReference iOwn = new AtomicReference(null); private final KeyspaceChangeListener listener; - private final Inbound inbound; private String nodeDirectory = null; private final ClusterInfoSession session; - public Subscriber(final Utils msutils, final Infrastructure infra, final AtomicBoolean isRunning, final KeyspaceChangeListener listener, - final Inbound inbound) { + public Subscriber(final Utils msutils, final Infrastructure infra, final AtomicBoolean isRunning, final KeyspaceChangeListener listener) { super(LOGGER, isRunning, infra.getScheduler(), 500); this.utils = msutils; this.session = msutils.session; this.thisNode = utils.thisNodeAddress; this.listener = listener; - this.inbound = inbound; } public boolean isReady() { @@ -185,7 +181,7 @@ private void callListener(final boolean[] prev, final boolean[] next) { } try { - listener.keyspaceChanged(shrink, grow, inbound); + listener.keyspaceChanged(shrink, grow); } catch (final RuntimeException rte) { LOGGER.error("Exception while notifying " + KeyspaceChangeListener.class.getSimpleName() + " of a change (" + (grow && shrink ? "gained and lost some shards)" : (grow ? "gained some shards)" : "lost some shards)")), rte); diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/blockingqueue/BlockingQueueReceiver.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/blockingqueue/BlockingQueueReceiver.java index 3362a13d..8089eb82 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/blockingqueue/BlockingQueueReceiver.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/blockingqueue/BlockingQueueReceiver.java @@ -23,7 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.dempsy.threading.ThreadingModel; +import net.dempsy.Infrastructure; import net.dempsy.transport.Listener; import net.dempsy.transport.MessageTransportException; import net.dempsy.transport.NodeAddress; @@ -119,7 +119,7 @@ public void run() { */ @SuppressWarnings({ "rawtypes", "unchecked" }) @Override - public synchronized void start(final Listener listener, final ThreadingModel threadingModel) { + public synchronized void start(final Listener listener, final Infrastructure infra) { if (listener == null) throw new IllegalArgumentException("Cannot pass null to " + BlockingQueueReceiver.class.getSimpleName() + ".setListener"); if (this.listener != null) @@ -127,7 +127,7 @@ public synchronized void start(final Listener listener, final ThreadingModel thr "Cannot set a new Listener (" + SafeString.objectDescription(listener) + ") on a " + BlockingQueueReceiver.class.getSimpleName() + " when there's one already set (" + SafeString.objectDescription(this.listener) + ")"); this.listener = listener; - threadingModel.runDaemon(this, "BQReceiver-" + address.getGuid()); + infra.getThreadingModel().runDaemon(this, "BQReceiver-" + address.getGuid()); } @Override diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/passthrough/PassthroughReceiver.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/passthrough/PassthroughReceiver.java index 8c3d04c5..81bac6b4 100644 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/passthrough/PassthroughReceiver.java +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/passthrough/PassthroughReceiver.java @@ -4,7 +4,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -import net.dempsy.threading.ThreadingModel; +import net.dempsy.Infrastructure; import net.dempsy.transport.Listener; import net.dempsy.transport.MessageTransportException; import net.dempsy.transport.NodeAddress; @@ -33,7 +33,7 @@ public NodeAddress getAddress() throws MessageTransportException { */ @SuppressWarnings("unchecked") @Override - public void start(final Listener listener, final ThreadingModel threadingModel) throws MessageTransportException { + public void start(final Listener listener, final Infrastructure infra) throws MessageTransportException { this.listener = (Listener) listener; } diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettyReceiver.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettyReceiver.java deleted file mode 100644 index e850b153..00000000 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettyReceiver.java +++ /dev/null @@ -1,242 +0,0 @@ -package net.dempsy.transport.tcp.netty; - -import java.io.IOException; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.List; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.ByteToMessageDecoder; -import io.netty.util.ReferenceCountUtil; -import io.netty.util.concurrent.Future; -import net.dempsy.DempsyException; -import net.dempsy.Router.RoutedMessage; -import net.dempsy.serialization.Serializer; -import net.dempsy.threading.ThreadingModel; -import net.dempsy.transport.Listener; -import net.dempsy.transport.MessageTransportException; -import net.dempsy.transport.Receiver; -import net.dempsy.transport.tcp.TcpAddress; -import net.dempsy.transport.tcp.TcpAddressResolver; -import net.dempsy.transport.tcp.TcpUtils; -import net.dempsy.util.io.MessageBufferInput; - -public class NettyReceiver implements Receiver { - private static final Logger LOGGER = LoggerFactory.getLogger(NettyReceiver.class); - private TcpAddressResolver resolver = a -> a; - private TcpAddress address = null; - private TcpAddress internal = null; - private boolean useLocalHost = false; - private int internalPort = -1; - private int numHandlers = 1; - private EventLoopGroup parentGroup = null; - private EventLoopGroup childGroup = null; - private Listener typedListener = null; - - private final Serializer serializer; - - public NettyReceiver(final Serializer serializer, final int port) { - this.internalPort = port; - this.serializer = serializer; - } - - public NettyReceiver(final Serializer serializer) { - this(serializer, -1); - } - - public NettyReceiver setUseLocalHost(final boolean useLocalHost) { - this.useLocalHost = useLocalHost; - return this; - } - - public NettyReceiver setNumHandlers(final int numHandlers) { - this.numHandlers = numHandlers; - return this; - } - - public NettyReceiver setResolver(final TcpAddressResolver resolver) { - this.resolver = resolver; - return this; - } - - @Override - public synchronized void close() throws Exception { - final Future cf = (childGroup != null) ? childGroup.shutdownGracefully() : null; - final Future pf = (parentGroup != null) ? parentGroup.shutdownGracefully() : null; - if (cf != null) - cf.await(); - if (pf != null) - pf.await(); - } - - @Override - public synchronized TcpAddress getAddress() { - if (internal == null) { - try { - final InetAddress addr = useLocalHost ? Inet4Address.getLocalHost() : TcpUtils.getFirstNonLocalhostInetAddress(); - final InetSocketAddress inetSocketAddress = doBind(addr, (internalPort < 0) ? 0 : internalPort); - internalPort = inetSocketAddress.getPort(); - final String serId = serializer.getClass().getPackage().getName(); - internal = new TcpAddress(addr, internalPort, serId); - address = resolver.getExternalAddresses(internal); - } catch (final IOException e) { - throw new DempsyException(e); - } - } - return address; - } - - private static AtomicLong acceptorThreadNum = new AtomicLong(0L); - private static AtomicLong globalHandlerGroupNum = new AtomicLong(0L); - - @Override - @SuppressWarnings("unchecked") - public synchronized void start(final Listener listener, final ThreadingModel threadingModel) throws MessageTransportException { - this.typedListener = (Listener) listener; - } - - private InetSocketAddress doBind(final InetAddress inetAddress, final int port) { - final ServerBootstrap b = new ServerBootstrap(); - parentGroup = new NioEventLoopGroup(1, (ThreadFactory) r -> new Thread(r, - NettyReceiver.class.getSimpleName() + "-Acceptor-" + acceptorThreadNum.getAndIncrement())); - final AtomicLong handlerNum = new AtomicLong(0); - final long ghgn = globalHandlerGroupNum.getAndIncrement(); - childGroup = new NioEventLoopGroup(numHandlers, (ThreadFactory) r -> new Thread(r, - NettyReceiver.class.getSimpleName() + "-Handler(" + ghgn + ")-" + handlerNum.getAndIncrement())); - - b.group(parentGroup, childGroup) - .channel(NioServerSocketChannel.class) - .option(ChannelOption.SO_REUSEADDR, true) - // .option(ChannelOption.ALLOCATOR, new PooledByteBufAllocator(false)) - .childHandler(new ChannelInitializer() { - - @Override - protected void initChannel(final SocketChannel ch) throws Exception { - // ch.config().setAllocator(new PooledByteBufAllocator(false) { - // @Override - // public ByteBuf ioBuffer() { - // return heapBuffer(); - // } - // - // @Override - // public ByteBuf ioBuffer(final int initialCapacity) { - // return heapBuffer(initialCapacity); - // } - // - // @Override - // public ByteBuf ioBuffer(final int initialCapacity, final int maxCapacity) { - // return heapBuffer(initialCapacity, maxCapacity); - // } - // - // }); - ch.pipeline().addLast(new Client(serializer, () -> typedListener)); - } - - }); - - final InetSocketAddress inetSocketAddress = new InetSocketAddress(inetAddress, port); - final InetSocketAddress localAddr; - // wait until the bind is complete. - try { - localAddr = (InetSocketAddress) b.bind(inetSocketAddress).sync().await().channel().localAddress(); - } catch (final InterruptedException e) { - throw new MessageTransportException("Interrupted while binding.", e); - } catch (final RuntimeException e) { - throw new MessageTransportException("Unexpected uncheked exception throw by netty.", e); - } catch (final Exception e) { - throw new MessageTransportException("Undeclared checked exception (yes, netty sucks) thrown by netty.", e); - } - - // spin until we have it. - return localAddr; - } - - private static class Client extends ByteToMessageDecoder { - public final Serializer serializer; - public final Listener listener; - - Client(final Serializer serializer, final Supplier> listener) { - this.serializer = serializer; - this.listener = listener.get(); - } - - static class ResetReaderIndex implements AutoCloseable { - final ByteBuf buf; - boolean clear = false; - - ResetReaderIndex(final ByteBuf buf) { - this.buf = buf; - } - - void clear() { - clear = true; - } - - @Override - public void close() { - if (!clear) - buf.resetReaderIndex(); - } - } - - @Override - protected void decode(final ChannelHandlerContext ctx, final ByteBuf in, final List out) throws Exception { - if (in.readableBytes() < 2) - return; - - try (ResetReaderIndex rr = new ResetReaderIndex(in.markReaderIndex());) { - final short tmpsize = in.readShort(); - final int size; - if (tmpsize == -1) { - if (in.readableBytes() < 4) - return; - - size = in.readInt(); - } else { - size = tmpsize; - } - - if (size < 1) { // we expect at least '1' - ReferenceCountUtil.release(in); - // assume we have a corrupt channel - throw new IOException("Read negative message size. Assuming a corrupt channel."); - } - - if (in.readableBytes() < size) - return; - - final byte[] data = new byte[size]; - in.readBytes(data); - rr.clear(); - try (MessageBufferInput mbi = new MessageBufferInput(data);) { - @SuppressWarnings("unchecked") - final T rm = (T) serializer.deserialize(mbi, RoutedMessage.class); - listener.onMessage(rm); // this should process the message asynchronously - } - } - } - - @Override - public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) throws InterruptedException { - // Close the connection when an exception is raised. - LOGGER.error("Exception processing client connection. Closing...", cause); - ctx.close().sync(); - } - - } -} diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettySender.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettySender.java deleted file mode 100644 index 1520caeb..00000000 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettySender.java +++ /dev/null @@ -1,220 +0,0 @@ -package net.dempsy.transport.tcp.netty; - -import java.util.List; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.bootstrap.Bootstrap; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.Unpooled; -import io.netty.buffer.UnpooledHeapByteBuf; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.handler.codec.MessageToMessageEncoder; -import net.dempsy.DempsyException; -import net.dempsy.Manager; -import net.dempsy.monitoring.NodeStatsCollector; -import net.dempsy.serialization.Serializer; -import net.dempsy.transport.MessageTransportException; -import net.dempsy.transport.Sender; -import net.dempsy.transport.tcp.TcpAddress; -import net.dempsy.util.io.MessageBufferOutput; - -public final class NettySender implements Sender { - private final static ConcurrentLinkedQueue pool = new ConcurrentLinkedQueue<>(); - - private final static Logger LOGGER = LoggerFactory.getLogger(NettySender.class); - private static final AtomicLong threadNum = new AtomicLong(0); - - private final NodeStatsCollector statsCollector; - private final TcpAddress addr; - private final Serializer serializer; - private final NettySenderFactory owner; - private final AtomicReference connection = new AtomicReference<>(null); - private final EventLoopGroup group; - - private boolean isRunning = true; - - NettySender(final TcpAddress addr, final NettySenderFactory parent, final NodeStatsCollector statsCollector, - final Manager serializerManger, final EventLoopGroup group) { - this.addr = addr; - serializer = serializerManger.getAssociatedInstance(addr.serializerId); - this.statsCollector = statsCollector; - this.owner = parent; - this.group = group; - reset(); - } - - @Override - public void send(final Object message) throws MessageTransportException { - try { - final Internal cur = connection.get(); - if (cur != null) { - connection.get().ch.writeAndFlush(message).sync(); - } - } catch (final InterruptedException e) { - throw new MessageTransportException(e); - } - } - - @Override - public synchronized void stop() { - isRunning = false; - reset(); - owner.imDone(addr); - } - - private void reset() { - final Internal previous = connection.getAndSet(isRunning ? new Internal() : null); - - if (previous != null) - previous.stop(); - } - - private MessageBufferOutput getPooled() { - final MessageBufferOutput tmp = pool.poll(); - if (tmp != null) { - tmp.reset(); - return tmp; - } else { - return new MessageBufferOutput(); - } - - } - - private static class MyByteBuf extends UnpooledHeapByteBuf { - final MessageBufferOutput toRelease; - - MyByteBuf(final ByteBufAllocator alloc, final MessageBufferOutput mbo) { - super(alloc, mbo.getBuffer(), mbo.getBuffer().length); - this.toRelease = mbo; - writerIndex(mbo.getPosition()); - } - - @Override - protected void deallocate() { - // hook me. - super.deallocate(); - pool.offer(toRelease); - } - } - - private class Internal { - Channel ch = null; - - Internal() { - reset(); - } - - synchronized void reset() { - try { - final Bootstrap b = new Bootstrap(); - b.group(group) - .channel(NioSocketChannel.class) - // .option(ChannelOption.ALLOCATOR, new PooledByteBufAllocator(false)) - .handler(new ChannelInitializer() { - - @Override - protected void initChannel(final SocketChannel ch) throws Exception { - final ChannelPipeline pipeline = ch.pipeline(); - pipeline.addLast(new MessageToMessageEncoder() { - - @Override - protected void encode(final ChannelHandlerContext ctx, final Object msg, final List out) - throws Exception { - - final MessageBufferOutput o = getPooled(); - serializer.serialize(msg, o); - ByteBuf preamble; - if (o.getPosition() > Short.MAX_VALUE) { - try (final MessageBufferOutput preamblembo = new MessageBufferOutput(6);) { - preamblembo.writeShort((short) -1); - preamblembo.writeInt(o.getPosition()); - preamble = Unpooled.wrappedBuffer(preamblembo.getBuffer()); - } - } else { - try (final MessageBufferOutput preamblembo = new MessageBufferOutput(2);) { - preamblembo.writeShort((short) o.getPosition()); - preamble = Unpooled.wrappedBuffer(preamblembo.getBuffer()); - } - } - out.add(preamble); - out.add(new MyByteBuf(ctx.alloc(), o)); - - if (statsCollector != null) - statsCollector.messageSent(msg); - } - - @Override - public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) throws Exception { - LOGGER.error("Failed writing to {}", addr, cause); - // TODO: pass the failed message over for another try. - NettySender.this.reset(); - } - }); - } - }); - - // Start the connection attempt. - ch = b.connect(addr.inetAddress, addr.port).sync().channel(); - - } catch (final InterruptedException ie) { - throw new DempsyException(ie); - } catch (final Exception ioe) { // Netty can throw an IOException here, but it uses UNSAFE to do it - // so we can't catch it because the 'connect' doesn't declare that - // it throws it. Stupid f-tards! - if (RuntimeException.class.isAssignableFrom(ioe.getClass())) { - throw (RuntimeException) ioe; - } else { - LOGGER.warn("Unexpected and undeclared Netty excpetion {}", ioe.getLocalizedMessage()); - throw new DempsyException(ioe); - } - } - } - - // REALLY REALLY STOP! - synchronized void stop() { - final Channel tmpCh = ch; // in case close throws. - ch = null; - try { - if (tmpCh != null) { - if (!tmpCh.close().await(1000)) { - LOGGER.warn("Had an issue closing the sender socket."); - startCloseThread(tmpCh, - "netty-sender-closer" + threadNum.getAndIncrement() + "-to(" + addr.getGuid() + ") from (" + owner.nodeId + ")"); - } - } - } catch (final Exception e) { - LOGGER.warn("Unexpected exception closing sender socket connection", e); - startCloseThread(tmpCh, - "netty-sender-closer" + threadNum.getAndIncrement() + "-to(" + addr.getGuid() + ") from (" + owner.nodeId + ")"); - return; - } - - } - } - - private static void startCloseThread(final Channel tmpCh, final String threadName) { - // close the damn thing in another thread insistently - new Thread(() -> { - while (tmpCh.isOpen()) { - try { - if (!tmpCh.close().await(1000)) - LOGGER.warn("Had an issue closing the sender socket."); - } catch (final Exception ee) { - LOGGER.warn("Had an issue closing the sender socket.", ee); - } - } - }, threadName).start(); - } -} \ No newline at end of file diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettySenderFactory.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettySenderFactory.java deleted file mode 100644 index 502fc751..00000000 --- a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/netty/NettySenderFactory.java +++ /dev/null @@ -1,136 +0,0 @@ -package net.dempsy.transport.tcp.netty; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import net.dempsy.Infrastructure; -import net.dempsy.Manager; -import net.dempsy.monitoring.NodeStatsCollector; -import net.dempsy.serialization.Serializer; -import net.dempsy.transport.MessageTransportException; -import net.dempsy.transport.NodeAddress; -import net.dempsy.transport.SenderFactory; -import net.dempsy.transport.tcp.TcpAddress; - -public class NettySenderFactory implements SenderFactory { - private final static Logger LOGGER = LoggerFactory.getLogger(NettySenderFactory.class); - - public static final String CONFIG_KEY_SENDER_THREADS = "num_sender_threads"; - public static final String DEFAULT_SENDER_THREADS = "1"; - - private final Manager serializerManager = new Manager(Serializer.class); - private final ConcurrentHashMap senders = new ConcurrentHashMap<>(); - - private NodeStatsCollector statsCollector; - private boolean running = true; - private EventLoopGroup group = null; - - String nodeId; - - @Override - public void close() { - final List snapshot; - synchronized (this) { - running = false; - snapshot = new ArrayList<>(senders.values()); - } - snapshot.forEach(s -> s.stop()); - - // we SHOULD be all done. - final boolean recurse; - synchronized (this) { - recurse = senders.size() > 0; - } - if (recurse) - close(); - - final EventLoopGroup tmpGr = group; - group = null; - if (tmpGr != null) { - try { - if (!tmpGr.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS).await(1000)) { - LOGGER.warn("Couldn't stop netty group for sender. Will try again."); - startGroupStopThread(tmpGr, "netty-sender-group-closer" + threadNum.getAndIncrement() + ") from (" + nodeId + ")"); - } - } catch (final Exception e) { - LOGGER.warn("Unexpected exception shutting down netty group", e); - startGroupStopThread(tmpGr, "netty-sender-group-closer" + threadNum.getAndIncrement() + ") from (" + nodeId + ")"); - } - } - - } - - @Override - public boolean isReady() { - return true; - } - - @Override - public NettySender getSender(final NodeAddress destination) throws MessageTransportException { - NettySender ret = senders.get(destination); - if (ret == null) { - synchronized (this) { - if (running) { - final TcpAddress tcpaddr = (TcpAddress) destination; - ret = new NettySender(tcpaddr, this, statsCollector, serializerManager, group); - final NettySender tmp = senders.putIfAbsent(tcpaddr, ret); - if (tmp != null) { - ret.stop(); - ret = tmp; - } - } else { - throw new IllegalStateException(NettySenderFactory.class.getSimpleName() + " is stopped."); - } - } - } - return ret; - } - - private final AtomicLong threadNum = new AtomicLong(0L); - - @Override - public void start(final Infrastructure infra) { - this.statsCollector = infra.getNodeStatsCollector(); - this.nodeId = infra.getNodeId(); - - final int numSenderThreads = Integer - .parseInt(infra.getConfigValue(NettySenderFactory.class, CONFIG_KEY_SENDER_THREADS, DEFAULT_SENDER_THREADS)); - - this.group = new NioEventLoopGroup(numSenderThreads, - (ThreadFactory) r -> new Thread(r, "netty-sender-" + threadNum.getAndIncrement() + " from (" + nodeId + ")")); - } - - void imDone(final TcpAddress tcp) { - senders.remove(tcp); - } - - private static void persistentStopGroup(final EventLoopGroup tmpGr) { - if (tmpGr == null) - return; - - while (!tmpGr.isShutdown()) { - try { - if (!tmpGr.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS).await(1000)) - LOGGER.warn("Couldn't stop netty group for sender. Will try again."); - } catch (final Exception ee) { - LOGGER.warn("Unexpected exception shutting down netty group", ee); - } - } - } - - private static void startGroupStopThread(final EventLoopGroup tmpGr, final String threadName) { - // close the damn thing in another thread insistently - new Thread(() -> { - persistentStopGroup(tmpGr); - }, threadName).start(); - } -} diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioAddress.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioAddress.java new file mode 100644 index 00000000..733347dd --- /dev/null +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioAddress.java @@ -0,0 +1,17 @@ +package net.dempsy.transport.tcp.nio; + +import java.net.InetAddress; + +import net.dempsy.transport.tcp.TcpAddress; + +public class NioAddress extends TcpAddress { + private static final long serialVersionUID = 1L; + + public NioAddress(final InetAddress inetAddress, final int port, final String serializerId, final int recvBufferSize) { + super(inetAddress, port, serializerId, recvBufferSize); + } + + @SuppressWarnings("unused") + private NioAddress() {} + +} diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioReceiver.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioReceiver.java new file mode 100644 index 00000000..fc1eaf67 --- /dev/null +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioReceiver.java @@ -0,0 +1,467 @@ +package net.dempsy.transport.tcp.nio; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.dempsy.DempsyException; +import net.dempsy.Infrastructure; +import net.dempsy.serialization.Serializer; +import net.dempsy.threading.ThreadingModel; +import net.dempsy.transport.Listener; +import net.dempsy.transport.MessageTransportException; +import net.dempsy.transport.RoutedMessage; +import net.dempsy.transport.tcp.AbstractTcpReceiver; +import net.dempsy.transport.tcp.TcpAddress; +import net.dempsy.transport.tcp.TcpUtils; +import net.dempsy.transport.tcp.nio.internal.NioUtils; +import net.dempsy.transport.tcp.nio.internal.NioUtils.ReturnableBufferOutput; +import net.dempsy.util.io.MessageBufferInput; + +public class NioReceiver extends AbstractTcpReceiver> { + private static Logger LOGGER = LoggerFactory.getLogger(NioReceiver.class); + + private final AtomicBoolean isRunning = new AtomicBoolean(true); + + private NioAddress internal = null; + private NioAddress address = null; + private Binding binding = null; + private Acceptor acceptor = null; + + @SuppressWarnings("unchecked") + private Reader[] readers = new Reader[2]; + + public NioReceiver(final Serializer serializer, final int port) { + super(serializer, port); + } + + public NioReceiver(final Serializer serializer) { + this(serializer, -1); + } + + @Override + public void close() { + isRunning.set(false); + if (acceptor != null) + acceptor.close(); + + for (int i = 0; i < readers.length; i++) { + final Reader r = readers[i]; + if (r != null) + r.close(); + } + } + + @Override + public synchronized TcpAddress getAddress() { + if (internal == null) { + try { + final InetAddress addr = useLocalHost ? Inet4Address.getLocalHost() + : (addrSupplier == null ? TcpUtils.getFirstNonLocalhostInetAddress() : addrSupplier.get()); + binding = new Binding(addr, internalPort); + final InetSocketAddress inetSocketAddress = binding.bound; + internalPort = inetSocketAddress.getPort(); + internal = new NioAddress(addr, internalPort, serId, binding.recvBufferSize); + address = resolver.getExternalAddresses(internal); + } catch (final IOException e) { + throw new DempsyException(e); + } + } + return address; + } + + @SuppressWarnings("unchecked") + @Override + public void start(final Listener listener, final Infrastructure infra) throws MessageTransportException { + if (!isRunning.get()) + throw new IllegalStateException("Cannot restart an " + NioReceiver.class.getSimpleName()); + + if (binding == null) + getAddress(); // sets binding via side affect. + + // before starting the acceptor, make sure we have Readers created. + try { + for (int i = 0; i < readers.length; i++) + readers[i] = new Reader(isRunning, address.getGuid(), (Listener) listener, serializer, maxMessageSize); + } catch (final IOException ioe) { + LOGGER.error(address.getGuid() + " failed to start up readers", ioe); + throw new MessageTransportException(address.getGuid() + " failed to start up readers", ioe); + } + + final ThreadingModel threadingModel = infra.getThreadingModel(); + // now start the readers. + for (int i = 0; i < readers.length; i++) + threadingModel.runDaemon(readers[i], "nio-reader-" + i + "-" + address); + + // start the acceptor + threadingModel.runDaemon(acceptor = new Acceptor(binding, isRunning, readers, address.getGuid()), "nio-acceptor-" + address); + } + + @Override + @SuppressWarnings("unchecked") + public NioReceiver setNumHandlers(final int numHandlers) { + readers = new Reader[numHandlers]; + return this; + } + + // ============================================================================= + // These classes manages accepting external connections. + // ============================================================================= + public static class Binding { + public final Selector selector; + public final ServerSocketChannel serverChannel; + public final InetSocketAddress bound; + public final int recvBufferSize; + + public Binding(final InetAddress addr, final int port) throws IOException { + final int lport = port < 0 ? 0 : port; + selector = Selector.open(); + + serverChannel = ServerSocketChannel.open(); + serverChannel.configureBlocking(false); + + final InetSocketAddress tobind = new InetSocketAddress(addr, lport); + final ServerSocket sock = serverChannel.socket(); + sock.bind(tobind); + bound = (InetSocketAddress) sock.getLocalSocketAddress(); + recvBufferSize = sock.getReceiveBufferSize(); + } + } + + private static class Acceptor implements Runnable { + final Binding binding; + final AtomicBoolean isRunning; + final Reader[] readers; + final long numReaders; + final AtomicLong messageNum = new AtomicLong(0); + final AtomicBoolean done = new AtomicBoolean(false); + final String thisNode; + + private Acceptor(final Binding binding, final AtomicBoolean isRunning, final Reader[] readers, final String thisNode) { + this.binding = binding; + this.isRunning = isRunning; + this.readers = readers; + this.numReaders = readers.length; + this.thisNode = thisNode; + } + + @Override + public void run() { + final Selector selector = binding.selector; + final ServerSocketChannel serverChannel = binding.serverChannel; + + try { + while (isRunning.get()) { + try { + serverChannel.register(selector, SelectionKey.OP_ACCEPT); + + while (isRunning.get()) { + final int numSelected = selector.select(); + + if (numSelected == 0) + continue; + + final Iterator keys = selector.selectedKeys().iterator(); + while (keys.hasNext()) { + final SelectionKey key = keys.next(); + + keys.remove(); + + if (!key.isValid()) + continue; + + if (key.isAcceptable()) { + accept(key); + } + } + } + } catch (final IOException ioe) { + LOGGER.error("Failed during accept loop.", ioe); + } + } + } finally { + try { + serverChannel.close(); + } catch (final IOException e) { + LOGGER.error(thisNode + " had an error trying to close the accept socket channel.", e); + } + done.set(true); + } + } + + private void accept(final SelectionKey key) throws IOException { + final Reader reader = readers[(int) (messageNum.getAndIncrement() % numReaders)]; + + final ServerSocketChannel serverChannel = (ServerSocketChannel) key.channel(); + final SocketChannel channel = serverChannel.accept(); + + LOGGER.trace(thisNode + " is accepting a connection from " + channel.getRemoteAddress()); + + reader.newClient(channel); + } + + // assumes isRunning is already set to false + private void close() { + while (!done.get()) { + binding.selector.wakeup(); + Thread.yield(); + } + } + } + // ============================================================================= + + // ============================================================================= + // A Client instance is attached to each socket in the selector's register + // ============================================================================= + private static class Client { + ReturnableBufferOutput partialRead = null; + private final String thisNode; + private final Listener typedListener; + private final Serializer serializer; + private final int maxMessageSize; + + private Client(final String thisNode, final Listener listener, final Serializer serializer, final int maxMessageSize) { + this.thisNode = thisNode; + this.typedListener = listener; + this.serializer = serializer; + this.maxMessageSize = maxMessageSize; + } + + /** + * Read the size + * @return -1 if there aren't enough bytes read in to figure out the size. -2 if the + * socket channel reached it's eof. Otherwise, the size actually read. + */ + private final int readSize(final SocketChannel channel, final ByteBuffer bb) throws IOException { + final int size; + + // if (bb.position() < 4) { + // bb.limit(4); + // if (channel.read(bb) == -1) + // return -2; + // } + // + // if (bb.position() >= 4) + // size = bb.getInt(0); + // else + // size = -1; + + if (bb.position() < 2) { + // read a Short + bb.limit(2); + if (channel.read(bb) == -1) + return -2; + } + + if (bb.position() >= 2) { // we read the full short in + final short ssize = bb.getShort(0); // read the short. + + if (ssize == -1) { // we need to read the int ... indication that an int size is there. + if (bb.position() < 6) { + bb.limit(6); // set the limit to read the int. + if (channel.read(bb) == -1) // read 4 more bytes. + return -2; + } + + if (bb.position() >= 6) // we have an int based size + size = bb.getInt(2); // read an int starting after the short. + else + // we need an int based size but don't have it all yet. + size = -1; // we're going to need to try again. + + } else { // the ssize contains the full size. + size = ssize; + } + } else { + // we already tried to read the short but didn't get enought bytes. + size = -1; // try again. + } + + return size; + } + + private void closeup(final SocketChannel channel, final SelectionKey key) { + final Socket socket = channel.socket(); + final SocketAddress remoteAddr = socket.getRemoteSocketAddress(); + LOGGER.debug(thisNode + " had a connection closed by client: " + remoteAddr); + try { + channel.close(); + } catch (final IOException ioe) { + LOGGER.error(thisNode + " failed to close the receiver channel receiving data from " + remoteAddr + ". Ingoring", ioe); + } + key.cancel(); + } + + public void read(final SelectionKey key) throws IOException { + final SocketChannel channel = (SocketChannel) key.channel(); + final ReturnableBufferOutput buf; + if (partialRead == null) { + buf = NioUtils.get(); + buf.getBb().limit(2); // set it to read the short for size initially + partialRead = buf; // set the partialRead. We'll unset this when we pass it on + } else + buf = partialRead; + ByteBuffer bb = buf.getBb(); + + if (bb.limit() <= 6) { // we haven't read the size yet. + final int size = readSize(channel, bb); + if (size == -2) { // indication we hit an eof + closeup(channel, key); + return; // we're done + } + if (size == -1) { // we didn't read the size yet so just go back. + return; + } + // if the results are less than zero or WAY to big, we need to assume a corrupt channel. + if (size <= 0 || size > maxMessageSize) { + // assume the channel is corrupted and close us out. + LOGGER.warn(thisNode + " received what appears to be a corrupt message because it's size is " + size); + closeup(channel, key); + return; + } + + final int limit = bb.limit(); + if (bb.capacity() < limit + size) { + // we need to grow the underlying buffer. + buf.grow(limit + size); + bb = buf.getBb(); + } + + buf.messageStart = bb.position(); + bb.limit(limit + size); // set the limit to read the entire message. + } + + if (bb.position() < bb.limit()) { + // continue reading + if (channel.read(bb) == -1) { + closeup(channel, key); + return; + } + } + + if (bb.position() < bb.limit()) + return; // we need to wait for more data. + + // otherwise we have a message ready to go. + final ReturnableBufferOutput toGo = partialRead; + partialRead = null; + typedListener.onMessage(() -> { + try (final ReturnableBufferOutput mbo = toGo; + final MessageBufferInput mbi = new MessageBufferInput(mbo.getBuffer(), mbo.messageStart, mbo.getBb().position());) { + @SuppressWarnings("unchecked") + final T rm = (T) serializer.deserialize(mbi, RoutedMessage.class); + return rm; + } catch (final IOException ioe) { + LOGGER.error(thisNode + " failed on deserialization", ioe); + throw new DempsyException(ioe); + } + }); + } + } + + public static class Reader implements Runnable { + + private final AtomicReference landing = new AtomicReference(null); + private final Selector selector; + private final AtomicBoolean isRunning; + private final String thisNode; + private final Listener typedListener; + private final Serializer serializer; + private final int maxMessageSize; + private final AtomicBoolean done = new AtomicBoolean(false); + + public Reader(final AtomicBoolean isRunning, final String thisNode, final Listener typedListener, final Serializer serializer, + final int maxMessageSize) + throws IOException { + selector = Selector.open(); + this.isRunning = isRunning; + this.thisNode = thisNode; + this.typedListener = typedListener; + this.serializer = serializer; + this.maxMessageSize = maxMessageSize; + } + + @Override + public void run() { + + try { + while (isRunning.get()) { + try { + final int numKeysSelected = selector.select(); + + if (numKeysSelected > 0) { + final Iterator keys = selector.selectedKeys().iterator(); + while (keys.hasNext()) { + final SelectionKey key = keys.next(); + + keys.remove(); + + if (!key.isValid()) + continue; + + if (key.isReadable()) { + ((Client) key.attachment()).read(key); + } else // this shouldn't be possible + LOGGER.info(thisNode + " reciever got an unexpexted selection key " + key); + } + } else if (isRunning.get() && !done.get()) { + + // if we processed no keys then maybe we have a new client passed over to us. + final SocketChannel newClient = landing.getAndSet(null); // mark it as retrieved. + if (newClient != null) { + // we have a new client + newClient.configureBlocking(false); + final Socket socket = newClient.socket(); + final SocketAddress remote = socket.getRemoteSocketAddress(); + LOGGER.debug(thisNode + " received connection from " + remote); + newClient.register(selector, SelectionKey.OP_READ, + new Client(thisNode, typedListener, serializer, maxMessageSize)); + } + } + } catch (final IOException ioe) { + LOGGER.error("Failed during read loop.", ioe); + } + } + } finally { + done.set(true); + } + } + + // assumes isRunning is already set to false + private void close() { + while (!done.get()) { + selector.wakeup(); + Thread.yield(); + } + } + + public synchronized void newClient(final SocketChannel newClient) { + // attempt to set the landing as long as it's null + while (landing.compareAndSet(null, newClient)) + Thread.yield(); + + // wait until the Reader runnable takes it. + while (landing.get() != null && isRunning.get() && !done.get()) { + selector.wakeup(); + Thread.yield(); + } + } + } + +} diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioSender.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioSender.java new file mode 100644 index 00000000..a6932efa --- /dev/null +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioSender.java @@ -0,0 +1,140 @@ +package net.dempsy.transport.tcp.nio; + +import static net.dempsy.transport.tcp.nio.internal.NioUtils.dontInterrupt; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.SocketChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.dempsy.monitoring.NodeStatsCollector; +import net.dempsy.serialization.Serializer; +import net.dempsy.transport.MessageTransportException; +import net.dempsy.transport.Sender; +import net.dempsy.transport.tcp.TcpAddress; +import net.dempsy.transport.tcp.nio.internal.NioUtils; + +public final class NioSender implements Sender { + private final static Logger LOGGER = LoggerFactory.getLogger(NioSender.class); + + private final NodeStatsCollector statsCollector; + private final TcpAddress addr; + private final NioSenderFactory owner; + private final String nodeId; + + public final Serializer serializer; + + final SocketChannel channel; + + private boolean connected = false; + private int sendBufferSize = -1; + private int recvBufferSize = -1; + + // read from Sending + BlockingQueue messages; + boolean running = true; + + NioSender(final TcpAddress addr, final NioSenderFactory parent) { + this.owner = parent; + this.addr = addr; + serializer = parent.serializerManager.getAssociatedInstance(addr.serializerId); + this.statsCollector = parent.statsCollector; + this.nodeId = parent.nodeId; + + // messages = new LinkedBlockingQueue<>(); + messages = new ArrayBlockingQueue<>(2); + try { + channel = SocketChannel.open(); + } catch (final IOException e) { + throw new MessageTransportException(e); // this is probably impossible + } + } + + @Override + public void send(final Object message) throws MessageTransportException { + boolean done = false; + while (running && !done) { + if (running) + dontInterrupt(() -> messages.put(message)); + done = true; + } + } + + @Override + public synchronized void stop() { + running = false; + dontInterrupt(() -> Thread.sleep(1)); + + final List drainTo = new ArrayList<>(); + + // in case we're being bombarded with sends from another thread, + // we'll keep trying this until everyone realizes we're stopped. + boolean stillNotDone = true; + final long stillNotDoneStartTime = System.currentTimeMillis(); + while (stillNotDone) { + for (boolean doneGettingStopMessageQueued = false; !doneGettingStopMessageQueued;) { + messages.drainTo(drainTo); + doneGettingStopMessageQueued = messages.offer(new StopMessage()); + } + final long startTime = System.currentTimeMillis(); + while (stillNotDone) { + if (!channel.isOpen() && channel.socket().isClosed()) + stillNotDone = false; + else if ((System.currentTimeMillis() - startTime) > 500) + break; + else + Thread.yield(); + } + + // if X seconds have passed let's just close it from this side and move on. + if ((System.currentTimeMillis() - stillNotDoneStartTime) > 3000) { + stillNotDone = false; + NioUtils.closeQuietly(channel, LOGGER, nodeId + " failed directly closing channel from " + NioSender.class); + if (!channel.socket().isClosed()) + NioUtils.closeQuietly(channel.socket(), LOGGER, nodeId + " failed directly closing socket from " + NioSender.class); + } + } + + drainTo.forEach(o -> statsCollector.messageNotSent()); + owner.imDone(addr); + } + + static class StopMessage {} + + void connect() throws IOException { + if (!connected) { + channel.configureBlocking(true); + channel.connect(new InetSocketAddress(addr.inetAddress, addr.port)); + channel.configureBlocking(false); + sendBufferSize = channel.socket().getSendBufferSize(); + recvBufferSize = addr.recvBufferSize; + connected = true; + owner.working.putIfAbsent(this, this); + } + } + + private int cachedBatchSize = -1; + + int getMaxBatchSize() { + if (cachedBatchSize < 0) { + int ret; + if (recvBufferSize <= 0) + ret = sendBufferSize; + else if (sendBufferSize <= 0) + ret = recvBufferSize; + else ret = Math.min(recvBufferSize, sendBufferSize); + if (ret <= 0) { + LOGGER.warn(nodeId + " sender to " + addr.getGuid() + " couldn't determine send and receieve buffer sizes. Setting batch size to "); + ret = owner.mtu; + } + cachedBatchSize = Math.min(ret, owner.mtu); + } + return cachedBatchSize; + } +} \ No newline at end of file diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioSenderFactory.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioSenderFactory.java new file mode 100644 index 00000000..a62f2fad --- /dev/null +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/NioSenderFactory.java @@ -0,0 +1,290 @@ +package net.dempsy.transport.tcp.nio; + +import static net.dempsy.transport.tcp.nio.internal.NioUtils.dontInterrupt; +import static net.dempsy.util.Functional.chain; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.dempsy.Infrastructure; +import net.dempsy.Manager; +import net.dempsy.monitoring.NodeStatsCollector; +import net.dempsy.serialization.Serializer; +import net.dempsy.transport.MessageTransportException; +import net.dempsy.transport.NodeAddress; +import net.dempsy.transport.SenderFactory; +import net.dempsy.transport.tcp.TcpAddress; +import net.dempsy.util.StupidHashMap; + +public class NioSenderFactory implements SenderFactory { + public final static Logger LOGGER = LoggerFactory.getLogger(NioSenderFactory.class); + + public static final String CONFIG_KEY_SENDER_THREADS = "send_threads"; + public static final String DEFAULT_SENDER_THREADS = "2"; + + // public static final String CONFIG_KEY_SENDER_BLOCKING = "send_blocking"; + // public static final String DEFAULT_SENDER_BLOCKING = "true"; + + public static final String CONFIG_KEY_SENDER_MAX_QUEUED = "send_max_queued"; + public static final String DEFAULT_SENDER_MAX_QUEUED = "1000"; + + public static final String CONFIG_KEY_SENDER_TCP_MTU = "tcp_mtu"; + public static final String DEFAULT_SENDER_TCP_MTU = "1400"; + + public static final String CONFIG_KEY_SENDER_STOP_TIMEOUT_MILLIS = "sender_stop_timeout_millis"; + public static final String DEFAULT_SENDER_STOP_TIMEOUT_MILLIS = "3000"; + + private final ConcurrentHashMap senders = new ConcurrentHashMap<>(); + + final StupidHashMap working = new StupidHashMap<>(); + + // ======================================= + // Read from NioSender + final Manager serializerManager = new Manager(Serializer.class); + final AtomicBoolean isRunning = new AtomicBoolean(true); + NodeStatsCollector statsCollector; + String nodeId; + int maxNumberOfQueuedOutgoing; + // boolean blocking; + int mtu = Integer.parseInt(DEFAULT_SENDER_TCP_MTU); + int stopTimeout = Integer.parseInt(DEFAULT_SENDER_STOP_TIMEOUT_MILLIS); + // ======================================= + + private Sending[] sendings; + private Thread[] sendingsThreads; + private final AtomicBoolean sendingsRunning = new AtomicBoolean(true); + + @Override + public void close() { + LOGGER.trace(nodeId + " stopping " + NioSenderFactory.class.getSimpleName()); + final List snapshot; + synchronized (this) { + isRunning.set(false); + snapshot = new ArrayList<>(senders.values()); + } + snapshot.forEach(s -> s.stop()); + + // we SHOULD be all done. + final boolean recurse; + synchronized (this) { + recurse = senders.size() > 0; + } + if (recurse) + close(); + + // all senders closed, we should stop the threads. + sendingsRunning.set(false); + final List threads = new ArrayList<>(Arrays.asList(sendingsThreads)); + boolean done = false; + final long startWaitTime = System.currentTimeMillis(); + while (threads.size() > 0 && !done) { + done = true; + for (final Iterator iter = threads.iterator(); iter.hasNext();) { + final Thread cur = iter.next(); + if (!cur.isAlive()) + iter.remove(); + else + done = false; + } + if ((System.currentTimeMillis() - startWaitTime) > stopTimeout) { + LOGGER.warn("Stopping without having been able to stop all sending threads."); + done = true; + } + } + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public NioSender getSender(final NodeAddress destination) throws MessageTransportException { + final TcpAddress tcpaddr = (TcpAddress) destination; + final NioSender ret; + if (isRunning.get()) { + ret = senders.computeIfAbsent(tcpaddr, a -> new NioSender(a, this)); + } else + throw new MessageTransportException(nodeId + " sender had getSender called while stopped."); + + try { + ret.connect(); + } catch (final IOException e) { + throw new MessageTransportException(nodeId + " sender failed to connect to " + destination.getGuid(), e); + } + return ret; + } + + @Override + public void start(final Infrastructure infra) { + this.statsCollector = infra.getNodeStatsCollector(); + this.nodeId = infra.getNodeId(); + + final int numSenderThreads = Integer + .parseInt(infra.getConfigValue(NioSender.class, CONFIG_KEY_SENDER_THREADS, DEFAULT_SENDER_THREADS)); + + mtu = Integer + .parseInt(infra.getConfigValue(NioSender.class, CONFIG_KEY_SENDER_TCP_MTU, DEFAULT_SENDER_TCP_MTU)); + + maxNumberOfQueuedOutgoing = Integer.parseInt(infra.getConfigValue(NioSender.class, CONFIG_KEY_SENDER_MAX_QUEUED, DEFAULT_SENDER_MAX_QUEUED)); + + // blocking = Boolean.parseBoolean(infra.getConfigValue(NioSender.class, CONFIG_KEY_SENDER_BLOCKING, DEFAULT_SENDER_BLOCKING)); + + stopTimeout = Integer + .parseInt(infra.getConfigValue(NioSender.class, CONFIG_KEY_SENDER_STOP_TIMEOUT_MILLIS, DEFAULT_SENDER_STOP_TIMEOUT_MILLIS)); + + sendings = new Sending[numSenderThreads]; + sendingsThreads = new Thread[numSenderThreads]; + + // now start the sending threads. + for (int i = 0; i < sendings.length; i++) + chain(sendingsThreads[i] = new Thread(sendings[i] = new Sending(sendingsRunning, nodeId, working, statsCollector), + "nio-sender-" + i + "-" + nodeId), t -> t.start()); + + } + + void imDone(final TcpAddress tcp) { + senders.remove(tcp); + } + + public static class Sending implements Runnable { + final AtomicBoolean isRunning; + final Selector selector; + final String nodeId; + final StupidHashMap idleSenders; + final NodeStatsCollector statsCollector; + + Sending(final AtomicBoolean isRunning, final String nodeId, final StupidHashMap working, + final NodeStatsCollector statsCollector) + throws MessageTransportException { + this.isRunning = isRunning; + this.nodeId = nodeId; + this.idleSenders = working; + this.statsCollector = statsCollector; + try { + this.selector = Selector.open(); + } catch (final IOException e) { + throw new MessageTransportException(e); + } + } + + @Override + public void run() { + int numNothing = 0; + while (isRunning.get()) { + try { + // blocking causes attempts to register to block creating a potential deadlock + final int numSelected = selector.selectNow(); + + // are there any sockets ready to write? + if (numSelected == 0) { + // ===================================================================== + // nothing ready ... might as well spend some time serializing messages + final Set keys = selector.keys(); + if (keys != null && keys.size() > 0) { + numNothing = 0; // reset the yield count since we have something to do + final SenderHolder thisOneCanSerialize = keys.stream() + .map(k -> (SenderHolder) k.attachment()) + .filter(s -> !s.readyToWrite(true)) // if we're ready to write then we don't need to do more. + .filter(s -> s.readyToSerialize()) + .findFirst() + .orElse(null); + if (thisOneCanSerialize != null) + thisOneCanSerialize.trySerialize(); + } + // ===================================================================== + else { // nothing to serialize, do we have any new senders that need handling? + if (!checkForNewSenders()) { // if we didn't do anything then sleep/yield based on how long we've been bord. + numNothing++; + if (numNothing > 1000) { + dontInterrupt(() -> Thread.sleep(1), ie -> { + if (isRunning.get()) + LOGGER.error(nodeId + " sender interrupted", ie); + }); + } else + Thread.yield(); + } else // otherwise we DID do something + numNothing = 0; + } + continue; + } else + numNothing = 0; // reset the yield count since we have something to do + + final Iterator keys = selector.selectedKeys().iterator(); + while (keys.hasNext()) { + final SelectionKey key = keys.next(); + + keys.remove(); + + if (!key.isValid()) + continue; + + if (key.isWritable()) { + final SenderHolder sh = (SenderHolder) key.attachment(); + if (sh.writeSomethingReturnDone(key, statsCollector)) { + idleSenders.putIfAbsent(sh.sender, sh.sender); + key.cancel(); + } + } + } + } catch (final IOException ioe) { + LOGGER.error(nodeId + " sender failed", ioe); + } finally { + // LOGGER.trace("looping sending thread:" + numNothing); + } + } + } + + private boolean checkForNewSenders() throws IOException { + boolean didSomething = false; + final Set curSenders = idleSenders.keySet(); + final Set newSenders = new HashSet<>(); + + try { // if we fail here we need to put the senders back or we'll loose them forever. + + // move any NioSenders with data from working and onto newSenders + curSenders.stream() + .filter(s -> s.messages.peek() != null) + .forEach(s -> { + final NioSender cur = idleSenders.remove(s); + // removing them means putting them on the newSenders set so we can track them + if (cur != null) + newSenders.add(cur); + }); + + // newSenders are now mine since they've been removed from working. + + // go through each new sender ... + for (final Iterator iter = newSenders.iterator(); iter.hasNext();) { + final NioSender cur = iter.next(); + + // ... if the new sender has messages ... + if (cur.messages.peek() != null) { + // ... register the channel for writing and attach the SenderHolder + new SenderHolder(cur).register(selector); + iter.remove(); + didSomething = true; // we did something. + } + } + } finally { + // any still on toWork need to be returned to working + newSenders.forEach(s -> idleSenders.putIfAbsent(s, s)); + } + + return didSomething; + } + + } +} diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/SenderHolder.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/SenderHolder.java new file mode 100644 index 00000000..61a79dd6 --- /dev/null +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/SenderHolder.java @@ -0,0 +1,159 @@ +package net.dempsy.transport.tcp.nio; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.util.LinkedList; + +import net.dempsy.monitoring.NodeStatsCollector; +import net.dempsy.serialization.Serializer; +import net.dempsy.transport.tcp.nio.NioSender.StopMessage; +import net.dempsy.transport.tcp.nio.internal.NioUtils; +import net.dempsy.transport.tcp.nio.internal.NioUtils.ReturnableBufferOutput; + +public class SenderHolder { + public final NioSender sender; + + private boolean previouslyWroteOddNumBufs = false; + + private int numBytesToWrite = 0; + private final LinkedList serializedMessages = new LinkedList<>(); + + public SenderHolder(final NioSender sender) { + this.sender = sender; + } + + private final void add(final ReturnableBufferOutput ob) { + numBytesToWrite += ob.getPosition(); + serializedMessages.add(ob); + } + + private final void addBack(final ReturnableBufferOutput ob, final int remaining) { + numBytesToWrite += remaining; + serializedMessages.add(ob); + } + + public final void register(final Selector selector) throws ClosedChannelException { + sender.channel.register(selector, SelectionKey.OP_WRITE, this); + } + + public final boolean shouldClose() { + final Object peek = sender.messages.peek(); + return (peek != null && (peek instanceof StopMessage)); + } + + public final boolean readyToSerialize() { + final Object peek = sender.messages.peek(); + return peek != null && !(peek instanceof StopMessage); + } + + public final boolean readyToWrite(final boolean considerMtu) { + return numBytesToWrite() >= (considerMtu ? sender.getMaxBatchSize() : 1); + } + + public final int numBytesToWrite() { + return numBytesToWrite; + } + + public void trySerialize() throws IOException { + prepareToWriteBestEffort(); + } + + public boolean writeSomethingReturnDone(final SelectionKey key, final NodeStatsCollector statsCollector) throws IOException { + prepareToWriteBestEffort(); + + if (readyToWrite(false)) { // if we have ANYTHING to write. + // ================================================== + // do a write pass + // ================================================== + // collect up the ByteBuffers to send. + final int numBb = serializedMessages.size(); + final ReturnableBufferOutput[] toSendRbos = new ReturnableBufferOutput[numBb]; + final ByteBuffer[] toSend = new ByteBuffer[numBb]; + int curIndex = 0; + + for (final ReturnableBufferOutput c : serializedMessages) { + toSend[curIndex] = c.getFloppedBb(); + toSendRbos[curIndex] = c; + curIndex++; + } + + final SocketChannel channel = (SocketChannel) key.channel(); + channel.write(toSend); // okay, let's see what we have now. + + numBytesToWrite = 0; + serializedMessages.clear(); + int numBufsCompletelyWritten = 0; + for (int i = 0; i < toSend.length; i++) { + final ByteBuffer curBb = toSend[i]; + final ReturnableBufferOutput curRob = toSendRbos[i]; + final int remaining = curBb.remaining(); + if (remaining != 0) { + addBack(curRob, remaining); + } else + numBufsCompletelyWritten++; + } + + // how many messages did we write? + if (previouslyWroteOddNumBufs) + numBufsCompletelyWritten++; + + previouslyWroteOddNumBufs = (numBufsCompletelyWritten & 0x1) == 0x1; // is numBufsCompletelyWritten now an odd number? + + final int numMessageDelivered = numBufsCompletelyWritten >> 1; + for (int i = 0; i < numMessageDelivered; i++) + statsCollector.messageSent(null); + + return !(readyToWrite(false) || readyToSerialize()); + + } else + return sender.messages.peek() == null; + + } + + private void prepareToWriteBestEffort() throws IOException { + while (true) { + if (!readyToWrite(true)) { + if (readyToSerialize()) { + serializeOne(); + continue; + } else + return; + } else + return; + } + } + + private boolean serializeOne() throws IOException { + if (shouldClose()) + return false; + + final Object toSer = sender.messages.poll(); + if (toSer != null) { + final ReturnableBufferOutput header = NioUtils.get(); + final ReturnableBufferOutput data = NioUtils.get(); + serialize(sender.serializer, toSer, header, data); + add(header); + add(data); + return true; + } + return false; + + } + + private static void serialize(final Serializer ser, final Object obj, final ReturnableBufferOutput header, final ReturnableBufferOutput data) + throws IOException { + header.reset(); + data.reset(); + ser.serialize(obj, data); + final int size = data.getPosition(); + if (size > Short.MAX_VALUE) { + header.writeShort((short) -1); + header.writeInt(size); + } else + header.writeShort((short) size); + } +} \ No newline at end of file diff --git a/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/internal/NioUtils.java b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/internal/NioUtils.java new file mode 100644 index 00000000..0b2ed276 --- /dev/null +++ b/dempsy-framework.impl/src/main/java/net/dempsy/transport/tcp/nio/internal/NioUtils.java @@ -0,0 +1,96 @@ +package net.dempsy.transport.tcp.nio.internal; + +import java.nio.ByteBuffer; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.Consumer; + +import org.slf4j.Logger; + +import net.dempsy.util.Functional.RunnableThrows; +import net.dempsy.util.io.MessageBufferOutput; + +public class NioUtils { + // ============================================================================= + // These classes manage the buffer pool used by the readers and clients + // ============================================================================= + private static ConcurrentLinkedQueue bufferPool = new ConcurrentLinkedQueue<>(); + + public static ReturnableBufferOutput get() { + ReturnableBufferOutput ret = bufferPool.poll(); + if (ret == null) + ret = new ReturnableBufferOutput(); + return ret; + } + + public static void closeQuietly(final AutoCloseable ac, final Logger LOGGER, final String failedMessage) { + try { + ac.close(); + } catch (final Exception e) { + LOGGER.warn(failedMessage, e); + } + } + + public static void dontInterrupt(final RunnableThrows runner) { + try { + runner.run(); + } catch (final InterruptedException ie) {} + } + + public static void dontInterrupt(final RunnableThrows runner, final Consumer handler) { + try { + runner.run(); + } catch (final InterruptedException ie) { + handler.accept(ie); + } + } + + public static class ReturnableBufferOutput extends MessageBufferOutput { + private ByteBuffer bb = null; + private boolean flopped = false; + + public int messageStart = -1; + + private ReturnableBufferOutput() { + super(2048); /// holds at least one full packet + } + + public ByteBuffer getBb() { + if (bb == null) + bb = ByteBuffer.wrap(getBuffer()); + return bb; + } + + public ByteBuffer getFloppedBb() { + final ByteBuffer lbb = getBb(); + if (!flopped) { + lbb.limit(getPosition()); + lbb.position(0); // position to zero. + flopped = true; + } + return lbb; + } + + @Override + public void close() { + super.close(); + reset(); + messageStart = -1; + bb = null; + flopped = false; + bufferPool.offer(this); + } + + @Override + public void grow(final int newcap) { + super.grow(newcap); + if (bb != null) { + final ByteBuffer obb = bb; + bb = ByteBuffer.wrap(getBuffer()); + bb.position(obb.position()); + bb.limit(obb.limit()); + flopped = false; + } + } + } + +} diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/DempsyBaseTest.java b/dempsy-framework.impl/src/test/java/net/dempsy/DempsyBaseTest.java index b2a79d58..51c2d1c1 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/DempsyBaseTest.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/DempsyBaseTest.java @@ -34,6 +34,7 @@ public abstract class DempsyBaseTest { * every runCombos call. This can make tests run for a loooooong time. */ public static boolean hardcore = false; + public static boolean butRotateSerializer = true; protected Logger LOGGER; @@ -95,14 +96,14 @@ public Combos(final String[] routers, final String[] containers, final String[] } public static List elasticRouterIds = Arrays.asList("managed"); - public static String[] transportsThatRequireSerializer = { "netty" }; + public static String[] transportsThatRequireSerializer = { "nio" }; public static Combos hardcore() { return new Combos( new String[] { "simple", "managed" }, new String[] { "locking", "nonlocking", "altnonlocking" }, new String[] { "local", "zookeeper" }, - new String[] { "bq", "passthrough", "netty" }, + new String[] { "bq", "passthrough", "nio" }, new String[] { "json", "java", "kryo" }); } @@ -111,7 +112,7 @@ public static Combos production() { new String[] { "managed" }, new String[] { "altnonlocking" }, new String[] { "zookeeper" }, - new String[] { "netty" }, + new String[] { "nio" }, new String[] { "kryo" }); } @@ -124,14 +125,24 @@ public static boolean requiresSerialization(final String transport) { public static Collection combos() { final Combos combos = (hardcore) ? hardcore() : production(); + // since serialization is orthogonal to the transport we wont test every transport + // with every serializer. Instead we'll rotate the serializers if that's requested. + int serializerIndex = 0; + final List ret = new ArrayList<>(); for (final String router : combos.routers) { for (final String container : combos.containers) { for (final String sessFact : combos.sessions) { for (final String tp : combos.transports) { if (requiresSerialization(tp)) { - for (final String ser : combos.serializers) - ret.add(new Object[] { router, container, sessFact, tp, ser }); + if (butRotateSerializer) { + ret.add(new Object[] { router, container, sessFact, tp, + combos.serializers[(serializerIndex % combos.serializers.length)] }); + serializerIndex++; + } else { + for (final String ser : combos.serializers) + ret.add(new Object[] { router, container, sessFact, tp, ser }); + } } else ret.add(new Object[] { router, container, sessFact, tp, "none" }); } diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/TestElasticity.java b/dempsy-framework.impl/src/test/java/net/dempsy/TestElasticity.java index 80f3e925..6b3e6af8 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/TestElasticity.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/TestElasticity.java @@ -241,7 +241,7 @@ protected void waitForEvenShardDistribution(final ClusterInfoSession session, fi // gather the collection of reachable ContainerAddresses from each Router to the given cluster. .map(r -> { if (showLog) - LOGGER.trace("From {}", r.thisNode()); + LOGGER.trace("From {}", r.thisNodeId()); return r.allReachable(cluster); }) // extract a set of NodeAddresses from each of the ContainerAddress collections @@ -279,7 +279,7 @@ public void testForProfiler() throws Throwable { final boolean ret = elasticRouterIds.contains(r); if (ret) { LOGGER.info("====================================================================================="); - LOGGER.info("======== Running testForProfiler with " + r + ", " + c + ", " + s + ", " + t); + LOGGER.info("======== Running testForProfiler with " + r + ", " + c + ", " + s + ", " + t + "/" + ser); } return ret; }, actxPath, new String[][][] { @@ -553,7 +553,10 @@ public void testExpansionPassivation() throws Exception { waitForEvenShardDistribution(session, "test-cluster1", 2, nodes); // about 1/2 should drop out. - assertTrue(poll(o -> sc.getMessageProcessorCount() < 15L)); + assertTrue(poll(o -> { + // System.err.println(sc.getMessageProcessorCount()); + return sc.getMessageProcessorCount() < 15L; + })); } } finally { diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/TestWordCount.java b/dempsy-framework.impl/src/test/java/net/dempsy/TestWordCount.java index 5c3bfd7b..ccd0a474 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/TestWordCount.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/TestWordCount.java @@ -127,6 +127,7 @@ public static class WordProducer implements Adaptor { private final AtomicBoolean isRunning = new AtomicBoolean(false); private Dispatcher dispatcher = null; private final AtomicBoolean done = new AtomicBoolean(false); + private final AtomicBoolean stopped = new AtomicBoolean(false); public boolean onePass = true; public static CountDownLatch latch = new CountDownLatch(0); KeyExtractor ke = new KeyExtractor(); @@ -150,29 +151,36 @@ public void setDispatcher(final Dispatcher dispatcher) { @Override public void start() { try { - latch.await(); - } catch (final InterruptedException ie) { - throw new RuntimeException(ie); - } - isRunning.set(true); - while (isRunning.get() && !done.get()) { - // obtain data from an external source - final String wordString = getNextWordFromSoucre(); try { - dispatcher.dispatch(ke.extract(new Word(wordString))); - numDispatched++; - if (numDispatched % 10000 == 0) - System.out.print("."); - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - LOGGER.error("Failed to dispatch", e); + latch.await(); + } catch (final InterruptedException ie) { + throw new RuntimeException(ie); + } + isRunning.set(true); + while (isRunning.get() && !done.get()) { + // obtain data from an external source + final String wordString = getNextWordFromSoucre(); + try { + dispatcher.dispatch(ke.extract(new Word(wordString))); + numDispatched++; + if (numDispatched % 10000 == 0) + System.out.print("."); + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + LOGGER.error("Failed to dispatch", e); + } } + System.out.println(); + } finally { + stopped.set(true); } - System.out.println(); } @Override public void stop() { isRunning.set(false); + + while (!stopped.get()) + Thread.yield(); } private int curCount = 0; diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainer.java b/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainer.java index a0f36c56..d78829f4 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainer.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainer.java @@ -51,6 +51,7 @@ import org.junit.runners.Parameterized.Parameters; import org.springframework.context.support.ClassPathXmlApplicationContext; +import net.dempsy.DempsyException; import net.dempsy.NodeManager; import net.dempsy.cluster.local.LocalClusterSessionFactory; import net.dempsy.config.ClusterId; @@ -73,6 +74,8 @@ import net.dempsy.lifecycle.annotation.utils.KeyExtractor; import net.dempsy.messages.Adaptor; import net.dempsy.messages.Dispatcher; +import net.dempsy.messages.KeyedMessage; +import net.dempsy.messages.KeyedMessageWithType; import net.dempsy.monitoring.ClusterStatsCollector; import net.dempsy.monitoring.basic.BasicNodeStatsCollector; import net.dempsy.transport.blockingqueue.BlockingQueueReceiver; @@ -112,6 +115,7 @@ public static Collection data() { public static Map cache = null; public static Set outputMessages = null; + public static RuntimeException justThrowMe = null; public TestContainer(final String containerId) { this.containerId = containerId; @@ -124,6 +128,7 @@ private T track(final T o) { @Before public void setUp() throws Exception { + justThrowMe = null; track(new SystemPropertyManager()).set("container-type", containerId); context = track(new ClassPathXmlApplicationContext(ctx)); sessionFactory = new LocalClusterSessionFactory(); @@ -138,6 +143,7 @@ public void setUp() throws Exception { public void tearDown() throws Exception { cache = null; outputMessages = null; + justThrowMe = null; recheck(() -> toClose.forEach(v -> uncheck(() -> v.close())), Exception.class); toClose.clear(); LocalClusterSessionFactory.completeReset(); @@ -273,6 +279,9 @@ public void handle(final ContainerTestMessage message) throws InterruptedExcepti @MessageHandler public ContainerTestMessage handle(final MyMessage message) throws InterruptedException { + if (justThrowMe != null) + throw justThrowMe; + myKey = message.getKey(); if (cache != null) @@ -334,6 +343,33 @@ public void stop() {} // ---------------------------------------------------------------------------- // Test Cases // ---------------------------------------------------------------------------- + public static final KeyExtractor ke = new KeyExtractor(); + + @Test + public void testWrongTypeMessage() throws Exception { + assertEquals(0, ((ClusterMetricGetters) container.statCollector).getMessageFailedCount()); + final KeyedMessageWithType kmwt = ke.extract(new MyMessage("YO")).get(0); + container.dispatch(new KeyedMessage(kmwt.key, new Object()), true); + assertEquals(1, ((ClusterMetricGetters) container.statCollector).getMessageFailedCount()); + } + + @Test + public void testMpThrowsDempsyException() throws Exception { + assertEquals(0, ((ClusterMetricGetters) container.statCollector).getMessageFailedCount()); + justThrowMe = new DempsyException("JustThrowMe!"); + final KeyedMessageWithType kmwt = ke.extract(new MyMessage("YO")).get(0); + container.dispatch(kmwt, true); + assertEquals(1, ((ClusterMetricGetters) container.statCollector).getMessageFailedCount()); + } + + @Test + public void testMpThrowsException() throws Exception { + assertEquals(0, ((ClusterMetricGetters) container.statCollector).getMessageFailedCount()); + justThrowMe = new RuntimeException("JustThrowMe!"); + final KeyedMessageWithType kmwt = ke.extract(new MyMessage("YO")).get(0); + container.dispatch(kmwt, true); + assertEquals(1, ((ClusterMetricGetters) container.statCollector).getMessageFailedCount()); + } @Test public void testConfiguration() throws Exception { diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainerLoadHandling.java b/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainerLoadHandling.java index bcccf22f..e6c8359b 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainerLoadHandling.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/container/TestContainerLoadHandling.java @@ -45,6 +45,7 @@ import net.dempsy.config.ClusterId; import net.dempsy.container.altnonlocking.NonLockingAltContainer; import net.dempsy.container.locking.LockingContainer; +import net.dempsy.container.mocks.DummyInbound; import net.dempsy.container.mocks.MockInputMessage; import net.dempsy.container.mocks.MockOutputMessage; import net.dempsy.container.nonlocking.NonLockingContainer; @@ -121,6 +122,8 @@ public void setUp() throws Exception { .setClusterId(cid); container.setDispatcher(dispatcher); + container.setInbound(new DummyInbound()); + container.start(new TestInfrastructure(null, null) { @Override diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/container/locking/TestInstanceManager.java b/dempsy-framework.impl/src/test/java/net/dempsy/container/locking/TestInstanceManager.java index ae9a72c1..0ba83101 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/container/locking/TestInstanceManager.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/container/locking/TestInstanceManager.java @@ -32,6 +32,7 @@ import net.dempsy.container.Container; import net.dempsy.container.ContainerException; import net.dempsy.container.locking.LockingContainer.InstanceWrapper; +import net.dempsy.container.mocks.DummyInbound; import net.dempsy.lifecycle.annotation.Activation; import net.dempsy.lifecycle.annotation.MessageHandler; import net.dempsy.lifecycle.annotation.MessageKey; @@ -50,7 +51,7 @@ public class TestInstanceManager { - private LockingContainer manager; + private LockingContainer container; // ---------------------------------------------------------------------------- // Test classes -- must be static/public for introspection @@ -244,9 +245,10 @@ public LockingContainer setupContainer(final MessageProcessorLifecycle protot statsCollector = new BasicClusterStatsCollector(); nodeStats = new BasicNodeStatsCollector(); - manager = (LockingContainer) new LockingContainer().setMessageProcessor(prototype).setClusterId(new ClusterId("test", "test")); - manager.setDispatcher(dispatcher); - manager.start(new TestInfrastructure(null, null) { + container = (LockingContainer) new LockingContainer().setMessageProcessor(prototype).setClusterId(new ClusterId("test", "test")); + container.setDispatcher(dispatcher); + container.setInbound(new DummyInbound()); + container.start(new TestInfrastructure(null, null) { @Override public BasicClusterStatsCollector getClusterStatsCollector(final ClusterId clusterId) { @@ -258,7 +260,7 @@ public NodeStatsCollector getNodeStatsCollector() { return nodeStats; } }); - return manager; + return container; } @Test diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/container/mocks/DummyInbound.java b/dempsy-framework.impl/src/test/java/net/dempsy/container/mocks/DummyInbound.java new file mode 100644 index 00000000..02e05d47 --- /dev/null +++ b/dempsy-framework.impl/src/test/java/net/dempsy/container/mocks/DummyInbound.java @@ -0,0 +1,29 @@ +package net.dempsy.container.mocks; + +import net.dempsy.Infrastructure; +import net.dempsy.KeyspaceChangeListener; +import net.dempsy.config.ClusterId; +import net.dempsy.router.RoutingStrategy; +import net.dempsy.router.RoutingStrategy.ContainerAddress; + +public class DummyInbound implements RoutingStrategy.Inbound { + + @Override + public void stop() {} + + @Override + public void start(final Infrastructure infra) {} + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setContainerDetails(final ClusterId clusterId, final ContainerAddress address, final KeyspaceChangeListener listener) {} + + @Override + public boolean doesMessageKeyBelongToNode(final Object messageKey) { + return true; + } +} diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/container/nonlocking/TestInstanceManager.java b/dempsy-framework.impl/src/test/java/net/dempsy/container/nonlocking/TestInstanceManager.java index 3e911590..b51daee2 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/container/nonlocking/TestInstanceManager.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/container/nonlocking/TestInstanceManager.java @@ -28,6 +28,7 @@ import net.dempsy.container.ClusterMetricGetters; import net.dempsy.container.Container; import net.dempsy.container.ContainerException; +import net.dempsy.container.mocks.DummyInbound; import net.dempsy.lifecycle.annotation.Activation; import net.dempsy.lifecycle.annotation.MessageHandler; import net.dempsy.lifecycle.annotation.MessageKey; @@ -241,6 +242,7 @@ public Container setupContainer(final MessageProcessorLifecycle prototype) th manager = new NonLockingContainer().setMessageProcessor(prototype).setClusterId(new ClusterId("test", "test")); manager.setDispatcher(dispatcher); + manager.setInbound(new DummyInbound()); manager.start(new TestInfrastructure(null, null) { BasicNodeStatsCollector nStats = new BasicNodeStatsCollector(); diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestLeaderAndSubscriber.java b/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestLeaderAndSubscriber.java index 928de87a..c9c738a7 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestLeaderAndSubscriber.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestLeaderAndSubscriber.java @@ -122,7 +122,7 @@ public void testLeaderWithSubscriber() throws Exception { final AtomicBoolean isRunning = new AtomicBoolean(true); try { - final Subscriber s = new Subscriber(utils, infra, isRunning, (l, m, i) -> {}, null); + final Subscriber s = new Subscriber(utils, infra, isRunning, (l, m) -> {}); s.process(); final Leader l = new Leader(utils, infra, isRunning); l.process(); @@ -153,7 +153,7 @@ public void testLeaderWithMutipleSubscribers() throws Exception { for (int i = 0; i < NUM_SUBS; i++) { final Utils utils = new Utils(makeInfra(session, sched), cid, new ContainerAddress(new DummyNodeAddress(), new int[] { 0 })); final Subscriber s; - subs.add(s = new Subscriber(utils, infra, isRunning, (l, m, xi) -> {}, null)); + subs.add(s = new Subscriber(utils, infra, isRunning, (l, m) -> {})); final Thread t = new Thread(() -> { // wait for it. while (!go.get()) { diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestManagedRoutingStrategy.java b/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestManagedRoutingStrategy.java index d35b1494..901d5ea1 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestManagedRoutingStrategy.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/router/managed/TestManagedRoutingStrategy.java @@ -56,7 +56,7 @@ public void testInboundSimpleHappyPathRegister() throws Exception { assertNotNull(ib); assertTrue(ManagedInbound.class.isAssignableFrom(ib.getClass())); - ib.setContainerDetails(clusterId, new ContainerAddress(new DummyNodeAddress("testInboundSimpleHappyPathRegister"), 0), (l, m, i) -> {}); + ib.setContainerDetails(clusterId, new ContainerAddress(new DummyNodeAddress("testInboundSimpleHappyPathRegister"), 0), (l, m) -> {}); ib.start(infra); assertTrue(waitForShards(session, msutils, numShardsToExpect)); @@ -124,12 +124,12 @@ public void testInboundDoubleHappyPathRegister() throws Exception { final ContainerAddress node1Ca = new ContainerAddress(new DummyNodeAddress("node1"), 0); final Utils utils = new Utils(infra, clusterId, node1Ca); - ib1.setContainerDetails(clusterId, node1Ca, (l, m, i) -> {}); + ib1.setContainerDetails(clusterId, node1Ca, (l, m) -> {}); ib1.start(infra); final ContainerAddress node2Ca = new ContainerAddress(new DummyNodeAddress("node2"), 0); - ib2.setContainerDetails(clusterId, node2Ca, (l, m, i) -> {}); + ib2.setContainerDetails(clusterId, node2Ca, (l, m) -> {}); try (final ClusterInfoSession session2 = sessFact.createSession();) { ib2.start(new TestInfrastructure(session2, infra.getScheduler())); @@ -163,7 +163,7 @@ public void testInboundResillience() throws Exception { final Infrastructure infra = makeInfra(session, sched); final Utils msutils = new Utils(infra, clusterId, ca); - ib.setContainerDetails(clusterId, ca, (l, m, i) -> {}); + ib.setContainerDetails(clusterId, ca, (l, m) -> {}); ib.start(infra); checkForShardDistribution(session, msutils, numShardsToExpect, 1); @@ -184,7 +184,7 @@ public void testInboundWithOutbound() throws Exception { final Infrastructure infra = makeInfra(session, sched); final Utils msutils = new Utils(infra, cid, oca); - ib.setContainerDetails(cid, oca, (l, m, i) -> {}); + ib.setContainerDetails(cid, oca, (l, m) -> {}); ib.start(infra); checkForShardDistribution(session, msutils, numShardsToExpect, 1); @@ -216,7 +216,7 @@ public void testInboundWithOutbound() throws Exception { try (ClusterInfoSession ses3 = sessFact.createSession(); RoutingStrategy.Inbound ib2 = manager.getAssociatedInstance(ManagedInbound.class.getPackage().getName())) { - ib2.setContainerDetails(cid, ca, (l, m, i) -> {}); + ib2.setContainerDetails(cid, ca, (l, m) -> {}); ib2.start(makeInfra(ses3, sched)); assertTrue(poll(o -> ob.selectDestinationForMessage(km) != null)); diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/router/simple/TestSimpleRoutingStrategy.java b/dempsy-framework.impl/src/test/java/net/dempsy/router/simple/TestSimpleRoutingStrategy.java index 358ac8ec..d1a9284f 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/router/simple/TestSimpleRoutingStrategy.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/router/simple/TestSimpleRoutingStrategy.java @@ -60,7 +60,7 @@ public void testInboundHappyPathRegister() throws Exception { assertNotNull(ib); assertTrue(SimpleInboundSide.class.isAssignableFrom(ib.getClass())); - ib.setContainerDetails(new ClusterId("test", "test"), new ContainerAddress(new DummyNodeAddress(), 0), (l, m, i) -> {}); + ib.setContainerDetails(new ClusterId("test", "test"), new ContainerAddress(new DummyNodeAddress(), 0), (l, m) -> {}); ib.start(infra); assertTrue(waitForReg(session)); @@ -75,7 +75,7 @@ public void testInboundResillience() throws Exception { assertNotNull(ib); assertTrue(SimpleInboundSide.class.isAssignableFrom(ib.getClass())); - ib.setContainerDetails(new ClusterId("test", "test"), new ContainerAddress(new DummyNodeAddress(), 0), (l, m, i) -> {}); + ib.setContainerDetails(new ClusterId("test", "test"), new ContainerAddress(new DummyNodeAddress(), 0), (l, m) -> {}); ib.start(infra); assertTrue(waitForReg(session)); @@ -103,7 +103,7 @@ public void testInboundWithOutbound() throws Exception { assertTrue(SimpleInboundSide.class.isAssignableFrom(ib.getClass())); final ClusterId cid = new ClusterId("test", "test"); - ib.setContainerDetails(cid, new ContainerAddress(new DummyNodeAddress("here"), 0), (l, m, i) -> {}); + ib.setContainerDetails(cid, new ContainerAddress(new DummyNodeAddress("here"), 0), (l, m) -> {}); ib.start(infra); assertTrue(waitForReg(session)); @@ -130,7 +130,7 @@ public void testInboundWithOutbound() throws Exception { try (ClusterInfoSession ses3 = sessFact.createSession(); RoutingStrategy.Inbound ib2 = manager.getAssociatedInstance(SimpleRoutingStrategy.class.getPackage().getName())) { - ib2.setContainerDetails(cid, ca, (l, m, i) -> {}); + ib2.setContainerDetails(cid, ca, (l, m) -> {}); ib2.start(makeInfra(ses3, sched)); assertTrue(poll(o -> ob.selectDestinationForMessage(km) != null)); diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/transport/blockingqueue/BlockingQueueTest.java b/dempsy-framework.impl/src/test/java/net/dempsy/transport/blockingqueue/BlockingQueueTest.java index 1eae5ed0..e7e1d597 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/transport/blockingqueue/BlockingQueueTest.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/transport/blockingqueue/BlockingQueueTest.java @@ -28,14 +28,13 @@ import org.junit.Test; +import net.dempsy.Infrastructure; import net.dempsy.threading.DefaultThreadingModel; -import net.dempsy.threading.ThreadingModel; import net.dempsy.transport.MessageTransportException; import net.dempsy.transport.Receiver; import net.dempsy.transport.Sender; import net.dempsy.transport.SenderFactory; import net.dempsy.transport.TransportManager; -import net.dempsy.transport.blockingqueue.BlockingQueueReceiver; import net.dempsy.util.TestInfrastructure; public class BlockingQueueTest { @@ -49,15 +48,15 @@ public class BlockingQueueTest { public void testBlockingQueue() throws Exception { final AtomicReference message = new AtomicReference(null); final ArrayBlockingQueue input = new ArrayBlockingQueue<>(16); - try (final ThreadingModel tm = new DefaultThreadingModel("BQTest-testBlockingQueue-"); - final Receiver r = new BlockingQueueReceiver(input); - final TransportManager tranMan = chain(new TransportManager(), c -> c.start(new TestInfrastructure(null, null))); + try (final Receiver r = new BlockingQueueReceiver(input); + final Infrastructure infra = new TestInfrastructure(new DefaultThreadingModel("BQTest-testBlockingQueue-")); + final TransportManager tranMan = chain(new TransportManager(), c -> c.start(infra)); SenderFactory sf = tranMan.getAssociatedInstance(transportTypeId);) { final Sender sender = sf.getSender(r.getAddress()); r.start((final String msg) -> { message.set(new String(msg)); return true; - }, tm); + }, infra); sender.send("Hello"); assertTrue(poll(o -> "Hello".equals(message.get()))); } @@ -73,9 +72,9 @@ public void testBlockingQueue() throws Exception { public void testBlockingQueueOverflow() throws Throwable { final AtomicReference message = new AtomicReference(null); final ArrayBlockingQueue input = new ArrayBlockingQueue<>(1); - try (final ThreadingModel tm = new DefaultThreadingModel("BQTest-testBlockingQueueOverflow-"); + try (final Infrastructure infra = new TestInfrastructure(new DefaultThreadingModel("BQTest-testBlockingQueueOverflow-")); final Receiver r = new BlockingQueueReceiver(input); - final TransportManager tranMan = chain(new TransportManager(), c -> c.start(new TestInfrastructure(null, null))); + final TransportManager tranMan = chain(new TransportManager(), c -> c.start(infra)); final SenderFactory sf = tranMan.getAssociatedInstance(transportTypeId);) { final Sender sender = sf.getSender(r.getAddress()); @@ -102,7 +101,7 @@ public void testBlockingQueueOverflow() throws Throwable { message.set(new String(msg)); receiveCount.incrementAndGet(); return true; - }, tm); + }, infra); // 2 messages should have been read and the 2nd should be "Hello again" assertTrue(poll(o -> "Hello again".equals(message.get()))); diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/transport/tcp/netty/NettyTransportTest.java b/dempsy-framework.impl/src/test/java/net/dempsy/transport/tcp/TcpTransportTest.java similarity index 50% rename from dempsy-framework.impl/src/test/java/net/dempsy/transport/tcp/netty/NettyTransportTest.java rename to dempsy-framework.impl/src/test/java/net/dempsy/transport/tcp/TcpTransportTest.java index 79c1556c..0f58f40f 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/transport/tcp/netty/NettyTransportTest.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/transport/tcp/TcpTransportTest.java @@ -1,86 +1,113 @@ -package net.dempsy.transport.tcp.netty; +package net.dempsy.transport.tcp; import static net.dempsy.util.Functional.chain; import static net.dempsy.utils.test.ConditionPoll.poll; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.dempsy.Router.RoutedMessage; import net.dempsy.ServiceTracker; import net.dempsy.TestWordCount; import net.dempsy.serialization.Serializer; import net.dempsy.serialization.jackson.JsonSerializer; import net.dempsy.serialization.kryo.KryoSerializer; import net.dempsy.threading.DefaultThreadingModel; +import net.dempsy.threading.ThreadingModel; import net.dempsy.transport.Listener; -import net.dempsy.transport.tcp.TcpAddress; +import net.dempsy.transport.Receiver; +import net.dempsy.transport.RoutedMessage; +import net.dempsy.transport.Sender; +import net.dempsy.transport.SenderFactory; +import net.dempsy.transport.tcp.nio.NioReceiver; +import net.dempsy.transport.tcp.nio.NioSenderFactory; import net.dempsy.util.TestInfrastructure; -public class NettyTransportTest { - private static final Logger LOGGER = LoggerFactory.getLogger(NettyTransportTest.class); +@RunWith(Parameterized.class) +public class TcpTransportTest { + private static final Logger LOGGER = LoggerFactory.getLogger(TcpTransportTest.class); + + private final Supplier senderFactory; + + private final Supplier> receiver; + + public TcpTransportTest(final String senderFactoryName, final Supplier senderFactory, final String receiverName, + final Supplier> receiver) { + this.senderFactory = senderFactory; + this.receiver = receiver; + } + + @Parameters(name = "{index}: senderfactory={0}, receiver={2}") + public static Collection combos() { + final Supplier nior = () -> new NioReceiver<>(new JsonSerializer()); + return Arrays.asList(new Object[][] { + { "nio", (Supplier) () -> new NioSenderFactory(), "nio", nior }, + }); + + } @Test public void testReceiverStart() throws Exception { final AtomicBoolean resolverCalled = new AtomicBoolean(false); try (ServiceTracker tr = new ServiceTracker();) { - final NettyReceiver r = tr.track(new NettyReceiver(new JsonSerializer())) - .setNumHandlers(2) + final AbstractTcpReceiver r = tr.track(receiver.get()) .setResolver(a -> { resolverCalled.set(true); return a; - }).setUseLocalHost(true); + }).setNumHandlers(2) + .setUseLocalHost(true); final TcpAddress addr = r.getAddress(); LOGGER.debug(addr.toString()); - r.start(null, tr.track(new DefaultThreadingModel(NettyTransportTest.class.getSimpleName() + ".testReceiverStart"))); - + r.start(null, tr.track(new TestInfrastructure(new DefaultThreadingModel(TcpTransportTest.class.getSimpleName() + ".testReceiverStart")))); assertTrue(resolverCalled.get()); } } @Test public void testMessage() throws Exception { - final AtomicBoolean resolverCalled = new AtomicBoolean(false); try (ServiceTracker tr = new ServiceTracker();) { - final NettyReceiver r = tr.track(new NettyReceiver(new KryoSerializer())) + final AbstractTcpReceiver r = tr.track(receiver.get()) .setNumHandlers(2) - .setResolver(a -> { - resolverCalled.set(true); - return a; - }).setUseLocalHost(true); + .setUseLocalHost(true); final TcpAddress addr = r.getAddress(); LOGGER.debug(addr.toString()); final AtomicReference rm = new AtomicReference<>(null); + final ThreadingModel tm = tr.track(new DefaultThreadingModel(TcpTransportTest.class.getSimpleName() + ".testMessage")); r.start((Listener) msg -> { rm.set(msg); return true; - }, tr.track(new DefaultThreadingModel(NettyTransportTest.class.getSimpleName() + ".testReceiverStart"))); + }, tr.track(new TestInfrastructure(tm))); - try (final NettySenderFactory sf = new NettySenderFactory();) { - sf.start(new TestInfrastructure(null, null) { + try (final SenderFactory sf = senderFactory.get();) { + sf.start(new TestInfrastructure(tm) { @Override public String getNodeId() { return "test"; } }); - final NettySender sender = sf.getSender(addr); + final Sender sender = sf.getSender(addr); sender.send(new RoutedMessage(new int[] { 0 }, "Hello", "Hello")); - assertTrue(resolverCalled.get()); assertTrue(poll(o -> rm.get() != null)); assertEquals("Hello", rm.get().message); } @@ -89,74 +116,78 @@ public String getNodeId() { @Test public void testLargeMessage() throws Exception { - final AtomicBoolean resolverCalled = new AtomicBoolean(false); final String huge = TestWordCount.readBible(); - try (ServiceTracker tr = new ServiceTracker();) { - final NettyReceiver r = tr.track(new NettyReceiver(new JsonSerializer())) + try (final ServiceTracker tr = new ServiceTracker();) { + final AbstractTcpReceiver r = tr.track(receiver.get()) .setNumHandlers(2) - .setResolver(a -> { - resolverCalled.set(true); - return a; - }).setUseLocalHost(true); + .setUseLocalHost(true) + .setMaxMessageSize(1024 * 1024 * 1024); final TcpAddress addr = r.getAddress(); LOGGER.debug(addr.toString()); final AtomicReference rm = new AtomicReference<>(null); + final ThreadingModel tm = tr.track(new DefaultThreadingModel(TcpTransportTest.class.getSimpleName() + ".testLargeMessage")); + r.start((Listener) msg -> { rm.set(msg); return true; - }, tr.track(new DefaultThreadingModel(NettyTransportTest.class.getSimpleName() + ".testReceiverStart"))); + }, tr.track(new TestInfrastructure(tm))); - try (final NettySenderFactory sf = new NettySenderFactory();) { + try (final SenderFactory sf = senderFactory.get();) { sf.start(new TestInfrastructure(null, null)); - final NettySender sender = sf.getSender(addr); + final Sender sender = sf.getSender(addr); sender.send(new RoutedMessage(new int[] { 0 }, "Hello", huge)); - assertTrue(resolverCalled.get()); assertTrue(poll(o -> rm.get() != null)); assertEquals(huge, rm.get().message); } } } - private void runMultiMessage(final int numThreads, final int numMessagePerThread, final String message, final Serializer serializer) - throws Exception { - final AtomicBoolean resolverCalled = new AtomicBoolean(false); + private static final String NUM_SENDER_THREADS = "2"; + + private void runMultiMessage(final String testName, final int numThreads, final int numMessagePerThread, final String message, + final Serializer serializer) throws Exception { try (final ServiceTracker tr = new ServiceTracker();) { - final NettyReceiver r = tr.track(new NettyReceiver(serializer)) + final AbstractTcpReceiver r = tr.track(receiver.get()) .setNumHandlers(2) - .setResolver(a -> { - resolverCalled.set(true); - return a; - }).setUseLocalHost(true); + .setUseLocalHost(true) + .setMaxMessageSize(1024 * 1024 * 1024); final TcpAddress addr = r.getAddress(); LOGGER.debug(addr.toString()); final AtomicLong msgCount = new AtomicLong(); + final ThreadingModel tm = tr.track(new DefaultThreadingModel(TcpTransportTest.class.getSimpleName() + "." + testName)); r.start((Listener) msg -> { msgCount.incrementAndGet(); return true; - }, tr.track(new DefaultThreadingModel(NettyTransportTest.class.getSimpleName() + ".testReceiverStart"))); + }, tr.track(new TestInfrastructure(tm))); final AtomicBoolean letMeGo = new AtomicBoolean(false); + final CountDownLatch waitToExit = new CountDownLatch(1); + final List threads = IntStream.range(0, numThreads).mapToObj(threadNum -> new Thread(() -> { - try (final NettySenderFactory sf = new NettySenderFactory();) { + try (final SenderFactory sf = senderFactory.get();) { sf.start(new TestInfrastructure(null, null) { - @Override public Map getConfiguration() { final Map ret = new HashMap<>(); - ret.put(NettySenderFactory.class.getPackage().getName() + "." + NettySenderFactory.CONFIG_KEY_SENDER_THREADS, "2"); + ret.put(sf.getClass().getPackage().getName() + "." + NioSenderFactory.CONFIG_KEY_SENDER_THREADS, NUM_SENDER_THREADS); return ret; } - }); - final NettySender sender = sf.getSender(addr); + final Sender sender = sf.getSender(addr); while (!letMeGo.get()) Thread.yield(); for (int i = 0; i < numMessagePerThread; i++) sender.send(new RoutedMessage(new int[] { 0 }, "Hello", message)); + // we need to keep the sender factory going until all messages were accounted for + + try { + waitToExit.await(); + } catch (final InterruptedException ie) {} + } }, "testMultiMessage-Sender-" + threadNum)) .map(th -> chain(th, t -> t.start())) @@ -166,21 +197,36 @@ public Map getConfiguration() { // here's we go. letMeGo.set(true); + // the total number of messages sent should be this count. + assertTrue(poll(new Long((long) numThreads * (long) numMessagePerThread), v -> { + try { + Thread.sleep(1000); + } catch (final InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return v.longValue() == msgCount.get(); + })); + + // let the threads exit + waitToExit.countDown(); + // all threads should eventually exit. assertTrue(poll(threads, o -> o.stream().filter(t -> t.isAlive()).count() == 0)); - // the total number of messages sent should be this count. - assertTrue(poll(new Long((long) numThreads * (long) numMessagePerThread), v -> v.longValue() == msgCount.get())); } } @Test public void testMultiMessage() throws Exception { - runMultiMessage(10, 10000, "Hello", new KryoSerializer()); + runMultiMessage("testMultiMessage", 10, 10000, "Hello", new KryoSerializer()); } + AtomicLong messageNum = new AtomicLong(); + @Test public void testMultiHugeMessage() throws Exception { - runMultiMessage(5, 100, TestWordCount.readBible(), new JsonSerializer()); + runMultiMessage("testMultiHugeMessage", 5, 100, "" + messageNum.incrementAndGet() + TestWordCount.readBible() + messageNum.incrementAndGet(), + new JsonSerializer()); } } diff --git a/dempsy-framework.impl/src/test/java/net/dempsy/util/TestInfrastructure.java b/dempsy-framework.impl/src/test/java/net/dempsy/util/TestInfrastructure.java index 746464c4..6efe9d9e 100644 --- a/dempsy-framework.impl/src/test/java/net/dempsy/util/TestInfrastructure.java +++ b/dempsy-framework.impl/src/test/java/net/dempsy/util/TestInfrastructure.java @@ -10,6 +10,7 @@ import net.dempsy.monitoring.NodeStatsCollector; import net.dempsy.monitoring.basic.BasicNodeStatsCollector; import net.dempsy.monitoring.basic.BasicStatsCollectorFactory; +import net.dempsy.threading.ThreadingModel; import net.dempsy.util.executor.AutoDisposeSingleThreadScheduler; public class TestInfrastructure implements Infrastructure { @@ -18,29 +19,32 @@ public class TestInfrastructure implements Infrastructure { final BasicStatsCollectorFactory statsFact; final BasicNodeStatsCollector nodeStats; final String application; + final ThreadingModel threading; - public TestInfrastructure(final ClusterInfoSession session, final AutoDisposeSingleThreadScheduler sched) { + public TestInfrastructure(final String testName, final ClusterInfoSession session, final AutoDisposeSingleThreadScheduler sched, + final ThreadingModel threading) { this.session = session; this.sched = sched; statsFact = new BasicStatsCollectorFactory(); nodeStats = new BasicNodeStatsCollector(); - application = "application"; + this.application = testName; + this.threading = threading; + } + + public TestInfrastructure(final ClusterInfoSession session, final AutoDisposeSingleThreadScheduler sched) { + this("application", session, sched, null); } public TestInfrastructure(final String testName, final ClusterInfoSession session, final AutoDisposeSingleThreadScheduler sched) { - this.session = session; - this.sched = sched; - statsFact = new BasicStatsCollectorFactory(); - nodeStats = new BasicNodeStatsCollector(); - this.application = testName; + this(testName, session, sched, null); } public TestInfrastructure(final ClusterId cid, final ClusterInfoSession session, final AutoDisposeSingleThreadScheduler sched) { - this.session = session; - this.sched = sched; - statsFact = new BasicStatsCollectorFactory(); - nodeStats = new BasicNodeStatsCollector(); - this.application = cid.applicationName; + this(cid.applicationName, session, sched, null); + } + + public TestInfrastructure(final ThreadingModel threading) { + this("application", null, null, threading); } @Override @@ -77,4 +81,15 @@ public NodeStatsCollector getNodeStatsCollector() { public String getNodeId() { return "test-infrastructure-fake-node-id"; } + + @Override + public ThreadingModel getThreadingModel() { + return threading; + } + + @Override + public void close() { + if (threading != null) + threading.close(); + } } \ No newline at end of file diff --git a/dempsy-framework.impl/src/test/resources/log4j.properties b/dempsy-framework.impl/src/test/resources/log4j.properties index a0006228..26f4ca2c 100755 --- a/dempsy-framework.impl/src/test/resources/log4j.properties +++ b/dempsy-framework.impl/src/test/resources/log4j.properties @@ -15,3 +15,4 @@ log4j.logger.org.apache.zookeeper.jmx.MBeanRegistry=ERROR #log4j.logger.net.dempsy.router.microshard.TestMicroshardingRoutingStrategy=TRACE #log4j.logger.net.dempsy.router.managed.TestManagedRoutingStrategy=TRACE #log4j.logger.net.dempsy.TestElasticity=TRACE +#log4j.logger.net.dempsy.container=TRACE diff --git a/dempsy-framework.impl/src/test/resources/td/node.xml b/dempsy-framework.impl/src/test/resources/td/node.xml index d23a174b..62752f27 100644 --- a/dempsy-framework.impl/src/test/resources/td/node.xml +++ b/dempsy-framework.impl/src/test/resources/td/node.xml @@ -21,6 +21,9 @@ + + + diff --git a/dempsy-framework.impl/src/test/resources/td/transport-netty.xml b/dempsy-framework.impl/src/test/resources/td/transport-nio.xml similarity index 85% rename from dempsy-framework.impl/src/test/resources/td/transport-netty.xml rename to dempsy-framework.impl/src/test/resources/td/transport-nio.xml index 826c75d4..b7f7d60b 100644 --- a/dempsy-framework.impl/src/test/resources/td/transport-netty.xml +++ b/dempsy-framework.impl/src/test/resources/td/transport-nio.xml @@ -7,7 +7,7 @@ - + diff --git a/pom.xml b/pom.xml index a87ac6f4..120166dc 100644 --- a/pom.xml +++ b/pom.xml @@ -9,12 +9,6 @@ pom Distributed Elastic Message Processing - Master Build - - org.sonatype.oss - oss-parent - 7 - - Apache 2 @@ -30,6 +24,7 @@ + 2.1.1 4.2.5.RELEASE 3.2.2 UTF-8 @@ -82,11 +77,12 @@ dempsy-framework.core ${project.version} - + + net.dempsy dempsy-commons-bom - 2.1.1 + ${dempsy-commons-bom.version} pom import @@ -102,11 +98,6 @@ slf4j-api 1.6.4 - - io.netty - netty-all - 4.1.9.Final - org.quartz-scheduler quartz