(issue #29) deterministic tie break asserts

nfsdb-core/src/main/java/com/nfsdb/net/JournalServer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015. Vlad Ilyushchenko
+ * Copyright (c) 2014. Vlad Ilyushchenko
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -56,6 +56,7 @@ public class JournalServer {
     private final AuthorizationHandler authorizationHandler;
     private final JournalServerLogger serverLogger = new JournalServerLogger();
     private final int serverInstance;
+    private final AtomicBoolean ignoreVoting = new AtomicBoolean(false);
     private ServerSocketChannel serverSocketChannel;
 
     public JournalServer(JournalReaderFactory factory) throws JournalNetworkException {
@@ -232,6 +233,15 @@ public int getServerInstance() {
         return serverInstance;
     }
 
+
+    public boolean isIgnoreVoting() {
+        return ignoreVoting.get();
+    }
+
+    public void setIgnoreVoting(boolean ignore) {
+        ignoreVoting.set(ignore);
+    }
+
     private class Acceptor implements Runnable {
         @Override
         public void run() {

nfsdb-core/src/main/java/com/nfsdb/net/JournalServerAgent.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015. Vlad Ilyushchenko
+ * Copyright (c) 2014. Vlad Ilyushchenko
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -94,7 +94,7 @@ public void process(ByteChannel channel) throws JournalNetworkException {
                     intResponseConsumer.read(channel);
                     if (intResponseConsumer.isComplete()) {
                         int inst = intResponseConsumer.getValue();
-                        boolean loss = inst > server.getServerInstance();
+                        boolean loss = !server.isIgnoreVoting() && inst > server.getServerInstance();
                         intResponseConsumer.reset();
                         commandConsumer.reset();
 

nfsdb-core/src/main/java/com/nfsdb/net/cluster/ClusterController.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015. Vlad Ilyushchenko
+ * Copyright (c) 2014. Vlad Ilyushchenko
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -136,15 +136,42 @@ private void up() throws JournalNetworkException {
             return;
         }
 
+        // after this point server cannot be voted out
+        server.setIgnoreVoting(true);
+
         if (client != null) {
             LOGGER.info(thisNode() + " Stopping client remnants");
             client.halt();
             client = null;
         }
+
+        if (activeNode != null) {
+            LOGGER.info("%s is waiting for %s to shutdown", thisNode(), activeNode);
+            waitTillDies(activeNode);
+        }
+
         LOGGER.info(thisNode() + " Activating callback");
         listener.onNodeActive();
     }
 
+    private void waitTillDies(final ClusterNode node) {
+        try {
+            JournalClient client = new JournalClient(new ClientConfig() {{
+                setHostname(node.getAddress());
+                setEnableMulticast(false);
+            }}, factory);
+
+            try {
+                while (client.pingServer()) {
+                    Thread.yield();
+                }
+            } finally {
+                client.halt();
+            }
+        } catch (JournalNetworkException ignore) {
+        }
+    }
+
     private ClusterNode getActiveNode() {
         // ping each cluster node except for current one
         try {

nfsdb-core/src/test/java/com/nfsdb/net/ClusterControllerTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015. Vlad Ilyushchenko
+ * Copyright (c) 2014. Vlad Ilyushchenko
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -78,22 +78,26 @@ public void testTiebreakFailOver() throws Exception {
         controller1.start();
         controller2.start();
 
-        active2Latch.await(10, TimeUnit.SECONDS);
-        Assert.assertEquals("Node 2 is expected to become active", 0, active2Latch.getCount());
+        do {
+            active1Latch.await(1, TimeUnit.MICROSECONDS);
+            active2Latch.await(1, TimeUnit.MICROSECONDS);
+        } while (active1Latch.getCount() > 0 && active2Latch.getCount() > 0);
 
-        // on slow servers controller 1 can go all the way to activation
-        // only to become standby after controller 2 comes online.
-        // however active1Latch becomes set irreversibly.
+        Assert.assertFalse("Two nodes are active simultaneously", active1Latch.getCount() == 0 && active2Latch.getCount() == 0);
 
-//        active1Latch.await(1, TimeUnit.SECONDS);
-//        Assert.assertEquals("Node 1 active() callback should not have been called", 1, active1Latch.getCount());
+        if (active1Latch.getCount() == 0) {
+            standby2Latch.await(200, TimeUnit.MILLISECONDS);
+            Assert.assertEquals("Node 2 is expected to be on standby", 0, standby2Latch.getCount());
 
-        standby1Latch.await(200, TimeUnit.MILLISECONDS);
-        Assert.assertEquals("Node 1 is expected to be on standby", 0, standby1Latch.getCount());
-
-        standby2Latch.await(200, TimeUnit.MILLISECONDS);
-        Assert.assertEquals("Node 2 onNodeStandingBy() is not expected to be called", 0, standby1Latch.getCount());
+            standby1Latch.await(200, TimeUnit.MILLISECONDS);
+            Assert.assertEquals("Node 1 is NOT expected to be on standby", 1, standby1Latch.getCount());
+        } else {
+            standby1Latch.await(200, TimeUnit.MILLISECONDS);
+            Assert.assertEquals("Node 1 is expected to be on standby", 0, standby1Latch.getCount());
 
+            standby2Latch.await(200, TimeUnit.MILLISECONDS);
+            Assert.assertEquals("Node 2 is NOT expected to be on standby", 1, standby2Latch.getCount());
+        }
 
         controller2.halt();