Fix dual masters race condition.

After a successful failover and the backup is active, a dual masters
exception could be thrown if:

1) a client is trying to connect to the primary server
2) the primary server is restarted before the client times out

The primary server would take the client connection as an event to
turn into a master, unaware that the backup was still active.

This patch adds a timeout - the primary server will not respond to
clients until two heartbeats have passed, allowing time for the backup
to notify the primary if it's active.

examples/Python/bstar.py

@@ -69,8 +69,13 @@ def __init__(self, primary, local, remote):
         self.statesub.on_recv(self.recv_state)
 
         # setup log formmater
-
+
+    def update_peer_expiry(self):
+        """Update peer expiry time to be 2 heartbeats from now."""
+        self.peer_expiry = time.time() + 2e-3 * HEARTBEAT
+
     def start(self):
+        self.update_peer_expiry()
         self.heartbeat.start()
         return self.loop.start()
 
@@ -94,10 +99,16 @@ def execute_fsm(self):
                 if (self.slave_callback):
                     self.loop.add_callback(self.slave_callback)
             elif (self.event == CLIENT_REQUEST):
-                print ("I: request from client, ready as master")
-                self.state = STATE_ACTIVE
-                if (self.master_callback):
-                    self.loop.add_callback(self.master_callback)
+                if (time.time () >= self.peer_expiry):
+                    print ("I: request from client, ready as master")
+                    self.state = STATE_ACTIVE
+                    if (self.master_callback):
+                        self.loop.add_callback(self.master_callback)
+                else:
+                    # don't respond to clients yet - we don't know if
+                    # the backup is currently Active as a result of
+                    # a successful failover
+                    accept = False
         elif (self.state == STATE_BACKUP):
             # Backup server is waiting for peer to connect
             # Rejects CLIENT_REQUEST events in this state
@@ -160,7 +171,7 @@ def recv_state (self, msg):
         state = msg[0]
         if state:
             self.event = int(state)
-            self.peer_expiry = time.time() + 2e-3 * HEARTBEAT
+            self.update_peer_expiry()
         self.execute_fsm()
 
     def voter_ready(self, msg):