[FIXED] Clustering: Possible panic leader election

If the leadership is lost while a node was just previously
elected leader, the election will fail but could result in panic.

Resolves #662

Signed-off-by: Ivan Kozlovic <ivan@synadia.com>

server/clustering_test.go

@@ -3885,3 +3885,56 @@ func TestClusteringStartReceiveNext(t *testing.T) {
 		})
 	}
 }
+
+type captureNoLeaderLog struct {
+	dummyLogger
+	gotIt bool
+}
+
+func (l *captureNoLeaderLog) Errorf(format string, args ...interface{}) {
+	l.dummyLogger.Lock()
+	trace := fmt.Sprintf(format, args...)
+	if strings.Contains(trace, raft.ErrNotLeader.Error()) {
+		l.gotIt = true
+	}
+	l.dummyLogger.Unlock()
+}
+
+func TestClusteringNotLeaderWhenLeadershipAcquired(t *testing.T) {
+	cleanupDatastore(t)
+	defer cleanupDatastore(t)
+	cleanupRaftLog(t)
+	defer cleanupRaftLog(t)
+
+	// For this test, use a central NATS server.
+	ns := natsdTest.RunDefaultServer()
+	defer ns.Shutdown()
+
+	// Configure first server
+	s1sOpts := getTestDefaultOptsForClustering("a", true)
+	l := &captureNoLeaderLog{}
+	s1sOpts.CustomLogger = l
+	s1 := runServerWithOpts(t, s1sOpts, nil)
+	defer s1.Shutdown()
+
+	ch1, ch2 := s1.sendSynchronziationRequest()
+	<-ch1
+
+	// Configure second server.
+	s2sOpts := getTestDefaultOptsForClustering("b", false)
+	s2 := runServerWithOpts(t, s2sOpts, nil)
+	s2.Shutdown()
+
+	time.Sleep(time.Second)
+	close(ch2)
+
+	waitFor(t, 2*time.Second, 100*time.Millisecond, func() error {
+		l.Lock()
+		gotIt := l.gotIt
+		l.Unlock()
+		if !gotIt {
+			return fmt.Errorf("Did not get the no leader error")
+		}
+		return nil
+	})
+}

server/server.go

@@ -1806,6 +1806,7 @@ func (s *StanServer) startRaftNode(hasStreamingState bool) error {
 							// Node shutdown, just return.
 							return
 						case err == raft.ErrLeadershipLost:
+						case err == raft.ErrNotLeader:
 							// Node lost leadership, continue loop.
 							continue
 						default: