diff --git a/server/consumer.go b/server/consumer.go index a5fd72a29e..b3b8b34beb 100644 --- a/server/consumer.go +++ b/server/consumer.go @@ -1564,7 +1564,7 @@ func (o *consumer) deleteNotActive() { // Don't think this needs to be a monitored go routine. go func() { const ( - startInterval = 5 * time.Second + startInterval = 30 * time.Second maxInterval = 5 * time.Minute ) jitter := time.Duration(rand.Int63n(int64(startInterval))) diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index 0a8a65bca8..e5f4bbe68a 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -3472,13 +3472,13 @@ func TestJetStreamClusterNoR1AssetsDuringLameDuck(t *testing.T) { // Make sure we do not have any R1 assets placed on the lameduck server. for s.isRunning() { - s.rnMu.RLock() + s.mu.RLock() if s.js == nil || s.js.srv == nil || s.js.srv.gacc == nil { - s.rnMu.RUnlock() + s.mu.RUnlock() break } hasAsset := len(s.js.srv.gacc.streams()) > 0 - s.rnMu.RUnlock() + s.mu.RUnlock() if hasAsset { t.Fatalf("Server had an R1 asset when it should not due to lameduck mode") } diff --git a/server/raft.go b/server/raft.go index 076b8e0df9..f7a0535372 100644 --- a/server/raft.go +++ b/server/raft.go @@ -588,14 +588,15 @@ func (s *Server) stepdownRaftNodes() { s.Debugf("Stepping down all leader raft nodes") } for _, n := range s.raftNodes { - if n.Leader() { - nodes = append(nodes, n) - } + nodes = append(nodes, n) } s.rnMu.RUnlock() for _, node := range nodes { - node.StepDown() + if node.Leader() { + node.StepDown() + } + node.SetObserver(true) } } diff --git a/server/server.go b/server/server.go index 5664e77c0e..f6880f3f29 100644 --- a/server/server.go +++ b/server/server.go @@ -2380,7 +2380,7 @@ func (s *Server) Shutdown() { accRes.Close() } - // Now check jetstream. + // Now check and shutdown jetstream. s.shutdownJetStream() // Now shutdown the nodes @@ -3944,7 +3944,8 @@ func (s *Server) isLameDuckMode() bool { } // This function will close the client listener then close the clients -// at some interval to avoid a reconnecting storm. +// at some interval to avoid a reconnect storm. +// We will also transfer any raft leaders and shutdown JetStream. func (s *Server) lameDuckMode() { s.mu.Lock() // Check if there is actually anything to do @@ -3985,6 +3986,12 @@ func (s *Server) lameDuckMode() { } } + // Now check and shutdown jetstream. + s.shutdownJetStream() + + // Now shutdown the nodes + s.shutdownRaftNodes() + // Wait for accept loops to be done to make sure that no new // client can connect for i := 0; i < expected; i++ {