Add reload lock to ensure new connections await completion of config …

…reload (#5207)

This may help with some problems during configuration reload with e.g.
OCSP stapling.

Signed-off-by: Neil Twigg <neil@nats.io>

locksordering.txt

@@ -16,3 +16,9 @@ possible with the normal account lock.
 accountLeafList -> client
 
 AccountResolver interface has various implementations, but assume: AccountResolver -> Server
+
+A reloadMu lock was added to prevent newly connecting clients racing with the configuration reload.
+This must be taken out as soon as a reload is about to happen before any other locks:
+
+    reloadMu -> Server
+    reloadMu -> optsMu

server/reload.go

@@ -999,6 +999,9 @@ func (s *Server) Reload() error {
 // type. This returns an error if an option which doesn't support
 // hot-swapping was changed.
 func (s *Server) ReloadOptions(newOpts *Options) error {
+	s.reloadMu.Lock()
+	defer s.reloadMu.Unlock()
+
 	s.mu.Lock()
 
 	curOpts := s.getOpts()

server/server.go

@@ -167,6 +167,7 @@ type Server struct {
 	stats
 	scStats
 	mu                  sync.RWMutex
+	reloadMu            sync.RWMutex // Write-locked when a config reload is taking place ONLY
 	kp                  nkeys.KeyPair
 	xkp                 nkeys.KeyPair
 	xpub                string
@@ -2727,7 +2728,9 @@ func (s *Server) acceptConnections(l net.Listener, acceptName string, createFunc
 		}
 		tmpDelay = ACCEPT_MIN_SLEEP
 		if !s.startGoRoutine(func() {
+			s.reloadMu.RLock()
 			createFunc(conn)
+			s.reloadMu.RUnlock()
 			s.grWG.Done()
 		}) {
 			conn.Close()