From c3a44924b601f326d05ac19ea9a6026b261e54c7 Mon Sep 17 00:00:00 2001
From: Jason Volk <jason@zemos.net>
Date: Fri, 18 Aug 2023 13:00:06 -0700
Subject: [PATCH] Add filesystem locking on jetstream fileStore instances.

Running two instances of the server which share the same directory (e.g.
default configuration `/tmp/nats/jetstream') will corrupt each other.

We mitigate by creating an empty file called LOCK in the directory and then
acquire a posix record lock on it; a common pattern.

Signed-off-by: Jason Volk <jason@zemos.net>
---
 server/filestore.go         | 17 +++++++++++++++
 server/filestore_posix.go   | 39 ++++++++++++++++++++++++++++++++++
 server/filestore_windows.go | 42 +++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+)
 create mode 100644 server/filestore_posix.go
 create mode 100644 server/filestore_windows.go

diff --git a/server/filestore.go b/server/filestore.go
index 7836db016a5..87d461846d9 100644
--- a/server/filestore.go
+++ b/server/filestore.go
@@ -174,6 +174,7 @@ type fileStore struct {
 	hh          hash.Hash64
 	qch         chan struct{}
 	cfs         []ConsumerStore
+	lfd         *os.File
 	sips        int
 	closed      bool
 	fip         bool
@@ -387,6 +388,10 @@ func newFileStoreWithCreated(fcfg FileStoreConfig, cfg StreamConfig, created tim
 		qch:    make(chan struct{}),
 	}
 
+	if err := fs.lockFileSystem(); err != nil {
+		return nil, err
+	}
+
 	// Set flush in place to AsyncFlush which by default is false.
 	fs.fip = !fcfg.AsyncFlush
 
@@ -7015,6 +7020,18 @@ func (fs *fileStore) SyncDeleted(dbs DeleteBlocks) {
 	}
 }
 
+func (fs *fileStore) lockFileSystem() error {
+	var err error
+	lpath := filepath.Join(fs.fcfg.StoreDir, "LOCK")
+	if fs.lfd, err = os.Create(lpath); err != nil {
+		return fmt.Errorf("could not create `%s': %v", lpath, err)
+	}
+	if err = lockFile(fs.lfd); err != nil {
+		return err
+	}
+	return nil
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Consumers
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/server/filestore_posix.go b/server/filestore_posix.go
new file mode 100644
index 00000000000..3f4334acd42
--- /dev/null
+++ b/server/filestore_posix.go
@@ -0,0 +1,39 @@
+// Copyright 2023 The NATS Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !windows
+// +build !windows
+
+package server
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"syscall"
+)
+
+// Acquire filesystem-level lock for the filestore to ensure exclusive access
+// for this server instance.
+func lockFile(f *os.File) error {
+	err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &syscall.Flock_t{
+		Type:   syscall.F_WRLCK,
+		Whence: io.SeekStart,
+		Start:  0,
+		Len:    0,
+	})
+	if err != nil {
+		return fmt.Errorf("lock `%s': %v", f.Name(), err)
+	}
+	return nil
+}
diff --git a/server/filestore_windows.go b/server/filestore_windows.go
new file mode 100644
index 00000000000..4aef381ce1d
--- /dev/null
+++ b/server/filestore_windows.go
@@ -0,0 +1,42 @@
+// Copyright 2023 The NATS Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+// +build windows
+
+package server
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+// Acquire filesystem-level lock for the filestore to ensure exclusive access
+// for this server instance.
+func lockFile(f *os.File) error {
+	var (
+		modkernel32  = syscall.NewLazyDLL("kernel32.dll")
+		prLockFileEx = modkernel32.NewProc("LockFileEx")
+		ol           = unsafe.Pointer(new(syscall.Overlapped))
+	)
+
+	a := prLockFileEx.Addr()
+	h := syscall.Handle(f.Fd())
+	r, _, e := syscall.Syscall6(a, 6, uintptr(h), 0x3, 0, ^uintptr(0), ^uintptr(0), uintptr(ol))
+	if r == 0 {
+		return fmt.Errorf("lock `%s': %v", f.Name(), error(e))
+	}
+	return nil
+}