diff --git a/changelog/13178.txt b/changelog/13178.txt new file mode 100644 index 0000000000000..10bb3b890daee --- /dev/null +++ b/changelog/13178.txt @@ -0,0 +1,3 @@ +```release-note:improvement +raft: set InitialMmapSize to 100GB on 64bit architectures +``` diff --git a/physical/raft/bolt_32bit_test.go b/physical/raft/bolt_32bit_test.go new file mode 100644 index 0000000000000..57ab4cee96bb6 --- /dev/null +++ b/physical/raft/bolt_32bit_test.go @@ -0,0 +1,45 @@ +// +build 386 arm + +package raft + +import ( + "os" + "strconv" + "testing" +) + +func Test_BoltOptions(t *testing.T) { + t.Parallel() + key := "VAULT_RAFT_INITIAL_MMAP_SIZE" + + testCases := []struct { + name string + env string + expectedSize int + }{ + {"none", "", 0}, + {"5MB", strconv.Itoa(5 * 1024 * 1024), 5 * 1024 * 1024}, + {"negative", "-1", 0}, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + if tc.env != "" { + current := os.Getenv(key) + defer os.Setenv(key, current) + os.Setenv(key, tc.env) + } + + o, err := boltOptions() + if err != nil { + t.Error(err) + } + + if o.InitialMmapSize != tc.expectedSize { + t.Errorf("expected InitialMmapSize to be %d but it was %d", tc.expectedSize, o.InitialMmapSize) + } + }) + } +} diff --git a/physical/raft/bolt_64bit_test.go b/physical/raft/bolt_64bit_test.go new file mode 100644 index 0000000000000..20ec4ea43f0bc --- /dev/null +++ b/physical/raft/bolt_64bit_test.go @@ -0,0 +1,42 @@ +// +build !386,!arm + +package raft + +import ( + "os" + "strconv" + "testing" +) + +func Test_BoltOptions(t *testing.T) { + t.Parallel() + key := "VAULT_RAFT_INITIAL_MMAP_SIZE" + + testCases := []struct { + name string + env string + expectedSize int + }{ + {"none", "", 100 * 1024 * 1024 * 1024}, + {"5MB", strconv.Itoa(5 * 1024 * 1024), 5 * 1024 * 1024}, + {"negative", "-1", 0}, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + if tc.env != "" { + current := os.Getenv(key) + defer os.Setenv(key, current) + os.Setenv(key, tc.env) + } + + o := boltOptions() + + if o.InitialMmapSize != tc.expectedSize { + t.Errorf("expected InitialMmapSize to be %d but it was %d", tc.expectedSize, o.InitialMmapSize) + } + }) + } +} diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index 66b2b4b9c5637..8c2e79fbde156 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -168,13 +168,9 @@ func (f *FSM) openDBFile(dbPath string) error { } } - freelistType, noFreelistSync := freelistOptions() + opts := boltOptions() start := time.Now() - boltDB, err := bolt.Open(dbPath, 0o600, &bolt.Options{ - Timeout: 1 * time.Second, - FreelistType: freelistType, - NoFreelistSync: noFreelistSync, - }) + boltDB, err := bolt.Open(dbPath, 0o600, opts) if err != nil { return err } diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 47b96d7c0dcbe..8c59fd07ae9c6 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -364,13 +364,10 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend } // Create the backend raft store for logs and stable storage. - freelistType, noFreelistSync := freelistOptions() + opts := boltOptions() raftOptions := raftboltdb.Options{ - Path: filepath.Join(path, "raft.db"), - BoltOptions: &bolt.Options{ - FreelistType: freelistType, - NoFreelistSync: noFreelistSync, - }, + Path: filepath.Join(path, "raft.db"), + BoltOptions: opts, } store, err := raftboltdb.New(raftOptions) if err != nil { @@ -1644,20 +1641,39 @@ func (s sealer) Open(ctx context.Context, ct []byte) ([]byte, error) { return s.access.Decrypt(ctx, &eblob, nil) } -// freelistOptions returns the freelist type and nofreelistsync values to use -// when opening boltdb files, based on our preferred defaults, and the possible -// presence of overriding environment variables. -func freelistOptions() (bolt.FreelistType, bool) { - freelistType := bolt.FreelistMapType - noFreelistSync := true +// boltOptions returns a bolt.Options struct, suitable for passing to +// bolt.Open(), pre-configured with all of our preferred defaults. +func boltOptions() *bolt.Options { + o := &bolt.Options{ + Timeout: 1 * time.Second, + FreelistType: bolt.FreelistMapType, + NoFreelistSync: true, + } if os.Getenv("VAULT_RAFT_FREELIST_TYPE") == "array" { - freelistType = bolt.FreelistArrayType + o.FreelistType = bolt.FreelistArrayType } if os.Getenv("VAULT_RAFT_FREELIST_SYNC") != "" { - noFreelistSync = false + o.NoFreelistSync = false + } + + // By default, we want to set InitialMmapSize to 100GB, but only on 64bit platforms. + // Otherwise, we set it to whatever the value of VAULT_RAFT_INITIAL_MMAP_SIZE + // is, assuming it can be parsed as an int. Bolt itself sets this to 0 by default, + // so if users are wanting to turn this off, they can also set it to 0. Setting it + // to a negative value is the same as not setting it at all. + if os.Getenv("VAULT_RAFT_INITIAL_MMAP_SIZE") == "" { + o.InitialMmapSize = initialMmapSize + } else { + imms, err := strconv.Atoi(os.Getenv("VAULT_RAFT_INITIAL_MMAP_SIZE")) + + // If there's an error here, it means they passed something that's not convertible to + // a number. Rather than fail startup, just ignore it. + if err == nil && imms > 0 { + o.InitialMmapSize = imms + } } - return freelistType, noFreelistSync + return o } diff --git a/physical/raft/vars_32bit.go b/physical/raft/vars_32bit.go new file mode 100644 index 0000000000000..27deb58873242 --- /dev/null +++ b/physical/raft/vars_32bit.go @@ -0,0 +1,5 @@ +// +build 386 arm + +package raft + +const initialMmapSize = 0 diff --git a/physical/raft/vars_64bit.go b/physical/raft/vars_64bit.go new file mode 100644 index 0000000000000..1a1716cd3000d --- /dev/null +++ b/physical/raft/vars_64bit.go @@ -0,0 +1,5 @@ +// +build !386,!arm + +package raft + +const initialMmapSize = 100 * 1024 * 1024 * 1024 // 100GB