Add InitialMmapSize to bolt options (hashicorp#13178)

changelog/13178.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+raft: set InitialMmapSize to 100GB on 64bit architectures
+```

physical/raft/bolt_32bit_test.go

@@ -0,0 +1,45 @@
+// +build 386 arm
+
+package raft
+
+import (
+	"os"
+	"strconv"
+	"testing"
+)
+
+func Test_BoltOptions(t *testing.T) {
+	t.Parallel()
+	key := "VAULT_RAFT_INITIAL_MMAP_SIZE"
+
+	testCases := []struct {
+		name         string
+		env          string
+		expectedSize int
+	}{
+		{"none", "", 0},
+		{"5MB", strconv.Itoa(5 * 1024 * 1024), 5 * 1024 * 1024},
+		{"negative", "-1", 0},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.env != "" {
+				current := os.Getenv(key)
+				defer os.Setenv(key, current)
+				os.Setenv(key, tc.env)
+			}
+
+			o, err := boltOptions()
+			if err != nil {
+				t.Error(err)
+			}
+
+			if o.InitialMmapSize != tc.expectedSize {
+				t.Errorf("expected InitialMmapSize to be %d but it was %d", tc.expectedSize, o.InitialMmapSize)
+			}
+		})
+	}
+}

physical/raft/bolt_64bit_test.go

@@ -0,0 +1,42 @@
+// +build !386,!arm
+
+package raft
+
+import (
+	"os"
+	"strconv"
+	"testing"
+)
+
+func Test_BoltOptions(t *testing.T) {
+	t.Parallel()
+	key := "VAULT_RAFT_INITIAL_MMAP_SIZE"
+
+	testCases := []struct {
+		name         string
+		env          string
+		expectedSize int
+	}{
+		{"none", "", 100 * 1024 * 1024 * 1024},
+		{"5MB", strconv.Itoa(5 * 1024 * 1024), 5 * 1024 * 1024},
+		{"negative", "-1", 0},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.env != "" {
+				current := os.Getenv(key)
+				defer os.Setenv(key, current)
+				os.Setenv(key, tc.env)
+			}
+
+			o := boltOptions()
+
+			if o.InitialMmapSize != tc.expectedSize {
+				t.Errorf("expected InitialMmapSize to be %d but it was %d", tc.expectedSize, o.InitialMmapSize)
+			}
+		})
+	}
+}

physical/raft/fsm.go

@@ -168,13 +168,9 @@ func (f *FSM) openDBFile(dbPath string) error {
 		}
 	}
 
-	freelistType, noFreelistSync := freelistOptions()
+	opts := boltOptions()
 	start := time.Now()
-	boltDB, err := bolt.Open(dbPath, 0o600, &bolt.Options{
-		Timeout:        1 * time.Second,
-		FreelistType:   freelistType,
-		NoFreelistSync: noFreelistSync,
-	})
+	boltDB, err := bolt.Open(dbPath, 0o600, opts)
 	if err != nil {
 		return err
 	}

physical/raft/raft.go

@@ -364,13 +364,10 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		}
 
 		// Create the backend raft store for logs and stable storage.
-		freelistType, noFreelistSync := freelistOptions()
+		opts := boltOptions()
 		raftOptions := raftboltdb.Options{
-			Path: filepath.Join(path, "raft.db"),
-			BoltOptions: &bolt.Options{
-				FreelistType:   freelistType,
-				NoFreelistSync: noFreelistSync,
-			},
+			Path:        filepath.Join(path, "raft.db"),
+			BoltOptions: opts,
 		}
 		store, err := raftboltdb.New(raftOptions)
 		if err != nil {
@@ -1644,20 +1641,39 @@ func (s sealer) Open(ctx context.Context, ct []byte) ([]byte, error) {
 	return s.access.Decrypt(ctx, &eblob, nil)
 }
 
-// freelistOptions returns the freelist type and nofreelistsync values to use
-// when opening boltdb files, based on our preferred defaults, and the possible
-// presence of overriding environment variables.
-func freelistOptions() (bolt.FreelistType, bool) {
-	freelistType := bolt.FreelistMapType
-	noFreelistSync := true
+// boltOptions returns a bolt.Options struct, suitable for passing to
+// bolt.Open(), pre-configured with all of our preferred defaults.
+func boltOptions() *bolt.Options {
+	o := &bolt.Options{
+		Timeout:        1 * time.Second,
+		FreelistType:   bolt.FreelistMapType,
+		NoFreelistSync: true,
+	}
 
 	if os.Getenv("VAULT_RAFT_FREELIST_TYPE") == "array" {
-		freelistType = bolt.FreelistArrayType
+		o.FreelistType = bolt.FreelistArrayType
 	}
 
 	if os.Getenv("VAULT_RAFT_FREELIST_SYNC") != "" {
-		noFreelistSync = false
+		o.NoFreelistSync = false
+	}
+
+	// By default, we want to set InitialMmapSize to 100GB, but only on 64bit platforms.
+	// Otherwise, we set it to whatever the value of VAULT_RAFT_INITIAL_MMAP_SIZE
+	// is, assuming it can be parsed as an int. Bolt itself sets this to 0 by default,
+	// so if users are wanting to turn this off, they can also set it to 0. Setting it
+	// to a negative value is the same as not setting it at all.
+	if os.Getenv("VAULT_RAFT_INITIAL_MMAP_SIZE") == "" {
+		o.InitialMmapSize = initialMmapSize
+	} else {
+		imms, err := strconv.Atoi(os.Getenv("VAULT_RAFT_INITIAL_MMAP_SIZE"))
+
+		// If there's an error here, it means they passed something that's not convertible to
+		// a number. Rather than fail startup, just ignore it.
+		if err == nil && imms > 0 {
+			o.InitialMmapSize = imms
+		}
 	}
 
-	return freelistType, noFreelistSync
+	return o
 }

physical/raft/vars_32bit.go

@@ -0,0 +1,5 @@
+// +build 386 arm
+
+package raft
+
+const initialMmapSize = 0

physical/raft/vars_64bit.go

@@ -0,0 +1,5 @@
+// +build !386,!arm
+
+package raft
+
+const initialMmapSize = 100 * 1024 * 1024 * 1024 // 100GB