db: clean up level options

The `Options.Level` field has questionable semantics. It's a slice of
arbitrary length and if not all levels are specified, the settings for
missing levels are derived at runtime based on the last specified
level (with target file size doubling for each level). But if we add
a level to the slice, that level gets the L0 defaults (including the
target file size).

This commits cleans this up: the field is now an array of size
`NumLevels` and the defaults are always calculated in the same way:
hardcoded defaults for L0, and for all other levels we take the
options from the level above and double the file size.

Author: RaduBerinde

cmd/pebble/db.go

@@ -65,7 +65,6 @@ func newPebbleDB(dir string) DB {
 		L0CompactionThreshold:       2,
 		L0StopWritesThreshold:       1000,
 		LBaseMaxBytes:               64 << 20, // 64 MB
-		Levels:                      make([]pebble.LevelOptions, 7),
 		MaxOpenFiles:                16384,
 		MemTableSize:                64 << 20,
 		MemTableStopWritesThreshold: 4,
@@ -100,10 +99,6 @@ func newPebbleDB(dir string) DB {
 		l.IndexBlockSize = 256 << 10 // 256 KB
 		l.FilterPolicy = bloom.FilterPolicy(10)
 		l.FilterType = pebble.TableFilter
-		if i > 0 {
-			l.TargetFileSize = opts.Levels[i-1].TargetFileSize * 2
-		}
-		l.EnsureDefaults()
 	}
 	opts.Levels[6].FilterPolicy = pebble.NoFilterPolicy
 	opts.FlushSplitBytes = opts.Levels[0].TargetFileSize

cmd/pebble/replay_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/stretchr/testify/require"
 )
 
@@ -48,18 +49,18 @@ func TestParseOptionsStr(t *testing.T) {
 		},
 		{
 			c: replayConfig{optionsString: `[Options] [Level "0"] target_file_size=222`},
-			options: &pebble.Options{Levels: []pebble.LevelOptions{
-				{TargetFileSize: 222},
+			options: &pebble.Options{Levels: [manifest.NumLevels]pebble.LevelOptions{
+				0: {TargetFileSize: 222},
 			}},
 		},
 		{
 			c: replayConfig{optionsString: `[Options] lbase_max_bytes=10  max_open_files=20  [Level "0"] target_file_size=30 [Level "1"] index_block_size=40`},
 			options: &pebble.Options{
 				LBaseMaxBytes: 10,
 				MaxOpenFiles:  20,
-				Levels: []pebble.LevelOptions{
-					{TargetFileSize: 30},
-					{IndexBlockSize: 40},
+				Levels: [manifest.NumLevels]pebble.LevelOptions{
+					0: {TargetFileSize: 30},
+					1: {IndexBlockSize: 40},
 				},
 			},
 		},

compaction.go

@@ -47,7 +47,7 @@ var gcLabels = pprof.Labels("pebble", "gc")
 // compacted files. We avoid expanding the lower level file set of a compaction
 // if it would make the total compaction cover more than this many bytes.
 func expandedCompactionByteSizeLimit(opts *Options, level int, availBytes uint64) uint64 {
-	v := uint64(25 * opts.Level(level).TargetFileSize)
+	v := uint64(25 * opts.Levels[level].TargetFileSize)
 
 	// Never expand a compaction beyond half the available capacity, divided
 	// by the maximum number of concurrent compactions. Each of the concurrent
@@ -69,13 +69,13 @@ func expandedCompactionByteSizeLimit(opts *Options, level int, availBytes uint64
 // maxGrandparentOverlapBytes is the maximum bytes of overlap with level+1
 // before we stop building a single file in a level-1 to level compaction.
 func maxGrandparentOverlapBytes(opts *Options, level int) uint64 {
-	return uint64(10 * opts.Level(level).TargetFileSize)
+	return uint64(10 * opts.Levels[level].TargetFileSize)
 }
 
 // maxReadCompactionBytes is used to prevent read compactions which
 // are too wide.
 func maxReadCompactionBytes(opts *Options, level int) uint64 {
-	return uint64(10 * opts.Level(level).TargetFileSize)
+	return uint64(10 * opts.Levels[level].TargetFileSize)
 }
 
 // noCloseIter wraps around a FragmentIterator, intercepting and eliding
@@ -712,7 +712,7 @@ func newFlush(
 	}
 
 	if opts.FlushSplitBytes > 0 {
-		c.maxOutputFileSize = uint64(opts.Level(0).TargetFileSize)
+		c.maxOutputFileSize = uint64(opts.Levels[0].TargetFileSize)
 		c.maxOverlapBytes = maxGrandparentOverlapBytes(opts, 0)
 		c.grandparents = c.version.Overlaps(baseLevel, c.userKeyBounds())
 		adjustGrandparentOverlapBytesForFlush(c, flushingBytes)

compaction_picker.go

@@ -246,7 +246,7 @@ func newPickedCompaction(
 		l0Organizer:            l0Organizer,
 		baseLevel:              baseLevel,
 		inputs:                 []compactionLevel{{level: startLevel}, {level: outputLevel}},
-		maxOutputFileSize:      uint64(opts.Level(adjustedLevel).TargetFileSize),
+		maxOutputFileSize:      uint64(opts.Levels[adjustedLevel].TargetFileSize),
 		maxOverlapBytes:        maxGrandparentOverlapBytes(opts, adjustedLevel),
 		maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedLevel),
 	}
@@ -1877,7 +1877,7 @@ func pickL0(
 	// counterproductive.
 	lcf = l0Organizer.PickIntraL0Compaction(env.earliestUnflushedSeqNum, minIntraL0Count, env.problemSpans)
 	if lcf != nil {
-		pc := newPickedCompactionFromL0(lcf, opts, vers, l0Organizer, 0, false)
+		pc := newPickedCompactionFromL0(lcf, opts, vers, l0Organizer, baseLevel, false)
 		if pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel, env.problemSpans) {
 			if pc.startLevel.files.Empty() {
 				opts.Logger.Fatalf("empty compaction chosen")

compaction_test.go

@@ -2212,8 +2212,7 @@ func TestCompactionErrorCleanup(t *testing.T) {
 	mem := vfs.NewMem()
 	ii := errorfs.OnIndex(math.MaxInt32) // start disabled
 	opts := &Options{
-		FS:     errorfs.Wrap(mem, errorfs.ErrInjected.If(ii)),
-		Levels: make([]LevelOptions, numLevels),
+		FS: errorfs.Wrap(mem, errorfs.ErrInjected.If(ii)),
 		EventListener: &EventListener{
 			TableCreated: func(info TableCreateInfo) {
 				t.Log(info)
@@ -2918,8 +2917,7 @@ func TestCompactionErrorStats(t *testing.T) {
 	mem := vfs.NewMem()
 	injector := &WriteErrorInjector{}
 	opts := &Options{
-		FS:     errorfs.Wrap(mem, injector),
-		Levels: make([]LevelOptions, numLevels),
+		FS: errorfs.Wrap(mem, injector),
 		EventListener: &EventListener{
 			TableCreated: func(info TableCreateInfo) {
 				t.Log(info)

data_test.go

@@ -1767,8 +1767,8 @@ func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error {
 			}
 			opts.Experimental.SpanPolicyFunc = MakeStaticSpanPolicyFunc(opts.Comparer.Compare, span, policy)
 		case "target-file-sizes":
-			if len(opts.Levels) < len(cmdArg.Vals) {
-				opts.Levels = slices.Grow(opts.Levels, len(cmdArg.Vals)-len(opts.Levels))[0:len(cmdArg.Vals)]
+			if len(cmdArg.Vals) > len(opts.Levels) {
+				return errors.New("too many target-file-sizes")
 			}
 			for i := range cmdArg.Vals {
 				size, err := strconv.ParseInt(cmdArg.Vals[i], 10, 64)
@@ -1777,6 +1777,12 @@ func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error {
 				}
 				opts.Levels[i].TargetFileSize = size
 			}
+			// Set the remaining file sizes. Normally, EnsureDefaults() would do that
+			// for us but it was already called and the target file sizes for all
+			// levels are now set to the defaults.
+			for i := len(cmdArg.Vals); i < len(opts.Levels); i++ {
+				opts.Levels[i].TargetFileSize = opts.Levels[i-1].TargetFileSize * 2
+			}
 		case "value-separation":
 			if len(cmdArg.Vals) != 3 {
 				return errors.New("value-separation-policy expects 3 arguments: (enabled, minimum-size, max-blob-reference-depth)")

excise_test.go

@@ -80,7 +80,8 @@ func TestExcise(t *testing.T) {
 			Logger:             testLogger{t},
 		}
 		if blockSize != 0 {
-			opts.Levels = append(opts.Levels, LevelOptions{BlockSize: blockSize, IndexBlockSize: 32 << 10})
+			opts.Levels[0].BlockSize = blockSize
+			opts.Levels[0].IndexBlockSize = 32 << 10
 		}
 		opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
 			"external-locator": remoteStorage,

external_test.go

@@ -188,7 +188,6 @@ func buildSeparatedValuesDB(
 		FS:                      vfs.NewMem(),
 		KeySchema:               cockroachkvs.KeySchema.Name,
 		KeySchemas:              sstable.MakeKeySchemas(&cockroachkvs.KeySchema),
-		Levels:                  make([]pebble.LevelOptions, 7),
 		MemTableSize:            2 << 20,
 		L0CompactionThreshold:   2,
 	}
@@ -199,17 +198,10 @@ func buildSeparatedValuesDB(
 			MaxBlobReferenceDepth: 10,
 		}
 	}
-	for i := 0; i < len(o.Levels); i++ {
-		l := &o.Levels[i]
-		l.BlockSize = 32 << 10       // 32 KB
-		l.IndexBlockSize = 256 << 10 // 256 KB
-		l.FilterPolicy = bloom.FilterPolicy(10)
-		l.FilterType = pebble.TableFilter
-		if i > 0 {
-			l.TargetFileSize = o.Levels[i-1].TargetFileSize * 2
-		}
-		l.EnsureDefaults()
-	}
+	o.Levels[0].BlockSize = 32 << 10       // 32 KB
+	o.Levels[0].IndexBlockSize = 256 << 10 // 256 KB
+	o.Levels[0].FilterPolicy = bloom.FilterPolicy(10)
+	o.Levels[0].FilterType = pebble.TableFilter
 	db, err := pebble.Open("", o)
 	require.NoError(tb, err)
 

iterator_test.go

@@ -969,8 +969,7 @@ func TestIteratorBlockIntervalFilter(t *testing.T) {
 			FormatMajorVersion:      internalFormatNewest,
 			BlockPropertyCollectors: bpCollectors,
 		}
-		lo := LevelOptions{BlockSize: 1, IndexBlockSize: 1}
-		opts.Levels = append(opts.Levels, lo)
+		opts.Levels[0] = LevelOptions{BlockSize: 1, IndexBlockSize: 1}
 
 		// Automatic compactions may compact away tombstones from L6, making
 		// some testcases non-deterministic.
@@ -1101,10 +1100,8 @@ func TestIteratorRandomizedBlockIntervalFilter(t *testing.T) {
 	opts.L0CompactionFileThreshold = 1 << rng.IntN(11) // 1-1024
 	opts.LBaseMaxBytes = 1 << rng.IntN(11)             // 1B - 1KB
 	opts.MemTableSize = 2 << 10                        // 2KB
-	var lopts LevelOptions
-	lopts.BlockSize = 1 << rng.IntN(8)      // 1B - 256B
-	lopts.IndexBlockSize = 1 << rng.IntN(8) // 1B - 256B
-	opts.Levels = []LevelOptions{lopts}
+	opts.Levels[0].BlockSize = 1 << rng.IntN(8)        // 1B - 256B
+	opts.Levels[0].IndexBlockSize = 1 << rng.IntN(8)   // 1B - 256B
 
 	d, err := Open("", opts)
 	require.NoError(t, err)
@@ -2138,7 +2135,7 @@ func TestRangeKeyMaskingRandomized(t *testing.T) {
 			sstable.NewTestKeysBlockPropertyCollector,
 		},
 	}
-	opts1.Levels = baseOpts.levelOpts
+	copy(opts1.Levels[:], baseOpts.levelOpts)
 	d1, err := Open("", opts1)
 	require.NoError(t, err)
 
@@ -2151,7 +2148,7 @@ func TestRangeKeyMaskingRandomized(t *testing.T) {
 			sstable.NewTestKeysBlockPropertyCollector,
 		},
 	}
-	opts2.Levels = randomOpts.levelOpts
+	copy(opts2.Levels[:], randomOpts.levelOpts)
 	d2, err := Open("", opts2)
 	require.NoError(t, err)
 

metamorphic/options.go

@@ -326,16 +326,14 @@ func defaultOptions(kf KeyFormat) *pebble.Options {
 		// Use an archive cleaner to ease post-mortem debugging.
 		Cleaner: base.ArchiveCleaner{},
 		// Always use our custom comparer which provides a Split method.
-		Comparer:           kf.Comparer,
-		KeySchema:          kf.KeySchema.Name,
-		KeySchemas:         sstable.MakeKeySchemas(kf.KeySchema),
-		FS:                 vfs.NewMem(),
-		FormatMajorVersion: defaultFormatMajorVersion,
-		Levels: []pebble.LevelOptions{{
-			FilterPolicy: bloom.FilterPolicy(10),
-		}},
+		Comparer:                kf.Comparer,
+		KeySchema:               kf.KeySchema.Name,
+		KeySchemas:              sstable.MakeKeySchemas(kf.KeySchema),
+		FS:                      vfs.NewMem(),
+		FormatMajorVersion:      defaultFormatMajorVersion,
 		BlockPropertyCollectors: kf.BlockPropertyCollectors,
 	}
+	opts.Levels[0].FilterPolicy = bloom.FilterPolicy(10)
 	opts.Experimental.IngestSplit = func() bool { return false }
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 
@@ -808,7 +806,7 @@ func RandomOptions(
 	default:
 		lopts.Compression = func() block.Compression { return pebble.SnappyCompression }
 	}
-	opts.Levels = []pebble.LevelOptions{lopts}
+	opts.Levels[0] = lopts
 
 	// Explicitly disable disk-backed FS's for the random configurations. The
 	// single standard test configuration that uses a disk-backed FS is