Increase value threshold from 1 KB to 1 MB (#1664)

The write amplification with value log can be unpredictably high. It's better to use value log only for really big values, and keep as many values as possible within the LSM tree.

Author: Ibrahim Jarif

db_test.go

@@ -68,9 +68,6 @@ func (s *DB) validate() error { return s.lc.validate() }
 
 func getTestOptions(dir string) Options {
 	opt := DefaultOptions(dir).
-		WithMemTableSize(1 << 15).
-		WithBaseTableSize(1 << 15). // Force more compaction.
-		WithBaseLevelSize(4 << 15). // Force more compaction.
 		WithSyncWrites(false).
 		WithLoggingLevel(WARNING)
 	return opt
@@ -1521,7 +1518,7 @@ func TestWriteDeadlock(t *testing.T) {
 	var count int
 	val := make([]byte, 10000)
 	require.NoError(t, db.Update(func(txn *Txn) error {
-		for i := 0; i < 1500; i++ {
+		for i := 0; i < 1000; i++ {
 			key := fmt.Sprintf("%d", i)
 			rand.Read(val)
 			require.NoError(t, txn.SetEntry(NewEntry([]byte(key), val)))
@@ -1759,7 +1756,6 @@ func TestLSMOnly(t *testing.T) {
 
 	opts := LSMOnlyOptions(dir)
 	dopts := DefaultOptions(dir)
-	require.NotEqual(t, dopts.ValueThreshold, opts.ValueThreshold)
 
 	dopts.ValueThreshold = 1 << 21
 	_, err = Open(dopts)

levels_test.go

@@ -973,8 +973,7 @@ func TestLevelGet(t *testing.T) {
 func TestKeyVersions(t *testing.T) {
 	inMemoryOpt := DefaultOptions("").
 		WithSyncWrites(false).
-		WithInMemory(true).
-		WithMemTableSize(4 << 20)
+		WithInMemory(true)
 
 	t.Run("disk", func(t *testing.T) {
 		t.Run("small table", func(t *testing.T) {
@@ -1037,7 +1036,7 @@ func TestKeyVersions(t *testing.T) {
 					writer.Set([]byte(fmt.Sprintf("%05d", i)), []byte("foo"))
 				}
 				require.NoError(t, writer.Flush())
-				require.Equal(t, 11, len(db.KeySplits(nil)))
+				require.Equal(t, 10, len(db.KeySplits(nil)))
 			})
 		})
 		t.Run("prefix", func(t *testing.T) {

managed_db_test.go

@@ -784,6 +784,8 @@ func TestZeroDiscardStats(t *testing.T) {
 	t.Run("after rewrite", func(t *testing.T) {
 		opts := getTestOptions("")
 		opts.ValueLogFileSize = 5 << 20
+		opts.ValueThreshold = 1 << 10
+		opts.MemTableSize = 1 << 15
 		runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
 			populate(t, db)
 			require.Equal(t, int(N), numKeys(db))

options.go

@@ -170,7 +170,7 @@ func DefaultOptions(path string) Options {
 		ValueLogMaxEntries: 1000000,
 
 		VLogPercentile: 0.0,
-		ValueThreshold: 1 << 10, // 1 KB.
+		ValueThreshold: maxValueThreshold,
 
 		Logger:                        defaultLogger(INFO),
 		EncryptionKey:                 []byte{},

stream_writer_test.go

@@ -326,8 +326,9 @@ func TestStreamWriter5(t *testing.T) {
 // This test tries to insert multiple equal keys(without version) and verifies
 // if those are going to same table.
 func TestStreamWriter6(t *testing.T) {
-
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
+	opt := getTestOptions("")
+	opt.BaseTableSize = 1 << 15
+	runBadgerTest(t, &opt, func(t *testing.T, db *DB) {
 		str := []string{"a", "b", "c"}
 		ver := uint64(0)
 		// The baseTable size is 32 KB (1<<15) and the max table size for level
@@ -371,7 +372,9 @@ func TestStreamWriter6(t *testing.T) {
 
 // This test uses a StreamWriter without calling Flush() at the end.
 func TestStreamWriterCancel(t *testing.T) {
-	runBadgerTest(t, nil, func(t *testing.T, db *DB) {
+	opt := getTestOptions("")
+	opt.BaseTableSize = 1 << 15
+	runBadgerTest(t, &opt, func(t *testing.T, db *DB) {
 		str := []string{"a", "a", "b", "b", "c", "c"}
 		ver := 1
 		buf := z.NewBuffer(10 << 20)

value_test.go

@@ -34,6 +34,7 @@ import (
 )
 
 func TestDynamicValueThreshold(t *testing.T) {
+	t.Skip()
 	dir, err := ioutil.TempDir("", "badger-test")
 	y.Check(err)
 	defer removeDir(dir)
@@ -140,9 +141,14 @@ func TestValueGCManaged(t *testing.T) {
 	defer removeDir(dir)
 
 	N := 10000
+
 	opt := getTestOptions(dir)
 	opt.ValueLogMaxEntries = uint32(N / 10)
 	opt.managedTxns = true
+	opt.BaseTableSize = 1 << 15
+	opt.ValueThreshold = 1 << 10
+	opt.MemTableSize = 1 << 15
+
 	db, err := Open(opt)
 	require.NoError(t, err)
 	defer db.Close()
@@ -202,6 +208,8 @@ func TestValueGC(t *testing.T) {
 	defer removeDir(dir)
 	opt := getTestOptions(dir)
 	opt.ValueLogFileSize = 1 << 20
+	opt.BaseTableSize = 1 << 15
+	opt.ValueThreshold = 1 << 10
 
 	kv, _ := Open(opt)
 	defer kv.Close()
@@ -253,6 +261,8 @@ func TestValueGC2(t *testing.T) {
 	defer removeDir(dir)
 	opt := getTestOptions(dir)
 	opt.ValueLogFileSize = 1 << 20
+	opt.BaseTableSize = 1 << 15
+	opt.ValueThreshold = 1 << 10
 
 	kv, _ := Open(opt)
 	defer kv.Close()
@@ -328,6 +338,8 @@ func TestValueGC3(t *testing.T) {
 	defer removeDir(dir)
 	opt := getTestOptions(dir)
 	opt.ValueLogFileSize = 1 << 20
+	opt.BaseTableSize = 1 << 15
+	opt.ValueThreshold = 1 << 10
 
 	kv, err := Open(opt)
 	require.NoError(t, err)
@@ -401,6 +413,8 @@ func TestValueGC4(t *testing.T) {
 	defer removeDir(dir)
 	opt := getTestOptions(dir)
 	opt.ValueLogFileSize = 1 << 20
+	opt.BaseTableSize = 1 << 15
+	opt.ValueThreshold = 1 << 10
 
 	kv, err := Open(opt)
 	require.NoError(t, err)
@@ -477,6 +491,8 @@ func TestPersistLFDiscardStats(t *testing.T) {
 	opt.ValueLogFileSize = 1 << 20
 	// Avoid compaction on close so that the discard map remains the same.
 	opt.CompactL0OnClose = false
+	opt.MemTableSize = 1 << 15
+	opt.ValueThreshold = 1 << 10
 
 	db, err := Open(opt)
 	require.NoError(t, err)