Use Badger's value log threshold of 1MB (#7415). (#7619)

The write amplification of using value log has been unpredictably high. With this change, we'd use 1MB as ValueThreshold to push as many things as possible into the LSM tree. Previously it was 1KB.

Co-authored-by: Manish R Jain <manish@dgraph.io>

dgraph/cmd/bulk/reduce.go

@@ -132,7 +132,6 @@ func (r *reducer) createBadgerInternal(dir string, compression bool) *badger.DB
 
 	opt := badger.DefaultOptions(dir).
 		WithSyncWrites(false).
-		WithValueThreshold(1 << 20 /* 1 KB */).
 		WithEncryptionKey(key).
 		WithBlockCacheSize(r.opt.BlockCacheSize).
 		WithIndexCacheSize(r.opt.IndexCacheSize)

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/OneOfOne/xxhash v1.2.5 // indirect
 	github.com/blevesearch/bleve v1.0.13
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd
-	github.com/dgraph-io/badger/v3 v3.2011.1
+	github.com/dgraph-io/badger/v3 v3.2011.2-0.20210210142907-44c9230e5a66
 	github.com/dgraph-io/dgo/v200 v200.0.0-20200805103119-a3544c464dd6
 	github.com/dgraph-io/gqlgen v0.13.2
 	github.com/dgraph-io/gqlparser/v2 v2.1.5

go.sum

@@ -116,8 +116,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgraph-io/badger/v3 v3.2011.1 h1:Hmyof0WMEF/QtutX5SQHzIMnJQxb/IrSzhjckV2SD6g=
-github.com/dgraph-io/badger/v3 v3.2011.1/go.mod h1:0rLLrQpKVQAL0or/lBLMQznhr6dWWX7h5AKnmnqx268=
+github.com/dgraph-io/badger/v3 v3.2011.2-0.20210210142907-44c9230e5a66 h1:k2FNYVVH2tKhcPoX8PDc0RUhCYei/+LebiKZEdFt+Ec=
+github.com/dgraph-io/badger/v3 v3.2011.2-0.20210210142907-44c9230e5a66/go.mod h1:0rLLrQpKVQAL0or/lBLMQznhr6dWWX7h5AKnmnqx268=
 github.com/dgraph-io/dgo/v200 v200.0.0-20200805103119-a3544c464dd6 h1:toHzMCdCUgYsjM0cW9+wafnKFXfp1HizIJUyzihN+vk=
 github.com/dgraph-io/dgo/v200 v200.0.0-20200805103119-a3544c464dd6/go.mod h1:rHa+h3kI4M8ASOirxyIyNeXBfHFgeskVUum2OrDMN3U=
 github.com/dgraph-io/gqlgen v0.13.2 h1:TNhndk+eHKj5qE7BenKKSYdSIdOGhLqxR1rCiMso9KM=

worker/file_handler.go

@@ -306,7 +306,6 @@ func (h *fileHandler) ExportBackup(backupDir, exportDir, format string,
 		// file reader and verifying the encryption in the backup file.
 		db, err := badger.OpenManaged(badger.DefaultOptions(dir).
 			WithSyncWrites(false).
-			WithValueThreshold(1 << 10).
 			WithNumVersionsToKeep(math.MaxInt32).
 			WithEncryptionKey(key))
 
@@ -358,7 +357,6 @@ func (h *fileHandler) ExportBackup(backupDir, exportDir, format string,
 			dir := filepath.Join(tmpDir, fmt.Sprintf("p%d", group))
 			db, err := badger.OpenManaged(badger.DefaultOptions(dir).
 				WithSyncWrites(false).
-				WithValueThreshold(1 << 10).
 				WithNumVersionsToKeep(math.MaxInt32).
 				WithEncryptionKey(key))
 

worker/restore.go

@@ -68,7 +68,6 @@ func RunRestore(pdir, location, backupId string, key x.SensitiveByteSlice, ctype
 				WithCompression(ctype).
 				WithZSTDCompressionLevel(clevel).
 				WithSyncWrites(false).
-				WithValueThreshold(1 << 10).
 				WithBlockCacheSize(100 * (1 << 20)).
 				WithIndexCacheSize(100 * (1 << 20)).
 				WithNumVersionsToKeep(math.MaxInt32).

worker/server_state.go

@@ -107,7 +107,6 @@ func (s *ServerState) initStorage() {
 		// for posting lists, so the cost of sync writes is amortized.
 		x.Check(os.MkdirAll(Config.PostingDir, 0700))
 		opt := badger.DefaultOptions(Config.PostingDir).
-			WithValueThreshold(1 << 10 /* 1KB */).
 			WithNumVersionsToKeep(math.MaxInt32).
 			WithBlockCacheSize(Config.PBlockCacheSize).
 			WithIndexCacheSize(Config.PIndexCacheSize)