-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Max entries per value log file: To ensure that we can GC a file in bounded time, we should restrict the max number of key-value pairs that can be contained by one log file. A new option now enforces that, taking the minimum of value log file size, and the number of entries contained to determine when a file should be rotated. New move keyspace: We should never write an entry with an older timestamp for the same key. We need to maintain this invariant to search for the latest value of a key, or else we need to search in all tables and find the max version among them. - Value log GC moves key-value pairs to new files. This used to cause older versions of keys to be written to the top of LSM tree. This change puts those moves into a separate keyspace, prefixed by badgerMove. When doing a value read, we can look into this keyspace if the previous value pointer is no longer valid (pointed to a deleted value log file). - Add an integration test, which runs for 3 mins and ensures that GC moves work as expected. - Switch db.get back to stopping as soon as we encounter the first key (instead of looking into all levels). - Collect value log GC stats when doing compactions. - Trace log the entire value log GC, so we know how the file was picked and the result of sampling. - Pick two value log files for GC. First from discard stats, and second randomly. Then try to GC both, and return on whichever is successful. - Add a function to delete move keys corresponding to a value log file deletion. - Increase the value threshold to 32 by default.
- Loading branch information
1 parent
1640484
commit 7af0076
Showing
9 changed files
with
464 additions
and
117 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| /testgc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "encoding/binary" | ||
| "fmt" | ||
| "log" | ||
| "math/rand" | ||
| "net/http" | ||
| _ "net/http/pprof" | ||
| "os" | ||
| "sync" | ||
| "sync/atomic" | ||
| "time" | ||
|
|
||
| "github.com/dgraph-io/badger" | ||
| "github.com/dgraph-io/badger/options" | ||
| "github.com/dgraph-io/badger/y" | ||
| ) | ||
|
|
||
| var Max int64 = 10000000 | ||
| var suffix = make([]byte, 128) | ||
|
|
||
| type S struct { | ||
| sync.Mutex | ||
| vals map[uint64]uint64 | ||
|
|
||
| count uint64 // Not under mutex lock. | ||
| } | ||
|
|
||
| func encoded(i uint64) []byte { | ||
| out := make([]byte, 8) | ||
| binary.BigEndian.PutUint64(out, i) | ||
| return out | ||
| } | ||
|
|
||
| func (s *S) write(db *badger.DB) error { | ||
| return db.Update(func(txn *badger.Txn) error { | ||
| for i := 0; i < 10; i++ { | ||
| // These keys would be overwritten. | ||
| keyi := uint64(rand.Int63n(Max)) | ||
| key := encoded(keyi) | ||
| vali := atomic.AddUint64(&s.count, 1) | ||
| val := encoded(vali) | ||
| val = append(val, suffix...) | ||
| if err := txn.Set(key, val); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| for i := 0; i < 20; i++ { | ||
| // These keys would be new and never overwritten. | ||
| keyi := atomic.AddUint64(&s.count, 1) | ||
| if keyi%1000000 == 0 { | ||
| log.Printf("Count: %d\n", keyi) | ||
| } | ||
| key := encoded(keyi) | ||
| val := append(key, suffix...) | ||
| if err := txn.Set(key, val); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| }) | ||
| } | ||
|
|
||
| func (s *S) read(db *badger.DB) error { | ||
| max := int64(atomic.LoadUint64(&s.count)) | ||
| keyi := uint64(rand.Int63n(max)) | ||
| key := encoded(keyi) | ||
|
|
||
| err := db.View(func(txn *badger.Txn) error { | ||
| item, err := txn.Get(key) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| val, err := item.Value() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| y.AssertTruef(len(val) == len(suffix)+8, "Found val of len: %d\n", len(val)) | ||
| vali := binary.BigEndian.Uint64(val[0:8]) | ||
| s.Lock() | ||
| expected := s.vals[keyi] | ||
| if vali < expected { | ||
| log.Fatalf("Expected: %d. Found: %d. Key: %d\n", expected, vali, keyi) | ||
| } else if vali == expected { | ||
| // pass | ||
| } else { | ||
| s.vals[keyi] = vali | ||
| } | ||
| s.Unlock() | ||
| return nil | ||
| }) | ||
| if err == badger.ErrKeyNotFound { | ||
| return nil | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| func main() { | ||
| fmt.Println("Badger Integration test for value log GC.") | ||
|
|
||
| // dir, err := ioutil.TempDir("./", "badger") | ||
| // if err != nil { | ||
| // log.Fatal(err) | ||
| // } | ||
| dir := "/mnt/drive/badgertest" | ||
| os.RemoveAll(dir) | ||
| // defer os.RemoveAll(dir) | ||
|
|
||
| opts := badger.DefaultOptions | ||
| opts.Dir = dir | ||
| opts.ValueDir = dir | ||
| opts.TableLoadingMode = options.MemoryMap | ||
| opts.ValueLogLoadingMode = options.FileIO | ||
| // opts.ValueLogFileSize = 64 << 20 // 64 MB. | ||
| opts.SyncWrites = false | ||
|
|
||
| db, err := badger.Open(opts) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| defer db.Close() | ||
|
|
||
| go http.ListenAndServe("localhost:8080", nil) | ||
|
|
||
| closer := y.NewCloser(11) | ||
| go func() { | ||
| // Run value log GC. | ||
| defer closer.Done() | ||
| var count int | ||
| ticker := time.NewTicker(30 * time.Second) | ||
| defer ticker.Stop() | ||
| for range ticker.C { | ||
| select { | ||
| case <-closer.HasBeenClosed(): | ||
| log.Printf("Num times value log GC was successful: %d\n", count) | ||
| return | ||
| default: | ||
| } | ||
| log.Printf("Starting a value log GC") | ||
| err := db.RunValueLogGC(0.1) | ||
| log.Printf("Result of value log GC: %v\n", err) | ||
| if err == nil { | ||
| count++ | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| s := S{ | ||
| count: uint64(Max), | ||
| vals: make(map[uint64]uint64), | ||
| } | ||
| var numLoops uint64 | ||
| ticker := time.NewTicker(5 * time.Second) | ||
| for i := 0; i < 10; i++ { | ||
| go func() { | ||
| defer closer.Done() | ||
| for { | ||
| if err := s.write(db); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| for j := 0; j < 10; j++ { | ||
| if err := s.read(db); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } | ||
| nl := atomic.AddUint64(&numLoops, 1) | ||
| select { | ||
| case <-closer.HasBeenClosed(): | ||
| return | ||
| case <-ticker.C: | ||
| log.Printf("Num loops: %d\n", nl) | ||
| default: | ||
| } | ||
| } | ||
| }() | ||
| } | ||
| time.Sleep(5 * time.Minute) | ||
| log.Println("Signaling...") | ||
| closer.SignalAndWait() | ||
| log.Println("Wait done. Now iterating over everything.") | ||
|
|
||
| err = db.View(func(txn *badger.Txn) error { | ||
| iopts := badger.DefaultIteratorOptions | ||
| itr := txn.NewIterator(iopts) | ||
| defer itr.Close() | ||
|
|
||
| var total, tested int | ||
| for itr.Rewind(); itr.Valid(); itr.Next() { | ||
| item := itr.Item() | ||
| key := item.Key() | ||
| keyi := binary.BigEndian.Uint64(key) | ||
| total++ | ||
|
|
||
| val, err := item.Value() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if len(val) < 8 { | ||
| log.Printf("Unexpected value: %x\n", val) | ||
| continue | ||
| } | ||
| vali := binary.BigEndian.Uint64(val[0:8]) | ||
|
|
||
| expected, ok := s.vals[keyi] // Not all keys must be in vals map. | ||
| if ok { | ||
| tested++ | ||
| if vali < expected { | ||
| // vali must be equal or greater than what's in the map. | ||
| log.Fatalf("Expected: %d. Got: %d. Key: %d\n", expected, vali, keyi) | ||
| } | ||
| } | ||
| } | ||
| log.Printf("Total iterated: %d. Tested values: %d\n", total, tested) | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| log.Fatalf("Error while iterating: %v", err) | ||
| } | ||
| log.Println("Iteration done. Test successful.") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.