Data loss while reading from DB #1126
Description
Hello. I have an issue. Badger loses data while reading data when app suddenly stops, via Ctrl+C (for example). And this happenes when I read data just through db.View (read only mode).
Go: 1.13.4
Badger: 2.0.0
Windows 10 build 1903
After stop, vlog file becomes too big (2 147 483 646 bytes aka 2 GB). And if i run app again, Badger gives an error
During db.vlog.open: Value log truncate required to run DB. This might result in data loss
And finally, If I run Badger with WithTruncate(true) option, data loses. Vlog file size becomes 20 bytes.
I want to use Badger in my app, but 'cause of this problem i can't. Any error/stop while reading can cause lossing data.
The problem can be reproduced with this code (Ctrl+C while program prints keys and values into console):
read.go
func main() {
db, err := badger.Open(badger.DefaultOptions("tmp/badger").WithSyncWrites(true))
if err != nil {
log.Panic(err)
}
defer db.Close()
err := db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.PrefetchSize = 10
it := txn.NewIterator(opts)
defer it.Close()
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
k := item.Key()
err := item.Value(func(v []byte) error {
fmt.Printf("key=%d, value=%s\n", k, v)
return nil
})
if err != nil {
return err
}
}
return nil
})
if err != nil {
log.Panic(err)
}
}
write.go
func main() {
db, err := badger.Open(badger.DefaultOptions("tmp/badger").WithSyncWrites(true))
if err != nil {
log.Panic(err)
}
defer db.Close()
seq, err := db.GetSequence([]byte("0"), 1)
if err != nil {
log.Panic(err)
}
defer seq.Release()
file, err := os.Open("some_big_data_file.txt")
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
err := db.Update(func(txn *badger.Txn) error {
for scanner.Scan() {
line := []byte(strings.TrimSpace(scanner.Text()))
if len(line) > 0 {
id, err := seq.Next()
if err != nil {
return err
}
key := make([]byte, 8)
binary.BigEndian.PutUint64(key, id)
err = txn.Set(key, line)
if err != nil {
return err
}
}
}
return scanner.Err()
})
if err != nil {
return err
}
}