Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix db.grow is unusable when NoFreelistSync is on #387

Merged
merged 1 commit into from
Jan 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion db.go
Original file line number Diff line number Diff line change
Expand Up @@ -1120,7 +1120,7 @@ func (db *DB) grow(sz int) error {

// If the data is smaller than the alloc size then only allocate what's needed.
// Once it goes over the allocation size then allocate in chunks.
if db.datasz < db.AllocSize {
if db.datasz <= db.AllocSize {
sz = db.datasz
} else {
sz += db.AllocSize
Expand Down
18 changes: 10 additions & 8 deletions tx.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@ func (tx *Tx) Commit() error {
tx.stats.IncRebalanceTime(time.Since(startTime))
}

opgid := tx.meta.pgid

// spill data onto dirty pages.
startTime = time.Now()
if err := tx.root.spill(); err != nil {
Expand All @@ -181,6 +183,14 @@ func (tx *Tx) Commit() error {
tx.meta.freelist = pgidNoFreelist
}

// If the high water mark has moved up then attempt to grow the database.
if tx.meta.pgid > opgid {
if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
tx.rollback()
return err
}
}

// Write dirty pages to disk.
startTime = time.Now()
if err := tx.write(); err != nil {
Expand Down Expand Up @@ -225,7 +235,6 @@ func (tx *Tx) Commit() error {
func (tx *Tx) commitFreelist() error {
// Allocate new pages for the new free list. This will overestimate
// the size of the freelist but not underestimate the size (which would be bad).
opgid := tx.meta.pgid
p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
if err != nil {
tx.rollback()
Expand All @@ -236,13 +245,6 @@ func (tx *Tx) commitFreelist() error {
return err
}
tx.meta.freelist = p.id
// If the high water mark has moved up then attempt to grow the database.
if tx.meta.pgid > opgid {
if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
tx.rollback()
return err
}
}

return nil
}
Expand Down
44 changes: 44 additions & 0 deletions tx_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ import (
"fmt"
"log"
"os"
"runtime"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

bolt "go.etcd.io/bbolt"
"go.etcd.io/bbolt/internal/btesting"
Expand Down Expand Up @@ -1009,3 +1011,45 @@ func TestTxStats_Sub(t *testing.T) {
assert.Equal(t, int64(10001), diff.GetWrite())
assert.Equal(t, 10009*time.Second, diff.GetWriteTime())
}

// TestTx_TruncateBeforeWrite ensures the file is truncated ahead whether we sync freelist or not.
func TestTx_TruncateBeforeWrite(t *testing.T) {
if runtime.GOOS == "windows" {
return
}
for _, isSyncFreelist := range []bool{false, true} {
t.Run(fmt.Sprintf("isSyncFreelist:%v", isSyncFreelist), func(t *testing.T) {
// Open the database.
db := btesting.MustCreateDBWithOption(t, &bolt.Options{
NoFreelistSync: isSyncFreelist,
})

bigvalue := make([]byte, db.AllocSize/100)
count := 0
for {
count++
tx, err := db.Begin(true)
require.NoError(t, err)
b, err := tx.CreateBucketIfNotExists([]byte("bucket"))
require.NoError(t, err)
err = b.Put([]byte{byte(count)}, bigvalue)
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)

size := fileSize(db.Path())

if size > int64(db.AllocSize) && size < int64(db.AllocSize)*2 {
// db.grow expands the file aggresively, that double the size while smaller than db.AllocSize,
// or increase with a step of db.AllocSize if larger, by which we can test if db.grow has run.
t.Fatalf("db.grow doesn't run when file size changes. file size: %d", size)
}
if size > int64(db.AllocSize) {
break
}
}
db.MustClose()
db.MustDeleteFile()
})
}
}