Skip to content

Commit

Permalink
Fix mmap resize calculation.
Browse files Browse the repository at this point in the history
This commit fixes an issue where the database would grow whenever it was opened. This was caused by
a recent change that performed a truncation when the database grew. Now there are fixed growth sizes
for the database (1MB, 2MB, 4MB, 8MB, etc) up to 1GB and then the database will grow by 1GB when it
resizes.

See also: 6bb2585
  • Loading branch information
benbjohnson committed Jan 28, 2015
1 parent 00c6357 commit 834b38e
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 9 deletions.
21 changes: 12 additions & 9 deletions db.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ import (
"unsafe"
)

// The smallest size that the mmap can be.
const minMmapSize = 1 << 22 // 4MB

// The largest step that can be taken when remapping the mmap.
const maxMmapStep = 1 << 30 // 1GB

Expand Down Expand Up @@ -222,15 +219,21 @@ func (db *DB) munmap() error {
// mmapSize determines the appropriate size for the mmap given the current size
// of the database. The minimum size is 4MB and doubles until it reaches 1GB.
func (db *DB) mmapSize(size int) int {
if size <= minMmapSize {
return minMmapSize
} else if size < maxMmapStep {
size *= 2
} else {
size += maxMmapStep
// Double the size from 1MB until 1GB.

This comment has been minimized.

Copy link
@tv42

tv42 Jan 28, 2015

The loop actually stops at 512MB and not 1GB. This is one of the reasons I don't like too much trickery with 1<<n..
http://play.golang.org/p/wN4OGcd9Ur

for i := uint(20); i < 30; i++ {
if size <= 1<<i {
return 1 << i
}
}

// If larger than 1GB then grow by 1GB at a time.
size += maxMmapStep

This comment has been minimized.

Copy link
@tv42

tv42 Jan 28, 2015

I think this will wraparound on 32-bit quite easily. The maxMapSize check is after the call to mmapSize.

if remainder := size % maxMmapStep; remainder > 0 {
size -= remainder
}

// Ensure that the mmap size is a multiple of the page size.
// This should always be true since we're incrementing in MBs.
if (size % db.pageSize) != 0 {
size = ((size / db.pageSize) + 1) * db.pageSize
}
Expand Down
41 changes: 41 additions & 0 deletions db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,39 @@ func TestOpen_Wait(t *testing.T) {
assert(t, time.Since(start) > 100*time.Millisecond, "")
}

// Ensure that opening a database does not increase its size.
// https://github.com/boltdb/bolt/issues/291
func TestOpen_Size(t *testing.T) {
// Open a data file.
db := NewTestDB()
path := db.Path()
defer db.Close()

// Insert until we get above the minimum 4MB size.
db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists([]byte("data"))
for i := 0; i < 10000; i++ {
_ = b.Put([]byte(fmt.Sprintf("%04d", i)), make([]byte, 1000))
}
return nil
})

// Close database and grab the size.
db.DB.Close()
sz := fileSize(path)

// Reopen database, update, and check size again.
db0, _ := bolt.Open(path, 0666, nil)
db0.Update(func(tx *bolt.Tx) error { return tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}) })
db0.Close()
newSz := fileSize(path)

// Compare the original size with the new size.
if sz != newSz {
t.Fatalf("unexpected file growth: %d => %d", sz, newSz)
}
}

// Ensure that a re-opened database is consistent.
func TestOpen_Check(t *testing.T) {
path := tempfile()
Expand Down Expand Up @@ -648,3 +681,11 @@ func trunc(b []byte, length int) []byte {
func truncDuration(d time.Duration) string {
return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1")
}

func fileSize(path string) int64 {
fi, err := os.Stat(path)
if err != nil {
return 0

This comment has been minimized.

Copy link
@tv42

tv42 Jan 28, 2015

I am not thrilled by dropping errors. If you accidentally get the path wrong in the caller, TestOpen_Size will always succeed because 0 == 0.

}
return fi.Size()
}

0 comments on commit 834b38e

Please sign in to comment.