forked from boltdb/bolt
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This commit fixes an issue where the database would grow whenever it was opened. This was caused by a recent change that performed a truncation when the database grew. Now there are fixed growth sizes for the database (1MB, 2MB, 4MB, 8MB, etc) up to 1GB and then the database will grow by 1GB when it resizes. See also: 6bb2585
- Loading branch information
1 parent
00c6357
commit 834b38e
Showing
2 changed files
with
53 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,9 +12,6 @@ import ( | |
"unsafe" | ||
) | ||
|
||
// The smallest size that the mmap can be. | ||
const minMmapSize = 1 << 22 // 4MB | ||
|
||
// The largest step that can be taken when remapping the mmap. | ||
const maxMmapStep = 1 << 30 // 1GB | ||
|
||
|
@@ -222,15 +219,21 @@ func (db *DB) munmap() error { | |
// mmapSize determines the appropriate size for the mmap given the current size | ||
// of the database. The minimum size is 4MB and doubles until it reaches 1GB. | ||
func (db *DB) mmapSize(size int) int { | ||
if size <= minMmapSize { | ||
return minMmapSize | ||
} else if size < maxMmapStep { | ||
size *= 2 | ||
} else { | ||
size += maxMmapStep | ||
// Double the size from 1MB until 1GB. | ||
This comment has been minimized.
Sorry, something went wrong. |
||
for i := uint(20); i < 30; i++ { | ||
if size <= 1<<i { | ||
return 1 << i | ||
} | ||
} | ||
|
||
// If larger than 1GB then grow by 1GB at a time. | ||
size += maxMmapStep | ||
This comment has been minimized.
Sorry, something went wrong.
tv42
|
||
if remainder := size % maxMmapStep; remainder > 0 { | ||
size -= remainder | ||
} | ||
|
||
// Ensure that the mmap size is a multiple of the page size. | ||
// This should always be true since we're incrementing in MBs. | ||
if (size % db.pageSize) != 0 { | ||
size = ((size / db.pageSize) + 1) * db.pageSize | ||
} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -85,6 +85,39 @@ func TestOpen_Wait(t *testing.T) { | |
assert(t, time.Since(start) > 100*time.Millisecond, "") | ||
} | ||
|
||
// Ensure that opening a database does not increase its size. | ||
// https://github.com/boltdb/bolt/issues/291 | ||
func TestOpen_Size(t *testing.T) { | ||
// Open a data file. | ||
db := NewTestDB() | ||
path := db.Path() | ||
defer db.Close() | ||
|
||
// Insert until we get above the minimum 4MB size. | ||
db.Update(func(tx *bolt.Tx) error { | ||
b, _ := tx.CreateBucketIfNotExists([]byte("data")) | ||
for i := 0; i < 10000; i++ { | ||
_ = b.Put([]byte(fmt.Sprintf("%04d", i)), make([]byte, 1000)) | ||
} | ||
return nil | ||
}) | ||
|
||
// Close database and grab the size. | ||
db.DB.Close() | ||
sz := fileSize(path) | ||
|
||
// Reopen database, update, and check size again. | ||
db0, _ := bolt.Open(path, 0666, nil) | ||
db0.Update(func(tx *bolt.Tx) error { return tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}) }) | ||
db0.Close() | ||
newSz := fileSize(path) | ||
|
||
// Compare the original size with the new size. | ||
if sz != newSz { | ||
t.Fatalf("unexpected file growth: %d => %d", sz, newSz) | ||
} | ||
} | ||
|
||
// Ensure that a re-opened database is consistent. | ||
func TestOpen_Check(t *testing.T) { | ||
path := tempfile() | ||
|
@@ -648,3 +681,11 @@ func trunc(b []byte, length int) []byte { | |
func truncDuration(d time.Duration) string { | ||
return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") | ||
} | ||
|
||
func fileSize(path string) int64 { | ||
fi, err := os.Stat(path) | ||
if err != nil { | ||
return 0 | ||
This comment has been minimized.
Sorry, something went wrong.
tv42
|
||
} | ||
return fi.Size() | ||
} |
The loop actually stops at 512MB and not 1GB. This is one of the reasons I don't like too much trickery with 1<<n..
http://play.golang.org/p/wN4OGcd9Ur