Merge pull request #102 from benbjohnson/fix-freelist

Write freelist after each commit.

cmd/bolt/pages.go

@@ -39,7 +39,10 @@ func Pages(path string) {
 				overflow = strconv.Itoa(p.OverflowCount)
 			}
 			printf("%-8d %-10s %-6d %-6s\n", p.ID, p.Type, p.Count, overflow)
-			id += 1 + p.OverflowCount
+			id += 1
+			if p.Type != "free" {
+				id += p.OverflowCount
+			}
 		}
 		return nil
 	})

db.go

@@ -271,6 +271,10 @@ func (db *DB) close() error {
 	db.freelist = nil
 	db.path = ""
 
+	// Clear ops.
+	db.ops.writeAt = nil
+	db.ops.metaWriteAt = nil
+
 	// Close the mmap.
 	if err := db.munmap(); err != nil {
 		return err
@@ -533,8 +537,12 @@ func (db *DB) Check() error {
 		reachable := make(map[pgid]*page)
 		reachable[0] = tx.page(0) // meta0
 		reachable[1] = tx.page(1) // meta1
-		reachable[tx.meta.buckets] = tx.page(tx.meta.buckets)
-		reachable[tx.meta.freelist] = tx.page(tx.meta.freelist)
+		for i := uint32(0); i <= tx.page(tx.meta.buckets).overflow; i++ {
+			reachable[tx.meta.buckets+pgid(i)] = tx.page(tx.meta.buckets)
+		}
+		for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
+			reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
+		}
 
 		// Check each reachable page within each bucket.
 		for _, bucket := range tx.Buckets() {

db_test.go

@@ -53,6 +53,18 @@ func TestDBReopen(t *testing.T) {
 	})
 }
 
+// Ensure that a re-opened database is consistent.
+func TestOpenCheck(t *testing.T) {
+	withDB(func(db *DB, path string) {
+		assert.NoError(t, db.Open(path, 0666))
+		assert.NoError(t, db.Check())
+		db.Close()
+
+		assert.NoError(t, db.Open(path, 0666))
+		assert.NoError(t, db.Check())
+	})
+}
+
 // Ensure that the database returns an error if the file handle cannot be open.
 func TestDBOpenFileError(t *testing.T) {
 	withDB(func(db *DB, path string) {
@@ -246,8 +258,8 @@ func TestDBStat(t *testing.T) {
 		// Obtain stats.
 		stat, err := db.Stat()
 		assert.NoError(t, err)
-		assert.Equal(t, 126, stat.PageCount)
-		assert.Equal(t, 3, stat.FreePageCount)
+		assert.Equal(t, 127, stat.PageCount)
+		assert.Equal(t, 4, stat.FreePageCount)
 		assert.Equal(t, 4096, stat.PageSize)
 		assert.Equal(t, 4194304, stat.MmapSize)
 		assert.Equal(t, 2, stat.TxCount)
@@ -305,21 +317,24 @@ func TestDBConsistency(t *testing.T) {
 				assert.Equal(t, "meta", p.Type)
 			}
 			if p, _ := tx.Page(2); assert.NotNil(t, p) {
-				assert.Equal(t, "freelist", p.Type)
+				assert.Equal(t, "free", p.Type)
 			}
 			if p, _ := tx.Page(3); assert.NotNil(t, p) {
 				assert.Equal(t, "free", p.Type)
 			}
 			if p, _ := tx.Page(4); assert.NotNil(t, p) {
-				assert.Equal(t, "buckets", p.Type)
+				assert.Equal(t, "freelist", p.Type)
 			}
 			if p, _ := tx.Page(5); assert.NotNil(t, p) {
-				assert.Equal(t, "leaf", p.Type)
+				assert.Equal(t, "buckets", p.Type)
 			}
 			if p, _ := tx.Page(6); assert.NotNil(t, p) {
+				assert.Equal(t, "leaf", p.Type)
+			}
+			if p, _ := tx.Page(7); assert.NotNil(t, p) {
 				assert.Equal(t, "free", p.Type)
 			}
-			p, _ := tx.Page(7)
+			p, _ := tx.Page(8)
 			assert.Nil(t, p)
 			return nil
 		})

freelist.go

@@ -12,6 +12,11 @@ type freelist struct {
 	pending map[txid][]pgid
 }
 
+// size returns the size of the page after serialization.
+func (f *freelist) size() int {
+	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * len(f.all()))
+}
+
 // all returns a list of all free ids and all pending ids in one sorted list.
 func (f *freelist) all() []pgid {
 	ids := make([]pgid, len(f.ids))

tx.go

@@ -199,6 +199,16 @@ func (t *Tx) Commit() error {
 	t.db.freelist.free(t.id(), t.page(t.meta.buckets))
 	t.meta.buckets = p.id
 
+	// Free the freelist and allocate new pages for it. This will overestimate
+	// the size of the freelist but not underestimate the size (which would be bad).
+	t.db.freelist.free(t.id(), t.page(t.meta.freelist))
+	p, err = t.allocate((t.db.freelist.size() / t.db.pageSize) + 1)
+	if err != nil {
+		return err
+	}
+	t.db.freelist.write(p)
+	t.meta.freelist = p.id
+
 	// Write dirty pages to disk.
 	if err := t.write(); err != nil {
 		return err

tx_test.go

@@ -230,7 +230,7 @@ func TestTxDeleteBucket(t *testing.T) {
 
 		db.Update(func(tx *Tx) error {
 			// Verify that the bucket's page is free.
-			assert.Equal(t, []pgid{6, root, 3}, db.freelist.all())
+			assert.Equal(t, []pgid{7, 6, root, 2}, db.freelist.all())
 
 			// Create the bucket again and make sure there's not a phantom value.
 			assert.NoError(t, tx.CreateBucket("widgets"))