use segregated hashmap to boost the freelist allocate and release per…

…formance

allocate_test.go

@@ -6,7 +6,8 @@ import (
 
 func TestTx_allocatePageStats(t *testing.T) {
 	f := newFreelist()
-	f.ids = []pgid{2, 3}
+	ids := []pgid{2, 3}
+	f.init(ids)
 
 	tx := &Tx{
 		db: &DB{

db.go

@@ -291,7 +291,7 @@ func (db *DB) loadFreelist() {
 			// Read free list from freelist page.
 			db.freelist.read(db.page(db.meta().freelist))
 		}
-		db.stats.FreePageN = len(db.freelist.ids)
+		db.stats.FreePageN = len(db.freelist.getFreePageIDs())
 	})
 }
 

freelist.go

@@ -14,21 +14,65 @@ type txPending struct {
 	lastReleaseBegin txid   // beginning txid of last matching releaseRange
 }
 
+// pidSet holds the set of starting pgids which have the same span size
+type pidSet map[pgid]struct{}
+
 // freelist represents a list of all pages that are available for allocation.
 // It also tracks pages that have been freed but are still in use by open transactions.
 type freelist struct {
-	ids     []pgid              // all free and available free page ids.
-	allocs  map[pgid]txid       // mapping of txid that allocated a pgid.
-	pending map[txid]*txPending // mapping of soon-to-be free page ids by tx.
-	cache   map[pgid]bool       // fast lookup of all free and pending page ids.
+	allocs      map[pgid]txid       // mapping of txid that allocated a pgid.
+	pending     map[txid]*txPending // mapping of soon-to-be free page ids by tx.
+	cache       map[pgid]bool       // fast lookup of all free and pending page ids.
+	freemaps    map[uint64]pidSet   // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size
+	forwardMap  map[pgid]uint64     // key is start pgid, value is its span size
+	backwardMap map[pgid]uint64     // key is end pgid, value is its span size
 }
 
 // newFreelist returns an empty, initialized freelist.
 func newFreelist() *freelist {
 	return &freelist{
-		allocs:  make(map[pgid]txid),
-		pending: make(map[txid]*txPending),
-		cache:   make(map[pgid]bool),
+		allocs:      make(map[pgid]txid),
+		pending:     make(map[txid]*txPending),
+		cache:       make(map[pgid]bool),
+		freemaps:    make(map[uint64]pidSet),
+		forwardMap:  make(map[pgid]uint64),
+		backwardMap: make(map[pgid]uint64),
+	}
+}
+
+// initial from pgids
+// pgids must be sorted
+func (f *freelist) init(pgids []pgid) {
+	if len(pgids) == 0 {
+		return
+	}
+
+	size := uint64(1)
+	start := pgids[0]
+
+	if !sort.SliceIsSorted([]pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) {
+		panic("pgids not sorted")
+	}
+
+	f.freemaps = make(map[uint64]pidSet)
+	f.forwardMap = make(map[pgid]uint64)
+	f.backwardMap = make(map[pgid]uint64)
+
+	for i := 1; i < len(pgids); i++ {
+		// continuous page
+		if pgids[i] == pgids[i-1]+1 {
+			size++
+		} else {
+			f.addSpan(start, size)
+
+			size = 1
+			start = pgids[i]
+		}
+	}
+
+	// init the tail
+	if size != 0 && start != 0 {
+		f.addSpan(start, size)
 	}
 }
 
@@ -49,7 +93,12 @@ func (f *freelist) count() int {
 
 // free_count returns count of free pages
 func (f *freelist) free_count() int {
-	return len(f.ids)
+	//use the forwardmap to get the total count
+	count := 0
+	for _, size := range f.forwardMap {
+		count += int(size)
+	}
+	return count
 }
 
 // pending_count returns count of pending pages
@@ -69,50 +118,68 @@ func (f *freelist) copyall(dst []pgid) {
 		m = append(m, txp.ids...)
 	}
 	sort.Sort(m)
-	mergepgids(dst, f.ids, m)
+	mergepgids(dst, f.getFreePageIDs(), m)
+}
+
+// getFreePageIDs returns the sorted free page ids
+func (f *freelist) getFreePageIDs() []pgid {
+	m := make([]pgid, 0, f.free_count())
+	for start, size := range f.forwardMap {
+		for i := 0; i < int(size); i++ {
+			m = append(m, start+pgid(i))
+		}
+	}
+	sort.Sort(pgids(m))
+
+	return m
 }
 
 // allocate returns the starting page id of a contiguous list of pages of a given size.
 // If a contiguous block cannot be found then 0 is returned.
 func (f *freelist) allocate(txid txid, n int) pgid {
-	if len(f.ids) == 0 {
+	if n == 0 {
 		return 0
 	}
 
-	var initial, previd pgid
-	for i, id := range f.ids {
-		if id <= 1 {
-			panic(fmt.Sprintf("invalid page allocation: %d", id))
+	// if we have a exact size match just return short path
+	if bm, ok := f.freemaps[uint64(n)]; ok {
+		for pid := range bm {
+			// remove the span
+			f.delSpan(pid, uint64(n))
+
+			f.allocs[pid] = txid
+
+			for i := pgid(0); i < pgid(n); i++ {
+				delete(f.cache, pid+pgid(i))
+			}
+			return pid
 		}
+	}
 
-		// Reset initial page if this is not contiguous.
-		if previd == 0 || id-previd != 1 {
-			initial = id
+	// lookup the map to find larger span
+	for size, bm := range f.freemaps {
+		if size < uint64(n) {
+			continue
 		}
 
-		// If we found a contiguous block then remove it and return it.
-		if (id-initial)+1 == pgid(n) {
-			// If we're allocating off the beginning then take the fast path
-			// and just adjust the existing slice. This will use extra memory
-			// temporarily but the append() in free() will realloc the slice
-			// as is necessary.
-			if (i + 1) == n {
-				f.ids = f.ids[i+1:]
-			} else {
-				copy(f.ids[i-n+1:], f.ids[i+1:])
-				f.ids = f.ids[:len(f.ids)-n]
-			}
+		for pid := range bm {
+			// remove the initial
+			f.delSpan(pid, uint64(size))
+
+			f.allocs[pid] = txid
+
+			remain := size - uint64(n)
+
+			// add remain span
+			f.addSpan(pid+pgid(n), remain)
 
-			// Remove from the free cache.
 			for i := pgid(0); i < pgid(n); i++ {
-				delete(f.cache, initial+i)
+				delete(f.cache, pid+pgid(i))
 			}
-			f.allocs[initial] = txid
-			return initial
+			return pid
 		}
-
-		previd = id
 	}
+
 	return 0
 }
 
@@ -160,8 +227,59 @@ func (f *freelist) release(txid txid) {
 			delete(f.pending, tid)
 		}
 	}
-	sort.Sort(m)
-	f.ids = pgids(f.ids).merge(m)
+
+	for _, pgid := range m {
+		// try to see if we can merge and update
+		f.mergeWithExistingSpan(pgid)
+	}
+}
+
+// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward
+func (f *freelist) mergeWithExistingSpan(pid pgid) {
+	prev := pid - 1
+	next := pid + 1
+
+	preSize, mergeWithPrev := f.backwardMap[prev]
+	nextSize, mergeWithNext := f.forwardMap[next]
+	newStart := pid
+	newSize := uint64(1)
+
+	if mergeWithPrev {
+		//merge with previous span
+		start := prev + 1 - pgid(preSize)
+		f.delSpan(start, preSize)
+
+		newStart -= pgid(preSize)
+		newSize += preSize
+	}
+
+	if mergeWithNext {
+		// merge with next span
+		f.delSpan(next, nextSize)
+		newSize += nextSize
+	}
+
+	f.addSpan(newStart, newSize)
+}
+
+func (f *freelist) addSpan(start pgid, size uint64) {
+	f.backwardMap[start-1+pgid(size)] = size
+	f.forwardMap[start] = size
+	if _, ok := f.freemaps[size]; !ok {
+		f.freemaps[size] = make(map[pgid]struct{})
+	}
+
+	f.freemaps[size][start] = struct{}{}
+}
+
+func (f *freelist) delSpan(start pgid, size uint64) {
+	delete(f.forwardMap, start)
+	delete(f.backwardMap, start+pgid(size-1))
+	delete(f.freemaps[size], start)
+	if len(f.freemaps[size]) == 0 {
+		delete(f.freemaps, size)
+	}
+
 }
 
 // releaseRange moves pending pages allocated within an extent [begin,end] to the free list.
@@ -194,8 +312,11 @@ func (f *freelist) releaseRange(begin, end txid) {
 			delete(f.pending, tid)
 		}
 	}
-	sort.Sort(m)
-	f.ids = pgids(f.ids).merge(m)
+
+	for _, pid := range m {
+		// try to see if we can merge and update
+		f.mergeWithExistingSpan(pid)
+	}
 }
 
 // rollback removes the pages from a given pending tx.
@@ -222,8 +343,11 @@ func (f *freelist) rollback(txid txid) {
 	}
 	// Remove pages from pending list and mark as free if allocated by txid.
 	delete(f.pending, txid)
-	sort.Sort(m)
-	f.ids = pgids(f.ids).merge(m)
+
+	for _, pid := range m {
+		// try to see if we can merge and update
+		f.mergeWithExistingSpan(pid)
+	}
 }
 
 // freed returns whether a given page is in the free list.
@@ -245,15 +369,9 @@ func (f *freelist) read(p *page) {
 	}
 
 	// Copy the list of page ids from the freelist.
-	if count == 0 {
-		f.ids = nil
-	} else {
+	if count != 0 {
 		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count]
-		f.ids = make([]pgid, len(ids))
-		copy(f.ids, ids)
-
-		// Make sure they're sorted.
-		sort.Sort(pgids(f.ids))
+		f.init(ids)
 	}
 
 	// Rebuild the page cache.
@@ -262,7 +380,7 @@ func (f *freelist) read(p *page) {
 
 // read initializes the freelist from a given list of ids.
 func (f *freelist) readIDs(ids []pgid) {
-	f.ids = ids
+	f.init(ids)
 	f.reindex()
 }
 
@@ -307,12 +425,12 @@ func (f *freelist) reload(p *page) {
 	// Check each page in the freelist and build a new available freelist
 	// with any pages not in the pending lists.
 	var a []pgid
-	for _, id := range f.ids {
+	for _, id := range f.getFreePageIDs() {
 		if !pcache[id] {
 			a = append(a, id)
 		}
 	}
-	f.ids = a
+	f.init(a)
 
 	// Once the available list is rebuilt then rebuild the free cache so that
 	// it includes the available and pending free pages.
@@ -321,9 +439,11 @@ func (f *freelist) reload(p *page) {
 
 // reindex rebuilds the free cache based on available and pending free lists.
 func (f *freelist) reindex() {
-	f.cache = make(map[pgid]bool, len(f.ids))
-	for _, id := range f.ids {
-		f.cache[id] = true
+	f.cache = make(map[pgid]bool, f.free_count())
+	for start, size := range f.forwardMap {
+		for i := 0; i < int(size); i++ {
+			f.cache[pgid(i)+start] = true
+		}
 	}
 	for _, txp := range f.pending {
 		for _, pendingID := range txp.ids {