add: add pkg package

internal/dict/bench_test.go

@@ -1,46 +1,45 @@
 package dict
 
 import (
+	"runtime"
 	"testing"
+
+	"github.com/cockroachdb/swiss"
 )
 
 const N = 100 * 10000
 
-func getStdmap(num int) map[string][]byte {
-	m := map[string][]byte{}
-	for i := 0; i < num; i++ {
-		k, v := genKV(i)
-		m[k] = v
+func getStdmap(n int) map[string]int {
+	m := make(map[string]int, n)
+	for i := 0; i < n; i++ {
+		k, _ := genKV(i)
+		m[k] = i
 	}
 	return m
 }
 
-func getDict(num int, options ...Options) *Dict {
-	opt := DefaultOptions
-	if len(options) > 0 {
-		opt = options[0]
-	}
-	m := New(opt)
-	for i := 0; i < num; i++ {
-		k, v := genKV(i)
-		m.Set(k, v)
+func getSwiss(n int) *swiss.Map[string, int] {
+	m := swiss.New[string, int](n)
+	for i := 0; i < n; i++ {
+		k, _ := genKV(i)
+		m.Put(k, i)
 	}
 	return m
 }
 
 func BenchmarkSet(b *testing.B) {
 	b.Run("stdmap", func(b *testing.B) {
-		m := map[string][]byte{}
+		m := make(map[string]int, 8)
 		for i := 0; i < b.N; i++ {
-			k, v := genKV(i)
-			m[k] = v
+			k, _ := genKV(i)
+			m[k] = i
 		}
 	})
-	b.Run("dict", func(b *testing.B) {
-		m := New(DefaultOptions)
+	b.Run("swiss", func(b *testing.B) {
+		m := swiss.New[string, int](8)
 		for i := 0; i < b.N; i++ {
-			k, v := genKV(i)
-			m.Set(k, v)
+			k, _ := genKV(i)
+			m.Put(k, i)
 		}
 	})
 }
@@ -54,8 +53,8 @@ func BenchmarkGet(b *testing.B) {
 			_ = m[k]
 		}
 	})
-	b.Run("dict", func(b *testing.B) {
-		m := getDict(N)
+	b.Run("swiss", func(b *testing.B) {
+		m := getSwiss(N)
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			k, _ := genKV(i)
@@ -64,42 +63,32 @@ func BenchmarkGet(b *testing.B) {
 	})
 }
 
-func BenchmarkScan(b *testing.B) {
-	b.Run("stdmap", func(b *testing.B) {
-		m := getStdmap(N)
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			for k, v := range m {
-				_, _ = k, v
-			}
+func BenchmarkGC(b *testing.B) {
+	b.Run("swiss1", func(b *testing.B) {
+		m := swiss.New[string, int](N)
+		data := make([]byte, 0)
+		for i := 0; i < N; i++ {
+			k, v := genKV(i)
+			m.Put(k, i)
+			data = append(data, v...)
 		}
-	})
-	b.Run("dict", func(b *testing.B) {
-		m := getDict(N)
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			m.Scan(func(s string, b []byte, i int64) bool {
-				return true
-			})
+			runtime.GC()
 		}
+		m.Put("xgz", len(data))
 	})
-}
 
-func BenchmarkRemove(b *testing.B) {
-	b.Run("stdmap", func(b *testing.B) {
-		m := getStdmap(N)
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			k, _ := genKV(i)
-			delete(m, k)
+	b.Run("swiss2", func(b *testing.B) {
+		m := swiss.New[string, []byte](N)
+		for i := 0; i < N; i++ {
+			k, v := genKV(i)
+			m.Put(k, v)
 		}
-	})
-	b.Run("dict", func(b *testing.B) {
-		m := getDict(N)
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			k, _ := genKV(i)
-			m.Remove(k)
+			runtime.GC()
 		}
+		m.Put("xgz", []byte("hello"))
 	})
 }

internal/dict/benchmark/main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cockroachdb/swiss"
 	"github.com/xgzlucario/rotom/internal/dict"
+	"github.com/xgzlucario/rotom/internal/pkg"
 )
 
 var previousPause time.Duration
@@ -38,34 +39,34 @@ func main() {
 	fmt.Println("entries:", entries)
 
 	start := time.Now()
+	q := pkg.NewQuantile(entries)
+
 	switch c {
 	case "dict":
 		m := dict.New(dict.DefaultOptions)
 		for i := 0; i < entries; i++ {
 			k, v := genKV(i)
+			start := time.Now()
 			m.Set(k, v)
+			q.Add(float64(time.Since(start)))
 		}
 
 	case "stdmap":
-		type Item struct {
-			val []byte
-			ts  int64
-		}
-		m := make(map[string]Item, entries)
+		m := make(map[string][]byte, 8)
 		for i := 0; i < entries; i++ {
 			k, v := genKV(i)
-			m[string(k)] = Item{val: v, ts: 0}
+			start := time.Now()
+			m[k] = v
+			q.Add(float64(time.Since(start)))
 		}
 
 	case "swiss":
-		type Item struct {
-			val []byte
-			ts  int64
-		}
-		m := swiss.New[string, Item](entries)
+		m := swiss.New[string, []byte](8)
 		for i := 0; i < entries; i++ {
 			k, v := genKV(i)
-			m.Put(string(k), Item{val: v, ts: 0})
+			start := time.Now()
+			m.Put(k, v)
+			q.Add(float64(time.Since(start)))
 		}
 	}
 	cost := time.Since(start)
@@ -76,11 +77,11 @@ func main() {
 	runtime.ReadMemStats(&mem)
 	debug.ReadGCStats(&stat)
 
-	fmt.Println("alloc:", mem.Alloc/1024/1024, "mb")
 	fmt.Println("gcsys:", mem.GCSys/1024/1024, "mb")
 	fmt.Println("heap inuse:", mem.HeapInuse/1024/1024, "mb")
 	fmt.Println("heap object:", mem.HeapObjects/1024, "k")
 	fmt.Println("gc:", stat.NumGC)
 	fmt.Println("pause:", gcPause())
 	fmt.Println("cost:", cost)
+	q.Print()
 }

internal/dict/dict.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/swiss"
+	"github.com/xgzlucario/rotom/internal/pkg"
 )
 
 const (
@@ -17,6 +18,10 @@ const (
 	maxFailed = 3
 )
 
+var (
+	dictAllocator = pkg.NewAllocator[string, Idx]()
+)
+
 // Dict is the hashmap for Rotom.
 type Dict struct {
 	mask   uint32
@@ -31,7 +36,7 @@ func New(options Options) *Dict {
 	for i := range dict.shards {
 		dict.shards[i] = &shard{
 			options: &options,
-			index:   swiss.New[string, Idx](options.IndexSize),
+			index:   swiss.New(options.IndexSize, swiss.WithAllocator(dictAllocator)),
 			data:    make([]byte, 0, options.BufferSize),
 		}
 	}
@@ -117,12 +122,6 @@ func (dict *Dict) Scan(callback Walker) {
 	}
 }
 
-func (dict *Dict) Migrate() {
-	for _, shard := range dict.shards {
-		shard.migrate()
-	}
-}
-
 func (dict *Dict) EvictExpired() {
 	id := rand.IntN(len(dict.shards))
 	dict.shards[id].evictExpired()
@@ -206,29 +205,31 @@ func (s *shard) evictExpired() {
 func (s *shard) migrate() {
 	newData := make([]byte, 0, len(s.data))
 	nanosec := time.Now().UnixNano()
+	newIndex := swiss.New(s.index.Len(), swiss.WithAllocator(dictAllocator))
 
 	s.index.All(func(key string, idx Idx) bool {
 		if idx.expiredWith(nanosec) {
-			s.index.Delete(key)
 			return true
 		}
-		s.index.Put(key, idx.setStart(len(newData)))
+		newIndex.Put(key, idx.setStart(len(newData)))
 		entry, _ := s.findEntry(idx)
 		newData = append(newData, entry...)
 		return true
 	})
 
+	s.index.Close()
+	s.index = newIndex
 	s.data = newData
 	s.unused = 0
 	s.migrations++
 }
 
 func (s *shard) findEntry(idx Idx) (entry, val []byte) {
 	pos := idx.start()
-	// read vlen
+	// read value len
 	vlen, n := binary.Uvarint(s.data[pos:])
 	pos += n
-	// read val
+	// read value
 	val = s.data[pos : pos+int(vlen)]
 	pos += int(vlen)
 

internal/dict/example/main.go

@@ -6,37 +6,13 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"runtime"
-	"slices"
 	"strconv"
 	"time"
 
 	"github.com/xgzlucario/rotom/internal/dict"
+	"github.com/xgzlucario/rotom/internal/pkg"
 )
 
-type Quantile struct {
-	f []float64
-}
-
-func NewQuantile(size int) *Quantile {
-	return &Quantile{f: make([]float64, 0, size)}
-}
-
-func (q *Quantile) Add(v float64) {
-	q.f = append(q.f, v)
-}
-
-func (q *Quantile) quantile(p float64) float64 {
-	r := q.f[int(float64(len(q.f))*p)]
-	return r
-}
-
-func (q *Quantile) Print() {
-	slices.Sort(q.f)
-	fmt.Printf("90th: %.0f ns\n", q.quantile(0.9))
-	fmt.Printf("99th: %.0f ns\n", q.quantile(0.99))
-	fmt.Printf("999th: %.0f ns\n", q.quantile(0.999))
-}
-
 const N = 100 * 10000
 
 func main() {
@@ -53,7 +29,7 @@ func main() {
 }
 
 func benchmark(options dict.Options) {
-	quant := NewQuantile(N)
+	quant := pkg.NewQuantile(N)
 
 	var count int64
 	var memStats runtime.MemStats

internal/hash/map.go

@@ -2,14 +2,19 @@ package hash
 
 import (
 	"github.com/cockroachdb/swiss"
+	"github.com/xgzlucario/rotom/internal/pkg"
+)
+
+var (
+	mapAllocator = pkg.NewAllocator[string, []byte]()
 )
 
 type Map struct {
 	m *swiss.Map[string, []byte]
 }
 
 func NewMap() *Map {
-	return &Map{m: swiss.New[string, []byte](8)}
+	return &Map{m: swiss.New(8, swiss.WithAllocator(mapAllocator))}
 }
 
 func (m *Map) Get(key string) ([]byte, bool) {
@@ -38,3 +43,7 @@ func (m *Map) Scan(fn func(key string, value []byte)) {
 		return true
 	})
 }
+
+func (m *Map) Free() {
+	m.m.Close()
+}

internal/hash/set.go

@@ -2,14 +2,19 @@ package hash
 
 import (
 	"github.com/cockroachdb/swiss"
+	"github.com/xgzlucario/rotom/internal/pkg"
+)
+
+var (
+	setAllocator = pkg.NewAllocator[string, struct{}]()
 )
 
 type Set struct {
 	m *swiss.Map[string, struct{}]
 }
 
 func NewSet() *Set {
-	return &Set{m: swiss.New[string, struct{}](8)}
+	return &Set{m: swiss.New[string, struct{}](8, swiss.WithAllocator(setAllocator))}
 }
 
 func (s *Set) Add(key string) bool {
@@ -28,3 +33,7 @@ func (s *Set) Pop() (item string, ok bool) {
 	})
 	return
 }
+
+func (s *Set) Free() {
+	s.m.Close()
+}