diff --git a/Makefile b/Makefile index 26bbef7..3449c62 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ run-gc: GODEBUG=gctrace=1 go run . test-cover: - go test -race -v -coverprofile=coverage.txt -covermode=atomic + go test ./... -race -v -coverprofile=coverage.txt -covermode=atomic go tool cover -html=coverage.txt -o coverage.html rm coverage.txt diff --git a/command.go b/command.go index 79b52fb..ae66e5c 100644 --- a/command.go +++ b/command.go @@ -179,8 +179,8 @@ func hgetallCommand(args []Arg) Value { } res := make([]Value, 0, 8) - hmap.Scan(func(key, value []byte) { - res = append(res, newBulkValue(key)) + hmap.Scan(func(key string, value []byte) { + res = append(res, newBulkValue([]byte(key))) res = append(res, newBulkValue(value)) }) return newArrayValue(res) diff --git a/dict/bench_test.go b/dict/bench_test.go new file mode 100644 index 0000000..ca3864c --- /dev/null +++ b/dict/bench_test.go @@ -0,0 +1,123 @@ +package dict + +import ( + "maps" + "testing" +) + +const N = 100 * 10000 + +func getStdmap(num int) map[string][]byte { + m := map[string][]byte{} + for i := 0; i < num; i++ { + k, v := genKV(i) + m[k] = v + } + return m +} + +func getDict(num int, options ...Options) *Dict { + opt := DefaultOptions + if len(options) > 0 { + opt = options[0] + } + m := New(opt) + for i := 0; i < num; i++ { + k, v := genKV(i) + m.Set(k, v) + } + return m +} + +func BenchmarkSet(b *testing.B) { + b.Run("stdmap", func(b *testing.B) { + m := map[string][]byte{} + for i := 0; i < b.N; i++ { + k, v := genKV(i) + m[k] = v + } + }) + b.Run("dict", func(b *testing.B) { + m := New(DefaultOptions) + for i := 0; i < b.N; i++ { + k, v := genKV(i) + m.Set(k, v) + } + }) +} + +func BenchmarkGet(b *testing.B) { + b.Run("stdmap", func(b *testing.B) { + m := getStdmap(N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + k, _ := genKV(i) + _ = m[k] + } + }) + b.Run("dict", func(b *testing.B) { + m := getDict(N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + k, _ := genKV(i) + m.Get(k) + } + }) +} + +func BenchmarkScan(b *testing.B) { + b.Run("stdmap", func(b *testing.B) { + m := getStdmap(N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + for k, v := range m { + _, _ = k, v + } + } + }) + b.Run("dict", func(b *testing.B) { + m := getDict(N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Scan(func(s string, b []byte, i int64) bool { + return true + }) + } + }) +} + +func BenchmarkRemove(b *testing.B) { + b.Run("stdmap", func(b *testing.B) { + m := getStdmap(N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + k, _ := genKV(i) + delete(m, k) + } + }) + b.Run("dict", func(b *testing.B) { + m := getDict(N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + k, _ := genKV(i) + m.Remove(k) + } + }) +} + +func BenchmarkMigrate(b *testing.B) { + b.Run("stdmap", func(b *testing.B) { + m := getStdmap(100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + maps.Clone(m) + } + }) + b.Run("dict", func(b *testing.B) { + m := getDict(100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Migrate() + } + }) +} diff --git a/dict/benchmark/main.go b/dict/benchmark/main.go new file mode 100644 index 0000000..b055fef --- /dev/null +++ b/dict/benchmark/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "flag" + "fmt" + "runtime" + "runtime/debug" + "time" + + "github.com/cockroachdb/swiss" + "github.com/xgzlucario/rotom/dict" +) + +var previousPause time.Duration + +func gcPause() time.Duration { + runtime.GC() + var stats debug.GCStats + debug.ReadGCStats(&stats) + pause := stats.PauseTotal - previousPause + previousPause = stats.PauseTotal + return pause +} + +func genKV(id int) (string, []byte) { + k := fmt.Sprintf("%08x", id) + return k, []byte(k) +} + +func main() { + c := "" + entries := 0 + flag.StringVar(&c, "cache", "dict", "map to bench.") + flag.IntVar(&entries, "entries", 2000*10000, "number of entries to test.") + flag.Parse() + + fmt.Println(c) + fmt.Println("entries:", entries) + + start := time.Now() + switch c { + case "dict": + m := dict.New(dict.DefaultOptions) + for i := 0; i < entries; i++ { + k, v := genKV(i) + m.Set(k, v) + } + + case "stdmap": + type Item struct { + val []byte + ts int64 + } + m := make(map[string]Item) + for i := 0; i < entries; i++ { + k, v := genKV(i) + m[string(k)] = Item{val: v, ts: 0} + } + + case "swiss": + type Item struct { + val []byte + ts int64 + } + m := swiss.New[string, Item](8) + for i := 0; i < entries; i++ { + k, v := genKV(i) + m.Put(string(k), Item{val: v, ts: 0}) + } + } + cost := time.Since(start) + + var mem runtime.MemStats + var stat debug.GCStats + + runtime.ReadMemStats(&mem) + debug.ReadGCStats(&stat) + + fmt.Println("alloc:", mem.Alloc/1024/1024, "mb") + fmt.Println("gcsys:", mem.GCSys/1024/1024, "mb") + fmt.Println("heap inuse:", mem.HeapInuse/1024/1024, "mb") + fmt.Println("heap object:", mem.HeapObjects/1024, "k") + fmt.Println("gc:", stat.NumGC) + fmt.Println("pause:", gcPause()) + fmt.Println("cost:", cost) +} diff --git a/dict/dict.go b/dict/dict.go new file mode 100644 index 0000000..211543e --- /dev/null +++ b/dict/dict.go @@ -0,0 +1,253 @@ +package dict + +import ( + "encoding/binary" + "math/rand/v2" + "time" + + "github.com/cockroachdb/swiss" +) + +const ( + noTTL = 0 + KB = 1024 + + // maxFailed indicates that the eviction algorithm breaks + // when consecutive unexpired key-value pairs are detected. + maxFailed = 3 +) + +// Dict is the hashmap for Rotom. +type Dict struct { + mask uint32 + shards []*shard +} + +func New(options Options) *Dict { + if err := validateOptions(options); err != nil { + panic(err) + } + dict := &Dict{ + mask: options.ShardCount - 1, + shards: make([]*shard, options.ShardCount), + } + for i := range dict.shards { + dict.shards[i] = &shard{ + options: &options, + index: swiss.New[string, Idx](options.IndexSize), + data: make([]byte, 0, options.BufferSize), + } + } + return dict +} + +func (dict *Dict) getShard(key string) *shard { + hash := MemHash(key) + return dict.shards[uint32(hash)&dict.mask] +} + +func (dict *Dict) Get(key string) ([]byte, int64, bool) { + shard := dict.getShard(key) + idx, ok := shard.index.Get(key) + if ok && !idx.expired() { + _, val := shard.findEntry(idx) + return val, idx.lo, ok + } + return nil, 0, false +} + +func (dict *Dict) SetTx(key string, val []byte, expiration int64) bool { + shard := dict.getShard(key) + idx, ok := shard.index.Get(key) + if ok { + entry, oldVal := shard.findEntry(idx) + // update value inplaced + if len(val) == len(oldVal) { + copy(oldVal, val) + shard.index.Put(key, idx.setTTL(expiration)) + return false + } + shard.unused += uint32(len(entry)) + } + + shard.index.Put(key, shard.appendEntry(val, expiration)) + return true +} + +func (dict *Dict) Set(kstr string, value []byte) bool { + return dict.SetTx(kstr, value, noTTL) +} + +func (dict *Dict) SetEx(kstr string, value []byte, duration time.Duration) bool { + return dict.SetTx(kstr, value, time.Now().Add(duration).UnixNano()) +} + +func (dict *Dict) Remove(key string) bool { + shard := dict.getShard(key) + idx, ok := shard.index.Get(key) + if ok { + shard.removeEntry(key, idx) + return !idx.expired() + } + return false +} + +func (dict *Dict) SetTTL(key string, expiration int64) bool { + shard := dict.getShard(key) + idx, ok := shard.index.Get(key) + if ok && !idx.expired() { + shard.index.Put(key, idx.setTTL(expiration)) + return true + } + return false +} + +type Walker func(key string, value []byte, ttl int64) (next bool) + +func (dict *Dict) Scan(callback Walker) { + for _, shard := range dict.shards { + next := shard.scan(callback) + if !next { + return + } + } +} + +func (dict *Dict) Migrate() { + for _, shard := range dict.shards { + shard.migrate() + } +} + +func (dict *Dict) EvictExpired() { + id := rand.IntN(len(dict.shards)) + shard := dict.shards[id] + shard.evictExpired() +} + +// Stats represents the runtime statistics of Dict. +type Stats struct { + Len int + Alloc uint64 + Unused uint64 + Migrates uint64 + Evictions uint64 + Probes uint64 +} + +// GetStats returns the current runtime statistics of Dict. +func (c *Dict) GetStats() (stats Stats) { + for _, shard := range c.shards { + stats.Len += shard.index.Len() + stats.Alloc += uint64(len(shard.data)) + stats.Unused += uint64(shard.unused) + stats.Migrates += uint64(shard.migrations) + stats.Evictions += shard.evictions + stats.Probes += shard.probes + } + return +} + +// UnusedRate calculates the percentage of unused space in the dict. +func (s Stats) UnusedRate() float64 { + return float64(s.Unused) / float64(s.Alloc) * 100 +} + +// EvictionRate calculates the percentage of evictions relative to probes. +func (s Stats) EvictionRate() float64 { + return float64(s.Evictions) / float64(s.Probes) * 100 +} + +// shard is the data container for Dict. +type shard struct { + options *Options + index *swiss.Map[string, Idx] + data []byte + unused uint32 + migrations uint32 + evictions uint64 + probes uint64 +} + +func (s *shard) appendEntry(val []byte, ts int64) Idx { + idx := newIdx(len(s.data), ts) + s.data = binary.AppendUvarint(s.data, uint64(len(val))) + s.data = append(s.data, val...) + return idx +} + +func (s *shard) scan(walker Walker) (next bool) { + next = true + s.index.All(func(key string, idx Idx) bool { + if idx.expired() { + return true + } + _, val := s.findEntry(idx) + next = walker(key, val, idx.lo) + return next + }) + return +} + +func (s *shard) evictExpired() { + var failed int + nanosec := time.Now().UnixNano() + + // probing + s.index.All(func(key string, idx Idx) bool { + s.probes++ + if idx.expiredWith(nanosec) { + s.removeEntry(key, idx) + s.evictions++ + failed = 0 + } else { + failed++ + } + return failed <= maxFailed + }) + + // check if migration is needed. + unusedRate := float64(s.unused) / float64(len(s.data)) + if unusedRate >= s.options.MigrateRatio { + s.migrate() + } +} + +// migrate transfers valid key-value pairs to a new container to save memory. +func (s *shard) migrate() { + newData := make([]byte, 0, len(s.data)) + nanosec := time.Now().UnixNano() + + s.index.All(func(key string, idx Idx) bool { + if idx.expiredWith(nanosec) { + s.index.Delete(key) + return true + } + s.index.Put(key, idx.setStart(len(newData))) + entry, _ := s.findEntry(idx) + newData = append(newData, entry...) + return true + }) + + s.data = newData + s.unused = 0 + s.migrations++ +} + +func (s *shard) findEntry(idx Idx) (entry, val []byte) { + pos := idx.start() + // read vlen + vlen, n := binary.Uvarint(s.data[pos:]) + pos += n + // read val + val = s.data[pos : pos+int(vlen)] + pos += int(vlen) + + return s.data[idx.start():pos], val +} + +func (s *shard) removeEntry(key string, idx Idx) { + entry, _ := s.findEntry(idx) + s.unused += uint32(len(entry)) + s.index.Delete(key) +} diff --git a/dict/dict_test.go b/dict/dict_test.go new file mode 100644 index 0000000..f687117 --- /dev/null +++ b/dict/dict_test.go @@ -0,0 +1,10 @@ +package dict + +import ( + "fmt" +) + +func genKV(i int) (string, []byte) { + k := fmt.Sprintf("%08x", i) + return k, []byte(k) +} diff --git a/dict/example/main.go b/dict/example/main.go new file mode 100644 index 0000000..68cc1cb --- /dev/null +++ b/dict/example/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "fmt" + "math/rand/v2" + "net/http" + _ "net/http/pprof" + "runtime" + "slices" + "strconv" + "time" + + "github.com/xgzlucario/rotom/dict" +) + +type Quantile struct { + f []float64 +} + +func NewQuantile(size int) *Quantile { + return &Quantile{f: make([]float64, 0, size)} +} + +func (q *Quantile) Add(v float64) { + q.f = append(q.f, v) +} + +func (q *Quantile) quantile(p float64) float64 { + r := q.f[int(float64(len(q.f))*p)] + return r +} + +func (q *Quantile) Print() { + slices.Sort(q.f) + fmt.Printf("90th: %.0f ns\n", q.quantile(0.9)) + fmt.Printf("99th: %.0f ns\n", q.quantile(0.99)) + fmt.Printf("999th: %.0f ns\n", q.quantile(0.999)) +} + +const N = 100 * 10000 + +func main() { + go func() { + _ = http.ListenAndServe("localhost:6060", nil) + }() + + options := dict.DefaultOptions + + fmt.Println("=====Options=====") + fmt.Printf("%+v\n", options) + benchmark(options) + runtime.GC() +} + +func benchmark(options dict.Options) { + quant := NewQuantile(N) + + var count int64 + var memStats runtime.MemStats + + dict := dict.New(options) + + // Set test + start := time.Now() + var now time.Time + for j := 0; ; j++ { + k := strconv.FormatUint(rand.Uint64(), 36) + + if j%10 == 0 { + now = time.Now() + if now.Sub(start) > time.Minute { + break + } + } + + dict.SetEx(k, []byte(k), time.Second) + count++ + + if j%10 == 0 { + cost := float64(time.Since(now)) / float64(time.Nanosecond) + quant.Add(cost) + } + + // evict + if j%10 == 0 { + dict.EvictExpired() + } + } + + // Stat + stat := dict.GetStats() + + fmt.Printf("[Cache] %.0fs | %dw | len: %dw | alloc: %v (unused: %.1f%%)\n", + time.Since(start).Seconds(), + count/1e4, + stat.Len/1e4, + formatSize(stat.Alloc), + stat.UnusedRate(), + ) + fmt.Printf("[Evict] probe: %vw / %vw (%.1f%%) | mgr: %d\n", + stat.Evictions/1e5, stat.Probes/1e5, stat.EvictionRate(), + stat.Migrates) + + // mem stats + runtime.ReadMemStats(&memStats) + fmt.Printf("[Mem] mem: %.0fMB | sys: %.0fMB | gc: %d | gcpause: %.0f us\n", + float64(memStats.Alloc)/1024/1024, + float64(memStats.Sys)/1024/1024, + memStats.NumGC, + float64(memStats.PauseTotalNs)/float64(memStats.NumGC)/1000) + + // quant print + quant.Print() + + fmt.Println("-----------------------------------------------------") +} + +const ( + KB = 1024 + MB = 1024 * KB +) + +// formatSize +func formatSize[T float64 | uint64](size T) string { + switch { + case size < KB: + return fmt.Sprintf("%.0fB", float64(size)) + case size < MB: + return fmt.Sprintf("%.1fKB", float64(size)/KB) + default: + return fmt.Sprintf("%.1fMB", float64(size)/MB) + } +} diff --git a/dict/index.go b/dict/index.go new file mode 100644 index 0000000..a9191f4 --- /dev/null +++ b/dict/index.go @@ -0,0 +1,45 @@ +package dict + +import ( + "math" + "time" +) + +type Idx struct { + hi uint32 // hi is position of data. + lo int64 // lo is timestamp of key. +} + +func (i Idx) start() int { + return int(i.hi) +} + +func (i Idx) expired() bool { + return i.lo > noTTL && i.lo < time.Now().UnixNano() +} + +func (i Idx) expiredWith(nanosec int64) bool { + return i.lo > noTTL && i.lo < nanosec +} + +func (i Idx) setTTL(ts int64) Idx { + i.lo = ts + return i +} + +func (i Idx) setStart(start int) Idx { + check(start) + i.hi = uint32(start) + return i +} + +func check(x int) { + if x > math.MaxUint32 { + panic("x overflows the limit of uint32") + } +} + +func newIdx(start int, ttl int64) Idx { + check(start) + return Idx{hi: uint32(start), lo: ttl} +} diff --git a/dict/utils.go b/dict/utils.go new file mode 100644 index 0000000..126b1d6 --- /dev/null +++ b/dict/utils.go @@ -0,0 +1,56 @@ +package dict + +import ( + "errors" + "math/bits" + "unsafe" +) + +type stringStruct struct { + str unsafe.Pointer + len int +} + +//go:noescape +//go:linkname memhash runtime.memhash +func memhash(p unsafe.Pointer, h, s uintptr) uintptr + +type HashFn func(string) uint64 + +// MemHash is the hash function used by go map, it utilizes available hardware instructions +// (behaves as aes hash if aes instruction is available). +// NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash. +func MemHash(str string) uint64 { + ss := (*stringStruct)(unsafe.Pointer(&str)) + return uint64(memhash(ss.str, 0, uintptr(ss.len))) +} + +// SizeUvarint +// See https://go-review.googlesource.com/c/go/+/572196/1/src/encoding/binary/varint.go#174 +func SizeUvarint(x uint64) int { + return int(9*uint32(bits.Len64(x))+64) / 64 +} + +type Options struct { + ShardCount uint32 + + // Default size of the bucket initial. + IndexSize int + BufferSize int + + // Migrate threshold for a bucket to trigger a migration. + MigrateRatio float64 +} + +var DefaultOptions = Options{ + ShardCount: 1024, + IndexSize: 1024, + BufferSize: 64 * KB, +} + +func validateOptions(options Options) error { + if options.ShardCount == 0 { + return errors.New("cache/options: invalid shard count") + } + return nil +} diff --git a/go.mod b/go.mod index ff3c766..61be74e 100644 --- a/go.mod +++ b/go.mod @@ -3,29 +3,28 @@ module github.com/xgzlucario/rotom go 1.22 require ( + github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/deckarep/golang-set/v2 v2.6.0 github.com/redis/go-redis/v9 v9.5.2 github.com/rs/zerolog v1.33.0 github.com/sakeven/RbTree v1.1.1 github.com/stretchr/testify v1.9.0 github.com/tidwall/mmap v0.3.0 - github.com/xgzlucario/GigaCache v0.0.0-20240614161308-886947cfb989 github.com/xgzlucario/quicklist v0.0.0-20240530174658-6f1a884f579b golang.org/x/sys v0.21.0 ) require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e343416..eb67f37 100644 --- a/go.sum +++ b/go.sum @@ -19,10 +19,11 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -36,6 +37,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.5.2 h1:L0L3fcSNReTRGyZ6AqAEN0K56wYeYAwapBIhkvh0f3E= github.com/redis/go-redis/v9 v9.5.2/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -47,25 +49,20 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/mmap v0.3.0 h1:XXt1YsiXCF5/UAu3pLbu6g7iulJ9jsbs6vt7UpiV0sY= github.com/tidwall/mmap v0.3.0/go.mod h1:2/dNzF5zA+te/JVHfrqNLcRkb8LjdH3c80vYHFQEZRk= -github.com/xgzlucario/GigaCache v0.0.0-20240614161308-886947cfb989 h1:oBY9eL6HxLYH7bATgeXwBrebPSKrz0IC+VTBNtINn/k= -github.com/xgzlucario/GigaCache v0.0.0-20240614161308-886947cfb989/go.mod h1:ChL9yLENqn4jVon23LCNQ6fPPEdA9u8vgvnpDIDlVEs= github.com/xgzlucario/quicklist v0.0.0-20240530174658-6f1a884f579b h1:C/+nN/kFJ6yrmEhIu+5Ra2jx/W8w+Ayu8pTiZfuU5Xc= github.com/xgzlucario/quicklist v0.0.0-20240530174658-6f1a884f579b/go.mod h1:1ZgyZNk91XIllYdOPpwP+9L2RCw6QGSy6alTYF+Z0iU= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/rotom.go b/rotom.go index 31da6f3..172da6c 100644 --- a/rotom.go +++ b/rotom.go @@ -3,7 +3,7 @@ package main import ( "io" - cache "github.com/xgzlucario/GigaCache" + "github.com/xgzlucario/rotom/dict" "github.com/xgzlucario/rotom/structx" ) @@ -20,7 +20,7 @@ type ( ) type DB struct { - strs *cache.GigaCache + strs *dict.Dict extras map[string]any aof *Aof } @@ -46,10 +46,7 @@ var ( // InitDB initializes database and redo appendonly files if nedded. func InitDB(config *Config) (err error) { - options := cache.DefaultOptions - options.ConcurrencySafe = false - options.DisableEvict = true - db.strs = cache.New(options) + db.strs = dict.New(dict.DefaultOptions) db.extras = make(map[string]any) if config.AppendOnly { @@ -219,5 +216,5 @@ func ServerCronFlush(loop *AeLoop, id int, extra interface{}) { } func ServerCronEvict(loop *AeLoop, id int, extra interface{}) { - db.strs.EvictExpiredKeys() + db.strs.EvictExpired() } diff --git a/structx/map.go b/structx/map.go index 36cb39a..e67accb 100644 --- a/structx/map.go +++ b/structx/map.go @@ -1,18 +1,15 @@ package structx import ( - cache "github.com/xgzlucario/GigaCache" + "github.com/xgzlucario/rotom/dict" ) type Map struct { - m *cache.GigaCache + m *dict.Dict } -func defaultOptions() cache.Options { - options := cache.DefaultOptions - options.ConcurrencySafe = false - // the hash keys is no need expire - options.DisableEvict = true +func defaultOptions() dict.Options { + options := dict.DefaultOptions options.ShardCount = 1 options.IndexSize = 8 options.BufferSize = 32 @@ -20,7 +17,7 @@ func defaultOptions() cache.Options { } func NewMap() (s *Map) { - return &Map{m: cache.New(defaultOptions())} + return &Map{m: dict.New(defaultOptions())} } func (m *Map) Get(key string) ([]byte, int64, bool) { @@ -35,8 +32,8 @@ func (m *Map) Remove(key string) bool { return m.m.Remove(key) } -func (m *Map) Scan(fn func(key, value []byte)) { - m.m.Scan(func(key, val []byte, _ int64) (next bool) { +func (m *Map) Scan(fn func(key string, value []byte)) { + m.m.Scan(func(key string, val []byte, _ int64) (next bool) { fn(key, val) return true })