From 7b021794616db73c7a50f2d46460f322fa93794f Mon Sep 17 00:00:00 2001 From: Horatiu Ion Date: Tue, 6 Jul 2021 15:37:24 +0200 Subject: [PATCH 1/4] add cache method to reload from file --- fastcache.go | 7 +++++++ file.go | 15 ++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/fastcache.go b/fastcache.go index 20a3c02..cc99cb9 100644 --- a/fastcache.go +++ b/fastcache.go @@ -211,6 +211,13 @@ func (c *Cache) UpdateStats(s *Stats) { s.InvalidValueHashErrors += atomic.LoadUint64(&c.bigStats.InvalidValueHashErrors) } +func (c *Cache) ReloadFromFile(path string) error { + c.Reset() + var err error + c, err = load(c, path, 0) + return err +} + type bucket struct { mu sync.RWMutex diff --git a/file.go b/file.go index bab5484..4aa8eca 100644 --- a/file.go +++ b/file.go @@ -80,7 +80,7 @@ func (c *Cache) SaveToFileConcurrent(filePath string, concurrency int) error { // // See SaveToFile* for saving cache data to file. func LoadFromFile(filePath string) (*Cache, error) { - return load(filePath, 0) + return load(nil, filePath, 0) } // LoadFromFileOrNew tries loading cache data from the given filePath. @@ -88,7 +88,7 @@ func LoadFromFile(filePath string) (*Cache, error) { // The function falls back to creating new cache with the given maxBytes // capacity if error occurs during loading the cache from file. func LoadFromFileOrNew(filePath string, maxBytes int) *Cache { - c, err := load(filePath, maxBytes) + c, err := load(nil, filePath, maxBytes) if err == nil { return c } @@ -125,7 +125,7 @@ func (c *Cache) save(dir string, workersCount int) error { return err } -func load(filePath string, maxBytes int) (*Cache, error) { +func load(oldCache *Cache, filePath string, maxBytes int) (*Cache, error) { maxBucketChunks, err := loadMetadata(filePath) if err != nil { return nil, err @@ -152,7 +152,12 @@ func load(filePath string, maxBytes int) (*Cache, error) { } results := make(chan error) workersCount := 0 - var c Cache + var c *Cache + if oldCache != nil { + c = oldCache + } else { + c = &Cache{} + } for _, fi := range fis { fn := fi.Name() if fi.IsDir() || !dataFileRegexp.MatchString(fn) { @@ -183,7 +188,7 @@ func load(filePath string, maxBytes int) (*Cache, error) { b.m = make(map[uint64]uint64) } } - return &c, nil + return c, nil } func saveMetadata(c *Cache, dir string) error { From e13ffce7df65f3dcbca08b7273fe1dd7cdf45651 Mon Sep 17 00:00:00 2001 From: Horatiu Ion Date: Tue, 6 Jul 2021 15:57:46 +0200 Subject: [PATCH 2/4] add `.Close` method to cache --- fastcache.go | 5 +++++ malloc_heap.go | 5 +++++ malloc_mmap.go | 19 ++++++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/fastcache.go b/fastcache.go index cc99cb9..dfaf9c8 100644 --- a/fastcache.go +++ b/fastcache.go @@ -218,6 +218,11 @@ func (c *Cache) ReloadFromFile(path string) error { return err } +func (c *Cache) Close() error { + c.Reset() + return clearChunks() +} + type bucket struct { mu sync.RWMutex diff --git a/malloc_heap.go b/malloc_heap.go index 79a7183..442a963 100644 --- a/malloc_heap.go +++ b/malloc_heap.go @@ -9,3 +9,8 @@ func getChunk() []byte { func putChunk(chunk []byte) { // No-op. } + +func clearChunks() error { + // No-op. + return nil +} diff --git a/malloc_mmap.go b/malloc_mmap.go index e0cd0e7..0c9d0c6 100644 --- a/malloc_mmap.go +++ b/malloc_mmap.go @@ -13,7 +13,10 @@ import ( const chunksPerAlloc = 1024 var ( - freeChunks []*[chunkSize]byte + // chunks to hand out to the library + freeChunks []*[chunkSize]byte + // orignal slice from Mmap for book-keeping + baseChunks [][]byte freeChunksLock sync.Mutex ) @@ -26,6 +29,7 @@ func getChunk() []byte { if err != nil { panic(fmt.Errorf("cannot allocate %d bytes via mmap: %s", chunkSize*chunksPerAlloc, err)) } + baseChunks = append(baseChunks, data) for len(data) > 0 { p := (*[chunkSize]byte)(unsafe.Pointer(&data[0])) freeChunks = append(freeChunks, p) @@ -51,3 +55,16 @@ func putChunk(chunk []byte) { freeChunks = append(freeChunks, p) freeChunksLock.Unlock() } + +func clearChunks() error { + freeChunksLock.Lock() + defer freeChunksLock.Unlock() + freeChunks = nil + for _, data := range baseChunks { + baseChunk := data + if err := unix.Munmap(baseChunk); err != nil { + return err + } + } + return nil +} From a6cb9793d5a00b8b84862cdfe05d2fcec110a258 Mon Sep 17 00:00:00 2001 From: Horatiu Ion Date: Tue, 6 Jul 2021 19:07:58 +0200 Subject: [PATCH 3/4] also clean baseChunks --- malloc_mmap.go | 1 + 1 file changed, 1 insertion(+) diff --git a/malloc_mmap.go b/malloc_mmap.go index 0c9d0c6..65bb75a 100644 --- a/malloc_mmap.go +++ b/malloc_mmap.go @@ -66,5 +66,6 @@ func clearChunks() error { return err } } + baseChunks = nil return nil } From 70f50021599ae9c0e6677a62fa69d5588e0d0aa0 Mon Sep 17 00:00:00 2001 From: Horatiu Ion Date: Tue, 6 Jul 2021 19:10:43 +0200 Subject: [PATCH 4/4] remove useless line --- malloc_mmap.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/malloc_mmap.go b/malloc_mmap.go index 65bb75a..ff2cc23 100644 --- a/malloc_mmap.go +++ b/malloc_mmap.go @@ -60,8 +60,7 @@ func clearChunks() error { freeChunksLock.Lock() defer freeChunksLock.Unlock() freeChunks = nil - for _, data := range baseChunks { - baseChunk := data + for _, baseChunk := range baseChunks { if err := unix.Munmap(baseChunk); err != nil { return err }