From b360cec6c3860fef778269bc14d9e5274fe0f5fe Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 5 Aug 2021 23:44:25 +0800 Subject: [PATCH 1/3] fix(lock):#294 fixed memory leak issue --- code/go/0chain.net/core/lock/lock.go | 65 +++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/core/lock/lock.go b/code/go/0chain.net/core/lock/lock.go index 933b85e30..7aba3cfea 100644 --- a/code/go/0chain.net/core/lock/lock.go +++ b/code/go/0chain.net/core/lock/lock.go @@ -1,17 +1,70 @@ package lock -import "sync" +import ( + "sync" + "time" +) -var lockPool = make(map[string]*sync.Mutex, 0) //nolint:gosimple // need more time to verify: declaring capacity is probably necessary? -var lockMutex = &sync.Mutex{} +var ( + lockPool = make(map[string]*Mutex) + unlockPool = make(map[string]time.Time) + lockMutex sync.Mutex +) -func GetMutex(tablename string, key string) *sync.Mutex { +// Mutex a mutual exclusion lock. +type Mutex struct { + // key lock key in pool + key string + sync.Mutex +} + +// Lock implements Locker.Lock +func (m *Mutex) Lock() { + m.Mutex.Lock() +} + +// Unlock implements Locker.Unlock, and mark mutex as unlock object +func (m *Mutex) Unlock() { + lockMutex.Lock() + defer lockMutex.Unlock() + + m.Mutex.Unlock() + //mark it as unlock object, it will be deleted in clean worker + unlockPool[m.key] = time.Now() +} + +// GetMutex get mutex by table and key +func GetMutex(tablename string, key string) *Mutex { lockKey := tablename + ":" + key lockMutex.Lock() defer lockMutex.Unlock() if eLock, ok := lockPool[lockKey]; ok { + // do NOT remove it from pool + delete(unlockPool, lockKey) return eLock } - lockPool[lockKey] = &sync.Mutex{} - return lockPool[lockKey] + + m := &Mutex{key: lockKey} + lockPool[lockKey] = m + + return m +} + +func init() { + go startLockPoolCleaner() +} + +func startLockPoolCleaner() { + for { + time.Sleep(1 * time.Hour) + + lockMutex.Lock() + + for key := range unlockPool { + delete(lockPool, key) + } + + unlockPool = make(map[string]time.Time) + lockMutex.Unlock() + } } From 9b1b388887a41df06f17fe18a0b262ea650034d1 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 17 Aug 2021 00:10:22 +0800 Subject: [PATCH 2/3] fix(lock):#294 updated clean works with counter;added unit tests --- code/go/0chain.net/core/lock/lock.go | 48 ++++++++++++++--------- code/go/0chain.net/core/lock/lock_test.go | 43 ++++++++++++++++++++ 2 files changed, 73 insertions(+), 18 deletions(-) create mode 100644 code/go/0chain.net/core/lock/lock_test.go diff --git a/code/go/0chain.net/core/lock/lock.go b/code/go/0chain.net/core/lock/lock.go index 7aba3cfea..dd03029c0 100644 --- a/code/go/0chain.net/core/lock/lock.go +++ b/code/go/0chain.net/core/lock/lock.go @@ -6,15 +6,21 @@ import ( ) var ( - lockPool = make(map[string]*Mutex) - unlockPool = make(map[string]time.Time) - lockMutex sync.Mutex + // MutexCleanInterval start to clean unsed mutex at specified interval + MutexCleanInterval = 10 * time.Minute + + lockPool = make(map[string]*Mutex) + counters = make(map[string]int) + lockMutex sync.Mutex ) // Mutex a mutual exclusion lock. type Mutex struct { // key lock key in pool key string + // usedby how objects it is used by + usedby int + sync.Mutex } @@ -28,43 +34,49 @@ func (m *Mutex) Unlock() { lockMutex.Lock() defer lockMutex.Unlock() + m.usedby-- m.Mutex.Unlock() - //mark it as unlock object, it will be deleted in clean worker - unlockPool[m.key] = time.Now() } // GetMutex get mutex by table and key func GetMutex(tablename string, key string) *Mutex { lockKey := tablename + ":" + key lockMutex.Lock() + defer lockMutex.Unlock() if eLock, ok := lockPool[lockKey]; ok { - // do NOT remove it from pool - delete(unlockPool, lockKey) + eLock.usedby++ return eLock } - m := &Mutex{key: lockKey} + m := &Mutex{key: lockKey, usedby: 1} + lockPool[lockKey] = m return m } func init() { - go startLockPoolCleaner() + go startWorker() } -func startLockPoolCleaner() { - for { - time.Sleep(1 * time.Hour) - - lockMutex.Lock() +func cleanUnusedMutexs() { + lockMutex.Lock() - for key := range unlockPool { - delete(lockPool, key) + for k, v := range lockPool { + if v.usedby < 1 { + delete(lockPool, k) } + } + + lockMutex.Unlock() +} + +func startWorker() { + for { + time.Sleep(MutexCleanInterval) + + cleanUnusedMutexs() - unlockPool = make(map[string]time.Time) - lockMutex.Unlock() } } diff --git a/code/go/0chain.net/core/lock/lock_test.go b/code/go/0chain.net/core/lock/lock_test.go new file mode 100644 index 000000000..d78eaff07 --- /dev/null +++ b/code/go/0chain.net/core/lock/lock_test.go @@ -0,0 +1,43 @@ +package lock + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLock(t *testing.T) { + + max := 100 + + for i := 0; i < max; i++ { + + lock1 := GetMutex("testlock", strconv.Itoa(i)) + + lock1.Lock() + + require.Equal(t, 1, lock1.usedby) + + lock1.Unlock() + + require.Equal(t, 0, lock1.usedby) + + lock2 := GetMutex("testlock", strconv.Itoa(i)) + lock2.Lock() + + require.Equal(t, 1, lock2.usedby) + + lock2.Unlock() + + require.Equal(t, 0, lock2.usedby) + } + + cleanUnusedMutexs() + + for i := 0; i < max; i++ { + _, ok := lockPool["testlock:"+strconv.Itoa(i)] + require.Equal(t, false, ok) + } + +} From 779f89ea1ca85d47fd93aea176ad14ed3c6e7093 Mon Sep 17 00:00:00 2001 From: Lz Date: Tue, 17 Aug 2021 00:12:08 +0800 Subject: [PATCH 3/3] fix(lock):#294 fixed golangci-lint issue --- code/go/0chain.net/core/lock/lock.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/core/lock/lock.go b/code/go/0chain.net/core/lock/lock.go index dd03029c0..25ca8ea9a 100644 --- a/code/go/0chain.net/core/lock/lock.go +++ b/code/go/0chain.net/core/lock/lock.go @@ -8,9 +8,10 @@ import ( var ( // MutexCleanInterval start to clean unsed mutex at specified interval MutexCleanInterval = 10 * time.Minute +) +var ( lockPool = make(map[string]*Mutex) - counters = make(map[string]int) lockMutex sync.Mutex )