Merge pull request #25 from TencentBlueKing/develop

v1.8.6

VERSION

@@ -1 +1 @@
-1.8.5
+1.8.6

build/support-files/sql/0015_iam_20210909-1800_mysql.sql

@@ -0,0 +1 @@
+ALTER TABLE `bkiam`.`policy` ADD INDEX `idx_update_at` (`updated_at`);

pkg/api/policy/handler/util_test.go

@@ -323,7 +323,7 @@ func Test_validateSystemMatchClient(t *testing.T) {
 		return []string{"test"}, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	impls.LocalSystemClientsCache = mockCache
 
 	type args struct {

pkg/cache/impls/init.go

@@ -12,14 +12,15 @@ package impls
 
 import (
 	"errors"
+	"math/rand"
 	"time"
 
 	gocache "github.com/patrickmn/go-cache"
-
 	log "github.com/sirupsen/logrus"
 
 	"iam/pkg/cache/cleaner"
 	"iam/pkg/cache/memory"
+	"iam/pkg/cache/memory/backend"
 	"iam/pkg/cache/redis"
 )
 
@@ -63,6 +64,12 @@ var (
 // ErrNotExceptedTypeFromCache ...
 var ErrNotExceptedTypeFromCache = errors.New("not expected type from cache")
 
+func newRandomDuration(seconds int) backend.RandomExpirationDurationFunc {
+	return func() time.Duration {
+		return time.Duration(rand.Intn(seconds*1000)) * time.Millisecond
+	}
+}
+
 // Cache should only know about get/retrieve data
 // ! DO NOT CARE ABOUT WHAT THE DATA WILL BE USED FOR
 func InitCaches(disabled bool) {
@@ -71,62 +78,87 @@ func InitCaches(disabled bool) {
 		disabled,
 		retrieveAppCodeAppSecret,
 		12*time.Hour,
+		nil,
 	)
 
+	// 影响: engine增量同步
+
 	LocalSubjectCache = memory.NewCache(
 		"local_subject",
 		disabled,
 		retrieveSubject,
 		1*time.Minute,
+		newRandomDuration(30),
 	)
 
-	LocalSubjectRoleCache = memory.NewCache(
-		"local_subject_role",
-		disabled,
-		retrieveSubjectRole,
-		1*time.Minute,
-	)
+	// 影响: job查询cmdb的资源进行鉴权
 
 	LocalRemoteResourceListCache = memory.NewCache(
 		"local_remote_resource_list",
 		disabled,
 		retrieveRemoteResourceList,
 		30*time.Second,
+		newRandomDuration(10),
 	)
 
+	// 影响: 每次鉴权
+
 	LocalSubjectPKCache = memory.NewCache(
 		"local_subject_pk",
 		disabled,
 		retrieveSubjectPK,
 		1*time.Minute,
+		newRandomDuration(30),
+	)
+
+	// 影响: 每次鉴权
+
+	LocalSubjectRoleCache = memory.NewCache(
+		"local_subject_role",
+		disabled,
+		retrieveSubjectRole,
+		1*time.Minute,
+		newRandomDuration(30),
 	)
 
+	// 影响: 每次鉴权 => system_id比较集中, singleflight可以防止大的并发落db
+
 	LocalSystemClientsCache = memory.NewCache(
 		"local_system_clients",
 		disabled,
 		retrieveSystemClients,
 		1*time.Minute,
+		newRandomDuration(30),
 	)
 
-	LocalAPIGatewayJWTClientIDCache = memory.NewCache(
-		"local_apigw_jwt_client_id",
-		disabled,
-		retrieveAPIGatewayJWTClientID,
-		30*time.Second,
-	)
+	// 影响: engine接口/policy查询接口
 
 	LocalActionCache = memory.NewCache(
 		"local_action",
 		disabled,
 		retrieveAction,
 		30*time.Minute,
+		newRandomDuration(30),
 	)
 
+	// 无影响, 重算而已不查db
+
+	LocalAPIGatewayJWTClientIDCache = memory.NewCache(
+		"local_apigw_jwt_client_id",
+		disabled,
+		retrieveAPIGatewayJWTClientID,
+		30*time.Second,
+		nil,
+	)
+
+	// 无影响, 重算而已不查db
+
 	LocalUnmarshaledExpressionCache = memory.NewCache(
 		"local_unmarshaled_expression",
 		disabled,
 		UnmarshalExpression,
 		30*time.Minute,
+		nil,
 	)
 
 	//  ==========================

pkg/cache/impls/local_action_test.go

@@ -37,7 +37,7 @@ func TestGetAction(t *testing.T) {
 		return svctypes.ThinAction{}, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalActionCache = mockCache
 
 	_, err := GetAction(1)
@@ -48,7 +48,7 @@ func TestGetAction(t *testing.T) {
 		return false, errors.New("error here")
 	}
 	mockCache = memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalActionCache = mockCache
 
 	_, err = GetAction(1)

pkg/cache/impls/local_apigw_jwt_client_id_test.go

@@ -51,7 +51,7 @@ var _ = Describe("LocalApigwJwtClientId", func() {
 				return true, nil
 			}
 			mockCache := memory.NewCache(
-				"mockCache", false, retrieveFunc, expiration)
+				"mockCache", false, retrieveFunc, expiration, nil)
 			LocalAPIGatewayJWTClientIDCache = mockCache
 		})
 

pkg/cache/impls/local_app_code_secret_test.go

@@ -39,7 +39,7 @@ func TestVerifyAppCodeAppSecret(t *testing.T) {
 		return true, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalAppCodeAppSecretCache = mockCache
 	assert.True(t, VerifyAppCodeAppSecret("test", "123"))
 
@@ -48,7 +48,7 @@ func TestVerifyAppCodeAppSecret(t *testing.T) {
 		return false, errors.New("error here")
 	}
 	mockCache = memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalAppCodeAppSecretCache = mockCache
 	assert.False(t, VerifyAppCodeAppSecret("test", "123"))
 }

pkg/cache/impls/local_remote_resource_list_test.go

@@ -43,7 +43,7 @@ func TestListRemoteResources(t *testing.T) {
 		return []map[string]interface{}{}, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalRemoteResourceListCache = mockCache
 
 	_, err := ListRemoteResources("test", "app", []string{"1", "2"}, []string{"id", "name"})
@@ -54,7 +54,7 @@ func TestListRemoteResources(t *testing.T) {
 		return false, errors.New("error here")
 	}
 	mockCache = memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalRemoteResourceListCache = mockCache
 
 	_, err = ListRemoteResources("test", "app", []string{"1", "2"}, []string{"id", "name"})

pkg/cache/impls/local_subject_role_test.go

@@ -39,7 +39,7 @@ func TestSubjectListSubjectRoleSystemID(t *testing.T) {
 		return []string{"bk_cmdb", "bk_job"}, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalSubjectRoleCache = mockCache
 
 	systemIDs, err := ListSubjectRoleSystemID("user", "admin")
@@ -51,7 +51,7 @@ func TestSubjectListSubjectRoleSystemID(t *testing.T) {
 		return []string{}, errors.New("error here")
 	}
 	mockCache = memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalSubjectRoleCache = mockCache
 
 	_, err = ListSubjectRoleSystemID("user", "admin")

pkg/cache/impls/local_subject_test.go

@@ -32,7 +32,7 @@ func TestGetSubjectByPK(t *testing.T) {
 		return svctypes.Subject{}, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalSubjectCache = mockCache
 
 	_, err := GetSubjectByPK(1)
@@ -43,7 +43,7 @@ func TestGetSubjectByPK(t *testing.T) {
 		return false, errors.New("error here")
 	}
 	mockCache = memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalSubjectCache = mockCache
 
 	_, err = GetSubjectByPK(1)

pkg/cache/impls/local_system_clients_test.go

@@ -31,7 +31,7 @@ func TestGetSystemClients(t *testing.T) {
 		return []string{"a", "b", "c"}, nil
 	}
 	mockCache := memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalSystemClientsCache = mockCache
 
 	cs, err := GetSystemClients("x")
@@ -46,7 +46,7 @@ func TestGetSystemClients(t *testing.T) {
 		return []string{}, errors.New("error here")
 	}
 	mockCache = memory.NewCache(
-		"mockCache", false, retrieveFunc, expiration)
+		"mockCache", false, retrieveFunc, expiration, nil)
 	LocalSystemClientsCache = mockCache
 
 	_, err = GetSystemClients("x")

pkg/cache/memory/backend/memory.go

@@ -21,6 +21,8 @@ const (
 	DefaultCleanupInterval = 5 * time.Minute
 )
 
+type RandomExpirationDurationFunc func() time.Duration
+
 // NewTTLCache create cache with expiration and cleanup interval,
 // if cleanupInterval is 0, will use DefaultCleanupInterval
 func newTTLCache(expiration time.Duration, cleanupInterval time.Duration) *gocache.Cache {
@@ -36,7 +38,8 @@ type MemoryBackend struct {
 	name  string
 	cache *gocache.Cache
 
-	defaultExpiration time.Duration
+	defaultExpiration  time.Duration
+	randomDurationFunc RandomExpirationDurationFunc
 }
 
 // Set ...
@@ -45,6 +48,10 @@ func (c *MemoryBackend) Set(key string, value interface{}, duration time.Duratio
 		duration = c.defaultExpiration
 	}
 
+	if c.randomDurationFunc != nil {
+		duration += c.randomDurationFunc()
+	}
+
 	c.cache.Set(key, value, duration)
 }
 
@@ -65,12 +72,17 @@ func (c *MemoryBackend) Delete(key string) error {
 }
 
 // NewMemoryBackend ...
-func NewMemoryBackend(name string, expiration time.Duration) *MemoryBackend {
+func NewMemoryBackend(
+	name string,
+	expiration time.Duration,
+	randomDurationFunc RandomExpirationDurationFunc,
+) *MemoryBackend {
 	cleanupInterval := expiration + (5 * time.Minute)
 
 	return &MemoryBackend{
-		name:              name,
-		cache:             newTTLCache(expiration, cleanupInterval),
-		defaultExpiration: expiration,
+		name:               name,
+		cache:              newTTLCache(expiration, cleanupInterval),
+		defaultExpiration:  expiration,
+		randomDurationFunc: randomDurationFunc,
 	}
 }

pkg/cache/memory/backend/memory_test.go

@@ -26,7 +26,7 @@ func TestNewTTLCache(t *testing.T) {
 }
 
 func TestMemoryBackend(t *testing.T) {
-	be := NewMemoryBackend("test", 5*time.Second)
+	be := NewMemoryBackend("test", 5*time.Second, nil)
 	assert.NotNil(t, be)
 
 	_, found := be.Get("not_exists")

pkg/cache/memory/base_cache_test.go

@@ -48,7 +48,7 @@ func retrieveError(k cache.Key) (interface{}, error) {
 func TestNewBaseCache(t *testing.T) {
 	expiration := 5 * time.Minute
 
-	be := backend.NewMemoryBackend("test", expiration)
+	be := backend.NewMemoryBackend("test", expiration, nil)
 
 	c := NewBaseCache(false, retrieveTest, be)
 

pkg/cache/memory/cache.go

@@ -17,15 +17,18 @@ import (
 )
 
 // NewCache create a memory cache
-func NewCache(name string, disabled bool, retrieveFunc RetrieveFunc,
-	expiration time.Duration) Cache {
-	be := backend.NewMemoryBackend(name, expiration)
+func NewCache(name string, disabled bool,
+	retrieveFunc RetrieveFunc,
+	expiration time.Duration,
+	randomDurationFunc backend.RandomExpirationDurationFunc,
+) Cache {
+	be := backend.NewMemoryBackend(name, expiration, randomDurationFunc)
 	return NewBaseCache(disabled, retrieveFunc, be)
 }
 
 // NewMockCache create a memory cache for mock
 func NewMockCache(retrieveFunc RetrieveFunc) Cache {
-	be := backend.NewMemoryBackend("mockCache", 5*time.Minute)
+	be := backend.NewMemoryBackend("mockCache", 5*time.Minute, nil)
 
 	return NewBaseCache(false, retrieveFunc, be)
 }

pkg/cache/memory/cache_test.go

@@ -26,6 +26,6 @@ func retrieveOK(k cache.Key) (interface{}, error) {
 func TestNewCache(t *testing.T) {
 	expiration := 5 * time.Minute
 
-	c := NewCache("test", false, retrieveOK, expiration)
+	c := NewCache("test", false, retrieveOK, expiration, nil)
 	assert.NotNil(t, c)
 }

release.md

@@ -1,3 +1,7 @@
+# 1.8.6
+
+- add: local cache expire with a random duration
+
 # 1.8.5
 
 - upgrade: update the expression table structure, delete useless columns