From ad3e767d511d45a3f8f671ca3219402ccc4bf750 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Tue, 15 Aug 2023 17:24:47 +0800 Subject: [PATCH] fix: support customize cache db for business (#19184) Support to configure the customized redis db for cache layer and other misc business for core, by default the behavior is same with previous(stored in db 0). Signed-off-by: chlins --- make/harbor.yml.tmpl | 15 +++++ .../migrations/version_2_8_0/harbor.yml.jinja | 60 +++++++++++++++++++ make/photon/prepare/templates/core/env.jinja | 6 ++ .../prepare/templates/jobservice/env.jinja | 3 + make/photon/prepare/utils/configs.py | 19 ++++-- src/core/main.go | 38 +++++++----- src/jobservice/main.go | 18 ------ src/lib/cache/cache.go | 35 +++++++++++ src/pkg/cached/base_manager.go | 16 ++--- 9 files changed, 166 insertions(+), 44 deletions(-) diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index d217d56d704..078df55af5b 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -174,6 +174,17 @@ _version: 2.8.0 # password: notary_server_db_password # ssl_mode: disable +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + # Uncomment external_redis if using external Redis server # external_redis: # # support redis, redis+sentinel @@ -191,6 +202,10 @@ _version: 2.8.0 # jobservice_db_index: 2 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: diff --git a/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja index be93c47866c..b3afd3549aa 100644 --- a/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja @@ -381,6 +381,49 @@ external_database: # ssl_mode: disable {% endif %} +{% if redis is defined %} +redis: +# # db_index 0 is for core, it's unchangeable +{% if redis.registry_db_index is defined %} + registry_db_index: {{ redis.registry_db_index }} +{% else %} +# # registry_db_index: 1 +{% endif %} +{% if redis.jobservice_db_index is defined %} + jobservice_db_index: {{ redis.jobservice_db_index }} +{% else %} +# # jobservice_db_index: 2 +{% endif %} +{% if redis.trivy_db_index is defined %} + trivy_db_index: {{ redis.trivy_db_index }} +{% else %} +# # trivy_db_index: 5 +{% endif %} +{% if redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} +{% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +{% endif %} +{% if redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} +{% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} +{% else %} +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} + {% if external_redis is defined %} external_redis: # support redis, redis+sentinel @@ -402,6 +445,19 @@ external_redis: jobservice_db_index: {{ external_redis.jobservice_db_index }} trivy_db_index: 5 idle_timeout_seconds: 30 + {% if external_redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} + {% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 + {% endif %} + {% if external_redis.harbor_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} + {% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + {% endif %} + {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -420,6 +476,10 @@ external_redis: # jobservice_db_index: 2 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 {% endif %} {% if uaa is defined %} diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index 28f38d43f8a..04b1c92548f 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -1,6 +1,9 @@ CONFIG_PATH=/etc/core/app.conf UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_harbor %} +_REDIS_URL_HARBOR={{redis_url_harbor}} +{% endif %} SYNC_QUOTA=true _REDIS_URL_REG={{redis_url_reg}} @@ -86,6 +89,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% endif %} {% if cache.enabled %} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index 563d99e51f2..3cfa9e32dda 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% if cache.enabled %} _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} \ No newline at end of file diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index f05042a2634..f35ae7cd820 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -306,7 +306,7 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy): config_dict['external_database'] = False # update redis configs - config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy)) + config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy)) # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) @@ -400,7 +400,7 @@ def get_redis_url_param(redis=None): return "" -def get_redis_configs(external_redis=None, with_trivy=True): +def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True): """Returns configs for redis >>> get_redis_configs()['external_redis'] @@ -433,6 +433,8 @@ def get_redis_configs(external_redis=None, with_trivy=True): >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False) True """ + + internal_redis = internal_redis or {} external_redis = external_redis or {} configs = dict(external_redis=bool(external_redis)) @@ -447,13 +449,22 @@ def get_redis_configs(external_redis=None, with_trivy=True): 'idle_timeout_seconds': 30, } - # overwriting existing keys by external_redis - redis.update({key: value for (key, value) in external_redis.items() if value}) + if len(internal_redis) > 0: + # overwriting existing keys by internal_redis + redis.update({key: value for (key, value) in internal_redis.items() if value}) + else: + # overwriting existing keys by external_redis + redis.update({key: value for (key, value) in external_redis.items() if value}) configs['redis_url_core'] = get_redis_url(0, redis) configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis) configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis) + if redis.get('harbor_db_index'): + configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis) + if redis.get('cache_layer_db_index'): + configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis) + if with_trivy: configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis) diff --git a/src/core/main.go b/src/core/main.go index b121e7b69f7..e9e9f401b93 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -127,25 +127,35 @@ func main() { web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName web.BConfig.MaxMemory = 1 << 35 // (32GB) web.BConfig.MaxUploadSize = 1 << 35 // (32GB) - - redisURL := os.Getenv("_REDIS_URL_CORE") - if len(redisURL) > 0 { - u, err := url.Parse(redisURL) + // the core db used for beego session + redisCoreURL := os.Getenv("_REDIS_URL_CORE") + if len(redisCoreURL) > 0 { + _, err := url.Parse(redisCoreURL) if err != nil { - panic("bad _REDIS_URL") + panic("bad _REDIS_URL_CORE") } - + // configure the beego session redis web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName - web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL + web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL + } - log.Info("initializing cache ...") - if err := cache.Initialize(u.Scheme, redisURL); err != nil { - log.Fatalf("failed to initialize cache: %v", err) - } - // when config/db init function is called, the cache is not ready, - // enable config cache explicitly when the cache is ready - dbCfg.EnableConfigCache() + log.Info("initializing cache ...") + // the harbor db used for harbor business, use core db if not specified + redisHarborURL := os.Getenv("_REDIS_URL_HARBOR") + if redisHarborURL == "" { + redisHarborURL = redisCoreURL } + u, err := url.Parse(redisHarborURL) + if err != nil { + panic("bad _REDIS_URL_HARBOR") + } + if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil { + log.Fatalf("failed to initialize cache: %v", err) + } + // when config/db init function is called, the cache is not ready, + // enable config cache explicitly when the cache is ready + dbCfg.EnableConfigCache() + web.AddTemplateExt("htm") log.Info("initializing configurations...") diff --git a/src/jobservice/main.go b/src/jobservice/main.go index ff6b5793e14..03810d67a37 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -19,8 +19,6 @@ import ( "errors" "flag" "fmt" - "net/url" - "os" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/jobservice/common/utils" @@ -29,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/job/impl" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/runtime" - "github.com/goharbor/harbor/src/lib/cache" cfgLib "github.com/goharbor/harbor/src/lib/config" tracelib "github.com/goharbor/harbor/src/lib/trace" _ "github.com/goharbor/harbor/src/pkg/accessory/model/base" @@ -45,21 +42,6 @@ func main() { panic(fmt.Sprintf("failed to load configuration, error: %v", err)) } - // init cache if cache layer enabled - // gc needs to delete artifact by artifact manager, but the artifact cache store in - // core redis db so here require core redis url and init default cache. - if cfgLib.CacheEnabled() { - cacheURL := os.Getenv("_REDIS_URL_CORE") - u, err := url.Parse(cacheURL) - if err != nil { - panic("bad _REDIS_URL_CORE") - } - - if err = cache.Initialize(u.Scheme, cacheURL); err != nil { - panic(fmt.Sprintf("failed to initialize cache: %v", err)) - } - } - // Get parameters configPath := flag.String("c", "", "Specify the yaml config file path") flag.Parse() diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 1e0dae56cc6..02a7b09264c 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/url" + "os" "sync" "time" @@ -137,3 +138,37 @@ func Initialize(typ, addr string) error { func Default() Cache { return cache } + +var ( + // cacheLayer is the global cache layer cache instance. + cacheLayer Cache + // cacheLayerOnce is the once condition for initializing instance. + cacheLayerOnce sync.Once +) + +// LayerCache is the global cache instance for cache layer. +func LayerCache() Cache { + // parse the redis url for cache layer, use the default cache if not specify + redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER") + if redisCacheURL == "" { + if cache != nil { + return cache + } + // use the core url if cache layer url not found + redisCacheURL = os.Getenv("_REDIS_URL_CORE") + } + + u, err := url.Parse(redisCacheURL) + if err != nil { + log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER") + } + + cacheLayerOnce.Do(func() { + cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:")) + if err != nil { + log.Fatalf("failed to initialize cache for cache layer, err: %v", err) + } + }) + + return cacheLayer +} diff --git a/src/pkg/cached/base_manager.go b/src/pkg/cached/base_manager.go index bc810372849..d7bf1caddbc 100644 --- a/src/pkg/cached/base_manager.go +++ b/src/pkg/cached/base_manager.go @@ -24,27 +24,27 @@ import ( ) // innerCache is the default cache client, -// actually it is a wrapper for cache.Default(). +// actually it is a wrapper for cache.LayerCache(). var innerCache cache.Cache = &cacheClient{} -// cacheClient is a interceptor for cache.Default, in order to implement specific +// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific // case for cache layer. type cacheClient struct{} func (*cacheClient) Contains(ctx context.Context, key string) bool { - return cache.Default().Contains(ctx, key) + return cache.LayerCache().Contains(ctx, key) } func (*cacheClient) Delete(ctx context.Context, key string) error { - return cache.Default().Delete(ctx, key) + return cache.LayerCache().Delete(ctx, key) } func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error { - return cache.Default().Fetch(ctx, key, value) + return cache.LayerCache().Fetch(ctx, key, value) } func (*cacheClient) Ping(ctx context.Context) error { - return cache.Default().Ping(ctx) + return cache.LayerCache().Ping(ctx) } func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error { @@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp return nil } - return cache.Default().Save(ctx, key, value, expiration...) + return cache.LayerCache().Save(ctx, key, value, expiration...) } func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) { - return cache.Default().Scan(ctx, match) + return cache.LayerCache().Scan(ctx, match) } var _ Manager = &BaseManager{}