fix: support customize cache db for business (#19184)

Support to configure the customized redis db for cache layer and other
misc business for core, by default the behavior is same with
previous(stored in db 0).

Signed-off-by: chlins <chenyuzh@vmware.com>

make/harbor.yml.tmpl

@@ -174,6 +174,17 @@ _version: 2.8.0
 #     password: notary_server_db_password
 #     ssl_mode: disable
 
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+
 # Uncomment external_redis if using external Redis server
 # external_redis:
 #   # support redis, redis+sentinel
@@ -191,6 +202,10 @@ _version: 2.8.0
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 
 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
 # uaa:

make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja

@@ -381,6 +381,49 @@ external_database:
 #     ssl_mode: disable
 {% endif %}
 
+{% if redis is defined %}
+redis:
+#   # db_index 0 is for core, it's unchangeable
+{% if redis.registry_db_index is defined %}
+  registry_db_index: {{ redis.registry_db_index }}
+{% else %}
+#   # registry_db_index: 1
+{% endif %}
+{% if redis.jobservice_db_index is defined %}
+  jobservice_db_index: {{ redis.jobservice_db_index }}
+{% else %}
+#   # jobservice_db_index: 2
+{% endif %}
+{% if redis.trivy_db_index is defined %}
+  trivy_db_index: {{ redis.trivy_db_index }}
+{% else %}
+#   # trivy_db_index: 5
+{% endif %}
+{% if redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+{% else %}
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+{% endif %}
+{% if redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+{% else %}
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+{% else %}
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+
 {% if external_redis is defined %}
 external_redis:
   # support redis, redis+sentinel
@@ -402,6 +445,19 @@ external_redis:
   jobservice_db_index: {{ external_redis.jobservice_db_index }}
   trivy_db_index: 5
   idle_timeout_seconds: 30
+  {% if external_redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+  {% else %}
+# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+# # harbor_db_index: 6
+  {% endif %}
+  {% if external_redis.harbor_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+  {% else %}
+# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+# # cache_layer_db_index: 7
+  {% endif %}
+
 {% else %}
 # Umcomments external_redis if using external Redis server
 # external_redis:
@@ -420,6 +476,10 @@ external_redis:
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 {% endif %}
 
 {% if uaa is defined %}

make/photon/prepare/templates/core/env.jinja

@@ -1,6 +1,9 @@
 CONFIG_PATH=/etc/core/app.conf
 UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_harbor %}
+_REDIS_URL_HARBOR={{redis_url_harbor}}
+{% endif %}
 SYNC_QUOTA=true
 _REDIS_URL_REG={{redis_url_reg}}
 
@@ -86,6 +89,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 {% endif %}
 
 {% if cache.enabled %}
+{% if redis_url_cache_layer %}
+_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/templates/jobservice/env.jinja

@@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 
 {% if cache.enabled %}
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_cache_layer %}
+_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/utils/configs.py

@@ -306,7 +306,7 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy):
         config_dict['external_database'] = False
 
     # update redis configs
-    config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
+    config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
 
     # auto generated secret string for core
     config_dict['core_secret'] = generate_random_string(16)
@@ -400,7 +400,7 @@ def get_redis_url_param(redis=None):
     return ""
 
 
-def get_redis_configs(external_redis=None, with_trivy=True):
+def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
     """Returns configs for redis
 
     >>> get_redis_configs()['external_redis']
@@ -433,6 +433,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
     >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
     True
     """
+
+    internal_redis = internal_redis or {}
     external_redis = external_redis or {}
 
     configs = dict(external_redis=bool(external_redis))
@@ -447,13 +449,22 @@ def get_redis_configs(external_redis=None, with_trivy=True):
         'idle_timeout_seconds': 30,
     }
 
-    # overwriting existing keys by external_redis
-    redis.update({key: value for (key, value) in external_redis.items() if value})
+    if len(internal_redis) > 0:
+        # overwriting existing keys by internal_redis
+        redis.update({key: value for (key, value) in internal_redis.items() if value})
+    else:
+        # overwriting existing keys by external_redis
+        redis.update({key: value for (key, value) in external_redis.items() if value})
 
     configs['redis_url_core'] = get_redis_url(0, redis)
     configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
     configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
 
+    if redis.get('harbor_db_index'):
+        configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
+    if redis.get('cache_layer_db_index'):
+        configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis)
+
     if with_trivy:
         configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
 

src/core/main.go

@@ -127,25 +127,35 @@ func main() {
 	web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
 	web.BConfig.MaxMemory = 1 << 35     // (32GB)
 	web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
-
-	redisURL := os.Getenv("_REDIS_URL_CORE")
-	if len(redisURL) > 0 {
-		u, err := url.Parse(redisURL)
+	// the core db used for beego session
+	redisCoreURL := os.Getenv("_REDIS_URL_CORE")
+	if len(redisCoreURL) > 0 {
+		_, err := url.Parse(redisCoreURL)
 		if err != nil {
-			panic("bad _REDIS_URL")
+			panic("bad _REDIS_URL_CORE")
 		}
-
+		// configure the beego session redis
 		web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
-		web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
+		web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL
+	}
 
-		log.Info("initializing cache ...")
-		if err := cache.Initialize(u.Scheme, redisURL); err != nil {
-			log.Fatalf("failed to initialize cache: %v", err)
-		}
-		// when config/db init function is called, the cache is not ready,
-		// enable config cache explicitly when the cache is ready
-		dbCfg.EnableConfigCache()
+	log.Info("initializing cache ...")
+	// the harbor db used for harbor business, use core db if not specified
+	redisHarborURL := os.Getenv("_REDIS_URL_HARBOR")
+	if redisHarborURL == "" {
+		redisHarborURL = redisCoreURL
 	}
+	u, err := url.Parse(redisHarborURL)
+	if err != nil {
+		panic("bad _REDIS_URL_HARBOR")
+	}
+	if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil {
+		log.Fatalf("failed to initialize cache: %v", err)
+	}
+	// when config/db init function is called, the cache is not ready,
+	// enable config cache explicitly when the cache is ready
+	dbCfg.EnableConfigCache()
+
 	web.AddTemplateExt("htm")
 
 	log.Info("initializing configurations...")

src/jobservice/main.go

@@ -19,8 +19,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"net/url"
-	"os"
 
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/jobservice/common/utils"
@@ -29,7 +27,6 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/job/impl"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	"github.com/goharbor/harbor/src/jobservice/runtime"
-	"github.com/goharbor/harbor/src/lib/cache"
 	cfgLib "github.com/goharbor/harbor/src/lib/config"
 	tracelib "github.com/goharbor/harbor/src/lib/trace"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
@@ -45,21 +42,6 @@ func main() {
 		panic(fmt.Sprintf("failed to load configuration, error: %v", err))
 	}
 
-	// init cache if cache layer enabled
-	// gc needs to delete artifact by artifact manager, but the artifact cache store in
-	// core redis db so here require core redis url and init default cache.
-	if cfgLib.CacheEnabled() {
-		cacheURL := os.Getenv("_REDIS_URL_CORE")
-		u, err := url.Parse(cacheURL)
-		if err != nil {
-			panic("bad _REDIS_URL_CORE")
-		}
-
-		if err = cache.Initialize(u.Scheme, cacheURL); err != nil {
-			panic(fmt.Sprintf("failed to initialize cache: %v", err))
-		}
-	}
-
 	// Get parameters
 	configPath := flag.String("c", "", "Specify the yaml config file path")
 	flag.Parse()

src/lib/cache/cache.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"os"
 	"sync"
 	"time"
 
@@ -137,3 +138,37 @@ func Initialize(typ, addr string) error {
 func Default() Cache {
 	return cache
 }
+
+var (
+	// cacheLayer is the global cache layer cache instance.
+	cacheLayer Cache
+	// cacheLayerOnce is the once condition for initializing instance.
+	cacheLayerOnce sync.Once
+)
+
+// LayerCache is the global cache instance for cache layer.
+func LayerCache() Cache {
+	// parse the redis url for cache layer, use the default cache if not specify
+	redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER")
+	if redisCacheURL == "" {
+		if cache != nil {
+			return cache
+		}
+		// use the core url if cache layer url not found
+		redisCacheURL = os.Getenv("_REDIS_URL_CORE")
+	}
+
+	u, err := url.Parse(redisCacheURL)
+	if err != nil {
+		log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER")
+	}
+
+	cacheLayerOnce.Do(func() {
+		cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:"))
+		if err != nil {
+			log.Fatalf("failed to initialize cache for cache layer, err: %v", err)
+		}
+	})
+
+	return cacheLayer
+}

src/pkg/cached/base_manager.go

@@ -24,27 +24,27 @@ import (
 )
 
 // innerCache is the default cache client,
-// actually it is a wrapper for cache.Default().
+// actually it is a wrapper for cache.LayerCache().
 var innerCache cache.Cache = &cacheClient{}
 
-// cacheClient is a interceptor for cache.Default, in order to implement specific
+// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific
 // case for cache layer.
 type cacheClient struct{}
 
 func (*cacheClient) Contains(ctx context.Context, key string) bool {
-	return cache.Default().Contains(ctx, key)
+	return cache.LayerCache().Contains(ctx, key)
 }
 
 func (*cacheClient) Delete(ctx context.Context, key string) error {
-	return cache.Default().Delete(ctx, key)
+	return cache.LayerCache().Delete(ctx, key)
 }
 
 func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
-	return cache.Default().Fetch(ctx, key, value)
+	return cache.LayerCache().Fetch(ctx, key, value)
 }
 
 func (*cacheClient) Ping(ctx context.Context) error {
-	return cache.Default().Ping(ctx)
+	return cache.LayerCache().Ping(ctx)
 }
 
 func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
@@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp
 		return nil
 	}
 
-	return cache.Default().Save(ctx, key, value, expiration...)
+	return cache.LayerCache().Save(ctx, key, value, expiration...)
 }
 
 func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
-	return cache.Default().Scan(ctx, match)
+	return cache.LayerCache().Scan(ctx, match)
 }
 
 var _ Manager = &BaseManager{}