fix: support customize cache db for business (#19182)

Support to configure the customized redis db for cache layer and other
misc business for core, by default the behavior is same with
previous(stored in db 0).

Signed-off-by: chlins <chenyuzh@vmware.com>

make/harbor.yml.tmpl

@@ -167,6 +167,28 @@ _version: 2.9.0
 #     max_idle_conns: 2
 #     max_open_conns: 0
 
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_db_index: 7
+
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+
 # Uncomment external_redis if using external Redis server
 # external_redis:
 #   # support redis, redis+sentinel
@@ -186,6 +208,10 @@ _version: 2.9.0
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 
 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
 # uaa:

make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja

@@ -371,6 +371,49 @@ external_database:
 #     ssl_mode: disable
 {% endif %}
 
+{% if redis is defined %}
+redis:
+#   # db_index 0 is for core, it's unchangeable
+{% if redis.registry_db_index is defined %}
+  registry_db_index: {{ redis.registry_db_index }}
+{% else %}
+#   # registry_db_index: 1
+{% endif %}
+{% if redis.jobservice_db_index is defined %}
+  jobservice_db_index: {{ redis.jobservice_db_index }}
+{% else %}
+#   # jobservice_db_index: 2
+{% endif %}
+{% if redis.trivy_db_index is defined %}
+  trivy_db_index: {{ redis.trivy_db_index }}
+{% else %}
+#   # trivy_db_index: 5
+{% endif %}
+{% if redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+{% else %}
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+{% endif %}
+{% if redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+{% else %}
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+{% else %}
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+
 {% if external_redis is defined %}
 external_redis:
   # support redis, redis+sentinel
@@ -390,6 +433,18 @@ external_redis:
   jobservice_db_index: {{ external_redis.jobservice_db_index }}
   trivy_db_index: 5
   idle_timeout_seconds: 30
+  {% if external_redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+  {% else %}
+# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+# # harbor_db_index: 6
+  {% endif %}
+  {% if external_redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+  {% else %}
+# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+# # cache_layer_db_index: 7
+  {% endif %}
 {% else %}
 # Umcomments external_redis if using external Redis server
 # external_redis:
@@ -406,6 +461,10 @@ external_redis:
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 {% endif %}
 
 {% if uaa is defined %}

make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja

@@ -381,6 +381,49 @@ external_database:
 #     ssl_mode: disable
 {% endif %}
 
+{% if redis is defined %}
+redis:
+#   # db_index 0 is for core, it's unchangeable
+{% if redis.registry_db_index is defined %}
+  registry_db_index: {{ redis.registry_db_index }}
+{% else %}
+#   # registry_db_index: 1
+{% endif %}
+{% if redis.jobservice_db_index is defined %}
+  jobservice_db_index: {{ redis.jobservice_db_index }}
+{% else %}
+#   # jobservice_db_index: 2
+{% endif %}
+{% if redis.trivy_db_index is defined %}
+  trivy_db_index: {{ redis.trivy_db_index }}
+{% else %}
+#   # trivy_db_index: 5
+{% endif %}
+{% if redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+{% else %}
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+{% endif %}
+{% if redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+{% else %}
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+{% else %}
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+
 {% if external_redis is defined %}
 external_redis:
   # support redis, redis+sentinel
@@ -406,6 +449,18 @@ external_redis:
   jobservice_db_index: {{ external_redis.jobservice_db_index }}
   trivy_db_index: 5
   idle_timeout_seconds: 30
+  {% if external_redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+  {% else %}
+# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+# # harbor_db_index: 6
+  {% endif %}
+  {% if external_redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+  {% else %}
+# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+# # cache_layer_db_index: 7
+  {% endif %}
 {% else %}
 # Umcomments external_redis if using external Redis server
 # external_redis:
@@ -424,6 +479,10 @@ external_redis:
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 {% endif %}
 
 {% if uaa is defined %}

make/photon/prepare/migrations/version_2_9_0/harbor.yml.jinja

@@ -378,6 +378,49 @@ external_database:
 #     max_open_conns: 0
 {% endif %}
 
+{% if redis is defined %}
+redis:
+#   # db_index 0 is for core, it's unchangeable
+{% if redis.registry_db_index is defined %}
+  registry_db_index: {{ redis.registry_db_index }}
+{% else %}
+#   # registry_db_index: 1
+{% endif %}
+{% if redis.jobservice_db_index is defined %}
+  jobservice_db_index: {{ redis.jobservice_db_index }}
+{% else %}
+#   # jobservice_db_index: 2
+{% endif %}
+{% if redis.trivy_db_index is defined %}
+  trivy_db_index: {{ redis.trivy_db_index }}
+{% else %}
+#   # trivy_db_index: 5
+{% endif %}
+{% if redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+{% else %}
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+{% endif %}
+{% if redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+{% else %}
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+{% else %}
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+
 {% if external_redis is defined %}
 external_redis:
   # support redis, redis+sentinel
@@ -399,6 +442,18 @@ external_redis:
   jobservice_db_index: {{ external_redis.jobservice_db_index }}
   trivy_db_index: 5
   idle_timeout_seconds: 30
+  {% if external_redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+  {% else %}
+# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+# # harbor_db_index: 6
+  {% endif %}
+  {% if external_redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+  {% else %}
+# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+# # cache_layer_db_index: 7
+  {% endif %}
 {% else %}
 # Umcomments external_redis if using external Redis server
 # external_redis:
@@ -417,6 +472,10 @@ external_redis:
 #   jobservice_db_index: 2
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 {% endif %}
 
 {% if uaa is defined %}

make/photon/prepare/templates/core/env.jinja

@@ -1,6 +1,9 @@
 CONFIG_PATH=/etc/core/app.conf
 UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_harbor %}
+_REDIS_URL_HARBOR={{redis_url_harbor}}
+{% endif %}
 SYNC_QUOTA=true
 _REDIS_URL_REG={{redis_url_reg}}
 
@@ -84,6 +87,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 {% endif %}
 
 {% if cache.enabled %}
+{% if redis_url_cache_layer %}
+_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/templates/jobservice/env.jinja

@@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 
 {% if cache.enabled %}
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_cache_layer %}
+_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/utils/configs.py

@@ -275,7 +275,7 @@ def parse_yaml_config(config_file_path, with_trivy):
         config_dict['external_database'] = False
 
     # update redis configs
-    config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
+    config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
 
     # auto generated secret string for core
     config_dict['core_secret'] = generate_random_string(16)
@@ -371,7 +371,7 @@ def get_redis_url_param(redis=None):
     return ""
 
 
-def get_redis_configs(external_redis=None, with_trivy=True):
+def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
     """Returns configs for redis
 
     >>> get_redis_configs()['external_redis']
@@ -404,6 +404,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
     >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
     True
     """
+
+    internal_redis = internal_redis or {}
     external_redis = external_redis or {}
 
     configs = dict(external_redis=bool(external_redis))
@@ -418,13 +420,22 @@ def get_redis_configs(external_redis=None, with_trivy=True):
         'idle_timeout_seconds': 30,
     }
 
-    # overwriting existing keys by external_redis
-    redis.update({key: value for (key, value) in external_redis.items() if value})
+    if len(internal_redis) > 0:
+        # overwriting existing keys by internal_redis
+        redis.update({key: value for (key, value) in internal_redis.items() if value})
+    else:
+        # overwriting existing keys by external_redis
+        redis.update({key: value for (key, value) in external_redis.items() if value})
 
     configs['redis_url_core'] = get_redis_url(0, redis)
     configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
     configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
 
+    if redis.get('harbor_db_index'):
+        configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
+    if redis.get('cache_layer_db_index'):
+        configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis)
+
     if with_trivy:
         configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
 

src/controller/quota/controller.go

@@ -262,7 +262,7 @@ func (c *controller) updateUsageByRedis(ctx context.Context, reference, referenc
 		return retry.Abort(ctx.Err())
 	}
 
-	client, err := libredis.GetCoreClient()
+	client, err := libredis.GetHarborClient()
 	if err != nil {
 		return retry.Abort(err)
 	}