feat: add cache layer for artifact

Signed-off-by: chlins <chenyuzh@vmware.com>

make/harbor.yml.tmpl

@@ -245,3 +245,8 @@ upload_purging:
   # the interval of the purge operations
   interval: 24h
   dryrun: false
+
+# Cache related config
+cache:
+  enabled: false
+  expire_hours: 24

make/photon/prepare/models.py

@@ -7,6 +7,7 @@
 from utils.misc import check_permission, owner_can_read, get_realpath, port_number_valid
 from utils.cert import san_existed
 
+
 class InternalTLS:
 
     harbor_certs_filename = {
@@ -137,8 +138,9 @@ def prepare(self):
             else:
                 os.chown(file, DEFAULT_UID, DEFAULT_GID)
 
+
 class Metric:
-    def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics" ):
+    def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics"):
         self.enabled = enabled
         self.port = port
         self.path = path
@@ -166,6 +168,7 @@ def validate(self):
         if self.endpoint and self.agent_host:
             raise Exception('Jaeger Colector Endpoint and Agent host both set, only can set one')
 
+
 class OtelExporter:
     def __init__(self, config: dict):
         if not config:
@@ -184,6 +187,7 @@ def validate(self):
         if not self.url_path:
             raise Exception('Trace url path not set')
 
+
 class Trace:
     def __init__(self, config: dict):
         self.enabled = config.get('enabled') or False
@@ -205,6 +209,7 @@ def validate(self):
         elif self.otel.enabled:
             self.otel.validate()
 
+
 class PurgeUpload:
     def __init__(self, config: dict):
         if not config:
@@ -223,14 +228,30 @@ def validate(self):
             raise Exception('purge upload age should set with with nh, n is the number of hour')
         # interval should larger than 2h
         age = self.age[:-1]
-        if not age.isnumeric() or int(age) < 2 :
+        if not age.isnumeric() or int(age) < 2:
             raise Exception('purge upload age should set with with nh, n is the number of hour and n should not be less than 2')
-        
+
         # interval should end with h
         if not isinstance(self.interval, str) or not self.interval.endswith('h'):
             raise Exception('purge upload interval should set with with nh, n is the number of hour')
         # interval should larger than 2h
         interval = self.interval[:-1]
-        if not interval.isnumeric() or int(interval) < 2 :
+        if not interval.isnumeric() or int(interval) < 2:
             raise Exception('purge upload interval should set with with nh, n is the number of hour and n should not beless than 2')
         return
+
+
+class Cache:
+    def __init__(self, config: dict):
+        if not config:
+            self.enabled = False
+        self.enabled = config.get('enabled')
+        self.expire_hours = config.get('expire_hours')
+
+    def validate(self):
+        if not self.enabled:
+            return
+
+        if not self.expire_hours.isnumeric():
+            raise Exception('cache expire hours should be number')
+        return

make/photon/prepare/templates/core/env.jinja

@@ -84,3 +84,8 @@ TRACE_OTEL_TIMEOUT={{ trace.otel.timeout }}
 TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 {% endif %}
 {% endif %}
+
+{% if cache.enabled %}
+CACHE_ENABLED=true
+CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
+{% endif %}

make/photon/prepare/utils/configs.py

@@ -3,11 +3,13 @@
 import yaml
 from urllib.parse import urlencode
 from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
-from models import InternalTLS, Metric, Trace, PurgeUpload
+from models import InternalTLS, Metric, Trace, PurgeUpload, Cache
 from utils.misc import generate_random_string, owner_can_read, other_can_read
 
-default_db_max_idle_conns = 2  # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
-default_db_max_open_conns = 0  # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
+# NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
+default_db_max_idle_conns = 2
+# NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
+default_db_max_open_conns = 0
 default_https_cert_path = '/your/certificate/path'
 default_https_key_path = '/your/certificate/path'
 
@@ -49,7 +51,8 @@ def validate(conf: dict, **kwargs):
         raise Exception("Error: storage driver %s is not supported, only the following ones are supported: %s" % (
             storage_provider_name, ",".join(valid_storage_drivers)))
 
-    storage_provider_config = conf.get("storage_provider_config") ## original is registry_storage_provider_config
+    # original is registry_storage_provider_config
+    storage_provider_config = conf.get("storage_provider_config")
     if storage_provider_name != "filesystem":
         if storage_provider_config == "":
             raise Exception(
@@ -78,10 +81,14 @@ def validate(conf: dict, **kwargs):
 
     if conf.get('trace'):
         conf['trace'].validate()
-        
+
     if conf.get('purge_upload'):
         conf['purge_upload'].validate()
 
+    if conf.get('cache'):
+        conf['cache'].validate()
+
+
 def parse_versions():
     if not versions_file_path.is_file():
         return {}
@@ -168,7 +175,6 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
             config_dict['notary_server_db_password'] = 'password'
             config_dict['notary_server_db_sslmode'] = 'disable'
 
-
     # Data path volume
     config_dict['data_volume'] = configs['data_volume']
 
@@ -214,9 +220,9 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
         all_no_proxy |= set(no_proxy_config.split(','))
 
     for proxy_component in proxy_components:
-      config_dict[proxy_component + '_http_proxy'] = proxy_config.get('http_proxy') or ''
-      config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
-      config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
+        config_dict[proxy_component + '_http_proxy'] = proxy_config.get('http_proxy') or ''
+        config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
+        config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
 
     # Trivy configs, optional
     trivy_configs = configs.get("trivy") or {}
@@ -354,6 +360,10 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
     purge_upload_config = configs.get('upload_purging')
     config_dict['purge_upload'] = PurgeUpload(purge_upload_config or {})
 
+    # cache configs
+    cache_config = configs.get('cache')
+    config_dict['cache'] = Cache(cache_config or {})
+
     return config_dict
 
 

src/common/const.go

@@ -196,4 +196,13 @@ const (
 	PullTimeUpdateDisable = "pull_time_update_disable"
 	// PullAuditLogDisable indicate if pull audit log is disable for pull request.
 	PullAuditLogDisable = "pull_audit_log_disable"
+
+	// Cache layer settings
+	// CacheEnabled indicate whether enable cache layer.
+	CacheEnabled = "cache_enabled"
+	// CacheExpireHours is the cache expiration time, unit is hour.
+	CacheExpireHours = "cache_expire_hours"
+	// DefaultCacheExpireHours is the default cache expire hours, default is
+	// 24h.
+	DefaultCacheExpireHours = 24
 )

src/controller/artifact/abstractor.go

@@ -18,7 +18,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+
 	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg"
 
 	"github.com/docker/distribution/manifest/manifestlist"
 	"github.com/docker/distribution/manifest/schema1"
@@ -40,7 +42,7 @@ type Abstractor interface {
 // NewAbstractor creates a new abstractor
 func NewAbstractor() Abstractor {
 	return &abstractor{
-		artMgr:  artifact.Mgr,
+		artMgr:  pkg.ArtifactMgr,
 		blobMgr: blob.Mgr,
 		regCli:  registry.Cli,
 	}

src/controller/artifact/controller.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/goharbor/harbor/src/pkg"
 	accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
 
 	"github.com/goharbor/harbor/src/controller/artifact/processor/chart"
@@ -116,7 +117,7 @@ func NewController() Controller {
 	return &controller{
 		tagCtl:       tag.Ctl,
 		repoMgr:      repository.Mgr,
-		artMgr:       artifact.Mgr,
+		artMgr:       pkg.ArtifactMgr,
 		artrashMgr:   artifactrash.Mgr,
 		blobMgr:      blob.Mgr,
 		sigMgr:       signature.GetManager(),

src/controller/event/handler/internal/artifact_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/lib/config"
 	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/pkg"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	_ "github.com/goharbor/harbor/src/pkg/config/db"
 	repo "github.com/goharbor/harbor/src/pkg/repository"
@@ -54,7 +55,7 @@ func (suite *ArtifactHandlerTestSuite) SetupSuite() {
 	suite.ctx = orm.NewContext(context.TODO(), beegoorm.NewOrm())
 
 	// mock artifact
-	_, err := artifact.Mgr.Create(suite.ctx, &artifact.Artifact{ID: 1, RepositoryID: 1})
+	_, err := pkg.ArtifactMgr.Create(suite.ctx, &artifact.Artifact{ID: 1, RepositoryID: 1})
 	suite.Nil(err)
 	// mock repository
 	_, err = repo.Mgr.Create(suite.ctx, &model.RepoRecord{RepositoryID: 1})
@@ -70,7 +71,7 @@ func (suite *ArtifactHandlerTestSuite) TearDownSuite() {
 	err := tag.Mgr.Delete(suite.ctx, 1)
 	suite.Nil(err)
 	// delete artifact
-	err = artifact.Mgr.Delete(suite.ctx, 1)
+	err = pkg.ArtifactMgr.Delete(suite.ctx, 1)
 	suite.Nil(err)
 	// delete repository
 	err = repo.Mgr.Delete(suite.ctx, 1)
@@ -107,7 +108,7 @@ func (suite *ArtifactHandlerTestSuite) TestOnPull() {
 	suite.Nil(err, "onPull should return nil")
 	// sync mode should update db immediately
 	// pull_time
-	art, err := artifact.Mgr.Get(suite.ctx, 1)
+	art, err := pkg.ArtifactMgr.Get(suite.ctx, 1)
 	suite.Nil(err)
 	suite.False(art.PullTime.IsZero(), "sync update pull_time")
 	lastPullTime := art.PullTime
@@ -122,7 +123,7 @@ func (suite *ArtifactHandlerTestSuite) TestOnPull() {
 	suite.Nil(err, "onPull should return nil")
 	// async mode should not update db immediately
 	// pull_time
-	art, err = artifact.Mgr.Get(suite.ctx, 1)
+	art, err = pkg.ArtifactMgr.Get(suite.ctx, 1)
 	suite.Nil(err)
 	suite.Equal(lastPullTime, art.PullTime, "pull_time should not be updated immediately")
 	// pull_count
@@ -131,7 +132,7 @@ func (suite *ArtifactHandlerTestSuite) TestOnPull() {
 	suite.Equal(int64(1), repository.PullCount, "pull_count should not be updated immediately")
 	// wait for db update
 	suite.Eventually(func() bool {
-		art, err = artifact.Mgr.Get(suite.ctx, 1)
+		art, err = pkg.ArtifactMgr.Get(suite.ctx, 1)
 		suite.Nil(err)
 		return art.PullTime.After(lastPullTime)
 	}, 3*asyncFlushDuration, asyncFlushDuration/2, "wait for pull_time async update")

src/controller/icon/controller.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/base64"
 	"image"
+
 	// import the gif format
 	_ "image/gif"
 	// import the jpeg format
@@ -31,6 +32,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/icon"
 	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/registry"
 	"github.com/nfnt/resize"
@@ -83,7 +85,7 @@ type Controller interface {
 // NewController creates a new instance of the icon controller
 func NewController() Controller {
 	return &controller{
-		artMgr: artifact.Mgr,
+		artMgr: pkg.ArtifactMgr,
 		regCli: registry.Cli,
 		cache:  sync.Map{},
 	}

src/controller/repository/controller.go

@@ -23,6 +23,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg"
 	art "github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/project"
 	"github.com/goharbor/harbor/src/pkg/repository"
@@ -61,7 +62,7 @@ func NewController() Controller {
 	return &controller{
 		proMgr:  project.Mgr,
 		repoMgr: repository.Mgr,
-		artMgr:  art.Mgr,
+		artMgr:  pkg.ArtifactMgr,
 		artCtl:  artifact.Ctl,
 	}
 }

src/controller/tag/controller.go

@@ -24,6 +24,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/lib/selector"
+	"github.com/goharbor/harbor/src/pkg"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/immutable/match"
 	"github.com/goharbor/harbor/src/pkg/immutable/match/rule"
@@ -61,7 +62,7 @@ type Controller interface {
 func NewController() Controller {
 	return &controller{
 		tagMgr:       tag.Mgr,
-		artMgr:       artifact.Mgr,
+		artMgr:       pkg.ArtifactMgr,
 		immutableMtr: rule.NewRuleMatcher(),
 	}
 }

src/lib/cache/cache.go

@@ -56,6 +56,9 @@ type Cache interface {
 
 	// Save cache the value by key
 	Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error
+
+	// Keys returns the key matched by prefixes
+	Keys(ctx context.Context, prefixes ...string) ([]string, error)
 }
 
 var (

src/lib/cache/memory/memory.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"strings"
 	"sync"
 	"time"
 
@@ -111,6 +112,29 @@ func (c *Cache) Save(ctx context.Context, key string, value interface{}, expirat
 	return nil
 }
 
+// Keys returns the key matched by prefixes.
+func (c *Cache) Keys(ctx context.Context, prefixes ...string) ([]string, error) {
+	// if no prefix, means match all keys.
+	matchAll := len(prefixes) == 0
+	// range map to get all keys
+	keys := make([]string, 0)
+	c.storage.Range(func(k, v interface{}) bool {
+		ks := k.(string)
+		if matchAll {
+			keys = append(keys, ks)
+		} else {
+			for _, p := range prefixes {
+				if strings.HasPrefix(ks, c.opts.Key(p)) {
+					keys = append(keys, ks)
+				}
+			}
+		}
+		return true
+	})
+
+	return keys, nil
+}
+
 // New returns memory cache
 func New(opts cache.Options) (cache.Cache, error) {
 	return &Cache{opts: &opts}, nil