pkg/chartmuseum,cmd: introduce the new keep-chart-always-up-to-date…

… flag and the default cache interval when not set. (#593) * pkg/chartmuseum,cmd: introduce the new `keep-chart-always-up-to-date` flag and the default cache interval when not set. * The flags works to always get the realtime index for the chart repository(which is the behavior before v0.13.0), it will cost more fs I/O. * Default interval is now set to 5m when `cache-interval` flag not set which we suggests in the issue list. The patch includes: * increases most of the museum api throughout by wrapping the event emitter into goroutine since the event listener already holds on the concurrent lock(#396). * adds the new server option `keep-chart-always-up-to-date` to force use the latest version. * `getIndexFile` rollbacks to fully reload index only if cacheinterval not set for better backward compatibility before v0.13.0(#444). Signed-off-by: scnace <scbizu@gmail.com> Signed-off-by: scbizu <scbizu@gmail.com> * pkg/chartmuseum/server: `getRepoObjectSlice` adds the read lock when index refresher works. Signed-off-by: scbizu <scbizu@gmail.com> * pkg/chartmuseum: rebuildIndex adds tenant lock Signed-off-by: scbizu <scbizu@gmail.com> * pkg/config: clarify the usage of always-regenerate-chart-index Co-authored-by: Casey Buto <cbuto@d2iq.com> Signed-off-by: Nace Sc <scbizu@icloud.com> Signed-off-by: scbizu <scbizu@gmail.com>
helm · Aug 1, 2022 · 3ae6ed2 · 3ae6ed2
1 parent 1ae6981
commit 3ae6ed2
Show file tree

Hide file tree

Showing 10 changed files with 126 additions and 24 deletions.
diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go
@@ -118,6 +118,7 @@ func cliHandler(c *cli.Context) {
 		PerChartLimit:          conf.GetInt("per-chart-limit"),
 		WebTemplatePath:        conf.GetString("web-template-path"),
 		ArtifactHubRepoID:      conf.GetStringMapString("artifact-hub-repo-id"),
+		AlwaysRegenerateIndex:  conf.GetBool("always-regenerate-chart-index"),
 	}
 
 	server, err := newServer(options)

diff --git a/pkg/chartmuseum/server.go b/pkg/chartmuseum/server.go
@@ -81,6 +81,9 @@ type (
 		Debug bool
 		// Deprecated: LogJSON is no longer effective. ServerOptions now requires the Logger field to be set and configured with LoggerOptions accordingly.
 		LogJSON bool
+		// AlwaysRegenerateIndex represents if the museum always return the up-to-date chart
+		// which means that the GetChart will increase its latency , be careful to enable this .
+		AlwaysRegenerateIndex bool
 	}
 
 	// Server is a generic interface for web servers
@@ -146,7 +149,8 @@ func NewServer(options ServerOptions) (Server, error) {
 		WebTemplatePath:        options.WebTemplatePath,
 		// Deprecated options
 		// EnforceSemver2 - see https://github.com/helm/chartmuseum/issues/485 for more info
-		EnforceSemver2: options.EnforceSemver2,
+		EnforceSemver2:        options.EnforceSemver2,
+		AlwaysRegenerateIndex: options.AlwaysRegenerateIndex,
 	})
 
 	return server, err

diff --git a/pkg/chartmuseum/server/multitenant/cache.go b/pkg/chartmuseum/server/multitenant/cache.go
@@ -477,10 +477,11 @@ func (server *MultiTenantServer) initCacheTimer() {
 		// (in case the files on the disk are manually manipulated)
 		go func() {
 			t := time.NewTicker(server.CacheInterval)
-			for _ = range t.C {
+			for range t.C {
 				server.rebuildIndex()
 			}
 		}()
+
 	}
 }
 
@@ -496,7 +497,6 @@ func (server *MultiTenantServer) emitEvent(c *gin.Context, repo string, operatio
 func (server *MultiTenantServer) startEventListener() {
 	server.Router.Logger.Debug("Starting internal event listener")
 	for {
-
 		e := <-server.EventChan
 		log := server.Logger.ContextLoggingFn(e.Context)
 
@@ -565,11 +565,13 @@ func (server *MultiTenantServer) startEventListener() {
 }
 
 func (server *MultiTenantServer) rebuildIndex() {
+	server.TenantCacheKeyLock.Lock()
+	defer server.TenantCacheKeyLock.Unlock()
 	if len(server.Tenants) == 0 {
 		return
 	}
 	server.Logger.Info("Rebuilding index for all tenants in cache")
-	for repo, _ := range server.Tenants {
+	for repo := range server.Tenants {
 		go server.rebuildIndexForTenant(repo)
 	}
 }
@@ -598,9 +600,7 @@ func (server *MultiTenantServer) refreshCacheEntry(log cm_logger.LoggingFn, repo
 		)
 		return
 	}
-	entry.RepoLock.Lock()
-	defer entry.RepoLock.Unlock()
-	objects := server.getRepoObjectSlice(entry)
+	objects := server.getRepoObjectSliceWithLock(entry)
 	diff := cm_storage.GetObjectSliceDiff(objects, fo.objects, server.TimestampTolerance)
 
 	// return fast if no changes
@@ -615,6 +615,9 @@ func (server *MultiTenantServer) refreshCacheEntry(log cm_logger.LoggingFn, repo
 		"repo", repo,
 	)
 
+	entry.RepoLock.Lock()
+	defer entry.RepoLock.Unlock()
+
 	ir := <-server.regenerateRepositoryIndex(log, entry, diff)
 	if ir.err != nil {
 		errStr := ir.err.Error()

diff --git a/pkg/chartmuseum/server/multitenant/handlers.go b/pkg/chartmuseum/server/multitenant/handlers.go
@@ -311,7 +311,7 @@ func (server *MultiTenantServer) deleteChartVersionRequestHandler(c *gin.Context
 		return
 	}
 
-	server.emitEvent(c, repo, deleteChart, &helm_repo.ChartVersion{
+	 server.emitEvent(c, repo, deleteChart, &helm_repo.ChartVersion{
 		Metadata: &chart.Metadata{
 			Name:    name,
 			Version: version,

diff --git a/pkg/chartmuseum/server/multitenant/handlers_test.go b/pkg/chartmuseum/server/multitenant/handlers_test.go
@@ -43,7 +43,9 @@ func (suite *HandlerTestSuite) getServer(depth int) *MultiTenantServer {
 		ChartPostFormFieldName: "chart",
 		ProvPostFormFieldName:  "prov",
 		IndexLimit:             1,
+		CacheInterval:          time.Second,
 	})
+	suite.NoError(err, "no error creating server")
 	return serverDepth
 }
 

diff --git a/pkg/chartmuseum/server/multitenant/index.go b/pkg/chartmuseum/server/multitenant/index.go
@@ -38,12 +38,13 @@ func (server *MultiTenantServer) getIndexFile(log cm_logger.LoggingFn, repo stri
 		)
 		return nil, &HTTPError{http.StatusInternalServerError, errStr}
 	}
-
 	entry.RepoLock.Lock()
 	defer entry.RepoLock.Unlock()
-	// if cache is nil, and not on a timer, regenerate it
-	if len(entry.RepoIndex.Entries) == 0 && server.CacheInterval == 0 {
 
+	// if the always-regenerate-chart-index flag is set, we always update the index file
+	// and ignore the chart cache
+	if server.AlwaysRegenerateIndex /* the flag is set */ ||
+		(!server.AlwaysRegenerateIndex && len(entry.RepoIndex.Entries) == 0) /* initial */ {
 		fo := <-server.getChartList(log, repo)
 
 		if fo.err != nil {
@@ -96,6 +97,13 @@ func (server *MultiTenantServer) saveStatefile(log cm_logger.LoggingFn, repo str
 	)
 }
 
+func (server *MultiTenantServer) getRepoObjectSliceWithLock(entry *cacheEntry) []cm_storage.Object {
+	entry.RepoLock.RLock()
+	defer entry.RepoLock.RUnlock()
+
+	return server.getRepoObjectSlice(entry)
+}
+
 func (server *MultiTenantServer) getRepoObjectSlice(entry *cacheEntry) []cm_storage.Object {
 	var objects []cm_storage.Object
 	for _, entry := range entry.RepoIndex.Entries {

diff --git a/pkg/chartmuseum/server/multitenant/server.go b/pkg/chartmuseum/server/multitenant/server.go
@@ -70,8 +70,9 @@ type (
 		ChartLimits            *ObjectsPerChartLimit
 		ArtifactHubRepoID      map[string]string
 		// Deprecated: see https://github.com/helm/chartmuseum/issues/485 for more info
-		EnforceSemver2  bool
-		WebTemplatePath string
+		EnforceSemver2        bool
+		WebTemplatePath       string
+		AlwaysRegenerateIndex bool
 	}
 
 	ObjectsPerChartLimit struct {
@@ -103,7 +104,8 @@ type (
 		ArtifactHubRepoID      map[string]string
 		WebTemplatePath        string
 		// Deprecated: see https://github.com/helm/chartmuseum/issues/485 for more info
-		EnforceSemver2 bool
+		EnforceSemver2        bool
+		AlwaysRegenerateIndex bool
 	}
 
 	tenantInternals struct {
@@ -163,6 +165,7 @@ func NewMultiTenantServer(options MultiTenantServerOptions) (*MultiTenantServer,
 		ChartLimits:            l,
 		WebTemplatePath:        options.WebTemplatePath,
 		ArtifactHubRepoID:      options.ArtifactHubRepoID,
+		AlwaysRegenerateIndex:  options.AlwaysRegenerateIndex,
 	}
 
 	if server.WebTemplatePath != "" {