fix: repo-server should allow not to skip hidden directories (argopro…

…j#17957) * Update repository.go Signed-off-by: BarAshkenazi <barashkenazi2@gmail.com> Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * add dir exclusion pattern Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> * support hidden directories in repo server Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> --------- Signed-off-by: BarAshkenazi <barashkenazi2@gmail.com> Signed-off-by: Bar <bar.ashkenazi@appsflyer.com> Co-authored-by: Bar <bar.ashkenazi@appsflyer.com> Signed-off-by: rumstead <37445536+rumstead@users.noreply.github.com>
rumstead · Jun 6, 2024 · 78c7a02 · 78c7a02
1 parent be844de
commit 78c7a02
Show file tree

Hide file tree

Showing 11 changed files with 78 additions and 3 deletions.
diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go
@@ -70,6 +70,7 @@ func NewCommand() *cobra.Command {
 		helmManifestMaxExtractedSize      string
 		helmRegistryMaxIndexSize          string
 		disableManifestMaxExtractedSize   bool
+		includeHiddenDirectories          bool
 	)
 	var command = cobra.Command{
 		Use:               cliName,
@@ -130,6 +131,7 @@ func NewCommand() *cobra.Command {
 				StreamedManifestMaxTarSize:                   streamedManifestMaxTarSizeQuantity.ToDec().Value(),
 				HelmManifestMaxExtractedSize:                 helmManifestMaxExtractedSizeQuantity.ToDec().Value(),
 				HelmRegistryMaxIndexSize:                     helmRegistryMaxIndexSizeQuantity.ToDec().Value(),
+				IncludeHiddenDirectories:                     includeHiddenDirectories,
 			}, askPassServer)
 			errors.CheckError(err)
 
@@ -215,6 +217,7 @@ func NewCommand() *cobra.Command {
 	command.Flags().StringVar(&helmManifestMaxExtractedSize, "helm-manifest-max-extracted-size", env.StringFromEnv("ARGOCD_REPO_SERVER_HELM_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of helm manifest archives when extracted")
 	command.Flags().StringVar(&helmRegistryMaxIndexSize, "helm-registry-max-index-size", env.StringFromEnv("ARGOCD_REPO_SERVER_HELM_MANIFEST_MAX_INDEX_SIZE", "1G"), "Maximum size of registry index file")
 	command.Flags().BoolVar(&disableManifestMaxExtractedSize, "disable-helm-manifest-max-extracted-size", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_HELM_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of helm manifest archives when extracted")
+	command.Flags().BoolVar(&includeHiddenDirectories, "include-hidden-directories", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES", false), "Include hidden directories from Git")
 	tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command)
 	cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, cacheutil.Options{
 		OnClientCreated: func(client *redis.Client) {

diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml
@@ -179,6 +179,8 @@ data:
   reposerver.git.lsremote.parallelism.limit: "0"
   # Git requests timeout.
   reposerver.git.request.timeout: "15s"
+  # Include hidden directories from Git
+  reposerver.include.hidden.directories: "false"
 
   # Disable TLS on the HTTP endpoint
   dexserver.disable.tls: "false"

diff --git a/docs/operator-manual/server-commands/argocd-repo-server.md b/docs/operator-manual/server-commands/argocd-repo-server.md
@@ -23,6 +23,7 @@ argocd-repo-server [flags]
       --helm-manifest-max-extracted-size string        Maximum size of helm manifest archives when extracted (default "1G")
       --helm-registry-max-index-size string            Maximum size of registry index file (default "1G")
   -h, --help                                           help for argocd-repo-server
+      --include-hidden-directories                     Include hidden directories from Git
       --logformat string                               Set the logging format. One of: text|json (default "text")
       --loglevel string                                Set the logging level. One of: debug|info|warn|error (default "info")
       --max-combined-directory-manifests-size string   Max combined size of manifest files in a directory-type Application (default "10M")

diff --git a/manifests/base/repo-server/argocd-repo-server-deployment.yaml b/manifests/base/repo-server/argocd-repo-server-deployment.yaml
@@ -209,6 +209,12 @@ spec:
                 key: reposerver.grpc.max.size
                 name: argocd-cmd-params-cm
                 optional: true
+          - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES
+            valueFrom:
+              configMapKeyRef:
+                key: reposerver.include.hidden.directories
+                name: argocd-cmd-params-cm
+                optional: true
           - name: HELM_CACHE_HOME
             value: /helm-working-dir
           - name: HELM_CONFIG_HOME

diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml
@@ -21622,6 +21622,12 @@ spec:
               key: reposerver.grpc.max.size
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.include.hidden.directories
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml
@@ -23197,6 +23197,12 @@ spec:
               key: reposerver.grpc.max.size
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.include.hidden.directories
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml
@@ -2279,6 +2279,12 @@ spec:
               key: reposerver.grpc.max.size
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.include.hidden.directories
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

diff --git a/manifests/install.yaml b/manifests/install.yaml
@@ -22267,6 +22267,12 @@ spec:
               key: reposerver.grpc.max.size
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.include.hidden.directories
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml
@@ -1349,6 +1349,12 @@ spec:
               key: reposerver.grpc.max.size
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.include.hidden.directories
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go
@@ -112,6 +112,7 @@ type RepoServerInitConstants struct {
 	HelmManifestMaxExtractedSize                 int64
 	HelmRegistryMaxIndexSize                     int64
 	DisableHelmManifestMaxExtractedSize          bool
+	IncludeHiddenDirectories                     bool
 }
 
 // NewService returns a new instance of the Manifest service
@@ -2650,9 +2651,8 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir
 			return nil
 		}
 
-		fname := entry.Name()
-		if strings.HasPrefix(fname, ".") { // Skip all folders starts with "."
-			return filepath.SkipDir
+		if !s.initConstants.IncludeHiddenDirectories && strings.HasPrefix(entry.Name(), ".") {
+			return filepath.SkipDir // Skip hidden directory
 		}
 
 		relativePath, err := filepath.Rel(repoRoot, path)

diff --git a/reposerver/repository/repository_test.go b/reposerver/repository/repository_test.go
@@ -3339,6 +3339,39 @@ func TestGetGitDirectories(t *testing.T) {
 	})
 }
 
+func TestGetGitDirectoriesWithHiddenDirSupported(t *testing.T) {
+	// test not using the cache
+	root := "./testdata/git-files-dirs"
+	s, _, cacheMocks := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) {
+		gitClient.On("Init").Return(nil)
+		gitClient.On("Fetch", mock.Anything).Return(nil)
+		gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil)
+		gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil)
+		gitClient.On("Root").Return(root)
+		paths.On("GetPath", mock.Anything).Return(root, nil)
+		paths.On("GetPathIfExists", mock.Anything).Return(root, nil)
+	}, root)
+	s.initConstants.IncludeHiddenDirectories = true
+	dirRequest := &apiclient.GitDirectoriesRequest{
+		Repo:             &argoappv1.Repository{Repo: "a-url.com"},
+		SubmoduleEnabled: false,
+		Revision:         "HEAD",
+	}
+	directories, err := s.GetGitDirectories(context.TODO(), dirRequest)
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, directories.GetPaths(), []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo", "app/bar/.hidden"})
+
+	// do the same request again to use the cache
+	// we only allow CheckOut to be called once in the mock
+	directories, err = s.GetGitDirectories(context.TODO(), dirRequest)
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo", "app/bar/.hidden"}, directories.GetPaths())
+	cacheMocks.mockCache.AssertCacheCalledTimes(t, &repositorymocks.CacheCallCounts{
+		ExternalSets: 1,
+		ExternalGets: 2,
+	})
+}
+
 func TestErrorGetGitFiles(t *testing.T) {
 	// test not using the cache
 	root := ""