From 1be3f89b506a0fd7e16410ae651ed4c19b0b2eeb Mon Sep 17 00:00:00 2001 From: BarAshkenazi Date: Wed, 24 Apr 2024 18:01:59 +0300 Subject: [PATCH 01/10] Update repository.go Signed-off-by: BarAshkenazi Signed-off-by: Bar --- reposerver/repository/repository.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 83cc149f68c8a..de986e3d9b853 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -2646,11 +2646,6 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir return nil } - fname := entry.Name() - if strings.HasPrefix(fname, ".") { // Skip all folders starts with "." - return filepath.SkipDir - } - relativePath, err := filepath.Rel(repoRoot, path) if err != nil { return fmt.Errorf("error constructing relative repo path: %w", err) From abb8aff9aab969276d3f6ac38223403f05f7a115 Mon Sep 17 00:00:00 2001 From: Bar Date: Thu, 25 Apr 2024 14:54:44 +0300 Subject: [PATCH 02/10] support hidden directories in repo server Signed-off-by: Bar --- reposerver/repository/repository_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reposerver/repository/repository_test.go b/reposerver/repository/repository_test.go index f99ce611777c2..817ef94c1c43c 100644 --- a/reposerver/repository/repository_test.go +++ b/reposerver/repository/repository_test.go @@ -3309,7 +3309,7 @@ func TestGetGitDirectories(t *testing.T) { } directories, err := s.GetGitDirectories(context.TODO(), dirRequest) assert.Nil(t, err) - assert.ElementsMatch(t, directories.GetPaths(), []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo"}) + assert.ElementsMatch(t, directories.GetPaths(), []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo", "app/bar/.hidden"}) // do the same request again to use the cache // we only allow CheckOut to be called once in the mock From bf0575d32fa6d268d1fbb2081772c0d75769fe7d Mon Sep 17 00:00:00 2001 From: Bar Date: Thu, 25 Apr 2024 15:27:24 +0300 Subject: [PATCH 03/10] support hidden directories in repo server Signed-off-by: Bar --- reposerver/repository/repository_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reposerver/repository/repository_test.go b/reposerver/repository/repository_test.go index 817ef94c1c43c..ef312beaae4e9 100644 --- a/reposerver/repository/repository_test.go +++ b/reposerver/repository/repository_test.go @@ -3315,7 +3315,7 @@ func TestGetGitDirectories(t *testing.T) { // we only allow CheckOut to be called once in the mock directories, err = s.GetGitDirectories(context.TODO(), dirRequest) assert.Nil(t, err) - assert.ElementsMatch(t, []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo"}, directories.GetPaths()) + assert.ElementsMatch(t, []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo", "app/bar/.hidden"}, directories.GetPaths()) cacheMocks.mockCache.AssertCacheCalledTimes(t, &repositorymocks.CacheCallCounts{ ExternalSets: 1, ExternalGets: 2, From 164fc809896fd2f976604fd63979320c3942753c Mon Sep 17 00:00:00 2001 From: Bar Date: Sun, 28 Apr 2024 14:38:21 +0300 Subject: [PATCH 04/10] add dir exclusion pattern Signed-off-by: Bar --- Procfile | 2 +- .../commands/argocd_repo_server.go | 13 +++++++++++++ reposerver/repository/repository.go | 11 +++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 4862b0230062f..642c7e771a949 100644 --- a/Procfile +++ b/Procfile @@ -2,7 +2,7 @@ controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}" dex: sh -c "ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/v2/cmd gendexcfg -o `pwd`/dist/dex.yaml && (test -f dist/dex.yaml || { echo 'Failed to generate dex configuration'; exit 1; }) && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:$(grep "image: ghcr.io/dexidp/dex" manifests/base/dex/argocd-dex-server-deployment.yaml | cut -d':' -f3) dex serve /dex.yaml" redis: bash -c "if [ \"$ARGOCD_REDIS_LOCAL\" = 'true' ]; then redis-server --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; else docker run --rm --name argocd-redis -i -p ${ARGOCD_E2E_REDIS_PORT:-6379}:${ARGOCD_E2E_REDIS_PORT:-6379} docker.io/library/redis:$(grep "image: redis" manifests/base/redis/argocd-redis-deployment.yaml | cut -d':' -f3) --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; fi" -repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-repo-server ARGOCD_GPG_ENABLED=${ARGOCD_GPG_ENABLED:-false} $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --otlp-address=${ARGOCD_OTLP_ADDRESS}" +repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES=true ARGOCD_REPO_SERVER_EXCLUDE_DIRECTORIES_PATTERN="" FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-repo-server ARGOCD_GPG_ENABLED=${ARGOCD_GPG_ENABLED:-false} $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --otlp-address=${ARGOCD_OTLP_ADDRESS}" cmp-server: [ "$ARGOCD_E2E_TEST" = 'true' ] && exit 0 || [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_BINARY_NAME=argocd-cmp-server ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} $COMMAND --config-dir-path ./test/cmp --loglevel debug --otlp-address=${ARGOCD_OTLP_ADDRESS}" ui: sh -c 'cd ui && ${ARGOCD_E2E_YARN_CMD:-yarn} start' git-server: test/fixture/testrepos/start-git.sh diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go index 2ba17cd9b64ba..b6e5e91ed4118 100644 --- a/cmd/argocd-repo-server/commands/argocd_repo_server.go +++ b/cmd/argocd-repo-server/commands/argocd_repo_server.go @@ -5,6 +5,7 @@ import ( "math" "net" "net/http" + "regexp" "time" "github.com/argoproj/pkg/stats" @@ -70,6 +71,9 @@ func NewCommand() *cobra.Command { helmManifestMaxExtractedSize string helmRegistryMaxIndexSize string disableManifestMaxExtractedSize bool + includeHiddenDirectories bool + dirExcludePattern string + dirExcludePatternCompiled *regexp.Regexp ) var command = cobra.Command{ Use: cliName, @@ -114,6 +118,11 @@ func NewCommand() *cobra.Command { helmRegistryMaxIndexSizeQuantity, err := resource.ParseQuantity(helmRegistryMaxIndexSize) errors.CheckError(err) + if dirExcludePattern != "" { + dirExcludePatternCompiled, err = regexp.Compile(dirExcludePattern) + errors.CheckError(err) + } + askPassServer := askpass.NewServer() metricsServer := metrics.NewMetricsServer() cacheutil.CollectMetrics(redisClient, metricsServer) @@ -130,6 +139,8 @@ func NewCommand() *cobra.Command { StreamedManifestMaxTarSize: streamedManifestMaxTarSizeQuantity.ToDec().Value(), HelmManifestMaxExtractedSize: helmManifestMaxExtractedSizeQuantity.ToDec().Value(), HelmRegistryMaxIndexSize: helmRegistryMaxIndexSizeQuantity.ToDec().Value(), + IncludeHiddenDirectories: includeHiddenDirectories, + DirExcludePattern: dirExcludePatternCompiled, }, askPassServer) errors.CheckError(err) @@ -215,6 +226,8 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&helmManifestMaxExtractedSize, "helm-manifest-max-extracted-size", env.StringFromEnv("ARGOCD_REPO_SERVER_HELM_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of helm manifest archives when extracted") command.Flags().StringVar(&helmRegistryMaxIndexSize, "helm-registry-max-index-size", env.StringFromEnv("ARGOCD_REPO_SERVER_HELM_MANIFEST_MAX_INDEX_SIZE", "1G"), "Maximum size of registry index file") command.Flags().BoolVar(&disableManifestMaxExtractedSize, "disable-helm-manifest-max-extracted-size", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_HELM_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of helm manifest archives when extracted") + command.Flags().BoolVar(&includeHiddenDirectories, "include-hidden-directories", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES", false), "Include hidden directories from Git") + command.Flags().StringVar(&dirExcludePattern, "dir-exclude-pattern", env.StringFromEnv("ARGOCD_REPO_SERVER_EXCLUDE_DIRECTORIES_PATTERN", "^\\..*", env.StringFromEnvOpts{AllowEmpty: true}), "Pattern to exclude directories in Git") tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command) cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, cacheutil.Options{ OnClientCreated: func(client *redis.Client) { diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index de986e3d9b853..70b7f54c5079d 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -112,6 +112,8 @@ type RepoServerInitConstants struct { HelmManifestMaxExtractedSize int64 HelmRegistryMaxIndexSize int64 DisableHelmManifestMaxExtractedSize bool + IncludeHiddenDirectories bool + DirExcludePattern *regexp.Regexp } // NewService returns a new instance of the Manifest service @@ -2604,6 +2606,7 @@ func (s *Service) GetGitFiles(_ context.Context, request *apiclient.GitFilesRequ } func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDirectoriesRequest) (*apiclient.GitDirectoriesResponse, error) { + log.Infof("Im new and env is %t", s.initConstants.IncludeHiddenDirectories) repo := request.GetRepo() revision := request.GetRevision() noRevisionCache := request.GetNoRevisionCache() @@ -2616,6 +2619,7 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir return nil, status.Errorf(codes.Internal, "unable to resolve git revision %s: %v", revision, err) } + log.Infof("Im new and env is %t", s.initConstants.IncludeHiddenDirectories) // check the cache and return the results if present if cachedPaths, err := s.cache.GetGitDirectories(repo.Repo, revision); err == nil { log.Debugf("cache hit for repo: %s revision: %s", repo.Repo, revision) @@ -2646,6 +2650,13 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir return nil } + if s.initConstants.DirExcludePattern != nil { + fname := entry.Name() + if skipDir := s.initConstants.DirExcludePattern.MatchString(fname); skipDir == true { + return filepath.SkipDir // Skip directories that match exclude pattern + } + } + relativePath, err := filepath.Rel(repoRoot, path) if err != nil { return fmt.Errorf("error constructing relative repo path: %w", err) From 0ba4fc5e7f0458ea010d90071009cd90fe3308db Mon Sep 17 00:00:00 2001 From: Bar Date: Mon, 29 Apr 2024 22:33:04 +0300 Subject: [PATCH 05/10] support hidden directories Signed-off-by: Bar --- .../commands/argocd_repo_server.go | 12 +++---- reposerver/repository/repository.go | 8 ++--- reposerver/repository/repository_test.go | 33 +++++++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go index b6e5e91ed4118..77e9151ca2920 100644 --- a/cmd/argocd-repo-server/commands/argocd_repo_server.go +++ b/cmd/argocd-repo-server/commands/argocd_repo_server.go @@ -72,8 +72,8 @@ func NewCommand() *cobra.Command { helmRegistryMaxIndexSize string disableManifestMaxExtractedSize bool includeHiddenDirectories bool - dirExcludePattern string - dirExcludePatternCompiled *regexp.Regexp + DirExclusionPattern string + DirExclusionPatternCompiled *regexp.Regexp ) var command = cobra.Command{ Use: cliName, @@ -118,8 +118,8 @@ func NewCommand() *cobra.Command { helmRegistryMaxIndexSizeQuantity, err := resource.ParseQuantity(helmRegistryMaxIndexSize) errors.CheckError(err) - if dirExcludePattern != "" { - dirExcludePatternCompiled, err = regexp.Compile(dirExcludePattern) + if DirExclusionPattern != "" { + DirExclusionPatternCompiled, err = regexp.Compile(DirExclusionPattern) errors.CheckError(err) } @@ -140,7 +140,7 @@ func NewCommand() *cobra.Command { HelmManifestMaxExtractedSize: helmManifestMaxExtractedSizeQuantity.ToDec().Value(), HelmRegistryMaxIndexSize: helmRegistryMaxIndexSizeQuantity.ToDec().Value(), IncludeHiddenDirectories: includeHiddenDirectories, - DirExcludePattern: dirExcludePatternCompiled, + DirExclusionPattern: DirExclusionPatternCompiled, }, askPassServer) errors.CheckError(err) @@ -227,7 +227,7 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&helmRegistryMaxIndexSize, "helm-registry-max-index-size", env.StringFromEnv("ARGOCD_REPO_SERVER_HELM_MANIFEST_MAX_INDEX_SIZE", "1G"), "Maximum size of registry index file") command.Flags().BoolVar(&disableManifestMaxExtractedSize, "disable-helm-manifest-max-extracted-size", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_HELM_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of helm manifest archives when extracted") command.Flags().BoolVar(&includeHiddenDirectories, "include-hidden-directories", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES", false), "Include hidden directories from Git") - command.Flags().StringVar(&dirExcludePattern, "dir-exclude-pattern", env.StringFromEnv("ARGOCD_REPO_SERVER_EXCLUDE_DIRECTORIES_PATTERN", "^\\..*", env.StringFromEnvOpts{AllowEmpty: true}), "Pattern to exclude directories in Git") + command.Flags().StringVar(&DirExclusionPattern, "dir-exclude-pattern", env.StringFromEnv("ARGOCD_REPO_SERVER_EXCLUDE_DIRECTORIES_PATTERN", "^\\..*", env.StringFromEnvOpts{AllowEmpty: true}), "Pattern to exclude directories in Git") tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command) cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, cacheutil.Options{ OnClientCreated: func(client *redis.Client) { diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 70b7f54c5079d..4432d51f14814 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -113,7 +113,7 @@ type RepoServerInitConstants struct { HelmRegistryMaxIndexSize int64 DisableHelmManifestMaxExtractedSize bool IncludeHiddenDirectories bool - DirExcludePattern *regexp.Regexp + DirExclusionPattern *regexp.Regexp } // NewService returns a new instance of the Manifest service @@ -2650,10 +2650,10 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir return nil } - if s.initConstants.DirExcludePattern != nil { + if s.initConstants.IncludeHiddenDirectories == false { fname := entry.Name() - if skipDir := s.initConstants.DirExcludePattern.MatchString(fname); skipDir == true { - return filepath.SkipDir // Skip directories that match exclude pattern + if skipDir := strings.HasPrefix(fname, "."); skipDir == true { + return filepath.SkipDir // Skip hidden directory } } diff --git a/reposerver/repository/repository_test.go b/reposerver/repository/repository_test.go index ef312beaae4e9..3a0c7526625d7 100644 --- a/reposerver/repository/repository_test.go +++ b/reposerver/repository/repository_test.go @@ -3309,6 +3309,39 @@ func TestGetGitDirectories(t *testing.T) { } directories, err := s.GetGitDirectories(context.TODO(), dirRequest) assert.Nil(t, err) + assert.ElementsMatch(t, directories.GetPaths(), []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo"}) + + // do the same request again to use the cache + // we only allow CheckOut to be called once in the mock + directories, err = s.GetGitDirectories(context.TODO(), dirRequest) + assert.Nil(t, err) + assert.ElementsMatch(t, []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo"}, directories.GetPaths()) + cacheMocks.mockCache.AssertCacheCalledTimes(t, &repositorymocks.CacheCallCounts{ + ExternalSets: 1, + ExternalGets: 2, + }) +} + +func TestGetGitDirectoriesWithHiddenDirSupported(t *testing.T) { + // test not using the cache + root := "./testdata/git-files-dirs" + s, _, cacheMocks := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) { + gitClient.On("Init").Return(nil) + gitClient.On("Fetch", mock.Anything).Return(nil) + gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil) + gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil) + gitClient.On("Root").Return(root) + paths.On("GetPath", mock.Anything).Return(root, nil) + paths.On("GetPathIfExists", mock.Anything).Return(root, nil) + }, root) + s.initConstants.IncludeHiddenDirectories = true + dirRequest := &apiclient.GitDirectoriesRequest{ + Repo: &argoappv1.Repository{Repo: "a-url.com"}, + SubmoduleEnabled: false, + Revision: "HEAD", + } + directories, err := s.GetGitDirectories(context.TODO(), dirRequest) + assert.Nil(t, err) assert.ElementsMatch(t, directories.GetPaths(), []string{"app", "app/bar", "app/foo/bar", "somedir", "app/foo", "app/bar/.hidden"}) // do the same request again to use the cache From 1492a4938394faffc4635eed5da911210abcde3c Mon Sep 17 00:00:00 2001 From: Bar Date: Mon, 29 Apr 2024 23:42:17 +0300 Subject: [PATCH 06/10] support hidden directories in repo server Signed-off-by: Bar --- cmd/argocd-repo-server/commands/argocd_repo_server.go | 9 --------- .../server-commands/argocd-repo-server.md | 1 + reposerver/repository/repository.go | 8 ++------ 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go index 77e9151ca2920..0aa291ef104c7 100644 --- a/cmd/argocd-repo-server/commands/argocd_repo_server.go +++ b/cmd/argocd-repo-server/commands/argocd_repo_server.go @@ -72,8 +72,6 @@ func NewCommand() *cobra.Command { helmRegistryMaxIndexSize string disableManifestMaxExtractedSize bool includeHiddenDirectories bool - DirExclusionPattern string - DirExclusionPatternCompiled *regexp.Regexp ) var command = cobra.Command{ Use: cliName, @@ -118,11 +116,6 @@ func NewCommand() *cobra.Command { helmRegistryMaxIndexSizeQuantity, err := resource.ParseQuantity(helmRegistryMaxIndexSize) errors.CheckError(err) - if DirExclusionPattern != "" { - DirExclusionPatternCompiled, err = regexp.Compile(DirExclusionPattern) - errors.CheckError(err) - } - askPassServer := askpass.NewServer() metricsServer := metrics.NewMetricsServer() cacheutil.CollectMetrics(redisClient, metricsServer) @@ -140,7 +133,6 @@ func NewCommand() *cobra.Command { HelmManifestMaxExtractedSize: helmManifestMaxExtractedSizeQuantity.ToDec().Value(), HelmRegistryMaxIndexSize: helmRegistryMaxIndexSizeQuantity.ToDec().Value(), IncludeHiddenDirectories: includeHiddenDirectories, - DirExclusionPattern: DirExclusionPatternCompiled, }, askPassServer) errors.CheckError(err) @@ -227,7 +219,6 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&helmRegistryMaxIndexSize, "helm-registry-max-index-size", env.StringFromEnv("ARGOCD_REPO_SERVER_HELM_MANIFEST_MAX_INDEX_SIZE", "1G"), "Maximum size of registry index file") command.Flags().BoolVar(&disableManifestMaxExtractedSize, "disable-helm-manifest-max-extracted-size", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_HELM_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of helm manifest archives when extracted") command.Flags().BoolVar(&includeHiddenDirectories, "include-hidden-directories", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES", false), "Include hidden directories from Git") - command.Flags().StringVar(&DirExclusionPattern, "dir-exclude-pattern", env.StringFromEnv("ARGOCD_REPO_SERVER_EXCLUDE_DIRECTORIES_PATTERN", "^\\..*", env.StringFromEnvOpts{AllowEmpty: true}), "Pattern to exclude directories in Git") tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command) cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, cacheutil.Options{ OnClientCreated: func(client *redis.Client) { diff --git a/docs/operator-manual/server-commands/argocd-repo-server.md b/docs/operator-manual/server-commands/argocd-repo-server.md index 189adbbc3370b..3532fc6c30b4a 100644 --- a/docs/operator-manual/server-commands/argocd-repo-server.md +++ b/docs/operator-manual/server-commands/argocd-repo-server.md @@ -23,6 +23,7 @@ argocd-repo-server [flags] --helm-manifest-max-extracted-size string Maximum size of helm manifest archives when extracted (default "1G") --helm-registry-max-index-size string Maximum size of registry index file (default "1G") -h, --help help for argocd-repo-server + --include-hidden-directories Include hidden directories from Git --logformat string Set the logging format. One of: text|json (default "text") --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") --max-combined-directory-manifests-size string Max combined size of manifest files in a directory-type Application (default "10M") diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 4432d51f14814..e172a9e92b5b2 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -113,7 +113,6 @@ type RepoServerInitConstants struct { HelmRegistryMaxIndexSize int64 DisableHelmManifestMaxExtractedSize bool IncludeHiddenDirectories bool - DirExclusionPattern *regexp.Regexp } // NewService returns a new instance of the Manifest service @@ -2650,11 +2649,8 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir return nil } - if s.initConstants.IncludeHiddenDirectories == false { - fname := entry.Name() - if skipDir := strings.HasPrefix(fname, "."); skipDir == true { - return filepath.SkipDir // Skip hidden directory - } + if !s.initConstants.IncludeHiddenDirectories && strings.HasPrefix(entry.Name(), ".") { + return filepath.SkipDir // Skip hidden directory } relativePath, err := filepath.Rel(repoRoot, path) From d780b386d79fcf939fc89e33d7f15ef14f85c5c3 Mon Sep 17 00:00:00 2001 From: Bar Date: Mon, 29 Apr 2024 23:43:25 +0300 Subject: [PATCH 07/10] support hidden directories in repo server Signed-off-by: Bar --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 642c7e771a949..4862b0230062f 100644 --- a/Procfile +++ b/Procfile @@ -2,7 +2,7 @@ controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}" dex: sh -c "ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/v2/cmd gendexcfg -o `pwd`/dist/dex.yaml && (test -f dist/dex.yaml || { echo 'Failed to generate dex configuration'; exit 1; }) && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:$(grep "image: ghcr.io/dexidp/dex" manifests/base/dex/argocd-dex-server-deployment.yaml | cut -d':' -f3) dex serve /dex.yaml" redis: bash -c "if [ \"$ARGOCD_REDIS_LOCAL\" = 'true' ]; then redis-server --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; else docker run --rm --name argocd-redis -i -p ${ARGOCD_E2E_REDIS_PORT:-6379}:${ARGOCD_E2E_REDIS_PORT:-6379} docker.io/library/redis:$(grep "image: redis" manifests/base/redis/argocd-redis-deployment.yaml | cut -d':' -f3) --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; fi" -repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES=true ARGOCD_REPO_SERVER_EXCLUDE_DIRECTORIES_PATTERN="" FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-repo-server ARGOCD_GPG_ENABLED=${ARGOCD_GPG_ENABLED:-false} $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --otlp-address=${ARGOCD_OTLP_ADDRESS}" +repo-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_GNUPGHOME=${ARGOCD_GNUPGHOME:-/tmp/argocd-local/gpg/keys} ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} ARGOCD_GPG_DATA_PATH=${ARGOCD_GPG_DATA_PATH:-/tmp/argocd-local/gpg/source} ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-repo-server ARGOCD_GPG_ENABLED=${ARGOCD_GPG_ENABLED:-false} $COMMAND --loglevel debug --port ${ARGOCD_E2E_REPOSERVER_PORT:-8081} --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --otlp-address=${ARGOCD_OTLP_ADDRESS}" cmp-server: [ "$ARGOCD_E2E_TEST" = 'true' ] && exit 0 || [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_BINARY_NAME=argocd-cmp-server ARGOCD_PLUGINSOCKFILEPATH=${ARGOCD_PLUGINSOCKFILEPATH:-./test/cmp} $COMMAND --config-dir-path ./test/cmp --loglevel debug --otlp-address=${ARGOCD_OTLP_ADDRESS}" ui: sh -c 'cd ui && ${ARGOCD_E2E_YARN_CMD:-yarn} start' git-server: test/fixture/testrepos/start-git.sh From 6136f05c54f57101fb457a4acbcaf1c4e04fcaaa Mon Sep 17 00:00:00 2001 From: Bar Date: Mon, 29 Apr 2024 23:45:10 +0300 Subject: [PATCH 08/10] support hidden directories in repo server Signed-off-by: Bar --- cmd/argocd-repo-server/commands/argocd_repo_server.go | 1 - reposerver/repository/repository.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go index 0aa291ef104c7..f80968cf76f37 100644 --- a/cmd/argocd-repo-server/commands/argocd_repo_server.go +++ b/cmd/argocd-repo-server/commands/argocd_repo_server.go @@ -5,7 +5,6 @@ import ( "math" "net" "net/http" - "regexp" "time" "github.com/argoproj/pkg/stats" diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index e172a9e92b5b2..eb65f408bd02d 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -2605,7 +2605,6 @@ func (s *Service) GetGitFiles(_ context.Context, request *apiclient.GitFilesRequ } func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDirectoriesRequest) (*apiclient.GitDirectoriesResponse, error) { - log.Infof("Im new and env is %t", s.initConstants.IncludeHiddenDirectories) repo := request.GetRepo() revision := request.GetRevision() noRevisionCache := request.GetNoRevisionCache() @@ -2618,7 +2617,6 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir return nil, status.Errorf(codes.Internal, "unable to resolve git revision %s: %v", revision, err) } - log.Infof("Im new and env is %t", s.initConstants.IncludeHiddenDirectories) // check the cache and return the results if present if cachedPaths, err := s.cache.GetGitDirectories(repo.Repo, revision); err == nil { log.Debugf("cache hit for repo: %s revision: %s", repo.Repo, revision) From ea7474dc1faa148c97af5daf374b1880f7c720f5 Mon Sep 17 00:00:00 2001 From: Bar Date: Fri, 24 May 2024 20:13:51 +0300 Subject: [PATCH 09/10] support hidden directories in repo server Signed-off-by: Bar --- docs/operator-manual/argocd-cmd-params-cm.yaml | 2 ++ .../base/repo-server/argocd-repo-server-deployment.yaml | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml index 3cb79d85f3150..e6e0034c24754 100644 --- a/docs/operator-manual/argocd-cmd-params-cm.yaml +++ b/docs/operator-manual/argocd-cmd-params-cm.yaml @@ -179,6 +179,8 @@ data: reposerver.git.lsremote.parallelism.limit: "0" # Git requests timeout. reposerver.git.request.timeout: "15s" + # Include hidden directories from Git + reposerver.include.hidden.directories: "false" # Disable TLS on the HTTP endpoint dexserver.disable.tls: "false" diff --git a/manifests/base/repo-server/argocd-repo-server-deployment.yaml b/manifests/base/repo-server/argocd-repo-server-deployment.yaml index 4e2f0afdf557e..0e86acd3e3b5e 100644 --- a/manifests/base/repo-server/argocd-repo-server-deployment.yaml +++ b/manifests/base/repo-server/argocd-repo-server-deployment.yaml @@ -209,6 +209,12 @@ spec: key: reposerver.grpc.max.size name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES + valueFrom: + configMapKeyRef: + key: reposerver.include.hidden.directories + name: argocd-cmd-params-cm + optional: true - name: HELM_CACHE_HOME value: /helm-working-dir - name: HELM_CONFIG_HOME From 4414c342b5887071f12b90d3a990a642c50229be Mon Sep 17 00:00:00 2001 From: Bar Date: Fri, 24 May 2024 21:13:39 +0300 Subject: [PATCH 10/10] support hidden directories in repo server Signed-off-by: Bar --- manifests/core-install.yaml | 6 ++++++ manifests/ha/install.yaml | 6 ++++++ manifests/ha/namespace-install.yaml | 6 ++++++ manifests/install.yaml | 6 ++++++ manifests/namespace-install.yaml | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml index cec04be68b118..ace900d2d65c9 100644 --- a/manifests/core-install.yaml +++ b/manifests/core-install.yaml @@ -21622,6 +21622,12 @@ spec: key: reposerver.grpc.max.size name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES + valueFrom: + configMapKeyRef: + key: reposerver.include.hidden.directories + name: argocd-cmd-params-cm + optional: true - name: HELM_CACHE_HOME value: /helm-working-dir - name: HELM_CONFIG_HOME diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index 707c7a9482359..dee5a101d83ff 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -23197,6 +23197,12 @@ spec: key: reposerver.grpc.max.size name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES + valueFrom: + configMapKeyRef: + key: reposerver.include.hidden.directories + name: argocd-cmd-params-cm + optional: true - name: HELM_CACHE_HOME value: /helm-working-dir - name: HELM_CONFIG_HOME diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml index 09c7559ca2765..157ebd9bad946 100644 --- a/manifests/ha/namespace-install.yaml +++ b/manifests/ha/namespace-install.yaml @@ -2279,6 +2279,12 @@ spec: key: reposerver.grpc.max.size name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES + valueFrom: + configMapKeyRef: + key: reposerver.include.hidden.directories + name: argocd-cmd-params-cm + optional: true - name: HELM_CACHE_HOME value: /helm-working-dir - name: HELM_CONFIG_HOME diff --git a/manifests/install.yaml b/manifests/install.yaml index e2e307577bf7a..7a30db2b5bf4c 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -22267,6 +22267,12 @@ spec: key: reposerver.grpc.max.size name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES + valueFrom: + configMapKeyRef: + key: reposerver.include.hidden.directories + name: argocd-cmd-params-cm + optional: true - name: HELM_CACHE_HOME value: /helm-working-dir - name: HELM_CONFIG_HOME diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml index ea69f57bd03f6..a1b2b31f0265d 100644 --- a/manifests/namespace-install.yaml +++ b/manifests/namespace-install.yaml @@ -1349,6 +1349,12 @@ spec: key: reposerver.grpc.max.size name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES + valueFrom: + configMapKeyRef: + key: reposerver.include.hidden.directories + name: argocd-cmd-params-cm + optional: true - name: HELM_CACHE_HOME value: /helm-working-dir - name: HELM_CONFIG_HOME