feat: Allow Matrix generator to have two Git child generators without conflict (#10522)

Makefile

@@ -342,7 +342,7 @@ lint-local:
 	golangci-lint --version
 	# NOTE: If you get a "Killed" OOM message, try reducing the value of GOGC
 	# See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
-	GOGC=$(ARGOCD_LINT_GOGC) GOMAXPROCS=2 golangci-lint run --fix --verbose --timeout 300s
+	GOGC=$(ARGOCD_LINT_GOGC) GOMAXPROCS=2 golangci-lint run --fix --verbose --timeout 360s
 
 .PHONY: lint-ui
 lint-ui: test-tools-image

applicationset/generators/git.go

@@ -85,6 +85,7 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
 		"total":    len(allPaths),
 		"repoURL":  appSetGenerator.Git.RepoURL,
 		"revision": appSetGenerator.Git.Revision,
+		"pathParamPrefix": appSetGenerator.Git.PathParamPrefix,
 	}).Info("applications result from the repo service")
 
 	requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths)
@@ -121,7 +122,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	for _, path := range allPaths {
 
 		// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
-		paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], useGoTemplate)
+		paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], useGoTemplate, appSetGenerator.Git.PathParamPrefix)
 		if err != nil {
 			return nil, fmt.Errorf("unable to process file '%s': %v", path, err)
 		}
@@ -133,7 +134,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	return res, nil
 }
 
-func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, useGoTemplate bool) ([]map[string]interface{}, error) {
+func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, useGoTemplate bool, pathParamPrefix string) ([]map[string]interface{}, error) {
 	objectsFound := []map[string]interface{}{}
 
 	// First, we attempt to parse as an array
@@ -167,7 +168,11 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 			paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(paramPath["path"].(string)))
 			paramPath["filenameNormalized"] = utils.SanitizeName(path.Base(paramPath["filename"].(string)))
 			paramPath["segments"] = strings.Split(paramPath["path"].(string), "/")
-			params["path"] = paramPath
+			if pathParamPrefix != "" {
+				params[pathParamPrefix] = map[string]interface{}{"path": paramPath}
+			} else {
+				params["path"] = paramPath
+			}
 		} else {
 			flat, err := flatten.Flatten(objectFound, "", flatten.DotStyle)
 			if err != nil {
@@ -176,14 +181,18 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 			for k, v := range flat {
 				params[k] = fmt.Sprintf("%v", v)
 			}
-			params["path"] = path.Dir(filePath)
-			params["path.basename"] = path.Base(params["path"].(string))
-			params["path.filename"] = path.Base(filePath)
-			params["path.basenameNormalized"] = utils.SanitizeName(path.Base(params["path"].(string)))
-			params["path.filenameNormalized"] = utils.SanitizeName(path.Base(params["path.filename"].(string)))
-			for k, v := range strings.Split(params["path"].(string), "/") {
+			pathParamName := "path"
+			if pathParamPrefix != "" {
+				pathParamName = pathParamPrefix+"."+pathParamName
+			}
+			params[pathParamName] = path.Dir(filePath)
+			params[pathParamName+".basename"] = path.Base(params[pathParamName].(string))
+			params[pathParamName+".filename"] = path.Base(filePath)
+			params[pathParamName+".basenameNormalized"] = utils.SanitizeName(path.Base(params[pathParamName].(string)))
+			params[pathParamName+".filenameNormalized"] = utils.SanitizeName(path.Base(params[pathParamName+".filename"].(string)))
+			for k, v := range strings.Split(params[pathParamName].(string), "/") {
 				if len(v) > 0 {
-					params["path["+strconv.Itoa(k)+"]"] = v
+					params[pathParamName+"["+strconv.Itoa(k)+"]"] = v
 				}
 			}
 		}
@@ -192,7 +201,6 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 	}
 
 	return res, nil
-
 }
 
 func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
@@ -223,9 +231,7 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG
 	return res
 }
 
-func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) []map[string]interface{} {
-	// TODO: At some point, the applicationSetGenerator param should be used
-
+func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) []map[string]interface{} {
 	res := make([]map[string]interface{}, len(requestedApps))
 	for i, a := range requestedApps {
 
@@ -237,14 +243,22 @@ func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argopro
 			paramPath["basename"] = path.Base(a)
 			paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(a))
 			paramPath["segments"] = strings.Split(paramPath["path"].(string), "/")
-			params["path"] = paramPath
+			if appSetGenerator.Git.PathParamPrefix != "" {
+				params[appSetGenerator.Git.PathParamPrefix] = map[string]interface{}{"path": paramPath}
+			} else {
+				params["path"] = paramPath
+			}
 		} else {
-			params["path"] = a
-			params["path.basename"] = path.Base(a)
-			params["path.basenameNormalized"] = utils.SanitizeName(path.Base(a))
-			for k, v := range strings.Split(params["path"].(string), "/") {
+			pathParamName := "path"
+			if appSetGenerator.Git.PathParamPrefix != "" {
+				pathParamName = appSetGenerator.Git.PathParamPrefix+"."+pathParamName
+			}
+			params[pathParamName] = a
+			params[pathParamName+".basename"] = path.Base(a)
+			params[pathParamName+".basenameNormalized"] = utils.SanitizeName(path.Base(a))
+			for k, v := range strings.Split(params[pathParamName].(string), "/") {
 				if len(v) > 0 {
-					params["path["+strconv.Itoa(k)+"]"] = v
+					params[pathParamName+"["+strconv.Itoa(k)+"]"] = v
 				}
 			}
 		}

applicationset/generators/git_test.go

@@ -51,7 +51,7 @@ func Test_generateParamsFromGitFile(t *testing.T) {
 	params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
 foo:
   bar: baz
-`), false)
+`), false, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -69,11 +69,33 @@ foo:
 	}, params)
 }
 
+func Test_generatePrefixedParamsFromGitFile(t *testing.T) {
+	params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
+foo:
+  bar: baz
+`), false, "myRepo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, []map[string]interface{}{
+		{
+			"foo.bar":                        "baz",
+			"myRepo.path":                    "path/dir",
+			"myRepo.path.basename":           "dir",
+			"myRepo.path.filename":           "file_name.yaml",
+			"myRepo.path.basenameNormalized": "dir",
+			"myRepo.path.filenameNormalized": "file-name.yaml",
+			"myRepo.path[0]":                 "path",
+			"myRepo.path[1]":                 "dir",
+		},
+	}, params)
+}
+
 func Test_generateParamsFromGitFileGoTemplate(t *testing.T) {
 	params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
 foo:
   bar: baz
-`), true)
+`), true, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -97,15 +119,46 @@ foo:
 	}, params)
 }
 
+func Test_generatePrefixedParamsFromGitFileGoTemplate(t *testing.T) {
+	params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
+foo:
+  bar: baz
+`), true, "myRepo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, []map[string]interface{}{
+		{
+			"foo": map[string]interface{}{
+				"bar": "baz",
+			},
+			"myRepo": map[string]interface{}{
+				"path": map[string]interface{}{
+					"path":               "path/dir",
+					"basename":           "dir",
+					"filename":           "file_name.yaml",
+					"basenameNormalized": "dir",
+					"filenameNormalized": "file-name.yaml",
+					"segments": []string{
+						"path",
+						"dir",
+					},
+				},
+			},
+		},
+	}, params)
+}
+
 func TestGitGenerateParamsFromDirectories(t *testing.T) {
 
 	cases := []struct {
-		name          string
-		directories   []argoprojiov1alpha1.GitDirectoryGeneratorItem
-		repoApps      []string
-		repoError     error
-		expected      []map[string]interface{}
-		expectedError error
+		name            string
+		directories     []argoprojiov1alpha1.GitDirectoryGeneratorItem
+		pathParamPrefix string
+		repoApps        []string
+		repoError       error
+		expected        []map[string]interface{}
+		expectedError   error
 	}{
 		{
 			name:        "happy flow - created apps",
@@ -124,6 +177,24 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 			},
 			expectedError: nil,
 		},
+		{
+			name:            "It prefixes path parameters with PathParamPrefix",
+			directories:     []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}},
+			pathParamPrefix: "myRepo",
+			repoApps: []string{
+				"app1",
+				"app2",
+				"app_3",
+				"p1/app4",
+			},
+			repoError: nil,
+			expected: []map[string]interface{}{
+				{"myRepo.path": "app1", "myRepo.path.basename": "app1", "myRepo.path.basenameNormalized": "app1", "myRepo.path[0]": "app1"},
+				{"myRepo.path": "app2", "myRepo.path.basename": "app2", "myRepo.path.basenameNormalized": "app2", "myRepo.path[0]": "app2"},
+				{"myRepo.path": "app_3", "myRepo.path.basename": "app_3", "myRepo.path.basenameNormalized": "app-3", "myRepo.path[0]": "app_3"},
+			},
+			expectedError: nil,
+		},
 		{
 			name:        "It filters application according to the paths",
 			directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}},
@@ -212,9 +283,10 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 				Spec: argoprojiov1alpha1.ApplicationSetSpec{
 					Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{
 						Git: &argoprojiov1alpha1.GitGenerator{
-							RepoURL:     "RepoURL",
-							Revision:    "Revision",
-							Directories: testCaseCopy.directories,
+							RepoURL:         "RepoURL",
+							Revision:        "Revision",
+							Directories:     testCaseCopy.directories,
+							PathParamPrefix: testCaseCopy.pathParamPrefix,
 						},
 					}},
 				},
@@ -237,12 +309,13 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
 
 	cases := []struct {
-		name          string
-		directories   []argoprojiov1alpha1.GitDirectoryGeneratorItem
-		repoApps      []string
-		repoError     error
-		expected      []map[string]interface{}
-		expectedError error
+		name            string
+		directories     []argoprojiov1alpha1.GitDirectoryGeneratorItem
+		pathParamPrefix string
+		repoApps        []string
+		repoError       error
+		expected        []map[string]interface{}
+		expectedError   error
 	}{
 		{
 			name:        "happy flow - created apps",
@@ -288,6 +361,57 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
 			},
 			expectedError: nil,
 		},
+		{
+			name:            "It prefixes path parameters with PathParamPrefix",
+			directories:     []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}},
+			pathParamPrefix: "myRepo",
+			repoApps: []string{
+				"app1",
+				"app2",
+				"app_3",
+				"p1/app4",
+			},
+			repoError: nil,
+			expected: []map[string]interface{}{
+				{
+					"myRepo": map[string]interface{}{
+						"path": map[string]interface{}{
+							"path":               "app1",
+							"basename":           "app1",
+							"basenameNormalized": "app1",
+							"segments": []string{
+								"app1",
+							},
+						},
+					},
+				},
+				{
+					"myRepo": map[string]interface{}{
+						"path": map[string]interface{}{
+							"path":               "app2",
+							"basename":           "app2",
+							"basenameNormalized": "app2",
+							"segments": []string{
+								"app2",
+							},
+						},
+					},
+				},
+				{
+					"myRepo": map[string]interface{}{
+						"path": map[string]interface{}{
+							"path":               "app_3",
+							"basename":           "app_3",
+							"basenameNormalized": "app-3",
+							"segments": []string{
+								"app_3",
+							},
+						},
+					},
+				},
+			},
+			expectedError: nil,
+		},
 		{
 			name:        "It filters application according to the paths",
 			directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}},
@@ -455,9 +579,10 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
 					GoTemplate: true,
 					Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{
 						Git: &argoprojiov1alpha1.GitGenerator{
-							RepoURL:     "RepoURL",
-							Revision:    "Revision",
-							Directories: testCaseCopy.directories,
+							RepoURL:         "RepoURL",
+							Revision:        "Revision",
+							Directories:     testCaseCopy.directories,
+							PathParamPrefix: testCaseCopy.pathParamPrefix,
 						},
 					}},
 				},

docs/operator-manual/applicationset/Generators-Git.md

@@ -67,6 +67,8 @@ The generator parameters are:
 
 **Note**: The right-most path name always becomes `{{path.basename}}`. For example, for `- path: /one/two/three/four`, `{{path.basename}}` is `four`.
 
+**Note**: If the `pathParamPrefix` option is specified, all `path`-related parameter names above will be prefixed with the specified value and a dot separator. E.g., if `pathParamPrefix` is `myRepo`, then the generated parameter name would be `myRepo.path` instead of `path`. Using this option is necessary in a Matrix generator where both child generators are Git generators (to avoid conflicts when merging the child generators’ items).
+
 Whenever a new Helm chart/Kustomize YAML/Application/plain subdirectory is added to the Git repository, the ApplicationSet controller will detect this change and automatically deploy the resulting manifests within new `Application` resources.
 
 As with other generators, clusters *must* already be defined within Argo CD, in order to generate Applications for them.
@@ -281,6 +283,7 @@ In addition to the flattened key/value pairs from the configuration file, the fo
 **Note**: The right-most *directory* name always becomes `{{path.basename}}`. For example, from `- path: /one/two/three/four/config.json`, `{{path.basename}}` will be `four`. 
 The filename can always be accessed using `{{path.filename}}`. 
 
+**Note**: If the `pathParamPrefix` option is specified, all `path`-related parameter names above will be prefixed with the specified value and a dot separator. E.g., if `pathParamPrefix` is `myRepo`, then the generated parameter name would be `myRepo.path` instead of `path`. Using this option is necessary in a Matrix generator where both child generators are Git generators (to avoid conflicts when merging the child generators’ items).
 
 ## Webhook Configuration
 

docs/operator-manual/applicationset/Generators-Matrix.md

@@ -11,6 +11,8 @@ By combining both generators parameters, to produce every possible combination,
 
 Any set of generators may be used, with the combined values of those generators inserted into the `template` parameters, as usual.
 
+**Note**: If both child generators are Git generators, one or both of them must use the `pathParamPrefix` option to avoid conflicts when merging the child generators’ items.
+
 ## Example: Git Directory generator + Cluster generator
 
 As an example, imagine that we have two clusters: