feat: Accept ElementsYaml for list generator to get dynamic content f…

…rom a git file (#12428) (#12490) * fix: use field-wise templating for child matrix generators (#11661) Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * test shouldn't use go template Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> * feat: extend List generator with ElementsJsonBase64 Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: proper field name and crd update Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: indentation Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: remove b64 encoding. Based on #12287 Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: generated with codegen Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: reset some of the generated files Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: elementsyaml to cover both yaml and json Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> * fix: regenerate code Signed-off-by: laurentiusoica <laurentiu@soica.ro> * Regenerate code Signed-off-by: laurentiusoica <laurentiu@soica.ro> * fix: update ApplicationSet docs Signed-off-by: laurentiusoica <laurentiu@soica.ro> * fix: elementsyaml to elementsYaml to be more consistent with other fields Signed-off-by: laurentiusoica <laurentiu@soica.ro> --------- Signed-off-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com> Signed-off-by: Laurentiu Soica <laurentiu@soica.ro> Signed-off-by: laurentiusoica <laurentiu@soica.ro> Co-authored-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com>
argoproj · Mar 22, 2023 · 019da5a · 019da5a
1 parent 304a742
commit 019da5a
Show file tree

Hide file tree

Showing 13 changed files with 943 additions and 612 deletions.
diff --git a/applicationset/examples/list-generator/list-elementsYaml-example.yaml b/applicationset/examples/list-generator/list-elementsYaml-example.yaml
@@ -0,0 +1,14 @@
+key:
+  components:
+    - name: component1
+      chart: podinfo
+      version: "6.3.2"
+      releaseName: component1
+      repoUrl: "https://stefanprodan.github.io/podinfo"
+      namespace: component1
+    - name: component2
+      chart: podinfo
+      version: "6.3.3"
+      releaseName: component2
+      repoUrl: "ghcr.io/stefanprodan/charts"
+      namespace: component2
diff --git a/applicationset/generators/list.go b/applicationset/generators/list.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
+	"sigs.k8s.io/yaml"
 )
 
 var _ Generator = (*ListGenerator)(nil)
@@ -73,5 +74,16 @@ func (g *ListGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Appli
 		}
 	}
 
+	// Append elements from ElementsYaml to the response
+	if len(appSetGenerator.List.ElementsYaml) > 0 {
+
+		var yamlElements []map[string]interface{}
+		err := yaml.Unmarshal([]byte(appSetGenerator.List.ElementsYaml), &yamlElements)
+		if err != nil {
+			return nil, fmt.Errorf("error unmarshling decoded ElementsYaml %v", err)
+		}
+		res = append(res, yamlElements...)	
+	}
+
 	return res, nil
 }
diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go
@@ -837,6 +837,174 @@ func TestInterpolatedMatrixGenerateGoTemplate(t *testing.T) {
 	}
 }
 
+func TestMatrixGenerateListElementsYaml(t *testing.T) {
+
+	gitGenerator := &argoprojiov1alpha1.GitGenerator{
+		RepoURL:  "RepoURL",
+		Revision: "Revision",
+		Files: []argoprojiov1alpha1.GitFileGeneratorItem{
+			{Path: "config.yaml"},
+		},
+	}
+
+	listGenerator := &argoprojiov1alpha1.ListGenerator{
+		Elements: []apiextensionsv1.JSON{},
+		ElementsYaml: "{{ .foo.bar | toJson }}",
+	}
+
+	testCases := []struct {
+		name           string
+		baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator
+		expectedErr    error
+		expected       []map[string]interface{}
+	}{
+		{
+			name: "happy flow - generate params",
+			baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{
+				{
+					Git: gitGenerator,
+				},
+				{
+					List: listGenerator,
+				},
+			},
+			expected: []map[string]interface{}{
+				{
+					"chart":         "a",
+					"version":         "1",
+					"foo": map[string]interface{}{
+						"bar": []interface{}{
+							map[string]interface{}{
+								"chart": "a",
+								"version": "1",
+							},
+							map[string]interface{}{
+								"chart": "b",
+								"version": "2",
+							},
+						},
+					},
+					"path": map[string]interface{}{
+						"basename": "dir",
+						"basenameNormalized": "dir",
+						"filename": "file_name.yaml",
+						"filenameNormalized": "file-name.yaml",
+						"path": "path/dir",
+						"segments": []string {
+							"path",
+							"dir",
+						},
+					},
+				},
+				{
+					"chart":         "b",
+					"version":         "2",
+					"foo": map[string]interface{}{
+						"bar": []interface{}{
+							map[string]interface{}{
+								"chart": "a",
+								"version": "1",
+							},
+							map[string]interface{}{
+								"chart": "b",
+								"version": "2",
+							},
+						},
+					},
+					"path": map[string]interface{}{
+						"basename": "dir",
+						"basenameNormalized": "dir",
+						"filename": "file_name.yaml",
+						"filenameNormalized": "file-name.yaml",
+						"path": "path/dir",
+						"segments": []string {
+							"path",
+							"dir",
+						},
+					},
+				},
+
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCaseCopy := testCase // Since tests may run in parallel
+
+		t.Run(testCaseCopy.name, func(t *testing.T) {
+			genMock := &generatorMock{}
+			appSet := &argoprojiov1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "set",
+				},
+				Spec: argoprojiov1alpha1.ApplicationSetSpec{
+					GoTemplate: true,
+				},
+			}
+
+			for _, g := range testCaseCopy.baseGenerators {
+
+				gitGeneratorSpec := argoprojiov1alpha1.ApplicationSetGenerator{
+					Git:  g.Git,
+					List: g.List,
+				}
+				genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]any{{
+					"foo": map[string]interface{}{
+						"bar": []interface{}{
+							map[string]interface{}{
+								"chart": "a",
+								"version": "1",
+							},
+							map[string]interface{}{
+								"chart": "b",
+								"version": "2",
+							},
+						},
+					},
+					"path": map[string]interface{}{
+						"basename": "dir",
+						"basenameNormalized": "dir",
+						"filename": "file_name.yaml",
+						"filenameNormalized": "file-name.yaml",
+						"path": "path/dir",
+						"segments": []string {
+							"path",
+							"dir",
+						},
+					},
+
+				}}, nil)
+				genMock.On("GetTemplate", &gitGeneratorSpec).
+					Return(&argoprojiov1alpha1.ApplicationSetTemplate{})
+
+			}
+
+			var matrixGenerator = NewMatrixGenerator(
+				map[string]Generator{
+					"Git":  genMock,
+					"List": &ListGenerator{},
+				},
+			)
+
+			got, err := matrixGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{
+				Matrix: &argoprojiov1alpha1.MatrixGenerator{
+					Generators: testCaseCopy.baseGenerators,
+					Template:   argoprojiov1alpha1.ApplicationSetTemplate{},
+				},
+			}, appSet)
+
+			if testCaseCopy.expectedErr != nil {
+				assert.ErrorIs(t, err, testCaseCopy.expectedErr)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, testCaseCopy.expected, got)
+			}
+
+		})
+
+	}
+}
+
 type generatorMock struct {
 	mock.Mock
 }

diff --git a/assets/swagger.json b/assets/swagger.json
@@ -7021,6 +7021,9 @@
             "$ref": "#/definitions/v1JSON"
           }
         },
+        "elementsYaml": {
+          "type": "string"
+        },
         "template": {
           "$ref": "#/definitions/v1alpha1ApplicationSetTemplate"
         }

diff --git a/docs/operator-manual/applicationset/Generators-List.md b/docs/operator-manual/applicationset/Generators-List.md
@@ -53,3 +53,63 @@ spec:
 
 !!! note "Clusters must be predefined in Argo CD"
     These clusters *must* already be defined within Argo CD, in order to generate applications for these values. The ApplicationSet controller does not create clusters within Argo CD (for instance, it does not have the credentials to do so).
+
+## Dynamically generated elements
+The List generator can also dynamically generate its elements based on a yaml/json it gets from a previous generator like git by combining the two with a matrix generator. In this example we are using the matrix generator with a git followed by a list generator and pass the content of a file in git as input to the `elementsYaml` field of the list generator:
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: elementsYaml
+  namespace: argocd
+spec:
+  goTemplate: true
+  generators:
+  - matrix:
+      generators:
+      - git:
+          repoURL: https://github.com/argoproj/argo-cd.git
+          revision: HEAD
+          files:
+          - path: applicationset/examples/list-generator/list-elementsYaml-example.yaml
+      - list:
+          elements: []
+          elementsYaml: "{{ .key.components | toJson }}"
+  template:
+    metadata:
+      name: '{{.name}}'
+    spec:
+      project: default
+      syncPolicy:
+        automated:
+          selfHeal: true    
+        syncOptions:
+        - CreateNamespace=true        
+      sources:
+        - chart: '{{.chart}}'
+          repoURL: '{{.repoUrl}}'
+          targetRevision: '{{.version}}'
+          helm:
+            releaseName: '{{.releaseName}}'
+      destination:
+        server: https://kubernetes.default.svc
+        namespace: '{{.namespace}}'
+```
+
+where `list-elementsYaml-example.yaml` content is:
+```yaml
+key:
+  components:
+    - name: component1
+      chart: podinfo
+      version: "6.3.2"
+      releaseName: component1
+      repoUrl: "https://stefanprodan.github.io/podinfo"
+      namespace: component1
+    - name: component2
+      chart: podinfo
+      version: "6.3.3"
+      releaseName: component2
+      repoUrl: "ghcr.io/stefanprodan/charts"
+      namespace: component2
+```
diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml
@@ -5545,6 +5545,8 @@ spec:
                           items:
                             x-kubernetes-preserve-unknown-fields: true
                           type: array
+                        elementsYaml:
+                          type: string
                         template:
                           properties:
                             metadata:
@@ -7451,6 +7453,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:
@@ -11027,6 +11031,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:

diff --git a/manifests/crds/applicationset-crd.yaml b/manifests/crds/applicationset-crd.yaml
@@ -1483,6 +1483,8 @@ spec:
                           items:
                             x-kubernetes-preserve-unknown-fields: true
                           type: array
+                        elementsYaml:
+                          type: string
                         template:
                           properties:
                             metadata:
@@ -3389,6 +3391,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:
@@ -6965,6 +6969,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:

diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml
@@ -5545,6 +5545,8 @@ spec:
                           items:
                             x-kubernetes-preserve-unknown-fields: true
                           type: array
+                        elementsYaml:
+                          type: string
                         template:
                           properties:
                             metadata:
@@ -7451,6 +7453,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:
@@ -11027,6 +11031,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:

diff --git a/manifests/install.yaml b/manifests/install.yaml
@@ -5545,6 +5545,8 @@ spec:
                           items:
                             x-kubernetes-preserve-unknown-fields: true
                           type: array
+                        elementsYaml:
+                          type: string
                         template:
                           properties:
                             metadata:
@@ -7451,6 +7453,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata:
@@ -11027,6 +11031,8 @@ spec:
                                     items:
                                       x-kubernetes-preserve-unknown-fields: true
                                     type: array
+                                  elementsYaml:
+                                    type: string
                                   template:
                                     properties:
                                       metadata: