Implement Git files discovery

Dockerfile

@@ -10,5 +10,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -a -o applic
 
 # Use distroless as minimal base image to package the manager binary
 FROM debian:10-slim
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y git-all && \
+    rm -r /var/lib/apt/lists /var/cache/apt/archives
 WORKDIR /
 COPY --from=builder /workspace/applicationset-controller /usr/local/bin/

api/v1alpha1/applicationset_types.go

@@ -65,6 +65,7 @@ type ClusterGenerator struct {
 type GitGenerator struct {
 	RepoURL             string                      `json:"repoURL"`
 	Directories         []GitDirectoryGeneratorItem `json:"directories,omitempty"`
+	Files               []GitFileGeneratorItem      `json:"files,omitempty"`
 	Revision            string                      `json:"revision"`
 	RequeueAfterSeconds int64                       `json:"requeueAfterSeconds,omitempty"`
 }
@@ -73,6 +74,10 @@ type GitDirectoryGeneratorItem struct {
 	Path string `json:"path"`
 }
 
+type GitFileGeneratorItem struct {
+	Path string `json:"path"`
+}
+
 // +kubebuilder:object:root=true
 
 // ApplicationSetList contains a list of ApplicationSet

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/argoproj/argo-cd v1.7.6
 	github.com/argoproj/gitops-engine v0.1.3-0.20200904164417-c04f859da9b2
 	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/jeremywohl/flatten v1.0.1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.6.0
@@ -22,6 +23,8 @@ require (
 )
 
 replace (
+	github.com/go-logr/logr => github.com/go-logr/logr v0.2.1
+	github.com/go-logr/zapr => github.com/go-logr/zapr v0.2.0
 	k8s.io/api => k8s.io/api v0.18.8
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.8
 	k8s.io/apimachinery => k8s.io/apimachinery v0.18.8
@@ -46,6 +49,4 @@ replace (
 	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.8
 	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.18.8
 	k8s.io/sample-controller => k8s.io/sample-controller v0.18.8
-	github.com/go-logr/logr => github.com/go-logr/logr v0.2.1
-	github.com/go-logr/zapr => github.com/go-logr/zapr v0.2.0
 )

go.sum

@@ -428,6 +428,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
+github.com/jeremywohl/flatten v1.0.1 h1:LrsxmB3hfwJuE+ptGOijix1PIfOoKLJ3Uee/mzbgtrs=
+github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

pkg/generators/git.go

@@ -2,21 +2,23 @@ package generators
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 	"time"
 
 	argoprojiov1alpha1 "github.com/argoproj-labs/applicationset/api/v1alpha1"
 	"github.com/argoproj-labs/applicationset/pkg/services"
+	"github.com/jeremywohl/flatten"
 	log "github.com/sirupsen/logrus"
 )
 
 var _ Generator = (*GitGenerator)(nil)
 
 type GitGenerator struct {
-	repos services.Apps
+	repos services.Repos
 }
 
-func NewGitGenerator(repos services.Apps) Generator {
+func NewGitGenerator(repos services.Repos) Generator {
 	g := &GitGenerator{
 		repos: repos,
 	}
@@ -37,6 +39,23 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic
 		return nil, EmptyAppSetGeneratorError
 	}
 
+	var err error
+	var res []map[string]string
+	if appSetGenerator.Git.Directories != nil {
+		res, err = g.generateParamsForGitDirectories(appSetGenerator)
+	} else if appSetGenerator.Git.Files != nil {
+		res, err = g.generateParamsForGitFiles(appSetGenerator)
+	} else {
+		return nil, EmptyAppSetGeneratorError
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) ([]map[string]string, error) {
 	allApps, err := g.repos.GetApps(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision)
 	if err != nil {
 		return nil, err
@@ -49,14 +68,63 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic
 		"revision": appSetGenerator.Git.Revision,
 	}).Info("applications result from the repo service")
 
-	requestedApps := g.filter(appSetGenerator.Git.Directories, allApps)
+	requestedApps := g.filterApps(appSetGenerator.Git.Directories, allApps)
 
-	res := g.generateParams(requestedApps, appSetGenerator)
+	res := g.generateParamsFromApps(requestedApps, appSetGenerator)
 
 	return res, nil
 }
 
-func (g *GitGenerator) filter(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allApps []string) []string {
+func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator) ([]map[string]string, error) {
+	allPaths := make(map[string]bool)
+	for _, requestedPath := range appSetGenerator.Git.Files {
+		paths, err := g.repos.GetPaths(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
+		if err != nil {
+			return nil, err
+		}
+		for _, path := range paths {
+			allPaths[path] = true
+		}
+	}
+
+	res := []map[string]string{}
+
+	for path, _ := range allPaths {
+		params, err := g.generateParamsFromGitFile(appSetGenerator, path)
+		if err != nil {
+			return nil, err
+		}
+
+		res = append(res, params)
+	}
+	return res, nil
+}
+
+func (g *GitGenerator) generateParamsFromGitFile(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, path string) (map[string]string, error) {
+	content, err := g.repos.GetFileContent(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, path)
+	if err != nil {
+		return nil, err
+	}
+
+	config := make(map[string]interface{})
+	err = json.Unmarshal(content, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	flat, err := flatten.Flatten(config, "", flatten.DotStyle)
+	if err != nil {
+		return nil, err
+	}
+	params := make(map[string]string)
+	for k, v := range flat {
+		params[k] = v.(string)
+	}
+
+	return params, nil
+}
+
+func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allApps []string) []string {
 	res := []string{}
 	for _, requestedPath := range Directories {
 		for _, appPath := range allApps {
@@ -74,7 +142,7 @@ func (g *GitGenerator) filter(Directories []argoprojiov1alpha1.GitDirectoryGener
 	return res
 }
 
-func (g *GitGenerator) generateParams(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator) []map[string]string {
+func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator) []map[string]string {
 	// TODO: At some point, the appicationSetGenerator param should be used
 
 	res := make([]map[string]string, len(requestedApps))

pkg/generators/git_test.go

@@ -29,7 +29,19 @@ func (a argoCDServiceMock) GetApps(ctx context.Context, repoURL string, revision
 	return args.Get(0).([]string), args.Error(1)
 }
 
-func TestGitGenerateParams(t *testing.T) {
+func (a argoCDServiceMock) GetPaths(ctx context.Context, repoURL string, revision string, pattern string) ([]string, error) {
+	args := a.mock.Called(ctx, repoURL, revision, pattern)
+
+	return args.Get(0).([]string), args.Error(1)
+}
+
+func (a argoCDServiceMock) GetFileContent(ctx context.Context, repoURL string, revision string, path string) ([]byte, error) {
+	args := a.mock.Called(ctx, repoURL, revision, path)
+
+	return args.Get(0).([]byte), args.Error(1)
+}
+
+func TestGitGenerateParamsFromDirectories(t *testing.T) {
 
 	cases := []struct {
 		name          string
@@ -124,3 +136,146 @@ func TestGitGenerateParams(t *testing.T) {
 	}
 
 }
+
+func TestGitGenerateParamsFromFiles(t *testing.T) {
+
+	cases := []struct {
+		name                   string
+		files                  []argoprojiov1alpha1.GitFileGeneratorItem
+		repoPaths              []string
+		repoFileContents       map[string][]byte
+		repoPathsError         error
+		repoFileContentsErrors map[string]error
+		expected               []map[string]string
+		expectedError          error
+	}{
+		{
+			name:  "happy flow: create params from git files",
+			files: []argoprojiov1alpha1.GitFileGeneratorItem{{"**/config.json"}},
+			repoPaths: []string{
+				"cluster-config/production/config.json",
+				"cluster-config/staging/config.json",
+			},
+			repoFileContents: map[string][]byte{
+				"cluster-config/production/config.json": []byte(`{
+   "cluster": {
+       "owner": "john.doe@example.com",
+       "name": "production",
+       "address": "https://kubernetes.default.svc"
+   },
+   "key1": "val1",
+   "key2": {
+       "key2_1": "val2_1",
+       "key2_2": {
+           "key2_2_1": "val2_2_1"
+       }
+   }
+}`),
+				"cluster-config/staging/config.json": []byte(`{
+   "cluster": {
+       "owner": "foo.bar@example.com",
+       "name": "staging",
+       "address": "https://kubernetes.default.svc"
+   }
+}`),
+			},
+			repoPathsError:         nil,
+			repoFileContentsErrors: nil,
+			expected: []map[string]string{
+				{
+					"cluster.owner":        "john.doe@example.com",
+					"cluster.name":         "production",
+					"cluster.address":      "https://kubernetes.default.svc",
+					"key1":                 "val1",
+					"key2.key2_1":          "val2_1",
+					"key2.key2_2.key2_2_1": "val2_2_1",
+				},
+				{
+					"cluster.owner":   "foo.bar@example.com",
+					"cluster.name":    "staging",
+					"cluster.address": "https://kubernetes.default.svc",
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                   "handles error during getting repo paths",
+			files:                  []argoprojiov1alpha1.GitFileGeneratorItem{{"**/config.json"}},
+			repoPaths:              []string{},
+			repoFileContents:       map[string][]byte{},
+			repoPathsError:         fmt.Errorf("paths error"),
+			repoFileContentsErrors: nil,
+			expected:               []map[string]string{},
+			expectedError:          fmt.Errorf("paths error"),
+		},
+		{
+			name:  "handles error during getting repo file contents",
+			files: []argoprojiov1alpha1.GitFileGeneratorItem{{"**/config.json"}},
+			repoPaths: []string{
+				"cluster-config/production/config.json",
+				"cluster-config/staging/config.json",
+			},
+			repoFileContents: map[string][]byte{
+				"cluster-config/production/config.json": []byte(`{
+   "cluster": {
+       "owner": "john.doe@example.com",
+       "name": "production",
+       "address": "https://kubernetes.default.svc"
+   }
+}`),
+				"cluster-config/staging/config.json": nil,
+			},
+			repoPathsError: nil,
+			repoFileContentsErrors: map[string]error{
+				"cluster-config/production/config.json": nil,
+				"cluster-config/staging/config.json":    fmt.Errorf("staging config file get content error"),
+			},
+			expected:      []map[string]string{},
+			expectedError: fmt.Errorf("staging config file get content error"),
+		},
+	}
+
+	for _, c := range cases {
+		cc := c
+		t.Run(cc.name, func(t *testing.T) {
+			argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}}
+			argoCDServiceMock.mock.On("GetPaths", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(c.repoPaths, c.repoPathsError)
+
+			if c.repoPaths != nil {
+				for _, repoPath := range c.repoPaths {
+					fmt.Println("repoPath: ", repoPath)
+					argoCDServiceMock.mock.On("GetFileContent", mock.Anything, mock.Anything, mock.Anything, repoPath).Return(c.repoFileContents[repoPath], c.repoFileContentsErrors[repoPath]).Once()
+				}
+			}
+
+			var gitGenerator = NewGitGenerator(argoCDServiceMock)
+			applicationSetInfo := argoprojiov1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "set",
+				},
+				Spec: argoprojiov1alpha1.ApplicationSetSpec{
+					Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{
+						Git: &argoprojiov1alpha1.GitGenerator{
+							RepoURL:  "RepoURL",
+							Revision: "Revision",
+							Files:    c.files,
+						},
+					}},
+				},
+			}
+
+			got, err := gitGenerator.GenerateParams(&applicationSetInfo.Spec.Generators[0])
+			fmt.Println(got, err)
+
+			if c.expectedError != nil {
+				assert.EqualError(t, err, c.expectedError.Error())
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, c.expected, got)
+			}
+
+			argoCDServiceMock.mock.AssertExpectations(t)
+		})
+	}
+
+}