feat: add gitRepository source

flanksource · Aug 18, 2021 · 04419bc · 04419bc
1 parent 06509f4
commit 04419bc
Show file tree

Hide file tree

Showing 10 changed files with 262 additions and 0 deletions.
diff --git a/api/v1/template_types.go b/api/v1/template_types.go
@@ -55,6 +55,7 @@ type TemplateStatus struct {
 }
 
 type ResourceSelector struct {
+	GitRepository      *GitRepository       `json:"gitRepository,omitempty"`
 	LabelSelector      metav1.LabelSelector `json:"labelSelector,omitempty"`
 	NamespaceSelector  metav1.LabelSelector `json:"namespaceSelector,omitempty"`
 	AnnotationSelector map[string]string    `json:"annotationSelector,omitempty"`
@@ -78,6 +79,12 @@ type CopyToNamespaces struct {
 	NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"`
 }
 
+type GitRepository struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+	Glob      string `json:"glob,omitempty"`
+}
+
 // +kubebuilder:object:root=true
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/templating.flanksource.com_templates.yaml b/config/crd/bases/templating.flanksource.com_templates.yaml
@@ -99,6 +99,15 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    gitRepository:
+                      properties:
+                        glob:
+                          type: string
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                      type: object
                     kind:
                       type: string
                     labelSelector:
@@ -184,6 +193,15 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    gitRepository:
+                      properties:
+                        glob:
+                          type: string
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                      type: object
                     kind:
                       type: string
                     labelSelector:

diff --git a/config/deploy/crd.yml b/config/deploy/crd.yml
@@ -296,6 +296,15 @@ spec:
                     type: string
                   fieldSelector:
                     type: string
+                  gitRepository:
+                    properties:
+                      glob:
+                        type: string
+                      name:
+                        type: string
+                      namespace:
+                        type: string
+                    type: object
                   kind:
                     type: string
                   labelSelector:
@@ -419,6 +428,15 @@ spec:
                     type: string
                   fieldSelector:
                     type: string
+                  gitRepository:
+                    properties:
+                      glob:
+                        type: string
+                      name:
+                        type: string
+                      namespace:
+                        type: string
+                    type: object
                   kind:
                     type: string
                   labelSelector:

diff --git a/config/deploy/operator.yml b/config/deploy/operator.yml
@@ -301,6 +301,15 @@ spec:
                     type: string
                   fieldSelector:
                     type: string
+                  gitRepository:
+                    properties:
+                      glob:
+                        type: string
+                      name:
+                        type: string
+                      namespace:
+                        type: string
+                    type: object
                   kind:
                     type: string
                   labelSelector:
@@ -424,6 +433,15 @@ spec:
                     type: string
                   fieldSelector:
                     type: string
+                  gitRepository:
+                    properties:
+                      glob:
+                        type: string
+                      name:
+                        type: string
+                      namespace:
+                        type: string
+                    type: object
                   kind:
                     type: string
                   labelSelector:

diff --git a/examples/git-repository.yaml b/examples/git-repository.yaml
@@ -0,0 +1,20 @@
+apiVersion: templating.flanksource.com/v1
+kind: Template
+metadata:
+  name: git-repository
+spec:
+  source:
+    gitRepository:
+      namespace: default
+      name: template-operator-dashboards
+      glob: "/grafana/dashboards/*.json"
+  resources:
+    - apiVersion: integreatly.org/v1alpha1
+      kind: GrafanaDashboard
+      metadata:
+        name: "{{ .filename | filepath.Base }}"
+        namespace: monitoring
+        labels:
+          app: grafana
+      spec:
+        json:  "{{ .content }}"
diff --git a/go.mod b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/go-logr/zapr v0.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.3
 	github.com/go-openapi/spec v0.19.3
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/pkg/errors v0.9.1

diff --git a/go.sum b/go.sum
@@ -280,6 +280,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=

diff --git a/k8s/template_manager.go b/k8s/template_manager.go
@@ -1,10 +1,15 @@
 package k8s
 
 import (
+	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
 	"regexp"
 	"sort"
 	"strconv"
@@ -19,6 +24,7 @@ import (
 	"github.com/flanksource/kommons/ktemplate"
 	templatev1 "github.com/flanksource/template-operator/api/v1"
 	"github.com/go-logr/logr"
+	"github.com/gobwas/glob"
 	"github.com/pkg/errors"
 	"github.com/tidwall/gjson"
 	v1 "k8s.io/api/core/v1"
@@ -184,6 +190,15 @@ func (tm *TemplateManager) selectResources(ctx context.Context, template *templa
 
 func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Template, cb CallbackFunc) (result ctrl.Result, err error) {
 	tm.Log.Info("Reconciling", "template", template.Name)
+	if template.Spec.Source.GitRepository != nil {
+		result, err := tm.handleGitRepository(ctx, template)
+		if err != nil {
+			return result, err
+		}
+		tm.Log.V(3).Info("Reconcile Complete", "template", template.Name)
+		return result, nil
+	}
+
 	sources, err := tm.selectResources(ctx, template, cb)
 	if err != nil {
 		return
@@ -567,6 +582,139 @@ func (tm *TemplateManager) JSONPath(object interface{}, jsonpath string) (*ForEa
 	return nil, errors.Errorf("field %s is not map or array", jsonpath)
 }
 
+func (tm *TemplateManager) handleGitRepository(ctx context.Context, template *templatev1.Template) (result ctrl.Result, err error) {
+	source := template.Spec.Source.GitRepository
+
+	gitRepository, err := tm.getGitRepository(ctx, source.Name, source.Namespace)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to get git repository")
+	}
+
+	status, ok := gitRepository.Object["status"].(map[string]interface{})
+	if !ok {
+		return ctrl.Result{}, errors.Errorf("failed to convert gitRepository.status to map")
+	}
+
+	artifact, ok := status["artifact"].(map[string]interface{})
+	if !ok {
+		return ctrl.Result{}, errors.Errorf("failed to get gitRepository.status.artifact")
+	}
+
+	url, found := artifact["url"].(string)
+	if !found {
+		return ctrl.Result{}, errors.Errorf("could not find url in gitRepository.status.artifact")
+	}
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if err != nil {
+		return ctrl.Result{}, errors.Errorf("failed to download gitRepository archive")
+	}
+
+	files, err := tm.getGitRepositoryFiles(ctx, source, resp.Body)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to get gitRepository files")
+	}
+
+	for filename, content := range files {
+		unstructuredTemplate, err := kommons.ToUnstructured(&unstructured.Unstructured{}, template)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to convert template to unstructured")
+		}
+		unstructuredTemplate.Object["filename"] = filename
+		unstructuredTemplate.Object["content"] = content
+
+		result, err := tm.HandleSource(ctx, template, *unstructuredTemplate)
+		if err != nil {
+			return result, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (tm *TemplateManager) getGitRepositoryFiles(ctx context.Context, source *templatev1.GitRepository, archive io.ReadCloser) (map[string]string, error) {
+	files := map[string]string{}
+	g, err := glob.Compile(source.Glob, '/')
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to compile glob pattern %s", source.Glob)
+	}
+
+	gzf, err := gzip.NewReader(archive)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read gzip archive")
+	}
+
+	tarReader := tar.NewReader(gzf)
+	i := 0
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to advance to next file in tar archive")
+		}
+
+		// name := "/" + header.Name
+		name := header.Name
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			continue
+		case tar.TypeReg:
+			if g.Match("/" + name) {
+				data, err := io.ReadAll(tarReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed to read file %s from tar archive", name)
+				}
+				files[name] = string(data)
+			}
+		default:
+			return nil, errors.Wrapf(err, "failed to find type of file %s", name)
+		}
+
+		i++
+	}
+
+	return files, nil
+}
+
+func (tm *TemplateManager) getGitRepository(ctx context.Context, name, namespace string) (*unstructured.Unstructured, error) {
+	dynamicClient, err := tm.Client.GetDynamicClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get dynamic client")
+	}
+	rm, _ := tm.Client.GetRestMapper()
+	gvk, err := rm.KindFor(schema.GroupVersionResource{
+		Group:    "source.toolkit.fluxcd.io",
+		Version:  "v1beta1",
+		Resource: "GitRepository",
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get kind for GitRepository")
+	}
+	gk := schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}
+	mapping, err := rm.RESTMapping(gk, gvk.Version)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get rest mapping for GitRepository")
+	}
+	resourceInterface, err := dynamicClient.Resource(mapping.Resource), nil
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get dynamic resource interface")
+	}
+
+	return resourceInterface.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
+}
+
 func labelSelectorToString(l metav1.LabelSelector) (string, error) {
 	labelMap, err := metav1.LabelSelectorAsMap(&l)
 	if err != nil {

diff --git a/test/fixtures/git-repository.yaml b/test/fixtures/git-repository.yaml
@@ -0,0 +1,10 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: template-operator-dashboards
+  namespace: default
+spec:
+  interval: 5m
+  url: https://github.com/flanksource/template-operator-e2e-test
+  ref:
+    branch: master