Skip to content
This repository has been archived by the owner on Oct 12, 2023. It is now read-only.

Commit

Permalink
Add git directory generator (#7)
Browse files Browse the repository at this point in the history
* add code

* run controller-gen

* fmt

* test run

* add tests

* fmt

* rename

* fix error log

* fix build

* call repo service directly

* add tests

* refactor after testing

* refactor

* fmt

* add support to filter paths

* PR fixes

* add config maps for argocd-cm
  • Loading branch information
OmerKahani committed Sep 8, 2020
1 parent 47c3908 commit 11c59c0
Show file tree
Hide file tree
Showing 12 changed files with 534 additions and 24 deletions.
1 change: 1 addition & 0 deletions api/v1alpha1/applicationset_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ type ClusterGenerator struct {
type GitGenerator struct {
RepoURL string `json:"repoURL"`
Directories []GitDirectoryGeneratorItem `json:"directories,omitempty"`
Revision string `json:"revision"`
}

type GitDirectoryGeneratorItem struct {
Expand Down
12 changes: 2 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,20 @@ go 1.13
require (
github.com/Masterminds/semver v1.5.0 // indirect
github.com/argoproj/argo-cd v1.5.6
github.com/argoproj/gitops-engine v0.1.3
github.com/argoproj/pkg v0.0.0-20200424003221-9b858eff18a1 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/gogo/protobuf v1.3.1 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.3.1 // indirect
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/pkg/errors v0.9.1
github.com/sergi/go-diff v1.1.0 // indirect
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.4.0
github.com/valyala/fasttemplate v1.1.1
github.com/yudai/gojsondiff v1.0.1-0.20180504020246-0525c875b75c // indirect
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect
google.golang.org/grpc v1.23.0
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
k8s.io/api v0.17.2
k8s.io/apimachinery v0.17.2
k8s.io/client-go v11.0.1-0.20190816222228-6d55c1b1f1ca+incompatible
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a // indirect
k8s.io/kubectl v0.16.6 // indirect
k8s.io/kubernetes v1.16.6
k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 // indirect
sigs.k8s.io/controller-runtime v0.5.0
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/argoproj/argo-cd v1.5.6 h1:yrzJuSTf/YBFLugB6DwiS8spDbRKFVyGg8Few5SPLfA=
github.com/argoproj/argo-cd v1.5.6/go.mod h1:UPOPiF6Y1y/oTL3X1KcuLbu7ljD7f4+SIaXvlowBmhE=
github.com/argoproj/gitops-engine v0.1.3 h1:eQp1bfqaeaATcu4XErlxNb6aVsN4rC7suL/Fqx/9E+k=
github.com/argoproj/gitops-engine v0.1.3/go.mod h1:UmBGlQLT/MPNiMmbnouZRWhkk3slPuozMsENdXMkIMs=
github.com/argoproj/pkg v0.0.0-20200102163130-2dd1f3f6b4de/go.mod h1:2EZ44RG/CcgtPTwrRR0apOc7oU6UIw8GjCUJWZ8X3bM=
github.com/argoproj/pkg v0.0.0-20200424003221-9b858eff18a1 h1:BCuMRvKYHPuPN2Gep3uY+XOScGNCBFbF/pXYlhmyJjg=
github.com/argoproj/pkg v0.0.0-20200424003221-9b858eff18a1/go.mod h1:2EZ44RG/CcgtPTwrRR0apOc7oU6UIw8GjCUJWZ8X3bM=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
Expand Down Expand Up @@ -211,6 +214,7 @@ github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoM
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
Expand Down Expand Up @@ -284,6 +288,7 @@ github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.m
github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79 h1:lR9ssWAqp9qL0bALxqEEkuudiP1eweOdv9jsRK3e7lE=
github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.3.0 h1:HJtP6RRwj2EpPCD/mhAWzSvLL/dFTdPm1UrWwanoFos=
Expand Down Expand Up @@ -510,6 +515,7 @@ github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jW
github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
Expand Down
16 changes: 13 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package main

import (
"context"
"flag"
"github.com/argoproj-labs/applicationset/pkg/services"
argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
"os"

Expand All @@ -13,6 +15,7 @@ import (

argoprojiov1alpha1 "github.com/argoproj-labs/applicationset/api/v1alpha1"
"github.com/argoproj-labs/applicationset/pkg/controllers"
"k8s.io/client-go/kubernetes"
// +kubebuilder:scaffold:imports
)

Expand All @@ -34,11 +37,15 @@ func main() {
var metricsAddr string
var probeBindAddr string
var enableLeaderElection bool
var namespace string
var argocdRepoServer string
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&metricsAddr, "probe-addr", ":8081", "The address the probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
flag.StringVar(&namespace, "namespace", "argocd", "Argo CD repo namesapce")
flag.StringVar(&argocdRepoServer, "argocd-repo-server", "argocd-repo-server:8081", "Argo CD repo server address")
flag.Parse()

ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
Expand All @@ -56,10 +63,13 @@ func main() {
os.Exit(1)
}

k8s := kubernetes.NewForConfigOrDie(mgr.GetConfig())

if err = (&controllers.ApplicationSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("applicationset-controller"),
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("applicationset-controller"),
AppsService: services.NewArgoCDService(context.Background(), k8s, namespace, argocdRepoServer),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ApplicationSet")
os.Exit(1)
Expand Down
1 change: 1 addition & 0 deletions manifests/base/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ rules:
- ''
resources:
- secrets
- configmaps
verbs:
- get
- list
Expand Down
1 change: 1 addition & 0 deletions manifests/cluster-rbac/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ rules:
- ''
resources:
- secrets
- configmaps
verbs:
- get
- list
Expand Down
8 changes: 7 additions & 1 deletion manifests/crds/applicationset-crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.3.0
creationTimestamp: null
labels:
app.kubernetes.io/name: applicationsets.argoproj.io
app.kubernetes.io/part-of: argocd-applicationset
Expand Down Expand Up @@ -113,8 +116,11 @@ spec:
type: array
repoURL:
type: string
revision:
type: string
required:
- repoURL
- repoURL
- revision
type: object
list:
description: ListGenerator include items info
Expand Down
15 changes: 12 additions & 3 deletions pkg/controllers/applicationset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"context"
"fmt"
"github.com/argoproj-labs/applicationset/pkg/generators"
"github.com/argoproj-labs/applicationset/pkg/services"
argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
Expand All @@ -34,14 +36,15 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"

argoprojiov1alpha1 "github.com/argoproj-labs/applicationset/api/v1alpha1"
argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
)

// ApplicationSetReconciler reconciles a ApplicationSet object
type ApplicationSetReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
Scheme *runtime.Scheme
Recorder record.EventRecorder
RepoServerAddr string
AppsService services.Apps
}

// +kubebuilder:rbac:groups=argoproj.io,resources=applicationsets,verbs=get;list;watch;create;update;patch;delete
Expand All @@ -60,21 +63,27 @@ func (r *ApplicationSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, err

listGenerator := generators.NewListGenerator()
clusterGenerator := generators.NewClusterGenerator(r.Client)
GitGenerator := generators.NewGitGenerator(r.AppsService)

// desiredApplications is the main list of all expected Applications from all generators in this appset.
var desiredApplications []argov1alpha1.Application

for _, tmpGenerator := range applicationSetInfo.Spec.Generators {
var apps []argov1alpha1.Application
var err error
if tmpGenerator.List != nil {
apps, err = listGenerator.GenerateApplications(&tmpGenerator, &applicationSetInfo)
} else if tmpGenerator.Clusters != nil {
apps, err = clusterGenerator.GenerateApplications(&tmpGenerator, &applicationSetInfo)
} else if tmpGenerator.Git != nil {
apps, err = GitGenerator.GenerateApplications(&tmpGenerator, &applicationSetInfo)
}
log.Infof("apps from generator: %+v", apps)
if err != nil {
log.WithError(err).Error("error generating applications")
continue
}

desiredApplications = append(desiredApplications, apps...)

}
Expand Down
89 changes: 82 additions & 7 deletions pkg/generators/git.go
Original file line number Diff line number Diff line change
@@ -1,27 +1,102 @@
package generators

import (
"context"
"fmt"
argoprojiov1alpha1 "github.com/argoproj-labs/applicationset/api/v1alpha1"
"github.com/argoproj-labs/applicationset/pkg/services"
"github.com/argoproj-labs/applicationset/pkg/utils"
argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
log "github.com/sirupsen/logrus"
"path"
)

var _ Generator = (*GitGenerator)(nil)

type GitGenerator struct {
repos services.Apps
}

func NewGitGenerator() Generator {
g := &GitGenerator{}
func NewGitGenerator(repos services.Apps) Generator {
g := &GitGenerator{
repos: repos,
}
return g
}

func (g *GitGenerator) GenerateApplications(
appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator,
appSet *argoprojiov1alpha1.ApplicationSet) ([]argov1alpha1.Application, error) {
if appSet == nil {
func (g *GitGenerator) GenerateApplications(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]argov1alpha1.Application, error) {
if appSetGenerator == nil || appSet == nil {
return nil, fmt.Errorf("ApplicationSet is empty")
}

return nil, nil
if appSetGenerator.Git == nil {
return nil, fmt.Errorf("git variable empty")
}

allApps, err := g.repos.GetApps(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision)
if err != nil {
return nil, err
}

log.WithFields(log.Fields{
"allAps": allApps,
"total": len(allApps),
"repoURL": appSetGenerator.Git.RepoURL,
"revision": appSetGenerator.Git.Revision,
}).Info("applications result from the repo service")

requestedApps := g.filter(appSetGenerator.Git.Directories, allApps)

res := g.generateApplications(requestedApps, appSet)

return res, nil
}

func (g *GitGenerator) filter(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allApps []string) []string {
res := []string{}
for _, requestedPath := range Directories {
for _, appPath := range allApps {
match, err := path.Match(requestedPath.Path, appPath)
if err != nil {
log.WithError(err).WithField("requestedPath", requestedPath).
WithField("appPath", appPath).Error("error while matching appPath to requestedPath")
continue
}
if match {
res = append(res, appPath)
}
}
}
return res
}

func (g *GitGenerator) generateApplications(requestedApps []string, appSet *argoprojiov1alpha1.ApplicationSet) ([]argov1alpha1.Application) {

res := make([]argov1alpha1.Application, len(requestedApps))
for i, a := range requestedApps {
app, err := g.generateApplication(appSet, a)
if err != nil {
log.WithError(err).WithField("app", a).Error("error while generating app")
continue
}
res[i] = *app
}

return res
}


func (g *GitGenerator) generateApplication(appSet *argoprojiov1alpha1.ApplicationSet, appPath string) (*argov1alpha1.Application, error) {
var tmplApplication argov1alpha1.Application
tmplApplication.Namespace = appSet.Spec.Template.Namespace
tmplApplication.Name = appSet.Spec.Template.Name
tmplApplication.Spec = appSet.Spec.Template.Spec

params := make(map[string]string, 2)
params["path"] = appPath
params["path.basename"] = path.Base(appPath)

tmpApplication, err := utils.RenderTemplateParams(&tmplApplication, params)

return tmpApplication, err
}

0 comments on commit 11c59c0

Please sign in to comment.