config templates

api/v1alpha1/clustertype_types.go

@@ -39,8 +39,8 @@ type ClusterTypeSpec struct {
 	//+kubebuilder:validation:MinLength=0
 	NamespaceService string `json:"namespaceService"`
 
-	//+optional
-	ConfigType ConfigType `json:"configType,omitempty"`
+	//+kubebuilder:validation:MinLength=0
+	ConfigType string `json:"configType"`
 }
 
 // ClusterTypeStatus defines the observed state of ClusterType

api/v1alpha1/template_types.go

@@ -20,18 +20,22 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// +kubebuilder:validation:Enum=reconciler;namespace
+// +kubebuilder:validation:Enum=reconciler;namespace;config
 type TemplateType string
 
 const (
 	ReconcilerTemplate TemplateType = "reconciler"
-	NemaspaceTemplate  TemplateType = "namespace"
+	NamespaceTemplate  TemplateType = "namespace"
+	ConfigTemplate     TemplateType = "config"
 )
 
 // TemplateSpec defines the desired state of Template
 type TemplateSpec struct {
 	Type TemplateType `json:"type"`
 
+	//+optional
+	ContentType string `json:"contentType"`
+
 	//+kubebuilder:pruning:PreserveUnknownFields
 	Manifests []string `json:"manifests"`
 }

config/crd/bases/scheduler.kalypso.io_clustertypes.yaml

@@ -36,9 +36,7 @@ spec:
             description: ClusterTypeSpec defines the desired state of ClusterType
             properties:
               configType:
-                enum:
-                - configMap
-                - envFile
+                minLength: 0
                 type: string
               namespaceService:
                 minLength: 0
@@ -47,6 +45,7 @@ spec:
                 minLength: 0
                 type: string
             required:
+            - configType
             - namespaceService
             - reconciler
             type: object

config/crd/bases/scheduler.kalypso.io_templates.yaml

@@ -35,6 +35,8 @@ spec:
           spec:
             description: TemplateSpec defines the desired state of Template
             properties:
+              contentType:
+                type: string
               manifests:
                 items:
                   type: string
@@ -44,6 +46,7 @@ spec:
                 enum:
                 - reconciler
                 - namespace
+                - config
                 type: string
             required:
             - manifests

controllers/assignment_controller.go

@@ -18,17 +18,13 @@ package controllers
 
 import (
 	"context"
-	"fmt"
 	"sort"
 	"time"
 
-	"gopkg.in/yaml.v2"
 	"k8s.io/apimachinery/pkg/api/errors"
 	meta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -114,7 +110,9 @@ func (r *AssignmentReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return r.manageFailure(ctx, reqLogger, assignment, err, "Failed to get DeploymentTarget")
 	}
 
-	templater, err := scheduler.NewTemplater(deploymentTarget, clusterType)
+	configData := r.getConfigData(ctx, clusterType, deploymentTarget)
+
+	templater, err := scheduler.NewTemplater(deploymentTarget, clusterType, configData)
 	if err != nil {
 		return r.manageFailure(ctx, reqLogger, assignment, err, "Failed to get templater")
 	}
@@ -138,7 +136,7 @@ func (r *AssignmentReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	reqLogger.Info("Namespace Manifests", "Manifests", namespaceManifests)
 
 	//get configManifests
-	configManifests, configContentType, err := r.getConfigManifests(ctx, clusterType, deploymentTarget, templater)
+	configManifests, configContentType, err := r.getConfigManifests(ctx, clusterType, templater)
 	if err != nil {
 		return r.manageFailure(ctx, reqLogger, assignment, err, "Failed to get config manifests")
 	}
@@ -265,36 +263,35 @@ func (r *AssignmentReconciler) getNamespaceManifests(ctx context.Context, cluste
 }
 
 // get the config manifests
-func (r *AssignmentReconciler) getConfigManifests(ctx context.Context, clusterType *schedulerv1alpha1.ClusterType, deploymentTarget *schedulerv1alpha1.DeploymentTarget, templater scheduler.Templater) ([]string, *string, error) {
-	// fetch all config maps in the cluster type namespace that have the label "platform-config: true"
-	configMaps := &corev1.ConfigMapList{}
-	err := r.List(ctx, configMaps, client.InNamespace(clusterType.Namespace), client.MatchingLabels{PlatformConfigLabel: "true"})
+func (r *AssignmentReconciler) getConfigManifests(ctx context.Context, clusterType *schedulerv1alpha1.ClusterType, templater scheduler.Templater) ([]string, *string, error) {
+
+	// fetch the cluster type namespace template
+	template := &schedulerv1alpha1.Template{}
+	err := r.Get(ctx, client.ObjectKey{Name: clusterType.Spec.ConfigType, Namespace: clusterType.Namespace}, template)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	configData := r.getConfigData(ctx, configMaps.Items, clusterType, deploymentTarget)
-
-	var manifests []string
-	var contentType string
-	if clusterType.Spec.ConfigType == schedulerv1alpha1.ConfigMapConfigType || clusterType.Spec.ConfigType == "" {
-		platformConfigMap := r.getPlatformConfigMap(PlatformConfigLabel, templater.GetTargetNamespace(), configData)
-		manifest, err := yaml.Marshal(platformConfigMap.Object)
-		if err != nil {
-			return nil, nil, err
-		}
-		manifests = append(manifests, string(manifest))
-		contentType = schedulerv1alpha1.YamlContentType
-	} else if clusterType.Spec.ConfigType == schedulerv1alpha1.EnvFileConfigType {
-		platformConfigEnv := r.getPlatformConfigEnv(PlatformConfigLabel, templater.GetTargetNamespace(), configData)
-		manifests = append(manifests, platformConfigEnv)
-		contentType = schedulerv1alpha1.EnvContentType
+	contentType := template.Spec.ContentType
+	if contentType == "" {
+		contentType = (string)(schedulerv1alpha1.ConfigMapConfigType)
 	}
 
+	manifests, err := templater.ProcessTemplate(ctx, template)
+
 	return manifests, &contentType, nil
 }
 
-func (r *AssignmentReconciler) getConfigData(ctx context.Context, configMaps []corev1.ConfigMap, clusterType *schedulerv1alpha1.ClusterType, deploymentTarget *schedulerv1alpha1.DeploymentTarget) map[string]string {
+func (r *AssignmentReconciler) getConfigData(ctx context.Context, clusterType *schedulerv1alpha1.ClusterType, deploymentTarget *schedulerv1alpha1.DeploymentTarget) map[string]string {
+	// fetch all config maps in the cluster type namespace that have the label "platform-config: true"
+	configMapsList := &corev1.ConfigMapList{}
+	err := r.List(ctx, configMapsList, client.InNamespace(clusterType.Namespace), client.MatchingLabels{PlatformConfigLabel: "true"})
+	if err != nil {
+		return nil
+	}
+
+	configMaps := configMapsList.Items
+
 	//sort config maps by name
 	sort.Slice(configMaps, func(i, j int) bool {
 		return configMaps[i].Name < configMaps[j].Name
@@ -328,31 +325,6 @@ func (r *AssignmentReconciler) getConfigData(ctx context.Context, configMaps []c
 	return sortedClusterConfigData
 }
 
-func (r *AssignmentReconciler) getPlatformConfigMap(name string, namespace string, clusterConfigData map[string]string) unstructured.Unstructured {
-	configMap := unstructured.Unstructured{}
-	configMap.SetGroupVersionKind(schema.GroupVersionKind{
-		Group:   "",
-		Version: "v1",
-		Kind:    "ConfigMap",
-	})
-
-	configMap.SetName(name)
-	configMap.SetNamespace(namespace)
-	configMap.Object["data"] = clusterConfigData
-
-	return configMap
-
-}
-
-func (r *AssignmentReconciler) getPlatformConfigEnv(name string, namespace string, clusterConfigData map[string]string) string {
-	var configEnv string
-
-	for key, value := range clusterConfigData {
-		configEnv += fmt.Sprintf("export %s=\"%s\"\n", key, value)
-	}
-	return configEnv
-}
-
 func (r *AssignmentReconciler) isConfigForClusterTypeAndTarget(config *corev1.ConfigMap, clusterType *schedulerv1alpha1.ClusterType, deploymentTarget *schedulerv1alpha1.DeploymentTarget) bool {
 	matches := true
 	for key, value := range config.Labels {

go.mod

@@ -3,6 +3,7 @@ module github.com/microsoft/kalypso-scheduler
 go 1.19
 
 require (
+	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/fluxcd/kustomize-controller/api v0.32.0
 	github.com/fluxcd/pkg/apis/meta v0.18.0
 	github.com/fluxcd/source-controller/api v0.33.0
@@ -13,11 +14,11 @@ require (
 	github.com/onsi/gomega v1.20.1
 	github.com/stretchr/testify v1.8.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
-	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.25.4
 	k8s.io/apimachinery v0.25.4
 	k8s.io/client-go v0.25.4
 	sigs.k8s.io/controller-runtime v0.13.1
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -28,6 +29,8 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -51,11 +54,14 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -79,6 +85,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.25.4 // indirect
 	k8s.io/component-base v0.25.4 // indirect
@@ -87,5 +94,4 @@ require (
 	k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -62,6 +62,12 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
+github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -240,6 +246,8 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
@@ -276,8 +284,12 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0=
 github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -808,5 +820,5 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

scheduler/githubrepo.go

@@ -305,11 +305,14 @@ func (g *githubRepo) getTree(ref *github.Reference, content *schedulerv1alpha1.R
 }
 
 func (g *githubRepo) getFullManifestsFileName(fileName string, contentType string) string {
-	if contentType == "" {
-		contentType = "yaml"
+	var fileExtension string
+	if contentType == schedulerv1alpha1.EnvContentType {
+		fileExtension = "sh"
+	} else {
+		fileExtension = "yaml"
 	}
 
-	return fileName + "." + contentType
+	return fileName + "." + fileExtension
 }
 
 func (g *githubRepo) addPromotedCommitId(existingEntries []*github.TreeEntry, content *schedulerv1alpha1.RepoContentType) (commitEntry *github.TreeEntry, isPromoted bool, err error) {