Merge pull request #759 from fluxcd/kustomize-envsubst-strict

kustomize: Implement envsubst strict mode

kustomize/go.mod

@@ -4,14 +4,15 @@ go 1.22
 
 replace (
 	github.com/fluxcd/pkg/apis/kustomize => ../apis/kustomize
+	github.com/fluxcd/pkg/envsubst => ../envsubst
 	github.com/fluxcd/pkg/sourceignore => ../sourceignore
 )
 
 require (
-	github.com/drone/envsubst v1.0.3
 	github.com/fluxcd/pkg/apis/kustomize v1.4.0
+	github.com/fluxcd/pkg/envsubst v1.0.0
 	github.com/fluxcd/pkg/sourceignore v0.6.0
-	github.com/go-git/go-git/v5 v5.11.0
+	github.com/go-git/go-git/v5 v5.12.0
 	github.com/onsi/gomega v1.32.0
 	github.com/otiai10/copy v1.14.0
 	k8s.io/api v0.29.3

kustomize/go.sum

@@ -16,8 +16,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g=
-github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -34,8 +32,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
-github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
-github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
+github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
+github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -142,8 +140,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
-github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
-github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
+github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -155,8 +153,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

kustomize/kustomize_varsub.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2022 The Flux authors
+Copyright 2024 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/drone/envsubst"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -32,6 +31,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/yaml"
+
+	"github.com/fluxcd/pkg/envsubst"
 )
 
 const (
@@ -48,16 +49,45 @@ const (
 	substituteAnnotationKey = "kustomize.toolkit.fluxcd.io/substitute"
 )
 
+// SubstituteOptions defines the options for the variable substitutions operation.
+type SubstituteOptions struct {
+	DryRun bool
+	Strict bool
+}
+
+type SubstituteOption func(a *SubstituteOptions)
+
+// SubstituteWithDryRun sets the dryRun option.
+// When dryRun is true, the substitution process will not attempt to talk to the cluster.
+func SubstituteWithDryRun(dryRun bool) SubstituteOption {
+	return func(a *SubstituteOptions) {
+		a.DryRun = dryRun
+	}
+}
+
+// SubstituteWithStrict sets the strict option.
+// When strict is true, the substitution process will fail if a var without a
+// default value is declared in files but is missing from the input vars.
+func SubstituteWithStrict(strict bool) SubstituteOption {
+	return func(a *SubstituteOptions) {
+		a.Strict = strict
+	}
+}
+
 // SubstituteVariables replaces the vars with their values in the specified resource.
 // If a resource is labeled or annotated with
 // 'kustomize.toolkit.fluxcd.io/substitute: disabled' the substitution is skipped.
-// if dryRun is true, this means we should not attempt to talk to the cluster.
 func SubstituteVariables(
 	ctx context.Context,
 	kubeClient client.Client,
 	kustomization unstructured.Unstructured,
 	res *resource.Resource,
-	dryRun bool) (*resource.Resource, error) {
+	opts ...SubstituteOption) (*resource.Resource, error) {
+	var options SubstituteOptions
+	for _, o := range opts {
+		o(&options)
+	}
+
 	resData, err := res.AsYAML()
 	if err != nil {
 		return nil, err
@@ -71,7 +101,7 @@ func SubstituteVariables(
 	// In dryRun mode this step is skipped. This might in different kind of errors.
 	// But if the user is using dryRun, he/she should know what he/she is doing, and we should comply.
 	var vars map[string]string
-	if !dryRun {
+	if !options.DryRun {
 		vars, err = loadVars(ctx, kubeClient, kustomization)
 		if err != nil {
 			return nil, err
@@ -94,9 +124,9 @@ func SubstituteVariables(
 
 	// run bash variable substitutions
 	if len(vars) > 0 {
-		jsonData, err := varSubstitution(resData, vars)
+		jsonData, err := varSubstitution(resData, vars, options.Strict)
 		if err != nil {
-			return nil, fmt.Errorf("YAMLToJSON: %w", err)
+			return nil, fmt.Errorf("envsubst error: %w", err)
 		}
 		err = res.UnmarshalJSON(jsonData)
 		if err != nil {
@@ -118,25 +148,25 @@ func loadVars(ctx context.Context, kubeClient client.Client, kustomization unstr
 		namespacedName := types.NamespacedName{Namespace: kustomization.GetNamespace(), Name: reference.Name}
 		switch reference.Kind {
 		case "ConfigMap":
-			resource := &corev1.ConfigMap{}
-			if err := kubeClient.Get(ctx, namespacedName, resource); err != nil {
+			cm := &corev1.ConfigMap{}
+			if err := kubeClient.Get(ctx, namespacedName, cm); err != nil {
 				if reference.Optional && apierrors.IsNotFound(err) {
 					continue
 				}
 				return nil, fmt.Errorf("substitute from 'ConfigMap/%s' error: %w", reference.Name, err)
 			}
-			for k, v := range resource.Data {
+			for k, v := range cm.Data {
 				vars[k] = strings.ReplaceAll(v, "\n", "")
 			}
 		case "Secret":
-			resource := &corev1.Secret{}
-			if err := kubeClient.Get(ctx, namespacedName, resource); err != nil {
+			secret := &corev1.Secret{}
+			if err := kubeClient.Get(ctx, namespacedName, secret); err != nil {
 				if reference.Optional && apierrors.IsNotFound(err) {
 					continue
 				}
 				return nil, fmt.Errorf("substitute from 'Secret/%s' error: %w", reference.Name, err)
 			}
-			for k, v := range resource.Data {
+			for k, v := range secret.Data {
 				vars[k] = strings.ReplaceAll(string(v), "\n", "")
 			}
 		}
@@ -145,16 +175,20 @@ func loadVars(ctx context.Context, kubeClient client.Client, kustomization unstr
 	return vars, nil
 }
 
-func varSubstitution(data []byte, vars map[string]string) ([]byte, error) {
+func varSubstitution(data []byte, vars map[string]string, strict bool) ([]byte, error) {
 	r, _ := regexp.Compile(varsubRegex)
 	for v := range vars {
 		if !r.MatchString(v) {
 			return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex)
 		}
 	}
 
-	output, err := envsubst.Eval(string(data), func(s string) string {
-		return vars[s]
+	output, err := envsubst.Eval(string(data), func(s string) (string, bool) {
+		if strict {
+			v, exists := vars[s]
+			return v, exists
+		}
+		return vars[s], true
 	})
 	if err != nil {
 		return nil, fmt.Errorf("variable substitution failed: %w", err)
@@ -194,5 +228,4 @@ func getSubstituteFrom(kustomization unstructured.Unstructured) ([]SubstituteRef
 	}
 
 	return nil, resultErr
-
 }

kustomize/kustomize_varsub_test.go

@@ -22,12 +22,13 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/fluxcd/pkg/kustomize"
 	. "github.com/onsi/gomega"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
+
+	"github.com/fluxcd/pkg/kustomize"
 )
 
-func TestKustomizationVarsub(t *testing.T) {
+func TestKustomization_Varsub(t *testing.T) {
 	g := NewWithT(t)
 
 	// Create a kustomization file with varsub
@@ -50,8 +51,8 @@ func TestKustomizationVarsub(t *testing.T) {
 	resMap, err := kustomize.Build(fs, "./testdata/resources/")
 	g.Expect(err).NotTo(HaveOccurred())
 	for _, res := range resMap.Resources() {
-		outRes, err := kustomize.SubstituteVariables(context.Background(), kubeClient, clientObjects[0], res, false)
-
+		outRes, err := kustomize.SubstituteVariables(context.Background(),
+			kubeClient, clientObjects[0], res)
 		g.Expect(err).NotTo(HaveOccurred())
 
 		if outRes != nil {
@@ -69,4 +70,17 @@ func TestKustomizationVarsub(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	g.Expect(string(resources)).To(Equal(string(expected)))
+
+	// Test with strict mode on
+	strictMapRes, err := kustomize.Build(fs, "./testdata/varsubstrict/")
+	g.Expect(err).NotTo(HaveOccurred())
+	_, err = kustomize.SubstituteVariables(context.Background(),
+		kubeClient, clientObjects[0], strictMapRes.Resources()[0], kustomize.SubstituteWithStrict(true))
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(ContainSubstring("variable not set"))
+
+	// Test with strict mode off
+	_, err = kustomize.SubstituteVariables(context.Background(),
+		kubeClient, clientObjects[0], strictMapRes.Resources()[0], kustomize.SubstituteWithStrict(false))
+	g.Expect(err).ToNot(HaveOccurred())
 }

kustomize/testdata/varsubstrict/config.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    environment: ${cluster_env:=dev}
+    region: ${cluster_region}
+  name: app-vars-strict
+  namespace: apps
+data:
+  missing: ${missing}

kustomize/testdata/varsubstrict/kustomization.yml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: apps
+resources:
+- ./config.yaml