Feat/add snapshot based testing for helm

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/oracle/oci-go-sdk v7.1.0+incompatible
 	github.com/pquerna/otp v1.2.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
 	github.com/tmccombs/hcl2json v0.3.3
 	github.com/urfave/cli v1.22.2
 	github.com/zclconf/go-cty v1.9.1
@@ -48,6 +48,8 @@ require (
 
 require (
 	cloud.google.com/go/cloudbuild v1.9.0
+	github.com/gonvenience/ytbx v1.4.4
+	github.com/homeport/dyff v1.6.0
 	github.com/slack-go/slack v0.10.3
 	gotest.tools/v3 v3.0.3
 )
@@ -63,6 +65,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
@@ -83,6 +86,11 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/gonvenience/bunt v1.3.5 // indirect
+	github.com/gonvenience/neat v1.3.12 // indirect
+	github.com/gonvenience/term v1.0.2 // indirect
+	github.com/gonvenience/text v1.0.7 // indirect
+	github.com/gonvenience/wrap v1.1.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -97,9 +105,14 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -110,10 +123,13 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/texttheater/golang-levenshtein v1.0.1 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect

modules/helm/options.go

@@ -18,4 +18,5 @@ type Options struct {
 	Logger            *logger.Logger      // Set a non-default logger that should be used. See the logger package for more info. Use logger.Discard to not print the output while executing the command.
 	ExtraArgs         map[string][]string // Extra arguments to pass to the helm install/upgrade/rollback/delete and helm repo add commands. The key signals the command (e.g., install) while the values are the extra arguments to pass through.
 	BuildDependencies bool                // If true, helm dependencies will be built before rendering template, installing or upgrade the chart.
+	SnapshotPath      string              // The path to the snapshot directory when using snapshot based testing. Empty string means use default ($PWD/__snapshot__).
 }

modules/helm/template.go

@@ -11,6 +11,11 @@ import (
 
 	"github.com/gruntwork-io/terratest/modules/files"
 	"github.com/gruntwork-io/terratest/modules/testing"
+
+	"os"
+
+	"github.com/gonvenience/ytbx"
+	"github.com/homeport/dyff/pkg/dyff"
 )
 
 // RenderTemplate runs `helm template` to render the template given the provided options and returns stdout/stderr from
@@ -134,3 +139,123 @@ func UnmarshalK8SYamlE(t testing.TestingT, yamlData string, destinationObj inter
 	}
 	return nil
 }
+
+// UpdateSnapshot creates or updates the k8s manifest snapshot of a chart (e.g bitnami/nginx).
+// It is one of the two functions needed to implement snapshot based testing for helm.
+// see https://github.com/gruntwork-io/terratest/issues/1377
+// A snapshot is used to compare the current manifests of a chart with the previous manifests.
+// A global diff is run against the two snapshosts and the number of differences is returned.
+func UpdateSnapshot(t testing.TestingT, options *Options, yamlData string, releaseName string) {
+	require.NoError(t, UpdateSnapshotE(t, options, yamlData, releaseName))
+}
+
+// UpdateSnapshot creates or updates the k8s manifest snapshot of a chart (e.g bitnami/nginx).
+// It is one of the two functions needed to implement snapshot based testing for helm.
+// see https://github.com/gruntwork-io/terratest/issues/1377
+// A snapshot is used to compare the current manifests of a chart with the previous manifests.
+// A global diff is run against the two snapshosts and the number of differences is returned.
+// It will failed the test if there is an error while writing the manifests' snapshot in the file system
+func UpdateSnapshotE(t testing.TestingT, options *Options, yamlData string, releaseName string) error {
+
+	var snapshotDir = "__snapshot__"
+	if options.SnapshotPath != "" {
+		snapshotDir = options.SnapshotPath
+	}
+	// Create a directory if not exists
+	if !files.FileExists(snapshotDir) {
+		if err := os.Mkdir(snapshotDir, 0755); err != nil {
+			return errors.WithStackTrace(err)
+		}
+	}
+
+	filename := filepath.Join(snapshotDir, releaseName+".yaml")
+	// Open a file in write mode
+	file, err := os.Create(filename)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	defer file.Close()
+
+	// Write the k8s manifest into the file
+	if _, err = file.WriteString(yamlData); err != nil {
+		return errors.WithStackTrace(err)
+	}
+
+	if options.Logger != nil {
+		options.Logger.Logf(t, "helm chart manifest written into file: %s", filename)
+	}
+	return nil
+}
+
+// DiffAgainstSnapshot compare the current manifests of a chart (e.g bitnami/nginx)
+// with the previous manifests stored in the snapshot.
+// see https://github.com/gruntwork-io/terratest/issues/1377
+// It returns the number of difference between the two manifest snaphost or -1 in case of error
+// It will failed the test if there is an error while reading or writing the two manifests in the file system
+func DiffAgainstSnapshot(t testing.TestingT, options *Options, yamlData string, releaseName string) int {
+	numberOfDiffs, err := DiffAgainstSnapshotE(t, options, yamlData, releaseName)
+	require.NoError(t, err)
+	return numberOfDiffs
+}
+
+// DiffAgainstSnapshotE compare the current manifests of a chart (e.g bitnami/nginx)
+// with the previous manifests stored in the snapshot.
+// see https://github.com/gruntwork-io/terratest/issues/1377
+// It returns the number of difference between the two manifest snaphost or -1 in case of error
+func DiffAgainstSnapshotE(t testing.TestingT, options *Options, yamlData string, releaseName string) (int, error) {
+
+	var snapshotDir = "__snapshot__"
+	if options.SnapshotPath != "" {
+		snapshotDir = options.SnapshotPath
+	}
+
+	// load the yaml snapshot file
+	snapshot := filepath.Join(snapshotDir, releaseName+".yaml")
+	from, err := ytbx.LoadFile(snapshot)
+	if err != nil {
+		return -1, errors.WithStackTrace(err)
+	}
+
+	// write the current manifest into a file as `dyff` does not support string input
+	currentManifests := releaseName + ".yaml"
+	file, err := os.Create(currentManifests)
+	if err != nil {
+		return -1, errors.WithStackTrace(err)
+	}
+
+	if _, err = file.WriteString(yamlData); err != nil {
+		return -1, errors.WithStackTrace(err)
+	}
+	defer file.Close()
+	defer os.Remove(currentManifests)
+
+	to, err := ytbx.LoadFile(currentManifests)
+	if err != nil {
+		return -1, errors.WithStackTrace(err)
+	}
+
+	// compare the two manifests using `dyff`
+	compOpt := dyff.KubernetesEntityDetection(false)
+
+	// create a report
+	report, err := dyff.CompareInputFiles(from, to, compOpt)
+	if err != nil {
+		return -1, errors.WithStackTrace(err)
+	}
+
+	// write any difference to stdout
+	reportWriter := &dyff.HumanReport{
+		Report:            report,
+		DoNotInspectCerts: false,
+		NoTableStyle:      false,
+		OmitHeader:        false,
+		UseGoPatchPaths:   false,
+	}
+
+	err = reportWriter.WriteReport(os.Stdout)
+	if err != nil {
+		return -1, errors.WithStackTrace(err)
+	}
+	// return the number of diffs to use in assertion while testing: 0 = no differences
+	return len(reportWriter.Diffs), nil
+}

modules/helm/template_test.go

@@ -17,6 +17,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/random"
 )
 
@@ -25,7 +26,7 @@ func TestRemoteChartRender(t *testing.T) {
 	const (
 		remoteChartSource  = "https://charts.bitnami.com/bitnami"
 		remoteChartName    = "nginx"
-		remoteChartVersion = "13.2.23"
+		remoteChartVersion = "13.2.24"
 	)
 
 	t.Parallel()
@@ -45,6 +46,7 @@ func TestRemoteChartRender(t *testing.T) {
 			"image.tag":        remoteChartVersion,
 		},
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+		Logger:         logger.Discard,
 	}
 
 	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
@@ -65,3 +67,81 @@ func TestRemoteChartRender(t *testing.T) {
 	require.Equal(t, len(deploymentContainers), 1)
 	require.Equal(t, deploymentContainers[0].Image, expectedContainerImage)
 }
+
+// Test that we can dump all the manifest locally a remote chart (e.g bitnami/nginx)
+// so that I can use them later to compare between two versions of the same chart for example
+func TestRemoteChartRenderDump(t *testing.T) {
+	const (
+		remoteChartSource  = "https://charts.bitnami.com/bitnami"
+		remoteChartName    = "nginx"
+		remoteChartVersion = "13.2.20"
+		// need to set a fix name for the namespace so it is not flag as a difference
+		namespaceName = "dump-ns"
+	)
+
+	releaseName := remoteChartName
+
+	options := &Options{
+		SetValues: map[string]string{
+			"image.repository": remoteChartName,
+			"image.registry":   "",
+			"image.tag":        remoteChartVersion,
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+		Logger:         logger.Discard,
+	}
+
+	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
+	// we want to assert that the template renders without any errors.
+	output := RenderRemoteTemplate(t, options, remoteChartSource, releaseName, []string{})
+
+	// Now we use kubernetes/client-go library to render the template output into the Deployment struct. This will
+	// ensure the Deployment resource is rendered correctly.
+	var deployment appsv1.Deployment
+	UnmarshalK8SYaml(t, output, &deployment)
+
+	// Verify the namespace matches the expected supplied namespace.
+	require.Equal(t, namespaceName, deployment.Namespace)
+
+	// write chart manifest to a local filesystem directory
+	options = &Options{
+		Logger:       logger.Default,
+		SnapshotPath: "__chart_manifests_snapshot__",
+	}
+	UpdateSnapshot(t, options, output, releaseName)
+}
+
+// Test that we can diff all the manifest to a local snapshot using a remote chart (e.g bitnami/nginx)
+func TestRemoteChartRenderDiff(t *testing.T) {
+	const (
+		remoteChartSource  = "https://charts.bitnami.com/bitnami"
+		remoteChartName    = "nginx"
+		remoteChartVersion = "13.2.24"
+		// need to set a fix name for the namespace so it is not flag as a difference
+		namespaceName = "dump-ns"
+	)
+
+	releaseName := remoteChartName
+	options := &Options{
+		SetValues: map[string]string{
+			"image.repository": remoteChartName,
+			"image.registry":   "",
+			"image.tag":        remoteChartVersion,
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+		Logger:         logger.Discard,
+		SnapshotPath:   "__chart_manifests_snapshot__",
+	}
+
+	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
+	// we want to assert that the template renders without any errors.
+	output := RenderRemoteTemplate(t, options, remoteChartSource, releaseName, []string{})
+
+	// Now we use kubernetes/client-go library to render the template output into the Deployment struct. This will
+	// ensure the Deployment resource is rendered correctly.
+	var deployment appsv1.Deployment
+	UnmarshalK8SYaml(t, output, &deployment)
+
+	// run the diff and assert there is only one difference: the image name
+	require.Equal(t, 1, DiffAgainstSnapshot(t, options, output, releaseName))
+}

test/fixtures/helm/keda-values.yaml

@@ -0,0 +1,30 @@
+metricsServer:
+  replicaCount: 3
+operator:
+  name: keda-operator
+  replicaCount: 3
+podAnnotations:
+  keda:
+    sidecar.istio.io/inject: "false"
+  metricsAdapter:
+    sidecar.istio.io/inject: "false"
+podDisruptionBudget:
+  metricServer:
+    minAvailable: 1
+  operator:
+    minAvailable: 1
+resources:
+  metricServer:
+    limits:
+      cpu: 100m
+      memory: 1234Mi
+    requests:
+      cpu: 50m
+      memory: 128Mi
+  operator:
+    limits:
+      cpu: 100m
+      memory: 1111Mi
+    requests:
+      cpu: 50m
+      memory: 888Mi