Feat/add snapshot based testing for helm

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/oracle/oci-go-sdk v7.1.0+incompatible
 	github.com/pquerna/otp v1.2.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
 	github.com/tmccombs/hcl2json v0.3.3
 	github.com/urfave/cli v1.22.2
 	github.com/zclconf/go-cty v1.9.1
@@ -47,6 +47,8 @@ require (
 )
 
 require (
+	github.com/gonvenience/ytbx v1.4.4
+	github.com/homeport/dyff v1.6.0
 	cloud.google.com/go/cloudbuild v1.9.0
 	github.com/slack-go/slack v0.10.3
 	gotest.tools/v3 v3.0.3
@@ -63,6 +65,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
@@ -83,6 +86,12 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/gonvenience/bunt v1.3.5 // indirect
+	github.com/gonvenience/neat v1.3.12 // indirect
+	github.com/gonvenience/term v1.0.2 // indirect
+	github.com/gonvenience/text v1.0.7 // indirect
+	github.com/gonvenience/wrap v1.1.2 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -97,9 +106,14 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -110,10 +124,13 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/texttheater/golang-levenshtein v1.0.1 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect

modules/helm/template.go

@@ -11,6 +11,12 @@ import (
 
 	"github.com/gruntwork-io/terratest/modules/files"
 	"github.com/gruntwork-io/terratest/modules/testing"
+
+	"fmt"
+	"os"
+
+	"github.com/gonvenience/ytbx"
+	"github.com/homeport/dyff/pkg/dyff"
 )
 
 // RenderTemplate runs `helm template` to render the template given the provided options and returns stdout/stderr from
@@ -134,3 +140,96 @@ func UnmarshalK8SYamlE(t testing.TestingT, yamlData string, destinationObj inter
 	}
 	return nil
 }
+
+// Add snapshot based testing for helm
+// see https://github.com/gruntwork-io/terratest/issues/1377
+// This function create/update the manifest snapshot of a chart (e.g bitnami/nginx)
+func UpdateSnapshot(yamlData string, releaseName string) {
+
+	snapshotDir := "__snapshot__"
+	// Create a directory if not exists
+	if !files.FileExists(snapshotDir) {
+		os.Mkdir(snapshotDir, 0755)
+	}
+
+	filename := snapshotDir + "/" + releaseName + ".yaml"
+	// Open a file in write mode
+	file, err := os.Create(filename)
+	if err != nil {
+		fmt.Println("Error creating file:", err)
+		return
+	}
+	defer file.Close()
+
+	// Write the k8s manifest into the file
+	_, err = file.WriteString(yamlData)
+	if err != nil {
+		fmt.Println("Error writing to file: ", filename, err)
+		return
+	}
+
+	fmt.Println("k8s manifest written into file: ", filename)
+}
+
+// Add snapshot based testing for helm
+// see https://github.com/gruntwork-io/terratest/issues/1377
+// This function compare the manifest snapshot of a chart (e.g bitnami/nginx) with the current manifest
+func DiffAgainstSnapshot(yamlData string, releaseName string) int {
+
+	snapshotDir := "__snapshot__"
+
+	// load the yaml snapshot file
+	snapshot := snapshotDir + "/" + releaseName + ".yaml"
+	from, err := ytbx.LoadFile(snapshot)
+	if err != nil {
+		fmt.Println("Error opening file:", err)
+		return 1
+	}
+
+	// write the current manifest into a file as `dyff` does not support string input
+	currentManifests := releaseName + ".yaml"
+	file, err := os.Create(currentManifests)
+	if err != nil {
+		fmt.Println("Error creating file:", err)
+		return 1
+	}
+	_, err = file.WriteString(yamlData)
+	if err != nil {
+		fmt.Println("Error writing to file: ", currentManifests, err)
+		return 1
+	}
+	defer file.Close()
+	defer os.Remove(currentManifests)
+
+	to, err := ytbx.LoadFile(currentManifests)
+	if err != nil {
+		fmt.Println("Error opening file:", err)
+		return 1
+	}
+
+	// compare the two manifests using `dyff`
+	compOpt := dyff.KubernetesEntityDetection(false)
+
+	// create a report
+	Report, err := dyff.CompareInputFiles(from, to, compOpt)
+	if err != nil {
+		fmt.Println("Error opening file:", err)
+		return 1
+	}
+
+	// write any difference to stdout
+	reportWriter := &dyff.HumanReport{
+		Report:            Report,
+		DoNotInspectCerts: false,
+		NoTableStyle:      false,
+		OmitHeader:        false,
+		UseGoPatchPaths:   false,
+	}
+
+	writer := os.Stdout
+	reportWriter.WriteReport(writer)
+
+	// return the number of diffs to use in in assertion while testing: 0 = no differences
+	number_of_diffs := len(reportWriter.Diffs)
+	return number_of_diffs
+}

modules/helm/template_test.go

@@ -65,3 +65,79 @@ func TestRemoteChartRender(t *testing.T) {
 	require.Equal(t, len(deploymentContainers), 1)
 	require.Equal(t, deploymentContainers[0].Image, expectedContainerImage)
 }
+
+// Test that we can dump all the manifest locally a remote chart (e.g bitnami/nginx)
+// so that I can use them later to compare between two versions of the same chart for example
+func TestRemoteChartRenderDump(t *testing.T) {
+	const (
+		remoteChartSource  = "https://charts.bitnami.com/bitnami"
+		remoteChartName    = "nginx"
+		remoteChartVersion = "13.2.20"
+		// need to set a fix name for the namespace so it is not flag as a difference
+		namespaceName = "dump-ns"
+	)
+
+	t.Parallel()
+
+	releaseName := remoteChartName
+
+	options := &Options{
+		SetValues: map[string]string{
+			"image.repository": remoteChartName,
+			"image.registry":   "",
+			"image.tag":        remoteChartVersion,
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
+	// we want to assert that the template renders without any errors.
+	output := RenderRemoteTemplate(t, options, remoteChartSource, releaseName, []string{})
+
+	// Now we use kubernetes/client-go library to render the template output into the Deployment struct. This will
+	// ensure the Deployment resource is rendered correctly.
+	var deployment appsv1.Deployment
+	UnmarshalK8SYaml(t, output, &deployment)
+
+	// Verify the namespace matches the expected supplied namespace.
+	require.Equal(t, namespaceName, deployment.Namespace)
+
+	// write chart manifest to a local filesystem directory
+	UpdateSnapshot(output, releaseName)
+}
+
+// Test that we can diff all the manifest to a local snapshot using a remote chart (e.g bitnami/nginx)
+func TestRemoteChartRenderDiff(t *testing.T) {
+	const (
+		remoteChartSource  = "https://charts.bitnami.com/bitnami"
+		remoteChartName    = "nginx"
+		remoteChartVersion = "13.2.23"
+		// need to set a fix name for the namespace so it is not flag as a difference
+		namespaceName = "dump-ns"
+	)
+
+	t.Parallel()
+
+	releaseName := remoteChartName
+	options := &Options{
+		SetValues: map[string]string{
+			"image.repository": remoteChartName,
+			"image.registry":   "",
+			"image.tag":        remoteChartVersion,
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
+	// we want to assert that the template renders without any errors.
+	output := RenderRemoteTemplate(t, options, remoteChartSource, releaseName, []string{})
+
+	// Now we use kubernetes/client-go library to render the template output into the Deployment struct. This will
+	// ensure the Deployment resource is rendered correctly.
+	var deployment appsv1.Deployment
+	UnmarshalK8SYaml(t, output, &deployment)
+
+	// run the diff and assert there is only one difference: the image name
+	number_of_diffs := DiffAgainstSnapshot(output, releaseName)
+	require.Equal(t, 1, number_of_diffs)
+}

test/fixtures/helm/keda-values.yaml

@@ -0,0 +1,30 @@
+metricsServer:
+  replicaCount: 3
+operator:
+  name: keda-operator
+  replicaCount: 3
+podAnnotations:
+  keda:
+    sidecar.istio.io/inject: "false"
+  metricsAdapter:
+    sidecar.istio.io/inject: "false"
+podDisruptionBudget:
+  metricServer:
+    minAvailable: 1
+  operator:
+    minAvailable: 1
+resources:
+  metricServer:
+    limits:
+      cpu: 100m
+      memory: 1234Mi
+    requests:
+      cpu: 50m
+      memory: 128Mi
+  operator:
+    limits:
+      cpu: 100m
+      memory: 1111Mi
+    requests:
+      cpu: 50m
+      memory: 888Mi