Merge pull request #155 from rquinio/feature/assert-schema-validation

Support failedTemplate assert for schema validation errors

DOCUMENT.md

@@ -183,7 +183,7 @@ Available assertion types are listed below:
 | `containsDocument` | **kind**: *string*. Expected `kind` of manifest.<br/> **apiVersion**: *string*. Expected `apiVersion` of manifest.<br/>**name**: *string, optional*. The value of the `metadata.name`.<br/>**namespace**: *string, optional*. The value of the `metadata.namespace`. | Asserts the documents rendered by the `kind` and `apiVersion` specified. | <pre>containsDocument:<br/>  kind: Deployment<br/>  apiVersion: apps/v1<br/>  name:foo<br/>  namespace: bar</pre> |
 | `matchSnapshot` | **path**: *string*. The `set` path for snapshot. | Assert the value of **path** is the same as snapshotted last time. Check [doc](./README.md#snapshot-testing) below. | <pre>matchSnapshot:<br/>  path: spec</pre> |
 | `matchSnapshotRaw` | | Assert the value in the NOTES.txt is the same as snapshotted last time. Check [doc](./README.md#snapshot-testing) below. | <pre>matchSnapshotRaw: {}<br/></pre> |
-| `failedTemplate` | **errorMessage**: *string*. The (human readable) `errorMessage` that should occur. | Assert the value of **errorMessage** is the same as the human readable template error. | <pre>failedTemplate:<br/>  errorMessage: Required value<br/></pre> |
+| `failedTemplate` | **errorMessage**: *string*. The (human readable) `errorMessage` that should occur. | Assert the value of **errorMessage** is the same as the human readable template rendering error. Also allows to match an error that would happen before template execution (ex: validation of values against schema) | <pre>failedTemplate:<br/>  errorMessage: Required value<br/></pre> |
 
 
 ### Antonym and `not`

pkg/unittest/.snapshots/TestV3RunJobWithSchema

@@ -0,0 +1,21 @@
+(*results.TestJobResult)({
+  DisplayName: (string) (len=11) "should work",
+  Index: (int) 0,
+  Passed: (bool) true,
+  ExecError: (*errors.errorString)(values don't meet the specifications of the schema(s) in the following chart(s):
+with-schema:
+- (root): image is required
+),
+  AssertsResult: ([]*results.AssertionResult) (len=1) {
+    (*results.AssertionResult)({
+      Index: (int) 0,
+      FailInfo: ([]string) {
+      },
+      Passed: (bool) true,
+      AssertType: (string) (len=14) "failedTemplate",
+      Not: (bool) false,
+      CustomInfo: (string) ""
+    })
+  },
+  Duration: (time.Duration) 0s
+})

pkg/unittest/.snapshots/TestV3RunJobWithTooLongReleaseName

@@ -0,0 +1,20 @@
+(*results.TestJobResult)({
+  DisplayName: (string) (len=19) "to long releasename",
+  Index: (int) 0,
+  Passed: (bool) false,
+  ExecError: (*errors.fundamental)(invalid release name, must match regex ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ and the length must not be longer than 53),
+  AssertsResult: ([]*results.AssertionResult) (len=1) {
+    (*results.AssertionResult)({
+      Index: (int) 0,
+      FailInfo: ([]string) (len=2) {
+        (string) (len=6) "Error:",
+        (string) (len=84) "\ttemplate \"basic/templates/crd_backup.yaml\" not exists or not selected in test suite"
+      },
+      Passed: (bool) false,
+      AssertType: (string) (len=12) "hasDocuments",
+      Not: (bool) false,
+      CustomInfo: (string) ""
+    })
+  },
+  Duration: (time.Duration) 0s
+})

pkg/unittest/assertion.go

@@ -28,6 +28,7 @@ func (a *Assertion) Assert(
 	templatesResult map[string][]common.K8sManifest,
 	snapshotComparer validators.SnapshotComparer,
 	renderSucceed bool,
+	renderError error,
 	result *results.AssertionResult,
 ) *results.AssertionResult {
 	result.AssertType = a.AssertType
@@ -41,7 +42,7 @@ func (a *Assertion) Assert(
 		rendered, ok := templatesResult[template]
 		var validatePassed bool
 		var singleFailInfo []string
-		if !ok {
+		if !ok && a.requireRenderSuccess {
 			noFile := []string{"Error:", a.noFileErrMessage(template)}
 			failInfo = append(failInfo, noFile...)
 			assertionPassed = false
@@ -59,6 +60,7 @@ func (a *Assertion) Assert(
 			Index:            a.DocumentIndex,
 			Negative:         a.Not != a.antonym,
 			SnapshotComparer: snapshotComparer,
+			RenderError:      renderError,
 		})
 
 		if !validatePassed {

pkg/unittest/assertion_test.go

@@ -24,7 +24,7 @@ func validateSucceededTestAssertions(
 	a.Nil(err)
 
 	for idx, assertion := range assertions {
-		result := assertion.Assert(renderedMap, fakeSnapshotComparer(true), true, &results.AssertionResult{Index: idx})
+		result := assertion.Assert(renderedMap, fakeSnapshotComparer(true), true, nil, &results.AssertionResult{Index: idx})
 		a.Equal(&results.AssertionResult{
 			Index:      idx,
 			FailInfo:   []string{},
@@ -285,7 +285,7 @@ equal:
 	a := assert.New(t)
 	a.Nil(err)
 
-	result := assertion.Assert(renderedMap, fakeSnapshotComparer(true), true, &results.AssertionResult{Index: 0})
+	result := assertion.Assert(renderedMap, fakeSnapshotComparer(true), true, nil, &results.AssertionResult{Index: 0})
 	a.Equal(&results.AssertionResult{
 		Index:      0,
 		FailInfo:   []string{"Error:", "\ttemplate \"not-existed.yaml\" not exists or not selected in test suite"},
@@ -306,7 +306,7 @@ func TestAssertionAssertWhenTemplateNotSpecifiedAndNoDefault(t *testing.T) {
 	yaml.Unmarshal([]byte(assertionYAML), &assertion)
 
 	a := assert.New(t)
-	result := assertion.Assert(renderedMap, fakeSnapshotComparer(true), true, &results.AssertionResult{Index: 0})
+	result := assertion.Assert(renderedMap, fakeSnapshotComparer(true), true, nil, &results.AssertionResult{Index: 0})
 	a.Equal(&results.AssertionResult{
 		Index:      0,
 		FailInfo:   []string{"Error:", "\tassertion.template must be given if testsuite.templates is empty"},

pkg/unittest/test_job.go

@@ -212,6 +212,7 @@ func (t *TestJob) RunV2(
 		manifestsOfFiles,
 		snapshotComparer,
 		renderSucceed,
+		nil,
 		failfast,
 	)
 
@@ -236,10 +237,10 @@ func (t *TestJob) RunV3(
 		return result
 	}
 
-	outputOfFiles, renderSucceed, err := t.renderV3Chart(targetChart, userValues)
-	if err != nil {
-		result.ExecError = err
-		return result
+	outputOfFiles, renderSucceed, renderError := t.renderV3Chart(targetChart, userValues)
+	if renderError != nil {
+		result.ExecError = renderError
+		// Continue to enable matching error via failedTemplate assert
 	}
 
 	manifestsOfFiles, err := t.parseManifestsFromOutputOfFiles(outputOfFiles)
@@ -253,6 +254,7 @@ func (t *TestJob) RunV3(
 		manifestsOfFiles,
 		snapshotComparer,
 		renderSucceed,
+		renderError,
 		failfast,
 	)
 
@@ -631,7 +633,7 @@ func (t *TestJob) parseManifestsFromOutputOfFiles(outputOfFiles map[string]strin
 func (t *TestJob) runAssertions(
 	manifestsOfFiles map[string][]common.K8sManifest,
 	snapshotComparer validators.SnapshotComparer,
-	renderSucceed, failfast bool,
+	renderSucceed bool, renderError error, failfast bool,
 ) (bool, []*results.AssertionResult) {
 	testPass := true
 	assertsResult := make([]*results.AssertionResult, 0)
@@ -641,6 +643,7 @@ func (t *TestJob) runAssertions(
 			manifestsOfFiles,
 			snapshotComparer,
 			renderSucceed,
+			renderError,
 			&results.AssertionResult{Index: idx},
 		)
 

pkg/unittest/test_job_test.go

@@ -702,7 +702,7 @@ asserts:
 	a.Equal(1, len(testResult.AssertsResult))
 }
 
-func TestV3RunJobWithToLongReleaseName(t *testing.T) {
+func TestV3RunJobWithTooLongReleaseName(t *testing.T) {
 	c, _ := loader.Load(testV3BasicChart)
 	manifest := `
 it: to long releasename
@@ -784,7 +784,7 @@ asserts:
 	a.Equal(2, len(testResult.AssertsResult))
 }
 
-func TestV3RunJobWithFailingTempalte(t *testing.T) {
+func TestV3RunJobWithFailingTemplate(t *testing.T) {
 	c, _ := loader.Load(testV3WithFailingTemplateChart)
 	manifest := `
 it: should work
@@ -805,3 +805,25 @@ asserts:
 	a.True(testResult.Passed)
 	a.Equal(1, len(testResult.AssertsResult))
 }
+
+func TestV3RunJobWithSchema(t *testing.T) {
+	c, _ := loader.Load(testV3WithSchemaChart)
+	manifest := `
+it: should work
+template: dummy.yaml
+asserts:
+  - failedTemplate:
+      errorMessage: "values don't meet the specifications of the schema(s) in the following chart(s):\nwith-schema:\n- (root): image is required\n"
+`
+	var tj TestJob
+	yaml.Unmarshal([]byte(manifest), &tj)
+
+	testResult := tj.RunV3(c, &snapshot.Cache{}, true, &results.TestJobResult{})
+
+	a := assert.New(t)
+	cupaloy.SnapshotT(t, makeTestJobResultSnapshotable(testResult))
+
+	a.NotNil(testResult.ExecError)
+	a.True(testResult.Passed)
+	a.Equal(1, len(testResult.AssertsResult))
+}

pkg/unittest/test_suite_test.go

@@ -32,6 +32,7 @@ const testV3WithSubChart string = "../../test/data/v3/with-subchart"
 const testV3WithSubFolderChart string = "../../test/data/v3/with-subfolder"
 const testV3WithSubSubFolderChart string = "../../test/data/v3/with-subsubcharts"
 const testV3WithFailingTemplateChart string = "../../test/data/v3/failing-template"
+const testV3WithSchemaChart string = "../../test/data/v3/with-schema"
 
 var tmpdir, _ = ioutil.TempDir("", testSuiteTests)
 

pkg/unittest/validators/common.go

@@ -27,6 +27,7 @@ type ValidateContext struct {
 	Index    int
 	Negative bool
 	SnapshotComparer
+	RenderError error
 }
 
 func (c *ValidateContext) getManifests() ([]common.K8sManifest, error) {

pkg/unittest/validators/failed_template_validator.go

@@ -40,17 +40,27 @@ func (a FailedTemplateValidator) Validate(context *ValidateContext) (bool, []str
 	validateSuccess := false
 	validateErrors := make([]string, 0)
 
-	for idx, manifest := range manifests {
-		actual := manifest[common.RAW]
-
-		if reflect.DeepEqual(a.ErrorMessage, actual) == context.Negative {
+	if context.RenderError != nil {
+		validateSuccess = true
+		if reflect.DeepEqual(a.ErrorMessage, context.RenderError.Error()) == context.Negative {
 			validateSuccess = false
-			errorMessage := a.failInfo(actual, idx, context.Negative)
+			errorMessage := a.failInfo(context.RenderError.Error(), -1, context.Negative)
 			validateErrors = append(validateErrors, errorMessage...)
-			continue
 		}
 
-		validateSuccess = determineSuccess(idx, validateSuccess, true)
+	} else {
+		for idx, manifest := range manifests {
+			actual := manifest[common.RAW]
+
+			if reflect.DeepEqual(a.ErrorMessage, actual) == context.Negative {
+				validateSuccess = false
+				errorMessage := a.failInfo(actual, idx, context.Negative)
+				validateErrors = append(validateErrors, errorMessage...)
+				continue
+			}
+
+			validateSuccess = determineSuccess(idx, validateSuccess, true)
+		}
 	}
 
 	return validateSuccess, validateErrors

pkg/unittest/validators/failed_template_validator_test.go

@@ -1,6 +1,7 @@
 package validators_test
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/lrills/helm-unittest/internal/common"
@@ -86,3 +87,15 @@ func TestFailedTemplateValidatorWhenInvalidIndex(t *testing.T) {
 		"	documentIndex 2 out of range",
 	}, diff)
 }
+
+func TestFailedTemplateValidatorWhenRenderError(t *testing.T) {
+	validator := FailedTemplateValidator{"values don't meet the specifications of the schema(s)"}
+	pass, diff := validator.Validate(&ValidateContext{
+		Docs:        []common.K8sManifest{},
+		Index:       -1,
+		RenderError: errors.New("values don't meet the specifications of the schema(s)"),
+	})
+
+	assert.True(t, pass)
+	assert.Equal(t, []string{}, diff)
+}

test/data/v3/with-schema/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v2
+description: A chart with a JSON schema
+name: with-schema
+version: 0.1.0

test/data/v3/with-schema/templates/dummy.yaml

@@ -0,0 +1 @@
+

test/data/v3/with-schema/values.schema.json

@@ -0,0 +1,26 @@
+{
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "required": [
+      "image"
+    ],
+    "properties": {
+      "image": {
+        "type": "object",
+        "required": [
+          "repository",
+          "pullPolicy"
+        ],
+        "properties": {
+          "repository": {
+            "type": "string",
+            "pattern": "^[a-z0-9-_]+$"
+          },
+          "pullPolicy": {
+            "type": "string",
+            "pattern": "^(Always|Never|IfNotPresent)$"
+          }
+        }
+      }
+    }
+  }