print snapshot summary, enhance printing

__fixtures__/basic/tests/deployment_test.yaml

@@ -9,7 +9,6 @@ tests:
       a.b.c: ABC
       x.y.z: XYZ
     asserts:
-      # - matchSnapshot:
       - equal:
           path: spec.template.spec.containers[0].image
           value: apache:latest

unittest/printer.go

@@ -16,6 +16,7 @@ type loggable interface {
 	warning(content string) string
 	warningBackground(content string) string
 	highlight(content string) string
+	faint(content string) string
 }
 
 type Printer struct {
@@ -71,6 +72,14 @@ func (p Printer) warningBackground(content string) string {
 	return "[" + content + "]"
 }
 
+var bold = color.New(color.Bold)
+
 func (p Printer) highlight(content string) string {
-	return color.WhiteString(content)
+	return bold.Sprint(content)
+}
+
+var faint = color.New(color.Faint)
+
+func (p Printer) faint(content string) string {
+	return faint.Sprint(content)
 }

unittest/snapshot/cache.go

@@ -12,7 +12,7 @@ import (
 type CompareResult struct {
 	Passed bool
 	Test   string
-	Index  int
+	Index  uint
 	New    string
 	Cached string
 }
@@ -22,8 +22,8 @@ type Cache struct {
 	Filepath   string
 	Existed    bool
 	IsUpdating bool
-	cached     map[string]map[int]string
-	new        map[string]map[int]string
+	cached     map[string]map[uint]string
+	new        map[string]map[uint]string
 	updated    bool
 }
 
@@ -44,7 +44,7 @@ func (s *Cache) RestoreFromFile() error {
 	return nil
 }
 
-func (s *Cache) getCached(test string, idx int) (string, bool) {
+func (s *Cache) getCached(test string, idx uint) (string, bool) {
 	if cachedByTest, ok := s.cached[test]; ok {
 		if cachedOfAssertion, ok := cachedByTest[idx]; ok {
 			return cachedOfAssertion, true
@@ -54,7 +54,7 @@ func (s *Cache) getCached(test string, idx int) (string, bool) {
 }
 
 // Compare compare content to cached last time, return CompareResult
-func (s *Cache) Compare(test string, idx int, content interface{}) *CompareResult {
+func (s *Cache) Compare(test string, idx uint, content interface{}) *CompareResult {
 	newSnapshot := s.saveNewCache(test, idx, content)
 	match, cachedSnapshot := s.compareToCached(test, idx, newSnapshot)
 	return &CompareResult{
@@ -66,7 +66,7 @@ func (s *Cache) Compare(test string, idx int, content interface{}) *CompareResul
 	}
 }
 
-func (s *Cache) compareToCached(test string, idx int, snapshot string) (bool, string) {
+func (s *Cache) compareToCached(test string, idx uint, snapshot string) (bool, string) {
 	cached, ok := s.getCached(test, idx)
 	if !ok {
 		s.updated = true
@@ -80,15 +80,15 @@ func (s *Cache) compareToCached(test string, idx int, snapshot string) (bool, st
 	return true, cached
 }
 
-func (s *Cache) saveNewCache(test string, idx int, content interface{}) string {
+func (s *Cache) saveNewCache(test string, idx uint, content interface{}) string {
 	snapshot := common.TrustedMarshalYAML(content)
 	if s.new == nil {
-		s.new = make(map[string]map[int]string)
+		s.new = make(map[string]map[uint]string)
 	}
 	if newCacheOfTest, ok := s.new[test]; ok {
 		newCacheOfTest[idx] = snapshot
 	} else {
-		s.new[test] = map[int]string{idx: snapshot}
+		s.new[test] = map[uint]string{idx: snapshot}
 	}
 	return snapshot
 }

unittest/test_job.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/lrills/helm-unittest/unittest/common"
 	"github.com/lrills/helm-unittest/unittest/snapshot"
+	"github.com/lrills/helm-unittest/unittest/validators"
 	"github.com/lrills/helm-unittest/unittest/valueutils"
 	yaml "gopkg.in/yaml.v2"
 	"k8s.io/helm/pkg/chartutil"
@@ -17,14 +18,26 @@ import (
 )
 
 type orderedSnapshotComparer struct {
-	cache   *snapshot.Cache
-	test    string
-	counter int
+	cache       *snapshot.Cache
+	test        string
+	counter     uint
+	failedCount uint
 }
 
-func (s orderedSnapshotComparer) CompareToSnapshot(content interface{}) *snapshot.CompareResult {
+func (s *orderedSnapshotComparer) CompareToSnapshot(content interface{}) *snapshot.CompareResult {
 	s.counter++
-	return s.cache.Compare(s.test, s.counter, content)
+	result := s.cache.Compare(s.test, s.counter, content)
+	if !result.Passed {
+		s.failedCount++
+	}
+	return result
+}
+
+func (s orderedSnapshotComparer) TatalCount() uint {
+	return s.counter
+}
+func (s orderedSnapshotComparer) FailedCount() uint {
+	return s.failedCount
 }
 
 // TestJob defintion of a test, including values and assertions
@@ -69,7 +82,15 @@ func (t *TestJob) Run(
 		return result
 	}
 
-	result.Passed, result.AssertsResult = t.runAssertions(manifestsOfFiles, cache)
+	snapshotComparer := &orderedSnapshotComparer{cache: cache, test: t.Name}
+	result.Passed, result.AssertsResult = t.runAssertions(
+		manifestsOfFiles,
+		snapshotComparer,
+	)
+
+	result.FailedSnapshotCount = snapshotComparer.FailedCount()
+	result.TotalSnapshotCount = snapshotComparer.TatalCount()
+
 	return result
 }
 
@@ -165,17 +186,18 @@ func (t *TestJob) parseManifestsFromOutputOfFiles(outputOfFiles map[string]strin
 
 func (t *TestJob) runAssertions(
 	manifestsOfFiles map[string][]common.K8sManifest,
-	cache *snapshot.Cache,
+	comparer validators.SnapshotComparer,
 ) (bool, []*AssertionResult) {
 	testPass := true
 	assertsResult := make([]*AssertionResult, len(t.Assertions))
 
 	for idx, assertion := range t.Assertions {
 		result := assertion.Assert(
 			manifestsOfFiles,
-			&orderedSnapshotComparer{cache: cache, test: t.Name},
+			comparer,
 			&AssertionResult{Index: idx},
 		)
+
 		assertsResult[idx] = result
 		testPass = testPass && result.Passed
 	}

unittest/test_job_result.go

@@ -1,11 +1,13 @@
 package unittest
 
 type TestJobResult struct {
-	DisplayName   string
-	Index         int
-	Passed        bool
-	ExecError     error
-	AssertsResult []*AssertionResult
+	DisplayName         string
+	Index               int
+	Passed              bool
+	ExecError           error
+	AssertsResult       []*AssertionResult
+	TotalSnapshotCount  uint
+	FailedSnapshotCount uint
 }
 
 func (tjr TestJobResult) print(printer loggable, verbosity int) {

unittest/test_runner.go

@@ -32,9 +32,10 @@ func getTestSuiteFiles(chartPath string, patterns []string) ([]string, error) {
 }
 
 type testUnitCounting struct {
-	passed  uint
-	failed  uint
-	errored uint
+	passed         uint
+	failed         uint
+	errored        uint
+	snapshotFailed uint
 }
 
 func (counting testUnitCounting) sprint(logger loggable) string {
@@ -53,11 +54,17 @@ func (counting testUnitCounting) sprint(logger loggable) string {
 	)
 }
 
+type testUnitCountingWithSnapshot struct {
+	testUnitCounting
+	snapshotFailed uint
+}
+
 type TestRunner struct {
-	ChartsPath    []string
-	suiteCounting testUnitCounting
-	testCounting  testUnitCounting
-	chartCounting testUnitCounting
+	ChartsPath       []string
+	suiteCounting    testUnitCountingWithSnapshot
+	testCounting     testUnitCountingWithSnapshot
+	chartCounting    testUnitCounting
+	snapshotCounting testUnitCounting
 }
 
 func (tr *TestRunner) Run(logger loggable, config TestConfig) bool {
@@ -81,17 +88,18 @@ func (tr *TestRunner) Run(logger loggable, config TestConfig) bool {
 		}
 
 		tr.printChartHeader(logger, chart, chartPath)
-		chartPassed, suitesResult := tr.runSuites(suiteFiles, chart, logger, config)
+		chartPassed, suitesResult := tr.testChart(suiteFiles, chart, logger, config)
 		tr.printSuiteResult(logger, suitesResult)
 
 		tr.countChart(chartPassed, nil)
 		allPassed = allPassed && chartPassed
 	}
+	tr.printSnapshotSummary(logger)
 	tr.printSummary(logger, time.Now().Sub(start))
 	return allPassed
 }
 
-func (tr *TestRunner) runSuites(
+func (tr *TestRunner) testChart(
 	suiteFiles []string,
 	chart *chart.Chart,
 	logger loggable,
@@ -139,6 +147,7 @@ func (tr *TestRunner) printSummary(logger loggable, elapsed time.Duration) {
 Charts:      %s
 Test Suites: %s
 Tests:       %s
+Snapshot:    %s
 Time:        %s
 `
 	logger.println(
@@ -147,6 +156,7 @@ Time:        %s
 			tr.chartCounting.sprint(logger),
 			tr.suiteCounting.sprint(logger),
 			tr.testCounting.sprint(logger),
+			tr.snapshotCounting.sprint(logger),
 			elapsed.String(),
 		),
 		0,
@@ -158,7 +168,7 @@ func (tr *TestRunner) printChartHeader(logger loggable, chart *chart.Chart, path
 	headerFormat := `
 ### Chart [ %s ] %s
 `
-	header := fmt.Sprintf(headerFormat, logger.highlight(chart.Metadata.Name), path)
+	header := fmt.Sprintf(headerFormat, logger.highlight(chart.Metadata.Name), logger.faint(path))
 	logger.println(header, 0)
 }
 
@@ -170,6 +180,19 @@ func (tr *TestRunner) printErroredChartHeader(logger loggable, err error) {
 	logger.println(header, 0)
 }
 
+func (tr *TestRunner) printSnapshotSummary(logger loggable) {
+	if tr.snapshotCounting.failed > 0 {
+		snapshotFormat := `
+		Snapshot Summary: %s`
+
+		summary := logger.danger(fmt.Sprintf("%d snapshot failed", tr.snapshotCounting.failed)) +
+			fmt.Sprintf(" in %d test suite.", tr.suiteCounting.snapshotFailed) +
+			logger.faint(" Check changes and use `-u` to update snapshot.")
+
+		logger.println(fmt.Sprintf(snapshotFormat, summary), 0)
+	}
+}
+
 func (tr *TestRunner) countSuite(suite *TestSuiteResult) {
 	if suite.Passed {
 		tr.suiteCounting.passed++
@@ -178,6 +201,9 @@ func (tr *TestRunner) countSuite(suite *TestSuiteResult) {
 		if suite.ExecError != nil {
 			tr.suiteCounting.errored++
 		}
+		if suite.HasSnapshotFail {
+			tr.suiteCounting.snapshotFailed++
+		}
 	}
 }
 
@@ -189,7 +215,12 @@ func (tr *TestRunner) countTest(test *TestJobResult) {
 		if test.ExecError != nil {
 			tr.testCounting.errored++
 		}
+		if test.FailedSnapshotCount > 0 {
+			tr.testCounting.snapshotFailed++
+		}
 	}
+	tr.snapshotCounting.failed += test.FailedSnapshotCount
+	tr.snapshotCounting.passed += test.TotalSnapshotCount - test.FailedSnapshotCount
 }
 
 func (tr *TestRunner) countChart(passed bool, err error) {

unittest/test_suite.go

@@ -56,7 +56,10 @@ func (s *TestSuite) Run(
 		return result
 	}
 
-	result.Passed, result.TestsResult = s.runTestJobs(preparedChart, snapshotCache)
+	result.Passed, result.HasSnapshotFail, result.TestsResult = s.runTestJobs(
+		preparedChart,
+		snapshotCache,
+	)
 	return result
 }
 
@@ -103,15 +106,24 @@ func (s *TestSuite) prepareChart(targetChart *chart.Chart) (*chart.Chart, error)
 	return copiedChart, nil
 }
 
-func (s *TestSuite) runTestJobs(chart *chart.Chart, cache *snapshot.Cache) (bool, []*TestJobResult) {
+func (s *TestSuite) runTestJobs(
+	chart *chart.Chart,
+	cache *snapshot.Cache,
+) (bool, bool, []*TestJobResult) {
 	suitePass := true
+	hasSnapshotFail := false
 	jobResults := make([]*TestJobResult, len(s.Tests))
+
 	for idx, testJob := range s.Tests {
 		jobResult := testJob.Run(chart, cache, &TestJobResult{Index: idx})
 		jobResults[idx] = jobResult
+
 		if !jobResult.Passed {
 			suitePass = false
+			if jobResult.FailedSnapshotCount > 0 {
+				hasSnapshotFail = true
+			}
 		}
 	}
-	return suitePass, jobResults
+	return suitePass, hasSnapshotFail, jobResults
 }

unittest/test_suite_result.go

@@ -7,11 +7,12 @@ import (
 )
 
 type TestSuiteResult struct {
-	DisplayName string
-	FilePath    string
-	Passed      bool
-	ExecError   error
-	TestsResult []*TestJobResult
+	DisplayName     string
+	FilePath        string
+	Passed          bool
+	ExecError       error
+	TestsResult     []*TestJobResult
+	HasSnapshotFail bool
 }
 
 func (tsr TestSuiteResult) print(printer loggable, verbosity int) {
@@ -36,9 +37,8 @@ func (tsr TestSuiteResult) printTitle(printer loggable) {
 	}
 	var pathToPrint string
 	if tsr.FilePath != "" {
-		pathToPrint = filepath.Dir(tsr.FilePath) +
-			string(os.PathSeparator) +
-			printer.highlight(filepath.Base(tsr.FilePath))
+		pathToPrint = printer.faint(filepath.Dir(tsr.FilePath)+string(os.PathSeparator)) +
+			filepath.Base(tsr.FilePath)
 	}
 	name := printer.highlight(tsr.DisplayName)
 	printer.println(

unittest/validators/snapshot_validator.go

@@ -19,7 +19,7 @@ func (v MatchSnapshotValidator) failInfo(compared *snapshot.CompareResult, not b
 	}
 	snapshotFailFormat := `
 Path:%s
-Expected` + notAnnotation + ` to match snapshot ` + strconv.Itoa(compared.Index) + `:
+Expected` + notAnnotation + ` to match snapshot ` + strconv.Itoa(int(compared.Index)) + `:
 %s
 `
 	var infoToShow string