Introducing FailureIsFatal feature

If set to true (via DefaultContextConfig.FailureIsFatal or using
T.FailureIsFatal), Fatal() method is called instead of Error() to
report tests failures.

Signed-off-by: Maxime Soulé <btik-git@scoubidou.com>

README.md

@@ -6,12 +6,25 @@ go-testdeep
 [![Go Report Card](https://goreportcard.com/badge/github.com/maxatome/go-testdeep)](https://goreportcard.com/report/github.com/maxatome/go-testdeep)
 [![GoDoc](https://godoc.org/github.com/maxatome/go-testdeep?status.svg)](https://godoc.org/github.com/maxatome/go-testdeep)
 
-Golang package `testdeep` allows extremely flexible deep comparison,
-built for testing.
+Extremely flexible golang deep comparison, extends the go testing package.
+
+- [Latest new](#latest-news)
+- [Synopsis](#synopsis)
+- [Installation](#installation)
+- [Presentation](#presentation)
+- [Available operators](#available-operators)
+- [License](#license)
 
 
 ## Latest news
 
+- 2018/06/19: new
+  [ContextConfig](https://godoc.org/github.com/maxatome/go-testdeep#ContextConfig)
+  feature `FailureIsFatal` available. See
+  [DefaultContextConfig](https://godoc.org/github.com/maxatome/go-testdeep#pkg-variables)
+  for default global value and
+  [`T.FailureIsFatal`](https://godoc.org/github.com/maxatome/go-testdeep#T.FailureIsFatal)
+  method;
 - 2018/06/17: new
   [`CmpPanic`](https://godoc.org/github.com/maxatome/go-testdeep#CmpPanic)
   &
@@ -27,7 +40,6 @@ built for testing.
   [`CmpSmuggle`](https://godoc.org/github.com/maxatome/go-testdeep#CmpSmuggle)
   &
   [`T.Smuggle`](https://godoc.org/github.com/maxatome/go-testdeep#T.Smuggle));
-- 2018/06/11: `DefaultContextConfig.MaxErrors` defaults to 10 (was 1);
 - see [commits history](https://github.com/maxatome/go-testdeep/commits/master)
   for other/older changes.
 
@@ -56,15 +68,16 @@ func TestCreateRecord(t *testing.T) {
   record, err := CreateRecord("Bob", 23)
 
   if td.CmpNoError(t, err) {
-    td.CmpStruct(t, record,
-      &Record{
-        Name: "Bob",
-        Age:  23,
-      },
-      td.StructFields{
-        "Id":        td.NotZero(),
-        "CreatedAt": td.Between(before, time.Now()),
-      },
+    td.CmpDeeply(t, record,
+      td.Struct(
+        &Record{
+          Name: "Bob",
+          Age:  23,
+        },
+        td.StructFields{
+          "Id":        td.NotZero(),
+          "CreatedAt": td.Between(before, time.Now()),
+        }),
       "Newly created record")
   }
 }
@@ -255,7 +268,7 @@ func TestCreateRecord(t *testing.T) {
 
   if td.CmpDeeply(t, err, nil) {
     td.CmpDeeply(t, record,
-      Struct(
+      td.Struct(
         Record{
           Name: "Bob",
           Age:  23,

cmp_deeply.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 )
 
-func formatError(t TestingT, err *Error, args ...interface{}) {
+func formatError(t TestingT, isFatal bool, err *Error, args ...interface{}) {
 	t.Helper()
 
 	const failedTest = "Failed test"
@@ -40,7 +40,11 @@ func formatError(t TestingT, err *Error, args ...interface{}) {
 
 	err.Append(buf, "")
 
-	t.Error(buf.String())
+	if isFatal {
+		t.Fatal(buf.String())
+	} else {
+		t.Error(buf.String())
+	}
 }
 
 func cmpDeeply(ctx Context, t TestingT, got, expected interface{},
@@ -51,7 +55,7 @@ func cmpDeeply(ctx Context, t TestingT, got, expected interface{},
 		return true
 	}
 	t.Helper()
-	formatError(t, err, args...)
+	formatError(t, ctx.failureIsFatal, err, args...)
 	return false
 }
 

cmp_deeply_test.go

@@ -8,22 +8,9 @@ package testdeep
 
 import (
 	"bytes"
-	"fmt"
 	"testing"
 )
 
-type TestTestingT struct {
-	LastMessage string
-}
-
-func (t *TestTestingT) Error(args ...interface{}) {
-	t.LastMessage = fmt.Sprint(args...)
-}
-
-func (t *TestTestingT) Helper() {
-	// Do nothing
-}
-
 func TestFormatError(t *testing.T) {
 	ttt := &TestTestingT{}
 
@@ -35,41 +22,49 @@ func TestFormatError(t *testing.T) {
 
 	nonStringName := bytes.NewBufferString("zip!")
 
-	//
-	// Without args
-	formatError(ttt, err)
-	equalStr(t, ttt.LastMessage, `Failed test
+	for _, fatal := range []bool{false, true} {
+		//
+		// Without args
+		formatError(ttt, fatal, err)
+		equalStr(t, ttt.LastMessage, `Failed test
 DATA: test error message
 	test error summary`)
+		equalBool(t, ttt.IsFatal, fatal)
 
-	//
-	// With one arg
-	formatError(ttt, err, "foo bar!")
-	equalStr(t, ttt.LastMessage, `Failed test 'foo bar!'
+		//
+		// With one arg
+		formatError(ttt, fatal, err, "foo bar!")
+		equalStr(t, ttt.LastMessage, `Failed test 'foo bar!'
 DATA: test error message
 	test error summary`)
+		equalBool(t, ttt.IsFatal, fatal)
 
-	formatError(ttt, err, nonStringName)
-	equalStr(t, ttt.LastMessage, `Failed test 'zip!'
+		formatError(ttt, fatal, err, nonStringName)
+		equalStr(t, ttt.LastMessage, `Failed test 'zip!'
 DATA: test error message
 	test error summary`)
+		equalBool(t, ttt.IsFatal, fatal)
 
-	//
-	// With several args & Printf format
-	formatError(ttt, err, "hello %d!", 123)
-	equalStr(t, ttt.LastMessage, `Failed test 'hello 123!'
+		//
+		// With several args & Printf format
+		formatError(ttt, fatal, err, "hello %d!", 123)
+		equalStr(t, ttt.LastMessage, `Failed test 'hello 123!'
 DATA: test error message
 	test error summary`)
+		equalBool(t, ttt.IsFatal, fatal)
 
-	//
-	// With several args without Printf format
-	formatError(ttt, err, "hello ", "world! ", 123)
-	equalStr(t, ttt.LastMessage, `Failed test 'hello world! 123'
+		//
+		// With several args without Printf format
+		formatError(ttt, fatal, err, "hello ", "world! ", 123)
+		equalStr(t, ttt.LastMessage, `Failed test 'hello world! 123'
 DATA: test error message
 	test error summary`)
+		equalBool(t, ttt.IsFatal, fatal)
 
-	formatError(ttt, err, nonStringName, "hello ", "world! ", 123)
-	equalStr(t, ttt.LastMessage, `Failed test 'zip!hello world! 123'
+		formatError(ttt, fatal, err, nonStringName, "hello ", "world! ", 123)
+		equalStr(t, ttt.LastMessage, `Failed test 'zip!hello world! 123'
 DATA: test error message
 	test error summary`)
+		equalBool(t, ttt.IsFatal, fatal)
+	}
 }

cmp_funcs_misc.go

@@ -45,6 +45,7 @@ func cmpError(ctx Context, t TestingT, got error, args ...interface{}) bool {
 
 	t.Helper()
 	formatError(t,
+		ctx.failureIsFatal,
 		&Error{
 			Context:  ctx,
 			Message:  "should be an error",
@@ -63,6 +64,7 @@ func cmpNoError(ctx Context, t TestingT, got error, args ...interface{}) bool {
 
 	t.Helper()
 	formatError(t,
+		ctx.failureIsFatal,
 		&Error{
 			Context:  ctx,
 			Message:  "should NOT be an error",
@@ -125,6 +127,7 @@ func cmpPanic(ctx Context, t TestingT, fn func(), expected interface{}, args ...
 
 	if !panicked {
 		formatError(t,
+			ctx.failureIsFatal,
 			&Error{
 				Context: ctx,
 				Message: "should have panicked",
@@ -174,6 +177,7 @@ func cmpNotPanic(ctx Context, t TestingT, fn func(), args ...interface{}) bool {
 	}
 
 	formatError(t,
+		ctx.failureIsFatal,
 		&Error{
 			Context:  ctx,
 			Message:  "should NOT have panicked",

context.go

@@ -36,6 +36,11 @@ type ContextConfig struct {
 	// Setting it to a negative number means no limit: all errors
 	// will be dumped.
 	MaxErrors int
+	// FailureIsFatal allows to Fatal() (instead of Error()) when a test
+	// fails. Using *testing.T instance as
+	// t.TestingFT value, FailNow() is called behind the scenes when
+	// Fatal() is called. See testing documentation for details.
+	FailureIsFatal bool
 }
 
 const (
@@ -59,8 +64,9 @@ func getMaxErrorsFromEnv() int {
 // tests failures. If overridden, new settings will impact all Cmp*
 // functions and *T methods (if not specifically configured.)
 var DefaultContextConfig = ContextConfig{
-	RootName:  contextDefaultRootName,
-	MaxErrors: getMaxErrorsFromEnv(),
+	RootName:       contextDefaultRootName,
+	MaxErrors:      getMaxErrorsFromEnv(),
+	FailureIsFatal: false,
 }
 
 func (c *ContextConfig) sanitize() {
@@ -96,6 +102,8 @@ type Context struct {
 	// < 0 do not stop until comparison ends.
 	maxErrors int
 	errors    *[]*Error
+	// See ContextConfig.FailureIsFatal for details
+	failureIsFatal bool
 }
 
 // NewContext creates a new Context using DefaultContextConfig configuration.
@@ -108,9 +116,10 @@ func NewContextWithConfig(config ContextConfig) (ctx Context) {
 	config.sanitize()
 
 	ctx = Context{
-		path:      config.RootName,
-		visited:   map[visit]bool{},
-		maxErrors: config.MaxErrors,
+		path:           config.RootName,
+		visited:        map[visit]bool{},
+		maxErrors:      config.MaxErrors,
+		failureIsFatal: config.FailureIsFatal,
 	}
 
 	ctx.initErrors()

context_test.go

@@ -35,6 +35,18 @@ func equalInt(t *testing.T, got, expected int) bool {
 	return false
 }
 
+func equalBool(t *testing.T, got, expected bool) bool {
+	if got == expected {
+		return true
+	}
+
+	t.Helper()
+	t.Errorf(`Failed test
+	     got: %t
+	expected: %t`, got, expected)
+	return false
+}
+
 func TestContext(t *testing.T) {
 	equalStr(t, NewContext().path, "DATA")
 	equalStr(t, NewBooleanContext().path, "")

t_struct.go

@@ -166,6 +166,37 @@ func (t *T) RootName(rootName string) *T {
 	return &new
 }
 
+// FailureIsFatal allows to choose whether t.TestingFT.Fatal() or
+// t.TestingFT.Error() will be used to print the next failure
+// reports. When "enable" is true (or missing) testing.Fatal() will be
+// called, else testing.Error(). Using *testing.T instance as
+// t.TestingFT value, FailNow() is called behind the scenes when
+// Fatal() is called. See testing documentation for details.
+//
+// It returns a new instance of *T so does not alter the original t
+// and used as follows:
+//
+//   // Following t.CmpDeeply() will call Fatal() if failure
+//   t = t.FailureIsFatal()
+//   t.CmpDeeply(...)
+//   t.CmpDeeply(...)
+//   // Following t.CmpDeeply() won't call Fatal() if failure
+//   t = t.FailureIsFatal(false)
+//   t.CmpDeeply(...)
+//
+// or, if only one call is critic:
+//
+//   // This CmpDeeply() call will call Fatal() if failure
+//   t.FailureIsFatal().CmpDeeply(...)
+//   // Following t.CmpDeeply() won't call Fatal() if failure
+//   t.CmpDeeply(...)
+//   t.CmpDeeply(...)
+func (t *T) FailureIsFatal(enable ...bool) *T {
+	new := *t
+	new.Config.FailureIsFatal = len(enable) == 0 || enable[0]
+	return &new
+}
+
 // CmpDeeply is mostly a shortcut for:
 //
 //   CmpDeeply(t.TestingFT, got, expected, args...)

t_struct_test.go

@@ -200,3 +200,39 @@ func TestRun(tt *testing.T) {
 	t.True(ok)
 	t.True(runPassed)
 }
+
+func TestFailureIsFatal(tt *testing.T) {
+	ttt := &TestTestingFT{}
+
+	// All t.True(false) tests of course fail
+
+	// Using default config
+	t := NewT(ttt)
+	t.True(false) // failure
+	CmpNotEmpty(tt, ttt.LastMessage)
+	CmpFalse(tt, ttt.IsFatal, "by default it not fatal")
+
+	// Using specific config
+	t = NewT(ttt, ContextConfig{FailureIsFatal: true})
+	t.True(false) // failure
+	CmpNotEmpty(tt, ttt.LastMessage)
+	CmpTrue(tt, ttt.IsFatal, "it must be fatal")
+
+	// Using FailureIsFatal()
+	t = NewT(ttt).FailureIsFatal()
+	t.True(false) // failure
+	CmpNotEmpty(tt, ttt.LastMessage)
+	CmpTrue(tt, ttt.IsFatal, "it must be fatal")
+
+	// Using FailureIsFatal(true)
+	t = NewT(ttt).FailureIsFatal(true)
+	t.True(false) // failure
+	CmpNotEmpty(tt, ttt.LastMessage)
+	CmpTrue(tt, ttt.IsFatal, "it must be fatal")
+
+	// Canceling specific config
+	t = NewT(ttt, ContextConfig{FailureIsFatal: false}).FailureIsFatal(false)
+	t.True(false) // failure
+	CmpNotEmpty(tt, ttt.LastMessage)
+	CmpFalse(tt, ttt.IsFatal, "it must be not fatal")
+}