Add -errors flag for testing error messages

cmd/toml-test/main.go

@@ -39,6 +39,7 @@ func parseFlags() (tomltest.Runner, []string, int, string, bool, bool) {
 		parallel    = f.Int(runtime.NumCPU(), "parallel")
 		printSkip   = f.Bool(false, "print-skip")
 		intAsFloat  = f.Bool(false, "int-as-float")
+		errors      = f.String("", "errors")
 		// TODO: ideally I'd like to set this even lower, but this stupid
 		// toml-rb is ridiculously slow and sometimes hits ~800ms on my laptop.
 		// See if we can improve that, and should probably split up some of
@@ -67,6 +68,21 @@ func parseFlags() (tomltest.Runner, []string, int, string, bool, bool) {
 	dur, err := time.ParseDuration(timeout.String())
 	zli.F(err)
 
+	var errs map[string]string
+	if errors.Set() {
+		fp, err := os.Open(errors.String())
+		zli.F(err)
+		func() {
+			defer fp.Close()
+			if strings.HasSuffix(errors.String(), ".json") {
+				err = json.NewDecoder(fp).Decode(&errs)
+			} else {
+				_, err = toml.NewDecoder(fp).Decode(&errs)
+			}
+			zli.F(err)
+		}()
+	}
+
 	r := tomltest.Runner{
 		Encoder:    encoder.Bool(),
 		RunTests:   run.StringsSplit(","),
@@ -77,6 +93,7 @@ func parseFlags() (tomltest.Runner, []string, int, string, bool, bool) {
 		Parser:     tomltest.NewCommandParser(fsys, f.Args),
 		Timeout:    dur,
 		IntAsFloat: intAsFloat.Bool(),
+		Errors:     errs,
 	}
 
 	if len(f.Args) == 0 && !listFiles.Bool() {

cmd/toml-test/usage.go

@@ -105,6 +105,22 @@ Flags:
 
     -int-as-float Treat all integers as floats, rather than integers.
 
+    -errors       TOML or JSON file with expected errors for invalid test
+                  files; an invalid test is considered to be "failed" if the
+                  output doesn't contain the string in the file. This is useful
+                  to ensure/test that your errors are what you expect them to
+                  be.
+
+                  The key is the file name, with or without invalid/ or .toml,
+                  and the value is the expected error. For example:
+
+                      "table/equals-sign"              = "expected error text"
+                      "invalid/float/exp-point-1.toml" = "error"
+
+                  It's not an error if a file is missing in the file, but it is
+                  an error if the filename in the errors.toml file doesn't
+                  exist.
+
     -color        Output color; possible values:
 
                        always   Show test failures in bold and red.

runner.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -45,15 +46,16 @@ func EmbeddedTests() fs.FS {
 // The validity of the parameters is not checked extensively; the caller should
 // verify this if need be. See ./cmd/toml-test for an example.
 type Runner struct {
-	Files      fs.FS         // Test files.
-	Encoder    bool          // Are we testing an encoder?
-	RunTests   []string      // Tests to run; run all if blank.
-	SkipTests  []string      // Tests to skip.
-	Parser     Parser        // Send data to a parser.
-	Version    string        // TOML version to run tests for.
-	Parallel   int           // Number of tests to run in parallel
-	Timeout    time.Duration // Maximum time for parse.
-	IntAsFloat bool          // Int values have type=float.
+	Files      fs.FS             // Test files.
+	Encoder    bool              // Are we testing an encoder?
+	RunTests   []string          // Tests to run; run all if blank.
+	SkipTests  []string          // Tests to skip.
+	Parser     Parser            // Send data to a parser.
+	Version    string            // TOML version to run tests for.
+	Parallel   int               // Number of tests to run in parallel
+	Timeout    time.Duration     // Maximum time for parse.
+	IntAsFloat bool              // Int values have type=float.
+	Errors     map[string]string // Expected errors list.
 }
 
 // A Parser instance is used to call the TOML parser we test.
@@ -172,6 +174,17 @@ func (r Runner) Run() (Tests, error) {
 	if r.Timeout == 0 {
 		r.Timeout = 1 * time.Second
 	}
+	if r.Errors == nil {
+		r.Errors = make(map[string]string)
+	}
+	nerr := make(map[string]string)
+	for k, v := range r.Errors {
+		if !strings.HasPrefix(k, "invalid/") {
+			k = path.Join("invalid", k)
+		}
+		nerr[strings.TrimSuffix(k, ".toml")] = v
+	}
+	r.Errors = nerr
 
 	var (
 		tests = Tests{
@@ -184,10 +197,17 @@ func (r Runner) Run() (Tests, error) {
 	)
 	for _, p := range r.RunTests {
 		invalid := strings.Contains(p, "invalid/")
+		t := Test{
+			Path:       p,
+			Encoder:    r.Encoder,
+			Timeout:    r.Timeout,
+			IntAsFloat: r.IntAsFloat,
+		}
 		if r.hasSkip(p) {
 			tests.Skipped++
 			mu.Lock()
-			tests.Tests = append(tests.Tests, Test{Path: p, Skipped: true, Encoder: r.Encoder, Timeout: r.Timeout, IntAsFloat: r.IntAsFloat})
+			t.Skipped = true
+			tests.Tests = append(tests.Tests, t)
 			mu.Unlock()
 			continue
 		}
@@ -197,10 +217,15 @@ func (r Runner) Run() (Tests, error) {
 		go func(p string) {
 			defer func() { <-limit; wg.Done() }()
 
-			t := Test{Path: p, Encoder: r.Encoder, Timeout: r.Timeout, IntAsFloat: r.IntAsFloat}.Run(r.Parser, r.Files)
+			t = t.Run(r.Parser, r.Files)
+
 			mu.Lock()
-			tests.Tests = append(tests.Tests, t)
+			if e, ok := r.Errors[p]; invalid && ok && !t.Failed() && !strings.Contains(t.Output, e) {
+				t.Failure = fmt.Sprintf("%q does not contain %q", t.Output, e)
+			}
+			delete(r.Errors, p)
 
+			tests.Tests = append(tests.Tests, t)
 			if t.Failed() {
 				if invalid {
 					tests.FailedInvalid++
@@ -224,6 +249,13 @@ func (r Runner) Run() (Tests, error) {
 			strings.Replace(tests.Tests[j].Path, "invalid/", "zinvalid", 1)
 	})
 
+	if len(r.Errors) > 0 {
+		keys := make([]string, 0, len(r.Errors))
+		for k := range r.Errors {
+			keys = append(keys, k)
+		}
+		return tests, fmt.Errorf("errors didn't match anything: %q", keys)
+	}
 	return tests, nil
 }
 

runner_test.go

@@ -1,8 +1,12 @@
 package tomltest
 
 import (
+	"context"
+	"fmt"
 	"os"
+	"strings"
 	"testing"
+	"testing/fstest"
 )
 
 func notInList(t *testing.T, list []string, str string) {
@@ -34,3 +38,73 @@ func TestVersion(t *testing.T) {
 	}
 	notInList(t, ls, "valid/string/escape-esc")
 }
+
+type testParser struct{}
+
+func (t *testParser) Encode(ctx context.Context, input string) (output string, outputIsError bool, err error) {
+	switch input {
+	case `a=1`:
+		return `{"a": {"type":"integer","value":"1"}}`, false, nil
+	case `a=`, `c=`:
+		return `oh noes: error one`, true, nil
+	case `b=`:
+		return `error two`, true, nil
+	default:
+		panic(fmt.Sprintf("unreachable: %q", input))
+	}
+}
+
+func (t testParser) Decode(ctx context.Context, input string) (string, bool, error) {
+	return t.Encode(ctx, input)
+}
+
+func TestErrors(t *testing.T) {
+	r := Runner{
+		Parser: &testParser{},
+		Files: fstest.MapFS{
+			"valid/a.toml":       &fstest.MapFile{Data: []byte(`a=1`)},
+			"valid/a.json":       &fstest.MapFile{Data: []byte(`{"a": {"type":"integer","value":"1"}}`)},
+			"invalid/a.toml":     &fstest.MapFile{Data: []byte(`a=`)},
+			"invalid/b.toml":     &fstest.MapFile{Data: []byte(`b=`)},
+			"invalid/dir/c.toml": &fstest.MapFile{Data: []byte(`c=`)},
+		},
+		Errors: map[string]string{
+			"invalid/a":  "oh noes",
+			"invalid/b":  "don't match",
+			"dir/c.toml": "oh noes",
+		},
+	}
+	tt, err := r.Run()
+	if err != nil {
+		t.Error(err)
+	}
+	for _, test := range tt.Tests {
+		if test.Path == "invalid/b" {
+			if !test.Failed() {
+				t.Errorf("expected failure for %q, but got none", test.Path)
+			}
+			continue
+		}
+
+		if test.Failed() {
+			t.Errorf("\n%s: %s", test.Path, test.Failure)
+		}
+	}
+
+	t.Run("non-existent", func(t *testing.T) {
+		r := Runner{
+			Parser: &testParser{},
+			Files:  fstest.MapFS{},
+			Errors: map[string]string{
+				"file/doesn/exist": "oh noes",
+			},
+		}
+		_, err := r.Run()
+		if err == nil {
+			t.Fatal("error is nil")
+		}
+		if !strings.Contains(err.Error(), "didn't match anything") {
+			t.Fatalf("wrong error: %s", err)
+		}
+	})
+}