all: add testing for compiler error messages

This is needed for some improvements I'm going to make next.

This commit also refactors error handling slightly to make it more
easily testable, this should hopefully not result in any actual changes
in behavior.

Author: aykevl

errors_test.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+)
+
+// Test the error messages of the TinyGo compiler.
+func TestErrors(t *testing.T) {
+	for _, name := range []string{
+		"cgo",
+		"syntax",
+		"types",
+	} {
+		t.Run(name, func(t *testing.T) {
+			testErrorMessages(t, "./testdata/errors/"+name+".go")
+		})
+	}
+}
+
+func testErrorMessages(t *testing.T, filename string) {
+	// Parse expected error messages.
+	expected := readErrorMessages(t, filename)
+
+	// Try to build a binary (this should fail with an error).
+	tmpdir := t.TempDir()
+	err := Build(filename, tmpdir+"/out", &compileopts.Options{
+		Target:        "wasip1",
+		Semaphore:     sema,
+		InterpTimeout: 180 * time.Second,
+		Debug:         true,
+		VerifyIR:      true,
+		Opt:           "z",
+	})
+	if err == nil {
+		t.Fatal("expected to get a compiler error")
+	}
+
+	// Get the full ./testdata/errors directory.
+	wd, absErr := filepath.Abs("testdata/errors")
+	if absErr != nil {
+		t.Fatal(absErr)
+	}
+
+	// Write error message out as plain text.
+	var buf bytes.Buffer
+	printCompilerError(err, func(v ...interface{}) {
+		fmt.Fprintln(&buf, v...)
+	}, wd)
+	actual := strings.TrimRight(buf.String(), "\n")
+
+	// Check whether the error is as expected.
+	if actual != expected {
+		t.Errorf("expected error:\n%s\ngot:\n%s", indentText(expected, "> "), indentText(actual, "> "))
+	}
+}
+
+// Indent the given text with a given indentation string.
+func indentText(text, indent string) string {
+	return indent + strings.ReplaceAll(text, "\n", "\n"+indent)
+}
+
+// Read "// ERROR:" prefixed messages from the given file.
+func readErrorMessages(t *testing.T, file string) string {
+	data, err := os.ReadFile(file)
+	if err != nil {
+		t.Fatal("could not read input file:", err)
+	}
+
+	var errors []string
+	for _, line := range strings.Split(string(data), "\n") {
+		if strings.HasPrefix(line, "// ERROR: ") {
+			errors = append(errors, strings.TrimRight(line[len("// ERROR: "):], "\r\n"))
+		}
+	}
+	return strings.Join(errors, "\n")
+}

main.go

@@ -1293,10 +1293,9 @@ func usage(command string) {
 
 // try to make the path relative to the current working directory. If any error
 // occurs, this error is ignored and the absolute path is returned instead.
-func tryToMakePathRelative(dir string) string {
-	wd, err := os.Getwd()
-	if err != nil {
-		return dir
+func tryToMakePathRelative(dir, wd string) string {
+	if wd == "" {
+		return dir // working directory not found
 	}
 	relpath, err := filepath.Rel(wd, dir)
 	if err != nil {
@@ -1307,28 +1306,25 @@ func tryToMakePathRelative(dir string) string {
 
 // printCompilerError prints compiler errors using the provided logger function
 // (similar to fmt.Println).
-//
-// There is one exception: interp errors may print to stderr unconditionally due
-// to limitations in the LLVM bindings.
-func printCompilerError(logln func(...interface{}), err error) {
+func printCompilerError(err error, logln func(...interface{}), wd string) {
 	switch err := err.(type) {
 	case types.Error:
-		printCompilerError(logln, scanner.Error{
+		printCompilerError(scanner.Error{
 			Pos: err.Fset.Position(err.Pos),
 			Msg: err.Msg,
-		})
+		}, logln, wd)
 	case scanner.Error:
 		if !strings.HasPrefix(err.Pos.Filename, filepath.Join(goenv.Get("GOROOT"), "src")) && !strings.HasPrefix(err.Pos.Filename, filepath.Join(goenv.Get("TINYGOROOT"), "src")) {
 			// This file is not from the standard library (either the GOROOT or
 			// the TINYGOROOT). Make the path relative, for easier reading.
 			// Ignore any errors in the process (falling back to the absolute
 			// path).
-			err.Pos.Filename = tryToMakePathRelative(err.Pos.Filename)
+			err.Pos.Filename = tryToMakePathRelative(err.Pos.Filename, wd)
 		}
 		logln(err)
 	case scanner.ErrorList:
 		for _, scannerErr := range err {
-			printCompilerError(logln, *scannerErr)
+			printCompilerError(*scannerErr, logln, wd)
 		}
 	case *interp.Error:
 		logln("#", err.ImportPath)
@@ -1346,7 +1342,7 @@ func printCompilerError(logln func(...interface{}), err error) {
 	case loader.Errors:
 		logln("#", err.Pkg.ImportPath)
 		for _, err := range err.Errs {
-			printCompilerError(logln, err)
+			printCompilerError(err, logln, wd)
 		}
 	case loader.Error:
 		logln(err.Err.Error())
@@ -1356,7 +1352,7 @@ func printCompilerError(logln func(...interface{}), err error) {
 		}
 	case *builder.MultiError:
 		for _, err := range err.Errs {
-			printCompilerError(logln, err)
+			printCompilerError(err, logln, wd)
 		}
 	default:
 		logln("error:", err)
@@ -1365,9 +1361,13 @@ func printCompilerError(logln func(...interface{}), err error) {
 
 func handleCompilerError(err error) {
 	if err != nil {
-		printCompilerError(func(args ...interface{}) {
+		wd, getwdErr := os.Getwd()
+		if getwdErr != nil {
+			wd = ""
+		}
+		printCompilerError(err, func(args ...interface{}) {
 			fmt.Fprintln(os.Stderr, args...)
-		}, err)
+		}, wd)
 		os.Exit(1)
 	}
 }
@@ -1769,9 +1769,13 @@ func main() {
 				stderr := (*testStderr)(buf)
 				passed, err := Test(pkgName, stdout, stderr, options, outpath)
 				if err != nil {
-					printCompilerError(func(args ...interface{}) {
+					wd, getwdErr := os.Getwd()
+					if getwdErr != nil {
+						wd = ""
+					}
+					printCompilerError(err, func(args ...interface{}) {
 						fmt.Fprintln(stderr, args...)
-					}, err)
+					}, wd)
 				}
 				if !passed {
 					select {

main_test.go

@@ -380,7 +380,7 @@ func runTestWithConfig(name string, t *testing.T, options compileopts.Options, c
 		return cmd.Run()
 	})
 	if err != nil {
-		printCompilerError(t.Log, err)
+		printCompilerError(err, t.Log, "")
 		t.Fail()
 		return
 	}

testdata/errors/cgo.go

@@ -0,0 +1,15 @@
+package main
+
+// #error hello
+// )))
+import "C"
+
+func main() {
+}
+
+// TODO: this error should be relative to the current directory (so cgo.go
+// instead of testdata/errors/cgo.go).
+
+// ERROR: # command-line-arguments
+// ERROR: testdata/errors/cgo.go:3:5: error: hello
+// ERROR: testdata/errors/cgo.go:4:4: error: expected identifier or '('

testdata/errors/syntax.go

@@ -0,0 +1,7 @@
+package main
+
+func main(var) { // syntax error
+}
+
+// ERROR: # command-line-arguments
+// ERROR: syntax.go:3:11: expected ')', found 'var'

testdata/errors/types.go

@@ -0,0 +1,12 @@
+package main
+
+func main() {
+	var a int
+	a = "foobar"
+	nonexisting()
+}
+
+// ERROR: # command-line-arguments
+// ERROR: types.go:5:6: cannot use "foobar" (untyped string constant) as int value in assignment
+// ERROR: types.go:6:2: undefined: nonexisting
+// ERROR: types.go:4:6: a declared and not used