Conformance tests for the runtime and generated lexers.

The goal is to have a single lexer definition that exercises all the
functionality of the stateful lexer and generated equivalent.

See #264

cmd/participle/files/codegen.go.tmpl

@@ -1,4 +1,6 @@
 // Code generated by Participle. DO NOT EDIT.
+{{if .Tags}}//go:build {{.Tags}}
+{{end -}}
 package {{.Package}}
 
 import (
@@ -12,6 +14,7 @@ import (
 )
 
 var _ syntax.Op
+const _ = utf8.RuneError
 
 var {{.Name}}Lexer lexer.Definition = lexer{{.Name}}DefinitionImpl{}
 

cmd/participle/gen_lexer_cmd.go

@@ -18,8 +18,9 @@ import (
 type genLexerCmd struct {
 	Name    string `help:"Name of the lexer."`
 	Output  string `short:"o" help:"Output file."`
+	Tags    string `help:"Build tags to include in the generated file."`
 	Package string `arg:"" required:"" help:"Go package for generated code."`
-	Lexer   string `arg:"" required:"" default:"-" type:"existingfile" help:"JSON representation of a Participle lexer."`
+	Lexer   string `arg:"" required:"" default:"-" type:"existingfile" help:"JSON representation of a Participle lexer (read from stdin if omitted)."`
 }
 
 func (c *genLexerCmd) Help() string {
@@ -52,7 +53,15 @@ func (c *genLexerCmd) Run() error {
 	if err != nil {
 		return err
 	}
-	err = generateLexer(os.Stdout, c.Package, def, c.Name)
+	out := os.Stdout
+	if c.Output != "" {
+		out, err = os.Create(c.Output)
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+	}
+	err = generateLexer(out, c.Package, def, c.Name, c.Tags)
 	if err != nil {
 		return err
 	}
@@ -89,14 +98,15 @@ var (
 	}).Parse(codegenTemplateSource))
 )
 
-func generateLexer(w io.Writer, pkg string, def *lexer.StatefulDefinition, name string) error {
+func generateLexer(w io.Writer, pkg string, def *lexer.StatefulDefinition, name, tags string) error {
 	type ctx struct {
 		Package string
 		Name    string
+		Tags    string
 		Def     *lexer.StatefulDefinition
 	}
 	rules := def.Rules()
-	err := codegenTemplate.Execute(w, ctx{pkg, name, def})
+	err := codegenTemplate.Execute(w, ctx{pkg, name, tags, def})
 	if err != nil {
 		return err
 	}

lexer/internal/conformance/conformance_codegen_test.go

@@ -0,0 +1,14 @@
+//go:build generated
+
+package conformance_test
+
+import (
+	"testing"
+
+	"github.com/alecthomas/participle/v2/lexer/internal/conformance"
+)
+
+// This should only be run by TestLexerConformanceGenerated.
+func TestLexerConformanceGeneratedInternal(t *testing.T) {
+	testLexer(t, conformance.GeneratedConformanceLexer)
+}

lexer/internal/conformance/conformance_test.go

@@ -0,0 +1,154 @@
+package conformance_test
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/alecthomas/assert/v2"
+	"github.com/alecthomas/participle/v2/lexer"
+)
+
+var conformanceLexer = lexer.MustStateful(lexer.Rules{
+	"Root": {
+		{"String", `"`, lexer.Push("String")},
+		{"Heredoc", `<<(\w+\b)`, lexer.Push("Heredoc")},
+	},
+	"String": {
+		{"Escaped", `\\.`, nil},
+		{"StringEnd", `"`, lexer.Pop()},
+		{"Expr", `\${`, lexer.Push("Expr")},
+		{"Char", `[^$"\\]+`, nil},
+	},
+	"Expr": {
+		lexer.Include("Root"),
+		{`Whitespace`, `\s+`, nil},
+		{`Oper`, `[-+/*%]`, nil},
+		{"Ident", `\w+`, lexer.Push("Reference")},
+		{"ExprEnd", `}`, lexer.Pop()},
+	},
+	"Reference": {
+		{"Dot", `\.`, nil},
+		{"Ident", `\w+`, nil},
+		lexer.Return(),
+	},
+	"Heredoc": {
+		{"End", `\b\1\b`, lexer.Pop()},
+		lexer.Include("Expr"),
+	},
+})
+
+type token struct {
+	Type  string
+	Value string
+}
+
+func testLexer(t *testing.T, lex lexer.Definition) {
+	t.Helper()
+	tests := []struct {
+		name     string
+		input    string
+		expected []token
+	}{
+		{"Push", `"${"Hello ${name + "!"}"}"`, []token{
+			{"String", "\""},
+			{"Expr", "${"},
+			{"String", "\""},
+			{"Char", "Hello "},
+			{"Expr", "${"},
+			{"Ident", "name"},
+			{"Whitespace", " "},
+			{"Oper", "+"},
+			{"Whitespace", " "},
+			{"String", "\""},
+			{"Char", "!"},
+			{"StringEnd", "\""},
+			{"ExprEnd", "}"},
+			{"StringEnd", "\""},
+			{"ExprEnd", "}"},
+			{"StringEnd", "\""},
+		}},
+		{"Reference", `"${user.name}"`, []token{
+			{"String", "\""},
+			{"Expr", "${"},
+			{"Ident", "user"},
+			{"Dot", "."},
+			{"Ident", "name"},
+			{"ExprEnd", "}"},
+			{"StringEnd", "\""},
+		}},
+	}
+	symbols := lexer.SymbolsByRune(lex)
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			l, err := lex.Lex(test.name, strings.NewReader(test.input))
+			assert.NoError(t, err)
+			tokens, err := lexer.ConsumeAll(l)
+			assert.NoError(t, err)
+			actual := make([]token, len(tokens)-1)
+			for i, t := range tokens {
+				if t.Type == lexer.EOF {
+					continue
+				}
+				actual[i] = token{Type: symbols[t.Type], Value: t.Value}
+			}
+			assert.Equal(t, test.expected, actual)
+		})
+	}
+}
+
+func TestLexerConformanceGenerated(t *testing.T) {
+	genLexer(t)
+	args := []string{"test", "-run", "TestLexerConformanceGeneratedInternal", "-tags", "generated"}
+	// Propagate test flags.
+	flag.CommandLine.VisitAll(func(f *flag.Flag) {
+		if f.Value.String() != f.DefValue {
+			args = append(args, fmt.Sprintf("-%s=%s", f.Name, f.Value.String()))
+		}
+	})
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	assert.NoError(t, err)
+}
+
+func TestLexerConformance(t *testing.T) {
+	testLexer(t, conformanceLexer)
+}
+
+func genLexer(t *testing.T) {
+	t.Helper()
+	lexerJSON, err := json.Marshal(conformanceLexer)
+	assert.NoError(t, err)
+	cwd, err := os.Getwd()
+	assert.NoError(t, err)
+	generatedConformanceLexer := filepath.Join(cwd, "conformance_lexer_gen.go")
+	t.Cleanup(func() {
+		_ = os.Remove(generatedConformanceLexer)
+	})
+	cmd := exec.Command(
+		"../../../scripts/participle",
+		"gen", "lexer", "conformance",
+		"--tags", "generated",
+		"--name", "GeneratedConformance",
+		"--output", generatedConformanceLexer)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	w, err := cmd.StdinPipe()
+	assert.NoError(t, err)
+	defer w.Close()
+	err = cmd.Start()
+	assert.NoError(t, err)
+	_, err = w.Write(lexerJSON)
+	assert.NoError(t, err)
+	err = w.Close()
+	assert.NoError(t, err)
+	err = cmd.Wait()
+	assert.NoError(t, err)
+}

scripts/participle

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -euo pipefail
-(cd "$(dirname $0)/../cmd/participle" && go install github.com/alecthomas/participle/v2/cmd/participle)
+(cd "$(dirname "$0")/../cmd/participle" && go install github.com/alecthomas/participle/v2/cmd/participle)
 exec "$(go env GOBIN)/participle" "$@"