Conformance tests for the runtime and generated lexers.

The goal is to have a single lexer definition that exercises all the
functionality of the stateful lexer and generated equivalent.

See #264

cmd/participle/files/codegen.go.tmpl → cmd/participle/codegen.go.tmpl

@@ -1,4 +1,6 @@
 // Code generated by Participle. DO NOT EDIT.
+{{if .Tags}}//go:build {{.Tags}}
+{{end -}}
 package {{.Package}}
 
 import (
@@ -12,6 +14,7 @@ import (
 )
 
 var _ syntax.Op
+const _ = utf8.RuneError
 
 var {{.Name}}Lexer lexer.Definition = lexer{{.Name}}DefinitionImpl{}
 

cmd/participle/gen_lexer_cmd.go

@@ -18,8 +18,9 @@ import (
 type genLexerCmd struct {
 	Name    string `help:"Name of the lexer."`
 	Output  string `short:"o" help:"Output file."`
+	Tags    string `help:"Build tags to include in the generated file."`
 	Package string `arg:"" required:"" help:"Go package for generated code."`
-	Lexer   string `arg:"" required:"" default:"-" type:"existingfile" help:"JSON representation of a Participle lexer."`
+	Lexer   string `arg:"" default:"-" type:"existingfile" help:"JSON representation of a Participle lexer (read from stdin if omitted)."`
 }
 
 func (c *genLexerCmd) Help() string {
@@ -52,18 +53,26 @@ func (c *genLexerCmd) Run() error {
 	if err != nil {
 		return err
 	}
-	err = generateLexer(os.Stdout, c.Package, def, c.Name)
+	out := os.Stdout
+	if c.Output != "" {
+		out, err = os.Create(c.Output)
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+	}
+	err = generateLexer(out, c.Package, def, c.Name, c.Tags)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
 var (
-	//go:embed files/codegen.go.tmpl
+	//go:embed codegen.go.tmpl
 	codegenTemplateSource string
-	codegenBackrefRe                         = regexp.MustCompile(`(\\+)(\d)`)
-	codegenTemplate       *template.Template = template.Must(template.New("lexgen").Funcs(template.FuncMap{
+	codegenBackrefRe      = regexp.MustCompile(`(\\+)(\d)`)
+	codegenTemplate       = template.Must(template.New("lexgen").Funcs(template.FuncMap{
 		"IsPush": func(r lexer.Rule) string {
 			if p, ok := r.Action.(lexer.ActionPush); ok {
 				return p.State
@@ -89,14 +98,15 @@ var (
 	}).Parse(codegenTemplateSource))
 )
 
-func generateLexer(w io.Writer, pkg string, def *lexer.StatefulDefinition, name string) error {
+func generateLexer(w io.Writer, pkg string, def *lexer.StatefulDefinition, name, tags string) error {
 	type ctx struct {
 		Package string
 		Name    string
+		Tags    string
 		Def     *lexer.StatefulDefinition
 	}
 	rules := def.Rules()
-	err := codegenTemplate.Execute(w, ctx{pkg, name, def})
+	err := codegenTemplate.Execute(w, ctx{pkg, name, tags, def})
 	if err != nil {
 		return err
 	}

cmd/participle/main.go

@@ -4,10 +4,12 @@ import "github.com/alecthomas/kong"
 
 var (
 	version string = "dev"
-	cli     struct {
+
+	cli struct {
 		Version kong.VersionFlag
-		Gen     struct {
-			Lexer genLexerCmd `cmd:""`
+
+		Gen struct {
+			Lexer genLexerCmd `cmd:"" help:"Generate a lexer."`
 		} `cmd:"" help:"Generate code to accelerate Participle."`
 	}
 )

lexer/internal/conformance/conformance_codegen_test.go

@@ -0,0 +1,14 @@
+//go:build generated
+
+package conformance_test
+
+import (
+	"testing"
+
+	"github.com/alecthomas/participle/v2/lexer/internal/conformance"
+)
+
+// This should only be run by TestLexerConformanceGenerated.
+func TestLexerConformanceGeneratedInternal(t *testing.T) {
+	testLexer(t, conformance.GeneratedConformanceLexer)
+}

lexer/internal/conformance/conformance_test.go

@@ -0,0 +1,154 @@
+package conformance_test
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/alecthomas/assert/v2"
+	"github.com/alecthomas/participle/v2/lexer"
+)
+
+var conformanceLexer = lexer.MustStateful(lexer.Rules{
+	"Root": {
+		{"String", `"`, lexer.Push("String")},
+		{"Heredoc", `<<(\w+\b)`, lexer.Push("Heredoc")},
+	},
+	"String": {
+		{"Escaped", `\\.`, nil},
+		{"StringEnd", `"`, lexer.Pop()},
+		{"Expr", `\${`, lexer.Push("Expr")},
+		{"Char", `[^$"\\]+`, nil},
+	},
+	"Expr": {
+		lexer.Include("Root"),
+		{`Whitespace`, `\s+`, nil},
+		{`Oper`, `[-+/*%]`, nil},
+		{"Ident", `\w+`, lexer.Push("Reference")},
+		{"ExprEnd", `}`, lexer.Pop()},
+	},
+	"Reference": {
+		{"Dot", `\.`, nil},
+		{"Ident", `\w+`, nil},
+		lexer.Return(),
+	},
+	"Heredoc": {
+		{"End", `\b\1\b`, lexer.Pop()},
+		lexer.Include("Expr"),
+	},
+})
+
+type token struct {
+	Type  string
+	Value string
+}
+
+func testLexer(t *testing.T, lex lexer.Definition) {
+	t.Helper()
+	tests := []struct {
+		name     string
+		input    string
+		expected []token
+	}{
+		{"Push", `"${"Hello ${name + "!"}"}"`, []token{
+			{"String", "\""},
+			{"Expr", "${"},
+			{"String", "\""},
+			{"Char", "Hello "},
+			{"Expr", "${"},
+			{"Ident", "name"},
+			{"Whitespace", " "},
+			{"Oper", "+"},
+			{"Whitespace", " "},
+			{"String", "\""},
+			{"Char", "!"},
+			{"StringEnd", "\""},
+			{"ExprEnd", "}"},
+			{"StringEnd", "\""},
+			{"ExprEnd", "}"},
+			{"StringEnd", "\""},
+		}},
+		{"Reference", `"${user.name}"`, []token{
+			{"String", "\""},
+			{"Expr", "${"},
+			{"Ident", "user"},
+			{"Dot", "."},
+			{"Ident", "name"},
+			{"ExprEnd", "}"},
+			{"StringEnd", "\""},
+		}},
+	}
+	symbols := lexer.SymbolsByRune(lex)
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			l, err := lex.Lex(test.name, strings.NewReader(test.input))
+			assert.NoError(t, err)
+			tokens, err := lexer.ConsumeAll(l)
+			assert.NoError(t, err)
+			actual := make([]token, len(tokens)-1)
+			for i, t := range tokens {
+				if t.Type == lexer.EOF {
+					continue
+				}
+				actual[i] = token{Type: symbols[t.Type], Value: t.Value}
+			}
+			assert.Equal(t, test.expected, actual)
+		})
+	}
+}
+
+func TestLexerConformanceGenerated(t *testing.T) {
+	genLexer(t)
+	args := []string{"test", "-run", "TestLexerConformanceGeneratedInternal", "-tags", "generated"}
+	// Propagate test flags.
+	flag.CommandLine.VisitAll(func(f *flag.Flag) {
+		if f.Value.String() != f.DefValue {
+			args = append(args, fmt.Sprintf("-%s=%s", f.Name, f.Value.String()))
+		}
+	})
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	assert.NoError(t, err)
+}
+
+func TestLexerConformance(t *testing.T) {
+	testLexer(t, conformanceLexer)
+}
+
+func genLexer(t *testing.T) {
+	t.Helper()
+	lexerJSON, err := json.Marshal(conformanceLexer)
+	assert.NoError(t, err)
+	cwd, err := os.Getwd()
+	assert.NoError(t, err)
+	generatedConformanceLexer := filepath.Join(cwd, "conformance_lexer_gen.go")
+	t.Cleanup(func() {
+		_ = os.Remove(generatedConformanceLexer)
+	})
+	cmd := exec.Command(
+		"../../../scripts/participle",
+		"gen", "lexer", "conformance",
+		"--tags", "generated",
+		"--name", "GeneratedConformance",
+		"--output", generatedConformanceLexer)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	w, err := cmd.StdinPipe()
+	assert.NoError(t, err)
+	defer w.Close()
+	err = cmd.Start()
+	assert.NoError(t, err)
+	_, err = w.Write(lexerJSON)
+	assert.NoError(t, err)
+	err = w.Close()
+	assert.NoError(t, err)
+	err = cmd.Wait()
+	assert.NoError(t, err)
+}

lexer/stateful.go

@@ -140,6 +140,10 @@ type RulesAction interface {
 	applyRules(state string, rule int, rules compiledRules) error
 }
 
+type validatingRule interface {
+	validate(rules Rules) error
+}
+
 // ActionPop pops to the previous state when the Rule matches.
 type ActionPop struct{}
 
@@ -177,6 +181,13 @@ func (p ActionPush) applyAction(lexer *StatefulLexer, groups []string) error {
 	return nil
 }
 
+func (p ActionPush) validate(rules Rules) error {
+	if _, ok := rules[p.State]; !ok {
+		return fmt.Errorf("push to unknown state %q", p.State)
+	}
+	return nil
+}
+
 // Push to the given state.
 //
 // The target state will then be the set of rules used for matching
@@ -232,6 +243,11 @@ func New(rules Rules) (*StatefulDefinition, error) {
 	compiled := compiledRules{}
 	for key, set := range rules {
 		for i, rule := range set {
+			if validate, ok := rule.Action.(validatingRule); ok {
+				if err := validate.validate(rules); err != nil {
+					return nil, fmt.Errorf("invalid action for rule %q: %w", rule.Name, err)
+				}
+			}
 			pattern := "^(?:" + rule.Pattern + ")"
 			var (
 				re  *regexp.Regexp

lexer/stateful_test.go

@@ -43,12 +43,17 @@ func TestMarshalUnmarshal(t *testing.T) {
 
 func TestStatefulLexer(t *testing.T) {
 	tests := []struct {
-		name   string
-		rules  lexer.Rules
-		input  string
-		tokens []string
-		err    string
+		name     string
+		rules    lexer.Rules
+		input    string
+		tokens   []string
+		err      string
+		buildErr string
 	}{
+		{name: "InvalidPushTarget",
+			buildErr: `invalid action for rule "foo": push to unknown state "Invalid"`,
+			rules:    lexer.Rules{"Root": {{`foo`, ``, lexer.Push("Invalid")}}},
+		},
 		{name: "BackrefNoGroups",
 			input: `hello`,
 			err:   `1:1: rule "Backref": invalid backref expansion: "\\1": invalid group 1 from parent with 0 groups`,
@@ -174,6 +179,12 @@ func TestStatefulLexer(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			def, err := lexer.New(test.rules)
+			if test.buildErr != "" {
+				require.EqualError(t, err, test.buildErr)
+				return
+			} else {
+				require.NoError(t, err)
+			}
 			require.NoError(t, err)
 			lex, err := def.Lex("", strings.NewReader(test.input))
 			require.NoError(t, err)

scripts/participle

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -euo pipefail
-(cd "$(dirname $0)/../cmd/participle" && go install github.com/alecthomas/participle/v2/cmd/participle)
+(cd "$(dirname "$0")/../cmd/participle" && go install github.com/alecthomas/participle/v2/cmd/participle)
 exec "$(go env GOBIN)/participle" "$@"