From 9547ee26e62b588a0ba50c2e3dccccb4031c732d Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 09:00:31 -0500
Subject: [PATCH 01/33] fix flaky test with matcher to close #68

---
 cmd/ictcc/main.go         |  19 +++---
 cmd/ictcc/version.go      |   2 +-
 fishi/codegen.go          |   9 ++-
 fishi/fishi.go            |  33 ++++++++++
 grammar/grammar_test.go   |  35 +++++++----
 internal/tmatch/tmatch.go | 124 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 199 insertions(+), 23 deletions(-)
 create mode 100644 internal/tmatch/tmatch.go

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index be8558a..115fa04 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -26,6 +26,10 @@ Flags:
 		checked for errors but no other action is taken (unless specified by
 		other flags).
 
+	-q/-quiet
+		Do not show progress messages. This does not affect error messages or
+		warning output.
+
 	-p/-parser FILE
 		Set the location of the pre-compiled parser cache to the given CFF
 		format file as opposed to the default of './parser.cff'.
@@ -265,6 +269,7 @@ var (
 )
 
 var (
+	quietMode     bool
 	noGen         bool
 	genAST        bool
 	genTree       bool
@@ -309,6 +314,7 @@ var (
 
 func init() {
 	const (
+		quietUsage       = "Do not print progress messages"
 		noGenUsage       = "Do not generate the parser"
 		genASTUsage      = "Print the AST of the analyzed fishi"
 		genTreeUsage     = "Print the parse trees of each analyzed fishi file"
@@ -330,6 +336,8 @@ func init() {
 	flag.StringVar(&parserCff, "p", parserCffDefault, parserCffUsage+" (shorthand)")
 	flag.StringVar(&lang, "lang", langDefault, langUsage)
 	flag.StringVar(&lang, "l", langDefault, langUsage+"(shorthand)")
+	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
+	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
 }
 
 func main() {
@@ -540,24 +548,17 @@ func main() {
 				if *hooksPath == "" {
 					fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing -hooks parameter\n")
 				} else {
-					genInfo, err := fishi.GenerateTestCompiler(spec, md, p, *hooksPath, *hooksTableName, cgOpts)
-					if err != nil {
-						fmt.Fprintf(os.Stderr, "%s\n", err.Error())
-						returnCode = ExitErrGeneration
-						return
-					}
 
 					di := trans.ValidationOptions{
-
 						ParseTrees:    *valSDTSShowTrees,
 						FullDepGraphs: *valSDTSShowGraphs,
 						ShowAllErrors: !*valSDTSFirstOnly,
 						SkipErrors:    *valSDTSSkip,
 					}
 
-					err = fishi.ExecuteTestCompiler(genInfo, di)
+					err := fishi.ValidateSimulatedInput(spec, md, p, *hooksPath, *hooksTableName, cgOpts, &di)
 					if err != nil {
-						fmt.Fprintf(os.Stderr, "%s\n", "ERROR: SDTS validation failed")
+						fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
 						returnCode = ExitErrGeneration
 						return
 					}
diff --git a/cmd/ictcc/version.go b/cmd/ictcc/version.go
index 8acbfee..1c72ccb 100644
--- a/cmd/ictcc/version.go
+++ b/cmd/ictcc/version.go
@@ -1,7 +1,7 @@
 package main
 
 const (
-	Version = "0.6.0"
+	Version = "0.6.0+dev"
 )
 
 func GetVersionString() string {
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 898108e..ba556e5 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -128,7 +128,14 @@ type GeneratedCodeInfo struct {
 	Path string
 }
 
-func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions trans.ValidationOptions) error {
+// ExecuteTestCompiler runs the compiler pointed to by gci in validation mode.
+//
+// If valOptions is nil, the default validation options are used.
+func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOptions) error {
+	if valOptions == nil {
+		valOptions = &trans.ValidationOptions{}
+	}
+
 	args := []string{"run", gci.MainFile, "-sim"}
 	if valOptions.FullDepGraphs {
 		args = append(args, "-sim-sdts-graphs")
diff --git a/fishi/fishi.go b/fishi/fishi.go
index c94acb3..50570be 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -37,6 +37,39 @@ type Options struct {
 	ParserTrace       bool
 }
 
+// ValidateSimulatedInput generates a lightweight compiler with the spec'd
+// frontend in a special directory (".sim" in the local directory) and then runs
+// SDTS validation on a variety of parse tree inputs designed to cover all the
+// productions of the grammar at least once.
+//
+// If running validation with the test compiler succeeds, it and the directory
+// it was generated in are deleted. If it fails, the directory is left in place
+// for inspection.
+//
+// IRType is required to be set in cgOpts.
+//
+// valOpts is not required to be set, and if nil will be treated as if it were
+// set to an empty struct.
+func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks, hooksTable string, cgOpts CodegenOptions, valOpts *trans.ValidationOptions) error {
+	genInfo, err := GenerateTestCompiler(spec, md, p, hooks, hooksTable, cgOpts)
+	if err != nil {
+		return fmt.Errorf("generate test compiler: %w", err)
+	}
+
+	err = ExecuteTestCompiler(genInfo, valOpts)
+	if err != nil {
+		return fmt.Errorf("execute test compiler: %w", err)
+	}
+
+	// if we got here, no errors. delete the test compiler and its directory
+	err = os.RemoveAll(genInfo.Path)
+	if err != nil {
+		return fmt.Errorf("remove test compiler: %w", err)
+	}
+
+	return nil
+}
+
 func GetFishiFromMarkdown(mdText []byte) []byte {
 	doc := markdown.Parse(mdText, mkparser.New())
 	var scanner fishiScanner
diff --git a/grammar/grammar_test.go b/grammar/grammar_test.go
index 666b0dc..0cd17ef 100644
--- a/grammar/grammar_test.go
+++ b/grammar/grammar_test.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/dekarrin/ictiobus/internal/box"
 	"github.com/dekarrin/ictiobus/internal/textfmt"
+	"github.com/dekarrin/ictiobus/internal/tmatch"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -340,10 +341,11 @@ func Test_Grammar_DeriveFullTree(t *testing.T) {
 
 func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) {
 	testCases := []struct {
-		name      string
-		input     Grammar
-		expect    map[string]Production
-		expectErr bool
+		name        string
+		input       Grammar
+		expect      map[string]Production
+		expectOneOf []map[string]Production // because this is testing a non-deterministic algorithm, there may be multiple possible outputs
+		expectErr   bool
 	}{
 		{
 			name: "inescapable derivation cycle in single rule",
@@ -378,9 +380,7 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) {
 				F -> ( E ) | id ;
 			`),
 			expect: map[string]Production{
-				"E": {"T"},
-				"T": {"F"},
-				"F": {"id"},
+				"E": {"T"}, "T": {"F"}, "F": {"id"},
 			},
 		},
 		{
@@ -390,10 +390,9 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) {
 				T -> T * F | F ;
 				F -> ( E ) | id | num;
 			`),
-			expect: map[string]Production{
-				"E": {"T"},
-				"T": {"F"},
-				"F": {"id"},
+			expectOneOf: []map[string]Production{
+				{"E": {"T"}, "T": {"F"}, "F": {"id"}},
+				{"E": {"T"}, "T": {"F"}, "F": {"num"}},
 			},
 		},
 	}
@@ -402,6 +401,11 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assert := assert.New(t)
 
+			// make sure we didnt accidentally make an invalid test
+			if !tc.expectErr && tc.expect == nil && tc.expectOneOf == nil {
+				panic(fmt.Sprintf("test case %s does not specify expectErr, expect, or expectOneOf", tc.name))
+			}
+
 			actual, err := tc.input.CreateFewestNonTermsAlternationsTable()
 			if tc.expectErr {
 				assert.Error(err)
@@ -410,7 +414,14 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) {
 				return
 			}
 
-			assert.Equal(tc.expect, actual)
+			// if only one, check that one
+			if tc.expect != nil {
+				assert.Equal(tc.expect, actual)
+			} else {
+				// otherwise, check that it is one of the possible ones
+				assertErr := tmatch.AnyStrMapV(actual, tc.expectOneOf, tmatch.Comparer(Production.Equal))
+				assert.NoError(assertErr)
+			}
 		})
 	}
 
diff --git a/internal/tmatch/tmatch.go b/internal/tmatch/tmatch.go
new file mode 100644
index 0000000..2d6c2d4
--- /dev/null
+++ b/internal/tmatch/tmatch.go
@@ -0,0 +1,124 @@
+// Package tmatch provides some quick and dirty matching functions with detailed
+// mismatch error messages for use with unit tests. It will probably eventually
+// be replaced with a well-vetted library like gomatch or gomega in the future.
+package tmatch
+
+import (
+	"fmt"
+
+	"github.com/dekarrin/ictiobus/internal/textfmt"
+)
+
+// CompFunc is a comparison function that returns true if the two values are
+// equal. Used for defining custom comparison for types that do not fulfill
+// the 'comparable' type constraint.
+type CompFunc func(v1, v2 any) bool
+
+// Comparer convertes the given function into a CompFunc. The returned CompFunc
+// ret to ensure the types are exactly as expected, and if not, considers
+// it a mismatch.
+func Comparer[E1 any, E2 any](fn func(v1 E1, v2 E2) bool) CompFunc {
+	return func(v1, v2 any) bool {
+		c1, ok := v1.(E1)
+		if !ok {
+			return false
+		}
+		c2, ok := v2.(E2)
+		if !ok {
+			return false
+		}
+
+		return fn(c1, c2)
+	}
+}
+
+// AnyStrMapV returns an error if the actual map does not match any of the maps
+// in expect, using vMatches to match elements.
+func AnyStrMapV[E any](actual map[string]E, expect []map[string]E, vMatches CompFunc) error {
+	foundAny := false
+	for _, expectMap := range expect {
+		candidateFailedMatch := false
+		// check that no key is present in one that is not present in the other
+		for key := range actual {
+			if _, ok := expectMap[key]; !ok {
+				candidateFailedMatch = true
+				break
+			}
+		}
+		if candidateFailedMatch {
+			continue
+		}
+		for key := range expectMap {
+			if _, ok := actual[key]; !ok {
+				candidateFailedMatch = true
+				break
+			}
+		}
+		if candidateFailedMatch {
+			continue
+		}
+
+		// keys are all the same, now check values
+		for key := range actual {
+			if !vMatches(actual[key], expectMap[key]) {
+				candidateFailedMatch = true
+				break
+			}
+		}
+		if !candidateFailedMatch {
+			foundAny = true
+			break
+		}
+	}
+
+	if !foundAny {
+		errMsg := "actual does not match any expected:\n     "
+		if len(expect) > 9 {
+			errMsg += " "
+		}
+		errMsg += "actual: "
+		if actual == nil {
+			errMsg += "nil"
+		} else {
+			errMsg += "{"
+			ordered := textfmt.OrderedKeys(actual)
+			for i, k := range ordered {
+				errMsg += fmt.Sprintf("%s: %v", k, actual[k])
+				if i+1 < len(ordered) {
+					errMsg += ", "
+				}
+			}
+			errMsg += "}"
+		}
+		errMsg += "\n"
+		for i, expectMap := range expect {
+			errMsg += fmt.Sprintf("expected[%d]: ", i)
+			if expectMap == nil {
+				errMsg += "nil"
+			} else {
+				errMsg += "{"
+				ordered := textfmt.OrderedKeys(expectMap)
+				for j, k := range ordered {
+					errMsg += fmt.Sprintf("%s: %v", k, expectMap[k])
+					if j+1 < len(ordered) {
+						errMsg += ", "
+					}
+				}
+				errMsg += "}"
+			}
+			errMsg += "\n"
+		}
+		return fmt.Errorf(errMsg)
+	}
+	return nil
+}
+
+// MatchAnyStrMap checks if the actual string map matches any of the expected
+// maps. All maps must have a comparable value type. If the value type is not
+// comparable, use MatchAnyStrMapV instead to provide a custom equality check.
+func AnyStrMap[E comparable](actual map[string]E, expect []map[string]E) error {
+
+	return AnyStrMapV(actual, expect, Comparer(func(v1, v2 E) bool {
+		return v1 == v2
+	}))
+}

From af7a1cf62b310c524d5b0a2a62af523569606db1 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 09:04:07 -0500
Subject: [PATCH 02/33] #16: dont run sdts validation on actively running fishi

---
 cmd/ictcc/main.go   | 15 +++++----------
 fishi/fishi.go      | 32 +++++---------------------------
 fishi/fishi_test.go |  6 +++---
 3 files changed, 13 insertions(+), 40 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 115fa04..d615fb8 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -386,16 +386,11 @@ func main() {
 	}
 
 	fo := fishi.Options{
-		ParserCFF:         parserCff,
-		ReadCache:         !*noCache,
-		WriteCache:        !*noCacheOutput,
-		SDTSValidate:      !*valSDTSOff,
-		SDTSValShowTrees:  *valSDTSShowTrees,
-		SDTSValShowGraphs: *valSDTSShowGraphs,
-		SDTSValAllTrees:   !*valSDTSFirstOnly,
-		SDTSValSkipTrees:  *valSDTSSkip,
-		LexerTrace:        *lexerTrace,
-		ParserTrace:       *parserTrace,
+		ParserCFF:   parserCff,
+		ReadCache:   !*noCache,
+		WriteCache:  !*noCacheOutput,
+		LexerTrace:  *lexerTrace,
+		ParserTrace: *parserTrace,
 	}
 
 	cgOpts := fishi.CodegenOptions{
diff --git a/fishi/fishi.go b/fishi/fishi.go
index 50570be..bcf3415 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -25,16 +25,11 @@ type Results struct {
 }
 
 type Options struct {
-	ParserCFF         string
-	ReadCache         bool
-	WriteCache        bool
-	SDTSValidate      bool
-	SDTSValShowTrees  bool
-	SDTSValShowGraphs bool
-	SDTSValAllTrees   bool
-	SDTSValSkipTrees  int
-	LexerTrace        bool
-	ParserTrace       bool
+	ParserCFF   string
+	ReadCache   bool
+	WriteCache  bool
+	LexerTrace  bool
+	ParserTrace bool
 }
 
 // ValidateSimulatedInput generates a lightweight compiler with the spec'd
@@ -207,23 +202,6 @@ func GetFrontend(opts Options) (ictiobus.Frontend[[]syntax.Block], error) {
 		}
 	}
 
-	// validate our SDTS if we were asked to
-	if opts.SDTSValidate {
-		valProd := fishiFront.Lexer.FakeLexemeProducer(true, "")
-
-		di := trans.ValidationOptions{
-			ParseTrees:    opts.SDTSValShowTrees,
-			FullDepGraphs: opts.SDTSValShowGraphs,
-			ShowAllErrors: opts.SDTSValAllTrees,
-			SkipErrors:    opts.SDTSValSkipTrees,
-		}
-
-		sddErr := fishiFront.SDT.Validate(fishiFront.Parser.Grammar(), fishiFront.IRAttribute, di, valProd)
-		if sddErr != nil {
-			return ictiobus.Frontend[[]syntax.Block]{}, fmt.Errorf("sdd validation error: %w", sddErr)
-		}
-	}
-
 	return fishiFront, nil
 }
 
diff --git a/fishi/fishi_test.go b/fishi/fishi_test.go
index 3db2697..e88e506 100644
--- a/fishi/fishi_test.go
+++ b/fishi/fishi_test.go
@@ -12,7 +12,7 @@ import (
 func Test_WithFakeInput(t *testing.T) {
 	assert := assert.New(t)
 
-	_, actual := Parse([]byte(testInput), Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true, SDTSValidate: true})
+	_, actual := Parse([]byte(testInput), Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true})
 
 	assert.NoError(actual)
 
@@ -27,7 +27,7 @@ func Test_WithFakeInput(t *testing.T) {
 func Test_SelfHostedMarkdown_Spec(t *testing.T) {
 	assert := assert.New(t)
 
-	res, err := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true, SDTSValidate: true})
+	res, err := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true})
 	if !assert.NoError(err) {
 		return
 	}
@@ -45,7 +45,7 @@ func Test_SelfHostedMarkdown_Spec(t *testing.T) {
 func Test_SelfHostedMarkdown(t *testing.T) {
 	assert := assert.New(t)
 
-	_, actual := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true, SDTSValidate: true})
+	_, actual := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true})
 
 	assert.NoError(actual)
 

From 74827b1ce135004a3e304650ef3a8d1dad71b0f8 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 09:09:12 -0500
Subject: [PATCH 03/33] #16: block creation in spec of attributes starting with
 '$'

---
 fishi/spec.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fishi/spec.go b/fishi/spec.go
index 9f4f9a7..bea9c8b 100644
--- a/fishi/spec.go
+++ b/fishi/spec.go
@@ -450,6 +450,11 @@ func analyzeASTActionsContentSlice(
 						synErr := types.NewSyntaxErrorFromToken("invalid attrRef: "+err.Error(), semAct.LHS.Src)
 						return nil, warnings, synErr
 					}
+					// make shore we aren't trying to set somefin starting with a '$'; those are reserved
+					if strings.HasPrefix(sdd.Attribute.Name, "$") {
+						synErr := types.NewSyntaxErrorFromToken("cannot create attribute starting with reserved marker '$'", semAct.LHS.Src)
+						return nil, warnings, synErr
+					}
 
 					// do the same for each arg to the hook
 					if len(semAct.With) > 0 {

From 9611ac432577b9419098ba029758e6d6bad25fee Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 09:27:59 -0500
Subject: [PATCH 04/33] #16: sdts verify spec does not refer to invalid
 attribute

---
 fishi/spec.go | 55 +++++++++++++++++++++++++++++++++++++++++----------
 fishigen.sh   | 11 +++++++++++
 2 files changed, 56 insertions(+), 10 deletions(-)
 create mode 100644 fishigen.sh

diff --git a/fishi/spec.go b/fishi/spec.go
index bea9c8b..c976516 100644
--- a/fishi/spec.go
+++ b/fishi/spec.go
@@ -727,25 +727,26 @@ func analzyeASTTokensContentSlice(
 
 // r is rule to check against, only first production is checked.
 func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.Rule) (trans.AttrRef, error) {
+	var ar trans.AttrRef
 	if astRef.Head {
-		return trans.AttrRef{
+		ar = trans.AttrRef{
 			Relation: trans.NodeRelation{
 				Type: trans.RelHead,
 			},
 			Name: astRef.Attribute,
-		}, nil
+		}
 	} else if astRef.SymInProd {
 		// make sure the rule has the right number of symbols
 		if astRef.Occurance >= len(r.Productions[0]) {
 			return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %d symbols", len(r.Productions[0]))
 		}
-		return trans.AttrRef{
+		ar = trans.AttrRef{
 			Relation: trans.NodeRelation{
 				Type:  trans.RelSymbol,
 				Index: astRef.Occurance,
 			},
 			Name: astRef.Attribute,
-		}, nil
+		}
 	} else if astRef.NontermInProd {
 		// make sure that the rule has that number of non-terminals
 		nontermCount := 0
@@ -757,13 +758,13 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R
 		if astRef.Occurance >= nontermCount {
 			return trans.AttrRef{}, fmt.Errorf("non-terminal index out of range; production only has %d non-terminals", nontermCount)
 		}
-		return trans.AttrRef{
+		ar = trans.AttrRef{
 			Relation: trans.NodeRelation{
 				Type:  trans.RelNonTerminal,
 				Index: astRef.Occurance,
 			},
 			Name: astRef.Attribute,
-		}, nil
+		}
 	} else if astRef.TermInProd {
 		// make sure that the rule has that number of terminals
 		termCount := 0
@@ -775,13 +776,13 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R
 		if astRef.Occurance >= termCount {
 			return trans.AttrRef{}, fmt.Errorf("terminal index out of range; production only has %d terminals", termCount)
 		}
-		return trans.AttrRef{
+		ar = trans.AttrRef{
 			Relation: trans.NodeRelation{
 				Type:  trans.RelTerminal,
 				Index: astRef.Occurance,
 			},
 			Name: astRef.Attribute,
-		}, nil
+		}
 	} else {
 		// it's an instance of a particular symbol. find out the symbol index.
 		symIndexes := []int{}
@@ -796,14 +797,48 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R
 		if astRef.Occurance >= len(symIndexes) {
 			return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %d instances of %s", len(symIndexes), astRef.Symbol)
 		}
-		return trans.AttrRef{
+		ar = trans.AttrRef{
 			Relation: trans.NodeRelation{
 				Type:  trans.RelSymbol,
 				Index: symIndexes[astRef.Occurance],
 			},
 			Name: astRef.Attribute,
-		}, nil
+		}
 	}
+
+	valErr := validateParsedAttrRef(ar, g, r)
+	if valErr != nil {
+		return ar, valErr
+	}
+	return ar, nil
+}
+
+// it is assumed that the parsed ref refers to an existing symbol; not checking
+// that here.
+func validateParsedAttrRef(ar trans.AttrRef, g grammar.Grammar, r grammar.Rule) error {
+	// need to know if we are dealing with a terminal or not
+	isTerm := false
+
+	// no need to check RelHead; by definition this cannot be a terminal
+	if ar.Relation.Type == trans.RelSymbol {
+		isTerm = g.IsTerminal(r.Productions[0][ar.Relation.Index])
+	} else {
+		isTerm = ar.Relation.Type == trans.RelTerminal
+	}
+
+	if isTerm {
+		// then anyfin we take from it, glub, must start with '$'
+		if ar.Name != "$text" && ar.Name != "$ft" {
+			return fmt.Errorf("referred-to terminal %q only has '$text' and '$ft' attributes", ar.Name)
+		}
+	} else {
+		// then we cannot take '$text' from it and it's an error.
+		if ar.Name == "$text" {
+			return fmt.Errorf("referred-to non-terminal %q does not have lexed text attribute \"$text\"", ar.Name)
+		}
+	}
+
+	return nil
 }
 
 func putEntryTokenInCorrectPosForDiscardCheck(first, second *types.Token, discardIsFirst *bool, tok types.Token) {
diff --git a/fishigen.sh b/fishigen.sh
new file mode 100644
index 0000000..f94d9a7
--- /dev/null
+++ b/fishigen.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# shortcut for running ictcc on fishi.md for when we build a new ictcc bin
+
+./ictcc -clr \
+	-ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \
+	-dest testout \
+	-l fishi -lang-ver 1.0.0 \
+	-hooks fishi/syntax \
+	fishi.md
+

From 701228f429bd659610838367cc85b492f443ee4f Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 09:32:46 -0500
Subject: [PATCH 05/33] #16: slightly betta error catching when prods are wrong

---
 fishi/spec.go              | 3 ++-
 fishigen.sh => genfishi.sh | 0
 2 files changed, 2 insertions(+), 1 deletion(-)
 rename fishigen.sh => genfishi.sh (100%)

diff --git a/fishi/spec.go b/fishi/spec.go
index c976516..b2c7b05 100644
--- a/fishi/spec.go
+++ b/fishi/spec.go
@@ -738,7 +738,8 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R
 	} else if astRef.SymInProd {
 		// make sure the rule has the right number of symbols
 		if astRef.Occurance >= len(r.Productions[0]) {
-			return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %d symbols", len(r.Productions[0]))
+			symCount := textfmt.Pluralize(len(r.Productions[0]), "symbol", "-s")
+			return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %s (%s)", symCount, r.Productions[0])
 		}
 		ar = trans.AttrRef{
 			Relation: trans.NodeRelation{
diff --git a/fishigen.sh b/genfishi.sh
similarity index 100%
rename from fishigen.sh
rename to genfishi.sh

From cf3ddaf07985dce4565ef5b20e0af20b3628aab5 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 10:05:45 -0500
Subject: [PATCH 06/33] #16: started lexer test for issue of lexing term
 attrrefs as terms only

---
 fishi/fe/lexer_test.go | 52 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 fishi/fe/lexer_test.go

diff --git a/fishi/fe/lexer_test.go b/fishi/fe/lexer_test.go
new file mode 100644
index 0000000..95f8d6f
--- /dev/null
+++ b/fishi/fe/lexer_test.go
@@ -0,0 +1,52 @@
+package fe
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/dekarrin/ictiobus/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Fishi_Lexer_AttrRef_Terminal(t *testing.T) {
+	testCases := []struct {
+		name   string
+		input  string
+		expect []types.Token
+	}{}
+
+	lx := CreateBootstrapLexer()
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// setup
+			assert := assert.New(t)
+			r := bytes.NewReader([]byte(tc.input))
+
+			// execute
+			tokStream, err := lx.Lex(r)
+
+			// verify
+			if !assert.NoError(err) {
+				return
+			}
+
+			// collect the tokens
+			toks := gatherTokens(tokStream)
+
+			// validate them
+			for i, tok := range toks {
+
+			}
+		})
+	}
+}
+
+func gatherTokens(stream types.TokenStream) []types.Token {
+	allTokens := []types.Token{}
+
+	for stream.HasNext() {
+		allTokens = append(allTokens, stream.Next())
+	}
+
+	return allTokens
+}

From 4ba40d0793e87770e66493918b0363d6f14b94b0 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 13:33:27 -0500
Subject: [PATCH 07/33] #16: added a test for attref lexing issue; it's
 probably a missing '^' in regex for attrref

---
 defexample.md          |  7 +++++++
 fishi/fe/lexer_test.go | 28 ++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/defexample.md b/defexample.md
index 9b244c0..0557d70 100644
--- a/defexample.md
+++ b/defexample.md
@@ -1,5 +1,12 @@
 Example Language
 ################
+
+*NOTE: this document is being kept for historical purproses, and some content
+may be correct, but it was the first attempt to standardize the fishi language
+and is heavily out of date. Refer to fishi.md instead of this file for an
+example; the correct parts of this file will eventually be worked into the manual
+for FISHI.*
+
 This is a complete example of a language specified in the special ictiobus
 format. It is a markdown based format that uses specially named sections to
 allow both specificiation of a language and freeform text to co-exist.
diff --git a/fishi/fe/lexer_test.go b/fishi/fe/lexer_test.go
index 95f8d6f..c79cb2f 100644
--- a/fishi/fe/lexer_test.go
+++ b/fishi/fe/lexer_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/dekarrin/ictiobus/lex"
 	"github.com/dekarrin/ictiobus/types"
 	"github.com/stretchr/testify/assert"
 )
@@ -13,7 +14,18 @@ func Test_Fishi_Lexer_AttrRef_Terminal(t *testing.T) {
 		name   string
 		input  string
 		expect []types.Token
-	}{}
+	}{
+		{
+			name: "single attr ref",
+			input: `%%actions
+					someAttrRef.value`,
+			expect: []types.Token{
+				lex.NewToken(TCHeaderActions, "%%actions", 0, 0, ""),
+				lex.NewToken(TCAttrRef, "someAttrRef.value", 0, 0, ""),
+				lex.NewToken(types.TokenEndOfText, "$", 0, 0, ""),
+			},
+		},
+	}
 
 	lx := CreateBootstrapLexer()
 	for _, tc := range testCases {
@@ -34,8 +46,20 @@ func Test_Fishi_Lexer_AttrRef_Terminal(t *testing.T) {
 			toks := gatherTokens(tokStream)
 
 			// validate them
-			for i, tok := range toks {
+			tokCount := len(toks)
+
+			// only check count, token class, and lexeme.
+			if !assert.Len(toks, len(tc.expect), "different number of tokens") {
+				if tokCount < len(tc.expect) {
+					tokCount = len(tc.expect)
+				}
+			}
 
+			for i := 0; i < tokCount; i++ {
+				if !assert.Equal(tc.expect[i].Class().ID(), toks[i].Class().ID(), "different token class for token #%d") {
+					return
+				}
+				assert.Equal(tc.expect[i].Lexeme(), toks[i].Lexeme(), "different lexemes for token #%d")
 			}
 		})
 	}

From d089bb25a6733838a78de676cb09ee45be8ef7a2 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 15:58:10 -0500
Subject: [PATCH 08/33] #16: fixed error with lex of attrrefs that are not what
 they should be

---
 .gitignore                  | 2 +-
 fishi.md                    | 2 +-
 fishi/fe/bootstrap_lexer.go | 2 +-
 fishi/fe/lexer_test.go      | 2 +-
 genfishi.sh                 | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index fe22f3b..6485fa6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,7 @@
 # vim
 *.swp
 
-/testout
+/.testout
 /ictcc
 /.sim
 
diff --git a/fishi.md b/fishi.md
index bf35f04..db24cd2 100644
--- a/fishi.md
+++ b/fishi.md
@@ -279,7 +279,7 @@ For actions state:
 
 \s+                      %discard
 
-(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]*
+(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[^\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]*
 %token attr-ref    %human attribute reference literal
 
 [0-9]+
diff --git a/fishi/fe/bootstrap_lexer.go b/fishi/fe/bootstrap_lexer.go
index 194aafe..e9b053a 100644
--- a/fishi/fe/bootstrap_lexer.go
+++ b/fishi/fe/bootstrap_lexer.go
@@ -100,7 +100,7 @@ func CreateBootstrapLexer() ictiobus.Lexer {
 
 	// actions patterns
 	bootLx.AddPattern(`\s+`, lex.Discard(), "actions", 0)
-	bootLx.AddPattern(`(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]*`, lex.LexAs(TCAttrRef.ID()), "actions", 0)
+	bootLx.AddPattern(`(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[^\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]*`, lex.LexAs(TCAttrRef.ID()), "actions", 0)
 	bootLx.AddPattern(`,`, lex.Discard(), "actions", 0)
 	bootLx.AddPattern(`[0-9]+`, lex.LexAs(TCInt.ID()), "actions", 0)
 	bootLx.AddPattern(`{[A-Za-z][^}]*}`, lex.LexAs(TCNonterminal.ID()), "actions", 0)
diff --git a/fishi/fe/lexer_test.go b/fishi/fe/lexer_test.go
index c79cb2f..087b0ba 100644
--- a/fishi/fe/lexer_test.go
+++ b/fishi/fe/lexer_test.go
@@ -22,7 +22,7 @@ func Test_Fishi_Lexer_AttrRef_Terminal(t *testing.T) {
 			expect: []types.Token{
 				lex.NewToken(TCHeaderActions, "%%actions", 0, 0, ""),
 				lex.NewToken(TCAttrRef, "someAttrRef.value", 0, 0, ""),
-				lex.NewToken(types.TokenEndOfText, "$", 0, 0, ""),
+				lex.NewToken(types.TokenEndOfText, "", 0, 0, ""),
 			},
 		},
 	}
diff --git a/genfishi.sh b/genfishi.sh
index f94d9a7..2d0c4b8 100644
--- a/genfishi.sh
+++ b/genfishi.sh
@@ -4,7 +4,7 @@
 
 ./ictcc -clr \
 	-ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \
-	-dest testout \
+	-dest .testout \
 	-l fishi -lang-ver 1.0.0 \
 	-hooks fishi/syntax \
 	fishi.md

From 392999bc101a37e1e855cee76ab2ca195bceda3f Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 17:00:17 -0500
Subject: [PATCH 09/33] #16: this is a pflag household now

---
 cmd/ictcc/main.go | 198 +++++++++++++++++++++++-----------------------
 genfishi.sh       |  13 ++-
 go.mod            |   1 +
 go.sum            |   2 +
 4 files changed, 110 insertions(+), 104 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index d615fb8..c02ee77 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -11,123 +11,123 @@ Usage:
 
 Flags:
 
-	-a/-ast
+	-a/--ast
 		Print the AST to stdout before generating the frontend.
 
-	-t/-tree
+	-t/--tree
 		Print the parse tree to stdout before generating the frontend.
 
-	-s/-spec
+	-s/--spec
 		Print the interpreted language specification to stdout before generating
 		the frontend.
 
-	-n/-no-gen
+	-n/--no-gen
 		Do not generate the parser. If this flag is set, the fishi is parsed and
 		checked for errors but no other action is taken (unless specified by
 		other flags).
 
-	-q/-quiet
+	-q/--quiet
 		Do not show progress messages. This does not affect error messages or
 		warning output.
 
-	-p/-parser FILE
+	-p/--parser FILE
 		Set the location of the pre-compiled parser cache to the given CFF
 		format file as opposed to the default of './parser.cff'.
 
-	-no-cache
+	--no-cache
 		Disable the loading of any cached frontend components, even if a
 		pre-built one is available.
 
-	-no-cache-out
+	--no-cache-out
 		Disable writing of any frontend components cache, even if a component
 		was built by the invocation.
 
-	-version
+	--version
 		Print the version of the ictiobus compiler-compiler and exit.
 
-	-val-sdts-off
+	--val-sdts-off
 		Disable validatione of the SDTS of the resulting fishi.
 
-	-val-sdts-trees
+	--val-sdts-trees
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, show the parse tree(s) that caused the problem.
 
 		Has no effect if -val-sdts-off is set.
 
-	-val-sdts-graphs
+	--val-sdts-graphs
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, show the full resulting dependency graph(s) that caused
 		the issue (if any).
 
 		Has no effect if -val-sdts-off is set.
 
-	-val-sdts-first
+	--val-sdts-first
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, show only the problem(s) found in the first simulated
 		parse tree (after any skipped by -val-sdts-skip) and then stop.
 
 		Has no effect if -val-sdts-off is set.
 
-	-val-sdts-skip N
+	--val-sdts-skip N
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, skip the first N simulated parse trees in the output.
 		Combine with -val-sdts-first to view a specific parse tree.
 
 		Has no effect if -val-sdts-off is set.
 
-	-debug-lexer
+	--debug-lexer
 		Enable debug mode for the lexer and print each token to standard out as
 		it is lexed. Note that if the lexer is not in lazy mode, all tokens will
 		be lexed before any parsing begins, and so with debug-lexer enabled will
 		all be printed to stdout before any parsing begins.
 
-	-debug-parser
+	--debug-parser
 		Enable debug mode for the parser and print each step of the parse to
 		stdout, including the symbol stack, manipulations of the stack, ACTION
 		selected in DFA based on the stack, and other information.
 
-	-pkg NAME
+	--pkg NAME
 		Set the name of the package to place generated files in. Defaults to
 		'fe'.
 
-	-dest DIR
+	--dest DIR
 		Set the destination directory to place generated files in. Defaults to a
 		directory named 'fe' in the current working directory.
 
-	-l/lang NAME
+	-l/--lang NAME
 		Set the name of the language to generate a frontend for. Defaults to
 		"Unspecified".
 
-	-lang-ver VERSION
+	--lang-ver VERSION
 		Set the version of the language to generate a frontend for. Defaults to
 		"v0.0.0".
 
-	-pre-format
+	--pre-format
 		Enable dumping of the fishi filled template files before they are passed
 		to the formatter. This allows debugging of the template files when
 		editing them, since they must be valid go code to be formatted.
 
-	-tmpl-tokens FILE
+	--tmpl-tokens FILE
 		Use the provided file as the template for outputting the generated
 		tokens file instead of the default embedded within the binary.
 
-	-tmpl-lexer FILE
+	--tmpl-lexer FILE
 		Use the provided file as the template for outputting the generated lexer
 		file instead of the default embedded within the binary.
 
-	-tmpl-parser FILE
+	--tmpl-parser FILE
 		Use the provided file as the template for outputting the generated
 		parser file instead of the default embedded within the binary.
 
-	-tmpl-sdts FILE
+	--tmpl-sdts FILE
 		Use the provided file as the template for outputting the generated SDTS
 		file instead of the default embedded within the binary.
 
-	-tmpl-frontend FILE
+	--tmpl-frontend FILE
 		Use the provided file as the template for outputting the generated
 		frontend file instead of the default embedded within the binary.
 
-	-no-ambig
+	--no-ambig
 		Disable the generation of a parser for an ambiguous language. Normally,
 		when generating an LR parser, an ambiguous grammar is allowed, with
 		shift-reduce conflicts resolved in favor of shift in all cases. If this
@@ -136,32 +136,32 @@ Flags:
 		ambiguous, so this flag has no effect if explicitly selecting an LL
 		parser.
 
-	-ll
+	--ll
 		Force the generation of an LL(1) parser instead of the default of trying
 		each parser type in sequence from most restrictive to least restrictive
 		and using the first one found.
 
-	-slr
+	--slr
 		Force the generation of an SLR(1) (simple LR) parser instead of the
 		default of trying each parser type in sequence from most restrictive to
 		least restrictive and using the first one found.
 
-	-clr
+	--clr
 		Force the generation of a CLR(1) (canonical LR) parser instead of the
 		default of trying each parser type in sequence from most restrictive to
 		least restrictive and using the first one found.
 
-	-lalr
+	--lalr
 		Force the generation of a LALR(1) (lookahead LR) parser instead of the
 		default of trying each parser type in sequence from most restrictive to
 		least restrictive and using the first one found.
 
-	-hooks PATH
+	--hooks PATH
 		Gives the filesystem path to the directory containing the package that
 		the hooks table is in. This is required for live validation of simulated
 		parse trees, but may be omitted if SDTS validation is disabled.
 
-	-hooks-table NAME
+	--hooks-table NAME
 		Gives the expression to retrieve the hooks table from the hooks package,
 		relative to the package that it is in. The NAME must be an exported var
 		of type map[string]trans.AttributeSetter, or a function call that
@@ -169,7 +169,7 @@ Flags:
 		expression; it will be automatically determined. The default value is
 		"HooksTable".
 
-	-ir TYPE
+	--ir TYPE
 		Gives the type of the intermediate representation returned by applying
 		the translation scheme to a parse tree. This is required for running
 		SDTS validation on simulated parse trees, but may be omitted if SDTS
@@ -187,8 +187,8 @@ separately but their resulting ASTs are combined into a single FISHI spec for a
 language.
 
 The spec is then used to generate a lexer, parser, and SDTS for the language.
-For the parser, if no specific parser is selected via the -ll, -slr, -clr, or
--lalr flags, the parser generator will attempt to generate each type of parser
+For the parser, if no specific parser is selected via the --ll, --slr, --clr, or
+--lalr flags, the parser generator will attempt to generate each type of parser
 in sequence from most restrictive to least restrictive (LL(1), simple LR(1),
 lookahead LR(1), and canonical LR(1), in that order), and use the first one it
 is able to generate. If a specific parser is selected, it will attempt to
@@ -204,15 +204,15 @@ correct exit code for the last file parsed.
 
 If files containing cached pre-built components of the frontend are available,
 they will be loaded and used unless -no-cache is set. The files are named
-'fishi-parser.cff' by default, and the names can be changed with the -parser/-p
+'fishi-parser.cff' by default, and the names can be changed with the --parser/-p
 flag if desired. Cache invalidation is non-sophisticated and cannot be
 automatically detected at this time. To force it to occur, the -no-cache flag
 must be manually used (or the file deleted).
 
 If new frontend components are generated from scratch, they will be cached by
-saving them to the files mentioned above unless -no-cache-out is set. Note that
+saving them to the files mentioned above unless --no-cache-out is set. Note that
 if the frontend components are loaded from cache files, they will not be output
-to cache files again regardless of whether -no-cache-out is present.
+to cache files again regardless of whether --no-cache-out is present.
 
 Once the input has been successfully parsed, the parser is generated using the
 options provided, unless the -n flag is set, in which case ictcc will
@@ -221,11 +221,12 @@ immediately exit with a success code after parsing the input.
 package main
 
 import (
-	"flag"
 	"fmt"
 	"os"
 	"strings"
 
+	"github.com/spf13/pflag"
+
 	"github.com/dekarrin/ictiobus"
 	"github.com/dekarrin/ictiobus/fishi"
 	"github.com/dekarrin/ictiobus/grammar"
@@ -273,43 +274,43 @@ var (
 	noGen         bool
 	genAST        bool
 	genTree       bool
-	genSpec       bool
+	showSpec      bool
 	parserCff     string
 	lang          string
-	dumpPreFormat *bool   = flag.Bool("pre-format", false, "Dump the generated code before running through gofmt")
-	pkg           *string = flag.String("pkg", "fe", "The name of the package to place generated files in")
-	dest          *string = flag.String("dest", "./fe", "The name of the directory to place the generated package in")
-	langVer       *string = flag.String("lang-ver", "v0.0.0", "The version of the language to generate")
-	noCache       *bool   = flag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
-	noCacheOutput *bool   = flag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
-
-	valSDTSOff        *bool = flag.Bool("val-sdts-off", false, "Disable validation of the SDTS of the resulting fishi")
-	valSDTSShowTrees  *bool = flag.Bool("val-sdts-trees", false, "Show trees that caused SDTS validation errors")
-	valSDTSShowGraphs *bool = flag.Bool("val-sdts-graphs", false, "Show full generated dependency graph output for parse trees that caused SDTS validation errors")
-	valSDTSFirstOnly  *bool = flag.Bool("val-sdts-first", false, "Show only the first error found in SDTS validation")
-	valSDTSSkip       *int  = flag.Int("val-sdts-skip", 0, "Skip the first N errors found in SDTS validation in output")
-
-	tmplTokens *string = flag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
-	tmplLexer  *string = flag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
-	tmplParser *string = flag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
-	tmplSDTS   *string = flag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
-	tmplFront  *string = flag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
-
-	parserLL      *bool = flag.Bool("ll", false, "Generate an LL(1) parser")
-	parserSLR     *bool = flag.Bool("slr", false, "Generate a simple LR(1) parser")
-	parserCLR     *bool = flag.Bool("clr", false, "Generate a canonical LR(1) parser")
-	parserLALR    *bool = flag.Bool("lalr", false, "Generate a canonical LR(1) parser")
-	parserNoAmbig *bool = flag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it")
-
-	lexerTrace  *bool = flag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
-	parserTrace *bool = flag.Bool("debug-parser", false, "Print the parser trace to stdout")
-
-	hooksPath      *string = flag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.")
-	hooksTableName *string = flag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.")
-
-	irType *string = flag.String("ir", "", "The fully-qualified type of IR to generate.")
-
-	version *bool = flag.Bool("version", false, "Print the version of ictcc and exit")
+	dumpPreFormat *bool   = pflag.Bool("pre-format", false, "Dump the generated code before running through gofmt")
+	pkg           *string = pflag.String("pkg", "fe", "The name of the package to place generated files in")
+	dest          *string = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
+	langVer       *string = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
+	noCache       *bool   = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
+	noCacheOutput *bool   = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
+
+	valSDTSOff        *bool = pflag.Bool("val-sdts-off", false, "Disable validation of the SDTS of the resulting fishi")
+	valSDTSShowTrees  *bool = pflag.Bool("val-sdts-trees", false, "Show trees that caused SDTS validation errors")
+	valSDTSShowGraphs *bool = pflag.Bool("val-sdts-graphs", false, "Show full generated dependency graph output for parse trees that caused SDTS validation errors")
+	valSDTSFirstOnly  *bool = pflag.Bool("val-sdts-first", false, "Show only the first error found in SDTS validation")
+	valSDTSSkip       *int  = pflag.Int("val-sdts-skip", 0, "Skip the first N errors found in SDTS validation in output")
+
+	tmplTokens *string = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
+	tmplLexer  *string = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
+	tmplParser *string = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
+	tmplSDTS   *string = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
+	tmplFront  *string = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
+
+	parserLL      *bool = pflag.Bool("ll", false, "Generate an LL(1) parser")
+	parserSLR     *bool = pflag.Bool("slr", false, "Generate a simple LR(1) parser")
+	parserCLR     *bool = pflag.Bool("clr", false, "Generate a canonical LR(1) parser")
+	parserLALR    *bool = pflag.Bool("lalr", false, "Generate a canonical LR(1) parser")
+	parserNoAmbig *bool = pflag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it")
+
+	lexerTrace  *bool = pflag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
+	parserTrace *bool = pflag.Bool("debug-parser", false, "Print the parser trace to stdout")
+
+	hooksPath      *string = pflag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.")
+	hooksTableName *string = pflag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.")
+
+	irType *string = pflag.String("ir", "", "The fully-qualified type of IR to generate.")
+
+	version *bool = pflag.Bool("version", false, "Print the version of ictcc and exit")
 )
 
 func init() {
@@ -324,20 +325,13 @@ func init() {
 		langUsage        = "The name of the languae being generated"
 		langDefault      = "Unspecified"
 	)
-	flag.BoolVar(&noGen, "no-gen", false, noGenUsage)
-	flag.BoolVar(&noGen, "n", false, noGenUsage+" (shorthand)")
-	flag.BoolVar(&genAST, "ast", false, genASTUsage)
-	flag.BoolVar(&genAST, "a", false, genASTUsage+" (shorthand)")
-	flag.BoolVar(&genSpec, "spec", false, genSpecUsage)
-	flag.BoolVar(&genSpec, "s", false, genSpecUsage+" (shorthand)")
-	flag.BoolVar(&genTree, "tree", false, genTreeUsage)
-	flag.BoolVar(&genTree, "t", false, genTreeUsage+" (shorthand)")
-	flag.StringVar(&parserCff, "parser", parserCffDefault, parserCffUsage)
-	flag.StringVar(&parserCff, "p", parserCffDefault, parserCffUsage+" (shorthand)")
-	flag.StringVar(&lang, "lang", langDefault, langUsage)
-	flag.StringVar(&lang, "l", langDefault, langUsage+"(shorthand)")
-	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
-	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
+	pflag.BoolVarP(&noGen, "no-gen", "n", false, noGenUsage)
+	pflag.BoolVarP(&genAST, "ast", "a", false, genASTUsage)
+	pflag.BoolVarP(&showSpec, "spec", "s", false, genSpecUsage)
+	pflag.BoolVarP(&genTree, "tree", "t", false, genTreeUsage)
+	pflag.StringVarP(&parserCff, "parser", "p", parserCffDefault, parserCffUsage)
+	pflag.StringVarP(&lang, "lang", "l", langDefault, langUsage)
+	pflag.BoolVarP(&quietMode, "quiet", "q", false, quietUsage)
 }
 
 func main() {
@@ -356,7 +350,7 @@ func main() {
 	// gather options and arguments
 	invocation := strings.Join(os.Args[1:], " ")
 
-	flag.Parse()
+	pflag.Parse()
 
 	if *version {
 		fmt.Println(GetVersionString())
@@ -370,7 +364,7 @@ func main() {
 		InvocationArgs: invocation,
 	}
 
-	args := flag.Args()
+	args := pflag.Args()
 
 	if len(args) < 1 {
 		fmt.Fprintf(os.Stderr, "No files given to process")
@@ -419,6 +413,10 @@ func main() {
 	}
 
 	// now that args are gathered, parse markdown files into an AST
+	if !quietMode {
+		files := textfmt.Pluralize(len(args), "FISHI input file", "-s")
+		fmt.Printf("Reading %s...\n", files)
+	}
 	var joinedAST *fishi.AST
 
 	for _, file := range args {
@@ -464,7 +462,9 @@ func main() {
 	}
 
 	// attempt to turn AST into a fishi.Spec
-
+	if !quietMode {
+		fmt.Printf("Generating language spec from FISHI...\n")
+	}
 	spec, warnings, err := fishi.NewSpec(*joinedAST)
 	// warnings may be valid even if there is an error
 	if len(warnings) > 0 {
@@ -496,22 +496,24 @@ func main() {
 	}
 
 	// we officially have a spec. try to print it if requested
-	if genSpec {
+	if showSpec {
 		printSpec(spec)
 	}
 
 	if !noGen {
-		// TODO: jello, first need to create a PRELIM generation along with hooks
-		// pkg because that is the only way to validate the SDTS.
-
 		// okay, first try to create a parser
-		// TODO: this should deffo be in fishi package, not the bin.
 		var p ictiobus.Parser
 		var parserWarns []fishi.Warning
 		// if one is selected, use that one
 		if parserType != nil {
+			if !quietMode {
+				fmt.Printf("Creating %s parser from spec...\n", *parserType)
+			}
 			p, parserWarns, err = spec.CreateParser(*parserType, allowAmbig)
 		} else {
+			if !quietMode {
+				fmt.Printf("Creating most restrictive parser from spec...\n")
+			}
 			p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig)
 		}
 
@@ -569,6 +571,8 @@ func main() {
 			return
 		}
 		fmt.Printf("(NOTE: complete frontend generation not implemented yet)\n")
+	} else if !quietMode {
+		fmt.Printf("(code generation skipped due to flags)\n")
 	}
 
 }
diff --git a/genfishi.sh b/genfishi.sh
index 2d0c4b8..f70507f 100644
--- a/genfishi.sh
+++ b/genfishi.sh
@@ -2,10 +2,9 @@
 
 # shortcut for running ictcc on fishi.md for when we build a new ictcc bin
 
-./ictcc -clr \
-	-ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \
-	-dest .testout \
-	-l fishi -lang-ver 1.0.0 \
-	-hooks fishi/syntax \
-	fishi.md
-
+./ictcc --clr \
+	--ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \
+	--dest .testout \
+	-l fishi --lang-ver 1.0.0 \
+	--hooks fishi/syntax \
+	fishi.md "$@"
diff --git a/go.mod b/go.mod
index e18a2a9..d63d7b6 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/dekarrin/rosed v1.2.1
 	github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.2
 	golang.org/x/text v0.8.0
 )
diff --git a/go.sum b/go.sum
index 6c9928a..7425a17 100644
--- a/go.sum
+++ b/go.sum
@@ -7,6 +7,8 @@ github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK
 github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

From c9d0c100f6eb391fbad9ec0fd509197e3c3072cb Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 17:13:26 -0500
Subject: [PATCH 10/33] #16: added ability to silence errors

---
 cmd/ictcc/main.go | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index c02ee77..9d0b566 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -535,17 +535,21 @@ func main() {
 			return
 		}
 
-		fmt.Printf("Successfully generated %s parser from grammar\n", p.Type().String())
+		if !quietMode {
+			fmt.Printf("Successfully generated %s parser from grammar\n", p.Type().String())
+		}
 
 		// create a test compiler and output it
 		if !*valSDTSOff {
 			if *irType == "" {
-				fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing -ir parameter\n")
+				fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --ir parameter\n")
 			} else {
 				if *hooksPath == "" {
-					fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing -hooks parameter\n")
+					fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --hooks parameter\n")
 				} else {
-
+					if !quietMode {
+						fmt.Printf("Generating parser simulation binary in .sim...\n")
+					}
 					di := trans.ValidationOptions{
 						ParseTrees:    *valSDTSShowTrees,
 						FullDepGraphs: *valSDTSShowGraphs,
@@ -559,18 +563,23 @@ func main() {
 						returnCode = ExitErrGeneration
 						return
 					}
+					if !quietMode {
+						fmt.Printf("Done simulating input; removing .sim...\n")
+					}
 				}
 			}
 		}
 
 		// assuming it worked, now generate the final output
+		if !quietMode {
+			fmt.Printf("Generating compiler frontend in %s...\n", *dest)
+		}
 		err := fishi.GenerateCompilerGo(spec, md, *pkg, *dest, &cgOpts)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
 			returnCode = ExitErrGeneration
 			return
 		}
-		fmt.Printf("(NOTE: complete frontend generation not implemented yet)\n")
 	} else if !quietMode {
 		fmt.Printf("(code generation skipped due to flags)\n")
 	}

From 3f9ca8465d4ea3df2c572be6555390f301dc6171 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 4 Apr 2023 17:41:01 -0500
Subject: [PATCH 11/33] #16: distracted commit

---
 cmd/ictcc/main.go              |   2 +-
 fishi/templates/irmain.go.tmpl | 117 +++++++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+), 1 deletion(-)
 create mode 100644 fishi/templates/irmain.go.tmpl

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 9d0b566..e77fcf4 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -367,7 +367,7 @@ func main() {
 	args := pflag.Args()
 
 	if len(args) < 1 {
-		fmt.Fprintf(os.Stderr, "No files given to process")
+		fmt.Fprintf(os.Stderr, "No files given to process\n")
 		returnCode = ExitErrNoFiles
 		return
 	}
diff --git a/fishi/templates/irmain.go.tmpl b/fishi/templates/irmain.go.tmpl
new file mode 100644
index 0000000..494584c
--- /dev/null
+++ b/fishi/templates/irmain.go.tmpl
@@ -0,0 +1,117 @@
+{{/* irmain.go.tmpl is the main entry point for generated code that has the
+ability to debug the lexer, parser, and output the printed IR once complete. */}}
+{{/* own binary. */ -}}
+
+/*
+{{ .BinName | title }} runs a compiler frontend generated by ictiobus on one or
+more files and outputs the parsed intermediate representation (A
+{{ .IRTypePackage }}.{{ .IRTypeExpr }}) as a string to stdout.
+
+Usage:
+
+	{{ .BinName }} [flags] file1.md file2.md ...
+*/
+package main
+
+import (
+	"fmt"
+	"bufio"
+    "os"
+	"flag"
+
+	"{{ .BinPkg }}/internal/{{ .HooksPkg }}"
+	"{{ .BinPkg }}/internal/{{ .FrontendPkg }}"
+	"{{ .IRTypePackage }}"
+)
+
+const (
+    ExitSuccess = iota
+	ExitErrNoFiles
+	ExitErrSyntax
+    ExitErr
+)
+
+const (
+	versionString = "{{ .BinName }} {{ .BinVersion }}"
+)
+
+var (
+    returnCode = ExitSuccess
+)
+
+// flags
+var (
+	quietMode bool
+	printTrees bool = flag.Bool("t", false, "Print the parse trees of each file read to stdout")
+	version *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
+	lexerTrace  *bool = flag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
+	parserTrace *bool = flag.Bool("debug-parser", false, "Print the parser trace to stdout")
+)
+
+// init flags
+func init() {
+	const (
+		quietUsage       = "Quiet mode"
+	)
+	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
+	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
+}
+
+func main() {
+	// preserve possibly-set exit code while also checking for panic and
+    // propagating it if there was one.
+	defer func() {
+		if panicErr := recover(); panicErr != nil {
+			// we are panicking, make sure we dont lose the panic just because
+			// we checked
+			panic("unrecoverable panic occured")
+		} else {
+			os.Exit(returnCode)
+		}
+	}()
+
+	flag.Parse()
+
+	if *version {
+		fmt.Println(versionString)
+		return
+	}
+
+	files := flag.Args()
+
+	if len(files) == 0 {
+		fmt.Fprintf(os.Stderr, "No files specified\n")
+		returnCode = ExitErrNoFiles
+		return
+	}
+
+	hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
+	langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil)
+	
+	for _, f := files {
+		file, err := os.Open(f)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error opening file %s: %s\n", f, err)
+			returnCode = ExitErr
+			return
+		}
+
+		r := bufio.NewReader(file)
+
+		ir, pt, err := langFront.Analyze(r)
+
+		// parse tree might be valid no matter what, so we print it first
+		if *printTrees {
+			fmt.Println(pt.String())
+		}
+
+		res, err := fishi.ParseMarkdownFile(file, fo)
+		if res.AST != nil {
+			if joinedAST == nil {
+				joinedAST = res.AST
+			} else {
+				joinedAST.Nodes = append(joinedAST.Nodes, res.AST.Nodes...)
+			}
+		}
+	}
+}
\ No newline at end of file

From b576b9555f4f91f5b101f5597e6ab27ed91cf25a Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Wed, 5 Apr 2023 09:45:05 -0500
Subject: [PATCH 12/33] #16: combine the codegens perhaps glub

---
 fishi/codegen.go               | 44 +++++++++++++-------
 fishi/fishi.go                 |  8 +++-
 fishi/templates/irmain.go.tmpl | 73 ++++++++++++++++++++++++++++------
 fishi/templates/main.go.tmpl   |  6 +--
 4 files changed, 98 insertions(+), 33 deletions(-)

diff --git a/fishi/codegen.go b/fishi/codegen.go
index ba556e5..1097208 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -156,6 +156,24 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 	return cmd.Run()
 }
 
+func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, pkgName string, binPath string, opts CodegenOptions) error {
+	binName := filepath.Base(binPath)
+	gci, err := GenerateBinaryMainGo(spec, md, p, hooksPkgDir, hooksExpr, pkgName, binName, ".gen", opts)
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command("go", "build", "-o", binName, gci.MainFile)
+	cmd.Dir = gci.Path
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // GenerateTestCompiler generates (but does not yet run) a test compiler for the
 // given spec and pre-created parser, using the provided hooks package. Once it
 // is created, it will be able to be executed by calling go run on the provided
@@ -170,7 +188,7 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 // or var name.
 //
 // opts must be non-nil and IRType must be set.
-func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, opts CodegenOptions) (GeneratedCodeInfo, error) {
+func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, fePkgName string, genPath string, binName string, opts CodegenOptions) (GeneratedCodeInfo, error) {
 	if opts.IRType == "" {
 		return GeneratedCodeInfo{}, fmt.Errorf("IRType must be set in options")
 	}
@@ -181,7 +199,6 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	}
 
 	gci := GeneratedCodeInfo{}
-	var fePkgName = "sim" + strings.ToLower(md.Language)
 
 	// what is the name of our hooks package? find out by reading the first go
 	// file in the package.
@@ -246,25 +263,24 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	}
 
 	// create a directory to save things in
-	tmpDir := ".sim"
-	err = os.RemoveAll(tmpDir)
+	err = os.RemoveAll(genPath)
 	if err != nil {
 		return gci, fmt.Errorf("removing old temp dir: %w", err)
 	}
-	err = os.MkdirAll(tmpDir, 0766)
+	err = os.MkdirAll(genPath, 0766)
 	if err != nil {
 		return gci, fmt.Errorf("creating temp dir: %w", err)
 	}
 
 	// start copying the hooks package
-	hooksDestPath := filepath.Join(tmpDir, "internal", hooksPkgName)
+	hooksDestPath := filepath.Join(genPath, "internal", hooksPkgName)
 	hooksDone, err := copyDirToTargetAsync(hooksPkgDir, hooksDestPath)
 	if err != nil {
 		return gci, fmt.Errorf("copying hooks package: %w", err)
 	}
 
 	// generate the compiler code
-	fePkgPath := filepath.Join(tmpDir, "internal", fePkgName)
+	fePkgPath := filepath.Join(genPath, "internal", fePkgName)
 	err = GenerateCompilerGo(spec, md, fePkgName, fePkgPath, &opts)
 	if err != nil {
 		return gci, fmt.Errorf("generating compiler: %w", err)
@@ -285,9 +301,9 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 
 	// export template with main file
 	mainFillData := cgMainData{
-		BinPkg:            "github.com/dekarrin/ictiobus/langtest/" + strings.ToLower(safePkgIdent(md.Language)),
-		BinName:           "test" + strings.ToLower(safePkgIdent(md.Language)),
-		BinVersion:        "(simulator version)",
+		BinPkg:            "github.com/dekarrin/ictiobus/langexec/" + safePkgIdent(md.Language),
+		BinName:           binName,
+		BinVersion:        md.Version,
 		HooksPkg:          hooksPkgName,
 		HooksTableExpr:    hooksExpr,
 		FrontendPkg:       fePkgName,
@@ -309,7 +325,7 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// finally, render the main file
 	rf := renderFiles[ComponentMainFile]
 	mainFileRelPath := rf.outFile
-	err = renderTemplateToFile(rf.tmpl, mainFillData, filepath.Join(tmpDir, mainFileRelPath), opts.DumpPreFormat)
+	err = renderTemplateToFile(rf.tmpl, mainFillData, filepath.Join(genPath, mainFileRelPath), opts.DumpPreFormat)
 	if err != nil {
 		return gci, err
 	}
@@ -320,21 +336,21 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// shell out to run go module stuff
 	goModInitCmd := exec.Command("go", "mod", "init", mainFillData.BinPkg)
 	goModInitCmd.Env = os.Environ()
-	goModInitCmd.Dir = tmpDir
+	goModInitCmd.Dir = genPath
 	goModInitOutput, err := goModInitCmd.CombinedOutput()
 	if err != nil {
 		return gci, fmt.Errorf("initializing generated module with binary: %w\n%s", err, string(goModInitOutput))
 	}
 	goModTidyCmd := exec.Command("go", "mod", "tidy")
 	goModTidyCmd.Env = os.Environ()
-	goModTidyCmd.Dir = tmpDir
+	goModTidyCmd.Dir = genPath
 	goModTidyOutput, err := goModTidyCmd.CombinedOutput()
 	if err != nil {
 		return gci, fmt.Errorf("tidying generated module with binary: %w\n%s", err, string(goModTidyOutput))
 	}
 
 	// if we got here, all output has been written to the temp dir.
-	gci.Path = tmpDir
+	gci.Path = genPath
 	gci.MainFile = mainFileRelPath
 
 	return gci, nil
diff --git a/fishi/fishi.go b/fishi/fishi.go
index bcf3415..efd9d6d 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -46,7 +46,13 @@ type Options struct {
 // valOpts is not required to be set, and if nil will be treated as if it were
 // set to an empty struct.
 func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks, hooksTable string, cgOpts CodegenOptions, valOpts *trans.ValidationOptions) error {
-	genInfo, err := GenerateTestCompiler(spec, md, p, hooks, hooksTable, cgOpts)
+	pkgName := "sim" + strings.ToLower(md.Language)
+
+	binName := safeTCIdentifierName(md.Language)
+	binName = binName[2:] // remove initial "tc".
+	binName = strings.ToLower(binName)
+	binName = "test" + binName
+	genInfo, err := GenerateBinaryMainGo(spec, md, p, hooks, hooksTable, pkgName, ".sim", binName, cgOpts)
 	if err != nil {
 		return fmt.Errorf("generate test compiler: %w", err)
 	}
diff --git a/fishi/templates/irmain.go.tmpl b/fishi/templates/irmain.go.tmpl
index 494584c..162cf01 100644
--- a/fishi/templates/irmain.go.tmpl
+++ b/fishi/templates/irmain.go.tmpl
@@ -22,6 +22,8 @@ import (
 	"{{ .BinPkg }}/internal/{{ .HooksPkg }}"
 	"{{ .BinPkg }}/internal/{{ .FrontendPkg }}"
 	"{{ .IRTypePackage }}"
+
+	"github.com/dekarrin/ictiobus/trans"
 )
 
 const (
@@ -42,19 +44,30 @@ var (
 // flags
 var (
 	quietMode bool
-	printTrees bool = flag.Bool("t", false, "Print the parse trees of each file read to stdout")
-	version *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
-	lexerTrace  *bool = flag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
-	parserTrace *bool = flag.Bool("debug-parser", false, "Print the parser trace to stdout")
+	lexerTrace  bool
+	parserTrace bool
+	printTrees bool         = flag.Bool("t", false, "Print the parse trees of each file read to stdout")
+	version			  *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
+	doSimulation      *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
+	simSDTSShowTrees  *bool = flag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
+	simSDTSShowGraphs *bool = flag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
+	simSDTSFirstOnly  *bool = flag.Bool("sim-first", false, "Show only the first error found in SDTS validation of simulated trees")
+	simSDTSSkip       *int  = flag.Int("sim-skip", 0, "Skip the first N errors found in SDTS validation of simulated trees")
 )
 
 // init flags
 func init() {
 	const (
-		quietUsage       = "Quiet mode"
+		quietUsage       = "Quiet mode; disables output of the IR"
+		lexerUsage		 = "Print the lexer trace to stdout"
+		parserUsage		 = "Print the parser trace to stdout"
 	)
 	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
 	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
+	flag.BoolVar(&lexerTrace, "debug-lexer", false, lexerUsage)
+	flag.BoolVar(&lexerTrace, "l", false, lexerUsage+" (shorthand)")
+	flag.BoolVar(&parserTrace, "debug-parser", false, parserUsage)
+	flag.BoolVar(&parserTrace, "p", false, parserUsage+" (shorthand)")
 }
 
 func main() {
@@ -85,13 +98,46 @@ func main() {
 		return
 	}
 
+	opts := {{ .FrontendPkg }}.FrontendOptions{
+		LexerTrace: *lexerTrace,
+		ParserTrace: *parserTrace,
+	}
+
 	hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
 	langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil)
+
+	if *doSimulation {
+		hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
+
+		langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil)
+
+		// hooks will be set, so run validation now
+		valProd := langFront.Lexer.FakeLexemeProducer(true, "")
+
+		di := trans.ValidationOptions{
+			ParseTrees:    *simSDTSShowTrees,
+			FullDepGraphs: *simSDTSShowGraphs,
+			ShowAllErrors: !*simSDTSFirstOnly,
+			SkipErrors:    *simSDTSSkip,
+		}
+
+		sddErr := langFront.SDT.Validate(langFront.Parser.Grammar(), langFront.IRAttribute, di, valProd)
+		if sddErr != nil {
+			fmt.Fprintf(os.Stderr, "validation error: %v\n", sddErr.Error())
+			returnCode = ExitErr
+			return
+		}
+
+		if !quietMode {
+			fmt.Printf("simulation completed with no errors\n")
+		}
+		return
+	}
 	
 	for _, f := files {
 		file, err := os.Open(f)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error opening file %s: %s\n", f, err)
+			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
 			returnCode = ExitErr
 			return
 		}
@@ -105,13 +151,14 @@ func main() {
 			fmt.Println(pt.String())
 		}
 
-		res, err := fishi.ParseMarkdownFile(file, fo)
-		if res.AST != nil {
-			if joinedAST == nil {
-				joinedAST = res.AST
-			} else {
-				joinedAST.Nodes = append(joinedAST.Nodes, res.AST.Nodes...)
-			}
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
+			returnCode = ExitErrSyntax
+			return
+		}
+
+		if !quietMode {
+			fmt.Println("----- %s -----\n%v\n\n"ir.String())
 		}
 	}
 }
\ No newline at end of file
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index ff60a9b..15c0193 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -36,11 +36,7 @@ var (
 	quietMode bool
 	version *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
 {{if .IncludeSimulation }}
-	doSimulation      *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules")
-	simSDTSShowTrees  *bool = flag.Bool("sim-sdts-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
-	simSDTSShowGraphs *bool = flag.Bool("sim-sdts-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
-	simSDTSFirstOnly  *bool = flag.Bool("sim-sdts-first", false, "Show only the first error found in SDTS validation of simulated trees")
-	simSDTSSkip       *int  = flag.Int("sim-sdts-skip", 0, "Skip the first N errors found in SDTS validation of simulated trees")
+
 {{end -}}
 )
 

From b63b53159af4b7e36e7c6cb4010523e56e29bd9c Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Thu, 6 Apr 2023 08:04:52 -0500
Subject: [PATCH 13/33] #16: making irmain be the new all-main

---
 cmd/ictcc/main.go              | 75 ++++++++++++++++++++--------------
 fishi/codegen.go               | 16 +++++---
 fishi/fishi.go                 | 14 +++++--
 fishi/templates/irmain.go.tmpl |  4 +-
 4 files changed, 67 insertions(+), 42 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index e77fcf4..67012c3 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -34,6 +34,10 @@ Flags:
 		Set the location of the pre-compiled parser cache to the given CFF
 		format file as opposed to the default of './parser.cff'.
 
+	--preserve-bin-source
+		Do not delete source files for any generated binary after compiling the
+		binary.
+
 	--no-cache
 		Disable the loading of any cached frontend components, even if a
 		pre-built one is available.
@@ -45,30 +49,32 @@ Flags:
 	--version
 		Print the version of the ictiobus compiler-compiler and exit.
 
-	--val-sdts-off
-		Disable validatione of the SDTS of the resulting fishi.
+	--sim-off
+		Disable simulation of the language once built. This will disable SDTS
+		validation, as live simulation is the only way to do this due to the
+		lack of support for dynamic loading of the hooks package in Go.
 
-	--val-sdts-trees
+	--sim-trees
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, show the parse tree(s) that caused the problem.
 
 		Has no effect if -val-sdts-off is set.
 
-	--val-sdts-graphs
+	--sim-graphs
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, show the full resulting dependency graph(s) that caused
 		the issue (if any).
 
 		Has no effect if -val-sdts-off is set.
 
-	--val-sdts-first
+	--sim-first-err
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, show only the problem(s) found in the first simulated
 		parse tree (after any skipped by -val-sdts-skip) and then stop.
 
 		Has no effect if -val-sdts-off is set.
 
-	--val-sdts-skip N
+	--sim-skip-errs N
 		If problems are detected with the SDTS of the resulting fishi during
 		SDTS validation, skip the first N simulated parse trees in the output.
 		Combine with -val-sdts-first to view a specific parse tree.
@@ -123,6 +129,10 @@ Flags:
 		Use the provided file as the template for outputting the generated SDTS
 		file instead of the default embedded within the binary.
 
+	--tmpl-main FILE
+		Use the provided file as the template for outputting generated binary
+		main file instead of the default embedded within the binary.
+
 	--tmpl-frontend FILE
 		Use the provided file as the template for outputting the generated
 		frontend file instead of the default embedded within the binary.
@@ -270,31 +280,33 @@ var (
 )
 
 var (
-	quietMode     bool
-	noGen         bool
-	genAST        bool
-	genTree       bool
-	showSpec      bool
-	parserCff     string
-	lang          string
-	dumpPreFormat *bool   = pflag.Bool("pre-format", false, "Dump the generated code before running through gofmt")
-	pkg           *string = pflag.String("pkg", "fe", "The name of the package to place generated files in")
-	dest          *string = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
-	langVer       *string = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
-	noCache       *bool   = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
-	noCacheOutput *bool   = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
-
-	valSDTSOff        *bool = pflag.Bool("val-sdts-off", false, "Disable validation of the SDTS of the resulting fishi")
-	valSDTSShowTrees  *bool = pflag.Bool("val-sdts-trees", false, "Show trees that caused SDTS validation errors")
-	valSDTSShowGraphs *bool = pflag.Bool("val-sdts-graphs", false, "Show full generated dependency graph output for parse trees that caused SDTS validation errors")
-	valSDTSFirstOnly  *bool = pflag.Bool("val-sdts-first", false, "Show only the first error found in SDTS validation")
-	valSDTSSkip       *int  = pflag.Int("val-sdts-skip", 0, "Skip the first N errors found in SDTS validation in output")
+	quietMode         bool
+	noGen             bool
+	genAST            bool
+	genTree           bool
+	showSpec          bool
+	parserCff         string
+	lang              string
+	preserveBinSource *bool   = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
+	dumpPreFormat     *bool   = pflag.Bool("pre-format", false, "Dump the generated code before running through gofmt")
+	pkg               *string = pflag.String("pkg", "fe", "The name of the package to place generated files in")
+	dest              *string = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
+	langVer           *string = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
+	noCache           *bool   = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
+	noCacheOutput     *bool   = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
+
+	valSDTSOff        *bool = pflag.Bool("sim-off", false, "Disable input simulation of the language once built")
+	valSDTSShowTrees  *bool = pflag.Bool("sim-trees", false, "Show parse trees that caused errors during simulation")
+	valSDTSShowGraphs *bool = pflag.Bool("sim-graphs", false, "Show full generated dependency graph output for parse trees that caused errors during simulation")
+	valSDTSFirstOnly  *bool = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation")
+	valSDTSSkip       *int  = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation in output")
 
 	tmplTokens *string = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
 	tmplLexer  *string = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
 	tmplParser *string = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
 	tmplSDTS   *string = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
 	tmplFront  *string = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
+	tmplMain   *string = pflag.String("tmpl-main", "", "A template file to replace the embedded main.go template with")
 
 	parserLL      *bool = pflag.Bool("ll", false, "Generate an LL(1) parser")
 	parserSLR     *bool = pflag.Bool("slr", false, "Generate a simple LR(1) parser")
@@ -388,9 +400,10 @@ func main() {
 	}
 
 	cgOpts := fishi.CodegenOptions{
-		DumpPreFormat: *dumpPreFormat,
-		IRType:        *irType,
-		TemplateFiles: map[string]string{},
+		DumpPreFormat:        *dumpPreFormat,
+		IRType:               *irType,
+		TemplateFiles:        map[string]string{},
+		PreserveBinarySource: *preserveBinSource,
 	}
 	if *tmplTokens != "" {
 		cgOpts.TemplateFiles[fishi.ComponentTokens] = *tmplTokens
@@ -407,6 +420,9 @@ func main() {
 	if *tmplFront != "" {
 		cgOpts.TemplateFiles[fishi.ComponentFrontend] = *tmplFront
 	}
+	if *tmplMain != "" {
+		cgOpts.TemplateFiles[fishi.ComponentMainFile] = *tmplMain
+	}
 	if len(cgOpts.TemplateFiles) == 0 {
 		// just nil it
 		cgOpts.TemplateFiles = nil
@@ -563,9 +579,6 @@ func main() {
 						returnCode = ExitErrGeneration
 						return
 					}
-					if !quietMode {
-						fmt.Printf("Done simulating input; removing .sim...\n")
-					}
 				}
 			}
 		}
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 1097208..b5a8bba 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -71,7 +71,7 @@ var (
 	//go:embed templates/frontend.go.tmpl
 	TemplateFrontend string
 
-	//go:embed templates/main.go.tmpl
+	//go:embed templates/irmain.go.tmpl
 	TemplateMainFile string
 )
 
@@ -117,6 +117,12 @@ type CodegenOptions struct {
 	// specific type instead of requiring an explicit type instantiation when
 	// called.
 	IRType string
+
+	// PreserveBinarySource is whether to keep the source files for any
+	// generated binary after the binary has been successfully
+	// compiled/executed. Normally, these files are removed, but preserving them
+	// allows for diagnostics on the generated source.
+	PreserveBinarySource bool
 }
 
 // GeneratedCodeInfo contains information about the generated code.
@@ -138,16 +144,16 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 
 	args := []string{"run", gci.MainFile, "-sim"}
 	if valOptions.FullDepGraphs {
-		args = append(args, "-sim-sdts-graphs")
+		args = append(args, "-sim-graphs")
 	}
 	if valOptions.ParseTrees {
-		args = append(args, "-sim-sdts-trees")
+		args = append(args, "-sim-trees")
 	}
 	if !valOptions.ShowAllErrors {
-		args = append(args, "-sim-sdts-first")
+		args = append(args, "-sim-first-err")
 	}
 	if valOptions.SkipErrors != 0 {
-		args = append(args, "-sim-sdts-skip", fmt.Sprintf("%d", valOptions.SkipErrors))
+		args = append(args, "-sim-skip-errs", fmt.Sprintf("%d", valOptions.SkipErrors))
 	}
 	cmd := exec.Command("go", args...)
 	cmd.Dir = gci.Path
diff --git a/fishi/fishi.go b/fishi/fishi.go
index efd9d6d..43ee072 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -45,6 +45,10 @@ type Options struct {
 //
 // valOpts is not required to be set, and if nil will be treated as if it were
 // set to an empty struct.
+//
+// No binary is generated as part of this, but source is which is then executed.
+// If PreserveBinarySource is set in cgOpts, the source will be left in the
+// .sim directory.
 func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks, hooksTable string, cgOpts CodegenOptions, valOpts *trans.ValidationOptions) error {
 	pkgName := "sim" + strings.ToLower(md.Language)
 
@@ -62,10 +66,12 @@ func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks
 		return fmt.Errorf("execute test compiler: %w", err)
 	}
 
-	// if we got here, no errors. delete the test compiler and its directory
-	err = os.RemoveAll(genInfo.Path)
-	if err != nil {
-		return fmt.Errorf("remove test compiler: %w", err)
+	if !cgOpts.PreserveBinarySource {
+		// if we got here, no errors. delete the test compiler and its directory
+		err = os.RemoveAll(genInfo.Path)
+		if err != nil {
+			return fmt.Errorf("remove test compiler: %w", err)
+		}
 	}
 
 	return nil
diff --git a/fishi/templates/irmain.go.tmpl b/fishi/templates/irmain.go.tmpl
index 162cf01..ae8a854 100644
--- a/fishi/templates/irmain.go.tmpl
+++ b/fishi/templates/irmain.go.tmpl
@@ -51,8 +51,8 @@ var (
 	doSimulation      *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
 	simSDTSShowTrees  *bool = flag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
 	simSDTSShowGraphs *bool = flag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
-	simSDTSFirstOnly  *bool = flag.Bool("sim-first", false, "Show only the first error found in SDTS validation of simulated trees")
-	simSDTSSkip       *int  = flag.Int("sim-skip", 0, "Skip the first N errors found in SDTS validation of simulated trees")
+	simSDTSFirstOnly  *bool = flag.Bool("sim-first-errs", false, "Show only the first error found in SDTS validation of simulated trees")
+	simSDTSSkip       *int  = flag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
 )
 
 // init flags

From 27d0e3c8d0fd5e6365b198c8cc6253dd689f3631 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Thu, 6 Apr 2023 08:45:02 -0500
Subject: [PATCH 14/33] #16: replace old main with new irmain

---
 fishi/codegen.go               |  24 +++--
 fishi/templates/irmain.go.tmpl | 164 ---------------------------------
 fishi/templates/main.go.tmpl   |  91 +++++++++++++++---
 3 files changed, 94 insertions(+), 185 deletions(-)
 delete mode 100644 fishi/templates/irmain.go.tmpl

diff --git a/fishi/codegen.go b/fishi/codegen.go
index b5a8bba..3053c01 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -71,7 +71,7 @@ var (
 	//go:embed templates/frontend.go.tmpl
 	TemplateFrontend string
 
-	//go:embed templates/irmain.go.tmpl
+	//go:embed templates/main.go.tmpl
 	TemplateMainFile string
 )
 
@@ -268,11 +268,6 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 		return strings.ToLower(s)
 	}
 
-	// create a directory to save things in
-	err = os.RemoveAll(genPath)
-	if err != nil {
-		return gci, fmt.Errorf("removing old temp dir: %w", err)
-	}
 	err = os.MkdirAll(genPath, 0766)
 	if err != nil {
 		return gci, fmt.Errorf("creating temp dir: %w", err)
@@ -339,6 +334,20 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// wait for the hooks package to be copied; we'll need it for go mod tidy
 	<-hooksDone
 
+	// wipe any existing go module stuff
+	err = os.RemoveAll(filepath.Join(genPath, "go.mod"))
+	if err != nil {
+		return gci, fmt.Errorf("removing existing go.mod: %w", err)
+	}
+	err = os.RemoveAll(filepath.Join(genPath, "go.sum"))
+	if err != nil {
+		return gci, fmt.Errorf("removing existing go.sum: %w", err)
+	}
+	err = os.RemoveAll(filepath.Join(genPath, "vendor"))
+	if err != nil {
+		return gci, fmt.Errorf("removing existing vendor directory: %w", err)
+	}
+
 	// shell out to run go module stuff
 	goModInitCmd := exec.Command("go", "mod", "init", mainFillData.BinPkg)
 	goModInitCmd.Env = os.Environ()
@@ -422,6 +431,9 @@ func createFuncMap() template.FuncMap {
 			s = strings.ReplaceAll(s, "`", "` + \"`\" + `")
 			return fmt.Sprintf("`%s`", s)
 		},
+		"title": func(s string) string {
+			return titleCaser.String(s)
+		},
 	}
 }
 
diff --git a/fishi/templates/irmain.go.tmpl b/fishi/templates/irmain.go.tmpl
deleted file mode 100644
index ae8a854..0000000
--- a/fishi/templates/irmain.go.tmpl
+++ /dev/null
@@ -1,164 +0,0 @@
-{{/* irmain.go.tmpl is the main entry point for generated code that has the
-ability to debug the lexer, parser, and output the printed IR once complete. */}}
-{{/* own binary. */ -}}
-
-/*
-{{ .BinName | title }} runs a compiler frontend generated by ictiobus on one or
-more files and outputs the parsed intermediate representation (A
-{{ .IRTypePackage }}.{{ .IRTypeExpr }}) as a string to stdout.
-
-Usage:
-
-	{{ .BinName }} [flags] file1.md file2.md ...
-*/
-package main
-
-import (
-	"fmt"
-	"bufio"
-    "os"
-	"flag"
-
-	"{{ .BinPkg }}/internal/{{ .HooksPkg }}"
-	"{{ .BinPkg }}/internal/{{ .FrontendPkg }}"
-	"{{ .IRTypePackage }}"
-
-	"github.com/dekarrin/ictiobus/trans"
-)
-
-const (
-    ExitSuccess = iota
-	ExitErrNoFiles
-	ExitErrSyntax
-    ExitErr
-)
-
-const (
-	versionString = "{{ .BinName }} {{ .BinVersion }}"
-)
-
-var (
-    returnCode = ExitSuccess
-)
-
-// flags
-var (
-	quietMode bool
-	lexerTrace  bool
-	parserTrace bool
-	printTrees bool         = flag.Bool("t", false, "Print the parse trees of each file read to stdout")
-	version			  *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
-	doSimulation      *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
-	simSDTSShowTrees  *bool = flag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
-	simSDTSShowGraphs *bool = flag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
-	simSDTSFirstOnly  *bool = flag.Bool("sim-first-errs", false, "Show only the first error found in SDTS validation of simulated trees")
-	simSDTSSkip       *int  = flag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
-)
-
-// init flags
-func init() {
-	const (
-		quietUsage       = "Quiet mode; disables output of the IR"
-		lexerUsage		 = "Print the lexer trace to stdout"
-		parserUsage		 = "Print the parser trace to stdout"
-	)
-	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
-	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
-	flag.BoolVar(&lexerTrace, "debug-lexer", false, lexerUsage)
-	flag.BoolVar(&lexerTrace, "l", false, lexerUsage+" (shorthand)")
-	flag.BoolVar(&parserTrace, "debug-parser", false, parserUsage)
-	flag.BoolVar(&parserTrace, "p", false, parserUsage+" (shorthand)")
-}
-
-func main() {
-	// preserve possibly-set exit code while also checking for panic and
-    // propagating it if there was one.
-	defer func() {
-		if panicErr := recover(); panicErr != nil {
-			// we are panicking, make sure we dont lose the panic just because
-			// we checked
-			panic("unrecoverable panic occured")
-		} else {
-			os.Exit(returnCode)
-		}
-	}()
-
-	flag.Parse()
-
-	if *version {
-		fmt.Println(versionString)
-		return
-	}
-
-	files := flag.Args()
-
-	if len(files) == 0 {
-		fmt.Fprintf(os.Stderr, "No files specified\n")
-		returnCode = ExitErrNoFiles
-		return
-	}
-
-	opts := {{ .FrontendPkg }}.FrontendOptions{
-		LexerTrace: *lexerTrace,
-		ParserTrace: *parserTrace,
-	}
-
-	hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
-	langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil)
-
-	if *doSimulation {
-		hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
-
-		langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil)
-
-		// hooks will be set, so run validation now
-		valProd := langFront.Lexer.FakeLexemeProducer(true, "")
-
-		di := trans.ValidationOptions{
-			ParseTrees:    *simSDTSShowTrees,
-			FullDepGraphs: *simSDTSShowGraphs,
-			ShowAllErrors: !*simSDTSFirstOnly,
-			SkipErrors:    *simSDTSSkip,
-		}
-
-		sddErr := langFront.SDT.Validate(langFront.Parser.Grammar(), langFront.IRAttribute, di, valProd)
-		if sddErr != nil {
-			fmt.Fprintf(os.Stderr, "validation error: %v\n", sddErr.Error())
-			returnCode = ExitErr
-			return
-		}
-
-		if !quietMode {
-			fmt.Printf("simulation completed with no errors\n")
-		}
-		return
-	}
-	
-	for _, f := files {
-		file, err := os.Open(f)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
-			returnCode = ExitErr
-			return
-		}
-
-		r := bufio.NewReader(file)
-
-		ir, pt, err := langFront.Analyze(r)
-
-		// parse tree might be valid no matter what, so we print it first
-		if *printTrees {
-			fmt.Println(pt.String())
-		}
-
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
-			returnCode = ExitErrSyntax
-			return
-		}
-
-		if !quietMode {
-			fmt.Println("----- %s -----\n%v\n\n"ir.String())
-		}
-	}
-}
\ No newline at end of file
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index 15c0193..e5b4239 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -1,25 +1,38 @@
-{{/* main.go.tmpl is the main entry point for generated code that uses its */}}
+{{/* irmain.go.tmpl is the main entry point for generated code that has the
+ability to debug the lexer, parser, and output the printed IR once complete. */}}
 {{/* own binary. */ -}}
 
+/*
+{{ .BinName | title }} runs a compiler frontend generated by ictiobus on one or
+more files and outputs the parsed intermediate representation (A
+{{ .IRTypePackage }}.{{ .IRType }}) as a string to stdout.
+
+Usage:
+
+	{{ .BinName }} [flags] file1.md file2.md ...
+*/
 package main
 
 import (
 	"fmt"
+	"bufio"
     "os"
 	"flag"
+	"path/filepath"
 
 	"{{ .BinPkg }}/internal/{{ .HooksPkg }}"
 	"{{ .BinPkg }}/internal/{{ .FrontendPkg }}"
-{{if and .IncludeSimulation .IRTypePackage }}
+{{if .IRTypePackage -}}
 	"{{ .IRTypePackage }}"
-{{end -}}
+{{- end}}
 
 	"github.com/dekarrin/ictiobus/trans"
 )
 
 const (
     ExitSuccess = iota
-
+	ExitErrNoFiles
+	ExitErrSyntax
     ExitErr
 )
 
@@ -34,19 +47,30 @@ var (
 // flags
 var (
 	quietMode bool
-	version *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
-{{if .IncludeSimulation }}
-
-{{end -}}
+	lexerTrace  bool
+	parserTrace bool
+	printTrees        *bool = flag.Bool("t", false, "Print the parse trees of each file read to stdout")
+	version			  *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
+	doSimulation      *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
+	simSDTSShowTrees  *bool = flag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
+	simSDTSShowGraphs *bool = flag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
+	simSDTSFirstOnly  *bool = flag.Bool("sim-first-errs", false, "Show only the first error found in SDTS validation of simulated trees")
+	simSDTSSkip       *int  = flag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
 )
 
 // init flags
 func init() {
 	const (
-		quietUsage       = "Quiet mode"
+		quietUsage       = "Quiet mode; disables output of the IR"
+		lexerUsage		 = "Print the lexer trace to stdout"
+		parserUsage		 = "Print the parser trace to stdout"
 	)
 	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
 	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
+	flag.BoolVar(&lexerTrace, "debug-lexer", false, lexerUsage)
+	flag.BoolVar(&lexerTrace, "l", false, lexerUsage+" (shorthand)")
+	flag.BoolVar(&parserTrace, "debug-parser", false, parserUsage)
+	flag.BoolVar(&parserTrace, "p", false, parserUsage+" (shorthand)")
 }
 
 func main() {
@@ -69,8 +93,14 @@ func main() {
 		return
 	}
 
-{{ if .IncludeSimulation }}
-	// do simulation if requested
+	opts := {{ .FrontendPkg }}.FrontendOptions{
+		LexerTrace: lexerTrace,
+		ParserTrace: parserTrace,
+	}
+
+	hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
+	langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, &opts)
+
 	if *doSimulation {
 		hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
 
@@ -94,12 +124,43 @@ func main() {
 		}
 
 		if !quietMode {
-			fmt.Printf("(simulation completed with 0 errors)\n")
+			fmt.Printf("Simulation completed with no errors\n")
 		}
+		return
 	}
-{{end -}}
 
-	if !quietMode {
-		fmt.Printf("({{.BinName}} executed successfully)\n")
+	files := flag.Args()
+	if len(files) == 0 {
+		fmt.Fprintf(os.Stderr, "No files specified\n")
+		returnCode = ExitErrNoFiles
+		return
+	}
+	
+	for _, f := range files {
+		file, err := os.Open(f)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
+			returnCode = ExitErr
+			return
+		}
+
+		r := bufio.NewReader(file)
+
+		ir, pt, err := langFront.Analyze(r)
+
+		// parse tree might be valid no matter what, so we print it first
+		if *printTrees {
+			fmt.Println(pt.String())
+		}
+
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
+			returnCode = ExitErrSyntax
+			return
+		}
+
+		if !quietMode {
+			fmt.Println("=== Analysis of %s ===\n%s\n\n", filepath.Base(f), ir)
+		}
 	}
 }
\ No newline at end of file

From 4896a92f1509b8869d5f75f1cec3d76a5d90fde4 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Thu, 6 Apr 2023 09:08:03 -0500
Subject: [PATCH 15/33] #16: should be able to gen diagbin now

---
 cmd/ictcc/main.go | 59 +++++++++++++++++++++++++++++++++++++++--------
 fishi/codegen.go  | 14 +++++++++++
 2 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 67012c3..3cfb591 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -26,6 +26,15 @@ Flags:
 		checked for errors but no other action is taken (unless specified by
 		other flags).
 
+	-d/--diag FILE
+		Generate a diagnostics binary for the target language. Assuming there
+		are no issues with the FISHI spec, this will generate a binary that can
+		analyze files written in the target language and output the result of
+		frontend analysis. This can be useful for testing out the frontend on
+		files quickly and efficiently, as it also includes further options
+		useful for debugging purposes, such as debugging lexed tokens and the
+		parser itself.
+
 	-q/--quiet
 		Do not show progress messages. This does not affect error messages or
 		warning output.
@@ -108,7 +117,7 @@ Flags:
 		Set the version of the language to generate a frontend for. Defaults to
 		"v0.0.0".
 
-	--pre-format
+	--debug-templates
 		Enable dumping of the fishi filled template files before they are passed
 		to the formatter. This allows debugging of the template files when
 		editing them, since they must be valid go code to be formatted.
@@ -280,15 +289,17 @@ var (
 )
 
 var (
-	quietMode         bool
-	noGen             bool
-	genAST            bool
-	genTree           bool
-	showSpec          bool
-	parserCff         string
-	lang              string
+	quietMode bool
+	noGen     bool
+	genAST    bool
+	genTree   bool
+	showSpec  bool
+	parserCff string
+	lang      string
+
+	diagnosticsBin    *string = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
 	preserveBinSource *bool   = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
-	dumpPreFormat     *bool   = pflag.Bool("pre-format", false, "Dump the generated code before running through gofmt")
+	debugTemplates    *bool   = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
 	pkg               *string = pflag.String("pkg", "fe", "The name of the package to place generated files in")
 	dest              *string = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
 	langVer           *string = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
@@ -369,6 +380,19 @@ func main() {
 		return
 	}
 
+	// mutually exclusive and required options for diagnostics bin generation.
+	if *diagnosticsBin != "" {
+		if noGen {
+			fmt.Fprintf(os.Stderr, "ERROR: Diagnostics binary generation canont be enabled if -n/--no-gen is specified\n")
+			returnCode = ExitErrInvalidFlags
+			return
+		} else if *irType == "" || *hooksPath == "" {
+			fmt.Fprintf(os.Stderr, "ERROR: diagnostics binary generation requires both --ir and --hooks to be set\n")
+			returnCode = ExitErrInvalidFlags
+			return
+		}
+	}
+
 	// create a spec metadata object
 	md := fishi.SpecMetadata{
 		Language:       lang,
@@ -400,7 +424,7 @@ func main() {
 	}
 
 	cgOpts := fishi.CodegenOptions{
-		DumpPreFormat:        *dumpPreFormat,
+		DumpPreFormat:        *debugTemplates,
 		IRType:               *irType,
 		TemplateFiles:        map[string]string{},
 		PreserveBinarySource: *preserveBinSource,
@@ -583,6 +607,21 @@ func main() {
 			}
 		}
 
+		// generate diagnostics output if requested
+		if *diagnosticsBin != "" {
+			// already checked required flags
+			if !quietMode {
+				fmt.Printf("Generating diagnostics binary in %s...\n", *diagnosticsBin)
+			}
+
+			err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *pkg, *diagnosticsBin, cgOpts)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
+				returnCode = ExitErrGeneration
+				return
+			}
+		}
+
 		// assuming it worked, now generate the final output
 		if !quietMode {
 			fmt.Printf("Generating compiler frontend in %s...\n", *dest)
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 3053c01..2bcc494 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -170,6 +170,7 @@ func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, ho
 	}
 
 	cmd := exec.Command("go", "build", "-o", binName, gci.MainFile)
+	//cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
 	cmd.Dir = gci.Path
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -177,6 +178,19 @@ func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, ho
 		return err
 	}
 
+	// Move it to the target location.
+	if err := os.Rename(filepath.Join(gci.Path, binName), binPath); err != nil {
+		return err
+	}
+
+	// unless requested to preserve the source, remove the generated source
+	// directory.
+	if !opts.PreserveBinarySource {
+		if err := os.RemoveAll(gci.Path); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

From 0568e4758e45dbfdbe202d7d2bde02107c49062d Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Thu, 6 Apr 2023 09:17:56 -0500
Subject: [PATCH 16/33] #16: this is a pflags household

---
 fishi/codegen.go             | 10 ++++----
 fishi/templates/main.go.tmpl | 50 +++++++++++++-----------------------
 2 files changed, 23 insertions(+), 37 deletions(-)

diff --git a/fishi/codegen.go b/fishi/codegen.go
index 2bcc494..4f7cf00 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -142,18 +142,18 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 		valOptions = &trans.ValidationOptions{}
 	}
 
-	args := []string{"run", gci.MainFile, "-sim"}
+	args := []string{"run", gci.MainFile, "--sim"}
 	if valOptions.FullDepGraphs {
-		args = append(args, "-sim-graphs")
+		args = append(args, "--sim-graphs")
 	}
 	if valOptions.ParseTrees {
-		args = append(args, "-sim-trees")
+		args = append(args, "--sim-trees")
 	}
 	if !valOptions.ShowAllErrors {
-		args = append(args, "-sim-first-err")
+		args = append(args, "--sim-first-err")
 	}
 	if valOptions.SkipErrors != 0 {
-		args = append(args, "-sim-skip-errs", fmt.Sprintf("%d", valOptions.SkipErrors))
+		args = append(args, "--sim-skip-errs", fmt.Sprintf("%d", valOptions.SkipErrors))
 	}
 	cmd := exec.Command("go", args...)
 	cmd.Dir = gci.Path
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index e5b4239..c39aa45 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -17,7 +17,6 @@ import (
 	"fmt"
 	"bufio"
     "os"
-	"flag"
 	"path/filepath"
 
 	"{{ .BinPkg }}/internal/{{ .HooksPkg }}"
@@ -27,6 +26,8 @@ import (
 {{- end}}
 
 	"github.com/dekarrin/ictiobus/trans"
+
+	"github.com/spf13/pflag"
 )
 
 const (
@@ -46,33 +47,18 @@ var (
 
 // flags
 var (
-	quietMode bool
-	lexerTrace  bool
-	parserTrace bool
-	printTrees        *bool = flag.Bool("t", false, "Print the parse trees of each file read to stdout")
-	version			  *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
-	doSimulation      *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
-	simSDTSShowTrees  *bool = flag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
-	simSDTSShowGraphs *bool = flag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
-	simSDTSFirstOnly  *bool = flag.Bool("sim-first-errs", false, "Show only the first error found in SDTS validation of simulated trees")
-	simSDTSSkip       *int  = flag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
+	quietMode         *bool = pflag.BoolP("quiet", "q", false, "Quiet mode; disables output of the IR")
+	lexerTrace        *bool = pflag.BoolP("debug-lexer", "l", false, "Print the lexer trace to stdout")
+	parserTrace       *bool = pflag.BoolP("debug-parser", "p", false, "Print the parser trace to stdout") 
+	printTrees        *bool = pflag.Bool("t", false, "Print the parse trees of each file read to stdout")
+	version			  *bool = pflag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
+	doSimulation      *bool = pflag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
+	simSDTSShowTrees  *bool = pflag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
+	simSDTSShowGraphs *bool = pflag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
+	simSDTSFirstOnly  *bool = pflag.Bool("sim-first-errs", false, "Show only the first error found in SDTS validation of simulated trees")
+	simSDTSSkip       *int  = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
 )
 
-// init flags
-func init() {
-	const (
-		quietUsage       = "Quiet mode; disables output of the IR"
-		lexerUsage		 = "Print the lexer trace to stdout"
-		parserUsage		 = "Print the parser trace to stdout"
-	)
-	flag.BoolVar(&quietMode, "quiet", false, quietUsage)
-	flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)")
-	flag.BoolVar(&lexerTrace, "debug-lexer", false, lexerUsage)
-	flag.BoolVar(&lexerTrace, "l", false, lexerUsage+" (shorthand)")
-	flag.BoolVar(&parserTrace, "debug-parser", false, parserUsage)
-	flag.BoolVar(&parserTrace, "p", false, parserUsage+" (shorthand)")
-}
-
 func main() {
 	// preserve possibly-set exit code while also checking for panic and
     // propagating it if there was one.
@@ -86,7 +72,7 @@ func main() {
 		}
 	}()
 
-	flag.Parse()
+	pflag.Parse()
 
 	if *version {
 		fmt.Println(versionString)
@@ -94,8 +80,8 @@ func main() {
 	}
 
 	opts := {{ .FrontendPkg }}.FrontendOptions{
-		LexerTrace: lexerTrace,
-		ParserTrace: parserTrace,
+		LexerTrace: *lexerTrace,
+		ParserTrace: *parserTrace,
 	}
 
 	hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
@@ -123,13 +109,13 @@ func main() {
 			return
 		}
 
-		if !quietMode {
+		if !*quietMode {
 			fmt.Printf("Simulation completed with no errors\n")
 		}
 		return
 	}
 
-	files := flag.Args()
+	files := pflag.Args()
 	if len(files) == 0 {
 		fmt.Fprintf(os.Stderr, "No files specified\n")
 		returnCode = ExitErrNoFiles
@@ -159,7 +145,7 @@ func main() {
 			return
 		}
 
-		if !quietMode {
+		if !*quietMode {
 			fmt.Println("=== Analysis of %s ===\n%s\n\n", filepath.Base(f), ir)
 		}
 	}

From 2d8c3ea03b2d687c5e684d94136725ce8f68d65d Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 11 Apr 2023 08:53:18 -0500
Subject: [PATCH 17/33] #16: swap args for diag building to correct order

---
 .gitignore       | 1 +
 fishi/codegen.go | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 6485fa6..726a7c9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 /.testout
 /ictcc
 /.sim
+/.gen
 
 # mac
 .DS_Store
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 4f7cf00..761d4d5 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -164,7 +164,7 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 
 func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, pkgName string, binPath string, opts CodegenOptions) error {
 	binName := filepath.Base(binPath)
-	gci, err := GenerateBinaryMainGo(spec, md, p, hooksPkgDir, hooksExpr, pkgName, binName, ".gen", opts)
+	gci, err := GenerateBinaryMainGo(spec, md, p, hooksPkgDir, hooksExpr, pkgName, ".gen", binName, opts)
 	if err != nil {
 		return err
 	}
@@ -178,6 +178,10 @@ func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, ho
 		return err
 	}
 
+	fmt.Printf("Generated binary: %s\n", gci.Path)
+	fmt.Printf("binName: %s\n", binName)
+	fmt.Printf("binPath: %s\n", binPath)
+
 	// Move it to the target location.
 	if err := os.Rename(filepath.Join(gci.Path, binName), binPath); err != nil {
 		return err

From d53072a41821bec2232721f65c69285f6fe10202 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 11 Apr 2023 09:06:04 -0500
Subject: [PATCH 18/33] #16: bin seems okay

---
 .gitignore        | 2 ++
 cmd/ictcc/main.go | 6 +++++-
 fishi/codegen.go  | 4 ----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 726a7c9..a5c1ad3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,8 @@
 /ictcc
 /.sim
 /.gen
+/diag
+/fishic
 
 # mac
 .DS_Store
diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 3cfb591..21f288b 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -611,7 +611,7 @@ func main() {
 		if *diagnosticsBin != "" {
 			// already checked required flags
 			if !quietMode {
-				fmt.Printf("Generating diagnostics binary in %s...\n", *diagnosticsBin)
+				fmt.Printf("Generating diagnostics binary code in .gen...\n")
 			}
 
 			err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *pkg, *diagnosticsBin, cgOpts)
@@ -620,6 +620,10 @@ func main() {
 				returnCode = ExitErrGeneration
 				return
 			}
+
+			if !quietMode {
+				fmt.Printf("Built diagnosticis binary '%s'\n", *diagnosticsBin)
+			}
 		}
 
 		// assuming it worked, now generate the final output
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 761d4d5..d3acff4 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -178,10 +178,6 @@ func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, ho
 		return err
 	}
 
-	fmt.Printf("Generated binary: %s\n", gci.Path)
-	fmt.Printf("binName: %s\n", binName)
-	fmt.Printf("binPath: %s\n", binPath)
-
 	// Move it to the target location.
 	if err := os.Rename(filepath.Join(gci.Path, binName), binPath); err != nil {
 		return err

From bbf6586ab9fdef99b6b95c24ed9ad102b2ce553b Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Tue, 11 Apr 2023 10:26:37 -0500
Subject: [PATCH 19/33] #16: get the new bins

---
 cmd/ictcc/main.go | 173 +++++++++++++++++++++++++---------------------
 fishi/codegen.go  |   9 ++-
 fishi/fishi.go    |  17 +++--
 3 files changed, 113 insertions(+), 86 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 21f288b..5c7627a 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -117,6 +117,11 @@ Flags:
 		Set the version of the language to generate a frontend for. Defaults to
 		"v0.0.0".
 
+	--prefix PATH
+		Set the prefix to use for all generated files. Defaults to the current
+		working directory name. If used, this path will be prepended to all
+		generated files. This prefix does *not* apply to input files.
+
 	--debug-templates
 		Enable dumping of the fishi filled template files before they are passed
 		to the formatter. This allows debugging of the template files when
@@ -242,6 +247,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/pflag"
@@ -297,6 +303,7 @@ var (
 	parserCff string
 	lang      string
 
+	pathPrefix                = pflag.String("prefix", "", "Path to prepend to path of all generated files")
 	diagnosticsBin    *string = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
 	preserveBinSource *bool   = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
 	debugTemplates    *bool   = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
@@ -540,106 +547,114 @@ func main() {
 		printSpec(spec)
 	}
 
-	if !noGen {
-		// okay, first try to create a parser
-		var p ictiobus.Parser
-		var parserWarns []fishi.Warning
-		// if one is selected, use that one
-		if parserType != nil {
-			if !quietMode {
-				fmt.Printf("Creating %s parser from spec...\n", *parserType)
-			}
-			p, parserWarns, err = spec.CreateParser(*parserType, allowAmbig)
-		} else {
-			if !quietMode {
-				fmt.Printf("Creating most restrictive parser from spec...\n")
-			}
-			p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig)
+	if noGen {
+		if !quietMode {
+			fmt.Printf("(code generation skipped due to flags)\n")
 		}
+		return
+	}
 
-		for _, warn := range parserWarns {
-			const warnPrefix = "WARN: "
-			// indent all except the first line
-			warnStr := rosed.Edit(warnPrefix+warn.Message).
-				LinesFrom(1).
-				IndentOpts(len(warnPrefix), rosed.Options{IndentStr: " "}).
-				String()
+	// code gen time!
 
-			fmt.Fprintf(os.Stderr, "%s\n", warnStr)
-		}
-		fmt.Fprintf(os.Stderr, "\n")
-
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
-			returnCode = ExitErrParser
-			return
+	// okay, first try to create a parser
+	var p ictiobus.Parser
+	var parserWarns []fishi.Warning
+	// if one is selected, use that one
+	if parserType != nil {
+		if !quietMode {
+			fmt.Printf("Creating %s parser from spec...\n", *parserType)
 		}
-
+		p, parserWarns, err = spec.CreateParser(*parserType, allowAmbig)
+	} else {
 		if !quietMode {
-			fmt.Printf("Successfully generated %s parser from grammar\n", p.Type().String())
+			fmt.Printf("Creating most restrictive parser from spec...\n")
 		}
+		p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig)
+	}
 
-		// create a test compiler and output it
-		if !*valSDTSOff {
-			if *irType == "" {
-				fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --ir parameter\n")
-			} else {
-				if *hooksPath == "" {
-					fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --hooks parameter\n")
-				} else {
-					if !quietMode {
-						fmt.Printf("Generating parser simulation binary in .sim...\n")
-					}
-					di := trans.ValidationOptions{
-						ParseTrees:    *valSDTSShowTrees,
-						FullDepGraphs: *valSDTSShowGraphs,
-						ShowAllErrors: !*valSDTSFirstOnly,
-						SkipErrors:    *valSDTSSkip,
-					}
-
-					err := fishi.ValidateSimulatedInput(spec, md, p, *hooksPath, *hooksTableName, cgOpts, &di)
-					if err != nil {
-						fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
-						returnCode = ExitErrGeneration
-						return
-					}
-				}
-			}
-		}
+	for _, warn := range parserWarns {
+		const warnPrefix = "WARN: "
+		// indent all except the first line
+		warnStr := rosed.Edit(warnPrefix+warn.Message).
+			LinesFrom(1).
+			IndentOpts(len(warnPrefix), rosed.Options{IndentStr: " "}).
+			String()
 
-		// generate diagnostics output if requested
-		if *diagnosticsBin != "" {
-			// already checked required flags
-			if !quietMode {
-				fmt.Printf("Generating diagnostics binary code in .gen...\n")
-			}
+		fmt.Fprintf(os.Stderr, "%s\n", warnStr)
+	}
+	fmt.Fprintf(os.Stderr, "\n")
 
-			err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *pkg, *diagnosticsBin, cgOpts)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
-				returnCode = ExitErrGeneration
-				return
-			}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+		returnCode = ExitErrParser
+		return
+	}
+
+	if !quietMode {
+		fmt.Printf("Successfully generated %s parser from grammar\n", p.Type().String())
+	}
 
-			if !quietMode {
-				fmt.Printf("Built diagnosticis binary '%s'\n", *diagnosticsBin)
+	// create a test compiler and output it
+	if !*valSDTSOff {
+		if *irType == "" {
+			fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --ir parameter\n")
+		} else {
+			if *hooksPath == "" {
+				fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --hooks parameter\n")
+			} else {
+				if !quietMode {
+					fmt.Printf("Generating parser simulation binary in .sim...\n")
+				}
+				di := trans.ValidationOptions{
+					ParseTrees:    *valSDTSShowTrees,
+					FullDepGraphs: *valSDTSShowGraphs,
+					ShowAllErrors: !*valSDTSFirstOnly,
+					SkipErrors:    *valSDTSSkip,
+				}
+
+				err := fishi.ValidateSimulatedInput(spec, md, p, *hooksPath, *hooksTableName, *pathPrefix, cgOpts, &di)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
+					returnCode = ExitErrGeneration
+					return
+				}
 			}
 		}
+	}
 
-		// assuming it worked, now generate the final output
+	// generate diagnostics output if requested
+	if *diagnosticsBin != "" {
+		// already checked required flags
 		if !quietMode {
-			fmt.Printf("Generating compiler frontend in %s...\n", *dest)
+			fmt.Printf("Generating diagnostics binary code in .gen...\n")
 		}
-		err := fishi.GenerateCompilerGo(spec, md, *pkg, *dest, &cgOpts)
+
+		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *pkg, *diagnosticsBin, *pathPrefix, cgOpts)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+			fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
 			returnCode = ExitErrGeneration
 			return
 		}
-	} else if !quietMode {
-		fmt.Printf("(code generation skipped due to flags)\n")
+
+		if !quietMode {
+			fmt.Printf("Built diagnosticis binary '%s'\n", *diagnosticsBin)
+		}
 	}
 
+	// assuming it worked, now generate the final output
+	if !quietMode {
+		fmt.Printf("Generating compiler frontend in %s...\n", *dest)
+	}
+	feDest := *dest
+	if *pathPrefix != "" {
+		feDest = filepath.Join(*pathPrefix, feDest)
+	}
+	err = fishi.GenerateCompilerGo(spec, md, *pkg, feDest, &cgOpts)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+		returnCode = ExitErrGeneration
+		return
+	}
 }
 
 // return from flags the parser type selected and whether ambiguity is allowed.
diff --git a/fishi/codegen.go b/fishi/codegen.go
index d3acff4..9df428e 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -162,9 +162,14 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 	return cmd.Run()
 }
 
-func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, pkgName string, binPath string, opts CodegenOptions) error {
+func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, pkgName string, binPath string, pathPrefix string, opts CodegenOptions) error {
 	binName := filepath.Base(binPath)
-	gci, err := GenerateBinaryMainGo(spec, md, p, hooksPkgDir, hooksExpr, pkgName, ".gen", binName, opts)
+
+	outDir := ".dir"
+	if pathPrefix != "" {
+		outDir = filepath.Join(pathPrefix, outDir)
+	}
+	gci, err := GenerateBinaryMainGo(spec, md, p, hooksPkgDir, hooksExpr, pkgName, outDir, binName, opts)
 	if err != nil {
 		return err
 	}
diff --git a/fishi/fishi.go b/fishi/fishi.go
index 43ee072..434f485 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/dekarrin/ictiobus"
@@ -33,9 +34,10 @@ type Options struct {
 }
 
 // ValidateSimulatedInput generates a lightweight compiler with the spec'd
-// frontend in a special directory (".sim" in the local directory) and then runs
-// SDTS validation on a variety of parse tree inputs designed to cover all the
-// productions of the grammar at least once.
+// frontend in a special directory (".sim" in the local directory or in the path
+// specified by pathPrefix, if set) and then runs SDTS validation on a variety
+// of parse tree inputs designed to cover all the productions of the grammar at
+// least once.
 //
 // If running validation with the test compiler succeeds, it and the directory
 // it was generated in are deleted. If it fails, the directory is left in place
@@ -49,14 +51,19 @@ type Options struct {
 // No binary is generated as part of this, but source is which is then executed.
 // If PreserveBinarySource is set in cgOpts, the source will be left in the
 // .sim directory.
-func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks, hooksTable string, cgOpts CodegenOptions, valOpts *trans.ValidationOptions) error {
+func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks, hooksTable string, pathPrefix string, cgOpts CodegenOptions, valOpts *trans.ValidationOptions) error {
 	pkgName := "sim" + strings.ToLower(md.Language)
 
 	binName := safeTCIdentifierName(md.Language)
 	binName = binName[2:] // remove initial "tc".
 	binName = strings.ToLower(binName)
 	binName = "test" + binName
-	genInfo, err := GenerateBinaryMainGo(spec, md, p, hooks, hooksTable, pkgName, ".sim", binName, cgOpts)
+
+	outDir := ".sim"
+	if pathPrefix != "" {
+		outDir = filepath.Join(pathPrefix, outDir)
+	}
+	genInfo, err := GenerateBinaryMainGo(spec, md, p, hooks, hooksTable, pkgName, outDir, binName, cgOpts)
 	if err != nil {
 		return fmt.Errorf("generate test compiler: %w", err)
 	}

From 74020a084c4843053d123e86d3475b6588ca50e1 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Thu, 13 Apr 2023 06:23:59 -0500
Subject: [PATCH 20/33] #16: diag binary is being built but it cannot read own
 source

---
 cmd/ictcc/main.go            | 30 +++++++++++++++++++++++-------
 fishi/codegen.go             |  2 +-
 fishi/templates/main.go.tmpl |  2 +-
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 5c7627a..51b9a70 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -118,9 +118,13 @@ Flags:
 		"v0.0.0".
 
 	--prefix PATH
-		Set the prefix to use for all generated files. Defaults to the current
-		working directory name. If used, this path will be prepended to all
-		generated files. This prefix does *not* apply to input files.
+		Set the prefix to use for all generated source files. Defaults to the
+		current working directory. If used, generated source files will be be
+		output to their location with this prefix instead of in a directory
+		(".sim", ".gen", and the generated frontend source package folder)
+		located in the current working directory. Combine with
+		--preserve-bin-source to aid in debugging. Does not affect diagnostic
+		binary output location.
 
 	--debug-templates
 		Enable dumping of the fishi filled template files before they are passed
@@ -303,7 +307,7 @@ var (
 	parserCff string
 	lang      string
 
-	pathPrefix                = pflag.String("prefix", "", "Path to prepend to path of all generated files")
+	pathPrefix                = pflag.String("prefix", "", "Path to prepend to path of all generated source files")
 	diagnosticsBin    *string = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
 	preserveBinSource *bool   = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
 	debugTemplates    *bool   = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
@@ -603,7 +607,11 @@ func main() {
 				fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --hooks parameter\n")
 			} else {
 				if !quietMode {
-					fmt.Printf("Generating parser simulation binary in .sim...\n")
+					simGenDir := ".sim"
+					if *pathPrefix != "" {
+						simGenDir = filepath.Join(*pathPrefix, simGenDir)
+					}
+					fmt.Printf("Generating parser simulation binary in %s...\n", simGenDir)
 				}
 				di := trans.ValidationOptions{
 					ParseTrees:    *valSDTSShowTrees,
@@ -626,7 +634,11 @@ func main() {
 	if *diagnosticsBin != "" {
 		// already checked required flags
 		if !quietMode {
-			fmt.Printf("Generating diagnostics binary code in .gen...\n")
+			diagGenDir := ".gen"
+			if *pathPrefix != "" {
+				diagGenDir = filepath.Join(*pathPrefix, diagGenDir)
+			}
+			fmt.Printf("Generating diagnostics binary code in %s...\n", diagGenDir)
 		}
 
 		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *pkg, *diagnosticsBin, *pathPrefix, cgOpts)
@@ -643,7 +655,11 @@ func main() {
 
 	// assuming it worked, now generate the final output
 	if !quietMode {
-		fmt.Printf("Generating compiler frontend in %s...\n", *dest)
+		feGenDir := *dest
+		if *pathPrefix != "" {
+			feGenDir = filepath.Join(*pathPrefix, feGenDir)
+		}
+		fmt.Printf("Generating compiler frontend in %s...\n", feGenDir)
 	}
 	feDest := *dest
 	if *pathPrefix != "" {
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 9df428e..c70af5c 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -165,7 +165,7 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, pkgName string, binPath string, pathPrefix string, opts CodegenOptions) error {
 	binName := filepath.Base(binPath)
 
-	outDir := ".dir"
+	outDir := ".gen"
 	if pathPrefix != "" {
 		outDir = filepath.Join(pathPrefix, outDir)
 	}
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index c39aa45..b3880bc 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -146,7 +146,7 @@ func main() {
 		}
 
 		if !*quietMode {
-			fmt.Println("=== Analysis of %s ===\n%s\n\n", filepath.Base(f), ir)
+			fmt.Printf("=== Analysis of %s ===\n%s\n\n", filepath.Base(f), ir)
 		}
 	}
 }
\ No newline at end of file

From 667e70d0e290c49a67ae56bb5e0f2599753b9460 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Fri, 14 Apr 2023 06:45:57 -0500
Subject: [PATCH 21/33] #16: begin to move preproc to own subpackage

---
 fishi/fishi.go               |  58 ------------
 fishi/fishi_test.go          |  59 -------------
 fishi/format/format.go       |  80 +++++++++++++++++
 fishi/format/scanner.go      |  39 ++++++++
 fishi/format/scanner_test.go | 166 +++++++++++++++++++++++++++++++++++
 5 files changed, 285 insertions(+), 117 deletions(-)
 create mode 100644 fishi/format/format.go
 create mode 100644 fishi/format/scanner.go
 create mode 100644 fishi/format/scanner_test.go

diff --git a/fishi/fishi.go b/fishi/fishi.go
index 434f485..1330533 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -1,11 +1,8 @@
 package fishi
 
 import (
-	"bufio"
-	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -15,9 +12,6 @@ import (
 	"github.com/dekarrin/ictiobus/fishi/syntax"
 	"github.com/dekarrin/ictiobus/trans"
 	"github.com/dekarrin/ictiobus/types"
-	"github.com/gomarkdown/markdown"
-	mkast "github.com/gomarkdown/markdown/ast"
-	mkparser "github.com/gomarkdown/markdown/parser"
 )
 
 type Results struct {
@@ -83,58 +77,6 @@ func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks
 
 	return nil
 }
-
-func GetFishiFromMarkdown(mdText []byte) []byte {
-	doc := markdown.Parse(mdText, mkparser.New())
-	var scanner fishiScanner
-	fishi := markdown.Render(doc, scanner)
-	return fishi
-}
-
-// Preprocess does a preprocess step on the source, which as of now includes
-// stripping comments and normalizing end of lines to \n.
-func Preprocess(source []byte) []byte {
-	toBuf := make([]byte, len(source))
-	copy(toBuf, source)
-	scanner := bufio.NewScanner(bytes.NewBuffer(toBuf))
-	var preprocessed strings.Builder
-
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.HasSuffix(line, "\r\n") || strings.HasPrefix(line, "\n\r") {
-			line = line[0 : len(line)-2]
-		} else {
-			line = strings.TrimSuffix(line, "\n")
-		}
-		line, _, _ = strings.Cut(line, "#")
-		preprocessed.WriteString(line)
-		preprocessed.WriteRune('\n')
-	}
-
-	return []byte(preprocessed.String())
-}
-
-type fishiScanner bool
-
-func (fs fishiScanner) RenderNode(w io.Writer, node mkast.Node, entering bool) mkast.WalkStatus {
-	if !entering {
-		return mkast.GoToNext
-	}
-
-	codeBlock, ok := node.(*mkast.CodeBlock)
-	if !ok || codeBlock == nil {
-		return mkast.GoToNext
-	}
-
-	if strings.ToLower(strings.TrimSpace(string(codeBlock.Info))) == "fishi" {
-		w.Write(codeBlock.Literal)
-	}
-	return mkast.GoToNext
-}
-
-func (fs fishiScanner) RenderHeader(w io.Writer, ast mkast.Node) {}
-func (fs fishiScanner) RenderFooter(w io.Writer, ast mkast.Node) {}
-
 func ParseMarkdownFile(filename string, opts Options) (Results, error) {
 	data, err := os.ReadFile(filename)
 	if err != nil {
diff --git a/fishi/fishi_test.go b/fishi/fishi_test.go
index e88e506..4e6114f 100644
--- a/fishi/fishi_test.go
+++ b/fishi/fishi_test.go
@@ -57,65 +57,6 @@ func Test_SelfHostedMarkdown(t *testing.T) {
 	}
 }
 
-func Test_GetFishiFromMarkdown(t *testing.T) {
-	testCases := []struct {
-		name   string
-		input  string
-		expect string
-	}{
-		{
-			name: "fishi and text",
-			input: "Test block\n" +
-				"only include the fishi block\n" +
-				"```fishi\n" +
-				"%%tokens\n" +
-				"\n" +
-				"%token test\n" +
-				"```\n",
-			expect: "%%tokens\n" +
-				"\n" +
-				"%token test\n",
-		},
-		{
-			name: "two fishi blocks",
-			input: "Test block\n" +
-				"only include the fishi blocks\n" +
-				"```fishi\n" +
-				"%%tokens\n" +
-				"\n" +
-				"%token test\n" +
-				"```\n" +
-				"some more text\n" +
-				"```fishi\n" +
-				"\n" +
-				"%token 7\n" +
-				"%%actions\n" +
-				"\n" +
-				"%set go\n" +
-				"```\n" +
-				"other text\n",
-			expect: "%%tokens\n" +
-				"\n" +
-				"%token test\n" +
-				"\n" +
-				"%token 7\n" +
-				"%%actions\n" +
-				"\n" +
-				"%set go\n",
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			assert := assert.New(t)
-
-			actual := GetFishiFromMarkdown([]byte(tc.input))
-
-			assert.Equal(tc.expect, string(actual))
-		})
-	}
-}
-
 const (
 	testInput = `%%actions
 	
diff --git a/fishi/format/format.go b/fishi/format/format.go
new file mode 100644
index 0000000..0b76782
--- /dev/null
+++ b/fishi/format/format.go
@@ -0,0 +1,80 @@
+// Package format contains functions for producing a CodeReader from a stream
+// that contains markdown files with fishi codeblocks. A CodeReader can be sent
+// directly to the frontend and handles all gathering of codeblocks and running
+// any preprocessing needed on it.
+package format
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"strings"
+)
+
+// CodeReader is a reader that can be used to read fishi code from a stream
+// containing markdown-formatted text with fishi codeblocks. It will gather all
+// fishi codeblocks immediately on open and then read bytes from them as Read is
+// called. Preprocessing may also be done at that time. The CodeReader will
+// return io.EOF when all bytes from fishi codeblocks in the stream have been
+// read.
+type CodeReader struct {
+	r *bytes.Reader
+}
+
+// Read reads bytes from the CodeReader. It will return io.EOF when all bytes
+// from fishi codeblocks in the stream have been read. It cannot return an
+// error as the actual underlying stream it was opened on is fully consumed at
+// the time of opening.
+func (cr *CodeReader) Read(p []byte) (n int, err error) {
+	return cr.r.Read(p)
+}
+
+// NewCodeReader creates a new CodeReader from a stream containing markdown
+// formatted text with fishi codeblocks. It will immediately read the provided
+// stream until it returns EOF and find all fishi codeblocks and run
+// preprocessing on them.
+//
+// Returns non-nil error if there is a problem reading the markdown or
+// preprocessing the code.
+func NewCodeReader(r io.Reader) (*CodeReader, error) {
+	// read the whole stream into a buffer
+	allInput := make([]byte, 0)
+
+	bufReader := make([]byte, 256)
+	var err error
+	for err != io.EOF {
+		var n int
+		n, err = r.Read(bufReader)
+
+		if n > 0 {
+			allInput = append(allInput, bufReader[:n]...)
+		}
+
+		if err != nil && err != io.EOF {
+
+		}
+	}
+}
+
+// Preprocess does a preprocess step on the source, which as of now includes
+// stripping comments and normalizing end of lines to \n.
+func Preprocess(source []byte) []byte {
+	toBuf := make([]byte, len(source))
+	copy(toBuf, source)
+	scanner := bufio.NewScanner(bytes.NewBuffer(toBuf))
+	var preprocessed strings.Builder
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasSuffix(line, "\r\n") || strings.HasPrefix(line, "\n\r") {
+			line = line[0 : len(line)-2]
+		} else {
+			line = strings.TrimSuffix(line, "\n")
+		}
+		line, _, _ = strings.Cut(line, "#")
+		preprocessed.WriteString(line)
+		preprocessed.WriteRune('\n')
+	}
+
+	return []byte(preprocessed.String())
+}
diff --git a/fishi/format/scanner.go b/fishi/format/scanner.go
new file mode 100644
index 0000000..089f8c6
--- /dev/null
+++ b/fishi/format/scanner.go
@@ -0,0 +1,39 @@
+package format
+
+import (
+	"io"
+	"strings"
+
+	"github.com/gomarkdown/markdown"
+	mkast "github.com/gomarkdown/markdown/ast"
+	mkparser "github.com/gomarkdown/markdown/parser"
+)
+
+type fishiScanner bool
+
+func (fs fishiScanner) RenderNode(w io.Writer, node mkast.Node, entering bool) mkast.WalkStatus {
+	if !entering {
+		return mkast.GoToNext
+	}
+
+	codeBlock, ok := node.(*mkast.CodeBlock)
+	if !ok || codeBlock == nil {
+		return mkast.GoToNext
+	}
+
+	if strings.ToLower(strings.TrimSpace(string(codeBlock.Info))) == "fishi" {
+		w.Write(codeBlock.Literal)
+	}
+	return mkast.GoToNext
+}
+
+func (fs fishiScanner) RenderHeader(w io.Writer, ast mkast.Node) {}
+func (fs fishiScanner) RenderFooter(w io.Writer, ast mkast.Node) {}
+
+// TODO: rename this to somefin that references the fact that it gathers fishi.
+func GetFishiFromMarkdown(mdText []byte) []byte {
+	doc := markdown.Parse(mdText, mkparser.New())
+	var scanner fishiScanner
+	fishi := markdown.Render(doc, scanner)
+	return fishi
+}
diff --git a/fishi/format/scanner_test.go b/fishi/format/scanner_test.go
new file mode 100644
index 0000000..d3e928e
--- /dev/null
+++ b/fishi/format/scanner_test.go
@@ -0,0 +1,166 @@
+package format
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_GetFishiFromMarkdown(t *testing.T) {
+	testCases := []struct {
+		name   string
+		input  string
+		expect string
+	}{
+		{
+			name: "fishi and text",
+			input: "Test block\n" +
+				"only include the fishi block\n" +
+				"```fishi\n" +
+				"%%tokens\n" +
+				"\n" +
+				"%token test\n" +
+				"```\n",
+			expect: "%%tokens\n" +
+				"\n" +
+				"%token test\n",
+		},
+		{
+			name: "two fishi blocks",
+			input: "Test block\n" +
+				"only include the fishi blocks\n" +
+				"```fishi\n" +
+				"%%tokens\n" +
+				"\n" +
+				"%token test\n" +
+				"```\n" +
+				"some more text\n" +
+				"```fishi\n" +
+				"\n" +
+				"%token 7\n" +
+				"%%actions\n" +
+				"\n" +
+				"%set go\n" +
+				"```\n" +
+				"other text\n",
+			expect: "%%tokens\n" +
+				"\n" +
+				"%token test\n" +
+				"\n" +
+				"%token 7\n" +
+				"%%actions\n" +
+				"\n" +
+				"%set go\n",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			actual := GetFishiFromMarkdown([]byte(tc.input))
+
+			assert.Equal(tc.expect, string(actual))
+		})
+	}
+}
+
+const (
+	testInput = `%%actions
+	
+						%symbol
+	
+	
+						{hey}
+						%prod  %index 8
+	
+					%set {thing}.thing %hook thing
+						%prod {}
+	
+					%set {thing}.thing %hook thing
+						%prod {test} this {THING}
+	
+						%set {thing}.thing %hook thing
+					%prod {ye} + {A}
+	
+					%set {thing}.thing %hook thing
+	
+							%symbol {yo}%prod + {EAT} ext
+	
+					%set {thing}.thing %hook thing
+					%%tokens
+					[somefin]
+	
+					%stateshift   someState
+	
+			%%tokens
+	
+			%!%[more]%!%bluggleb*shi{2,4}   %stateshift glub
+			%token lovely %human Something for this
+	
+				%%tokens
+	
+					glub  %discard
+	
+	
+					[some]{FREEFORM}idk[^bullshit]text\*
+					%discard
+	
+					%!%[more]%!%bluggleb*shi{2,4}   %stateshift glub
+				%token lovely %human Something nice
+					%priority 1
+	
+				%state this
+	
+				[yo] %discard
+	
+				%%grammar
+				%state glub
+				{RULE} =   {SOMEBULLSHIT}
+	
+							%%grammar
+							{RULE}=                           {WOAH} | n
+							{R2}				= =+  {DAMN} cool | okaythen + 2 | {}
+											 | {SOMEFIN ELSE}
+	
+							%state someState
+	
+							{ANOTHER}=		{HMM}
+	
+	
+	
+	
+				%%actions
+	
+				%symbol {text-element}
+				%prod FREEFORM_TEXT
+				%set {text-element}.str
+				%hook identity  %with {0}.$text
+	
+				%prod ESCSEQ
+				%set {text-element}.str
+				%hook unescape  %with {.}.$test
+	
+	
+				%symbol {OTHER}
+				%prod EHHH
+				%set {OTHER}.str
+				%hook identity  %with {9}.$text
+	
+				%prod ESCSEQ
+				%set {text-element$12}.str
+				%hook unescape  %with {^}.$test
+	
+				%state someGoodState
+	
+				%symbol {text-element}
+				%prod FREEFORM_TEXT
+				%set {text-element}.str
+				%hook identity  %with {ANON$12}.$text
+	
+				%prod ESCSEQ
+				%set {text-element}.str
+				%hook unescape  %with {ESCSEQ}.$test
+	
+				`
+)

From 8b4375427c408dbf172261222c342b6e089c1d7a Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Fri, 14 Apr 2023 08:02:47 -0500
Subject: [PATCH 22/33] #16: moved preformat processing into stream-based API

---
 fishi/fishi.go         | 44 ++++++++++++++++++------------------------
 fishi/fishi_test.go    |  9 ++++++++-
 fishi/format/format.go | 11 ++++++++++-
 3 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/fishi/fishi.go b/fishi/fishi.go
index 1330533..2f673bb 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -1,14 +1,17 @@
 package fishi
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/dekarrin/ictiobus"
 	"github.com/dekarrin/ictiobus/fishi/fe"
+	"github.com/dekarrin/ictiobus/fishi/format"
 	"github.com/dekarrin/ictiobus/fishi/syntax"
 	"github.com/dekarrin/ictiobus/trans"
 	"github.com/dekarrin/ictiobus/types"
@@ -78,12 +81,18 @@ func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks
 	return nil
 }
 func ParseMarkdownFile(filename string, opts Options) (Results, error) {
-	data, err := os.ReadFile(filename)
+	f, err := os.Open(filename)
 	if err != nil {
 		return Results{}, err
 	}
 
-	res, err := ParseMarkdown(data, opts)
+	bufF := bufio.NewReader(f)
+	r, err := format.NewCodeReader(bufF)
+	if err != nil {
+		return Results{}, err
+	}
+
+	res, err := Parse(r, opts)
 	if err != nil {
 		return res, err
 	}
@@ -91,41 +100,26 @@ func ParseMarkdownFile(filename string, opts Options) (Results, error) {
 	return res, nil
 }
 
-func ParseMarkdown(mdText []byte, opts Options) (Results, error) {
-
-	// TODO: read in filename, based on it check for cached version
-
-	// debug steps: output source after preprocess
-	// output token stream
-	// output grammar constructed
-	// output parser table and type
-
-	source := GetFishiFromMarkdown(mdText)
-	return Parse(source, opts)
-}
-
-// Parse converts the fishi source code provided into an AST.
-func Parse(source []byte, opts Options) (Results, error) {
+// Parse converts the fishi source code read from the given reader into an AST.
+func Parse(r io.Reader, opts Options) (Results, error) {
 	// get the frontend
 	fishiFront, err := GetFrontend(opts)
 	if err != nil {
 		return Results{}, fmt.Errorf("could not get frontend: %w", err)
 	}
 
-	preprocessedSource := Preprocess(source)
-
-	r := Results{}
+	res := Results{}
 	// now, try to make a parse tree
-	nodes, pt, err := fishiFront.AnalyzeString(string(preprocessedSource))
-	r.Tree = pt // need to do this before we return
+	nodes, pt, err := fishiFront.Analyze(r)
+	res.Tree = pt // need to do this before we return
 	if err != nil {
-		return r, err
+		return res, err
 	}
-	r.AST = &AST{
+	res.AST = &AST{
 		Nodes: nodes,
 	}
 
-	return r, nil
+	return res, nil
 }
 
 // GetFrontend gets the frontend for the fishi compiler-compiler. If cffFile is
diff --git a/fishi/fishi_test.go b/fishi/fishi_test.go
index 4e6114f..8fe184e 100644
--- a/fishi/fishi_test.go
+++ b/fishi/fishi_test.go
@@ -1,9 +1,11 @@
 package fishi
 
 import (
+	"bytes"
 	"fmt"
 	"testing"
 
+	"github.com/dekarrin/ictiobus/fishi/format"
 	"github.com/dekarrin/ictiobus/types"
 
 	"github.com/stretchr/testify/assert"
@@ -12,7 +14,12 @@ import (
 func Test_WithFakeInput(t *testing.T) {
 	assert := assert.New(t)
 
-	_, actual := Parse([]byte(testInput), Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true})
+	r, err := format.NewCodeReader(bytes.NewReader([]byte(testInput)))
+	if !assert.NoError(err) {
+		return
+	}
+
+	_, actual := Parse(r, Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true})
 
 	assert.NoError(actual)
 
diff --git a/fishi/format/format.go b/fishi/format/format.go
index 0b76782..a266b5d 100644
--- a/fishi/format/format.go
+++ b/fishi/format/format.go
@@ -51,9 +51,18 @@ func NewCodeReader(r io.Reader) (*CodeReader, error) {
 		}
 
 		if err != nil && err != io.EOF {
-
+			return nil, err
 		}
 	}
+
+	gatheredFishi := GetFishiFromMarkdown(allInput)
+	fishiSource := Preprocess(gatheredFishi)
+
+	cr := &CodeReader{
+		r: bytes.NewReader(fishiSource),
+	}
+
+	return cr, nil
 }
 
 // Preprocess does a preprocess step on the source, which as of now includes

From c54fe9bf37e0434e5ec0bdc1b4d32b60d420bb97 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Fri, 14 Apr 2023 08:04:06 -0500
Subject: [PATCH 23/33] #16: fix test case for fishi

---
 fishi/fishi_test.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/fishi/fishi_test.go b/fishi/fishi_test.go
index 8fe184e..5dfdbf5 100644
--- a/fishi/fishi_test.go
+++ b/fishi/fishi_test.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/dekarrin/ictiobus/fishi/format"
 	"github.com/dekarrin/ictiobus/types"
 
 	"github.com/stretchr/testify/assert"
@@ -14,10 +13,7 @@ import (
 func Test_WithFakeInput(t *testing.T) {
 	assert := assert.New(t)
 
-	r, err := format.NewCodeReader(bytes.NewReader([]byte(testInput)))
-	if !assert.NoError(err) {
-		return
-	}
+	r := bytes.NewReader([]byte(testInput))
 
 	_, actual := Parse(r, Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true})
 

From db7eff072ce3868088ecf2436a6169d8f639aa26 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Fri, 14 Apr 2023 08:15:53 -0500
Subject: [PATCH 24/33] #16: updating template to allow use of format package

---
 fishi/codegen.go             | 113 +++++++++++++++++++----------------
 fishi/templates/main.go.tmpl |  17 +++++-
 2 files changed, 79 insertions(+), 51 deletions(-)

diff --git a/fishi/codegen.go b/fishi/codegen.go
index c70af5c..58dee30 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -213,7 +213,9 @@ func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, ho
 // or var name.
 //
 // opts must be non-nil and IRType must be set.
-func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, fePkgName string, genPath string, binName string, opts CodegenOptions) (GeneratedCodeInfo, error) {
+//
+// TODO: turn this ugly signature into a struct.
+func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, formatPkgDir string, formatCall string, fePkgName string, genPath string, binName string, opts CodegenOptions) (GeneratedCodeInfo, error) {
 	if opts.IRType == "" {
 		return GeneratedCodeInfo{}, fmt.Errorf("IRType must be set in options")
 	}
@@ -225,56 +227,9 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 
 	gci := GeneratedCodeInfo{}
 
-	// what is the name of our hooks package? find out by reading the first go
-	// file in the package.
-	hooksDirItems, err := os.ReadDir(hooksPkgDir)
+	hooksPkgName, err := readPackageName(hooksPkgDir)
 	if err != nil {
-		return gci, fmt.Errorf("reading hooks package dir: %w", err)
-	}
-
-	var hooksPkgName string
-	for _, item := range hooksDirItems {
-		if !item.IsDir() && strings.ToLower(filepath.Ext(item.Name())) == ".go" {
-			// read the file to find the package name
-			goFilePath := filepath.Join(hooksPkgDir, item.Name())
-			goFile, err := os.Open(goFilePath)
-			if err != nil {
-				return gci, fmt.Errorf("reading go file in hooks package: %w", err)
-			}
-
-			// buffered reading
-			r := bufio.NewReader(goFile)
-
-			// now find the package name in the file
-			for hooksPkgName == "" {
-				str, err := r.ReadString('\n')
-				strTrimmed := strings.TrimSpace(str)
-
-				// is it a line starting with "package"?
-				if strings.HasPrefix(strTrimmed, "package") {
-					lineItems := strings.Split(strTrimmed, " ")
-					if len(lineItems) == 2 {
-						hooksPkgName = lineItems[1]
-						break
-					}
-				}
-
-				// ofc if err is somefin else
-				if err != nil {
-					if err == io.EOF {
-						break
-					}
-					return gci, fmt.Errorf("reading go file in hooks package: %w", err)
-				}
-			}
-		}
-
-		if hooksPkgName != "" {
-			break
-		}
-	}
-	if hooksPkgName == "" {
-		return gci, fmt.Errorf("could not find package name for hooks package; make sure files are gofmt'd")
+		return gci, fmt.Errorf("reading hooks package name: %w", err)
 	}
 	if hooksPkgName == fePkgName {
 		// double it to avoid name collision
@@ -708,6 +663,8 @@ type cgMainData struct {
 	IRTypePackage     string
 	IRType            string
 	IncludeSimulation bool
+	FormatPackage     string
+	FormatCall        string
 }
 
 // codegenData for template fill.
@@ -896,3 +853,59 @@ func copyDirToTargetAsync(srcDir string, targetDir string) (copyResult chan erro
 
 	return ch, nil
 }
+
+func readPackageName(dir string) (string, error) {
+	// what is the name of our hooks package? find out by reading the first go
+	// file in the package.
+	dirItems, err := os.ReadDir(dir)
+	if err != nil {
+		return "", err
+	}
+
+	var pkgName string
+	for _, item := range dirItems {
+		if !item.IsDir() && strings.ToLower(filepath.Ext(item.Name())) == ".go" {
+			// read the file to find the package name
+			goFilePath := filepath.Join(dir, item.Name())
+			goFile, err := os.Open(goFilePath)
+			if err != nil {
+				return "", err
+			}
+
+			// buffered reading
+			r := bufio.NewReader(goFile)
+
+			// now find the package name in the file
+			for pkgName == "" {
+				str, err := r.ReadString('\n')
+				strTrimmed := strings.TrimSpace(str)
+
+				// is it a line starting with "package"?
+				if strings.HasPrefix(strTrimmed, "package") {
+					lineItems := strings.Split(strTrimmed, " ")
+					if len(lineItems) == 2 {
+						pkgName = lineItems[1]
+						break
+					}
+				}
+
+				// ofc if err is somefin else
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+					return "", err
+				}
+			}
+		}
+
+		if pkgName != "" {
+			break
+		}
+	}
+	if pkgName == "" {
+		return "", fmt.Errorf("could not find package name; make sure files are gofmt'd")
+	}
+
+	return pkgName, nil
+}
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index b3880bc..87c40b2 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -25,6 +25,10 @@ import (
 	"{{ .IRTypePackage }}"
 {{- end}}
 
+{{if .FormatPackage -}}
+	"{{ .FormatPackage }}"
+{{- end}}
+
 	"github.com/dekarrin/ictiobus/trans"
 
 	"github.com/spf13/pflag"
@@ -122,6 +126,7 @@ func main() {
 		return
 	}
 	
+	var r io.Reader
 	for _, f := range files {
 		file, err := os.Open(f)
 		if err != nil {
@@ -130,7 +135,17 @@ func main() {
 			return
 		}
 
-		r := bufio.NewReader(file)
+		r = bufio.NewReader(file)
+
+{{if .FormatCall -}}
+		// format the input
+		r, err = {{ .FormatCall }}(r)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
+			returnCode = ExitErr
+			return
+		}
+{{- end}}
 
 		ir, pt, err := langFront.Analyze(r)
 

From 16cc0f01671dcb47e97aa1e56560dd7045411cef Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 09:00:38 -0500
Subject: [PATCH 25/33] #16: should be able to set diag bin to do preformatting

---
 cmd/ictcc/main.go            | 172 ++++++++++++++++++++++++-----------
 fishi/cgstructs.go           |  50 ++++++++++
 fishi/codegen.go             | 128 ++++++++++++++++++++------
 fishi/fishi.go               |  13 ++-
 fishi/format/format.go       |   2 +-
 fishi/templates/main.go.tmpl |   6 +-
 6 files changed, 284 insertions(+), 87 deletions(-)
 create mode 100644 fishi/cgstructs.go

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 51b9a70..faccb72 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -33,7 +33,10 @@ Flags:
 		frontend analysis. This can be useful for testing out the frontend on
 		files quickly and efficiently, as it also includes further options
 		useful for debugging purposes, such as debugging lexed tokens and the
-		parser itself.
+		parser itself. Note that by default the diagnostics binary will only
+		accept text input in the language accepted by the specified frontend; to
+		allow it to perform reading of specialized formats and/or perform
+		preprocessing, use the -f/--diag-format-pkg flag.
 
 	-q/--quiet
 		Do not show progress messages. This does not affect error messages or
@@ -43,6 +46,37 @@ Flags:
 		Set the location of the pre-compiled parser cache to the given CFF
 		format file as opposed to the default of './parser.cff'.
 
+	-f/--diag-format-pkg DIR
+		Enable special format reading in the generated diagnostics binary by
+		wrapping any io.Reader opened on input files in another io.Reader that
+		handles reading the format of the input file. This is performed by
+		calling a function in the package located in the specified file, by
+		default this function is called 'NewCodeReader' but can be changed by
+		specifying the --diag-format-call flag. The function must take an
+		io.Reader and return a new io.Reader that reads source code from the
+		given io.Reader and performs any preprocessing required on it. This
+		allows the diagnostics binary to read files that are not simply text
+		files directly ready to be accepted by the frontend. If not set, the
+		diagnostics binary will not perform any preprocessing on input files and
+		assumes that any input can be directly accepted by the frontend. This
+		flag is only useful if -d/--diag is also set.
+
+	-c/--diag-format-call NAME
+		Set the name of the function to call in the package specified by
+		-f/--diag-format-pkg to get an io.Reader that can read specialized
+		formats. Defaults to 'NewCodeReader'. This function is used by the
+		diagnostics binary to do format reading and preprocessing on input prior
+		to analysis by the frontend. This flag is only useful if -d/--diag is
+		also set.
+
+	-l/--lang NAME
+		Set the name of the language to generate a frontend for. Defaults to
+		"Unspecified".
+
+	--lang-ver VERSION
+		Set the version of the language to generate a frontend for. Defaults to
+		"v0.0.0".
+
 	--preserve-bin-source
 		Do not delete source files for any generated binary after compiling the
 		binary.
@@ -109,14 +143,6 @@ Flags:
 		Set the destination directory to place generated files in. Defaults to a
 		directory named 'fe' in the current working directory.
 
-	-l/--lang NAME
-		Set the name of the language to generate a frontend for. Defaults to
-		"Unspecified".
-
-	--lang-ver VERSION
-		Set the version of the language to generate a frontend for. Defaults to
-		"v0.0.0".
-
 	--prefix PATH
 		Set the prefix to use for all generated source files. Defaults to the
 		current working directory. If used, generated source files will be be
@@ -307,44 +333,47 @@ var (
 	parserCff string
 	lang      string
 
-	pathPrefix                = pflag.String("prefix", "", "Path to prepend to path of all generated source files")
-	diagnosticsBin    *string = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
-	preserveBinSource *bool   = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
-	debugTemplates    *bool   = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
-	pkg               *string = pflag.String("pkg", "fe", "The name of the package to place generated files in")
-	dest              *string = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
-	langVer           *string = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
-	noCache           *bool   = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
-	noCacheOutput     *bool   = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
-
-	valSDTSOff        *bool = pflag.Bool("sim-off", false, "Disable input simulation of the language once built")
-	valSDTSShowTrees  *bool = pflag.Bool("sim-trees", false, "Show parse trees that caused errors during simulation")
-	valSDTSShowGraphs *bool = pflag.Bool("sim-graphs", false, "Show full generated dependency graph output for parse trees that caused errors during simulation")
-	valSDTSFirstOnly  *bool = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation")
-	valSDTSSkip       *int  = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation in output")
-
-	tmplTokens *string = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
-	tmplLexer  *string = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
-	tmplParser *string = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
-	tmplSDTS   *string = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
-	tmplFront  *string = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
-	tmplMain   *string = pflag.String("tmpl-main", "", "A template file to replace the embedded main.go template with")
-
-	parserLL      *bool = pflag.Bool("ll", false, "Generate an LL(1) parser")
-	parserSLR     *bool = pflag.Bool("slr", false, "Generate a simple LR(1) parser")
-	parserCLR     *bool = pflag.Bool("clr", false, "Generate a canonical LR(1) parser")
-	parserLALR    *bool = pflag.Bool("lalr", false, "Generate a canonical LR(1) parser")
-	parserNoAmbig *bool = pflag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it")
-
-	lexerTrace  *bool = pflag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
-	parserTrace *bool = pflag.Bool("debug-parser", false, "Print the parser trace to stdout")
-
-	hooksPath      *string = pflag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.")
-	hooksTableName *string = pflag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.")
-
-	irType *string = pflag.String("ir", "", "The fully-qualified type of IR to generate.")
-
-	version *bool = pflag.Bool("version", false, "Print the version of ictcc and exit")
+	diagnosticsBin = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
+	diagFormatPkg  = pflag.StringP("diag-format-pkg", "f", "", "The package containing format functions for the diagnostic binary to call on input prior to passing to frontend analysis")
+	diagFormatCall = pflag.StringP("diag-format-call", "c", "NewCodeReader", "The function within the diag-format-pkg to call to open a reader on input prior to passing to frontend analysis")
+
+	pathPrefix        = pflag.String("prefix", "", "Path to prepend to path of all generated source files")
+	preserveBinSource = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
+	debugTemplates    = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
+	pkg               = pflag.String("pkg", "fe", "The name of the package to place generated files in")
+	dest              = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
+	langVer           = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
+	noCache           = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
+	noCacheOutput     = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
+
+	valSDTSOff        = pflag.Bool("sim-off", false, "Disable input simulation of the language once built")
+	valSDTSShowTrees  = pflag.Bool("sim-trees", false, "Show parse trees that caused errors during simulation")
+	valSDTSShowGraphs = pflag.Bool("sim-graphs", false, "Show full generated dependency graph output for parse trees that caused errors during simulation")
+	valSDTSFirstOnly  = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation")
+	valSDTSSkip       = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation in output")
+
+	tmplTokens = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
+	tmplLexer  = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
+	tmplParser = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
+	tmplSDTS   = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
+	tmplFront  = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
+	tmplMain   = pflag.String("tmpl-main", "", "A template file to replace the embedded main.go template with")
+
+	parserLL      = pflag.Bool("ll", false, "Generate an LL(1) parser")
+	parserSLR     = pflag.Bool("slr", false, "Generate a simple LR(1) parser")
+	parserCLR     = pflag.Bool("clr", false, "Generate a canonical LR(1) parser")
+	parserLALR    = pflag.Bool("lalr", false, "Generate a canonical LR(1) parser")
+	parserNoAmbig = pflag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it")
+
+	lexerTrace  = pflag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
+	parserTrace = pflag.Bool("debug-parser", false, "Print the parser trace to stdout")
+
+	hooksPath      = pflag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.")
+	hooksTableName = pflag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.")
+
+	irType = pflag.String("ir", "", "The fully-qualified type of IR to generate.")
+
+	version = pflag.Bool("version", false, "Print the version of ictcc and exit")
 )
 
 func init() {
@@ -402,6 +431,29 @@ func main() {
 			returnCode = ExitErrInvalidFlags
 			return
 		}
+
+		// you cannot set ONLY the formatting call
+		flagInfoDiagFormatCall := pflag.Lookup("diag-format-call")
+		// don't error check; all we'd do is panic
+		if flagInfoDiagFormatCall.Changed && *diagFormatPkg == "" {
+			fmt.Fprintf(os.Stderr, "ERROR: -c/--diag-format-call cannot be set without -f/--diag-format-pkg\n")
+			returnCode = ExitErrInvalidFlags
+			return
+		}
+	} else {
+		// otherwise, it makes no sense to set --diag-format-pkg or --diag-format-call; disallow this
+		flagInfoDiagFormatPkg := pflag.Lookup("diag-format-pkg")
+		flagInfoDiagFormatCall := pflag.Lookup("diag-format-call")
+		if flagInfoDiagFormatPkg.Changed {
+			fmt.Fprintf(os.Stderr, "ERROR: -f/--diag-format-pkg cannot be set without -d/--diagnostics-bin\n")
+			returnCode = ExitErrInvalidFlags
+			return
+		}
+		if flagInfoDiagFormatCall.Changed {
+			fmt.Fprintf(os.Stderr, "ERROR: -c/--diag-format-call cannot be set without -d/--diagnostics-bin\n")
+			returnCode = ExitErrInvalidFlags
+			return
+		}
 	}
 
 	// create a spec metadata object
@@ -558,9 +610,7 @@ func main() {
 		return
 	}
 
-	// code gen time!
-
-	// okay, first try to create a parser
+	// spec completed and no-gen not set; try to create a parser
 	var p ictiobus.Parser
 	var parserWarns []fishi.Warning
 	// if one is selected, use that one
@@ -576,6 +626,8 @@ func main() {
 		p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig)
 	}
 
+	// code gen time! 38D
+
 	for _, warn := range parserWarns {
 		const warnPrefix = "WARN: "
 		// indent all except the first line
@@ -634,6 +686,14 @@ func main() {
 	if *diagnosticsBin != "" {
 		// already checked required flags
 		if !quietMode {
+			// tell user if the diagnostic binary cannot do preformatting based
+			// on flags
+			if *diagFormatPkg != "" {
+				fmt.Printf("Format preprocessing disabled in diagnostics bin; set -f to enable\n")
+			}
+
+			// TODO: probably should make this a constant in fishi instead of
+			// having ictcc just magically know about it
 			diagGenDir := ".gen"
 			if *pathPrefix != "" {
 				diagGenDir = filepath.Join(*pathPrefix, diagGenDir)
@@ -641,7 +701,15 @@ func main() {
 			fmt.Printf("Generating diagnostics binary code in %s...\n", diagGenDir)
 		}
 
-		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *pkg, *diagnosticsBin, *pathPrefix, cgOpts)
+		// only specify a format call if a format package was specified,
+		// otherwise we'll always pass in a non-empty string for the format call
+		// even when diagFormatPkg is empty, which is not allowed.
+		var formatCall string
+		if *diagFormatPkg != "" {
+			formatCall = *diagFormatCall
+		}
+
+		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *diagFormatPkg, formatCall, *pkg, *diagnosticsBin, *pathPrefix, cgOpts)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
 			returnCode = ExitErrGeneration
@@ -649,7 +717,7 @@ func main() {
 		}
 
 		if !quietMode {
-			fmt.Printf("Built diagnosticis binary '%s'\n", *diagnosticsBin)
+			fmt.Printf("Built diagnostics binary '%s'\n", *diagnosticsBin)
 		}
 	}
 
diff --git a/fishi/cgstructs.go b/fishi/cgstructs.go
new file mode 100644
index 0000000..7a9d49d
--- /dev/null
+++ b/fishi/cgstructs.go
@@ -0,0 +1,50 @@
+package fishi
+
+import "github.com/dekarrin/ictiobus"
+
+// File cgstructs.go contains structs used as part of code generation.
+
+// MainBinaryParams is paramters for generating a main.go file for a binary.
+// Unless otherwise specified, all fields are required.
+type MainBinaryParams struct {
+	// Parser is the parser to use for the generated compiler.
+	Parser ictiobus.Parser
+
+	// HooksPkgDir is the path to the directory containing the hooks package.
+	HooksPkgDir string
+
+	// HooksExpr is the expression to use to get the hooks map. This can be a
+	// function call, constant name, or var name.
+	HooksExpr string
+
+	// FormatPkgDir is the path to the directory containing the format package.
+	// It is completely optional; if not set, the generated main will not
+	// contain any pre-formatting code and will assume files are directly ready
+	// to be fed into the frontend. Must be set if FormatCall is set.
+	FormatPkgDir string
+
+	// FormatCall is the name of a function within the package specified by
+	// FormatPkgDir that gets an io.Reader that will run any required
+	// pre-formatting on an input io.Reader to get code that can be analyzed by
+	// the frontend. Is is optional; if not set, the generated main will not
+	// contain any pre-formatting code and will assume files are directly ready
+	// to be fed into the frontend. Must be set if FormatPkgDir is set.
+	FormatCall string
+
+	// FrontendPkgName is the name of the package to place generated frontend
+	// code in.
+	FrontendPkgName string
+
+	// GenPath is the path to a directory to generate code in. If it does not
+	// exist, it will be created. If it does exist, any existing files in it
+	// will be removed will be emptied before code is generated.
+	GenPath string
+
+	// BinName is the name of the binary being generated. This will be used
+	// within code for showing help output and other messages.
+	BinName string
+
+	// Opts are options for code generation. This must be set and its IRType
+	// field is required to be set, but all other fields within it are optional.
+	Opts CodegenOptions
+}
diff --git a/fishi/codegen.go b/fishi/codegen.go
index 58dee30..c442aff 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -162,14 +162,40 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOpti
 	return cmd.Run()
 }
 
-func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, pkgName string, binPath string, pathPrefix string, opts CodegenOptions) error {
+// GenerateDiagnosticsBinary generates a binary that can read input written in
+// the language specified by the given Spec and SpecMetadata and print out basic
+// information about the analysis, with the goal of printing out the
+// constructed intermediate representation from analyzed files.
+//
+// The args formatPkgDir and formatCall are used to specify preformatting for
+// code that that the generated binary will analyze. If set, io.Readers that are
+// opened on code input will be passed to the function specified by formatCall.
+// If formatCall is set, formatPkgDir must also be set, even if it is already
+// specified by another parameter. formatCall must be the name of a function
+// within the package specified by formatPkgDir which takes an io.Reader and
+// returns a new io.Reader that wraps the one passed in and returns preformatted
+// code ready to be analyzed by the generated frontend.
+//
+// TODO: turn this huge signature into a struct for everyfin from p to opts.
+func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, formatPkgDir string, formatCall string, pkgName string, binPath string, pathPrefix string, opts CodegenOptions) error {
 	binName := filepath.Base(binPath)
 
 	outDir := ".gen"
 	if pathPrefix != "" {
 		outDir = filepath.Join(pathPrefix, outDir)
 	}
-	gci, err := GenerateBinaryMainGo(spec, md, p, hooksPkgDir, hooksExpr, pkgName, outDir, binName, opts)
+
+	gci, err := GenerateBinaryMainGo(spec, md, MainBinaryParams{
+		Parser:          p,
+		HooksPkgDir:     hooksPkgDir,
+		HooksExpr:       hooksExpr,
+		FormatPkgDir:    formatPkgDir,
+		FormatCall:      formatCall,
+		FrontendPkgName: pkgName,
+		GenPath:         outDir,
+		BinName:         binName,
+		Opts:            opts,
+	})
 	if err != nil {
 		return err
 	}
@@ -213,27 +239,50 @@ func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, ho
 // or var name.
 //
 // opts must be non-nil and IRType must be set.
-//
-// TODO: turn this ugly signature into a struct.
-func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, formatPkgDir string, formatCall string, fePkgName string, genPath string, binName string, opts CodegenOptions) (GeneratedCodeInfo, error) {
-	if opts.IRType == "" {
+func GenerateBinaryMainGo(spec Spec, md SpecMetadata, params MainBinaryParams) (GeneratedCodeInfo, error) {
+	if params.Opts.IRType == "" {
 		return GeneratedCodeInfo{}, fmt.Errorf("IRType must be set in options")
 	}
+	if params.FormatPkgDir != "" && params.FormatCall == "" {
+		return GeneratedCodeInfo{}, fmt.Errorf("formatCall must be set if formatPkgDir is set")
+	}
+	if params.FormatPkgDir == "" && params.FormatCall != "" {
+		return GeneratedCodeInfo{}, fmt.Errorf("formatPkgDir must be set if formatCall is set")
+	}
 
-	irFQPackage, irType, irPackage, irErr := ParseFQType(opts.IRType)
+	// we need a separate import for the format package only if it's not the same
+	// as the hooks package.
+	separateFormatImport := params.FormatPkgDir != params.HooksPkgDir && params.FormatPkgDir != ""
+
+	irFQPackage, irType, irPackage, irErr := ParseFQType(params.Opts.IRType)
 	if irErr != nil {
 		return GeneratedCodeInfo{}, fmt.Errorf("parsing IRType: %w", irErr)
 	}
 
 	gci := GeneratedCodeInfo{}
 
-	hooksPkgName, err := readPackageName(hooksPkgDir)
+	hooksPkgName, err := readPackageName(params.HooksPkgDir)
 	if err != nil {
 		return gci, fmt.Errorf("reading hooks package name: %w", err)
 	}
-	if hooksPkgName == fePkgName {
+
+	var formatPkgName string
+	if params.FormatPkgDir != "" {
+		formatPkgName, err = readPackageName(params.FormatPkgDir)
+		if err != nil {
+			return gci, fmt.Errorf("reading format package name: %w", err)
+		}
+	}
+
+	if hooksPkgName == params.FrontendPkgName {
 		// double it to avoid name collision
-		fePkgName += "_" + fePkgName
+		params.FrontendPkgName += "_" + params.FrontendPkgName
+	}
+
+	// only worry about formatPkgName if the dir is not the same as hooks.
+	if params.FormatPkgDir != params.HooksPkgDir && formatPkgName == params.FrontendPkgName {
+		// double it to avoid name collision
+		params.FrontendPkgName += "_" + params.FrontendPkgName
 	}
 
 	safePkgIdent := func(s string) string {
@@ -242,21 +291,31 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 		return strings.ToLower(s)
 	}
 
-	err = os.MkdirAll(genPath, 0766)
+	err = os.MkdirAll(params.GenPath, 0766)
 	if err != nil {
-		return gci, fmt.Errorf("creating temp dir: %w", err)
+		return gci, fmt.Errorf("creating dir for generated code: %w", err)
 	}
 
 	// start copying the hooks package
-	hooksDestPath := filepath.Join(genPath, "internal", hooksPkgName)
-	hooksDone, err := copyDirToTargetAsync(hooksPkgDir, hooksDestPath)
+	hooksDestPath := filepath.Join(params.GenPath, "internal", hooksPkgName)
+	hooksDone, err := copyDirToTargetAsync(params.HooksPkgDir, hooksDestPath)
 	if err != nil {
 		return gci, fmt.Errorf("copying hooks package: %w", err)
 	}
+	// start copying the format package if set and if it's not the same as the
+	// hooks package.
+	var formatDone <-chan error
+	if separateFormatImport {
+		formatDestPath := filepath.Join(params.GenPath, "internal", formatPkgName)
+		formatDone, err = copyDirToTargetAsync(params.FormatPkgDir, formatDestPath)
+		if err != nil {
+			return gci, fmt.Errorf("copying format package: %w", err)
+		}
+	}
 
 	// generate the compiler code
-	fePkgPath := filepath.Join(genPath, "internal", fePkgName)
-	err = GenerateCompilerGo(spec, md, fePkgName, fePkgPath, &opts)
+	fePkgPath := filepath.Join(params.GenPath, "internal", params.FrontendPkgName)
+	err = GenerateCompilerGo(spec, md, params.FrontendPkgName, fePkgPath, &params.Opts)
 	if err != nil {
 		return gci, fmt.Errorf("generating compiler: %w", err)
 	}
@@ -264,7 +323,7 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// since GenerateCompilerGo ensures the directory exists, we can now copy
 	// the encoded parser into it as well.
 	parserPath := filepath.Join(fePkgPath, "parser.cff")
-	err = ictiobus.SaveParserToDisk(p, parserPath)
+	err = ictiobus.SaveParserToDisk(params.Parser, parserPath)
 	if err != nil {
 		return gci, fmt.Errorf("writing parser: %w", err)
 	}
@@ -277,11 +336,14 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// export template with main file
 	mainFillData := cgMainData{
 		BinPkg:            "github.com/dekarrin/ictiobus/langexec/" + safePkgIdent(md.Language),
-		BinName:           binName,
+		BinName:           params.BinName,
 		BinVersion:        md.Version,
 		HooksPkg:          hooksPkgName,
-		HooksTableExpr:    hooksExpr,
-		FrontendPkg:       fePkgName,
+		HooksTableExpr:    params.HooksExpr,
+		FormatPkg:         formatPkgName,
+		FormatCall:        params.FormatCall,
+		ImportFormatPkg:   separateFormatImport,
+		FrontendPkg:       params.FrontendPkgName,
 		IRTypePackage:     irFQPackage,
 		IRType:            irType,
 		IncludeSimulation: true,
@@ -292,7 +354,7 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	}
 
 	// initialize templates
-	err = initTemplates(renderFiles, fnMap, opts.TemplateFiles)
+	err = initTemplates(renderFiles, fnMap, params.Opts.TemplateFiles)
 	if err != nil {
 		return gci, err
 	}
@@ -300,7 +362,7 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// finally, render the main file
 	rf := renderFiles[ComponentMainFile]
 	mainFileRelPath := rf.outFile
-	err = renderTemplateToFile(rf.tmpl, mainFillData, filepath.Join(genPath, mainFileRelPath), opts.DumpPreFormat)
+	err = renderTemplateToFile(rf.tmpl, mainFillData, filepath.Join(params.GenPath, mainFileRelPath), params.Opts.DumpPreFormat)
 	if err != nil {
 		return gci, err
 	}
@@ -308,16 +370,21 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// wait for the hooks package to be copied; we'll need it for go mod tidy
 	<-hooksDone
 
+	// if we have a format package, wait for it to be copied
+	if formatDone != nil {
+		<-formatDone
+	}
+
 	// wipe any existing go module stuff
-	err = os.RemoveAll(filepath.Join(genPath, "go.mod"))
+	err = os.RemoveAll(filepath.Join(params.GenPath, "go.mod"))
 	if err != nil {
 		return gci, fmt.Errorf("removing existing go.mod: %w", err)
 	}
-	err = os.RemoveAll(filepath.Join(genPath, "go.sum"))
+	err = os.RemoveAll(filepath.Join(params.GenPath, "go.sum"))
 	if err != nil {
 		return gci, fmt.Errorf("removing existing go.sum: %w", err)
 	}
-	err = os.RemoveAll(filepath.Join(genPath, "vendor"))
+	err = os.RemoveAll(filepath.Join(params.GenPath, "vendor"))
 	if err != nil {
 		return gci, fmt.Errorf("removing existing vendor directory: %w", err)
 	}
@@ -325,21 +392,21 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk
 	// shell out to run go module stuff
 	goModInitCmd := exec.Command("go", "mod", "init", mainFillData.BinPkg)
 	goModInitCmd.Env = os.Environ()
-	goModInitCmd.Dir = genPath
+	goModInitCmd.Dir = params.GenPath
 	goModInitOutput, err := goModInitCmd.CombinedOutput()
 	if err != nil {
 		return gci, fmt.Errorf("initializing generated module with binary: %w\n%s", err, string(goModInitOutput))
 	}
 	goModTidyCmd := exec.Command("go", "mod", "tidy")
 	goModTidyCmd.Env = os.Environ()
-	goModTidyCmd.Dir = genPath
+	goModTidyCmd.Dir = params.GenPath
 	goModTidyOutput, err := goModTidyCmd.CombinedOutput()
 	if err != nil {
 		return gci, fmt.Errorf("tidying generated module with binary: %w\n%s", err, string(goModTidyOutput))
 	}
 
 	// if we got here, all output has been written to the temp dir.
-	gci.Path = genPath
+	gci.Path = params.GenPath
 	gci.MainFile = mainFileRelPath
 
 	return gci, nil
@@ -659,12 +726,13 @@ type cgMainData struct {
 	BinVersion        string
 	HooksPkg          string
 	HooksTableExpr    string
+	ImportFormatPkg   bool
+	FormatPkg         string
+	FormatCall        string
 	FrontendPkg       string
 	IRTypePackage     string
 	IRType            string
 	IncludeSimulation bool
-	FormatPackage     string
-	FormatCall        string
 }
 
 // codegenData for template fill.
diff --git a/fishi/fishi.go b/fishi/fishi.go
index 2f673bb..e29daf6 100644
--- a/fishi/fishi.go
+++ b/fishi/fishi.go
@@ -60,7 +60,18 @@ func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks
 	if pathPrefix != "" {
 		outDir = filepath.Join(pathPrefix, outDir)
 	}
-	genInfo, err := GenerateBinaryMainGo(spec, md, p, hooks, hooksTable, pkgName, outDir, binName, cgOpts)
+
+	// not setting the format package and call here because we don't need
+	// preformatting to run verification simulation.
+	genInfo, err := GenerateBinaryMainGo(spec, md, MainBinaryParams{
+		Parser:          p,
+		HooksPkgDir:     hooks,
+		HooksExpr:       hooksTable,
+		FrontendPkgName: pkgName,
+		GenPath:         outDir,
+		BinName:         binName,
+		Opts:            cgOpts,
+	})
 	if err != nil {
 		return fmt.Errorf("generate test compiler: %w", err)
 	}
diff --git a/fishi/format/format.go b/fishi/format/format.go
index a266b5d..94a4c21 100644
--- a/fishi/format/format.go
+++ b/fishi/format/format.go
@@ -11,7 +11,7 @@ import (
 	"strings"
 )
 
-// CodeReader is a reader that can be used to read fishi code from a stream
+// CodeReader is an implementation of io.Reader that reads fishi code from input
 // containing markdown-formatted text with fishi codeblocks. It will gather all
 // fishi codeblocks immediately on open and then read bytes from them as Read is
 // called. Preprocessing may also be done at that time. The CodeReader will
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index 87c40b2..7195502 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -25,8 +25,8 @@ import (
 	"{{ .IRTypePackage }}"
 {{- end}}
 
-{{if .FormatPackage -}}
-	"{{ .FormatPackage }}"
+{{if .ImportFormatPkg -}}
+	"{{ .BinPkg }}/internal/{{ .FormatPkg }}"
 {{- end}}
 
 	"github.com/dekarrin/ictiobus/trans"
@@ -139,7 +139,7 @@ func main() {
 
 {{if .FormatCall -}}
 		// format the input
-		r, err = {{ .FormatCall }}(r)
+		r, err = {{ .FormatPkg }}.{{ .FormatCall }}(r)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
 			returnCode = ExitErr

From 2e0ffb7a08138c565ce3dc9064f6f47f3b643c89 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 09:11:42 -0500
Subject: [PATCH 26/33] #16: cleanup args in ictcc main

---
 cmd/ictcc/main.go | 269 +++++++++++++++++++++-------------------------
 1 file changed, 124 insertions(+), 145 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index faccb72..823c8e2 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -325,78 +325,57 @@ var (
 )
 
 var (
-	quietMode bool
-	noGen     bool
-	genAST    bool
-	genTree   bool
-	showSpec  bool
-	parserCff string
-	lang      string
-
-	diagnosticsBin = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
-	diagFormatPkg  = pflag.StringP("diag-format-pkg", "f", "", "The package containing format functions for the diagnostic binary to call on input prior to passing to frontend analysis")
-	diagFormatCall = pflag.StringP("diag-format-call", "c", "NewCodeReader", "The function within the diag-format-pkg to call to open a reader on input prior to passing to frontend analysis")
-
-	pathPrefix        = pflag.String("prefix", "", "Path to prepend to path of all generated source files")
-	preserveBinSource = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
-	debugTemplates    = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
-	pkg               = pflag.String("pkg", "fe", "The name of the package to place generated files in")
-	dest              = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
-	langVer           = pflag.String("lang-ver", "v0.0.0", "The version of the language to generate")
-	noCache           = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
-	noCacheOutput     = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
-
-	valSDTSOff        = pflag.Bool("sim-off", false, "Disable input simulation of the language once built")
-	valSDTSShowTrees  = pflag.Bool("sim-trees", false, "Show parse trees that caused errors during simulation")
-	valSDTSShowGraphs = pflag.Bool("sim-graphs", false, "Show full generated dependency graph output for parse trees that caused errors during simulation")
-	valSDTSFirstOnly  = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation")
-	valSDTSSkip       = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation in output")
-
-	tmplTokens = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
-	tmplLexer  = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
-	tmplParser = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
-	tmplSDTS   = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
-	tmplFront  = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
-	tmplMain   = pflag.String("tmpl-main", "", "A template file to replace the embedded main.go template with")
-
-	parserLL      = pflag.Bool("ll", false, "Generate an LL(1) parser")
-	parserSLR     = pflag.Bool("slr", false, "Generate a simple LR(1) parser")
-	parserCLR     = pflag.Bool("clr", false, "Generate a canonical LR(1) parser")
-	parserLALR    = pflag.Bool("lalr", false, "Generate a canonical LR(1) parser")
-	parserNoAmbig = pflag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it")
-
-	lexerTrace  = pflag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
-	parserTrace = pflag.Bool("debug-parser", false, "Print the parser trace to stdout")
-
-	hooksPath      = pflag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.")
-	hooksTableName = pflag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.")
-
-	irType = pflag.String("ir", "", "The fully-qualified type of IR to generate.")
-
-	version = pflag.Bool("version", false, "Print the version of ictcc and exit")
+	flagQuietMode = pflag.BoolP("quiet", "q", false, "Suppress progress messages and other supplementary output")
+	flagNoGen     = pflag.BoolP("no-gen", "n", false, "Do not attempt to generate the parser")
+	flagGenAST    = pflag.BoolP("ast", "a", false, "Print the AST of the analyzed fishi")
+	flagGenTree   = pflag.BoolP("tree", "t", false, "Print the parse trees of each analyzed fishi file")
+	flagShowSpec  = pflag.BoolP("spec", "s", false, "Print the FISHI spec interpreted from the analyzed fishi")
+	flagParserCff = pflag.StringP("parser", "p", "fishi-parser.cff", "Use the specified parser CFF cache file instead of default")
+	flagLang      = pflag.StringP("lang", "l", "Unspecified", "The name of the languae being generated")
+	flagLangVer   = pflag.StringP("lang-ver", "v", "v0.0.0", "The version of the language to generate")
+
+	flagDiagBin        = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language")
+	flagDiagFormatPkg  = pflag.StringP("diag-format-pkg", "f", "", "The package containing format functions for the diagnostic binary to call on input prior to passing to frontend analysis")
+	flagDiagFormatCall = pflag.StringP("diag-format-call", "c", "NewCodeReader", "The function within the diag-format-pkg to call to open a reader on input prior to passing to frontend analysis")
+
+	flagPathPrefix        = pflag.String("prefix", "", "Path to prepend to path of all generated source files")
+	flagPreserveBinSource = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files")
+	flagDebugTemplates    = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt")
+	flagPkg               = pflag.String("pkg", "fe", "The name of the package to place generated files in")
+	flagDest              = pflag.String("dest", "./fe", "The name of the directory to place the generated package in")
+	flagNoCache           = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available")
+	flagNoCacheOutput     = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated")
+
+	flagSimOff          = pflag.Bool("sim-off", false, "Disable input simulation of the language once built")
+	flagSimTrees        = pflag.Bool("sim-trees", false, "Show parse trees that caused errors during simulation")
+	flagSimGraphs       = pflag.Bool("sim-graphs", false, "Show full generated dependency graph output for parse trees that caused errors during simulation")
+	flagSimFirstErrOnly = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation")
+	flagSimSkipErrs     = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation in output")
+
+	flagTmplTokens = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with")
+	flagTmplLexer  = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with")
+	flagTmplParser = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with")
+	flagTmplSDTS   = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with")
+	flagTmplFront  = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with")
+	flagTmplMain   = pflag.String("tmpl-main", "", "A template file to replace the embedded main.go template with")
+
+	flagParserLL      = pflag.Bool("ll", false, "Generate an LL(1) parser")
+	flagParserSLR     = pflag.Bool("slr", false, "Generate a simple LR(1) parser")
+	flagParserCLR     = pflag.Bool("clr", false, "Generate a canonical LR(1) parser")
+	flagParserLALR    = pflag.Bool("lalr", false, "Generate a canonical LR(1) parser")
+	flagParserNoAmbig = pflag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it")
+
+	flagLexerTrace  = pflag.Bool("debug-lexer", false, "Print the lexer trace to stdout")
+	flagParserTrace = pflag.Bool("debug-parser", false, "Print the parser trace to stdout")
+
+	flagHooksPath      = pflag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.")
+	flagHooksTableName = pflag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.")
+
+	flagIRType = pflag.String("ir", "", "The fully-qualified type of IR to generate.")
+
+	flagVersion = pflag.Bool("version", false, "Print the version of ictcc and exit")
 )
 
-func init() {
-	const (
-		quietUsage       = "Do not print progress messages"
-		noGenUsage       = "Do not generate the parser"
-		genASTUsage      = "Print the AST of the analyzed fishi"
-		genTreeUsage     = "Print the parse trees of each analyzed fishi file"
-		genSpecUsage     = "Print the FISHI spec interpreted from the analyzed fishi"
-		parserCffUsage   = "Use the specified parser CFF cache file instead of default"
-		parserCffDefault = "fishi-parser.cff"
-		langUsage        = "The name of the languae being generated"
-		langDefault      = "Unspecified"
-	)
-	pflag.BoolVarP(&noGen, "no-gen", "n", false, noGenUsage)
-	pflag.BoolVarP(&genAST, "ast", "a", false, genASTUsage)
-	pflag.BoolVarP(&showSpec, "spec", "s", false, genSpecUsage)
-	pflag.BoolVarP(&genTree, "tree", "t", false, genTreeUsage)
-	pflag.StringVarP(&parserCff, "parser", "p", parserCffDefault, parserCffUsage)
-	pflag.StringVarP(&lang, "lang", "l", langDefault, langUsage)
-	pflag.BoolVarP(&quietMode, "quiet", "q", false, quietUsage)
-}
-
 func main() {
 	// basic function to check if panic is happening and recover it while also
 	// preserving possibly-set exit code.
@@ -415,18 +394,18 @@ func main() {
 
 	pflag.Parse()
 
-	if *version {
+	if *flagVersion {
 		fmt.Println(GetVersionString())
 		return
 	}
 
 	// mutually exclusive and required options for diagnostics bin generation.
-	if *diagnosticsBin != "" {
-		if noGen {
+	if *flagDiagBin != "" {
+		if *flagNoGen {
 			fmt.Fprintf(os.Stderr, "ERROR: Diagnostics binary generation canont be enabled if -n/--no-gen is specified\n")
 			returnCode = ExitErrInvalidFlags
 			return
-		} else if *irType == "" || *hooksPath == "" {
+		} else if *flagIRType == "" || *flagHooksPath == "" {
 			fmt.Fprintf(os.Stderr, "ERROR: diagnostics binary generation requires both --ir and --hooks to be set\n")
 			returnCode = ExitErrInvalidFlags
 			return
@@ -435,7 +414,7 @@ func main() {
 		// you cannot set ONLY the formatting call
 		flagInfoDiagFormatCall := pflag.Lookup("diag-format-call")
 		// don't error check; all we'd do is panic
-		if flagInfoDiagFormatCall.Changed && *diagFormatPkg == "" {
+		if flagInfoDiagFormatCall.Changed && *flagDiagFormatPkg == "" {
 			fmt.Fprintf(os.Stderr, "ERROR: -c/--diag-format-call cannot be set without -f/--diag-format-pkg\n")
 			returnCode = ExitErrInvalidFlags
 			return
@@ -458,8 +437,8 @@ func main() {
 
 	// create a spec metadata object
 	md := fishi.SpecMetadata{
-		Language:       lang,
-		Version:        *langVer,
+		Language:       *flagLang,
+		Version:        *flagLangVer,
 		InvocationArgs: invocation,
 	}
 
@@ -479,36 +458,36 @@ func main() {
 	}
 
 	fo := fishi.Options{
-		ParserCFF:   parserCff,
-		ReadCache:   !*noCache,
-		WriteCache:  !*noCacheOutput,
-		LexerTrace:  *lexerTrace,
-		ParserTrace: *parserTrace,
+		ParserCFF:   *flagParserCff,
+		ReadCache:   !*flagNoCache,
+		WriteCache:  !*flagNoCacheOutput,
+		LexerTrace:  *flagLexerTrace,
+		ParserTrace: *flagParserTrace,
 	}
 
 	cgOpts := fishi.CodegenOptions{
-		DumpPreFormat:        *debugTemplates,
-		IRType:               *irType,
+		DumpPreFormat:        *flagDebugTemplates,
+		IRType:               *flagIRType,
 		TemplateFiles:        map[string]string{},
-		PreserveBinarySource: *preserveBinSource,
+		PreserveBinarySource: *flagPreserveBinSource,
 	}
-	if *tmplTokens != "" {
-		cgOpts.TemplateFiles[fishi.ComponentTokens] = *tmplTokens
+	if *flagTmplTokens != "" {
+		cgOpts.TemplateFiles[fishi.ComponentTokens] = *flagTmplTokens
 	}
-	if *tmplLexer != "" {
-		cgOpts.TemplateFiles[fishi.ComponentLexer] = *tmplLexer
+	if *flagTmplLexer != "" {
+		cgOpts.TemplateFiles[fishi.ComponentLexer] = *flagTmplLexer
 	}
-	if *tmplParser != "" {
-		cgOpts.TemplateFiles[fishi.ComponentParser] = *tmplParser
+	if *flagTmplParser != "" {
+		cgOpts.TemplateFiles[fishi.ComponentParser] = *flagTmplParser
 	}
-	if *tmplSDTS != "" {
-		cgOpts.TemplateFiles[fishi.ComponentSDTS] = *tmplSDTS
+	if *flagTmplSDTS != "" {
+		cgOpts.TemplateFiles[fishi.ComponentSDTS] = *flagTmplSDTS
 	}
-	if *tmplFront != "" {
-		cgOpts.TemplateFiles[fishi.ComponentFrontend] = *tmplFront
+	if *flagTmplFront != "" {
+		cgOpts.TemplateFiles[fishi.ComponentFrontend] = *flagTmplFront
 	}
-	if *tmplMain != "" {
-		cgOpts.TemplateFiles[fishi.ComponentMainFile] = *tmplMain
+	if *flagTmplMain != "" {
+		cgOpts.TemplateFiles[fishi.ComponentMainFile] = *flagTmplMain
 	}
 	if len(cgOpts.TemplateFiles) == 0 {
 		// just nil it
@@ -516,7 +495,7 @@ func main() {
 	}
 
 	// now that args are gathered, parse markdown files into an AST
-	if !quietMode {
+	if !*flagQuietMode {
 		files := textfmt.Pluralize(len(args), "FISHI input file", "-s")
 		fmt.Printf("Reading %s...\n", files)
 	}
@@ -535,13 +514,13 @@ func main() {
 
 		// parse tree is per-file, so we do this immediately even on error, as
 		// it may be useful
-		if res.Tree != nil && genTree {
+		if res.Tree != nil && *flagGenTree {
 			fmt.Printf("%s\n", trans.AddAttributes(*res.Tree).String())
 		}
 
 		if err != nil {
 			// results may be valid even if there is an error
-			if joinedAST != nil && genAST {
+			if joinedAST != nil && *flagGenAST {
 				fmt.Printf("%s\n", res.AST.String())
 			}
 
@@ -560,12 +539,12 @@ func main() {
 		panic("joinedAST is nil; should not be possible")
 	}
 
-	if genAST {
+	if *flagGenAST {
 		fmt.Printf("%s\n", joinedAST.String())
 	}
 
 	// attempt to turn AST into a fishi.Spec
-	if !quietMode {
+	if !*flagQuietMode {
 		fmt.Printf("Generating language spec from FISHI...\n")
 	}
 	spec, warnings, err := fishi.NewSpec(*joinedAST)
@@ -599,12 +578,12 @@ func main() {
 	}
 
 	// we officially have a spec. try to print it if requested
-	if showSpec {
+	if *flagShowSpec {
 		printSpec(spec)
 	}
 
-	if noGen {
-		if !quietMode {
+	if *flagNoGen {
+		if !*flagQuietMode {
 			fmt.Printf("(code generation skipped due to flags)\n")
 		}
 		return
@@ -615,12 +594,12 @@ func main() {
 	var parserWarns []fishi.Warning
 	// if one is selected, use that one
 	if parserType != nil {
-		if !quietMode {
+		if !*flagQuietMode {
 			fmt.Printf("Creating %s parser from spec...\n", *parserType)
 		}
 		p, parserWarns, err = spec.CreateParser(*parserType, allowAmbig)
 	} else {
-		if !quietMode {
+		if !*flagQuietMode {
 			fmt.Printf("Creating most restrictive parser from spec...\n")
 		}
 		p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig)
@@ -646,33 +625,33 @@ func main() {
 		return
 	}
 
-	if !quietMode {
+	if !*flagQuietMode {
 		fmt.Printf("Successfully generated %s parser from grammar\n", p.Type().String())
 	}
 
 	// create a test compiler and output it
-	if !*valSDTSOff {
-		if *irType == "" {
+	if !*flagSimOff {
+		if *flagIRType == "" {
 			fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --ir parameter\n")
 		} else {
-			if *hooksPath == "" {
+			if *flagHooksPath == "" {
 				fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --hooks parameter\n")
 			} else {
-				if !quietMode {
+				if !*flagQuietMode {
 					simGenDir := ".sim"
-					if *pathPrefix != "" {
-						simGenDir = filepath.Join(*pathPrefix, simGenDir)
+					if *flagPathPrefix != "" {
+						simGenDir = filepath.Join(*flagPathPrefix, simGenDir)
 					}
 					fmt.Printf("Generating parser simulation binary in %s...\n", simGenDir)
 				}
 				di := trans.ValidationOptions{
-					ParseTrees:    *valSDTSShowTrees,
-					FullDepGraphs: *valSDTSShowGraphs,
-					ShowAllErrors: !*valSDTSFirstOnly,
-					SkipErrors:    *valSDTSSkip,
+					ParseTrees:    *flagSimTrees,
+					FullDepGraphs: *flagSimGraphs,
+					ShowAllErrors: !*flagSimFirstErrOnly,
+					SkipErrors:    *flagSimSkipErrs,
 				}
 
-				err := fishi.ValidateSimulatedInput(spec, md, p, *hooksPath, *hooksTableName, *pathPrefix, cgOpts, &di)
+				err := fishi.ValidateSimulatedInput(spec, md, p, *flagHooksPath, *flagHooksTableName, *flagPathPrefix, cgOpts, &di)
 				if err != nil {
 					fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
 					returnCode = ExitErrGeneration
@@ -683,20 +662,20 @@ func main() {
 	}
 
 	// generate diagnostics output if requested
-	if *diagnosticsBin != "" {
+	if *flagDiagBin != "" {
 		// already checked required flags
-		if !quietMode {
+		if !*flagQuietMode {
 			// tell user if the diagnostic binary cannot do preformatting based
 			// on flags
-			if *diagFormatPkg != "" {
+			if *flagDiagFormatPkg != "" {
 				fmt.Printf("Format preprocessing disabled in diagnostics bin; set -f to enable\n")
 			}
 
 			// TODO: probably should make this a constant in fishi instead of
 			// having ictcc just magically know about it
 			diagGenDir := ".gen"
-			if *pathPrefix != "" {
-				diagGenDir = filepath.Join(*pathPrefix, diagGenDir)
+			if *flagPathPrefix != "" {
+				diagGenDir = filepath.Join(*flagPathPrefix, diagGenDir)
 			}
 			fmt.Printf("Generating diagnostics binary code in %s...\n", diagGenDir)
 		}
@@ -705,35 +684,35 @@ func main() {
 		// otherwise we'll always pass in a non-empty string for the format call
 		// even when diagFormatPkg is empty, which is not allowed.
 		var formatCall string
-		if *diagFormatPkg != "" {
-			formatCall = *diagFormatCall
+		if *flagDiagFormatPkg != "" {
+			formatCall = *flagDiagFormatCall
 		}
 
-		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *hooksPath, *hooksTableName, *diagFormatPkg, formatCall, *pkg, *diagnosticsBin, *pathPrefix, cgOpts)
+		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *flagHooksPath, *flagHooksTableName, *flagDiagFormatPkg, formatCall, *flagPkg, *flagDiagBin, *flagPathPrefix, cgOpts)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
 			returnCode = ExitErrGeneration
 			return
 		}
 
-		if !quietMode {
-			fmt.Printf("Built diagnostics binary '%s'\n", *diagnosticsBin)
+		if !*flagQuietMode {
+			fmt.Printf("Built diagnostics binary '%s'\n", *flagDiagBin)
 		}
 	}
 
 	// assuming it worked, now generate the final output
-	if !quietMode {
-		feGenDir := *dest
-		if *pathPrefix != "" {
-			feGenDir = filepath.Join(*pathPrefix, feGenDir)
+	if !*flagQuietMode {
+		feGenDir := *flagDest
+		if *flagPathPrefix != "" {
+			feGenDir = filepath.Join(*flagPathPrefix, feGenDir)
 		}
 		fmt.Printf("Generating compiler frontend in %s...\n", feGenDir)
 	}
-	feDest := *dest
-	if *pathPrefix != "" {
-		feDest = filepath.Join(*pathPrefix, feDest)
+	feDest := *flagDest
+	if *flagPathPrefix != "" {
+		feDest = filepath.Join(*flagPathPrefix, feDest)
 	}
-	err = fishi.GenerateCompilerGo(spec, md, *pkg, feDest, &cgOpts)
+	err = fishi.GenerateCompilerGo(spec, md, *flagPkg, feDest, &cgOpts)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
 		returnCode = ExitErrGeneration
@@ -748,29 +727,29 @@ func main() {
 // err will be non-nil if there is an invalid combination of CLI flags.
 func parserSelectionFromFlags() (t *types.ParserType, allowAmbig bool, err error) {
 	// enforce mutual exclusion of cli args
-	if (*parserLL && (*parserCLR || *parserSLR || *parserLALR)) ||
-		(*parserCLR && (*parserSLR || *parserLALR)) ||
-		(*parserSLR && *parserLALR) {
+	if (*flagParserLL && (*flagParserCLR || *flagParserSLR || *flagParserLALR)) ||
+		(*flagParserCLR && (*flagParserSLR || *flagParserLALR)) ||
+		(*flagParserSLR && *flagParserLALR) {
 
 		err = fmt.Errorf("cannot specify more than one parser type")
 		return
 	}
 
-	allowAmbig = !*parserNoAmbig
+	allowAmbig = !*flagParserNoAmbig
 
-	if *parserLL {
+	if *flagParserLL {
 		t = new(types.ParserType)
 		*t = types.ParserLL1
 
 		// allowAmbig auto false for LL(1)
 		allowAmbig = false
-	} else if *parserSLR {
+	} else if *flagParserSLR {
 		t = new(types.ParserType)
 		*t = types.ParserSLR1
-	} else if *parserCLR {
+	} else if *flagParserCLR {
 		t = new(types.ParserType)
 		*t = types.ParserCLR1
-	} else if *parserLALR {
+	} else if *flagParserLALR {
 		t = new(types.ParserType)
 		*t = types.ParserLALR1
 	}

From db0a1525831c68694a0cc333f31d0a65fc1ad9a1 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 09:44:27 -0500
Subject: [PATCH 27/33] #16: slightly cleaner error handling in ictcc main

---
 cmd/ictcc/main.go   | 106 +++++++++-----------------------------
 cmd/ictcc/status.go | 122 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 147 insertions(+), 81 deletions(-)
 create mode 100644 cmd/ictcc/status.go

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 823c8e2..9eec843 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -292,38 +292,6 @@ import (
 	"github.com/dekarrin/rosed"
 )
 
-const (
-	// ExitSuccess is the exit code for a successful run.
-	ExitSuccess = iota
-
-	// ExitErrNoFiles is the code returned as exit status when no files are
-	// provided to the invocation.
-	ExitErrNoFiles
-
-	// ExitErrInvalidFlags is used if the combination of flags specified is
-	// invalid.
-	ExitErrInvalidFlags
-
-	// ExitErrSyntax is the code returned as exit status when a syntax error
-	// occurs.
-	ExitErrSyntax
-
-	// ExitErrParser is the code returned as exit status when there is an error
-	// generating the parser.
-	ExitErrParser
-
-	// ExitErrGeneration is the code returned as exit status when there is an
-	// error creating the generated files.
-	ExitErrGeneration
-
-	// ExitErrOther is a generic error code for any other error.
-	ExitErrOther
-)
-
-var (
-	returnCode = ExitSuccess
-)
-
 var (
 	flagQuietMode = pflag.BoolP("quiet", "q", false, "Suppress progress messages and other supplementary output")
 	flagNoGen     = pflag.BoolP("no-gen", "n", false, "Do not attempt to generate the parser")
@@ -377,17 +345,7 @@ var (
 )
 
 func main() {
-	// basic function to check if panic is happening and recover it while also
-	// preserving possibly-set exit code.
-	defer func() {
-		if panicErr := recover(); panicErr != nil {
-			// we are panicking, make sure we dont lose the panic just because
-			// we checked
-			panic("unrecoverable panic occured")
-		} else {
-			os.Exit(returnCode)
-		}
-	}()
+	defer preservePanicOrExitWithStatus()
 
 	// gather options and arguments
 	invocation := strings.Join(os.Args[1:], " ")
@@ -402,12 +360,10 @@ func main() {
 	// mutually exclusive and required options for diagnostics bin generation.
 	if *flagDiagBin != "" {
 		if *flagNoGen {
-			fmt.Fprintf(os.Stderr, "ERROR: Diagnostics binary generation canont be enabled if -n/--no-gen is specified\n")
-			returnCode = ExitErrInvalidFlags
+			errInvalidFlags("Diagnostics bin generation cannot be enabled due to -n/--no-gen")
 			return
 		} else if *flagIRType == "" || *flagHooksPath == "" {
-			fmt.Fprintf(os.Stderr, "ERROR: diagnostics binary generation requires both --ir and --hooks to be set\n")
-			returnCode = ExitErrInvalidFlags
+			errInvalidFlags("Diagnostics bin generation requires both --ir and --hooks to be set")
 			return
 		}
 
@@ -415,8 +371,7 @@ func main() {
 		flagInfoDiagFormatCall := pflag.Lookup("diag-format-call")
 		// don't error check; all we'd do is panic
 		if flagInfoDiagFormatCall.Changed && *flagDiagFormatPkg == "" {
-			fmt.Fprintf(os.Stderr, "ERROR: -c/--diag-format-call cannot be set without -f/--diag-format-pkg\n")
-			returnCode = ExitErrInvalidFlags
+			errInvalidFlags("-c/--diag-format-call cannot be set without -f/--diag-format-pkg")
 			return
 		}
 	} else {
@@ -424,37 +379,34 @@ func main() {
 		flagInfoDiagFormatPkg := pflag.Lookup("diag-format-pkg")
 		flagInfoDiagFormatCall := pflag.Lookup("diag-format-call")
 		if flagInfoDiagFormatPkg.Changed {
-			fmt.Fprintf(os.Stderr, "ERROR: -f/--diag-format-pkg cannot be set without -d/--diagnostics-bin\n")
-			returnCode = ExitErrInvalidFlags
+			errInvalidFlags("-f/--diag-format-pkg cannot be set without -d/--diagnostics-bin")
 			return
 		}
 		if flagInfoDiagFormatCall.Changed {
-			fmt.Fprintf(os.Stderr, "ERROR: -c/--diag-format-call cannot be set without -d/--diagnostics-bin\n")
-			returnCode = ExitErrInvalidFlags
+			errInvalidFlags("-c/--diag-format-call cannot be set without -d/--diagnostics-bin")
 			return
 		}
 	}
 
-	// create a spec metadata object
-	md := fishi.SpecMetadata{
-		Language:       *flagLang,
-		Version:        *flagLangVer,
-		InvocationArgs: invocation,
+	parserType, allowAmbig, err := parserSelectionFromFlags()
+	if err != nil {
+		errInvalidFlags(err.Error())
+		return
 	}
 
+	// check args before gathering flags
 	args := pflag.Args()
 
 	if len(args) < 1 {
-		fmt.Fprintf(os.Stderr, "No files given to process\n")
-		returnCode = ExitErrNoFiles
+		errNoFiles("No files given to process")
 		return
 	}
 
-	parserType, allowAmbig, err := parserSelectionFromFlags()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
-		returnCode = ExitErrInvalidFlags
-		return
+	// create a spec metadata object
+	md := fishi.SpecMetadata{
+		Language:       *flagLang,
+		Version:        *flagLangVer,
+		InvocationArgs: invocation,
 	}
 
 	fo := fishi.Options{
@@ -525,11 +477,9 @@ func main() {
 			}
 
 			if syntaxErr, ok := err.(*types.SyntaxError); ok {
-				fmt.Fprintf(os.Stderr, "%s:\n%s\n", file, syntaxErr.FullMessage())
-				returnCode = ExitErrSyntax
+				errSyntax(file, syntaxErr.FullMessage())
 			} else {
-				fmt.Fprintf(os.Stderr, "%s: %s\n", file, err.Error())
-				returnCode = ExitErrOther
+				errOther(fmt.Sprintf("%s: %s", file, err.Error()))
 			}
 			return
 		}
@@ -568,11 +518,9 @@ func main() {
 		// token/syntax error output. Allow specification of file in anyfin that
 		// can return a SyntaxError and have all token sources include that.
 		if syntaxErr, ok := err.(*types.SyntaxError); ok {
-			fmt.Fprintf(os.Stderr, "%s\n", syntaxErr.FullMessage())
-			returnCode = ExitErrSyntax
+			errSyntax("", syntaxErr.FullMessage())
 		} else {
-			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
-			returnCode = ExitErrOther
+			errOther(err.Error())
 		}
 		return
 	}
@@ -620,8 +568,7 @@ func main() {
 	fmt.Fprintf(os.Stderr, "\n")
 
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
-		returnCode = ExitErrParser
+		errParser(err.Error())
 		return
 	}
 
@@ -653,8 +600,7 @@ func main() {
 
 				err := fishi.ValidateSimulatedInput(spec, md, p, *flagHooksPath, *flagHooksTableName, *flagPathPrefix, cgOpts, &di)
 				if err != nil {
-					fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
-					returnCode = ExitErrGeneration
+					errGeneration(err.Error())
 					return
 				}
 			}
@@ -690,8 +636,7 @@ func main() {
 
 		err := fishi.GenerateDiagnosticsBinary(spec, md, p, *flagHooksPath, *flagHooksTableName, *flagDiagFormatPkg, formatCall, *flagPkg, *flagDiagBin, *flagPathPrefix, cgOpts)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
-			returnCode = ExitErrGeneration
+			errGeneration(err.Error())
 			return
 		}
 
@@ -714,8 +659,7 @@ func main() {
 	}
 	err = fishi.GenerateCompilerGo(spec, md, *flagPkg, feDest, &cgOpts)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
-		returnCode = ExitErrGeneration
+		errGeneration(err.Error())
 		return
 	}
 }
diff --git a/cmd/ictcc/status.go b/cmd/ictcc/status.go
new file mode 100644
index 0000000..5f44a8e
--- /dev/null
+++ b/cmd/ictcc/status.go
@@ -0,0 +1,122 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+const (
+	// ExitSuccess is the exit code for a successful run.
+	ExitSuccess = iota
+
+	// ExitErrNoFiles is the code returned as exit status when no files are
+	// provided to the invocation.
+	ExitErrNoFiles
+
+	// ExitErrInvalidFlags is used if the combination of flags specified is
+	// invalid.
+	ExitErrInvalidFlags
+
+	// ExitErrSyntax is the code returned as exit status when a syntax error
+	// occurs.
+	ExitErrSyntax
+
+	// ExitErrParser is the code returned as exit status when there is an error
+	// generating the parser.
+	ExitErrParser
+
+	// ExitErrGeneration is the code returned as exit status when there is an
+	// error creating the generated files.
+	ExitErrGeneration
+
+	// ExitErrOther is a generic error code for any other error.
+	ExitErrOther
+)
+
+var (
+	exitStatus = ExitSuccess
+)
+
+// errNoFiles sets the exit status to ExitErrNoFiles and prints the given error
+// message to stderr by calling exitErr.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func errNoFiles(msg string) {
+	exitErr(ExitErrNoFiles, msg)
+}
+
+// errInvalidFlags sets the exit status to ExitErrInvalidFlags and prints the
+// given error message to stderr by calling exitErr.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func errInvalidFlags(msg string) {
+	exitErr(ExitErrInvalidFlags, msg)
+}
+
+// errSyntax sets the exit status to ExitErrSyntax and prints the given error
+// message to stderr. Does *NOT* prepend with "ERROR: " in order to allow the
+// filename to be printed before the error message, but will still automatically
+// end the message with a newline. If filename is set to "", it will not be
+// prepended and the msg will be printed as-is.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func errSyntax(filename string, msg string) {
+	if filename != "" {
+		fmt.Fprintf(os.Stderr, "%s:\n", filename)
+	}
+	fmt.Fprintf(os.Stderr, "%s\n", msg)
+	exitStatus = ExitErrSyntax
+}
+
+// errParser sets the exit status to ExitErrParser and prints the given error
+// message to stderr by calling exitErr.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func errParser(msg string) {
+	exitErr(ExitErrParser, msg)
+}
+
+// errGeneration sets the exit status to ExitErrGeneration and prints the given
+// error message to stderr by calling exitErr.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func errGeneration(msg string) {
+	exitErr(ExitErrGeneration, msg)
+}
+
+// errOther sets the exit status to ExitErrOther and prints the given error
+// message to stderr by calling exitErr.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func errOther(msg string) {
+	exitErr(ExitErrOther, msg)
+}
+
+// exitErr sets the exit status and prints "ERROR: " followed by the given
+// error message to stderr. Automatically ends printed message with a newline.
+//
+// Caller is responsible for exiting main immediately after this function
+// returns.
+func exitErr(statusCode int, msg string) {
+	fmt.Fprintf(os.Stderr, "ERROR: %s\n", msg)
+	exitStatus = statusCode
+}
+
+// basic function to check if panic is happening and recover it while also
+// preserving possibly-set exit code. Immediately call this as defered as first
+// statement in main.
+func preservePanicOrExitWithStatus() {
+	if panicErr := recover(); panicErr != nil {
+		// we are panicking, make sure we dont lose the panic just because
+		// we checked
+		panic("unrecoverable panic occured")
+	} else {
+		os.Exit(exitStatus)
+	}
+}

From c97496c55ef4b5d402a94a26da710daa4a6009ee Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 09:46:01 -0500
Subject: [PATCH 28/33] #16: updated genfishi to use new args and diag format

---
 genfishi.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/genfishi.sh b/genfishi.sh
index f70507f..1264cd4 100644
--- a/genfishi.sh
+++ b/genfishi.sh
@@ -5,6 +5,8 @@
 ./ictcc --clr \
 	--ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \
 	--dest .testout \
-	-l fishi --lang-ver 1.0.0 \
+	-l fishi -v 1.0.0 \
 	--hooks fishi/syntax \
+	-d diag \
+	-f fishi/format \
 	fishi.md "$@"

From 34071e08c354ca306089d3a53e92e6cc31fbbfa3 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 09:54:23 -0500
Subject: [PATCH 29/33] #16: update diag template to properly refer to io

---
 cmd/ictcc/main.go            | 2 +-
 fishi/templates/main.go.tmpl | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 9eec843..51306dd 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -613,7 +613,7 @@ func main() {
 		if !*flagQuietMode {
 			// tell user if the diagnostic binary cannot do preformatting based
 			// on flags
-			if *flagDiagFormatPkg != "" {
+			if *flagDiagFormatPkg == "" {
 				fmt.Printf("Format preprocessing disabled in diagnostics bin; set -f to enable\n")
 			}
 
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index 7195502..4dfa5e9 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"bufio"
     "os"
+	"io"
 	"path/filepath"
 
 	"{{ .BinPkg }}/internal/{{ .HooksPkg }}"

From 1a5ce15f1482e99775afe93082da9341000eefa2 Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 10:07:31 -0500
Subject: [PATCH 30/33] #16: clean output bin

---
 fishi/cgstructs.go           |  2 ++
 fishi/codegen.go             |  7 ++++--
 fishi/templates/main.go.tmpl | 44 ++++++++++++++++++------------------
 genfishi.sh                  |  2 +-
 4 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/fishi/cgstructs.go b/fishi/cgstructs.go
index 7a9d49d..4d016ae 100644
--- a/fishi/cgstructs.go
+++ b/fishi/cgstructs.go
@@ -4,6 +4,8 @@ import "github.com/dekarrin/ictiobus"
 
 // File cgstructs.go contains structs used as part of code generation.
 
+// TODO: move most structs from codegen to here.
+
 // MainBinaryParams is paramters for generating a main.go file for a binary.
 // Unless otherwise specified, all fields are required.
 type MainBinaryParams struct {
diff --git a/fishi/codegen.go b/fishi/codegen.go
index c442aff..bd4e677 100644
--- a/fishi/codegen.go
+++ b/fishi/codegen.go
@@ -337,7 +337,8 @@ func GenerateBinaryMainGo(spec Spec, md SpecMetadata, params MainBinaryParams) (
 	mainFillData := cgMainData{
 		BinPkg:            "github.com/dekarrin/ictiobus/langexec/" + safePkgIdent(md.Language),
 		BinName:           params.BinName,
-		BinVersion:        md.Version,
+		Version:           md.Version,
+		Lang:              md.Language,
 		HooksPkg:          hooksPkgName,
 		HooksTableExpr:    params.HooksExpr,
 		FormatPkg:         formatPkgName,
@@ -720,10 +721,12 @@ type codegenTemplate struct {
 }
 
 // codegen data for template fill of main.go
+// TODO: combine with cgData?
 type cgMainData struct {
 	BinPkg            string
 	BinName           string
-	BinVersion        string
+	Version           string
+	Lang              string
 	HooksPkg          string
 	HooksTableExpr    string
 	ImportFormatPkg   bool
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index 4dfa5e9..699a32a 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -43,7 +43,7 @@ const (
 )
 
 const (
-	versionString = "{{ .BinName }} {{ .BinVersion }}"
+	versionString = "{{ .BinName }} {{ .Version }}{{if ne .Lang "Unspecified" }} (for {{ .Lang }}){{end}}"
 )
 
 var (
@@ -52,16 +52,16 @@ var (
 
 // flags
 var (
-	quietMode         *bool = pflag.BoolP("quiet", "q", false, "Quiet mode; disables output of the IR")
-	lexerTrace        *bool = pflag.BoolP("debug-lexer", "l", false, "Print the lexer trace to stdout")
-	parserTrace       *bool = pflag.BoolP("debug-parser", "p", false, "Print the parser trace to stdout") 
-	printTrees        *bool = pflag.Bool("t", false, "Print the parse trees of each file read to stdout")
-	version			  *bool = pflag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
-	doSimulation      *bool = pflag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
-	simSDTSShowTrees  *bool = pflag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
-	simSDTSShowGraphs *bool = pflag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
-	simSDTSFirstOnly  *bool = pflag.Bool("sim-first-errs", false, "Show only the first error found in SDTS validation of simulated trees")
-	simSDTSSkip       *int  = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
+	flagQuietMode		= pflag.BoolP("quiet", "q", false, "Quiet mode; disables output of the IR")
+	flagLexerTrace		= pflag.BoolP("debug-lexer", "l", false, "Print the lexer trace to stdout")
+	flagParserTrace		= pflag.BoolP("debug-parser", "p", false, "Print the parser trace to stdout") 
+	flagPrintTrees		= pflag.Bool("t", false, "Print the parse trees of each file read to stdout")
+	flagVersion			= pflag.Bool("version", false, "Print the version of {{ .BinName }} and exit")
+	flagSim				= pflag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit")
+	flagSimTrees		= pflag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors")
+	flagSimGraphs 		= pflag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors")
+	flagSimFirstErr  	= pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation of simulated trees")
+	flagSimSkipErrs		= pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees")
 )
 
 func main() {
@@ -79,20 +79,20 @@ func main() {
 
 	pflag.Parse()
 
-	if *version {
+	if *flagVersion {
 		fmt.Println(versionString)
 		return
 	}
 
 	opts := {{ .FrontendPkg }}.FrontendOptions{
-		LexerTrace: *lexerTrace,
-		ParserTrace: *parserTrace,
+		LexerTrace: *flagLexerTrace,
+		ParserTrace: *flagParserTrace,
 	}
 
 	hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
 	langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, &opts)
 
-	if *doSimulation {
+	if *flagSim {
 		hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }}
 
 		langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil)
@@ -101,10 +101,10 @@ func main() {
 		valProd := langFront.Lexer.FakeLexemeProducer(true, "")
 
 		di := trans.ValidationOptions{
-			ParseTrees:    *simSDTSShowTrees,
-			FullDepGraphs: *simSDTSShowGraphs,
-			ShowAllErrors: !*simSDTSFirstOnly,
-			SkipErrors:    *simSDTSSkip,
+			ParseTrees:    *flagSimTrees,
+			FullDepGraphs: *flagSimGraphs,
+			ShowAllErrors: !*flagSimFirstErr,
+			SkipErrors:    *flagSimSkipErrs,
 		}
 
 		sddErr := langFront.SDT.Validate(langFront.Parser.Grammar(), langFront.IRAttribute, di, valProd)
@@ -114,7 +114,7 @@ func main() {
 			return
 		}
 
-		if !*quietMode {
+		if !*flagQuietMode {
 			fmt.Printf("Simulation completed with no errors\n")
 		}
 		return
@@ -151,7 +151,7 @@ func main() {
 		ir, pt, err := langFront.Analyze(r)
 
 		// parse tree might be valid no matter what, so we print it first
-		if *printTrees {
+		if *flagPrintTrees {
 			fmt.Println(pt.String())
 		}
 
@@ -161,7 +161,7 @@ func main() {
 			return
 		}
 
-		if !*quietMode {
+		if !*flagQuietMode {
 			fmt.Printf("=== Analysis of %s ===\n%s\n\n", filepath.Base(f), ir)
 		}
 	}
diff --git a/genfishi.sh b/genfishi.sh
index 1264cd4..185b44f 100644
--- a/genfishi.sh
+++ b/genfishi.sh
@@ -7,6 +7,6 @@
 	--dest .testout \
 	-l fishi -v 1.0.0 \
 	--hooks fishi/syntax \
-	-d diag \
+	-d fishic \
 	-f fishi/format \
 	fishi.md "$@"

From a73ad41ea6ff16741f7d65ec4bce2c274d34018f Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 10:09:06 -0500
Subject: [PATCH 31/33] #16: Proper name of language is FISHI not fishi

---
 genfishi.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/genfishi.sh b/genfishi.sh
index 185b44f..92b1262 100644
--- a/genfishi.sh
+++ b/genfishi.sh
@@ -5,7 +5,7 @@
 ./ictcc --clr \
 	--ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \
 	--dest .testout \
-	-l fishi -v 1.0.0 \
+	-l FISHI -v 1.0.0 \
 	--hooks fishi/syntax \
 	-d fishic \
 	-f fishi/format \

From ccafd194717dfa4e9c8df6e6e41c2233c68790ff Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 11:00:33 -0500
Subject: [PATCH 32/33] #16: need to commit all to allow fishic to use new
 error msg

---
 cmd/ictcc/main.go            |  4 ++--
 cmd/ictcc/status.go          | 17 ++++++++---------
 fishi.md                     |  2 +-
 fishi/fe/bootstrap_lexer.go  |  1 -
 fishi/templates/main.go.tmpl | 10 ++++++++--
 types/syntaxerror.go         | 14 ++++++++++++++
 6 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go
index 51306dd..6e1f0a2 100644
--- a/cmd/ictcc/main.go
+++ b/cmd/ictcc/main.go
@@ -477,7 +477,7 @@ func main() {
 			}
 
 			if syntaxErr, ok := err.(*types.SyntaxError); ok {
-				errSyntax(file, syntaxErr.FullMessage())
+				errSyntax(file, syntaxErr)
 			} else {
 				errOther(fmt.Sprintf("%s: %s", file, err.Error()))
 			}
@@ -518,7 +518,7 @@ func main() {
 		// token/syntax error output. Allow specification of file in anyfin that
 		// can return a SyntaxError and have all token sources include that.
 		if syntaxErr, ok := err.(*types.SyntaxError); ok {
-			errSyntax("", syntaxErr.FullMessage())
+			errSyntax("", syntaxErr)
 		} else {
 			errOther(err.Error())
 		}
diff --git a/cmd/ictcc/status.go b/cmd/ictcc/status.go
index 5f44a8e..ea7feeb 100644
--- a/cmd/ictcc/status.go
+++ b/cmd/ictcc/status.go
@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"os"
+
+	"github.com/dekarrin/ictiobus/types"
 )
 
 const (
@@ -55,19 +57,16 @@ func errInvalidFlags(msg string) {
 	exitErr(ExitErrInvalidFlags, msg)
 }
 
-// errSyntax sets the exit status to ExitErrSyntax and prints the given error
-// message to stderr. Does *NOT* prepend with "ERROR: " in order to allow the
-// filename to be printed before the error message, but will still automatically
-// end the message with a newline. If filename is set to "", it will not be
-// prepended and the msg will be printed as-is.
+// errSyntax sets the exit status to ExitErrSyntax and prints an error message
+// given by the syntax error to stderr.
 //
 // Caller is responsible for exiting main immediately after this function
 // returns.
-func errSyntax(filename string, msg string) {
-	if filename != "" {
-		fmt.Fprintf(os.Stderr, "%s:\n", filename)
+func errSyntax(filename string, synErr *types.SyntaxError) {
+	if filename == "" {
+		filename = "<INPUT>"
 	}
-	fmt.Fprintf(os.Stderr, "%s\n", msg)
+	fmt.Fprintf(os.Stderr, "%s\n", synErr.MessageForFile(filename))
 	exitStatus = ExitErrSyntax
 }
 
diff --git a/fishi.md b/fishi.md
index db24cd2..ed73394 100644
--- a/fishi.md
+++ b/fishi.md
@@ -235,7 +235,7 @@ For tokens state:
 %!%[Tt][Oo][Kk][Ee][Nn]                            %token dir-token
 %human %!%token directive
 
-%!%![Dd][Ii][Ss][Cc][Aa][Rr][Dd]                   %token dir-discard
+%!%[Dd][Ii][Ss][Cc][Aa][Rr][Dd]                   %token dir-discard
 %human %!%discard directive
 
 %!%[Pp][Rr][Ii][Oo][Rr][Ii][Tt][Yy]                %token dir-priority
diff --git a/fishi/fe/bootstrap_lexer.go b/fishi/fe/bootstrap_lexer.go
index e9b053a..10d6dab 100644
--- a/fishi/fe/bootstrap_lexer.go
+++ b/fishi/fe/bootstrap_lexer.go
@@ -44,7 +44,6 @@ func CreateBootstrapLexer() ictiobus.Lexer {
 	//bootLx.RegisterClass(tcDirDefault, "tokens")
 	bootLx.RegisterClass(TCDirDiscard, "tokens")
 	bootLx.RegisterClass(TCDirPriority, "tokens")
-	bootLx.RegisterClass(TCInt, "tokens")
 	bootLx.RegisterClass(TCDirState, "tokens")
 	bootLx.RegisterClass(TCLineStartFreeformText, "tokens")
 	bootLx.RegisterClass(TCLineStartEscseq, "tokens")
diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl
index 699a32a..a720bed 100644
--- a/fishi/templates/main.go.tmpl
+++ b/fishi/templates/main.go.tmpl
@@ -31,6 +31,7 @@ import (
 {{- end}}
 
 	"github.com/dekarrin/ictiobus/trans"
+	"github.com/dekarrin/ictiobus/types"
 
 	"github.com/spf13/pflag"
 )
@@ -156,8 +157,13 @@ func main() {
 		}
 
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
-			returnCode = ExitErrSyntax
+			if syntaxErr, ok := err.(*types.SyntaxError); ok {
+				fmt.Fprintf(os.Stderr, "%s\n", syntaxErr.MessageForFile(f))
+				returnCode = ExitErrSyntax
+			} else {
+				fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err)
+				returnCode = ExitErr
+			}
 			return
 		}
 
diff --git a/types/syntaxerror.go b/types/syntaxerror.go
index b8b1a4e..0f72ca1 100644
--- a/types/syntaxerror.go
+++ b/types/syntaxerror.go
@@ -57,6 +57,20 @@ func (se SyntaxError) FullMessage() string {
 	return errMsg
 }
 
+// MessageForFile returns the full error message in the format of
+// filename:line:pos: message, followed by the syntax error itself.
+func (se SyntaxError) MessageForFile(filename string) string {
+	var msg string
+
+	if se.line != 0 {
+		msg = fmt.Sprintf("%s:%d:%d: %s\n%s", filename, se.line, se.pos, se.message, se.SourceLineWithCursor())
+	} else {
+		msg = fmt.Sprintf("%s: %s", filename, msg)
+	}
+
+	return msg
+}
+
 // SourceLineWithCursor returns the source offending code on one line and
 // directly under it a cursor showing where the error occured.
 //

From f56ecb62f09756041ae9323bee330fe8f91fa26f Mon Sep 17 00:00:00 2001
From: dekarrin <dekarrin.irl@gmail.com>
Date: Sat, 15 Apr 2023 11:01:11 -0500
Subject: [PATCH 33/33] #16: version bump prior to merge for external use

---
 cmd/ictcc/version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/ictcc/version.go b/cmd/ictcc/version.go
index 1c72ccb..83fc221 100644
--- a/cmd/ictcc/version.go
+++ b/cmd/ictcc/version.go
@@ -1,7 +1,7 @@
 package main
 
 const (
-	Version = "0.6.0+dev"
+	Version = "0.6.1"
 )
 
 func GetVersionString() string {