diff --git a/.gitignore b/.gitignore index fe22f3b..a5c1ad3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,12 @@ # vim *.swp -/testout +/.testout /ictcc /.sim +/.gen +/diag +/fishic # mac .DS_Store diff --git a/cmd/ictcc/main.go b/cmd/ictcc/main.go index be8558a..6e1f0a2 100644 --- a/cmd/ictcc/main.go +++ b/cmd/ictcc/main.go @@ -11,119 +11,177 @@ Usage: Flags: - -a/-ast + -a/--ast Print the AST to stdout before generating the frontend. - -t/-tree + -t/--tree Print the parse tree to stdout before generating the frontend. - -s/-spec + -s/--spec Print the interpreted language specification to stdout before generating the frontend. - -n/-no-gen + -n/--no-gen Do not generate the parser. If this flag is set, the fishi is parsed and checked for errors but no other action is taken (unless specified by other flags). - -p/-parser FILE + -d/--diag FILE + Generate a diagnostics binary for the target language. Assuming there + are no issues with the FISHI spec, this will generate a binary that can + analyze files written in the target language and output the result of + frontend analysis. This can be useful for testing out the frontend on + files quickly and efficiently, as it also includes further options + useful for debugging purposes, such as debugging lexed tokens and the + parser itself. Note that by default the diagnostics binary will only + accept text input in the language accepted by the specified frontend; to + allow it to perform reading of specialized formats and/or perform + preprocessing, use the -f/--diag-format-pkg flag. + + -q/--quiet + Do not show progress messages. This does not affect error messages or + warning output. + + -p/--parser FILE Set the location of the pre-compiled parser cache to the given CFF format file as opposed to the default of './parser.cff'. - -no-cache + -f/--diag-format-pkg DIR + Enable special format reading in the generated diagnostics binary by + wrapping any io.Reader opened on input files in another io.Reader that + handles reading the format of the input file. This is performed by + calling a function in the package located in the specified file, by + default this function is called 'NewCodeReader' but can be changed by + specifying the --diag-format-call flag. The function must take an + io.Reader and return a new io.Reader that reads source code from the + given io.Reader and performs any preprocessing required on it. This + allows the diagnostics binary to read files that are not simply text + files directly ready to be accepted by the frontend. If not set, the + diagnostics binary will not perform any preprocessing on input files and + assumes that any input can be directly accepted by the frontend. This + flag is only useful if -d/--diag is also set. + + -c/--diag-format-call NAME + Set the name of the function to call in the package specified by + -f/--diag-format-pkg to get an io.Reader that can read specialized + formats. Defaults to 'NewCodeReader'. This function is used by the + diagnostics binary to do format reading and preprocessing on input prior + to analysis by the frontend. This flag is only useful if -d/--diag is + also set. + + -l/--lang NAME + Set the name of the language to generate a frontend for. Defaults to + "Unspecified". + + --lang-ver VERSION + Set the version of the language to generate a frontend for. Defaults to + "v0.0.0". + + --preserve-bin-source + Do not delete source files for any generated binary after compiling the + binary. + + --no-cache Disable the loading of any cached frontend components, even if a pre-built one is available. - -no-cache-out + --no-cache-out Disable writing of any frontend components cache, even if a component was built by the invocation. - -version + --version Print the version of the ictiobus compiler-compiler and exit. - -val-sdts-off - Disable validatione of the SDTS of the resulting fishi. + --sim-off + Disable simulation of the language once built. This will disable SDTS + validation, as live simulation is the only way to do this due to the + lack of support for dynamic loading of the hooks package in Go. - -val-sdts-trees + --sim-trees If problems are detected with the SDTS of the resulting fishi during SDTS validation, show the parse tree(s) that caused the problem. Has no effect if -val-sdts-off is set. - -val-sdts-graphs + --sim-graphs If problems are detected with the SDTS of the resulting fishi during SDTS validation, show the full resulting dependency graph(s) that caused the issue (if any). Has no effect if -val-sdts-off is set. - -val-sdts-first + --sim-first-err If problems are detected with the SDTS of the resulting fishi during SDTS validation, show only the problem(s) found in the first simulated parse tree (after any skipped by -val-sdts-skip) and then stop. Has no effect if -val-sdts-off is set. - -val-sdts-skip N + --sim-skip-errs N If problems are detected with the SDTS of the resulting fishi during SDTS validation, skip the first N simulated parse trees in the output. Combine with -val-sdts-first to view a specific parse tree. Has no effect if -val-sdts-off is set. - -debug-lexer + --debug-lexer Enable debug mode for the lexer and print each token to standard out as it is lexed. Note that if the lexer is not in lazy mode, all tokens will be lexed before any parsing begins, and so with debug-lexer enabled will all be printed to stdout before any parsing begins. - -debug-parser + --debug-parser Enable debug mode for the parser and print each step of the parse to stdout, including the symbol stack, manipulations of the stack, ACTION selected in DFA based on the stack, and other information. - -pkg NAME + --pkg NAME Set the name of the package to place generated files in. Defaults to 'fe'. - -dest DIR + --dest DIR Set the destination directory to place generated files in. Defaults to a directory named 'fe' in the current working directory. - -l/lang NAME - Set the name of the language to generate a frontend for. Defaults to - "Unspecified". - - -lang-ver VERSION - Set the version of the language to generate a frontend for. Defaults to - "v0.0.0". + --prefix PATH + Set the prefix to use for all generated source files. Defaults to the + current working directory. If used, generated source files will be be + output to their location with this prefix instead of in a directory + (".sim", ".gen", and the generated frontend source package folder) + located in the current working directory. Combine with + --preserve-bin-source to aid in debugging. Does not affect diagnostic + binary output location. - -pre-format + --debug-templates Enable dumping of the fishi filled template files before they are passed to the formatter. This allows debugging of the template files when editing them, since they must be valid go code to be formatted. - -tmpl-tokens FILE + --tmpl-tokens FILE Use the provided file as the template for outputting the generated tokens file instead of the default embedded within the binary. - -tmpl-lexer FILE + --tmpl-lexer FILE Use the provided file as the template for outputting the generated lexer file instead of the default embedded within the binary. - -tmpl-parser FILE + --tmpl-parser FILE Use the provided file as the template for outputting the generated parser file instead of the default embedded within the binary. - -tmpl-sdts FILE + --tmpl-sdts FILE Use the provided file as the template for outputting the generated SDTS file instead of the default embedded within the binary. - -tmpl-frontend FILE + --tmpl-main FILE + Use the provided file as the template for outputting generated binary + main file instead of the default embedded within the binary. + + --tmpl-frontend FILE Use the provided file as the template for outputting the generated frontend file instead of the default embedded within the binary. - -no-ambig + --no-ambig Disable the generation of a parser for an ambiguous language. Normally, when generating an LR parser, an ambiguous grammar is allowed, with shift-reduce conflicts resolved in favor of shift in all cases. If this @@ -132,32 +190,32 @@ Flags: ambiguous, so this flag has no effect if explicitly selecting an LL parser. - -ll + --ll Force the generation of an LL(1) parser instead of the default of trying each parser type in sequence from most restrictive to least restrictive and using the first one found. - -slr + --slr Force the generation of an SLR(1) (simple LR) parser instead of the default of trying each parser type in sequence from most restrictive to least restrictive and using the first one found. - -clr + --clr Force the generation of a CLR(1) (canonical LR) parser instead of the default of trying each parser type in sequence from most restrictive to least restrictive and using the first one found. - -lalr + --lalr Force the generation of a LALR(1) (lookahead LR) parser instead of the default of trying each parser type in sequence from most restrictive to least restrictive and using the first one found. - -hooks PATH + --hooks PATH Gives the filesystem path to the directory containing the package that the hooks table is in. This is required for live validation of simulated parse trees, but may be omitted if SDTS validation is disabled. - -hooks-table NAME + --hooks-table NAME Gives the expression to retrieve the hooks table from the hooks package, relative to the package that it is in. The NAME must be an exported var of type map[string]trans.AttributeSetter, or a function call that @@ -165,7 +223,7 @@ Flags: expression; it will be automatically determined. The default value is "HooksTable". - -ir TYPE + --ir TYPE Gives the type of the intermediate representation returned by applying the translation scheme to a parse tree. This is required for running SDTS validation on simulated parse trees, but may be omitted if SDTS @@ -183,8 +241,8 @@ separately but their resulting ASTs are combined into a single FISHI spec for a language. The spec is then used to generate a lexer, parser, and SDTS for the language. -For the parser, if no specific parser is selected via the -ll, -slr, -clr, or --lalr flags, the parser generator will attempt to generate each type of parser +For the parser, if no specific parser is selected via the --ll, --slr, --clr, or +--lalr flags, the parser generator will attempt to generate each type of parser in sequence from most restrictive to least restrictive (LL(1), simple LR(1), lookahead LR(1), and canonical LR(1), in that order), and use the first one it is able to generate. If a specific parser is selected, it will attempt to @@ -200,15 +258,15 @@ correct exit code for the last file parsed. If files containing cached pre-built components of the frontend are available, they will be loaded and used unless -no-cache is set. The files are named -'fishi-parser.cff' by default, and the names can be changed with the -parser/-p +'fishi-parser.cff' by default, and the names can be changed with the --parser/-p flag if desired. Cache invalidation is non-sophisticated and cannot be automatically detected at this time. To force it to occur, the -no-cache flag must be manually used (or the file deleted). If new frontend components are generated from scratch, they will be cached by -saving them to the files mentioned above unless -no-cache-out is set. Note that +saving them to the files mentioned above unless --no-cache-out is set. Note that if the frontend components are loaded from cache files, they will not be output -to cache files again regardless of whether -no-cache-out is present. +to cache files again regardless of whether --no-cache-out is present. Once the input has been successfully parsed, the parser is generated using the options provided, unless the -n flag is set, in which case ictcc will @@ -217,11 +275,13 @@ immediately exit with a success code after parsing the input. package main import ( - "flag" "fmt" "os" + "path/filepath" "strings" + "github.com/spf13/pflag" + "github.com/dekarrin/ictiobus" "github.com/dekarrin/ictiobus/fishi" "github.com/dekarrin/ictiobus/grammar" @@ -232,183 +292,154 @@ import ( "github.com/dekarrin/rosed" ) -const ( - // ExitSuccess is the exit code for a successful run. - ExitSuccess = iota - - // ExitErrNoFiles is the code returned as exit status when no files are - // provided to the invocation. - ExitErrNoFiles - - // ExitErrInvalidFlags is used if the combination of flags specified is - // invalid. - ExitErrInvalidFlags - - // ExitErrSyntax is the code returned as exit status when a syntax error - // occurs. - ExitErrSyntax - - // ExitErrParser is the code returned as exit status when there is an error - // generating the parser. - ExitErrParser - - // ExitErrGeneration is the code returned as exit status when there is an - // error creating the generated files. - ExitErrGeneration - - // ExitErrOther is a generic error code for any other error. - ExitErrOther -) - -var ( - returnCode = ExitSuccess -) - var ( - noGen bool - genAST bool - genTree bool - genSpec bool - parserCff string - lang string - dumpPreFormat *bool = flag.Bool("pre-format", false, "Dump the generated code before running through gofmt") - pkg *string = flag.String("pkg", "fe", "The name of the package to place generated files in") - dest *string = flag.String("dest", "./fe", "The name of the directory to place the generated package in") - langVer *string = flag.String("lang-ver", "v0.0.0", "The version of the language to generate") - noCache *bool = flag.Bool("no-cache", false, "Disable use of cached frontend components, even if available") - noCacheOutput *bool = flag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated") - - valSDTSOff *bool = flag.Bool("val-sdts-off", false, "Disable validation of the SDTS of the resulting fishi") - valSDTSShowTrees *bool = flag.Bool("val-sdts-trees", false, "Show trees that caused SDTS validation errors") - valSDTSShowGraphs *bool = flag.Bool("val-sdts-graphs", false, "Show full generated dependency graph output for parse trees that caused SDTS validation errors") - valSDTSFirstOnly *bool = flag.Bool("val-sdts-first", false, "Show only the first error found in SDTS validation") - valSDTSSkip *int = flag.Int("val-sdts-skip", 0, "Skip the first N errors found in SDTS validation in output") - - tmplTokens *string = flag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with") - tmplLexer *string = flag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with") - tmplParser *string = flag.String("tmpl-parser", "", "A template file to replace the embedded parser template with") - tmplSDTS *string = flag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with") - tmplFront *string = flag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with") - - parserLL *bool = flag.Bool("ll", false, "Generate an LL(1) parser") - parserSLR *bool = flag.Bool("slr", false, "Generate a simple LR(1) parser") - parserCLR *bool = flag.Bool("clr", false, "Generate a canonical LR(1) parser") - parserLALR *bool = flag.Bool("lalr", false, "Generate a canonical LR(1) parser") - parserNoAmbig *bool = flag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it") - - lexerTrace *bool = flag.Bool("debug-lexer", false, "Print the lexer trace to stdout") - parserTrace *bool = flag.Bool("debug-parser", false, "Print the parser trace to stdout") - - hooksPath *string = flag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.") - hooksTableName *string = flag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.") - - irType *string = flag.String("ir", "", "The fully-qualified type of IR to generate.") - - version *bool = flag.Bool("version", false, "Print the version of ictcc and exit") + flagQuietMode = pflag.BoolP("quiet", "q", false, "Suppress progress messages and other supplementary output") + flagNoGen = pflag.BoolP("no-gen", "n", false, "Do not attempt to generate the parser") + flagGenAST = pflag.BoolP("ast", "a", false, "Print the AST of the analyzed fishi") + flagGenTree = pflag.BoolP("tree", "t", false, "Print the parse trees of each analyzed fishi file") + flagShowSpec = pflag.BoolP("spec", "s", false, "Print the FISHI spec interpreted from the analyzed fishi") + flagParserCff = pflag.StringP("parser", "p", "fishi-parser.cff", "Use the specified parser CFF cache file instead of default") + flagLang = pflag.StringP("lang", "l", "Unspecified", "The name of the languae being generated") + flagLangVer = pflag.StringP("lang-ver", "v", "v0.0.0", "The version of the language to generate") + + flagDiagBin = pflag.StringP("diag", "d", "", "Generate binary that has the generated frontend and uses it to analyze the target language") + flagDiagFormatPkg = pflag.StringP("diag-format-pkg", "f", "", "The package containing format functions for the diagnostic binary to call on input prior to passing to frontend analysis") + flagDiagFormatCall = pflag.StringP("diag-format-call", "c", "NewCodeReader", "The function within the diag-format-pkg to call to open a reader on input prior to passing to frontend analysis") + + flagPathPrefix = pflag.String("prefix", "", "Path to prepend to path of all generated source files") + flagPreserveBinSource = pflag.Bool("preserve-bin-source", false, "Preserve the source of any generated binary files") + flagDebugTemplates = pflag.Bool("debug-templates", false, "Dump the filled templates before running through gofmt") + flagPkg = pflag.String("pkg", "fe", "The name of the package to place generated files in") + flagDest = pflag.String("dest", "./fe", "The name of the directory to place the generated package in") + flagNoCache = pflag.Bool("no-cache", false, "Disable use of cached frontend components, even if available") + flagNoCacheOutput = pflag.Bool("no-cache-out", false, "Disable writing of cached frontend components, even if one was generated") + + flagSimOff = pflag.Bool("sim-off", false, "Disable input simulation of the language once built") + flagSimTrees = pflag.Bool("sim-trees", false, "Show parse trees that caused errors during simulation") + flagSimGraphs = pflag.Bool("sim-graphs", false, "Show full generated dependency graph output for parse trees that caused errors during simulation") + flagSimFirstErrOnly = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation") + flagSimSkipErrs = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation in output") + + flagTmplTokens = pflag.String("tmpl-tokens", "", "A template file to replace the embedded tokens template with") + flagTmplLexer = pflag.String("tmpl-lexer", "", "A template file to replace the embedded lexer template with") + flagTmplParser = pflag.String("tmpl-parser", "", "A template file to replace the embedded parser template with") + flagTmplSDTS = pflag.String("tmpl-sdts", "", "A template file to replace the embedded SDTS template with") + flagTmplFront = pflag.String("tmpl-frontend", "", "A template file to replace the embedded frontend template with") + flagTmplMain = pflag.String("tmpl-main", "", "A template file to replace the embedded main.go template with") + + flagParserLL = pflag.Bool("ll", false, "Generate an LL(1) parser") + flagParserSLR = pflag.Bool("slr", false, "Generate a simple LR(1) parser") + flagParserCLR = pflag.Bool("clr", false, "Generate a canonical LR(1) parser") + flagParserLALR = pflag.Bool("lalr", false, "Generate a canonical LR(1) parser") + flagParserNoAmbig = pflag.Bool("no-ambig", false, "Disallow ambiguity in grammar even if creating a parser that can auto-resolve it") + + flagLexerTrace = pflag.Bool("debug-lexer", false, "Print the lexer trace to stdout") + flagParserTrace = pflag.Bool("debug-parser", false, "Print the parser trace to stdout") + + flagHooksPath = pflag.String("hooks", "", "The path to the hooks directory to use for the generated parser. Required for SDTS validation.") + flagHooksTableName = pflag.String("hooks-table", "HooksTable", "Function call or name of exported var in 'hooks' that has the hooks table.") + + flagIRType = pflag.String("ir", "", "The fully-qualified type of IR to generate.") + + flagVersion = pflag.Bool("version", false, "Print the version of ictcc and exit") ) -func init() { - const ( - noGenUsage = "Do not generate the parser" - genASTUsage = "Print the AST of the analyzed fishi" - genTreeUsage = "Print the parse trees of each analyzed fishi file" - genSpecUsage = "Print the FISHI spec interpreted from the analyzed fishi" - parserCffUsage = "Use the specified parser CFF cache file instead of default" - parserCffDefault = "fishi-parser.cff" - langUsage = "The name of the languae being generated" - langDefault = "Unspecified" - ) - flag.BoolVar(&noGen, "no-gen", false, noGenUsage) - flag.BoolVar(&noGen, "n", false, noGenUsage+" (shorthand)") - flag.BoolVar(&genAST, "ast", false, genASTUsage) - flag.BoolVar(&genAST, "a", false, genASTUsage+" (shorthand)") - flag.BoolVar(&genSpec, "spec", false, genSpecUsage) - flag.BoolVar(&genSpec, "s", false, genSpecUsage+" (shorthand)") - flag.BoolVar(&genTree, "tree", false, genTreeUsage) - flag.BoolVar(&genTree, "t", false, genTreeUsage+" (shorthand)") - flag.StringVar(&parserCff, "parser", parserCffDefault, parserCffUsage) - flag.StringVar(&parserCff, "p", parserCffDefault, parserCffUsage+" (shorthand)") - flag.StringVar(&lang, "lang", langDefault, langUsage) - flag.StringVar(&lang, "l", langDefault, langUsage+"(shorthand)") -} - func main() { - // basic function to check if panic is happening and recover it while also - // preserving possibly-set exit code. - defer func() { - if panicErr := recover(); panicErr != nil { - // we are panicking, make sure we dont lose the panic just because - // we checked - panic("unrecoverable panic occured") - } else { - os.Exit(returnCode) - } - }() + defer preservePanicOrExitWithStatus() // gather options and arguments invocation := strings.Join(os.Args[1:], " ") - flag.Parse() + pflag.Parse() - if *version { + if *flagVersion { fmt.Println(GetVersionString()) return } - // create a spec metadata object - md := fishi.SpecMetadata{ - Language: lang, - Version: *langVer, - InvocationArgs: invocation, + // mutually exclusive and required options for diagnostics bin generation. + if *flagDiagBin != "" { + if *flagNoGen { + errInvalidFlags("Diagnostics bin generation cannot be enabled due to -n/--no-gen") + return + } else if *flagIRType == "" || *flagHooksPath == "" { + errInvalidFlags("Diagnostics bin generation requires both --ir and --hooks to be set") + return + } + + // you cannot set ONLY the formatting call + flagInfoDiagFormatCall := pflag.Lookup("diag-format-call") + // don't error check; all we'd do is panic + if flagInfoDiagFormatCall.Changed && *flagDiagFormatPkg == "" { + errInvalidFlags("-c/--diag-format-call cannot be set without -f/--diag-format-pkg") + return + } + } else { + // otherwise, it makes no sense to set --diag-format-pkg or --diag-format-call; disallow this + flagInfoDiagFormatPkg := pflag.Lookup("diag-format-pkg") + flagInfoDiagFormatCall := pflag.Lookup("diag-format-call") + if flagInfoDiagFormatPkg.Changed { + errInvalidFlags("-f/--diag-format-pkg cannot be set without -d/--diagnostics-bin") + return + } + if flagInfoDiagFormatCall.Changed { + errInvalidFlags("-c/--diag-format-call cannot be set without -d/--diagnostics-bin") + return + } } - args := flag.Args() + parserType, allowAmbig, err := parserSelectionFromFlags() + if err != nil { + errInvalidFlags(err.Error()) + return + } + + // check args before gathering flags + args := pflag.Args() if len(args) < 1 { - fmt.Fprintf(os.Stderr, "No files given to process") - returnCode = ExitErrNoFiles + errNoFiles("No files given to process") return } - parserType, allowAmbig, err := parserSelectionFromFlags() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - returnCode = ExitErrInvalidFlags - return + // create a spec metadata object + md := fishi.SpecMetadata{ + Language: *flagLang, + Version: *flagLangVer, + InvocationArgs: invocation, } fo := fishi.Options{ - ParserCFF: parserCff, - ReadCache: !*noCache, - WriteCache: !*noCacheOutput, - SDTSValidate: !*valSDTSOff, - SDTSValShowTrees: *valSDTSShowTrees, - SDTSValShowGraphs: *valSDTSShowGraphs, - SDTSValAllTrees: !*valSDTSFirstOnly, - SDTSValSkipTrees: *valSDTSSkip, - LexerTrace: *lexerTrace, - ParserTrace: *parserTrace, + ParserCFF: *flagParserCff, + ReadCache: !*flagNoCache, + WriteCache: !*flagNoCacheOutput, + LexerTrace: *flagLexerTrace, + ParserTrace: *flagParserTrace, } cgOpts := fishi.CodegenOptions{ - DumpPreFormat: *dumpPreFormat, - IRType: *irType, - TemplateFiles: map[string]string{}, + DumpPreFormat: *flagDebugTemplates, + IRType: *flagIRType, + TemplateFiles: map[string]string{}, + PreserveBinarySource: *flagPreserveBinSource, } - if *tmplTokens != "" { - cgOpts.TemplateFiles[fishi.ComponentTokens] = *tmplTokens + if *flagTmplTokens != "" { + cgOpts.TemplateFiles[fishi.ComponentTokens] = *flagTmplTokens } - if *tmplLexer != "" { - cgOpts.TemplateFiles[fishi.ComponentLexer] = *tmplLexer + if *flagTmplLexer != "" { + cgOpts.TemplateFiles[fishi.ComponentLexer] = *flagTmplLexer } - if *tmplParser != "" { - cgOpts.TemplateFiles[fishi.ComponentParser] = *tmplParser + if *flagTmplParser != "" { + cgOpts.TemplateFiles[fishi.ComponentParser] = *flagTmplParser } - if *tmplSDTS != "" { - cgOpts.TemplateFiles[fishi.ComponentSDTS] = *tmplSDTS + if *flagTmplSDTS != "" { + cgOpts.TemplateFiles[fishi.ComponentSDTS] = *flagTmplSDTS } - if *tmplFront != "" { - cgOpts.TemplateFiles[fishi.ComponentFrontend] = *tmplFront + if *flagTmplFront != "" { + cgOpts.TemplateFiles[fishi.ComponentFrontend] = *flagTmplFront + } + if *flagTmplMain != "" { + cgOpts.TemplateFiles[fishi.ComponentMainFile] = *flagTmplMain } if len(cgOpts.TemplateFiles) == 0 { // just nil it @@ -416,6 +447,10 @@ func main() { } // now that args are gathered, parse markdown files into an AST + if !*flagQuietMode { + files := textfmt.Pluralize(len(args), "FISHI input file", "-s") + fmt.Printf("Reading %s...\n", files) + } var joinedAST *fishi.AST for _, file := range args { @@ -431,22 +466,20 @@ func main() { // parse tree is per-file, so we do this immediately even on error, as // it may be useful - if res.Tree != nil && genTree { + if res.Tree != nil && *flagGenTree { fmt.Printf("%s\n", trans.AddAttributes(*res.Tree).String()) } if err != nil { // results may be valid even if there is an error - if joinedAST != nil && genAST { + if joinedAST != nil && *flagGenAST { fmt.Printf("%s\n", res.AST.String()) } if syntaxErr, ok := err.(*types.SyntaxError); ok { - fmt.Fprintf(os.Stderr, "%s:\n%s\n", file, syntaxErr.FullMessage()) - returnCode = ExitErrSyntax + errSyntax(file, syntaxErr) } else { - fmt.Fprintf(os.Stderr, "%s: %s\n", file, err.Error()) - returnCode = ExitErrOther + errOther(fmt.Sprintf("%s: %s", file, err.Error())) } return } @@ -456,12 +489,14 @@ func main() { panic("joinedAST is nil; should not be possible") } - if genAST { + if *flagGenAST { fmt.Printf("%s\n", joinedAST.String()) } // attempt to turn AST into a fishi.Spec - + if !*flagQuietMode { + fmt.Printf("Generating language spec from FISHI...\n") + } spec, warnings, err := fishi.NewSpec(*joinedAST) // warnings may be valid even if there is an error if len(warnings) > 0 { @@ -483,98 +518,150 @@ func main() { // token/syntax error output. Allow specification of file in anyfin that // can return a SyntaxError and have all token sources include that. if syntaxErr, ok := err.(*types.SyntaxError); ok { - fmt.Fprintf(os.Stderr, "%s\n", syntaxErr.FullMessage()) - returnCode = ExitErrSyntax + errSyntax("", syntaxErr) } else { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - returnCode = ExitErrOther + errOther(err.Error()) } return } // we officially have a spec. try to print it if requested - if genSpec { + if *flagShowSpec { printSpec(spec) } - if !noGen { - // TODO: jello, first need to create a PRELIM generation along with hooks - // pkg because that is the only way to validate the SDTS. + if *flagNoGen { + if !*flagQuietMode { + fmt.Printf("(code generation skipped due to flags)\n") + } + return + } - // okay, first try to create a parser - // TODO: this should deffo be in fishi package, not the bin. - var p ictiobus.Parser - var parserWarns []fishi.Warning - // if one is selected, use that one - if parserType != nil { - p, parserWarns, err = spec.CreateParser(*parserType, allowAmbig) - } else { - p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig) + // spec completed and no-gen not set; try to create a parser + var p ictiobus.Parser + var parserWarns []fishi.Warning + // if one is selected, use that one + if parserType != nil { + if !*flagQuietMode { + fmt.Printf("Creating %s parser from spec...\n", *parserType) } + p, parserWarns, err = spec.CreateParser(*parserType, allowAmbig) + } else { + if !*flagQuietMode { + fmt.Printf("Creating most restrictive parser from spec...\n") + } + p, parserWarns, err = spec.CreateMostRestrictiveParser(allowAmbig) + } - for _, warn := range parserWarns { - const warnPrefix = "WARN: " - // indent all except the first line - warnStr := rosed.Edit(warnPrefix+warn.Message). - LinesFrom(1). - IndentOpts(len(warnPrefix), rosed.Options{IndentStr: " "}). - String() + // code gen time! 38D - fmt.Fprintf(os.Stderr, "%s\n", warnStr) - } - fmt.Fprintf(os.Stderr, "\n") + for _, warn := range parserWarns { + const warnPrefix = "WARN: " + // indent all except the first line + warnStr := rosed.Edit(warnPrefix+warn.Message). + LinesFrom(1). + IndentOpts(len(warnPrefix), rosed.Options{IndentStr: " "}). + String() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - returnCode = ExitErrParser - return - } + fmt.Fprintf(os.Stderr, "%s\n", warnStr) + } + fmt.Fprintf(os.Stderr, "\n") + if err != nil { + errParser(err.Error()) + return + } + + if !*flagQuietMode { fmt.Printf("Successfully generated %s parser from grammar\n", p.Type().String()) + } - // create a test compiler and output it - if !*valSDTSOff { - if *irType == "" { - fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing -ir parameter\n") + // create a test compiler and output it + if !*flagSimOff { + if *flagIRType == "" { + fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --ir parameter\n") + } else { + if *flagHooksPath == "" { + fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing --hooks parameter\n") } else { - if *hooksPath == "" { - fmt.Fprintf(os.Stderr, "WARN: skipping SDTS validation due to missing -hooks parameter\n") - } else { - genInfo, err := fishi.GenerateTestCompiler(spec, md, p, *hooksPath, *hooksTableName, cgOpts) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - returnCode = ExitErrGeneration - return + if !*flagQuietMode { + simGenDir := ".sim" + if *flagPathPrefix != "" { + simGenDir = filepath.Join(*flagPathPrefix, simGenDir) } + fmt.Printf("Generating parser simulation binary in %s...\n", simGenDir) + } + di := trans.ValidationOptions{ + ParseTrees: *flagSimTrees, + FullDepGraphs: *flagSimGraphs, + ShowAllErrors: !*flagSimFirstErrOnly, + SkipErrors: *flagSimSkipErrs, + } - di := trans.ValidationOptions{ + err := fishi.ValidateSimulatedInput(spec, md, p, *flagHooksPath, *flagHooksTableName, *flagPathPrefix, cgOpts, &di) + if err != nil { + errGeneration(err.Error()) + return + } + } + } + } - ParseTrees: *valSDTSShowTrees, - FullDepGraphs: *valSDTSShowGraphs, - ShowAllErrors: !*valSDTSFirstOnly, - SkipErrors: *valSDTSSkip, - } + // generate diagnostics output if requested + if *flagDiagBin != "" { + // already checked required flags + if !*flagQuietMode { + // tell user if the diagnostic binary cannot do preformatting based + // on flags + if *flagDiagFormatPkg == "" { + fmt.Printf("Format preprocessing disabled in diagnostics bin; set -f to enable\n") + } - err = fishi.ExecuteTestCompiler(genInfo, di) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", "ERROR: SDTS validation failed") - returnCode = ExitErrGeneration - return - } - } + // TODO: probably should make this a constant in fishi instead of + // having ictcc just magically know about it + diagGenDir := ".gen" + if *flagPathPrefix != "" { + diagGenDir = filepath.Join(*flagPathPrefix, diagGenDir) } + fmt.Printf("Generating diagnostics binary code in %s...\n", diagGenDir) + } + + // only specify a format call if a format package was specified, + // otherwise we'll always pass in a non-empty string for the format call + // even when diagFormatPkg is empty, which is not allowed. + var formatCall string + if *flagDiagFormatPkg != "" { + formatCall = *flagDiagFormatCall } - // assuming it worked, now generate the final output - err := fishi.GenerateCompilerGo(spec, md, *pkg, *dest, &cgOpts) + err := fishi.GenerateDiagnosticsBinary(spec, md, p, *flagHooksPath, *flagHooksTableName, *flagDiagFormatPkg, formatCall, *flagPkg, *flagDiagBin, *flagPathPrefix, cgOpts) if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - returnCode = ExitErrGeneration + errGeneration(err.Error()) return } - fmt.Printf("(NOTE: complete frontend generation not implemented yet)\n") + + if !*flagQuietMode { + fmt.Printf("Built diagnostics binary '%s'\n", *flagDiagBin) + } } + // assuming it worked, now generate the final output + if !*flagQuietMode { + feGenDir := *flagDest + if *flagPathPrefix != "" { + feGenDir = filepath.Join(*flagPathPrefix, feGenDir) + } + fmt.Printf("Generating compiler frontend in %s...\n", feGenDir) + } + feDest := *flagDest + if *flagPathPrefix != "" { + feDest = filepath.Join(*flagPathPrefix, feDest) + } + err = fishi.GenerateCompilerGo(spec, md, *flagPkg, feDest, &cgOpts) + if err != nil { + errGeneration(err.Error()) + return + } } // return from flags the parser type selected and whether ambiguity is allowed. @@ -584,29 +671,29 @@ func main() { // err will be non-nil if there is an invalid combination of CLI flags. func parserSelectionFromFlags() (t *types.ParserType, allowAmbig bool, err error) { // enforce mutual exclusion of cli args - if (*parserLL && (*parserCLR || *parserSLR || *parserLALR)) || - (*parserCLR && (*parserSLR || *parserLALR)) || - (*parserSLR && *parserLALR) { + if (*flagParserLL && (*flagParserCLR || *flagParserSLR || *flagParserLALR)) || + (*flagParserCLR && (*flagParserSLR || *flagParserLALR)) || + (*flagParserSLR && *flagParserLALR) { err = fmt.Errorf("cannot specify more than one parser type") return } - allowAmbig = !*parserNoAmbig + allowAmbig = !*flagParserNoAmbig - if *parserLL { + if *flagParserLL { t = new(types.ParserType) *t = types.ParserLL1 // allowAmbig auto false for LL(1) allowAmbig = false - } else if *parserSLR { + } else if *flagParserSLR { t = new(types.ParserType) *t = types.ParserSLR1 - } else if *parserCLR { + } else if *flagParserCLR { t = new(types.ParserType) *t = types.ParserCLR1 - } else if *parserLALR { + } else if *flagParserLALR { t = new(types.ParserType) *t = types.ParserLALR1 } diff --git a/cmd/ictcc/status.go b/cmd/ictcc/status.go new file mode 100644 index 0000000..ea7feeb --- /dev/null +++ b/cmd/ictcc/status.go @@ -0,0 +1,121 @@ +package main + +import ( + "fmt" + "os" + + "github.com/dekarrin/ictiobus/types" +) + +const ( + // ExitSuccess is the exit code for a successful run. + ExitSuccess = iota + + // ExitErrNoFiles is the code returned as exit status when no files are + // provided to the invocation. + ExitErrNoFiles + + // ExitErrInvalidFlags is used if the combination of flags specified is + // invalid. + ExitErrInvalidFlags + + // ExitErrSyntax is the code returned as exit status when a syntax error + // occurs. + ExitErrSyntax + + // ExitErrParser is the code returned as exit status when there is an error + // generating the parser. + ExitErrParser + + // ExitErrGeneration is the code returned as exit status when there is an + // error creating the generated files. + ExitErrGeneration + + // ExitErrOther is a generic error code for any other error. + ExitErrOther +) + +var ( + exitStatus = ExitSuccess +) + +// errNoFiles sets the exit status to ExitErrNoFiles and prints the given error +// message to stderr by calling exitErr. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func errNoFiles(msg string) { + exitErr(ExitErrNoFiles, msg) +} + +// errInvalidFlags sets the exit status to ExitErrInvalidFlags and prints the +// given error message to stderr by calling exitErr. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func errInvalidFlags(msg string) { + exitErr(ExitErrInvalidFlags, msg) +} + +// errSyntax sets the exit status to ExitErrSyntax and prints an error message +// given by the syntax error to stderr. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func errSyntax(filename string, synErr *types.SyntaxError) { + if filename == "" { + filename = "" + } + fmt.Fprintf(os.Stderr, "%s\n", synErr.MessageForFile(filename)) + exitStatus = ExitErrSyntax +} + +// errParser sets the exit status to ExitErrParser and prints the given error +// message to stderr by calling exitErr. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func errParser(msg string) { + exitErr(ExitErrParser, msg) +} + +// errGeneration sets the exit status to ExitErrGeneration and prints the given +// error message to stderr by calling exitErr. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func errGeneration(msg string) { + exitErr(ExitErrGeneration, msg) +} + +// errOther sets the exit status to ExitErrOther and prints the given error +// message to stderr by calling exitErr. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func errOther(msg string) { + exitErr(ExitErrOther, msg) +} + +// exitErr sets the exit status and prints "ERROR: " followed by the given +// error message to stderr. Automatically ends printed message with a newline. +// +// Caller is responsible for exiting main immediately after this function +// returns. +func exitErr(statusCode int, msg string) { + fmt.Fprintf(os.Stderr, "ERROR: %s\n", msg) + exitStatus = statusCode +} + +// basic function to check if panic is happening and recover it while also +// preserving possibly-set exit code. Immediately call this as defered as first +// statement in main. +func preservePanicOrExitWithStatus() { + if panicErr := recover(); panicErr != nil { + // we are panicking, make sure we dont lose the panic just because + // we checked + panic("unrecoverable panic occured") + } else { + os.Exit(exitStatus) + } +} diff --git a/cmd/ictcc/version.go b/cmd/ictcc/version.go index 8acbfee..83fc221 100644 --- a/cmd/ictcc/version.go +++ b/cmd/ictcc/version.go @@ -1,7 +1,7 @@ package main const ( - Version = "0.6.0" + Version = "0.6.1" ) func GetVersionString() string { diff --git a/defexample.md b/defexample.md index 9b244c0..0557d70 100644 --- a/defexample.md +++ b/defexample.md @@ -1,5 +1,12 @@ Example Language ################ + +*NOTE: this document is being kept for historical purproses, and some content +may be correct, but it was the first attempt to standardize the fishi language +and is heavily out of date. Refer to fishi.md instead of this file for an +example; the correct parts of this file will eventually be worked into the manual +for FISHI.* + This is a complete example of a language specified in the special ictiobus format. It is a markdown based format that uses specially named sections to allow both specificiation of a language and freeform text to co-exist. diff --git a/fishi.md b/fishi.md index bf35f04..ed73394 100644 --- a/fishi.md +++ b/fishi.md @@ -235,7 +235,7 @@ For tokens state: %!%[Tt][Oo][Kk][Ee][Nn] %token dir-token %human %!%token directive -%!%![Dd][Ii][Ss][Cc][Aa][Rr][Dd] %token dir-discard +%!%[Dd][Ii][Ss][Cc][Aa][Rr][Dd] %token dir-discard %human %!%discard directive %!%[Pp][Rr][Ii][Oo][Rr][Ii][Tt][Yy] %token dir-priority @@ -279,7 +279,7 @@ For actions state: \s+ %discard -(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]* +(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[^\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]* %token attr-ref %human attribute reference literal [0-9]+ diff --git a/fishi/cgstructs.go b/fishi/cgstructs.go new file mode 100644 index 0000000..4d016ae --- /dev/null +++ b/fishi/cgstructs.go @@ -0,0 +1,52 @@ +package fishi + +import "github.com/dekarrin/ictiobus" + +// File cgstructs.go contains structs used as part of code generation. + +// TODO: move most structs from codegen to here. + +// MainBinaryParams is paramters for generating a main.go file for a binary. +// Unless otherwise specified, all fields are required. +type MainBinaryParams struct { + // Parser is the parser to use for the generated compiler. + Parser ictiobus.Parser + + // HooksPkgDir is the path to the directory containing the hooks package. + HooksPkgDir string + + // HooksExpr is the expression to use to get the hooks map. This can be a + // function call, constant name, or var name. + HooksExpr string + + // FormatPkgDir is the path to the directory containing the format package. + // It is completely optional; if not set, the generated main will not + // contain any pre-formatting code and will assume files are directly ready + // to be fed into the frontend. Must be set if FormatCall is set. + FormatPkgDir string + + // FormatCall is the name of a function within the package specified by + // FormatPkgDir that gets an io.Reader that will run any required + // pre-formatting on an input io.Reader to get code that can be analyzed by + // the frontend. Is is optional; if not set, the generated main will not + // contain any pre-formatting code and will assume files are directly ready + // to be fed into the frontend. Must be set if FormatPkgDir is set. + FormatCall string + + // FrontendPkgName is the name of the package to place generated frontend + // code in. + FrontendPkgName string + + // GenPath is the path to a directory to generate code in. If it does not + // exist, it will be created. If it does exist, any existing files in it + // will be removed will be emptied before code is generated. + GenPath string + + // BinName is the name of the binary being generated. This will be used + // within code for showing help output and other messages. + BinName string + + // Opts are options for code generation. This must be set and its IRType + // field is required to be set, but all other fields within it are optional. + Opts CodegenOptions +} diff --git a/fishi/codegen.go b/fishi/codegen.go index 898108e..bd4e677 100644 --- a/fishi/codegen.go +++ b/fishi/codegen.go @@ -117,6 +117,12 @@ type CodegenOptions struct { // specific type instead of requiring an explicit type instantiation when // called. IRType string + + // PreserveBinarySource is whether to keep the source files for any + // generated binary after the binary has been successfully + // compiled/executed. Normally, these files are removed, but preserving them + // allows for diagnostics on the generated source. + PreserveBinarySource bool } // GeneratedCodeInfo contains information about the generated code. @@ -128,19 +134,26 @@ type GeneratedCodeInfo struct { Path string } -func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions trans.ValidationOptions) error { - args := []string{"run", gci.MainFile, "-sim"} +// ExecuteTestCompiler runs the compiler pointed to by gci in validation mode. +// +// If valOptions is nil, the default validation options are used. +func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions *trans.ValidationOptions) error { + if valOptions == nil { + valOptions = &trans.ValidationOptions{} + } + + args := []string{"run", gci.MainFile, "--sim"} if valOptions.FullDepGraphs { - args = append(args, "-sim-sdts-graphs") + args = append(args, "--sim-graphs") } if valOptions.ParseTrees { - args = append(args, "-sim-sdts-trees") + args = append(args, "--sim-trees") } if !valOptions.ShowAllErrors { - args = append(args, "-sim-sdts-first") + args = append(args, "--sim-first-err") } if valOptions.SkipErrors != 0 { - args = append(args, "-sim-sdts-skip", fmt.Sprintf("%d", valOptions.SkipErrors)) + args = append(args, "--sim-skip-errs", fmt.Sprintf("%d", valOptions.SkipErrors)) } cmd := exec.Command("go", args...) cmd.Dir = gci.Path @@ -149,6 +162,69 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions trans.ValidationOptio return cmd.Run() } +// GenerateDiagnosticsBinary generates a binary that can read input written in +// the language specified by the given Spec and SpecMetadata and print out basic +// information about the analysis, with the goal of printing out the +// constructed intermediate representation from analyzed files. +// +// The args formatPkgDir and formatCall are used to specify preformatting for +// code that that the generated binary will analyze. If set, io.Readers that are +// opened on code input will be passed to the function specified by formatCall. +// If formatCall is set, formatPkgDir must also be set, even if it is already +// specified by another parameter. formatCall must be the name of a function +// within the package specified by formatPkgDir which takes an io.Reader and +// returns a new io.Reader that wraps the one passed in and returns preformatted +// code ready to be analyzed by the generated frontend. +// +// TODO: turn this huge signature into a struct for everyfin from p to opts. +func GenerateDiagnosticsBinary(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, formatPkgDir string, formatCall string, pkgName string, binPath string, pathPrefix string, opts CodegenOptions) error { + binName := filepath.Base(binPath) + + outDir := ".gen" + if pathPrefix != "" { + outDir = filepath.Join(pathPrefix, outDir) + } + + gci, err := GenerateBinaryMainGo(spec, md, MainBinaryParams{ + Parser: p, + HooksPkgDir: hooksPkgDir, + HooksExpr: hooksExpr, + FormatPkgDir: formatPkgDir, + FormatCall: formatCall, + FrontendPkgName: pkgName, + GenPath: outDir, + BinName: binName, + Opts: opts, + }) + if err != nil { + return err + } + + cmd := exec.Command("go", "build", "-o", binName, gci.MainFile) + //cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + cmd.Dir = gci.Path + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + // Move it to the target location. + if err := os.Rename(filepath.Join(gci.Path, binName), binPath); err != nil { + return err + } + + // unless requested to preserve the source, remove the generated source + // directory. + if !opts.PreserveBinarySource { + if err := os.RemoveAll(gci.Path); err != nil { + return err + } + } + + return nil +} + // GenerateTestCompiler generates (but does not yet run) a test compiler for the // given spec and pre-created parser, using the provided hooks package. Once it // is created, it will be able to be executed by calling go run on the provided @@ -163,73 +239,50 @@ func ExecuteTestCompiler(gci GeneratedCodeInfo, valOptions trans.ValidationOptio // or var name. // // opts must be non-nil and IRType must be set. -func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPkgDir string, hooksExpr string, opts CodegenOptions) (GeneratedCodeInfo, error) { - if opts.IRType == "" { +func GenerateBinaryMainGo(spec Spec, md SpecMetadata, params MainBinaryParams) (GeneratedCodeInfo, error) { + if params.Opts.IRType == "" { return GeneratedCodeInfo{}, fmt.Errorf("IRType must be set in options") } + if params.FormatPkgDir != "" && params.FormatCall == "" { + return GeneratedCodeInfo{}, fmt.Errorf("formatCall must be set if formatPkgDir is set") + } + if params.FormatPkgDir == "" && params.FormatCall != "" { + return GeneratedCodeInfo{}, fmt.Errorf("formatPkgDir must be set if formatCall is set") + } - irFQPackage, irType, irPackage, irErr := ParseFQType(opts.IRType) + // we need a separate import for the format package only if it's not the same + // as the hooks package. + separateFormatImport := params.FormatPkgDir != params.HooksPkgDir && params.FormatPkgDir != "" + + irFQPackage, irType, irPackage, irErr := ParseFQType(params.Opts.IRType) if irErr != nil { return GeneratedCodeInfo{}, fmt.Errorf("parsing IRType: %w", irErr) } gci := GeneratedCodeInfo{} - var fePkgName = "sim" + strings.ToLower(md.Language) - // what is the name of our hooks package? find out by reading the first go - // file in the package. - hooksDirItems, err := os.ReadDir(hooksPkgDir) + hooksPkgName, err := readPackageName(params.HooksPkgDir) if err != nil { - return gci, fmt.Errorf("reading hooks package dir: %w", err) + return gci, fmt.Errorf("reading hooks package name: %w", err) } - var hooksPkgName string - for _, item := range hooksDirItems { - if !item.IsDir() && strings.ToLower(filepath.Ext(item.Name())) == ".go" { - // read the file to find the package name - goFilePath := filepath.Join(hooksPkgDir, item.Name()) - goFile, err := os.Open(goFilePath) - if err != nil { - return gci, fmt.Errorf("reading go file in hooks package: %w", err) - } - - // buffered reading - r := bufio.NewReader(goFile) - - // now find the package name in the file - for hooksPkgName == "" { - str, err := r.ReadString('\n') - strTrimmed := strings.TrimSpace(str) - - // is it a line starting with "package"? - if strings.HasPrefix(strTrimmed, "package") { - lineItems := strings.Split(strTrimmed, " ") - if len(lineItems) == 2 { - hooksPkgName = lineItems[1] - break - } - } - - // ofc if err is somefin else - if err != nil { - if err == io.EOF { - break - } - return gci, fmt.Errorf("reading go file in hooks package: %w", err) - } - } - } - - if hooksPkgName != "" { - break + var formatPkgName string + if params.FormatPkgDir != "" { + formatPkgName, err = readPackageName(params.FormatPkgDir) + if err != nil { + return gci, fmt.Errorf("reading format package name: %w", err) } } - if hooksPkgName == "" { - return gci, fmt.Errorf("could not find package name for hooks package; make sure files are gofmt'd") + + if hooksPkgName == params.FrontendPkgName { + // double it to avoid name collision + params.FrontendPkgName += "_" + params.FrontendPkgName } - if hooksPkgName == fePkgName { + + // only worry about formatPkgName if the dir is not the same as hooks. + if params.FormatPkgDir != params.HooksPkgDir && formatPkgName == params.FrontendPkgName { // double it to avoid name collision - fePkgName += "_" + fePkgName + params.FrontendPkgName += "_" + params.FrontendPkgName } safePkgIdent := func(s string) string { @@ -238,27 +291,31 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk return strings.ToLower(s) } - // create a directory to save things in - tmpDir := ".sim" - err = os.RemoveAll(tmpDir) + err = os.MkdirAll(params.GenPath, 0766) if err != nil { - return gci, fmt.Errorf("removing old temp dir: %w", err) - } - err = os.MkdirAll(tmpDir, 0766) - if err != nil { - return gci, fmt.Errorf("creating temp dir: %w", err) + return gci, fmt.Errorf("creating dir for generated code: %w", err) } // start copying the hooks package - hooksDestPath := filepath.Join(tmpDir, "internal", hooksPkgName) - hooksDone, err := copyDirToTargetAsync(hooksPkgDir, hooksDestPath) + hooksDestPath := filepath.Join(params.GenPath, "internal", hooksPkgName) + hooksDone, err := copyDirToTargetAsync(params.HooksPkgDir, hooksDestPath) if err != nil { return gci, fmt.Errorf("copying hooks package: %w", err) } + // start copying the format package if set and if it's not the same as the + // hooks package. + var formatDone <-chan error + if separateFormatImport { + formatDestPath := filepath.Join(params.GenPath, "internal", formatPkgName) + formatDone, err = copyDirToTargetAsync(params.FormatPkgDir, formatDestPath) + if err != nil { + return gci, fmt.Errorf("copying format package: %w", err) + } + } // generate the compiler code - fePkgPath := filepath.Join(tmpDir, "internal", fePkgName) - err = GenerateCompilerGo(spec, md, fePkgName, fePkgPath, &opts) + fePkgPath := filepath.Join(params.GenPath, "internal", params.FrontendPkgName) + err = GenerateCompilerGo(spec, md, params.FrontendPkgName, fePkgPath, ¶ms.Opts) if err != nil { return gci, fmt.Errorf("generating compiler: %w", err) } @@ -266,7 +323,7 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk // since GenerateCompilerGo ensures the directory exists, we can now copy // the encoded parser into it as well. parserPath := filepath.Join(fePkgPath, "parser.cff") - err = ictiobus.SaveParserToDisk(p, parserPath) + err = ictiobus.SaveParserToDisk(params.Parser, parserPath) if err != nil { return gci, fmt.Errorf("writing parser: %w", err) } @@ -278,12 +335,16 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk // export template with main file mainFillData := cgMainData{ - BinPkg: "github.com/dekarrin/ictiobus/langtest/" + strings.ToLower(safePkgIdent(md.Language)), - BinName: "test" + strings.ToLower(safePkgIdent(md.Language)), - BinVersion: "(simulator version)", + BinPkg: "github.com/dekarrin/ictiobus/langexec/" + safePkgIdent(md.Language), + BinName: params.BinName, + Version: md.Version, + Lang: md.Language, HooksPkg: hooksPkgName, - HooksTableExpr: hooksExpr, - FrontendPkg: fePkgName, + HooksTableExpr: params.HooksExpr, + FormatPkg: formatPkgName, + FormatCall: params.FormatCall, + ImportFormatPkg: separateFormatImport, + FrontendPkg: params.FrontendPkgName, IRTypePackage: irFQPackage, IRType: irType, IncludeSimulation: true, @@ -294,7 +355,7 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk } // initialize templates - err = initTemplates(renderFiles, fnMap, opts.TemplateFiles) + err = initTemplates(renderFiles, fnMap, params.Opts.TemplateFiles) if err != nil { return gci, err } @@ -302,7 +363,7 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk // finally, render the main file rf := renderFiles[ComponentMainFile] mainFileRelPath := rf.outFile - err = renderTemplateToFile(rf.tmpl, mainFillData, filepath.Join(tmpDir, mainFileRelPath), opts.DumpPreFormat) + err = renderTemplateToFile(rf.tmpl, mainFillData, filepath.Join(params.GenPath, mainFileRelPath), params.Opts.DumpPreFormat) if err != nil { return gci, err } @@ -310,24 +371,43 @@ func GenerateTestCompiler(spec Spec, md SpecMetadata, p ictiobus.Parser, hooksPk // wait for the hooks package to be copied; we'll need it for go mod tidy <-hooksDone + // if we have a format package, wait for it to be copied + if formatDone != nil { + <-formatDone + } + + // wipe any existing go module stuff + err = os.RemoveAll(filepath.Join(params.GenPath, "go.mod")) + if err != nil { + return gci, fmt.Errorf("removing existing go.mod: %w", err) + } + err = os.RemoveAll(filepath.Join(params.GenPath, "go.sum")) + if err != nil { + return gci, fmt.Errorf("removing existing go.sum: %w", err) + } + err = os.RemoveAll(filepath.Join(params.GenPath, "vendor")) + if err != nil { + return gci, fmt.Errorf("removing existing vendor directory: %w", err) + } + // shell out to run go module stuff goModInitCmd := exec.Command("go", "mod", "init", mainFillData.BinPkg) goModInitCmd.Env = os.Environ() - goModInitCmd.Dir = tmpDir + goModInitCmd.Dir = params.GenPath goModInitOutput, err := goModInitCmd.CombinedOutput() if err != nil { return gci, fmt.Errorf("initializing generated module with binary: %w\n%s", err, string(goModInitOutput)) } goModTidyCmd := exec.Command("go", "mod", "tidy") goModTidyCmd.Env = os.Environ() - goModTidyCmd.Dir = tmpDir + goModTidyCmd.Dir = params.GenPath goModTidyOutput, err := goModTidyCmd.CombinedOutput() if err != nil { return gci, fmt.Errorf("tidying generated module with binary: %w\n%s", err, string(goModTidyOutput)) } // if we got here, all output has been written to the temp dir. - gci.Path = tmpDir + gci.Path = params.GenPath gci.MainFile = mainFileRelPath return gci, nil @@ -393,6 +473,9 @@ func createFuncMap() template.FuncMap { s = strings.ReplaceAll(s, "`", "` + \"`\" + `") return fmt.Sprintf("`%s`", s) }, + "title": func(s string) string { + return titleCaser.String(s) + }, } } @@ -638,12 +721,17 @@ type codegenTemplate struct { } // codegen data for template fill of main.go +// TODO: combine with cgData? type cgMainData struct { BinPkg string BinName string - BinVersion string + Version string + Lang string HooksPkg string HooksTableExpr string + ImportFormatPkg bool + FormatPkg string + FormatCall string FrontendPkg string IRTypePackage string IRType string @@ -836,3 +924,59 @@ func copyDirToTargetAsync(srcDir string, targetDir string) (copyResult chan erro return ch, nil } + +func readPackageName(dir string) (string, error) { + // what is the name of our hooks package? find out by reading the first go + // file in the package. + dirItems, err := os.ReadDir(dir) + if err != nil { + return "", err + } + + var pkgName string + for _, item := range dirItems { + if !item.IsDir() && strings.ToLower(filepath.Ext(item.Name())) == ".go" { + // read the file to find the package name + goFilePath := filepath.Join(dir, item.Name()) + goFile, err := os.Open(goFilePath) + if err != nil { + return "", err + } + + // buffered reading + r := bufio.NewReader(goFile) + + // now find the package name in the file + for pkgName == "" { + str, err := r.ReadString('\n') + strTrimmed := strings.TrimSpace(str) + + // is it a line starting with "package"? + if strings.HasPrefix(strTrimmed, "package") { + lineItems := strings.Split(strTrimmed, " ") + if len(lineItems) == 2 { + pkgName = lineItems[1] + break + } + } + + // ofc if err is somefin else + if err != nil { + if err == io.EOF { + break + } + return "", err + } + } + } + + if pkgName != "" { + break + } + } + if pkgName == "" { + return "", fmt.Errorf("could not find package name; make sure files are gofmt'd") + } + + return pkgName, nil +} diff --git a/fishi/fe/bootstrap_lexer.go b/fishi/fe/bootstrap_lexer.go index 194aafe..10d6dab 100644 --- a/fishi/fe/bootstrap_lexer.go +++ b/fishi/fe/bootstrap_lexer.go @@ -44,7 +44,6 @@ func CreateBootstrapLexer() ictiobus.Lexer { //bootLx.RegisterClass(tcDirDefault, "tokens") bootLx.RegisterClass(TCDirDiscard, "tokens") bootLx.RegisterClass(TCDirPriority, "tokens") - bootLx.RegisterClass(TCInt, "tokens") bootLx.RegisterClass(TCDirState, "tokens") bootLx.RegisterClass(TCLineStartFreeformText, "tokens") bootLx.RegisterClass(TCLineStartEscseq, "tokens") @@ -100,7 +99,7 @@ func CreateBootstrapLexer() ictiobus.Lexer { // actions patterns bootLx.AddPattern(`\s+`, lex.Discard(), "actions", 0) - bootLx.AddPattern(`(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]*`, lex.LexAs(TCAttrRef.ID()), "actions", 0) + bootLx.AddPattern(`(?:{(?:&|\.)(?:[0-9]+)?}|{[0-9]+}|{\^}|{[A-Za-z][^{}]*}|[^\s{}]+)\.[\$A-Za-z][\$A-Za-z0-9_]*`, lex.LexAs(TCAttrRef.ID()), "actions", 0) bootLx.AddPattern(`,`, lex.Discard(), "actions", 0) bootLx.AddPattern(`[0-9]+`, lex.LexAs(TCInt.ID()), "actions", 0) bootLx.AddPattern(`{[A-Za-z][^}]*}`, lex.LexAs(TCNonterminal.ID()), "actions", 0) diff --git a/fishi/fe/lexer_test.go b/fishi/fe/lexer_test.go new file mode 100644 index 0000000..087b0ba --- /dev/null +++ b/fishi/fe/lexer_test.go @@ -0,0 +1,76 @@ +package fe + +import ( + "bytes" + "testing" + + "github.com/dekarrin/ictiobus/lex" + "github.com/dekarrin/ictiobus/types" + "github.com/stretchr/testify/assert" +) + +func Test_Fishi_Lexer_AttrRef_Terminal(t *testing.T) { + testCases := []struct { + name string + input string + expect []types.Token + }{ + { + name: "single attr ref", + input: `%%actions + someAttrRef.value`, + expect: []types.Token{ + lex.NewToken(TCHeaderActions, "%%actions", 0, 0, ""), + lex.NewToken(TCAttrRef, "someAttrRef.value", 0, 0, ""), + lex.NewToken(types.TokenEndOfText, "", 0, 0, ""), + }, + }, + } + + lx := CreateBootstrapLexer() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // setup + assert := assert.New(t) + r := bytes.NewReader([]byte(tc.input)) + + // execute + tokStream, err := lx.Lex(r) + + // verify + if !assert.NoError(err) { + return + } + + // collect the tokens + toks := gatherTokens(tokStream) + + // validate them + tokCount := len(toks) + + // only check count, token class, and lexeme. + if !assert.Len(toks, len(tc.expect), "different number of tokens") { + if tokCount < len(tc.expect) { + tokCount = len(tc.expect) + } + } + + for i := 0; i < tokCount; i++ { + if !assert.Equal(tc.expect[i].Class().ID(), toks[i].Class().ID(), "different token class for token #%d") { + return + } + assert.Equal(tc.expect[i].Lexeme(), toks[i].Lexeme(), "different lexemes for token #%d") + } + }) + } +} + +func gatherTokens(stream types.TokenStream) []types.Token { + allTokens := []types.Token{} + + for stream.HasNext() { + allTokens = append(allTokens, stream.Next()) + } + + return allTokens +} diff --git a/fishi/fishi.go b/fishi/fishi.go index c94acb3..e29daf6 100644 --- a/fishi/fishi.go +++ b/fishi/fishi.go @@ -2,21 +2,19 @@ package fishi import ( "bufio" - "bytes" "errors" "fmt" "io" "os" + "path/filepath" "strings" "github.com/dekarrin/ictiobus" "github.com/dekarrin/ictiobus/fishi/fe" + "github.com/dekarrin/ictiobus/fishi/format" "github.com/dekarrin/ictiobus/fishi/syntax" "github.com/dekarrin/ictiobus/trans" "github.com/dekarrin/ictiobus/types" - "github.com/gomarkdown/markdown" - mkast "github.com/gomarkdown/markdown/ast" - mkparser "github.com/gomarkdown/markdown/parser" ) type Results struct { @@ -25,76 +23,87 @@ type Results struct { } type Options struct { - ParserCFF string - ReadCache bool - WriteCache bool - SDTSValidate bool - SDTSValShowTrees bool - SDTSValShowGraphs bool - SDTSValAllTrees bool - SDTSValSkipTrees int - LexerTrace bool - ParserTrace bool + ParserCFF string + ReadCache bool + WriteCache bool + LexerTrace bool + ParserTrace bool } -func GetFishiFromMarkdown(mdText []byte) []byte { - doc := markdown.Parse(mdText, mkparser.New()) - var scanner fishiScanner - fishi := markdown.Render(doc, scanner) - return fishi -} - -// Preprocess does a preprocess step on the source, which as of now includes -// stripping comments and normalizing end of lines to \n. -func Preprocess(source []byte) []byte { - toBuf := make([]byte, len(source)) - copy(toBuf, source) - scanner := bufio.NewScanner(bytes.NewBuffer(toBuf)) - var preprocessed strings.Builder - - for scanner.Scan() { - line := scanner.Text() - if strings.HasSuffix(line, "\r\n") || strings.HasPrefix(line, "\n\r") { - line = line[0 : len(line)-2] - } else { - line = strings.TrimSuffix(line, "\n") - } - line, _, _ = strings.Cut(line, "#") - preprocessed.WriteString(line) - preprocessed.WriteRune('\n') +// ValidateSimulatedInput generates a lightweight compiler with the spec'd +// frontend in a special directory (".sim" in the local directory or in the path +// specified by pathPrefix, if set) and then runs SDTS validation on a variety +// of parse tree inputs designed to cover all the productions of the grammar at +// least once. +// +// If running validation with the test compiler succeeds, it and the directory +// it was generated in are deleted. If it fails, the directory is left in place +// for inspection. +// +// IRType is required to be set in cgOpts. +// +// valOpts is not required to be set, and if nil will be treated as if it were +// set to an empty struct. +// +// No binary is generated as part of this, but source is which is then executed. +// If PreserveBinarySource is set in cgOpts, the source will be left in the +// .sim directory. +func ValidateSimulatedInput(spec Spec, md SpecMetadata, p ictiobus.Parser, hooks, hooksTable string, pathPrefix string, cgOpts CodegenOptions, valOpts *trans.ValidationOptions) error { + pkgName := "sim" + strings.ToLower(md.Language) + + binName := safeTCIdentifierName(md.Language) + binName = binName[2:] // remove initial "tc". + binName = strings.ToLower(binName) + binName = "test" + binName + + outDir := ".sim" + if pathPrefix != "" { + outDir = filepath.Join(pathPrefix, outDir) } - return []byte(preprocessed.String()) -} - -type fishiScanner bool - -func (fs fishiScanner) RenderNode(w io.Writer, node mkast.Node, entering bool) mkast.WalkStatus { - if !entering { - return mkast.GoToNext + // not setting the format package and call here because we don't need + // preformatting to run verification simulation. + genInfo, err := GenerateBinaryMainGo(spec, md, MainBinaryParams{ + Parser: p, + HooksPkgDir: hooks, + HooksExpr: hooksTable, + FrontendPkgName: pkgName, + GenPath: outDir, + BinName: binName, + Opts: cgOpts, + }) + if err != nil { + return fmt.Errorf("generate test compiler: %w", err) } - codeBlock, ok := node.(*mkast.CodeBlock) - if !ok || codeBlock == nil { - return mkast.GoToNext + err = ExecuteTestCompiler(genInfo, valOpts) + if err != nil { + return fmt.Errorf("execute test compiler: %w", err) } - if strings.ToLower(strings.TrimSpace(string(codeBlock.Info))) == "fishi" { - w.Write(codeBlock.Literal) + if !cgOpts.PreserveBinarySource { + // if we got here, no errors. delete the test compiler and its directory + err = os.RemoveAll(genInfo.Path) + if err != nil { + return fmt.Errorf("remove test compiler: %w", err) + } } - return mkast.GoToNext -} - -func (fs fishiScanner) RenderHeader(w io.Writer, ast mkast.Node) {} -func (fs fishiScanner) RenderFooter(w io.Writer, ast mkast.Node) {} + return nil +} func ParseMarkdownFile(filename string, opts Options) (Results, error) { - data, err := os.ReadFile(filename) + f, err := os.Open(filename) if err != nil { return Results{}, err } - res, err := ParseMarkdown(data, opts) + bufF := bufio.NewReader(f) + r, err := format.NewCodeReader(bufF) + if err != nil { + return Results{}, err + } + + res, err := Parse(r, opts) if err != nil { return res, err } @@ -102,41 +111,26 @@ func ParseMarkdownFile(filename string, opts Options) (Results, error) { return res, nil } -func ParseMarkdown(mdText []byte, opts Options) (Results, error) { - - // TODO: read in filename, based on it check for cached version - - // debug steps: output source after preprocess - // output token stream - // output grammar constructed - // output parser table and type - - source := GetFishiFromMarkdown(mdText) - return Parse(source, opts) -} - -// Parse converts the fishi source code provided into an AST. -func Parse(source []byte, opts Options) (Results, error) { +// Parse converts the fishi source code read from the given reader into an AST. +func Parse(r io.Reader, opts Options) (Results, error) { // get the frontend fishiFront, err := GetFrontend(opts) if err != nil { return Results{}, fmt.Errorf("could not get frontend: %w", err) } - preprocessedSource := Preprocess(source) - - r := Results{} + res := Results{} // now, try to make a parse tree - nodes, pt, err := fishiFront.AnalyzeString(string(preprocessedSource)) - r.Tree = pt // need to do this before we return + nodes, pt, err := fishiFront.Analyze(r) + res.Tree = pt // need to do this before we return if err != nil { - return r, err + return res, err } - r.AST = &AST{ + res.AST = &AST{ Nodes: nodes, } - return r, nil + return res, nil } // GetFrontend gets the frontend for the fishi compiler-compiler. If cffFile is @@ -174,23 +168,6 @@ func GetFrontend(opts Options) (ictiobus.Frontend[[]syntax.Block], error) { } } - // validate our SDTS if we were asked to - if opts.SDTSValidate { - valProd := fishiFront.Lexer.FakeLexemeProducer(true, "") - - di := trans.ValidationOptions{ - ParseTrees: opts.SDTSValShowTrees, - FullDepGraphs: opts.SDTSValShowGraphs, - ShowAllErrors: opts.SDTSValAllTrees, - SkipErrors: opts.SDTSValSkipTrees, - } - - sddErr := fishiFront.SDT.Validate(fishiFront.Parser.Grammar(), fishiFront.IRAttribute, di, valProd) - if sddErr != nil { - return ictiobus.Frontend[[]syntax.Block]{}, fmt.Errorf("sdd validation error: %w", sddErr) - } - } - return fishiFront, nil } diff --git a/fishi/fishi_test.go b/fishi/fishi_test.go index 3db2697..5dfdbf5 100644 --- a/fishi/fishi_test.go +++ b/fishi/fishi_test.go @@ -1,6 +1,7 @@ package fishi import ( + "bytes" "fmt" "testing" @@ -12,7 +13,9 @@ import ( func Test_WithFakeInput(t *testing.T) { assert := assert.New(t) - _, actual := Parse([]byte(testInput), Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true, SDTSValidate: true}) + r := bytes.NewReader([]byte(testInput)) + + _, actual := Parse(r, Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true}) assert.NoError(actual) @@ -27,7 +30,7 @@ func Test_WithFakeInput(t *testing.T) { func Test_SelfHostedMarkdown_Spec(t *testing.T) { assert := assert.New(t) - res, err := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true, SDTSValidate: true}) + res, err := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true}) if !assert.NoError(err) { return } @@ -45,7 +48,7 @@ func Test_SelfHostedMarkdown_Spec(t *testing.T) { func Test_SelfHostedMarkdown(t *testing.T) { assert := assert.New(t) - _, actual := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true, SDTSValidate: true}) + _, actual := ParseMarkdownFile("../fishi.md", Options{ParserCFF: "../fishi-parser.cff", ReadCache: true, WriteCache: true}) assert.NoError(actual) @@ -57,65 +60,6 @@ func Test_SelfHostedMarkdown(t *testing.T) { } } -func Test_GetFishiFromMarkdown(t *testing.T) { - testCases := []struct { - name string - input string - expect string - }{ - { - name: "fishi and text", - input: "Test block\n" + - "only include the fishi block\n" + - "```fishi\n" + - "%%tokens\n" + - "\n" + - "%token test\n" + - "```\n", - expect: "%%tokens\n" + - "\n" + - "%token test\n", - }, - { - name: "two fishi blocks", - input: "Test block\n" + - "only include the fishi blocks\n" + - "```fishi\n" + - "%%tokens\n" + - "\n" + - "%token test\n" + - "```\n" + - "some more text\n" + - "```fishi\n" + - "\n" + - "%token 7\n" + - "%%actions\n" + - "\n" + - "%set go\n" + - "```\n" + - "other text\n", - expect: "%%tokens\n" + - "\n" + - "%token test\n" + - "\n" + - "%token 7\n" + - "%%actions\n" + - "\n" + - "%set go\n", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert := assert.New(t) - - actual := GetFishiFromMarkdown([]byte(tc.input)) - - assert.Equal(tc.expect, string(actual)) - }) - } -} - const ( testInput = `%%actions diff --git a/fishi/format/format.go b/fishi/format/format.go new file mode 100644 index 0000000..94a4c21 --- /dev/null +++ b/fishi/format/format.go @@ -0,0 +1,89 @@ +// Package format contains functions for producing a CodeReader from a stream +// that contains markdown files with fishi codeblocks. A CodeReader can be sent +// directly to the frontend and handles all gathering of codeblocks and running +// any preprocessing needed on it. +package format + +import ( + "bufio" + "bytes" + "io" + "strings" +) + +// CodeReader is an implementation of io.Reader that reads fishi code from input +// containing markdown-formatted text with fishi codeblocks. It will gather all +// fishi codeblocks immediately on open and then read bytes from them as Read is +// called. Preprocessing may also be done at that time. The CodeReader will +// return io.EOF when all bytes from fishi codeblocks in the stream have been +// read. +type CodeReader struct { + r *bytes.Reader +} + +// Read reads bytes from the CodeReader. It will return io.EOF when all bytes +// from fishi codeblocks in the stream have been read. It cannot return an +// error as the actual underlying stream it was opened on is fully consumed at +// the time of opening. +func (cr *CodeReader) Read(p []byte) (n int, err error) { + return cr.r.Read(p) +} + +// NewCodeReader creates a new CodeReader from a stream containing markdown +// formatted text with fishi codeblocks. It will immediately read the provided +// stream until it returns EOF and find all fishi codeblocks and run +// preprocessing on them. +// +// Returns non-nil error if there is a problem reading the markdown or +// preprocessing the code. +func NewCodeReader(r io.Reader) (*CodeReader, error) { + // read the whole stream into a buffer + allInput := make([]byte, 0) + + bufReader := make([]byte, 256) + var err error + for err != io.EOF { + var n int + n, err = r.Read(bufReader) + + if n > 0 { + allInput = append(allInput, bufReader[:n]...) + } + + if err != nil && err != io.EOF { + return nil, err + } + } + + gatheredFishi := GetFishiFromMarkdown(allInput) + fishiSource := Preprocess(gatheredFishi) + + cr := &CodeReader{ + r: bytes.NewReader(fishiSource), + } + + return cr, nil +} + +// Preprocess does a preprocess step on the source, which as of now includes +// stripping comments and normalizing end of lines to \n. +func Preprocess(source []byte) []byte { + toBuf := make([]byte, len(source)) + copy(toBuf, source) + scanner := bufio.NewScanner(bytes.NewBuffer(toBuf)) + var preprocessed strings.Builder + + for scanner.Scan() { + line := scanner.Text() + if strings.HasSuffix(line, "\r\n") || strings.HasPrefix(line, "\n\r") { + line = line[0 : len(line)-2] + } else { + line = strings.TrimSuffix(line, "\n") + } + line, _, _ = strings.Cut(line, "#") + preprocessed.WriteString(line) + preprocessed.WriteRune('\n') + } + + return []byte(preprocessed.String()) +} diff --git a/fishi/format/scanner.go b/fishi/format/scanner.go new file mode 100644 index 0000000..089f8c6 --- /dev/null +++ b/fishi/format/scanner.go @@ -0,0 +1,39 @@ +package format + +import ( + "io" + "strings" + + "github.com/gomarkdown/markdown" + mkast "github.com/gomarkdown/markdown/ast" + mkparser "github.com/gomarkdown/markdown/parser" +) + +type fishiScanner bool + +func (fs fishiScanner) RenderNode(w io.Writer, node mkast.Node, entering bool) mkast.WalkStatus { + if !entering { + return mkast.GoToNext + } + + codeBlock, ok := node.(*mkast.CodeBlock) + if !ok || codeBlock == nil { + return mkast.GoToNext + } + + if strings.ToLower(strings.TrimSpace(string(codeBlock.Info))) == "fishi" { + w.Write(codeBlock.Literal) + } + return mkast.GoToNext +} + +func (fs fishiScanner) RenderHeader(w io.Writer, ast mkast.Node) {} +func (fs fishiScanner) RenderFooter(w io.Writer, ast mkast.Node) {} + +// TODO: rename this to somefin that references the fact that it gathers fishi. +func GetFishiFromMarkdown(mdText []byte) []byte { + doc := markdown.Parse(mdText, mkparser.New()) + var scanner fishiScanner + fishi := markdown.Render(doc, scanner) + return fishi +} diff --git a/fishi/format/scanner_test.go b/fishi/format/scanner_test.go new file mode 100644 index 0000000..d3e928e --- /dev/null +++ b/fishi/format/scanner_test.go @@ -0,0 +1,166 @@ +package format + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_GetFishiFromMarkdown(t *testing.T) { + testCases := []struct { + name string + input string + expect string + }{ + { + name: "fishi and text", + input: "Test block\n" + + "only include the fishi block\n" + + "```fishi\n" + + "%%tokens\n" + + "\n" + + "%token test\n" + + "```\n", + expect: "%%tokens\n" + + "\n" + + "%token test\n", + }, + { + name: "two fishi blocks", + input: "Test block\n" + + "only include the fishi blocks\n" + + "```fishi\n" + + "%%tokens\n" + + "\n" + + "%token test\n" + + "```\n" + + "some more text\n" + + "```fishi\n" + + "\n" + + "%token 7\n" + + "%%actions\n" + + "\n" + + "%set go\n" + + "```\n" + + "other text\n", + expect: "%%tokens\n" + + "\n" + + "%token test\n" + + "\n" + + "%token 7\n" + + "%%actions\n" + + "\n" + + "%set go\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + + actual := GetFishiFromMarkdown([]byte(tc.input)) + + assert.Equal(tc.expect, string(actual)) + }) + } +} + +const ( + testInput = `%%actions + + %symbol + + + {hey} + %prod %index 8 + + %set {thing}.thing %hook thing + %prod {} + + %set {thing}.thing %hook thing + %prod {test} this {THING} + + %set {thing}.thing %hook thing + %prod {ye} + {A} + + %set {thing}.thing %hook thing + + %symbol {yo}%prod + {EAT} ext + + %set {thing}.thing %hook thing + %%tokens + [somefin] + + %stateshift someState + + %%tokens + + %!%[more]%!%bluggleb*shi{2,4} %stateshift glub + %token lovely %human Something for this + + %%tokens + + glub %discard + + + [some]{FREEFORM}idk[^bullshit]text\* + %discard + + %!%[more]%!%bluggleb*shi{2,4} %stateshift glub + %token lovely %human Something nice + %priority 1 + + %state this + + [yo] %discard + + %%grammar + %state glub + {RULE} = {SOMEBULLSHIT} + + %%grammar + {RULE}= {WOAH} | n + {R2} = =+ {DAMN} cool | okaythen + 2 | {} + | {SOMEFIN ELSE} + + %state someState + + {ANOTHER}= {HMM} + + + + + %%actions + + %symbol {text-element} + %prod FREEFORM_TEXT + %set {text-element}.str + %hook identity %with {0}.$text + + %prod ESCSEQ + %set {text-element}.str + %hook unescape %with {.}.$test + + + %symbol {OTHER} + %prod EHHH + %set {OTHER}.str + %hook identity %with {9}.$text + + %prod ESCSEQ + %set {text-element$12}.str + %hook unescape %with {^}.$test + + %state someGoodState + + %symbol {text-element} + %prod FREEFORM_TEXT + %set {text-element}.str + %hook identity %with {ANON$12}.$text + + %prod ESCSEQ + %set {text-element}.str + %hook unescape %with {ESCSEQ}.$test + + ` +) diff --git a/fishi/spec.go b/fishi/spec.go index 9f4f9a7..b2c7b05 100644 --- a/fishi/spec.go +++ b/fishi/spec.go @@ -450,6 +450,11 @@ func analyzeASTActionsContentSlice( synErr := types.NewSyntaxErrorFromToken("invalid attrRef: "+err.Error(), semAct.LHS.Src) return nil, warnings, synErr } + // make shore we aren't trying to set somefin starting with a '$'; those are reserved + if strings.HasPrefix(sdd.Attribute.Name, "$") { + synErr := types.NewSyntaxErrorFromToken("cannot create attribute starting with reserved marker '$'", semAct.LHS.Src) + return nil, warnings, synErr + } // do the same for each arg to the hook if len(semAct.With) > 0 { @@ -722,25 +727,27 @@ func analzyeASTTokensContentSlice( // r is rule to check against, only first production is checked. func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.Rule) (trans.AttrRef, error) { + var ar trans.AttrRef if astRef.Head { - return trans.AttrRef{ + ar = trans.AttrRef{ Relation: trans.NodeRelation{ Type: trans.RelHead, }, Name: astRef.Attribute, - }, nil + } } else if astRef.SymInProd { // make sure the rule has the right number of symbols if astRef.Occurance >= len(r.Productions[0]) { - return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %d symbols", len(r.Productions[0])) + symCount := textfmt.Pluralize(len(r.Productions[0]), "symbol", "-s") + return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %s (%s)", symCount, r.Productions[0]) } - return trans.AttrRef{ + ar = trans.AttrRef{ Relation: trans.NodeRelation{ Type: trans.RelSymbol, Index: astRef.Occurance, }, Name: astRef.Attribute, - }, nil + } } else if astRef.NontermInProd { // make sure that the rule has that number of non-terminals nontermCount := 0 @@ -752,13 +759,13 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R if astRef.Occurance >= nontermCount { return trans.AttrRef{}, fmt.Errorf("non-terminal index out of range; production only has %d non-terminals", nontermCount) } - return trans.AttrRef{ + ar = trans.AttrRef{ Relation: trans.NodeRelation{ Type: trans.RelNonTerminal, Index: astRef.Occurance, }, Name: astRef.Attribute, - }, nil + } } else if astRef.TermInProd { // make sure that the rule has that number of terminals termCount := 0 @@ -770,13 +777,13 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R if astRef.Occurance >= termCount { return trans.AttrRef{}, fmt.Errorf("terminal index out of range; production only has %d terminals", termCount) } - return trans.AttrRef{ + ar = trans.AttrRef{ Relation: trans.NodeRelation{ Type: trans.RelTerminal, Index: astRef.Occurance, }, Name: astRef.Attribute, - }, nil + } } else { // it's an instance of a particular symbol. find out the symbol index. symIndexes := []int{} @@ -791,14 +798,48 @@ func attrRefFromASTAttrRef(astRef syntax.AttrRef, g grammar.Grammar, r grammar.R if astRef.Occurance >= len(symIndexes) { return trans.AttrRef{}, fmt.Errorf("symbol index out of range; production only has %d instances of %s", len(symIndexes), astRef.Symbol) } - return trans.AttrRef{ + ar = trans.AttrRef{ Relation: trans.NodeRelation{ Type: trans.RelSymbol, Index: symIndexes[astRef.Occurance], }, Name: astRef.Attribute, - }, nil + } + } + + valErr := validateParsedAttrRef(ar, g, r) + if valErr != nil { + return ar, valErr } + return ar, nil +} + +// it is assumed that the parsed ref refers to an existing symbol; not checking +// that here. +func validateParsedAttrRef(ar trans.AttrRef, g grammar.Grammar, r grammar.Rule) error { + // need to know if we are dealing with a terminal or not + isTerm := false + + // no need to check RelHead; by definition this cannot be a terminal + if ar.Relation.Type == trans.RelSymbol { + isTerm = g.IsTerminal(r.Productions[0][ar.Relation.Index]) + } else { + isTerm = ar.Relation.Type == trans.RelTerminal + } + + if isTerm { + // then anyfin we take from it, glub, must start with '$' + if ar.Name != "$text" && ar.Name != "$ft" { + return fmt.Errorf("referred-to terminal %q only has '$text' and '$ft' attributes", ar.Name) + } + } else { + // then we cannot take '$text' from it and it's an error. + if ar.Name == "$text" { + return fmt.Errorf("referred-to non-terminal %q does not have lexed text attribute \"$text\"", ar.Name) + } + } + + return nil } func putEntryTokenInCorrectPosForDiscardCheck(first, second *types.Token, discardIsFirst *bool, tok types.Token) { diff --git a/fishi/templates/main.go.tmpl b/fishi/templates/main.go.tmpl index ff60a9b..a720bed 100644 --- a/fishi/templates/main.go.tmpl +++ b/fishi/templates/main.go.tmpl @@ -1,30 +1,50 @@ -{{/* main.go.tmpl is the main entry point for generated code that uses its */}} +{{/* irmain.go.tmpl is the main entry point for generated code that has the +ability to debug the lexer, parser, and output the printed IR once complete. */}} {{/* own binary. */ -}} +/* +{{ .BinName | title }} runs a compiler frontend generated by ictiobus on one or +more files and outputs the parsed intermediate representation (A +{{ .IRTypePackage }}.{{ .IRType }}) as a string to stdout. + +Usage: + + {{ .BinName }} [flags] file1.md file2.md ... +*/ package main import ( "fmt" + "bufio" "os" - "flag" + "io" + "path/filepath" "{{ .BinPkg }}/internal/{{ .HooksPkg }}" "{{ .BinPkg }}/internal/{{ .FrontendPkg }}" -{{if and .IncludeSimulation .IRTypePackage }} +{{if .IRTypePackage -}} "{{ .IRTypePackage }}" -{{end -}} +{{- end}} + +{{if .ImportFormatPkg -}} + "{{ .BinPkg }}/internal/{{ .FormatPkg }}" +{{- end}} "github.com/dekarrin/ictiobus/trans" + "github.com/dekarrin/ictiobus/types" + + "github.com/spf13/pflag" ) const ( ExitSuccess = iota - + ExitErrNoFiles + ExitErrSyntax ExitErr ) const ( - versionString = "{{ .BinName }} {{ .BinVersion }}" + versionString = "{{ .BinName }} {{ .Version }}{{if ne .Lang "Unspecified" }} (for {{ .Lang }}){{end}}" ) var ( @@ -33,26 +53,18 @@ var ( // flags var ( - quietMode bool - version *bool = flag.Bool("version", false, "Print the version of {{ .BinName }} and exit") -{{if .IncludeSimulation }} - doSimulation *bool = flag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules") - simSDTSShowTrees *bool = flag.Bool("sim-sdts-trees", false, "Show full simulated parse trees that caused SDTS validation errors") - simSDTSShowGraphs *bool = flag.Bool("sim-sdts-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors") - simSDTSFirstOnly *bool = flag.Bool("sim-sdts-first", false, "Show only the first error found in SDTS validation of simulated trees") - simSDTSSkip *int = flag.Int("sim-sdts-skip", 0, "Skip the first N errors found in SDTS validation of simulated trees") -{{end -}} + flagQuietMode = pflag.BoolP("quiet", "q", false, "Quiet mode; disables output of the IR") + flagLexerTrace = pflag.BoolP("debug-lexer", "l", false, "Print the lexer trace to stdout") + flagParserTrace = pflag.BoolP("debug-parser", "p", false, "Print the parser trace to stdout") + flagPrintTrees = pflag.Bool("t", false, "Print the parse trees of each file read to stdout") + flagVersion = pflag.Bool("version", false, "Print the version of {{ .BinName }} and exit") + flagSim = pflag.Bool("sim", false, "Run analysis on a series of simulated parse trees intended to cover all rules and then exit") + flagSimTrees = pflag.Bool("sim-trees", false, "Show full simulated parse trees that caused SDTS validation errors") + flagSimGraphs = pflag.Bool("sim-graphs", false, "Show dependency graph for each simulated tree that caused SDTS validation errors") + flagSimFirstErr = pflag.Bool("sim-first-err", false, "Show only the first error found in SDTS validation of simulated trees") + flagSimSkipErrs = pflag.Int("sim-skip-errs", 0, "Skip the first N errors found in SDTS validation of simulated trees") ) -// init flags -func init() { - const ( - quietUsage = "Quiet mode" - ) - flag.BoolVar(&quietMode, "quiet", false, quietUsage) - flag.BoolVar(&quietMode, "q", false, quietUsage+" (shorthand)") -} - func main() { // preserve possibly-set exit code while also checking for panic and // propagating it if there was one. @@ -66,16 +78,22 @@ func main() { } }() - flag.Parse() + pflag.Parse() - if *version { + if *flagVersion { fmt.Println(versionString) return } -{{ if .IncludeSimulation }} - // do simulation if requested - if *doSimulation { + opts := {{ .FrontendPkg }}.FrontendOptions{ + LexerTrace: *flagLexerTrace, + ParserTrace: *flagParserTrace, + } + + hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }} + langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, &opts) + + if *flagSim { hooksMapping := {{ .HooksPkg }}.{{ .HooksTableExpr }} langFront := {{ .FrontendPkg }}.Frontend(hooksMapping, nil) @@ -84,10 +102,10 @@ func main() { valProd := langFront.Lexer.FakeLexemeProducer(true, "") di := trans.ValidationOptions{ - ParseTrees: *simSDTSShowTrees, - FullDepGraphs: *simSDTSShowGraphs, - ShowAllErrors: !*simSDTSFirstOnly, - SkipErrors: *simSDTSSkip, + ParseTrees: *flagSimTrees, + FullDepGraphs: *flagSimGraphs, + ShowAllErrors: !*flagSimFirstErr, + SkipErrors: *flagSimSkipErrs, } sddErr := langFront.SDT.Validate(langFront.Parser.Grammar(), langFront.IRAttribute, di, valProd) @@ -97,13 +115,60 @@ func main() { return } - if !quietMode { - fmt.Printf("(simulation completed with 0 errors)\n") + if !*flagQuietMode { + fmt.Printf("Simulation completed with no errors\n") } + return } -{{end -}} - if !quietMode { - fmt.Printf("({{.BinName}} executed successfully)\n") + files := pflag.Args() + if len(files) == 0 { + fmt.Fprintf(os.Stderr, "No files specified\n") + returnCode = ExitErrNoFiles + return + } + + var r io.Reader + for _, f := range files { + file, err := os.Open(f) + if err != nil { + fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err) + returnCode = ExitErr + return + } + + r = bufio.NewReader(file) + +{{if .FormatCall -}} + // format the input + r, err = {{ .FormatPkg }}.{{ .FormatCall }}(r) + if err != nil { + fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err) + returnCode = ExitErr + return + } +{{- end}} + + ir, pt, err := langFront.Analyze(r) + + // parse tree might be valid no matter what, so we print it first + if *flagPrintTrees { + fmt.Println(pt.String()) + } + + if err != nil { + if syntaxErr, ok := err.(*types.SyntaxError); ok { + fmt.Fprintf(os.Stderr, "%s\n", syntaxErr.MessageForFile(f)) + returnCode = ExitErrSyntax + } else { + fmt.Fprintf(os.Stderr, "ERR: %s: %s\n", f, err) + returnCode = ExitErr + } + return + } + + if !*flagQuietMode { + fmt.Printf("=== Analysis of %s ===\n%s\n\n", filepath.Base(f), ir) + } } } \ No newline at end of file diff --git a/genfishi.sh b/genfishi.sh new file mode 100644 index 0000000..92b1262 --- /dev/null +++ b/genfishi.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# shortcut for running ictcc on fishi.md for when we build a new ictcc bin + +./ictcc --clr \ + --ir '[]github.com/dekarrin/ictiobus/fishi/syntax.Block' \ + --dest .testout \ + -l FISHI -v 1.0.0 \ + --hooks fishi/syntax \ + -d fishic \ + -f fishi/format \ + fishi.md "$@" diff --git a/go.mod b/go.mod index e18a2a9..d63d7b6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/dekarrin/rosed v1.2.1 github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.2 golang.org/x/text v0.8.0 ) diff --git a/go.sum b/go.sum index 6c9928a..7425a17 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/grammar/grammar_test.go b/grammar/grammar_test.go index 666b0dc..0cd17ef 100644 --- a/grammar/grammar_test.go +++ b/grammar/grammar_test.go @@ -9,6 +9,7 @@ import ( "github.com/dekarrin/ictiobus/internal/box" "github.com/dekarrin/ictiobus/internal/textfmt" + "github.com/dekarrin/ictiobus/internal/tmatch" "github.com/stretchr/testify/assert" ) @@ -340,10 +341,11 @@ func Test_Grammar_DeriveFullTree(t *testing.T) { func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) { testCases := []struct { - name string - input Grammar - expect map[string]Production - expectErr bool + name string + input Grammar + expect map[string]Production + expectOneOf []map[string]Production // because this is testing a non-deterministic algorithm, there may be multiple possible outputs + expectErr bool }{ { name: "inescapable derivation cycle in single rule", @@ -378,9 +380,7 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) { F -> ( E ) | id ; `), expect: map[string]Production{ - "E": {"T"}, - "T": {"F"}, - "F": {"id"}, + "E": {"T"}, "T": {"F"}, "F": {"id"}, }, }, { @@ -390,10 +390,9 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) { T -> T * F | F ; F -> ( E ) | id | num; `), - expect: map[string]Production{ - "E": {"T"}, - "T": {"F"}, - "F": {"id"}, + expectOneOf: []map[string]Production{ + {"E": {"T"}, "T": {"F"}, "F": {"id"}}, + {"E": {"T"}, "T": {"F"}, "F": {"num"}}, }, }, } @@ -402,6 +401,11 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) + // make sure we didnt accidentally make an invalid test + if !tc.expectErr && tc.expect == nil && tc.expectOneOf == nil { + panic(fmt.Sprintf("test case %s does not specify expectErr, expect, or expectOneOf", tc.name)) + } + actual, err := tc.input.CreateFewestNonTermsAlternationsTable() if tc.expectErr { assert.Error(err) @@ -410,7 +414,14 @@ func Test_Grammar_CreateFewestNonTermsAlternationsTable(t *testing.T) { return } - assert.Equal(tc.expect, actual) + // if only one, check that one + if tc.expect != nil { + assert.Equal(tc.expect, actual) + } else { + // otherwise, check that it is one of the possible ones + assertErr := tmatch.AnyStrMapV(actual, tc.expectOneOf, tmatch.Comparer(Production.Equal)) + assert.NoError(assertErr) + } }) } diff --git a/internal/tmatch/tmatch.go b/internal/tmatch/tmatch.go new file mode 100644 index 0000000..2d6c2d4 --- /dev/null +++ b/internal/tmatch/tmatch.go @@ -0,0 +1,124 @@ +// Package tmatch provides some quick and dirty matching functions with detailed +// mismatch error messages for use with unit tests. It will probably eventually +// be replaced with a well-vetted library like gomatch or gomega in the future. +package tmatch + +import ( + "fmt" + + "github.com/dekarrin/ictiobus/internal/textfmt" +) + +// CompFunc is a comparison function that returns true if the two values are +// equal. Used for defining custom comparison for types that do not fulfill +// the 'comparable' type constraint. +type CompFunc func(v1, v2 any) bool + +// Comparer convertes the given function into a CompFunc. The returned CompFunc +// ret to ensure the types are exactly as expected, and if not, considers +// it a mismatch. +func Comparer[E1 any, E2 any](fn func(v1 E1, v2 E2) bool) CompFunc { + return func(v1, v2 any) bool { + c1, ok := v1.(E1) + if !ok { + return false + } + c2, ok := v2.(E2) + if !ok { + return false + } + + return fn(c1, c2) + } +} + +// AnyStrMapV returns an error if the actual map does not match any of the maps +// in expect, using vMatches to match elements. +func AnyStrMapV[E any](actual map[string]E, expect []map[string]E, vMatches CompFunc) error { + foundAny := false + for _, expectMap := range expect { + candidateFailedMatch := false + // check that no key is present in one that is not present in the other + for key := range actual { + if _, ok := expectMap[key]; !ok { + candidateFailedMatch = true + break + } + } + if candidateFailedMatch { + continue + } + for key := range expectMap { + if _, ok := actual[key]; !ok { + candidateFailedMatch = true + break + } + } + if candidateFailedMatch { + continue + } + + // keys are all the same, now check values + for key := range actual { + if !vMatches(actual[key], expectMap[key]) { + candidateFailedMatch = true + break + } + } + if !candidateFailedMatch { + foundAny = true + break + } + } + + if !foundAny { + errMsg := "actual does not match any expected:\n " + if len(expect) > 9 { + errMsg += " " + } + errMsg += "actual: " + if actual == nil { + errMsg += "nil" + } else { + errMsg += "{" + ordered := textfmt.OrderedKeys(actual) + for i, k := range ordered { + errMsg += fmt.Sprintf("%s: %v", k, actual[k]) + if i+1 < len(ordered) { + errMsg += ", " + } + } + errMsg += "}" + } + errMsg += "\n" + for i, expectMap := range expect { + errMsg += fmt.Sprintf("expected[%d]: ", i) + if expectMap == nil { + errMsg += "nil" + } else { + errMsg += "{" + ordered := textfmt.OrderedKeys(expectMap) + for j, k := range ordered { + errMsg += fmt.Sprintf("%s: %v", k, expectMap[k]) + if j+1 < len(ordered) { + errMsg += ", " + } + } + errMsg += "}" + } + errMsg += "\n" + } + return fmt.Errorf(errMsg) + } + return nil +} + +// MatchAnyStrMap checks if the actual string map matches any of the expected +// maps. All maps must have a comparable value type. If the value type is not +// comparable, use MatchAnyStrMapV instead to provide a custom equality check. +func AnyStrMap[E comparable](actual map[string]E, expect []map[string]E) error { + + return AnyStrMapV(actual, expect, Comparer(func(v1, v2 E) bool { + return v1 == v2 + })) +} diff --git a/types/syntaxerror.go b/types/syntaxerror.go index b8b1a4e..0f72ca1 100644 --- a/types/syntaxerror.go +++ b/types/syntaxerror.go @@ -57,6 +57,20 @@ func (se SyntaxError) FullMessage() string { return errMsg } +// MessageForFile returns the full error message in the format of +// filename:line:pos: message, followed by the syntax error itself. +func (se SyntaxError) MessageForFile(filename string) string { + var msg string + + if se.line != 0 { + msg = fmt.Sprintf("%s:%d:%d: %s\n%s", filename, se.line, se.pos, se.message, se.SourceLineWithCursor()) + } else { + msg = fmt.Sprintf("%s: %s", filename, msg) + } + + return msg +} + // SourceLineWithCursor returns the source offending code on one line and // directly under it a cursor showing where the error occured. //