diff --git a/bytecode.go b/bytecode.go index 20f505e..bdd688f 100644 --- a/bytecode.go +++ b/bytecode.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/source" ) // Bytecode holds the compiled functions and constants. @@ -216,16 +217,16 @@ func (o *CompiledFunction) Equal(right Object) bool { } // SourcePos returns the source position of the instruction at ip. -func (o *CompiledFunction) SourcePos(ip int) parser.Pos { +func (o *CompiledFunction) SourcePos(ip int) source.Pos { begin: if ip >= 0 { if p, ok := o.SourceMap[ip]; ok { - return parser.Pos(p) + return source.Pos(p) } ip-- goto begin } - return parser.NoPos + return source.NoPos } // Fprint writes constants and instructions to given Writer in a human readable form. diff --git a/compiler.go b/compiler.go index a54b393..031f552 100644 --- a/compiler.go +++ b/compiler.go @@ -12,6 +12,9 @@ import ( "reflect" "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/node" + "github.com/gad-lang/gad/parser/source" "github.com/gad-lang/gad/token" ) @@ -62,7 +65,7 @@ type ( opts CompilerOptions trace io.Writer indent int - stack []parser.Node + stack []ast.Node selectorStack [][][]func() } @@ -80,7 +83,7 @@ type ( OptimizeConst bool OptimizeExpr bool Mixed bool - MixedWriteFunction parser.Expr + MixedWriteFunction node.Expr moduleStore *moduleStore constsCache map[Object]int } @@ -88,7 +91,7 @@ type ( // CompilerError represents a compiler error. CompilerError struct { FileSet *parser.SourceFileSet - Node parser.Node + Node ast.Node Err error } @@ -292,7 +295,7 @@ func (c *Compiler) Bytecode() *Bytecode { } // CompileStmts compiles parser.Stmt and builds Bytecode. -func (c *Compiler) compileStmts(stmt ...parser.Stmt) (err error) { +func (c *Compiler) compileStmts(stmt ...node.Stmt) (err error) { l := len(stmt) if l == 0 { @@ -302,36 +305,36 @@ func (c *Compiler) compileStmts(stmt ...parser.Stmt) (err error) { stmts: for i := 0; i < l; i++ { switch stmt[i].(type) { - case *parser.TextStmt, *parser.ExprToTextStmt: + case *node.TextStmt, *node.ExprToTextStmt: var j = i + 1 l2: for j < l { switch stmt[j].(type) { - case *parser.TextStmt, *parser.ExprToTextStmt: + case *node.TextStmt, *node.ExprToTextStmt: j++ default: break l2 } } - var exprs = make([]parser.Expr, j-i) + var exprs = make([]node.Expr, j-i) for z, s := range stmt[i:j] { switch t := s.(type) { - case *parser.TextStmt: - exprs[z] = &parser.StringLit{Value: t.Literal} - case *parser.ExprToTextStmt: + case *node.TextStmt: + exprs[z] = &node.StringLit{Value: t.Literal} + case *node.ExprToTextStmt: exprs[z] = t.Expr } } wf := c.opts.MixedWriteFunction if wf == nil { - wf = &parser.Ident{Name: "write"} + wf = &node.Ident{Name: "write"} } - err = c.compileCallExpr(&parser.CallExpr{ - Func: wf, - Args: parser.CallExprArgs{Values: exprs}, + err = c.compileCallExpr(&node.CallExpr{ + Func: wf, + CallArgs: node.CallArgs{Args: node.CallExprArgs{Values: exprs}}, }) if err != nil { return @@ -349,149 +352,152 @@ stmts: } // Compile compiles parser.Node and builds Bytecode. -func (c *Compiler) Compile(node parser.Node) error { - defer c.at(node)() +func (c *Compiler) Compile(nd ast.Node) error { + defer c.at(nd)() if c.trace != nil { - if node != nil { + if nd != nil { defer untracec(tracec(c, fmt.Sprintf("%s (%s)", - node.String(), reflect.TypeOf(node).Elem().Name()))) + nd.String(), reflect.TypeOf(nd).Elem().Name()))) } else { defer untracec(tracec(c, "")) } } - switch node := node.(type) { + switch nt := nd.(type) { case *parser.File: - if err := c.compileStmts(node.Stmts...); err != nil { + if err := c.compileStmts(nt.Stmts...); err != nil { return err } - case *parser.ExprStmt: - if err := c.Compile(node.Expr); err != nil { + case *node.ExprStmt: + if err := c.Compile(nt.Expr); err != nil { return err } - c.emit(node, OpPop) - case *parser.IncDecStmt: + if f, _ := nt.Expr.(*node.FuncLit); f != nil && f.Type.Ident != nil { + return nil + } + c.emit(nt, OpPop) + case *node.IncDecStmt: op := token.AddAssign - if node.Token == token.Dec { + if nt.Token == token.Dec { op = token.SubAssign } return c.compileAssignStmt( - node, - []parser.Expr{node.Expr}, - []parser.Expr{&parser.IntLit{Value: 1, ValuePos: node.TokenPos}}, + nt, + []node.Expr{nt.Expr}, + []node.Expr{&node.IntLit{Value: 1, ValuePos: nt.TokenPos}}, token.Var, op, ) - case *parser.ParenExpr: - return c.Compile(node.Expr) - case *parser.BinaryExpr: - switch node.Token { + case *node.ParenExpr: + return c.Compile(nt.Expr) + case *node.BinaryExpr: + switch nt.Token { case token.LAnd, token.LOr, token.NullichCoalesce: - return c.compileLogical(node) + return c.compileLogical(nt) default: - return c.compileBinaryExpr(node) + return c.compileBinaryExpr(nt) } - case *parser.IntLit: - c.emit(node, OpConstant, c.addConstant(Int(node.Value))) - case *parser.UintLit: - c.emit(node, OpConstant, c.addConstant(Uint(node.Value))) - case *parser.FloatLit: - c.emit(node, OpConstant, c.addConstant(Float(node.Value))) - case *parser.DecimalLit: - c.emit(node, OpConstant, c.addConstant(Decimal(node.Value))) - case *parser.BoolLit: - if node.Value { - c.emit(node, OpTrue) + case *node.IntLit: + c.emit(nt, OpConstant, c.addConstant(Int(nt.Value))) + case *node.UintLit: + c.emit(nt, OpConstant, c.addConstant(Uint(nt.Value))) + case *node.FloatLit: + c.emit(nt, OpConstant, c.addConstant(Float(nt.Value))) + case *node.DecimalLit: + c.emit(nt, OpConstant, c.addConstant(Decimal(nt.Value))) + case *node.BoolLit: + if nt.Value { + c.emit(nt, OpTrue) } else { - c.emit(node, OpFalse) + c.emit(nt, OpFalse) } - case *parser.StringLit: - c.emit(node, OpConstant, c.addConstant(String(node.Value))) - case *parser.CharLit: - c.emit(node, OpConstant, c.addConstant(Char(node.Value))) - case *parser.NilLit: - c.emit(node, OpNull) - case *parser.StdInLit: - c.emit(node, OpStdIn) - case *parser.StdOutLit: - c.emit(node, OpStdOut) - case *parser.StdErrLit: - c.emit(node, OpStdErr) - case *parser.CalleeKeyword: - c.emit(node, OpCallee) - case *parser.ArgsKeyword: - c.emit(node, OpArgs) - case *parser.NamedArgsKeyword: - c.emit(node, OpNamedArgs) - case *parser.UnaryExpr: - return c.compileUnaryExpr(node) - case *parser.IfStmt: - return c.compileIfStmt(node) - case *parser.TryStmt: - return c.compileTryStmt(node) - case *parser.CatchStmt: - return c.compileCatchStmt(node) - case *parser.FinallyStmt: - return c.compileFinallyStmt(node) - case *parser.ThrowStmt: - return c.compileThrowStmt(node) - case *parser.ForStmt: - return c.compileForStmt(node) - case *parser.ForInStmt: - return c.compileForInStmt(node) - case *parser.BranchStmt: - return c.compileBranchStmt(node) - case *parser.BlockStmt: - return c.compileBlockStmt(node) - case *parser.DeclStmt: - return c.compileDeclStmt(node) - case *parser.AssignStmt: - return c.compileAssignStmt(node, - node.LHS, node.RHS, token.Var, node.Token) - case *parser.Ident: - return c.compileIdent(node) - case *parser.ArrayLit: - return c.compileArrayLit(node) - case *parser.MapLit: - return c.compileMapLit(node) - case *parser.KeyValueArrayLit: - return c.compileKeyValueArrayLit(node) - case *parser.SelectorExpr: // selector on RHS side - return c.compileSelectorExpr(node) - case *parser.NullishSelectorExpr: // selector on RHS side - return c.compileNullishSelectorExpr(node) - case *parser.IndexExpr: - return c.compileIndexExpr(node) - case *parser.SliceExpr: - return c.compileSliceExpr(node) - case *parser.FuncLit: - return c.compileFuncLit(node) - case *parser.ClosureLit: - return c.compileClosureLit(node) - case *parser.ReturnStmt: - return c.compileReturnStmt(node) - case *parser.CallExpr: - return c.compileCallExpr(node) - case *parser.ImportExpr: - return c.compileImportExpr(node) - case *parser.CondExpr: - return c.compileCondExpr(node) - case *parser.TextStmt: - return c.Compile(&parser.StringLit{Value: node.Literal}) - case *parser.EmptyStmt: - case *parser.ConfigStmt: - if node.Options.WriteFunc != nil { - c.opts.MixedWriteFunction = node.Options.WriteFunc + case *node.StringLit: + c.emit(nt, OpConstant, c.addConstant(String(nt.Value))) + case *node.CharLit: + c.emit(nt, OpConstant, c.addConstant(Char(nt.Value))) + case *node.NilLit: + c.emit(nt, OpNull) + case *node.StdInLit: + c.emit(nt, OpStdIn) + case *node.StdOutLit: + c.emit(nt, OpStdOut) + case *node.StdErrLit: + c.emit(nt, OpStdErr) + case *node.CalleeKeyword: + c.emit(nt, OpCallee) + case *node.ArgsKeyword: + c.emit(nt, OpArgs) + case *node.NamedArgsKeyword: + c.emit(nt, OpNamedArgs) + case *node.UnaryExpr: + return c.compileUnaryExpr(nt) + case *node.IfStmt: + return c.compileIfStmt(nt) + case *node.TryStmt: + return c.compileTryStmt(nt) + case *node.CatchStmt: + return c.compileCatchStmt(nt) + case *node.FinallyStmt: + return c.compileFinallyStmt(nt) + case *node.ThrowStmt: + return c.compileThrowStmt(nt) + case *node.ForStmt: + return c.compileForStmt(nt) + case *node.ForInStmt: + return c.compileForInStmt(nt) + case *node.BranchStmt: + return c.compileBranchStmt(nt) + case *node.BlockStmt: + return c.compileBlockStmt(nt) + case *node.DeclStmt: + return c.compileDeclStmt(nt) + case *node.AssignStmt: + return c.compileAssignStmt(nt, + nt.LHS, nt.RHS, token.Var, nt.Token) + case *node.Ident: + return c.compileIdent(nt) + case *node.ArrayLit: + return c.compileArrayLit(nt) + case *node.MapLit: + return c.compileMapLit(nt) + case *node.KeyValueArrayLit: + return c.compileKeyValueArrayLit(nt) + case *node.SelectorExpr: // selector on RHS side + return c.compileSelectorExpr(nt) + case *node.NullishSelectorExpr: // selector on RHS side + return c.compileNullishSelectorExpr(nt) + case *node.IndexExpr: + return c.compileIndexExpr(nt) + case *node.SliceExpr: + return c.compileSliceExpr(nt) + case *node.FuncLit: + return c.compileFuncLit(nt) + case *node.ClosureLit: + return c.compileClosureLit(nt) + case *node.ReturnStmt: + return c.compileReturnStmt(nt) + case *node.CallExpr: + return c.compileCallExpr(nt) + case *node.ImportExpr: + return c.compileImportExpr(nt) + case *node.CondExpr: + return c.compileCondExpr(nt) + case *node.TextStmt: + return c.Compile(&node.StringLit{Value: nt.Literal}) + case *node.EmptyStmt: + case *node.ConfigStmt: + if nt.Options.WriteFunc != nil { + c.opts.MixedWriteFunction = nt.Options.WriteFunc } case nil: default: - return c.errorf(node, `%[1]T "%[1]v" not implemented`, node) + return c.errorf(nt, `%[1]T "%[1]v" not implemented`, nt) } return nil } -func (c *Compiler) at(node parser.Node) func() { - c.stack = append(c.stack, node) +func (c *Compiler) at(nd ast.Node) func() { + c.stack = append(c.stack, nd) return func() { c.stack = c.stack[:len(c.stack)-1] } @@ -578,10 +584,10 @@ func (c *Compiler) addCompiledFunction(obj Object) (index int) { return } -func (c *Compiler) emit(node parser.Node, opcode Opcode, operands ...int) int { - filePos := parser.NoPos - if node != nil { - filePos = node.Pos() +func (c *Compiler) emit(nd ast.Node, opcode Opcode, operands ...int) int { + filePos := source.NoPos + if nd != nil { + filePos = nd.Pos() } inst := make([]byte, 0, 8) @@ -606,11 +612,11 @@ func (c *Compiler) addInstruction(b []byte) int { return posNewIns } -func (c *Compiler) checkCyclicImports(node parser.Node, modulePath string) error { +func (c *Compiler) checkCyclicImports(nd ast.Node, modulePath string) error { if c.modulePath == modulePath { - return c.errorf(node, "cyclic module import: %s", modulePath) + return c.errorf(nd, "cyclic module import: %s", modulePath) } else if c.parent != nil { - return c.parent.checkCyclicImports(node, modulePath) + return c.parent.checkCyclicImports(nd, modulePath) } return nil } @@ -639,13 +645,13 @@ func (c *Compiler) baseModuleMap() *ModuleMap { } func (c *Compiler) compileModule( - node parser.Node, + nd ast.Node, modulePath string, moduleMap *ModuleMap, src []byte, ) (int, error) { var err error - if err = c.checkCyclicImports(node, modulePath); err != nil { + if err = c.checkCyclicImports(nd, modulePath); err != nil { return 0, err } @@ -668,7 +674,7 @@ func (c *Compiler) compileModule( fork := c.fork(modFile, modulePath, moduleMap, symbolTable) err = fork.optimize(file) if err != nil && err != errSkip { - return 0, c.error(node, err) + return 0, c.error(nd, err) } if err = fork.Compile(file); err != nil { @@ -677,7 +683,7 @@ func (c *Compiler) compileModule( bc := fork.Bytecode() if bc.Main.NumLocals > 256 { - return 0, c.error(node, ErrSymbolLimit) + return 0, c.error(nd, ErrSymbolLimit) } c.constants = bc.Constants @@ -742,22 +748,22 @@ func (c *Compiler) fork( return child } -func (c *Compiler) error(node parser.Node, err error) error { +func (c *Compiler) error(nd ast.Node, err error) error { return &CompilerError{ FileSet: c.file.Set(), - Node: node, + Node: nd, Err: err, } } func (c *Compiler) errorf( - node parser.Node, + nd ast.Node, format string, args ...any, ) error { return &CompilerError{ FileSet: c.file.Set(), - Node: node, + Node: nd, Err: fmt.Errorf(format, args...), } } @@ -817,7 +823,7 @@ func MakeInstruction(buf []byte, op Opcode, args ...int) ([]byte, error) { buf = append(buf, byte(args[0]>>8)) buf = append(buf, byte(args[0])) return buf, nil - case OpLoadModule, OpSetupTry: + case OpLoadModule, OpSetupTry, OpIterNextElse: buf = append(buf, byte(args[0]>>8)) buf = append(buf, byte(args[0])) buf = append(buf, byte(args[1]>>8)) diff --git a/compiler_nodes.go b/compiler_nodes.go index c247e44..c4f3f4a 100644 --- a/compiler_nodes.go +++ b/compiler_nodes.go @@ -7,28 +7,29 @@ package gad import ( "strconv" - "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/node" "github.com/gad-lang/gad/token" ) -func (c *Compiler) compileIfStmt(node *parser.IfStmt) error { +func (c *Compiler) compileIfStmt(nd *node.IfStmt) error { // open new symbol table for the statement c.symbolTable = c.symbolTable.Fork(true) defer func() { c.symbolTable = c.symbolTable.Parent(false) }() - if node.Init != nil { - if err := c.Compile(node.Init); err != nil { + if nd.Init != nil { + if err := c.Compile(nd.Init); err != nil { return err } } jumpPos1 := -1 var skipElse bool - if v, ok := node.Cond.(*parser.BoolLit); !ok { + if v, ok := nd.Cond.(*node.BoolLit); !ok { op := OpJumpFalsy - if v, ok := simplifyExpr(node.Cond).(*parser.UnaryExpr); ok && v.Token.Is(token.Null, token.NotNull) { + if v, ok := simplifyExpr(nd.Cond).(*node.UnaryExpr); ok && v.Token.Is(token.Null, token.NotNull) { if err := c.Compile(v.Expr); err != nil { return err } @@ -37,34 +38,34 @@ func (c *Compiler) compileIfStmt(node *parser.IfStmt) error { if v.Token == token.NotNull { op = OpJumpNil } - } else if err := c.Compile(node.Cond); err != nil { + } else if err := c.Compile(nd.Cond); err != nil { return err } // first jump placeholder - jumpPos1 = c.emit(node, op, 0) - if err := c.Compile(node.Body); err != nil { + jumpPos1 = c.emit(nd, op, 0) + if err := c.Compile(nd.Body); err != nil { return err } } else if v.Value { - if err := c.Compile(node.Body); err != nil { + if err := c.Compile(nd.Body); err != nil { return err } skipElse = true } else { - jumpPos1 = c.emit(node, OpJump, 0) + jumpPos1 = c.emit(nd, OpJump, 0) } - if !skipElse && node.Else != nil { + if !skipElse && nd.Else != nil { // second jump placeholder - jumpPos2 := c.emit(node, OpJump, 0) + jumpPos2 := c.emit(nd, OpJump, 0) if jumpPos1 > -1 { // update first jump offset curPos := len(c.instructions) c.changeOperand(jumpPos1, curPos) } - if err := c.Compile(node.Else); err != nil { + if err := c.Compile(nd.Else); err != nil { return err } // update second jump offset @@ -80,7 +81,7 @@ func (c *Compiler) compileIfStmt(node *parser.IfStmt) error { return nil } -func (c *Compiler) compileTryStmt(node *parser.TryStmt) error { +func (c *Compiler) compileTryStmt(nd *node.TryStmt) error { /* // create a single symbol table for try-catch-finally // any `return` statement in finally block ignores already thrown error. @@ -110,14 +111,14 @@ func (c *Compiler) compileTryStmt(node *parser.TryStmt) error { c.tryCatchIndex++ defer func() { c.symbolTable = c.symbolTable.Parent(false) - c.emit(node, OpThrow, 0) // implicit re-throw + c.emit(nd, OpThrow, 0) // implicit re-throw }() - optry := c.emit(node, OpSetupTry, 0, 0) + optry := c.emit(nd, OpSetupTry, 0, 0) var catchPos, finallyPos int - if node.Body != nil && len(node.Body.Stmts) > 0 { + if nd.Body != nil && len(nd.Body.Stmts) > 0 { // in order not to fork symbol table in Body, compile stmts here instead of in *BlockStmt - for _, stmt := range node.Body.Stmts { + for _, stmt := range nd.Body.Stmts { if err := c.Compile(stmt); err != nil { return err } @@ -125,64 +126,64 @@ func (c *Compiler) compileTryStmt(node *parser.TryStmt) error { } var opjump int - if node.Catch != nil { + if nd.Catch != nil { // if there is no thrown error before catch statement, set catch ident to nil // otherwise jumping to finally and accessing ident in finally access previous set same index variable. - if node.Catch.Ident != nil { - c.emit(node.Catch, OpNull) - symbol, exists := c.symbolTable.DefineLocal(node.Catch.Ident.Name) + if nd.Catch.Ident != nil { + c.emit(nd.Catch, OpNull) + symbol, exists := c.symbolTable.DefineLocal(nd.Catch.Ident.Name) if exists { - c.emit(node, OpSetLocal, symbol.Index) + c.emit(nd, OpSetLocal, symbol.Index) } else { - c.emit(node, OpDefineLocal, symbol.Index) + c.emit(nd, OpDefineLocal, symbol.Index) } } - opjump = c.emit(node, OpJump, 0) + opjump = c.emit(nd, OpJump, 0) catchPos = len(c.instructions) - if err := c.Compile(node.Catch); err != nil { + if err := c.Compile(nd.Catch); err != nil { return err } } c.tryCatchIndex-- // always emit OpSetupFinally to cleanup - if node.Finally != nil { - finallyPos = c.emit(node.Finally, OpSetupFinally) - if err := c.Compile(node.Finally); err != nil { + if nd.Finally != nil { + finallyPos = c.emit(nd.Finally, OpSetupFinally) + if err := c.Compile(nd.Finally); err != nil { return err } } else { - finallyPos = c.emit(node, OpSetupFinally) + finallyPos = c.emit(nd, OpSetupFinally) } c.changeOperand(optry, catchPos, finallyPos) - if node.Catch != nil { + if nd.Catch != nil { // no need jumping if catch is not defined c.changeOperand(opjump, finallyPos) } return nil } -func (c *Compiler) compileCatchStmt(node *parser.CatchStmt) error { - c.emit(node, OpSetupCatch) - if node.Ident != nil { - symbol, exists := c.symbolTable.DefineLocal(node.Ident.Name) +func (c *Compiler) compileCatchStmt(nd *node.CatchStmt) error { + c.emit(nd, OpSetupCatch) + if nd.Ident != nil { + symbol, exists := c.symbolTable.DefineLocal(nd.Ident.Name) if exists { - c.emit(node, OpSetLocal, symbol.Index) + c.emit(nd, OpSetLocal, symbol.Index) } else { - c.emit(node, OpDefineLocal, symbol.Index) + c.emit(nd, OpDefineLocal, symbol.Index) } } else { - c.emit(node, OpPop) + c.emit(nd, OpPop) } - if node.Body == nil { + if nd.Body == nil { return nil } // in order not to fork symbol table in Body, compile stmts here instead of in *BlockStmt - for _, stmt := range node.Body.Stmts { + for _, stmt := range nd.Body.Stmts { if err := c.Compile(stmt); err != nil { return err } @@ -190,13 +191,13 @@ func (c *Compiler) compileCatchStmt(node *parser.CatchStmt) error { return nil } -func (c *Compiler) compileFinallyStmt(node *parser.FinallyStmt) error { - if node.Body == nil { +func (c *Compiler) compileFinallyStmt(nd *node.FinallyStmt) error { + if nd.Body == nil { return nil } // in order not to fork symbol table in Body, compile stmts here instead of in *BlockStmt - for _, stmt := range node.Body.Stmts { + for _, stmt := range nd.Body.Stmts { if err := c.Compile(stmt); err != nil { return err } @@ -204,20 +205,20 @@ func (c *Compiler) compileFinallyStmt(node *parser.FinallyStmt) error { return nil } -func (c *Compiler) compileThrowStmt(node *parser.ThrowStmt) error { - if node.Expr != nil { - if err := c.Compile(node.Expr); err != nil { +func (c *Compiler) compileThrowStmt(nd *node.ThrowStmt) error { + if nd.Expr != nil { + if err := c.Compile(nd.Expr); err != nil { return err } } - c.emit(node, OpThrow, 1) + c.emit(nd, OpThrow, 1) return nil } -func (c *Compiler) compileDeclStmt(node *parser.DeclStmt) error { - decl := node.Decl.(*parser.GenDecl) +func (c *Compiler) compileDeclStmt(nd *node.DeclStmt) error { + decl := nd.Decl.(*node.GenDecl) if len(decl.Specs) == 0 { - return c.errorf(node, "empty declaration not allowed") + return c.errorf(nd, "empty declaration not allowed") } switch decl.Tok { @@ -231,26 +232,26 @@ func (c *Compiler) compileDeclStmt(node *parser.DeclStmt) error { return nil } -func (c *Compiler) compileDeclParam(node *parser.GenDecl) error { +func (c *Compiler) compileDeclParam(nd *node.GenDecl) error { if c.symbolTable.parent != nil { - return c.errorf(node, "param not allowed in this scope") + return c.errorf(nd, "param not allowed in this scope") } var ( - names = make([]string, 0, len(node.Specs)) - namedSpec []parser.Spec + names = make([]string, 0, len(nd.Specs)) + namedSpec []node.Spec ) - for i, sp := range node.Specs { - if np, _ := sp.(*parser.NamedParamSpec); np != nil { - namedSpec = node.Specs[i:] + for i, sp := range nd.Specs { + if np, _ := sp.(*node.NamedParamSpec); np != nil { + namedSpec = nd.Specs[i:] break } else { - spec := sp.(*parser.ParamSpec) + spec := sp.(*node.ParamSpec) names = append(names, spec.Ident.Name) if spec.Variadic { if c.variadic { - return c.errorf(node, + return c.errorf(nd, "multiple variadic param declaration") } c.variadic = true @@ -259,7 +260,7 @@ func (c *Compiler) compileDeclParam(node *parser.GenDecl) error { } if err := c.symbolTable.SetParams(c.variadic, names...); err != nil { - return c.error(node, err) + return c.error(nd, err) } namedSpecCount := len(namedSpec) @@ -271,11 +272,11 @@ func (c *Compiler) compileDeclParam(node *parser.GenDecl) error { named := make([]*NamedParam, len(namedSpec)) for i, sp := range namedSpec { - spec := sp.(*parser.NamedParamSpec) + spec := sp.(*node.NamedParamSpec) if spec.Value == nil { namedSpecCount-- if c.varNamedParams { - return c.errorf(node, + return c.errorf(nd, "multiple variadic named param declaration") } named[i] = &NamedParam{Name: spec.Ident.Name} @@ -286,27 +287,27 @@ func (c *Compiler) compileDeclParam(node *parser.GenDecl) error { } if err := c.symbolTable.SetNamedParams(named...); err != nil { - return c.error(node, err) + return c.error(nd, err) } - stmts := c.helperBuildKwargsIfUndefinedStmts(namedSpecCount, func(index int) (name string, value parser.Expr) { - spec := namedSpec[index].(*parser.NamedParamSpec) + stmts := c.helperBuildKwargsIfUndefinedStmts(namedSpecCount, func(index int) (name string, value node.Expr) { + spec := namedSpec[index].(*node.NamedParamSpec) return spec.Ident.Name, spec.Value }) - return c.Compile(&parser.BlockStmt{Stmts: stmts}) + return c.Compile(&node.BlockStmt{Stmts: stmts}) } -func (c *Compiler) compileDeclGlobal(node *parser.GenDecl) error { +func (c *Compiler) compileDeclGlobal(nd *node.GenDecl) error { if c.symbolTable.parent != nil { - return c.errorf(node, "global not allowed in this scope") + return c.errorf(nd, "global not allowed in this scope") } - for _, sp := range node.Specs { - spec := sp.(*parser.ParamSpec) + for _, sp := range nd.Specs { + spec := sp.(*node.ParamSpec) symbol, err := c.symbolTable.DefineGlobal(spec.Ident.Name) if err != nil { - return c.error(node, err) + return c.error(nd, err) } idx := c.addConstant(String(spec.Ident.Name)) @@ -315,28 +316,28 @@ func (c *Compiler) compileDeclGlobal(node *parser.GenDecl) error { return nil } -func (c *Compiler) compileDeclValue(node *parser.GenDecl) error { +func (c *Compiler) compileDeclValue(nd *node.GenDecl) error { var ( isConst bool - lastExpr parser.Expr + lastExpr node.Expr ) - if node.Tok == token.Const { + if nd.Tok == token.Const { isConst = true defer func() { c.iotaVal = -1 }() } - for _, sp := range node.Specs { - spec := sp.(*parser.ValueSpec) + for _, sp := range nd.Specs { + spec := sp.(*node.ValueSpec) if isConst { if v, ok := spec.Data.(int); ok { c.iotaVal = v } else { - return c.errorf(node, "invalid iota value") + return c.errorf(nd, "invalid iota value") } } for i, ident := range spec.Idents { - leftExpr := []parser.Expr{ident} - var v parser.Expr + leftExpr := []node.Expr{ident} + var v node.Expr if i < len(spec.Values) { v = spec.Values[i] } @@ -345,14 +346,14 @@ func (c *Compiler) compileDeclValue(node *parser.GenDecl) error { if isConst && lastExpr != nil { v = lastExpr } else { - v = &parser.NilLit{TokenPos: ident.Pos()} + v = &node.NilLit{TokenPos: ident.Pos()} } } else { lastExpr = v } - rightExpr := []parser.Expr{v} - err := c.compileAssignStmt(node, leftExpr, rightExpr, node.Tok, token.Define) + rightExpr := []node.Expr{v} + err := c.compileAssignStmt(nd, leftExpr, rightExpr, nd.Tok, token.Define) if err != nil { return err } @@ -362,15 +363,14 @@ func (c *Compiler) compileDeclValue(node *parser.GenDecl) error { } func (c *Compiler) checkAssignment( - node parser.Node, - lhs []parser.Expr, - rhs []parser.Expr, - keyword token.Token, + nd ast.Node, + lhs []node.Expr, + rhs []node.Expr, op token.Token, ) (bool, error) { _, numRHS := len(lhs), len(rhs) if numRHS > 1 { - return false, c.errorf(node, + return false, c.errorf(nd, "multiple expressions on the right side not supported") } @@ -378,7 +378,7 @@ func (c *Compiler) checkAssignment( Loop: for _, expr := range lhs { switch expr.(type) { - case *parser.SelectorExpr, *parser.IndexExpr: + case *node.SelectorExpr, *node.IndexExpr: selector = true break Loop } @@ -387,7 +387,7 @@ Loop: if selector { if op == token.Define { // using selector on new variable does not make sense - return false, c.errorf(node, "operator ':=' not allowed with selector") + return false, c.errorf(nd, "operator ':=' not allowed with selector") } } @@ -395,13 +395,13 @@ Loop: } func (c *Compiler) compileAssignStmt( - node parser.Node, - lhs []parser.Expr, - rhs []parser.Expr, + nd ast.Node, + lhs []node.Expr, + rhs []node.Expr, keyword token.Token, op token.Token, ) error { - compile, err := c.checkAssignment(node, lhs, rhs, keyword, op) + compile, err := c.checkAssignment(nd, lhs, rhs, op) if err != nil || !compile { return err } @@ -420,26 +420,26 @@ func (c *Compiler) compileAssignStmt( tempArrSymbol, _ = c.symbolTable.DefineLocal(":array") // ignore disabled builtins of symbol table for BuiltinMakeArray because // it is required to handle destructuring assignment. - c.emit(node, OpGetBuiltin, int(BuiltinMakeArray)) - c.emit(node, OpConstant, c.addConstant(Int(len(lhs)))) + c.emit(nd, OpGetBuiltin, int(BuiltinMakeArray)) + c.emit(nd, OpConstant, c.addConstant(Int(len(lhs)))) } if op == token.Assign { switch lhs[0].(type) { - case *parser.StdInLit, *parser.StdOutLit, *parser.StdErrLit: + case *node.StdInLit, *node.StdOutLit, *node.StdErrLit: var fd int64 switch lhs[0].(type) { - case *parser.StdOutLit: + case *node.StdOutLit: fd = 1 - case *parser.StdErrLit: + case *node.StdErrLit: fd = 2 } - return c.compileCallExpr(&parser.CallExpr{ - Func: &parser.Ident{Name: BuiltinStdIO.String()}, - Args: parser.CallExprArgs{Values: []parser.Expr{ - &parser.IntLit{Value: fd}, + return c.compileCallExpr(&node.CallExpr{ + Func: &node.Ident{Name: BuiltinStdIO.String()}, + CallArgs: node.CallArgs{Args: node.CallExprArgs{Values: []node.Expr{ + &node.IntLit{Value: fd}, rhs[0], - }}, + }}}, }) } } @@ -449,14 +449,14 @@ func (c *Compiler) compileAssignStmt( if op == token.LOrAssign { op2 = OpOrJump } - jumpPos := c.emit(node, op2, 0) + jumpPos := c.emit(nd, op2, 0) // compile RHSs for _, expr := range rhs { if err := c.Compile(expr); err != nil { return err } } - if err := c.compileDefineAssign(node, lhs[0], keyword, token.Assign, false); err != nil { + if err := c.compileDefineAssign(nd, lhs[0], keyword, token.Assign, false); err != nil { return err } c.changeOperand(jumpPos, len(c.instructions)) @@ -471,73 +471,73 @@ func (c *Compiler) compileAssignStmt( } if isArrDestruct { - return c.compileDestructuring(node, lhs, tempArrSymbol, keyword, op) + return c.compileDestructuring(nd, lhs, tempArrSymbol, keyword, op) } if op != token.Assign && op != token.Define { - c.compileCompoundAssignment(node, op) + c.compileCompoundAssignment(nd, op) } - return c.compileDefineAssign(node, lhs[0], keyword, op, false) + return c.compileDefineAssign(nd, lhs[0], keyword, op, false) } func (c *Compiler) compileCompoundAssignment( - node parser.Node, + nd ast.Node, op token.Token, ) { switch op { case token.AddAssign: - c.emit(node, OpBinaryOp, int(token.Add)) + c.emit(nd, OpBinaryOp, int(token.Add)) case token.SubAssign: - c.emit(node, OpBinaryOp, int(token.Sub)) + c.emit(nd, OpBinaryOp, int(token.Sub)) case token.MulAssign: - c.emit(node, OpBinaryOp, int(token.Mul)) + c.emit(nd, OpBinaryOp, int(token.Mul)) case token.QuoAssign: - c.emit(node, OpBinaryOp, int(token.Quo)) + c.emit(nd, OpBinaryOp, int(token.Quo)) case token.RemAssign: - c.emit(node, OpBinaryOp, int(token.Rem)) + c.emit(nd, OpBinaryOp, int(token.Rem)) case token.AndAssign: - c.emit(node, OpBinaryOp, int(token.And)) + c.emit(nd, OpBinaryOp, int(token.And)) case token.OrAssign: - c.emit(node, OpBinaryOp, int(token.Or)) + c.emit(nd, OpBinaryOp, int(token.Or)) case token.AndNotAssign: - c.emit(node, OpBinaryOp, int(token.AndNot)) + c.emit(nd, OpBinaryOp, int(token.AndNot)) case token.XorAssign: - c.emit(node, OpBinaryOp, int(token.Xor)) + c.emit(nd, OpBinaryOp, int(token.Xor)) case token.ShlAssign: - c.emit(node, OpBinaryOp, int(token.Shl)) + c.emit(nd, OpBinaryOp, int(token.Shl)) case token.ShrAssign: - c.emit(node, OpBinaryOp, int(token.Shr)) + c.emit(nd, OpBinaryOp, int(token.Shr)) } } func (c *Compiler) compileDestructuring( - node parser.Node, - lhs []parser.Expr, + nd ast.Node, + lhs []node.Expr, tempArrSymbol *Symbol, keyword token.Token, op token.Token, ) error { - c.emit(node, OpCall, 2, 0) - c.emit(node, OpDefineLocal, tempArrSymbol.Index) + c.emit(nd, OpCall, 2, 0) + c.emit(nd, OpDefineLocal, tempArrSymbol.Index) numLHS := len(lhs) var found int for lhsIndex, expr := range lhs { if op == token.Define { - if term, ok := expr.(*parser.Ident); ok { + if term, ok := expr.(*node.Ident); ok { if _, ok = c.symbolTable.find(term.Name); ok { found++ } } if found == numLHS { - return c.errorf(node, "no new variable on left side") + return c.errorf(nd, "no new variable on left side") } } - c.emit(node, OpGetLocal, tempArrSymbol.Index) - c.emit(node, OpConstant, c.addConstant(Int(lhsIndex))) - c.emit(node, OpGetIndex, 1) - err := c.compileDefineAssign(node, expr, keyword, op, keyword != token.Const) + c.emit(nd, OpGetLocal, tempArrSymbol.Index) + c.emit(nd, OpConstant, c.addConstant(Int(lhsIndex))) + c.emit(nd, OpGetIndex, 1) + err := c.compileDefineAssign(nd, expr, keyword, op, keyword != token.Const) if err != nil { return err } @@ -545,51 +545,51 @@ func (c *Compiler) compileDestructuring( if !c.symbolTable.InBlock() { // blocks set nil to variables defined in it after block - c.emit(node, OpNull) - c.emit(node, OpSetLocal, tempArrSymbol.Index) + c.emit(nd, OpNull) + c.emit(nd, OpSetLocal, tempArrSymbol.Index) } return nil } func (c *Compiler) compileDefine( - node parser.Node, + nd ast.Node, ident string, allowRedefine bool, keyword token.Token, ) error { symbol, exists := c.symbolTable.DefineLocal(ident) if !allowRedefine && exists && ident != "_" { - return c.errorf(node, "%q redeclared in this block", ident) + return c.errorf(nd, "%q redeclared in this block", ident) } if symbol.Constant { - return c.errorf(node, "assignment to constant variable %q", ident) + return c.errorf(nd, "assignment to constant variable %q", ident) } if c.iotaVal > -1 && ident == "iota" && keyword == token.Const { - return c.errorf(node, "assignment to iota") + return c.errorf(nd, "assignment to iota") } - c.emit(node, OpDefineLocal, symbol.Index) + c.emit(nd, OpDefineLocal, symbol.Index) symbol.Assigned = true symbol.Constant = keyword == token.Const && ident != "_" return nil } func (c *Compiler) compileAssign( - node parser.Node, + nd ast.Node, symbol *Symbol, ident string, ) error { if symbol.Constant { - return c.errorf(node, "assignment to constant variable %q", ident) + return c.errorf(nd, "assignment to constant variable %q", ident) } switch symbol.Scope { case ScopeLocal: - c.emit(node, OpSetLocal, symbol.Index) + c.emit(nd, OpSetLocal, symbol.Index) symbol.Assigned = true case ScopeFree: - c.emit(node, OpSetFree, symbol.Index) + c.emit(nd, OpSetFree, symbol.Index) symbol.Assigned = true s := symbol for s != nil { @@ -599,17 +599,17 @@ func (c *Compiler) compileAssign( s = s.Original } case ScopeGlobal: - c.emit(node, OpSetGlobal, symbol.Index) + c.emit(nd, OpSetGlobal, symbol.Index) symbol.Assigned = true default: - return c.errorf(node, "unresolved reference %q", ident) + return c.errorf(nd, "unresolved reference %q", ident) } return nil } func (c *Compiler) compileDefineAssign( - node parser.Node, - lhs parser.Expr, + nd ast.Node, + lhs node.Expr, keyword token.Token, op token.Token, allowRedefine bool, @@ -617,28 +617,28 @@ func (c *Compiler) compileDefineAssign( ident, selectors := resolveAssignLHS(lhs) numSel := len(selectors) if numSel == 0 && op == token.Define { - return c.compileDefine(node, ident, allowRedefine, keyword) + return c.compileDefine(nd, ident, allowRedefine, keyword) } symbol, ok := c.symbolTable.Resolve(ident) if !ok { - return c.errorf(node, "unresolved reference %q", ident) + return c.errorf(nd, "unresolved reference %q", ident) } if numSel == 0 { - return c.compileAssign(node, symbol, ident) + return c.compileAssign(nd, symbol, ident) } // get indexes until last one and set the value to the last index switch symbol.Scope { case ScopeLocal: - c.emit(node, OpGetLocal, symbol.Index) + c.emit(nd, OpGetLocal, symbol.Index) case ScopeFree: - c.emit(node, OpGetFree, symbol.Index) + c.emit(nd, OpGetFree, symbol.Index) case ScopeGlobal: - c.emit(node, OpGetGlobal, symbol.Index) + c.emit(nd, OpGetGlobal, symbol.Index) default: - return c.errorf(node, "unexpected scope %q for symbol %q", + return c.errorf(nd, "unexpected scope %q for symbol %q", symbol.Scope, ident) } @@ -648,74 +648,74 @@ func (c *Compiler) compileDefineAssign( return err } } - c.emit(node, OpGetIndex, numSel-1) + c.emit(nd, OpGetIndex, numSel-1) } if err := c.Compile(selectors[numSel-1]); err != nil { return err } - c.emit(node, OpSetIndex) + c.emit(nd, OpSetIndex) return nil } -func resolveAssignLHS(expr parser.Expr) (name string, selectors []parser.Expr) { +func resolveAssignLHS(expr node.Expr) (name string, selectors []node.Expr) { switch term := expr.(type) { - case *parser.SelectorExpr: + case *node.SelectorExpr: name, selectors = resolveAssignLHS(term.Expr) selectors = append(selectors, term.Sel) - case *parser.IndexExpr: + case *node.IndexExpr: name, selectors = resolveAssignLHS(term.Expr) selectors = append(selectors, term.Index) - case *parser.Ident: + case *node.Ident: name = term.Name } return } -func (c *Compiler) compileBranchStmt(node *parser.BranchStmt) error { - switch node.Token { +func (c *Compiler) compileBranchStmt(nd *node.BranchStmt) error { + switch nd.Token { case token.Break: curLoop := c.currentLoop() if curLoop == nil { - return c.errorf(node, "break not allowed outside loop") + return c.errorf(nd, "break not allowed outside loop") } var pos int if curLoop.lastTryCatchIndex == c.tryCatchIndex { - pos = c.emit(node, OpJump, 0) + pos = c.emit(nd, OpJump, 0) } else { - c.emit(node, OpFinalizer, curLoop.lastTryCatchIndex+1) - pos = c.emit(node, OpJump, 0) + c.emit(nd, OpFinalizer, curLoop.lastTryCatchIndex+1) + pos = c.emit(nd, OpJump, 0) } curLoop.breaks = append(curLoop.breaks, pos) case token.Continue: curLoop := c.currentLoop() if curLoop == nil { - return c.errorf(node, "continue not allowed outside loop") + return c.errorf(nd, "continue not allowed outside loop") } var pos int if curLoop.lastTryCatchIndex == c.tryCatchIndex { - pos = c.emit(node, OpJump, 0) + pos = c.emit(nd, OpJump, 0) } else { - c.emit(node, OpFinalizer, curLoop.lastTryCatchIndex+1) - pos = c.emit(node, OpJump, 0) + c.emit(nd, OpFinalizer, curLoop.lastTryCatchIndex+1) + pos = c.emit(nd, OpJump, 0) } curLoop.continues = append(curLoop.continues, pos) default: - return c.errorf(node, "invalid branch statement: %s", node.Token.String()) + return c.errorf(nd, "invalid branch statement: %s", nd.Token.String()) } return nil } -func (c *Compiler) compileBlockStmt(node *parser.BlockStmt) error { - if len(node.Stmts) == 0 { +func (c *Compiler) compileBlockStmt(nd *node.BlockStmt) error { + if len(nd.Stmts) == 0 { return nil } c.symbolTable = c.symbolTable.Fork(true) - if err := c.compileStmts(node.Stmts...); err != nil { + if err := c.compileStmts(nd.Stmts...); err != nil { return err } @@ -723,28 +723,28 @@ func (c *Compiler) compileBlockStmt(node *parser.BlockStmt) error { return nil } -func (c *Compiler) compileReturnStmt(node *parser.ReturnStmt) error { - if node.Result == nil { +func (c *Compiler) compileReturnStmt(nd *node.ReturnStmt) error { + if nd.Result == nil { if c.tryCatchIndex > -1 { - c.emit(node, OpFinalizer, 0) + c.emit(nd, OpFinalizer, 0) } - c.emit(node, OpReturn, 0) + c.emit(nd, OpReturn, 0) return nil } - if err := c.Compile(node.Result); err != nil { + if err := c.Compile(nd.Result); err != nil { return err } if c.tryCatchIndex > -1 { - c.emit(node, OpFinalizer, 0) + c.emit(nd, OpFinalizer, 0) } - c.emit(node, OpReturn, 1) + c.emit(nd, OpReturn, 1) return nil } -func (c *Compiler) compileForStmt(stmt *parser.ForStmt) error { +func (c *Compiler) compileForStmt(stmt *node.ForStmt) error { c.symbolTable = c.symbolTable.Fork(true) defer func() { c.symbolTable = c.symbolTable.Parent(false) @@ -811,7 +811,7 @@ func (c *Compiler) compileForStmt(stmt *parser.ForStmt) error { return nil } -func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error { +func (c *Compiler) compileForInStmt(stmt *node.ForInStmt) error { c.symbolTable = c.symbolTable.Fork(true) defer func() { c.symbolTable = c.symbolTable.Parent(false) @@ -842,6 +842,17 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error { c.emit(stmt, OpIterInit) c.emit(stmt, OpDefineLocal, itSymbol.Index) + var ( + iterNextElsePos, + truePos, + falsePos int + ) + + if stmt.Else != nil { + c.emit(stmt, OpGetLocal, itSymbol.Index) + iterNextElsePos = c.emit(stmt, OpIterNextElse, 0, 0) + } + // pre-condition position preCondPos := len(c.instructions) @@ -853,6 +864,13 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error { // condition jump position postCondPos := c.emit(stmt, OpJumpFalsy, 0) + if stmt.Else != nil { + truePos = len(c.instructions) + defer func() { + c.changeOperand(iterNextElsePos, truePos, falsePos) + }() + } + // enter loop loop := c.enterLoop() @@ -894,6 +912,14 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error { // back to condition c.emit(stmt, OpJump, preCondPos) + // else stmt + if stmt.Else != nil { + falsePos = len(c.instructions) + if err := c.Compile(stmt.Else); err != nil { + return err + } + } + // post-statement position postStmtPos := len(c.instructions) c.changeOperand(postCondPos, postStmtPos) @@ -909,27 +935,40 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error { return nil } -func (c *Compiler) compileFuncLit(node *parser.FuncLit) error { - return c.compileFunc(node, node.Type, node.Body) +func (c *Compiler) compileFuncLit(nd *node.FuncLit) error { + if ident := nd.Type.Ident; ident != nil { + nd.Type.Ident = nil + defer func() { + nd.Type.Ident = ident + }() + ass := &node.AssignStmt{ + LHS: []node.Expr{ident}, + RHS: []node.Expr{nd}, + Token: token.Define, + } + return c.compileAssignStmt(ass, + ass.LHS, ass.RHS, token.Var, ass.Token) + } + return c.compileFunc(nd, nd.Type, nd.Body) } -func (c *Compiler) compileClosureLit(node *parser.ClosureLit) error { - var stmts []parser.Stmt - if b, ok := node.Body.(*parser.BlockExpr); ok { +func (c *Compiler) compileClosureLit(nd *node.ClosureLit) error { + var stmts []node.Stmt + if b, ok := nd.Body.(*node.BlockExpr); ok { stmts = b.Stmts if l := len(stmts); l > 0 { switch t := stmts[l-1].(type) { - case *parser.ExprStmt: - stmts[l-1] = &parser.ReturnStmt{Result: t.Expr} + case *node.ExprStmt: + stmts[l-1] = &node.ReturnStmt{Result: t.Expr} } } } else { - stmts = append(stmts, &parser.ReturnStmt{Result: node.Body}) + stmts = append(stmts, &node.ReturnStmt{Result: nd.Body}) } - return c.compileFunc(node, node.Type, &parser.BlockStmt{Stmts: stmts}) + return c.compileFunc(nd, nd.Type, &node.BlockStmt{Stmts: stmts}) } -func (c *Compiler) compileFunc(node parser.Node, typ *parser.FuncType, body *parser.BlockStmt) error { +func (c *Compiler) compileFunc(nd ast.Node, typ *node.FuncType, body *node.BlockStmt) error { var ( params = make([]string, len(typ.Params.Args.Values)) namedParams = make([]*NamedParam, len(typ.Params.NamedArgs.Names)) @@ -945,7 +984,7 @@ func (c *Compiler) compileFunc(node parser.Node, typ *parser.FuncType, body *par } if err := symbolTable.SetParams(typ.Params.Args.Var != nil, params...); err != nil { - return c.error(node, err) + return c.error(nd, err) } for i, name := range typ.Params.NamedArgs.Names { @@ -958,12 +997,12 @@ func (c *Compiler) compileFunc(node parser.Node, typ *parser.FuncType, body *par if len(namedParams) > 0 { if err := symbolTable.SetNamedParams(namedParams...); err != nil { - return c.error(node, err) + return c.error(nd, err) } } if count := len(typ.Params.NamedArgs.Values); count > 0 { - body.Stmts = append(c.helperBuildKwargsStmts(count, func(index int) (name string, value parser.Expr) { + body.Stmts = append(c.helperBuildKwargsStmts(count, func(index int) (name string, value node.Expr) { return typ.Params.NamedArgs.Names[index].Name, typ.Params.NamedArgs.Values[index] }), body.Stmts...) } @@ -977,97 +1016,97 @@ func (c *Compiler) compileFunc(node parser.Node, typ *parser.FuncType, body *par for _, s := range freeSymbols { switch s.Scope { case ScopeLocal: - c.emit(node, OpGetLocalPtr, s.Index) + c.emit(nd, OpGetLocalPtr, s.Index) case ScopeFree: - c.emit(node, OpGetFreePtr, s.Index) + c.emit(nd, OpGetFreePtr, s.Index) } } bc := fork.Bytecode() if bc.Main.NumLocals > 256 { - return c.error(node, ErrSymbolLimit) + return c.error(nd, ErrSymbolLimit) } c.constants = bc.Constants index := c.addConstant(bc.Main) if len(freeSymbols) > 0 { - c.emit(node, OpClosure, index, len(freeSymbols)) + c.emit(nd, OpClosure, index, len(freeSymbols)) } else { - c.emit(node, OpConstant, index) + c.emit(nd, OpConstant, index) } return nil } -func (c *Compiler) compileLogical(node *parser.BinaryExpr) error { +func (c *Compiler) compileLogical(nd *node.BinaryExpr) error { // left side term - if err := c.Compile(node.LHS); err != nil { + if err := c.Compile(nd.LHS); err != nil { return err } // jump position var jumpPos int - switch node.Token { + switch nd.Token { case token.LAnd: - jumpPos = c.emit(node, OpAndJump, 0) + jumpPos = c.emit(nd, OpAndJump, 0) case token.NullichCoalesce: - jumpPos = c.emit(node, OpJumpNotNil, 0) + jumpPos = c.emit(nd, OpJumpNotNil, 0) default: - jumpPos = c.emit(node, OpOrJump, 0) + jumpPos = c.emit(nd, OpOrJump, 0) } // right side term - if err := c.Compile(node.RHS); err != nil { + if err := c.Compile(nd.RHS); err != nil { return err } c.changeOperand(jumpPos, len(c.instructions)) return nil } -func (c *Compiler) compileBinaryExpr(node *parser.BinaryExpr) error { - if err := c.Compile(node.LHS); err != nil { +func (c *Compiler) compileBinaryExpr(nd *node.BinaryExpr) error { + if err := c.Compile(nd.LHS); err != nil { return err } - if err := c.Compile(node.RHS); err != nil { + if err := c.Compile(nd.RHS); err != nil { return err } - switch node.Token { + switch nd.Token { case token.Equal: - c.emit(node, OpEqual) + c.emit(nd, OpEqual) case token.NotEqual: - c.emit(node, OpNotEqual) + c.emit(nd, OpNotEqual) default: - if !node.Token.IsBinaryOperator() { - return c.errorf(node, "invalid binary operator: %s", - node.Token.String()) + if !nd.Token.IsBinaryOperator() { + return c.errorf(nd, "invalid binary operator: %s", + nd.Token.String()) } - c.emit(node, OpBinaryOp, int(node.Token)) + c.emit(nd, OpBinaryOp, int(nd.Token)) } return nil } -func (c *Compiler) compileUnaryExpr(node *parser.UnaryExpr) error { - if err := c.Compile(node.Expr); err != nil { +func (c *Compiler) compileUnaryExpr(nd *node.UnaryExpr) error { + if err := c.Compile(nd.Expr); err != nil { return err } - switch node.Token { + switch nd.Token { case token.Not, token.Sub, token.Xor, token.Add: - c.emit(node, OpUnary, int(node.Token)) + c.emit(nd, OpUnary, int(nd.Token)) case token.Null: - c.emit(node, OpIsNil) + c.emit(nd, OpIsNil) case token.NotNull: - c.emit(node, OpNotIsNil) + c.emit(nd, OpNotIsNil) default: - return c.errorf(node, - "invalid unary operator: %s", node.Token.String()) + return c.errorf(nd, + "invalid unary operator: %s", nd.Token.String()) } return nil } -func (c *Compiler) compileSelectorExpr(node *parser.SelectorExpr) error { +func (c *Compiler) compileSelectorExpr(nd *node.SelectorExpr) error { defer c.pushSelector()() - expr, selectors := resolveSelectorExprs(node) + expr, selectors := resolveSelectorExprs(nd) if err := c.Compile(expr); err != nil { return err @@ -1077,7 +1116,7 @@ func (c *Compiler) compileSelectorExpr(node *parser.SelectorExpr) error { return err } } - c.emit(node, OpGetIndex, len(selectors)) + c.emit(nd, OpGetIndex, len(selectors)) return nil } @@ -1087,7 +1126,7 @@ func (c *Compiler) pushSelector() func() { stackLen = len(c.stack) ) switch c.stack[stackLen-2].(type) { - case *parser.SelectorExpr, *parser.NullishSelectorExpr: + case *node.SelectorExpr, *node.NullishSelectorExpr: default: increases = true c.selectorStack = append(c.selectorStack, nil) @@ -1111,10 +1150,10 @@ func (c *Compiler) selectorHandler(f func()) { c.selectorStack[l][0] = append(c.selectorStack[l][0], f) } -func (c *Compiler) compileNullishSelectorExpr(node *parser.NullishSelectorExpr) error { +func (c *Compiler) compileNullishSelectorExpr(nd *node.NullishSelectorExpr) error { defer c.pushSelector()() - expr, selectors := resolveSelectorExprs(node) + expr, selectors := resolveSelectorExprs(nd) var jumpPos int @@ -1128,7 +1167,7 @@ func (c *Compiler) compileNullishSelectorExpr(node *parser.NullishSelectorExpr) } } - jumpPos = c.emit(node, OpJumpNil, 0) + jumpPos = c.emit(nd, OpJumpNil, 0) c.selectorHandler(func() { c.changeOperand(jumpPos, len(c.instructions)) }) @@ -1136,25 +1175,25 @@ func (c *Compiler) compileNullishSelectorExpr(node *parser.NullishSelectorExpr) if err := c.Compile(selectors[len(selectors)-1]); err != nil { return err } - c.emit(node, OpGetIndex, len(selectors)) + c.emit(nd, OpGetIndex, len(selectors)) return nil } -func resolveSelectorExprs(node parser.Expr) (expr parser.Expr, selectors []parser.Expr) { - expr = node - switch v := node.(type) { - case *parser.SelectorExpr: +func resolveSelectorExprs(nd node.Expr) (expr node.Expr, selectors []node.Expr) { + expr = nd + switch v := nd.(type) { + case *node.SelectorExpr: expr, selectors = resolveIndexExprs(v.Expr) selectors = append(selectors, v.Sel) - case *parser.NullishSelectorExpr: + case *node.NullishSelectorExpr: expr, selectors = resolveIndexExprs(v.Expr) selectors = append(selectors, v.Sel) } return } -func (c *Compiler) compileIndexExpr(node *parser.IndexExpr) error { - expr, indexes := resolveIndexExprs(node) +func (c *Compiler) compileIndexExpr(nd *node.IndexExpr) error { + expr, indexes := resolveIndexExprs(nd) if err := c.Compile(expr); err != nil { return err } @@ -1163,56 +1202,56 @@ func (c *Compiler) compileIndexExpr(node *parser.IndexExpr) error { return err } } - c.emit(node, OpGetIndex, len(indexes)) + c.emit(nd, OpGetIndex, len(indexes)) return nil } -func resolveIndexExprs(node parser.Expr) (expr parser.Expr, indexes []parser.Expr) { - expr = node - if v, ok := node.(*parser.IndexExpr); ok { +func resolveIndexExprs(nd node.Expr) (expr node.Expr, indexes []node.Expr) { + expr = nd + if v, ok := nd.(*node.IndexExpr); ok { expr, indexes = resolveIndexExprs(v.Expr) indexes = append(indexes, v.Index) } return } -func (c *Compiler) compileSliceExpr(node *parser.SliceExpr) error { - if err := c.Compile(node.Expr); err != nil { +func (c *Compiler) compileSliceExpr(nd *node.SliceExpr) error { + if err := c.Compile(nd.Expr); err != nil { return err } - if node.Low != nil { - if err := c.Compile(node.Low); err != nil { + if nd.Low != nil { + if err := c.Compile(nd.Low); err != nil { return err } } else { - c.emit(node, OpNull) + c.emit(nd, OpNull) } - if node.High != nil { - if err := c.Compile(node.High); err != nil { + if nd.High != nil { + if err := c.Compile(nd.High); err != nil { return err } } else { - c.emit(node, OpNull) + c.emit(nd, OpNull) } - c.emit(node, OpSliceIndex) + c.emit(nd, OpSliceIndex) return nil } -func (c *Compiler) compileCallExpr(node *parser.CallExpr) error { +func (c *Compiler) compileCallExpr(nd *node.CallExpr) error { var ( - selExpr *parser.SelectorExpr + selExpr *node.SelectorExpr isSelector bool flags OpCallFlag op = OpCall - numArgs = len(node.Args.Values) + numArgs = len(nd.Args.Values) ) - if node.Func != nil { - selExpr, isSelector = node.Func.(*parser.SelectorExpr) + if nd.Func != nil { + selExpr, isSelector = nd.Func.(*node.SelectorExpr) } if isSelector { if err := c.Compile(selExpr.Expr); err != nil { @@ -1220,36 +1259,36 @@ func (c *Compiler) compileCallExpr(node *parser.CallExpr) error { } op = OpCallName } else { - if err := c.Compile(node.Func); err != nil { + if err := c.Compile(nd.Func); err != nil { return err } } - for _, arg := range node.Args.Values { + for _, arg := range nd.Args.Values { if err := c.Compile(arg); err != nil { return err } } - if node.Args.Ellipsis != nil { + if nd.Args.Ellipsis != nil { numArgs++ flags |= OpCallFlagVarArgs - if err := c.Compile(node.Args.Ellipsis.Value); err != nil { + if err := c.Compile(nd.Args.Ellipsis.Value); err != nil { return err } } - if numKwargs := len(node.NamedArgs.Names); numKwargs > 0 { + if numKwargs := len(nd.NamedArgs.Names); numKwargs > 0 { flags |= OpCallFlagNamedArgs - namedArgs := &parser.ArrayLit{Elements: make([]parser.Expr, numKwargs)} + namedArgs := &node.ArrayLit{Elements: make([]node.Expr, numKwargs)} - for i, name := range node.NamedArgs.Names { - value := node.NamedArgs.Values[i] + for i, name := range nd.NamedArgs.Names { + value := nd.NamedArgs.Values[i] if value == nil { // is flag - value = &parser.BoolLit{Value: true} + value = &node.BoolLit{Value: true} } - namedArgs.Elements[i] = &parser.ArrayLit{Elements: []parser.Expr{name.NameString(), value}} + namedArgs.Elements[i] = &node.ArrayLit{Elements: []node.Expr{name.NameString(), value}} } if err := c.Compile(namedArgs); err != nil { @@ -1257,9 +1296,9 @@ func (c *Compiler) compileCallExpr(node *parser.CallExpr) error { } } - if node.NamedArgs.Ellipsis != nil { + if nd.NamedArgs.Ellipsis != nil { flags |= OpCallFlagVarNamedArgs - if err := c.Compile(node.NamedArgs.Ellipsis.Value); err != nil { + if err := c.Compile(nd.NamedArgs.Ellipsis.Value); err != nil { return err } } @@ -1270,19 +1309,19 @@ func (c *Compiler) compileCallExpr(node *parser.CallExpr) error { } } - c.emit(node, op, numArgs, int(flags)) + c.emit(nd, op, numArgs, int(flags)) return nil } -func (c *Compiler) compileImportExpr(node *parser.ImportExpr) error { - moduleName := node.ModuleName +func (c *Compiler) compileImportExpr(nd *node.ImportExpr) error { + moduleName := nd.ModuleName if moduleName == "" { - return c.errorf(node, "empty module name") + return c.errorf(nd, "empty module name") } importer := c.moduleMap.Get(moduleName) if importer == nil { - return c.errorf(node, "module '%s' not found", moduleName) + return c.errorf(nd, "module '%s' not found", moduleName) } extImp, isExt := importer.(ExtImporter) @@ -1296,7 +1335,7 @@ func (c *Compiler) compileImportExpr(node *parser.ImportExpr) error { if !exists { mod, err := importer.Import(moduleName) if err != nil { - return c.error(node, err) + return c.error(nd, err) } switch v := mod.(type) { case []byte: @@ -1306,7 +1345,7 @@ func (c *Compiler) compileImportExpr(node *parser.ImportExpr) error { } else { moduleMap = c.baseModuleMap() } - cidx, err := c.compileModule(node, moduleName, moduleMap, v) + cidx, err := c.compileModule(nd, moduleName, moduleMap, v) if err != nil { return err } @@ -1314,7 +1353,7 @@ func (c *Compiler) compileImportExpr(node *parser.ImportExpr) error { case Object: module = c.addModule(moduleName, 2, c.addConstant(v)) default: - return c.errorf(node, "invalid import value type: %T", v) + return c.errorf(nd, "invalid import value type: %T", v) } } @@ -1331,40 +1370,40 @@ func (c *Compiler) compileImportExpr(node *parser.ImportExpr) error { // load module // if module is already stored, load from VM.modulesCache otherwise call compiled function // and store copy of result to VM.modulesCache. - c.emit(node, OpLoadModule, module.constantIndex, module.moduleIndex) - jumpPos := c.emit(node, OpJumpFalsy, 0) + c.emit(nd, OpLoadModule, module.constantIndex, module.moduleIndex) + jumpPos := c.emit(nd, OpJumpFalsy, 0) // modules should not accept parameters, to suppress the wrong number of arguments error // set all params to nil for i := 0; i < numParams; i++ { - c.emit(node, OpNull) + c.emit(nd, OpNull) } - c.emit(node, OpCall, numParams, 0) - c.emit(node, OpStoreModule, module.moduleIndex) + c.emit(nd, OpCall, numParams, 0) + c.emit(nd, OpStoreModule, module.moduleIndex) c.changeOperand(jumpPos, len(c.instructions)) case 2: // load module // if module is already stored, load from VM.modulesCache otherwise copy object // and store it to VM.modulesCache. - c.emit(node, OpLoadModule, module.constantIndex, module.moduleIndex) - jumpPos := c.emit(node, OpJumpFalsy, 0) - c.emit(node, OpStoreModule, module.moduleIndex) + c.emit(nd, OpLoadModule, module.constantIndex, module.moduleIndex) + jumpPos := c.emit(nd, OpJumpFalsy, 0) + c.emit(nd, OpStoreModule, module.moduleIndex) c.changeOperand(jumpPos, len(c.instructions)) default: - return c.errorf(node, "invalid module type: %v", module.typ) + return c.errorf(nd, "invalid module type: %v", module.typ) } return nil } -func (c *Compiler) compileCondExpr(node *parser.CondExpr) error { - if v, ok := node.Cond.(*parser.BoolLit); ok { +func (c *Compiler) compileCondExpr(nd *node.CondExpr) error { + if v, ok := nd.Cond.(*node.BoolLit); ok { if v.Value { - return c.Compile(node.True) + return c.Compile(nd.True) } - return c.Compile(node.False) + return c.Compile(nd.False) } op := OpJumpFalsy - if v, ok := simplifyExpr(node.Cond).(*parser.UnaryExpr); ok && v.Token.Is(token.Null, token.NotNull) { + if v, ok := simplifyExpr(nd.Cond).(*node.UnaryExpr); ok && v.Token.Is(token.Null, token.NotNull) { if err := c.Compile(v.Expr); err != nil { return err } @@ -1373,23 +1412,23 @@ func (c *Compiler) compileCondExpr(node *parser.CondExpr) error { if v.Token == token.NotNull { op = OpJumpNil } - } else if err := c.Compile(node.Cond); err != nil { + } else if err := c.Compile(nd.Cond); err != nil { return err } // first jump placeholder - jumpPos1 := c.emit(node, op, 0) - if err := c.Compile(node.True); err != nil { + jumpPos1 := c.emit(nd, op, 0) + if err := c.Compile(nd.True); err != nil { return err } // second jump placeholder - jumpPos2 := c.emit(node, OpJump, 0) + jumpPos2 := c.emit(nd, OpJump, 0) // update first jump offset curPos := len(c.instructions) c.changeOperand(jumpPos1, curPos) - if err := c.Compile(node.False); err != nil { + if err := c.Compile(nd.False); err != nil { return err } // update second jump offset @@ -1398,59 +1437,59 @@ func (c *Compiler) compileCondExpr(node *parser.CondExpr) error { return nil } -func (c *Compiler) compileIdent(node *parser.Ident) error { - symbol, ok := c.symbolTable.Resolve(node.Name) +func (c *Compiler) compileIdent(nd *node.Ident) error { + symbol, ok := c.symbolTable.Resolve(nd.Name) if !ok { - if c.iotaVal < 0 || node.Name != "iota" { - return c.errorf(node, "unresolved reference %q", node.Name) + if c.iotaVal < 0 || nd.Name != "iota" { + return c.errorf(nd, "unresolved reference %q", nd.Name) } - c.emit(node, OpConstant, c.addConstant(Int(c.iotaVal))) + c.emit(nd, OpConstant, c.addConstant(Int(c.iotaVal))) return nil } switch symbol.Scope { case ScopeGlobal: - c.emit(node, OpGetGlobal, symbol.Index) + c.emit(nd, OpGetGlobal, symbol.Index) case ScopeLocal: - c.emit(node, OpGetLocal, symbol.Index) + c.emit(nd, OpGetLocal, symbol.Index) case ScopeBuiltin: - c.emit(node, OpGetBuiltin, symbol.Index) + c.emit(nd, OpGetBuiltin, symbol.Index) case ScopeFree: - c.emit(node, OpGetFree, symbol.Index) + c.emit(nd, OpGetFree, symbol.Index) } return nil } -func (c *Compiler) compileArrayLit(node *parser.ArrayLit) error { - for _, elem := range node.Elements { +func (c *Compiler) compileArrayLit(nd *node.ArrayLit) error { + for _, elem := range nd.Elements { if err := c.Compile(elem); err != nil { return err } } - c.emit(node, OpArray, len(node.Elements)) + c.emit(nd, OpArray, len(nd.Elements)) return nil } -func (c *Compiler) compileMapLit(node *parser.MapLit) error { - for _, elt := range node.Elements { +func (c *Compiler) compileMapLit(nd *node.MapLit) error { + for _, elt := range nd.Elements { // key - c.emit(node, OpConstant, c.addConstant(String(elt.Key))) + c.emit(nd, OpConstant, c.addConstant(String(elt.Key))) // value if err := c.Compile(elt.Value); err != nil { return err } } - c.emit(node, OpMap, len(node.Elements)*2) + c.emit(nd, OpMap, len(nd.Elements)*2) return nil } -func (c *Compiler) compileKeyValueArrayLit(node *parser.KeyValueArrayLit) (err error) { - for _, elt := range node.Elements { +func (c *Compiler) compileKeyValueArrayLit(nd *node.KeyValueArrayLit) (err error) { + for _, elt := range nd.Elements { // key switch t := elt.Key.(type) { - case *parser.Ident: - c.emit(node, OpConstant, c.addConstant(String(t.Name))) + case *node.Ident: + c.emit(nd, OpConstant, c.addConstant(String(t.Name))) default: if err = c.Compile(elt.Key); err != nil { return @@ -1459,30 +1498,30 @@ func (c *Compiler) compileKeyValueArrayLit(node *parser.KeyValueArrayLit) (err e // value if elt.Value == nil { - c.emit(node, OpConstant, c.addConstant(True)) + c.emit(nd, OpConstant, c.addConstant(True)) } else if err = c.Compile(elt.Value); err != nil { return err } } - c.emit(node, OpKeyValueArray, len(node.Elements)*2) + c.emit(nd, OpKeyValueArray, len(nd.Elements)*2) return nil } -func (c *Compiler) helperBuildKwargsStmts(count int, get func(index int) (name string, value parser.Expr)) (stmts []parser.Stmt) { +func (c *Compiler) helperBuildKwargsStmts(count int, get func(index int) (name string, value node.Expr)) (stmts []node.Stmt) { for i := 0; i < count; i++ { name, value := get(i) - nameLit := &parser.StringLit{Literal: strconv.Quote(name), Value: name} - stmts = append(stmts, &parser.AssignStmt{ + nameLit := &node.StringLit{Literal: strconv.Quote(name), Value: name} + stmts = append(stmts, &node.AssignStmt{ Token: token.NullichAssign, - LHS: []parser.Expr{&parser.Ident{Name: name}}, - RHS: []parser.Expr{&parser.BinaryExpr{ + LHS: []node.Expr{&node.Ident{Name: name}}, + RHS: []node.Expr{&node.BinaryExpr{ Token: token.NullichCoalesce, - LHS: &parser.CallExpr{ - Func: &parser.NamedArgsKeyword{}, - Args: parser.CallExprArgs{ - Values: []parser.Expr{nameLit}, - }, + LHS: &node.CallExpr{ + Func: &node.NamedArgsKeyword{}, + CallArgs: node.CallArgs{Args: node.CallExprArgs{ + Values: []node.Expr{nameLit}, + }}, }, RHS: value, }}, @@ -1491,25 +1530,25 @@ func (c *Compiler) helperBuildKwargsStmts(count int, get func(index int) (name s return } -func (c *Compiler) helperBuildKwargsIfUndefinedStmts(count int, get func(index int) (name string, value parser.Expr)) (stmts []parser.Stmt) { +func (c *Compiler) helperBuildKwargsIfUndefinedStmts(count int, get func(index int) (name string, value node.Expr)) (stmts []node.Stmt) { for i := 0; i < count; i++ { name, value := get(i) - stmts = append(stmts, &parser.AssignStmt{ + stmts = append(stmts, &node.AssignStmt{ Token: token.NullichAssign, - LHS: []parser.Expr{&parser.Ident{Name: name}}, - RHS: []parser.Expr{value}, + LHS: []node.Expr{&node.Ident{Name: name}}, + RHS: []node.Expr{value}, }) } return } -func simplifyExpr(e parser.Expr) parser.Expr { +func simplifyExpr(e node.Expr) node.Expr { do: switch t := e.(type) { - case *parser.ParenExpr: + case *node.ParenExpr: switch t2 := t.Expr.(type) { - case *parser.ParenExpr, *parser.UnaryExpr: + case *node.ParenExpr, *node.UnaryExpr: e = t2 goto do } diff --git a/compiler_test.go b/compiler_test.go index edc4faa..1b4a16b 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -193,7 +193,7 @@ func TestCompiler_CompileIfNull(t *testing.T) { } func TestCompiler_Mixed(t *testing.T) { - expectCompileMixed(t, `# gad: writer=myfn; #{- var myfn -} a`, bytecode( + expectCompileMixed(t, "# gad: writer=myfn\n#{- var myfn -} a", bytecode( Array{String("a")}, compFunc(concatInsts( makeInst(OpNull), @@ -234,7 +234,7 @@ func TestCompiler_Mixed(t *testing.T) { ), withLocals(1)), )) - expectCompile(t, `# gad: mixed, writer=myfn; #{- var myfn -} a`, bytecode( + expectCompile(t, "# gad: mixed, writer=myfn\n#{ var myfn -} a", bytecode( Array{String("a")}, compFunc(concatInsts( makeInst(OpNull), @@ -246,7 +246,7 @@ func TestCompiler_Mixed(t *testing.T) { ), withLocals(1)), )) - expectCompile(t, `# gad: mixed; #{- a := begin} a #{end}`, bytecode( + expectCompile(t, "# gad: mixed\n#{- a := begin} a #{end}", bytecode( Array{String(" a ")}, compFunc(concatInsts( makeInst(OpConstant, 0), @@ -255,7 +255,7 @@ func TestCompiler_Mixed(t *testing.T) { ), withLocals(1)), )) - expectCompile(t, `# gad: mixed; #{- a := begin -} a #{- end}`, bytecode( + expectCompile(t, "# gad: mixed\n#{- a := begin -} a #{- end}", bytecode( Array{String("a")}, compFunc(concatInsts( makeInst(OpConstant, 0), @@ -264,7 +264,7 @@ func TestCompiler_Mixed(t *testing.T) { ), withLocals(1)), )) - expectCompile(t, `# gad: mixed; #{- a := begin -} a #{- end; return a}`, bytecode( + expectCompile(t, "# gad: mixed\n#{- a := begin -} a #{- end; return a}", bytecode( Array{String("a")}, compFunc(concatInsts( makeInst(OpConstant, 0), @@ -274,7 +274,7 @@ func TestCompiler_Mixed(t *testing.T) { ), withLocals(1)), )) - expectCompile(t, `# gad: mixed; #{- a := begin -} a #{- end}#{return a}`, bytecode( + expectCompile(t, "# gad: mixed\n#{- a := begin -} a #{- end}#{return a}", bytecode( Array{String("a")}, compFunc(concatInsts( makeInst(OpConstant, 0), @@ -284,7 +284,7 @@ func TestCompiler_Mixed(t *testing.T) { ), withLocals(1)), )) - expectCompile(t, `# gad: mixed; #{- a := begin -} a #{- end} b #{return a}`, bytecode( + expectCompile(t, "# gad: mixed\n#{- a := begin -} a #{- end} b #{return a}", bytecode( Array{String("a"), String(" b ")}, compFunc(concatInsts( makeInst(OpConstant, 0), @@ -1387,6 +1387,27 @@ func TestCompiler_Compile(t *testing.T) { )), )) + expectCompile(t, `func f () { 24 }; f();`, bytecode( + Array{ + Int(24), + compFunc(concatInsts( + makeInst(OpConstant, 0), + makeInst(OpPop), + makeInst(OpReturn, 0), + )), + }, + compFunc(concatInsts( + makeInst(OpConstant, 1), + makeInst(OpDefineLocal, 0), + makeInst(OpGetLocal, 0), + makeInst(OpCall, 0, 0), + makeInst(OpPop), + makeInst(OpReturn, 0), + ), + withLocals(1), + ), + )) + expectCompile(t, `f := func() { 24 }; f();`, bytecode( Array{ Int(24), @@ -1924,7 +1945,7 @@ func TestCompiler_Compile(t *testing.T) { withLocals(1), ), )) - expectCompileError(t, `try {}`, `Parse Error: expected 'finally', found ';'`) + expectCompileError(t, `try {};`, `Parse Error: expected 'finally', found ';'`) expectCompileError(t, `catch {}`, `Parse Error: expected statement, found 'catch'`) expectCompileError(t, `finally {}`, `Parse Error: expected statement, found 'finally'`) // catch and finally must in the same line with right brace. @@ -2229,6 +2250,45 @@ func TestCompiler_Compile(t *testing.T) { )) } +func TestCompilerFor(t *testing.T) { + expectCompile(t, `var r = ""; for x in [] { r += string(x) } else { r += "@"}; r+="#"; return r`, bytecode( + Array{String(""), String("@"), String("#")}, + compFunc(concatInsts( + makeInst(OpConstant, 0), + makeInst(OpDefineLocal, 0), + makeInst(OpArray, 0), + makeInst(OpIterInit), + makeInst(OpDefineLocal, 1), + makeInst(OpGetLocal, 1), + makeInst(OpIterNextElse, 24, 46), + makeInst(OpGetLocal, 1), + makeInst(OpIterNext), + makeInst(OpJumpFalsy, 55), + makeInst(OpGetLocal, 1), + makeInst(OpIterValue), + makeInst(OpDefineLocal, 2), + makeInst(OpGetLocal, 0), + makeInst(OpGetBuiltin, int(BuiltinString)), + makeInst(OpGetLocal, 2), + makeInst(OpCall, 1, 0), + makeInst(OpBinaryOp, int(token.Add)), + makeInst(OpSetLocal, 0), + makeInst(OpJump, 18), + makeInst(OpGetLocal, 0), + makeInst(OpConstant, 1), + makeInst(OpBinaryOp, int(token.Add)), + makeInst(OpSetLocal, 0), + makeInst(OpGetLocal, 0), + makeInst(OpConstant, 2), + makeInst(OpBinaryOp, int(token.Add)), + makeInst(OpSetLocal, 0), + makeInst(OpGetLocal, 0), + makeInst(OpReturn, 1), + makeInst(OpReturn, 0), + ), + withLocals(3), + ))) +} func TestCompilerNullishSelector(t *testing.T) { expectCompile(t, `var a; (a["I"+"DX"])?.d`, bytecode( Array{ diff --git a/objects.go b/objects.go index f4f962e..a082aed 100644 --- a/objects.go +++ b/objects.go @@ -17,6 +17,7 @@ import ( "github.com/gad-lang/gad/internal/compat" "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/source" "github.com/gad-lang/gad/token" ) @@ -1250,7 +1251,7 @@ func (o *Error) NewError(messages ...string) *Error { type RuntimeError struct { Err *Error fileSet *parser.SourceFileSet - Trace []parser.Pos + Trace []source.Pos } var ( @@ -1265,7 +1266,7 @@ func (o *RuntimeError) Unwrap() error { return nil } -func (o *RuntimeError) addTrace(pos parser.Pos) { +func (o *RuntimeError) addTrace(pos source.Pos) { if len(o.Trace) > 0 { if o.Trace[len(o.Trace)-1] == pos { return @@ -1292,7 +1293,7 @@ func (o *RuntimeError) Copy() Object { return &RuntimeError{ Err: err, fileSet: o.fileSet, - Trace: append([]parser.Pos{}, o.Trace...), + Trace: append([]source.Pos{}, o.Trace...), } } diff --git a/opcodes.go b/opcodes.go index 55bd405..4204737 100644 --- a/opcodes.go +++ b/opcodes.go @@ -54,6 +54,7 @@ const ( OpClosure OpIterInit OpIterNext + OpIterNextElse OpIterKey OpIterValue OpLoadModule @@ -114,6 +115,7 @@ var OpcodeNames = [...]string{ OpClosure: "CLOSURE", OpIterInit: "ITERINIT", OpIterNext: "ITERNEXT", + OpIterNextElse: "ITERNEXTELSE", OpIterKey: "ITERKEY", OpIterValue: "ITERVALUE", OpLoadModule: "LOADMODULE", @@ -175,6 +177,7 @@ var OpcodeOperands = [...][]int{ OpClosure: {2, 1}, // constant index, item count OpIterInit: {}, OpIterNext: {}, + OpIterNextElse: {2, 2}, // true pos, false pos OpIterKey: {}, OpIterValue: {}, OpLoadModule: {2, 2}, // constant index, module index diff --git a/optimizer.go b/optimizer.go index 73d8585..85ce793 100644 --- a/optimizer.go +++ b/optimizer.go @@ -12,13 +12,15 @@ import ( "strconv" "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/node" "github.com/gad-lang/gad/token" ) // OptimizerError represents an optimizer error. type OptimizerError struct { FilePos parser.SourceFilePos - Node parser.Node + Node ast.Node Err error } @@ -69,7 +71,7 @@ type SimpleOptimizer struct { constants []Object instructions []byte moduleStore *moduleStore - returnStmt parser.ReturnStmt + returnStmt node.ReturnStmt file *parser.File errors multipleErr trace io.Writer @@ -112,19 +114,19 @@ func NewOptimizer( } } -func canOptimizeExpr(expr parser.Expr) bool { - if parser.IsStatement(expr) { +func canOptimizeExpr(expr node.Expr) bool { + if node.IsStatement(expr) { return false } switch expr.(type) { - case *parser.BoolLit, - *parser.IntLit, - *parser.UintLit, - *parser.FloatLit, - *parser.CharLit, - *parser.StringLit, - *parser.NilLit: + case *node.BoolLit, + *node.IntLit, + *node.UintLit, + *node.FloatLit, + *node.CharLit, + *node.StringLit, + *node.NilLit: return false } return true @@ -189,7 +191,7 @@ func canOptimizeInsts(constants []Object, insts []byte) bool { return canOptimize } -func (so *SimpleOptimizer) evalExpr(expr parser.Expr) (parser.Expr, bool) { +func (so *SimpleOptimizer) evalExpr(expr node.Expr) (node.Expr, bool) { if !so.optimExpr { return nil, false } @@ -225,7 +227,7 @@ func (so *SimpleOptimizer) evalExpr(expr parser.Expr) (parser.Expr, bool) { return x, ok } -func (so *SimpleOptimizer) slowEvalExpr(expr parser.Expr) (parser.Expr, bool) { +func (so *SimpleOptimizer) slowEvalExpr(expr node.Expr) (node.Expr, bool) { st := NewSymbolTable(so.builtins). EnableParams(false). DisableBuiltin(so.disabledBuiltins...). @@ -276,44 +278,44 @@ func (so *SimpleOptimizer) slowEvalExpr(expr parser.Expr) (parser.Expr, bool) { switch v := obj.(type) { case String: l := strconv.Quote(string(v)) - expr = &parser.StringLit{ + expr = &node.StringLit{ Value: string(v), Literal: l, ValuePos: expr.Pos(), } case *NilType: - expr = &parser.NilLit{TokenPos: expr.Pos()} + expr = &node.NilLit{TokenPos: expr.Pos()} case Bool: l := strconv.FormatBool(bool(v)) - expr = &parser.BoolLit{ + expr = &node.BoolLit{ Value: bool(v), Literal: l, ValuePos: expr.Pos(), } case Int: l := strconv.FormatInt(int64(v), 10) - expr = &parser.IntLit{ + expr = &node.IntLit{ Value: int64(v), Literal: l, ValuePos: expr.Pos(), } case Uint: l := strconv.FormatUint(uint64(v), 10) - expr = &parser.UintLit{ + expr = &node.UintLit{ Value: uint64(v), Literal: l, ValuePos: expr.Pos(), } case Float: l := strconv.FormatFloat(float64(v), 'f', -1, 64) - expr = &parser.FloatLit{ + expr = &node.FloatLit{ Value: float64(v), Literal: l, ValuePos: expr.Pos(), } case Char: l := strconv.QuoteRune(rune(v)) - expr = &parser.CharLit{ + expr = &node.CharLit{ Value: rune(v), Literal: l, ValuePos: expr.Pos(), @@ -400,8 +402,8 @@ func (so *SimpleOptimizer) Optimize() error { func (so *SimpleOptimizer) binaryopInts( op token.Token, - left, right *parser.IntLit, -) (parser.Expr, bool) { + left, right *node.IntLit, +) (node.Expr, bool) { var val int64 switch op { @@ -432,13 +434,13 @@ func (so *SimpleOptimizer) binaryopInts( return nil, false } l := strconv.FormatInt(val, 10) - return &parser.IntLit{Value: val, Literal: l, ValuePos: left.ValuePos}, true + return &node.IntLit{Value: val, Literal: l, ValuePos: left.ValuePos}, true } func (so *SimpleOptimizer) binaryopFloats( op token.Token, - left, right *parser.FloatLit, -) (parser.Expr, bool) { + left, right *node.FloatLit, +) (node.Expr, bool) { var val float64 switch op { @@ -457,7 +459,7 @@ func (so *SimpleOptimizer) binaryopFloats( return nil, false } - return &parser.FloatLit{ + return &node.FloatLit{ Value: val, Literal: strconv.FormatFloat(val, 'f', -1, 64), ValuePos: left.ValuePos, @@ -466,27 +468,27 @@ func (so *SimpleOptimizer) binaryopFloats( func (so *SimpleOptimizer) binaryop( op token.Token, - left, right parser.Expr, -) (parser.Expr, bool) { + left, right node.Expr, +) (node.Expr, bool) { if !so.optimConsts { return nil, false } switch left := left.(type) { - case *parser.IntLit: - if right, ok := right.(*parser.IntLit); ok { + case *node.IntLit: + if right, ok := right.(*node.IntLit); ok { return so.binaryopInts(op, left, right) } - case *parser.FloatLit: - if right, ok := right.(*parser.FloatLit); ok { + case *node.FloatLit: + if right, ok := right.(*node.FloatLit); ok { return so.binaryopFloats(op, left, right) } - case *parser.StringLit: - right, ok := right.(*parser.StringLit) + case *node.StringLit: + right, ok := right.(*node.StringLit) if ok && op == token.Add { v := left.Value + right.Value - return &parser.StringLit{ + return &node.StringLit{ Value: v, Literal: strconv.Quote(v), ValuePos: left.ValuePos, @@ -498,19 +500,19 @@ func (so *SimpleOptimizer) binaryop( func (so *SimpleOptimizer) unaryop( op token.Token, - expr parser.Expr, -) (parser.Expr, bool) { + expr node.Expr, +) (node.Expr, bool) { if !so.optimConsts { return nil, false } switch expr := expr.(type) { - case *parser.IntLit: + case *node.IntLit: switch op { case token.Not: v := expr.Value == 0 - return &parser.BoolLit{ + return &node.BoolLit{ Value: v, Literal: strconv.FormatBool(v), ValuePos: expr.ValuePos, @@ -518,7 +520,7 @@ func (so *SimpleOptimizer) unaryop( case token.Sub: v := -expr.Value l := strconv.FormatInt(v, 10) - return &parser.IntLit{ + return &node.IntLit{ Value: v, Literal: l, ValuePos: expr.ValuePos, @@ -526,17 +528,17 @@ func (so *SimpleOptimizer) unaryop( case token.Xor: v := ^expr.Value l := strconv.FormatInt(v, 10) - return &parser.IntLit{ + return &node.IntLit{ Value: v, Literal: l, ValuePos: expr.ValuePos, }, true } - case *parser.UintLit: + case *node.UintLit: switch op { case token.Not: v := expr.Value == 0 - return &parser.BoolLit{ + return &node.BoolLit{ Value: v, Literal: strconv.FormatBool(v), ValuePos: expr.ValuePos, @@ -544,7 +546,7 @@ func (so *SimpleOptimizer) unaryop( case token.Sub: v := -expr.Value l := strconv.FormatUint(v, 10) - return &parser.UintLit{ + return &node.UintLit{ Value: v, Literal: l, ValuePos: expr.ValuePos, @@ -552,18 +554,18 @@ func (so *SimpleOptimizer) unaryop( case token.Xor: v := ^expr.Value l := strconv.FormatUint(v, 10) - return &parser.UintLit{ + return &node.UintLit{ Value: v, Literal: l, ValuePos: expr.ValuePos, }, true } - case *parser.FloatLit: + case *node.FloatLit: switch op { case token.Sub: v := -expr.Value l := strconv.FormatFloat(v, 'f', -1, 64) - return &parser.FloatLit{ + return &node.FloatLit{ Value: v, Literal: l, ValuePos: expr.ValuePos, @@ -573,166 +575,169 @@ func (so *SimpleOptimizer) unaryop( return nil, false } -func (so *SimpleOptimizer) optimize(node parser.Node) (parser.Expr, bool) { +func (so *SimpleOptimizer) optimize(nd ast.Node) (node.Expr, bool) { if so.trace != nil { - if node != nil { + if nd != nil { defer untraceoptim(traceoptim(so, fmt.Sprintf("%s (%s)", - node.String(), reflect.TypeOf(node).Elem().Name()))) + nd.String(), reflect.TypeOf(nd).Elem().Name()))) } else { defer untraceoptim(traceoptim(so, "")) } } - if !parser.IsStatement(node) { + if !node.IsStatement(nd) { so.enterExprLevel() defer so.leaveExprLevel() } var ( - expr parser.Expr + expr node.Expr ok bool ) - switch node := node.(type) { + switch nd := nd.(type) { case *parser.File: - for _, stmt := range node.Stmts { + for _, stmt := range nd.Stmts { _, _ = so.optimize(stmt) } - case *parser.ExprStmt: - if node.Expr != nil { - if expr, ok = so.optimize(node.Expr); ok { - node.Expr = expr + case *node.ExprStmt: + if nd.Expr != nil { + if expr, ok = so.optimize(nd.Expr); ok { + nd.Expr = expr } - if expr, ok = so.evalExpr(node.Expr); ok { - node.Expr = expr + if expr, ok = so.evalExpr(nd.Expr); ok { + nd.Expr = expr } } - case *parser.ParenExpr: - if node.Expr != nil { - return so.optimize(node.Expr) + case *node.ParenExpr: + if nd.Expr != nil { + return so.optimize(nd.Expr) } - case *parser.BinaryExpr: - if expr, ok = so.optimize(node.LHS); ok { - node.LHS = expr + case *node.BinaryExpr: + if expr, ok = so.optimize(nd.LHS); ok { + nd.LHS = expr } - if expr, ok = so.optimize(node.RHS); ok { - node.RHS = expr + if expr, ok = so.optimize(nd.RHS); ok { + nd.RHS = expr } - if expr, ok = so.binaryop(node.Token, node.LHS, node.RHS); ok { + if expr, ok = so.binaryop(nd.Token, nd.LHS, nd.RHS); ok { so.count++ return expr, ok } - return so.evalExpr(node) - case *parser.UnaryExpr: - if expr, ok = so.optimize(node.Expr); ok { - node.Expr = expr + return so.evalExpr(nd) + case *node.UnaryExpr: + if expr, ok = so.optimize(nd.Expr); ok { + nd.Expr = expr } - if expr, ok = so.unaryop(node.Token, node.Expr); ok { + if expr, ok = so.unaryop(nd.Token, nd.Expr); ok { so.count++ return expr, ok } - return so.evalExpr(node) - case *parser.IfStmt: - if node.Init != nil { - _, _ = so.optimize(node.Init) + return so.evalExpr(nd) + case *node.IfStmt: + if nd.Init != nil { + _, _ = so.optimize(nd.Init) } - if expr, ok = so.optimize(node.Cond); ok { - node.Cond = expr + if expr, ok = so.optimize(nd.Cond); ok { + nd.Cond = expr } - if expr, ok = so.evalExpr(node.Cond); ok { - node.Cond = expr + if expr, ok = so.evalExpr(nd.Cond); ok { + nd.Cond = expr } - if falsy, ok := isLitFalsy(node.Cond); ok { + if falsy, ok := isLitFalsy(nd.Cond); ok { // convert expression to BoolLit so that Compiler skips if block - node.Cond = &parser.BoolLit{ + nd.Cond = &node.BoolLit{ Value: !falsy, Literal: strconv.FormatBool(!falsy), - ValuePos: node.Cond.Pos(), + ValuePos: nd.Cond.Pos(), } } - if node.Body != nil { - _, _ = so.optimize(node.Body) + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - if node.Else != nil { - _, _ = so.optimize(node.Else) + if nd.Else != nil { + _, _ = so.optimize(nd.Else) } - case *parser.TryStmt: - if node.Body != nil { - _, _ = so.optimize(node.Body) + case *node.TryStmt: + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - if node.Catch != nil { - _, _ = so.optimize(node.Catch) + if nd.Catch != nil { + _, _ = so.optimize(nd.Catch) } - if node.Finally != nil { - _, _ = so.optimize(node.Finally) + if nd.Finally != nil { + _, _ = so.optimize(nd.Finally) } - case *parser.CatchStmt: - if node.Body != nil { - _, _ = so.optimize(node.Body) + case *node.CatchStmt: + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - case *parser.FinallyStmt: - if node.Body != nil { - _, _ = so.optimize(node.Body) + case *node.FinallyStmt: + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - case *parser.ThrowStmt: - if node.Expr != nil { - if expr, ok = so.optimize(node.Expr); ok { - node.Expr = expr + case *node.ThrowStmt: + if nd.Expr != nil { + if expr, ok = so.optimize(nd.Expr); ok { + nd.Expr = expr } - if expr, ok = so.evalExpr(node.Expr); ok { - node.Expr = expr + if expr, ok = so.evalExpr(nd.Expr); ok { + nd.Expr = expr } } - case *parser.ForStmt: - if node.Init != nil { - _, _ = so.optimize(node.Init) + case *node.ForStmt: + if nd.Init != nil { + _, _ = so.optimize(nd.Init) } - if node.Cond != nil { - if expr, ok = so.optimize(node.Cond); ok { - node.Cond = expr + if nd.Cond != nil { + if expr, ok = so.optimize(nd.Cond); ok { + nd.Cond = expr } } - if node.Post != nil { - _, _ = so.optimize(node.Post) + if nd.Post != nil { + _, _ = so.optimize(nd.Post) } - if node.Body != nil { - _, _ = so.optimize(node.Body) + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - case *parser.ForInStmt: - if node.Body != nil { - _, _ = so.optimize(node.Body) + case *node.ForInStmt: + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - case *parser.BlockStmt: - for _, stmt := range node.Stmts { + if nd.Else != nil { + _, _ = so.optimize(nd.Else) + } + case *node.BlockStmt: + for _, stmt := range nd.Stmts { _, _ = so.optimize(stmt) } - case *parser.AssignStmt: - for _, lhs := range node.LHS { - if ident, ok := lhs.(*parser.Ident); ok { + case *node.AssignStmt: + for _, lhs := range nd.LHS { + if ident, ok := lhs.(*node.Ident); ok { so.scope.define(ident.Name) } } - for i, rhs := range node.RHS { + for i, rhs := range nd.RHS { if expr, ok = so.optimize(rhs); ok { - node.RHS[i] = expr + nd.RHS[i] = expr } } - for i, rhs := range node.RHS { + for i, rhs := range nd.RHS { if expr, ok = so.evalExpr(rhs); ok { - node.RHS[i] = expr + nd.RHS[i] = expr } } - case *parser.DeclStmt: - decl := node.Decl.(*parser.GenDecl) + case *node.DeclStmt: + decl := nd.Decl.(*node.GenDecl) switch decl.Tok { case token.Param, token.Global: for _, sp := range decl.Specs { - spec := sp.(*parser.ParamSpec) + spec := sp.(*node.ParamSpec) so.scope.define(spec.Ident.Name) } case token.Var, token.Const: for _, sp := range decl.Specs { - spec := sp.(*parser.ValueSpec) + spec := sp.(*node.ValueSpec) for i := range spec.Idents { so.scope.define(spec.Idents[i].Name) if i < len(spec.Values) && spec.Values[i] != nil { @@ -748,116 +753,116 @@ func (so *SimpleOptimizer) optimize(node parser.Node) (parser.Expr, bool) { } } } - case *parser.ArrayLit: - for i := range node.Elements { - if expr, ok = so.optimize(node.Elements[i]); ok { - node.Elements[i] = expr + case *node.ArrayLit: + for i := range nd.Elements { + if expr, ok = so.optimize(nd.Elements[i]); ok { + nd.Elements[i] = expr } - if expr, ok = so.evalExpr(node.Elements[i]); ok { - node.Elements[i] = expr + if expr, ok = so.evalExpr(nd.Elements[i]); ok { + nd.Elements[i] = expr } } - case *parser.MapLit: - for i := range node.Elements { - if expr, ok = so.optimize(node.Elements[i].Value); ok { - node.Elements[i].Value = expr + case *node.MapLit: + for i := range nd.Elements { + if expr, ok = so.optimize(nd.Elements[i].Value); ok { + nd.Elements[i].Value = expr } - if expr, ok = so.evalExpr(node.Elements[i].Value); ok { - node.Elements[i].Value = expr + if expr, ok = so.evalExpr(nd.Elements[i].Value); ok { + nd.Elements[i].Value = expr } } - case *parser.IndexExpr: - if expr, ok = so.optimize(node.Index); ok { - node.Index = expr + case *node.IndexExpr: + if expr, ok = so.optimize(nd.Index); ok { + nd.Index = expr } - if expr, ok = so.evalExpr(node.Index); ok { - node.Index = expr + if expr, ok = so.evalExpr(nd.Index); ok { + nd.Index = expr } - case *parser.SliceExpr: - if node.Low != nil { - if expr, ok = so.optimize(node.Low); ok { - node.Low = expr + case *node.SliceExpr: + if nd.Low != nil { + if expr, ok = so.optimize(nd.Low); ok { + nd.Low = expr } - if expr, ok = so.evalExpr(node.Low); ok { - node.Low = expr + if expr, ok = so.evalExpr(nd.Low); ok { + nd.Low = expr } } - if node.High != nil { - if expr, ok = so.optimize(node.High); ok { - node.High = expr + if nd.High != nil { + if expr, ok = so.optimize(nd.High); ok { + nd.High = expr } - if expr, ok = so.evalExpr(node.High); ok { - node.High = expr + if expr, ok = so.evalExpr(nd.High); ok { + nd.High = expr } } - case *parser.FuncLit: + case *node.FuncLit: so.enterScope() defer so.leaveScope() - for _, ident := range node.Type.Params.Args.Values { + for _, ident := range nd.Type.Params.Args.Values { so.scope.define(ident.Name) } - for _, ident := range node.Type.Params.NamedArgs.Names { + for _, ident := range nd.Type.Params.NamedArgs.Names { so.scope.define(ident.Name) } - if node.Body != nil { - _, _ = so.optimize(node.Body) + if nd.Body != nil { + _, _ = so.optimize(nd.Body) } - case *parser.ReturnStmt: - if node.Result != nil { - if expr, ok = so.optimize(node.Result); ok { - node.Result = expr + case *node.ReturnStmt: + if nd.Result != nil { + if expr, ok = so.optimize(nd.Result); ok { + nd.Result = expr } - if expr, ok = so.evalExpr(node.Result); ok { - node.Result = expr + if expr, ok = so.evalExpr(nd.Result); ok { + nd.Result = expr } } - case *parser.CallExpr: - if node.Func != nil { - _, _ = so.optimize(node.Func) + case *node.CallExpr: + if nd.Func != nil { + _, _ = so.optimize(nd.Func) } - for i := range node.Args.Values { - if expr, ok = so.optimize(node.Args.Values[i]); ok { - node.Args.Values[i] = expr + for i := range nd.Args.Values { + if expr, ok = so.optimize(nd.Args.Values[i]); ok { + nd.Args.Values[i] = expr } - if expr, ok = so.evalExpr(node.Args.Values[i]); ok { - node.Args.Values[i] = expr + if expr, ok = so.evalExpr(nd.Args.Values[i]); ok { + nd.Args.Values[i] = expr } } - for i := range node.NamedArgs.Values { - if expr, ok = so.optimize(node.NamedArgs.Values[i]); ok { - node.NamedArgs.Values[i] = expr + for i := range nd.NamedArgs.Values { + if expr, ok = so.optimize(nd.NamedArgs.Values[i]); ok { + nd.NamedArgs.Values[i] = expr } - if expr, ok = so.evalExpr(node.NamedArgs.Values[i]); ok { - node.NamedArgs.Values[i] = expr + if expr, ok = so.evalExpr(nd.NamedArgs.Values[i]); ok { + nd.NamedArgs.Values[i] = expr } } - case *parser.CondExpr: - if expr, ok = so.optimize(node.Cond); ok { - node.Cond = expr + case *node.CondExpr: + if expr, ok = so.optimize(nd.Cond); ok { + nd.Cond = expr } - if expr, ok = so.evalExpr(node.Cond); ok { - node.Cond = expr + if expr, ok = so.evalExpr(nd.Cond); ok { + nd.Cond = expr } - if falsy, ok := isLitFalsy(node.Cond); ok { + if falsy, ok := isLitFalsy(nd.Cond); ok { // convert expression to BoolLit so that Compiler skips expressions - node.Cond = &parser.BoolLit{ + nd.Cond = &node.BoolLit{ Value: !falsy, Literal: strconv.FormatBool(!falsy), - ValuePos: node.Cond.Pos(), + ValuePos: nd.Cond.Pos(), } } - if expr, ok = so.optimize(node.True); ok { - node.True = expr + if expr, ok = so.optimize(nd.True); ok { + nd.True = expr } - if expr, ok = so.evalExpr(node.True); ok { - node.True = expr + if expr, ok = so.evalExpr(nd.True); ok { + nd.True = expr } - if expr, ok = so.optimize(node.False); ok { - node.False = expr + if expr, ok = so.optimize(nd.False); ok { + nd.False = expr } - if expr, ok = so.evalExpr(node.False); ok { - node.False = expr + if expr, ok = so.evalExpr(nd.False); ok { + nd.False = expr } } return nil, false @@ -876,11 +881,11 @@ func (so *SimpleOptimizer) Total() int { return so.total } -func (so *SimpleOptimizer) error(node parser.Node, err error) error { - pos := so.file.InputFile.Set().Position(node.Pos()) +func (so *SimpleOptimizer) error(nd ast.Node, err error) error { + pos := so.file.InputFile.Set().Position(nd.Pos()) return &OptimizerError{ FilePos: pos, - Node: node, + Node: nd, Err: err, } } @@ -921,25 +926,25 @@ func isObjectConstant(obj Object) bool { return false } -func isLitFalsy(expr parser.Expr) (bool, bool) { +func isLitFalsy(expr node.Expr) (bool, bool) { if expr == nil { return false, false } switch v := expr.(type) { - case *parser.BoolLit: + case *node.BoolLit: return !v.Value, true - case *parser.IntLit: + case *node.IntLit: return Int(v.Value).IsFalsy(), true - case *parser.UintLit: + case *node.UintLit: return Uint(v.Value).IsFalsy(), true - case *parser.FloatLit: + case *node.FloatLit: return Float(v.Value).IsFalsy(), true - case *parser.StringLit: + case *node.StringLit: return String(v.Value).IsFalsy(), true - case *parser.CharLit: + case *node.CharLit: return Char(v.Value).IsFalsy(), true - case *parser.NilLit: + case *node.NilLit: return Nil.IsFalsy(), true } return false, false diff --git a/parser/ast.go b/parser/ast/ast.go similarity index 69% rename from parser/ast.go rename to parser/ast/ast.go index 69ce4b6..8df09f7 100644 --- a/parser/ast.go +++ b/parser/ast/ast.go @@ -10,91 +10,39 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE.golang file. -package parser +package ast import ( "strings" -) -const ( - nullRep = "" + "github.com/gad-lang/gad/parser/source" ) // Node represents a node in the AST. type Node interface { // Pos returns the position of first character belonging to the node. - Pos() Pos + Pos() source.Pos // End returns the position of first character immediately after the node. - End() Pos + End() source.Pos // String returns a string representation of the node. String() string } -// IdentList represents a list of identifiers. -type IdentList struct { - LParen Pos - VarArgs bool - List []*Ident - RParen Pos -} - -// Pos returns the position of first character belonging to the node. -func (n *IdentList) Pos() Pos { - if n.LParen.IsValid() { - return n.LParen - } - if len(n.List) > 0 { - return n.List[0].Pos() - } - return NoPos -} - -// End returns the position of first character immediately after the node. -func (n *IdentList) End() Pos { - if n.RParen.IsValid() { - return n.RParen + 1 - } - if l := len(n.List); l > 0 { - return n.List[l-1].End() - } - return NoPos -} - -// NumFields returns the number of fields. -func (n *IdentList) NumFields() int { - if n == nil { - return 0 - } - return len(n.List) -} - -func (n *IdentList) String() string { - var list []string - for i, e := range n.List { - if n.VarArgs && i == len(n.List)-1 { - list = append(list, "..."+e.String()) - } else { - list = append(list, e.String()) - } - } - return "(" + strings.Join(list, ", ") + ")" -} - // ---------------------------------------------------------------------------- // Comments // A Comment node represents a single //-style or /*-style comment. type Comment struct { - Slash Pos // position of "/" starting the comment - Text string // comment text (excluding '\n' for //-style comments) + Slash source.Pos // position of "/" starting the comment + Text string // comment text (excluding '\n' for //-style comments) } // Pos returns the position of the comment's slash. -func (c *Comment) Pos() Pos { return c.Slash } +func (c *Comment) Pos() source.Pos { return c.Slash } // End returns the position of first character immediately after the comment. -func (c *Comment) End() Pos { - return Pos(int(c.Slash) + len(c.Text)) +func (c *Comment) End() source.Pos { + return source.Pos(int(c.Slash) + len(c.Text)) } // A CommentGroup represents a sequence of comments @@ -104,12 +52,12 @@ type CommentGroup struct { } // Pos returns the position of the first comment. -func (g *CommentGroup) Pos() Pos { +func (g *CommentGroup) Pos() source.Pos { return g.List[0].Pos() } // End returns the position of last comment's end position. -func (g *CommentGroup) End() Pos { +func (g *CommentGroup) End() source.Pos { return g.List[len(g.List)-1].End() } @@ -190,9 +138,9 @@ func stripTrailingWhitespace(s string) string { type Literal struct { Value string - Pos Pos + Pos source.Pos } -func (l Literal) End() Pos { - return l.Pos + Pos(len(l.Value)) +func (l Literal) End() source.Pos { + return l.Pos + source.Pos(len(l.Value)) } diff --git a/parser/ast_test.go b/parser/ast_test.go deleted file mode 100644 index e8922ff..0000000 --- a/parser/ast_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package parser_test - -import ( - "testing" - - "github.com/gad-lang/gad/parser" -) - -func TestIdentListString(t *testing.T) { - identListVar := &parser.IdentList{ - List: []*parser.Ident{ - {Name: "a"}, - {Name: "b"}, - {Name: "c"}, - }, - VarArgs: true, - } - - expectedVar := "(a, b, ...c)" - if str := identListVar.String(); str != expectedVar { - t.Fatalf("expected string of %#v to be %s, got %s", - identListVar, expectedVar, str) - } - - identList := &parser.IdentList{ - List: []*parser.Ident{ - {Name: "a"}, - {Name: "b"}, - {Name: "c"}, - }, - VarArgs: false, - } - - expected := "(a, b, c)" - if str := identList.String(); str != expected { - t.Fatalf("expected string of %#v to be %s, got %s", - identList, expected, str) - } -} diff --git a/parser/file.go b/parser/file.go index 629b063..1278e60 100644 --- a/parser/file.go +++ b/parser/file.go @@ -14,29 +14,36 @@ package parser import ( "strings" + + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/node" + "github.com/gad-lang/gad/parser/source" + "github.com/gad-lang/gad/stringw" ) // File represents a file unit. type File struct { InputFile *SourceFile - Stmts []Stmt - Comments []*CommentGroup + Stmts []node.Stmt + Comments []*ast.CommentGroup } // Pos returns the position of first character belonging to the node. -func (n *File) Pos() Pos { - return Pos(n.InputFile.Base) +func (n *File) Pos() source.Pos { + return source.Pos(n.InputFile.Base) } // End returns the position of first character immediately after the node. -func (n *File) End() Pos { - return Pos(n.InputFile.Base + n.InputFile.Size) +func (n *File) End() source.Pos { + return source.Pos(n.InputFile.Base + n.InputFile.Size) +} + +func (n *File) StringTo(w stringw.StringWriter) { + stringw.ToStringSlice(w, "; ", n.Stmts) } func (n *File) String() string { - var stmts []string - for _, e := range n.Stmts { - stmts = append(stmts, e.String()) - } - return strings.Join(stmts, "; ") + var s strings.Builder + n.StringTo(&s) + return s.String() } diff --git a/parser/expr.go b/parser/node/expr.go similarity index 68% rename from parser/expr.go rename to parser/node/expr.go index cfbd985..c823be8 100644 --- a/parser/expr.go +++ b/parser/node/expr.go @@ -10,38 +10,42 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE.golang file. -package parser +package node import ( "bytes" + "io" "strings" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/source" + "github.com/gad-lang/gad/parser/utils" "github.com/gad-lang/gad/token" "github.com/shopspring/decimal" ) // Expr represents an expression node in the AST. type Expr interface { - Node - exprNode() + ast.Node + ExprNode() } // ArrayLit represents an array literal. type ArrayLit struct { Elements []Expr - LBrack Pos - RBrack Pos + LBrack source.Pos + RBrack source.Pos } -func (e *ArrayLit) exprNode() {} +func (e *ArrayLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *ArrayLit) Pos() Pos { +func (e *ArrayLit) Pos() source.Pos { return e.LBrack } // End returns the position of first character immediately after the node. -func (e *ArrayLit) End() Pos { +func (e *ArrayLit) End() source.Pos { return e.RBrack + 1 } @@ -55,19 +59,19 @@ func (e *ArrayLit) String() string { // BadExpr represents a bad expression. type BadExpr struct { - From Pos - To Pos + From source.Pos + To source.Pos } -func (e *BadExpr) exprNode() {} +func (e *BadExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *BadExpr) Pos() Pos { +func (e *BadExpr) Pos() source.Pos { return e.From } // End returns the position of first character immediately after the node. -func (e *BadExpr) End() Pos { +func (e *BadExpr) End() source.Pos { return e.To } @@ -80,18 +84,18 @@ type BinaryExpr struct { LHS Expr RHS Expr Token token.Token - TokenPos Pos + TokenPos source.Pos } -func (e *BinaryExpr) exprNode() {} +func (e *BinaryExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *BinaryExpr) Pos() Pos { +func (e *BinaryExpr) Pos() source.Pos { return e.LHS.Pos() } // End returns the position of first character immediately after the node. -func (e *BinaryExpr) End() Pos { +func (e *BinaryExpr) End() source.Pos { return e.RHS.End() } @@ -103,78 +107,108 @@ func (e *BinaryExpr) String() string { // BoolLit represents a boolean literal. type BoolLit struct { Value bool - ValuePos Pos + ValuePos source.Pos Literal string } -func (e *BoolLit) exprNode() {} +func (e *BoolLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *BoolLit) Pos() Pos { +func (e *BoolLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *BoolLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *BoolLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *BoolLit) String() string { return e.Literal } -// CallExpr represents a function call expression. -type CallExpr struct { - Func Expr - LParen Pos +type CallArgs struct { + LParen source.Pos Args CallExprArgs NamedArgs CallExprNamedArgs - RParen Pos + RParen source.Pos +} + +// Pos returns the position of first character belonging to the node. +func (c *CallArgs) Pos() source.Pos { + return c.LParen +} + +// End returns the position of first character immediately after the node. +func (c *CallArgs) End() source.Pos { + return c.RParen + 1 +} + +func (c *CallArgs) String() string { + var buf strings.Builder + c.StringW(&buf) + return buf.String() +} + +func (c *CallArgs) StringW(w io.Writer) { + c.StringArg(w, "(", ")", true) +} + +func (c *CallArgs) StringArg(w io.Writer, lbrace, rbrace string, nasep bool) { + io.WriteString(w, lbrace) + if c.Args.Valid() { + io.WriteString(w, c.Args.String()) + } + if c.NamedArgs.Valid() { + if nasep { + io.WriteString(w, "; ") + } + io.WriteString(w, c.NamedArgs.String()) + } + io.WriteString(w, rbrace) +} + +// CallExpr represents a function call expression. +type CallExpr struct { + Func Expr + CallArgs } -func (e *CallExpr) exprNode() {} +func (e *CallExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *CallExpr) Pos() Pos { +func (e *CallExpr) Pos() source.Pos { return e.Func.Pos() } // End returns the position of first character immediately after the node. -func (e *CallExpr) End() Pos { +func (e *CallExpr) End() source.Pos { return e.RParen + 1 } func (e *CallExpr) String() string { var buf = bytes.NewBufferString(e.Func.String()) - buf.WriteString("(") - if e.Args.Valid() { - buf.WriteString(e.Args.String()) - } - if e.NamedArgs.Valid() { - buf.WriteString("; ") - buf.WriteString(e.NamedArgs.String()) - } - buf.WriteString(")") + e.CallArgs.StringW(buf) return buf.String() } // CharLit represents a character literal. type CharLit struct { Value rune - ValuePos Pos + ValuePos source.Pos Literal string } -func (e *CharLit) exprNode() {} +func (e *CharLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *CharLit) Pos() Pos { +func (e *CharLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *CharLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *CharLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *CharLit) String() string { @@ -186,19 +220,19 @@ type CondExpr struct { Cond Expr True Expr False Expr - QuestionPos Pos - ColonPos Pos + QuestionPos source.Pos + ColonPos source.Pos } -func (e *CondExpr) exprNode() {} +func (e *CondExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *CondExpr) Pos() Pos { +func (e *CondExpr) Pos() source.Pos { return e.Cond.Pos() } // End returns the position of first character immediately after the node. -func (e *CondExpr) End() Pos { +func (e *CondExpr) End() source.Pos { return e.False.End() } @@ -210,20 +244,20 @@ func (e *CondExpr) String() string { // FloatLit represents a floating point literal. type FloatLit struct { Value float64 - ValuePos Pos + ValuePos source.Pos Literal string } -func (e *FloatLit) exprNode() {} +func (e *FloatLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *FloatLit) Pos() Pos { +func (e *FloatLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *FloatLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *FloatLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *FloatLit) String() string { @@ -233,20 +267,20 @@ func (e *FloatLit) String() string { // DecimalLit represents a floating point literal. type DecimalLit struct { Value decimal.Decimal - ValuePos Pos + ValuePos source.Pos Literal string } -func (e *DecimalLit) exprNode() {} +func (e *DecimalLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *DecimalLit) Pos() Pos { +func (e *DecimalLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *DecimalLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *DecimalLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *DecimalLit) String() string { @@ -259,20 +293,20 @@ type FuncLit struct { Body *BlockStmt } -func (e *FuncLit) exprNode() {} +func (e *FuncLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *FuncLit) Pos() Pos { +func (e *FuncLit) Pos() source.Pos { return e.Type.Pos() } // End returns the position of first character immediately after the node. -func (e *FuncLit) End() Pos { +func (e *FuncLit) End() source.Pos { return e.Body.End() } func (e *FuncLit) String() string { - return "func" + e.Type.Params.String() + " " + e.Body.String() + return "func" + e.Type.String() + " " + e.Body.String() } // ClosureLit represents a function closure literal. @@ -281,15 +315,15 @@ type ClosureLit struct { Body Expr } -func (e *ClosureLit) exprNode() {} +func (e *ClosureLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *ClosureLit) Pos() Pos { +func (e *ClosureLit) Pos() source.Pos { return e.Type.Pos() } // End returns the position of first character immediately after the node. -func (e *ClosureLit) End() Pos { +func (e *ClosureLit) End() source.Pos { return e.Body.End() } @@ -304,23 +338,23 @@ type ArgsList struct { } // Pos returns the position of first character belonging to the node. -func (n *ArgsList) Pos() Pos { +func (n *ArgsList) Pos() source.Pos { if len(n.Values) > 0 { return n.Values[0].Pos() } else if n.Var != nil { return n.Var.Pos() } - return NoPos + return source.NoPos } // End returns the position of first character immediately after the node. -func (n *ArgsList) End() Pos { +func (n *ArgsList) End() source.Pos { if n.Var != nil { return n.Var.End() } else if l := len(n.Values); l > 0 { return n.Values[l-1].End() } - return NoPos + return source.NoPos } // NumFields returns the number of fields. @@ -356,17 +390,17 @@ func (n *NamedArgsList) Add(name *Ident, value Expr) *NamedArgsList { } // Pos returns the position of first character belonging to the node. -func (n *NamedArgsList) Pos() Pos { +func (n *NamedArgsList) Pos() source.Pos { if len(n.Names) > 0 { return n.Names[0].Pos() } else if n.Var != nil { return n.Var.Pos() } - return NoPos + return source.NoPos } // End returns the position of first character immediately after the node. -func (n *NamedArgsList) End() Pos { +func (n *NamedArgsList) End() source.Pos { if n.Var != nil { return n.Var.End() } @@ -376,7 +410,7 @@ func (n *NamedArgsList) End() Pos { } return n.Values[l-1].End() } - return NoPos + return source.NoPos } // NumFields returns the number of fields. @@ -400,44 +434,44 @@ func (n *NamedArgsList) String() string { // FuncParams represents a function paramsw. type FuncParams struct { - LParen Pos + LParen source.Pos Args ArgsList NamedArgs NamedArgsList - RParen Pos + RParen source.Pos } // Pos returns the position of first character belonging to the node. -func (n *FuncParams) Pos() (pos Pos) { +func (n *FuncParams) Pos() (pos source.Pos) { if n.LParen.IsValid() { return n.LParen } - if pos = n.Args.Pos(); pos != NoPos { + if pos = n.Args.Pos(); pos != source.NoPos { return pos } - if pos = n.NamedArgs.Pos(); pos != NoPos { + if pos = n.NamedArgs.Pos(); pos != source.NoPos { return pos } - return NoPos + return source.NoPos } // End returns the position of first character immediately after the node. -func (n *FuncParams) End() (pos Pos) { +func (n *FuncParams) End() (pos source.Pos) { if n.RParen.IsValid() { return n.RParen + 1 } - if pos = n.NamedArgs.End(); pos != NoPos { + if pos = n.NamedArgs.End(); pos != source.NoPos { return pos } - if pos = n.Args.End(); pos != NoPos { + if pos = n.Args.End(); pos != source.NoPos { return pos } - return NoPos + return source.NoPos } func (n *FuncParams) String() string { buf := bytes.NewBufferString("(") buf.WriteString(n.Args.String()) - if buf.Len() > 1 && n.NamedArgs.Pos() != NoPos { + if buf.Len() > 1 && n.NamedArgs.Pos() != source.NoPos { buf.WriteString("; ") } buf.WriteString(n.NamedArgs.String()) @@ -447,42 +481,48 @@ func (n *FuncParams) String() string { // FuncType represents a function type definition. type FuncType struct { - FuncPos Pos + FuncPos source.Pos + Ident *Ident Params FuncParams } -func (e *FuncType) exprNode() {} +func (e *FuncType) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *FuncType) Pos() Pos { +func (e *FuncType) Pos() source.Pos { return e.FuncPos } // End returns the position of first character immediately after the node. -func (e *FuncType) End() Pos { +func (e *FuncType) End() source.Pos { return e.Params.End() } func (e *FuncType) String() string { - return "func" + e.Params.String() + var s string + if e.Ident != nil { + s += " " + s += e.Ident.String() + } + return s + e.Params.String() } // Ident represents an identifier. type Ident struct { Name string - NamePos Pos + NamePos source.Pos } -func (e *Ident) exprNode() {} +func (e *Ident) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *Ident) Pos() Pos { +func (e *Ident) Pos() source.Pos { return e.NamePos } // End returns the position of first character immediately after the node. -func (e *Ident) End() Pos { - return Pos(int(e.NamePos) + len(e.Name)) +func (e *Ident) End() source.Pos { + return source.Pos(int(e.NamePos) + len(e.Name)) } func (e *Ident) String() string { @@ -496,20 +536,20 @@ func (e *Ident) String() string { type ImportExpr struct { ModuleName string Token token.Token - TokenPos Pos + TokenPos source.Pos } -func (e *ImportExpr) exprNode() {} +func (e *ImportExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *ImportExpr) Pos() Pos { +func (e *ImportExpr) Pos() source.Pos { return e.TokenPos } // End returns the position of first character immediately after the node. -func (e *ImportExpr) End() Pos { +func (e *ImportExpr) End() source.Pos { // import("moduleName") - return Pos(int(e.TokenPos) + 10 + len(e.ModuleName)) + return source.Pos(int(e.TokenPos) + 10 + len(e.ModuleName)) } func (e *ImportExpr) String() string { @@ -519,20 +559,20 @@ func (e *ImportExpr) String() string { // IndexExpr represents an index expression. type IndexExpr struct { Expr Expr - LBrack Pos + LBrack source.Pos Index Expr - RBrack Pos + RBrack source.Pos } -func (e *IndexExpr) exprNode() {} +func (e *IndexExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *IndexExpr) Pos() Pos { +func (e *IndexExpr) Pos() source.Pos { return e.Expr.Pos() } // End returns the position of first character immediately after the node. -func (e *IndexExpr) End() Pos { +func (e *IndexExpr) End() source.Pos { return e.RBrack + 1 } @@ -547,20 +587,20 @@ func (e *IndexExpr) String() string { // IntLit represents an integer literal. type IntLit struct { Value int64 - ValuePos Pos + ValuePos source.Pos Literal string } -func (e *IntLit) exprNode() {} +func (e *IntLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *IntLit) Pos() Pos { +func (e *IntLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *IntLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *IntLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *IntLit) String() string { @@ -570,20 +610,20 @@ func (e *IntLit) String() string { // UintLit represents an unsigned integer literal. type UintLit struct { Value uint64 - ValuePos Pos + ValuePos source.Pos Literal string } -func (e *UintLit) exprNode() {} +func (e *UintLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *UintLit) Pos() Pos { +func (e *UintLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *UintLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *UintLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *UintLit) String() string { @@ -593,20 +633,20 @@ func (e *UintLit) String() string { // MapElementLit represents a map element. type MapElementLit struct { Key string - KeyPos Pos - ColonPos Pos + KeyPos source.Pos + ColonPos source.Pos Value Expr } -func (e *MapElementLit) exprNode() {} +func (e *MapElementLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *MapElementLit) Pos() Pos { +func (e *MapElementLit) Pos() source.Pos { return e.KeyPos } // End returns the position of first character immediately after the node. -func (e *MapElementLit) End() Pos { +func (e *MapElementLit) End() source.Pos { return e.Value.End() } @@ -616,20 +656,20 @@ func (e *MapElementLit) String() string { // MapLit represents a map literal. type MapLit struct { - LBrace Pos + LBrace source.Pos Elements []*MapElementLit - RBrace Pos + RBrace source.Pos } -func (e *MapLit) exprNode() {} +func (e *MapLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *MapLit) Pos() Pos { +func (e *MapLit) Pos() source.Pos { return e.LBrace } // End returns the position of first character immediately after the node. -func (e *MapLit) End() Pos { +func (e *MapLit) End() source.Pos { return e.RBrace + 1 } @@ -644,19 +684,19 @@ func (e *MapLit) String() string { // ParenExpr represents a parenthesis wrapped expression. type ParenExpr struct { Expr Expr - LParen Pos - RParen Pos + LParen source.Pos + RParen source.Pos } -func (e *ParenExpr) exprNode() {} +func (e *ParenExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *ParenExpr) Pos() Pos { +func (e *ParenExpr) Pos() source.Pos { return e.LParen } // End returns the position of first character immediately after the node. -func (e *ParenExpr) End() Pos { +func (e *ParenExpr) End() source.Pos { return e.RParen + 1 } @@ -675,15 +715,15 @@ type SelectorExpr struct { Sel Expr } -func (e *SelectorExpr) exprNode() {} +func (e *SelectorExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *SelectorExpr) Pos() Pos { +func (e *SelectorExpr) Pos() source.Pos { return e.Expr.Pos() } // End returns the position of first character immediately after the node. -func (e *SelectorExpr) End() Pos { +func (e *SelectorExpr) End() source.Pos { return e.Sel.End() } @@ -708,15 +748,15 @@ type NullishSelectorExpr struct { Sel Expr } -func (e *NullishSelectorExpr) exprNode() {} +func (e *NullishSelectorExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *NullishSelectorExpr) Pos() Pos { +func (e *NullishSelectorExpr) Pos() source.Pos { return e.Expr.Pos() } // End returns the position of first character immediately after the node. -func (e *NullishSelectorExpr) End() Pos { +func (e *NullishSelectorExpr) End() source.Pos { return e.Sel.End() } @@ -738,21 +778,21 @@ func (e *NullishSelectorExpr) SelectorExpr() Expr { // SliceExpr represents a slice expression. type SliceExpr struct { Expr Expr - LBrack Pos + LBrack source.Pos Low Expr High Expr - RBrack Pos + RBrack source.Pos } -func (e *SliceExpr) exprNode() {} +func (e *SliceExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *SliceExpr) Pos() Pos { +func (e *SliceExpr) Pos() source.Pos { return e.Expr.Pos() } // End returns the position of first character immediately after the node. -func (e *SliceExpr) End() Pos { +func (e *SliceExpr) End() source.Pos { return e.RBrack + 1 } @@ -770,29 +810,29 @@ func (e *SliceExpr) String() string { // StringLit represents a string literal. type StringLit struct { Value string - ValuePos Pos + ValuePos source.Pos Literal string } func (e *StringLit) CanIdent() bool { for _, r := range e.Value { - if !isLetter(r) { + if !utils.IsLetter(r) { return false } } return true } -func (e *StringLit) exprNode() {} +func (e *StringLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *StringLit) Pos() Pos { +func (e *StringLit) Pos() source.Pos { return e.ValuePos } // End returns the position of first character immediately after the node. -func (e *StringLit) End() Pos { - return Pos(int(e.ValuePos) + len(e.Literal)) +func (e *StringLit) End() source.Pos { + return source.Pos(int(e.ValuePos) + len(e.Literal)) } func (e *StringLit) String() string { @@ -803,18 +843,18 @@ func (e *StringLit) String() string { type UnaryExpr struct { Expr Expr Token token.Token - TokenPos Pos + TokenPos source.Pos } -func (e *UnaryExpr) exprNode() {} +func (e *UnaryExpr) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *UnaryExpr) Pos() Pos { +func (e *UnaryExpr) Pos() source.Pos { return e.Expr.Pos() } // End returns the position of first character immediately after the node. -func (e *UnaryExpr) End() Pos { +func (e *UnaryExpr) End() source.Pos { return e.Expr.End() } @@ -830,18 +870,18 @@ func (e *UnaryExpr) String() string { // NilLit represents an nil literal. type NilLit struct { - TokenPos Pos + TokenPos source.Pos } -func (e *NilLit) exprNode() {} +func (e *NilLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *NilLit) Pos() Pos { +func (e *NilLit) Pos() source.Pos { return e.TokenPos } // End returns the position of first character immediately after the node. -func (e *NilLit) End() Pos { +func (e *NilLit) End() source.Pos { return e.TokenPos + 9 // len(nil) == 9 } @@ -850,7 +890,7 @@ func (e *NilLit) String() string { } type EllipsisValue struct { - Pos Pos + Pos source.Pos Value Expr } @@ -876,27 +916,31 @@ func (a *CallExprArgs) String() string { } type NamedArgExpr struct { - String *StringLit - Ident *Ident + Lit *StringLit + Ident *Ident } func (e *NamedArgExpr) Name() string { - if e.String != nil { - return e.String.Value + if e.Lit != nil { + return e.Lit.Value } return e.Ident.Name } func (e *NamedArgExpr) NameString() *StringLit { - if e.String != nil { - return e.String + if e.Lit != nil { + return e.Lit } return &StringLit{Value: e.Ident.Name, ValuePos: e.Ident.NamePos} } +func (e *NamedArgExpr) String() string { + return e.Expr().String() +} + func (e *NamedArgExpr) Expr() Expr { - if e.String != nil { - return e.String + if e.Lit != nil { + return e.Lit } return e.Ident } @@ -908,6 +952,27 @@ type CallExprNamedArgs struct { Ellipsis *EllipsisValue } +func (a *CallExprNamedArgs) Append(name NamedArgExpr, value Expr) { + a.Names = append(a.Names, name) + a.Values = append(a.Values, value) +} + +func (a *CallExprNamedArgs) Prepend(name NamedArgExpr, value Expr) { + a.Names = append([]NamedArgExpr{name}, a.Names...) + a.Values = append([]Expr{value}, a.Values...) +} + +func (a *CallExprNamedArgs) Get(name NamedArgExpr) (index int, value Expr) { + names := name.String() + index = -1 + for i, expr := range a.Names { + if expr.String() == names { + return i, a.Values[i] + } + } + return +} + func (a *CallExprNamedArgs) Valid() bool { return len(a.Names) > 0 || a.Ellipsis != nil } @@ -936,21 +1001,22 @@ func (a *CallExprNamedArgs) String() string { // KeyValueLit represents a key value element. type KeyValueLit struct { - Key Expr - KeyPos Pos - ColonPos Pos - Value Expr + Key Expr + Value Expr } -func (e *KeyValueLit) exprNode() {} +func (e *KeyValueLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *KeyValueLit) Pos() Pos { - return e.KeyPos +func (e *KeyValueLit) Pos() source.Pos { + return e.Key.Pos() } // End returns the position of first character immediately after the node. -func (e *KeyValueLit) End() Pos { +func (e *KeyValueLit) End() source.Pos { + if e.Value == nil { + return e.Key.End() + } return e.Value.End() } @@ -963,20 +1029,20 @@ func (e *KeyValueLit) String() string { // KeyValueArrayLit represents a key value array literal. type KeyValueArrayLit struct { - LBrace Pos + LBrace source.Pos Elements []*KeyValueLit - RBrace Pos + RBrace source.Pos } -func (e *KeyValueArrayLit) exprNode() {} +func (e *KeyValueArrayLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *KeyValueArrayLit) Pos() Pos { +func (e *KeyValueArrayLit) Pos() source.Pos { return e.LBrace } // End returns the position of first character immediately after the node. -func (e *KeyValueArrayLit) End() Pos { +func (e *KeyValueArrayLit) End() source.Pos { return e.RBrace + 1 } @@ -989,85 +1055,85 @@ func (e *KeyValueArrayLit) String() string { } type CalleeKeyword struct { - TokenPos Pos + TokenPos source.Pos Literal string } -func (c *CalleeKeyword) Pos() Pos { +func (c *CalleeKeyword) Pos() source.Pos { return c.TokenPos } -func (c *CalleeKeyword) End() Pos { - return c.TokenPos + Pos(len(token.Callee.String())) +func (c *CalleeKeyword) End() source.Pos { + return c.TokenPos + source.Pos(len(token.Callee.String())) } func (c *CalleeKeyword) String() string { return c.Literal } -func (c *CalleeKeyword) exprNode() { +func (c *CalleeKeyword) ExprNode() { } type ArgsKeyword struct { - TokenPos Pos + TokenPos source.Pos Literal string } -func (c *ArgsKeyword) Pos() Pos { +func (c *ArgsKeyword) Pos() source.Pos { return c.TokenPos } -func (c *ArgsKeyword) End() Pos { - return c.TokenPos + Pos(len(c.Literal)) +func (c *ArgsKeyword) End() source.Pos { + return c.TokenPos + source.Pos(len(c.Literal)) } func (c *ArgsKeyword) String() string { return c.Literal } -func (c *ArgsKeyword) exprNode() { +func (c *ArgsKeyword) ExprNode() { } type NamedArgsKeyword struct { - TokenPos Pos + TokenPos source.Pos Literal string } -func (c *NamedArgsKeyword) Pos() Pos { +func (c *NamedArgsKeyword) Pos() source.Pos { return c.TokenPos } -func (c *NamedArgsKeyword) End() Pos { - return c.TokenPos + Pos(len(c.Literal)) +func (c *NamedArgsKeyword) End() source.Pos { + return c.TokenPos + source.Pos(len(c.Literal)) } func (c *NamedArgsKeyword) String() string { return c.Literal } -func (c *NamedArgsKeyword) exprNode() { +func (c *NamedArgsKeyword) ExprNode() { } type BlockExpr struct { *BlockStmt } -func (b BlockExpr) exprNode() {} +func (b BlockExpr) ExprNode() {} // StdInLit represents an STDIN literal. type StdInLit struct { - TokenPos Pos + TokenPos source.Pos } -func (e *StdInLit) exprNode() {} +func (e *StdInLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *StdInLit) Pos() Pos { +func (e *StdInLit) Pos() source.Pos { return e.TokenPos } // End returns the position of first character immediately after the node. -func (e *StdInLit) End() Pos { +func (e *StdInLit) End() source.Pos { return e.TokenPos + 5 // len(STDIN) == 5 } @@ -1077,18 +1143,18 @@ func (e *StdInLit) String() string { // StdOutLit represents an STDOUT literal. type StdOutLit struct { - TokenPos Pos + TokenPos source.Pos } -func (e *StdOutLit) exprNode() {} +func (e *StdOutLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *StdOutLit) Pos() Pos { +func (e *StdOutLit) Pos() source.Pos { return e.TokenPos } // End returns the position of first character immediately after the node. -func (e *StdOutLit) End() Pos { +func (e *StdOutLit) End() source.Pos { return e.TokenPos + 6 // len(STDOUT) == 6 } @@ -1098,18 +1164,18 @@ func (e *StdOutLit) String() string { // StdErrLit represents an STDERR literal. type StdErrLit struct { - TokenPos Pos + TokenPos source.Pos } -func (e *StdErrLit) exprNode() {} +func (e *StdErrLit) ExprNode() {} // Pos returns the position of first character belonging to the node. -func (e *StdErrLit) Pos() Pos { +func (e *StdErrLit) Pos() source.Pos { return e.TokenPos } // End StdErrLit the position of first character immediately after the node. -func (e *StdErrLit) End() Pos { +func (e *StdErrLit) End() source.Pos { return e.TokenPos + 6 // len(STDERR) == 6 } diff --git a/parser/node/helpers.go b/parser/node/helpers.go new file mode 100644 index 0000000..9eb1215 --- /dev/null +++ b/parser/node/helpers.go @@ -0,0 +1,57 @@ +package node + +import ( + "strings" + + "github.com/gad-lang/gad/parser/source" +) + +// IdentList represents a list of identifiers. +type IdentList struct { + LParen source.Pos + VarArgs bool + List []*Ident + RParen source.Pos +} + +// Pos returns the position of first character belonging to the node. +func (n *IdentList) Pos() source.Pos { + if n.LParen.IsValid() { + return n.LParen + } + if len(n.List) > 0 { + return n.List[0].Pos() + } + return source.NoPos +} + +// End returns the position of first character immediately after the node. +func (n *IdentList) End() source.Pos { + if n.RParen.IsValid() { + return n.RParen + 1 + } + if l := len(n.List); l > 0 { + return n.List[l-1].End() + } + return source.NoPos +} + +// NumFields returns the number of fields. +func (n *IdentList) NumFields() int { + if n == nil { + return 0 + } + return len(n.List) +} + +func (n *IdentList) String() string { + var list []string + for i, e := range n.List { + if n.VarArgs && i == len(n.List)-1 { + list = append(list, "..."+e.String()) + } else { + list = append(list, e.String()) + } + } + return "(" + strings.Join(list, ", ") + ")" +} diff --git a/parser/node/pkg.go b/parser/node/pkg.go new file mode 100644 index 0000000..0a852ef --- /dev/null +++ b/parser/node/pkg.go @@ -0,0 +1,5 @@ +package node + +const ( + nullRep = "" +) diff --git a/parser/stmt.go b/parser/node/stmt.go similarity index 64% rename from parser/stmt.go rename to parser/node/stmt.go index 0ddd96c..3aca9c5 100644 --- a/parser/stmt.go +++ b/parser/node/stmt.go @@ -10,22 +10,26 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE.golang file. -package parser +package node import ( + "fmt" "strconv" "strings" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/source" + "github.com/gad-lang/gad/parser/utils" "github.com/gad-lang/gad/token" ) // Stmt represents a statement in the AST. type Stmt interface { - Node - stmtNode() + ast.Node + StmtNode() } -// IsStatement returns true if given value is implements interface{ stmtNode() }. +// IsStatement returns true if given value is implements interface{ StmtNode() }. func IsStatement(v any) bool { _, ok := v.(interface { stmtNode() @@ -38,18 +42,18 @@ type AssignStmt struct { LHS []Expr RHS []Expr Token token.Token - TokenPos Pos + TokenPos source.Pos } -func (s *AssignStmt) stmtNode() {} +func (s *AssignStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *AssignStmt) Pos() Pos { +func (s *AssignStmt) Pos() source.Pos { return s.LHS[0].Pos() } // End returns the position of first character immediately after the node. -func (s *AssignStmt) End() Pos { +func (s *AssignStmt) End() source.Pos { return s.RHS[len(s.RHS)-1].End() } @@ -67,19 +71,19 @@ func (s *AssignStmt) String() string { // BadStmt represents a bad statement. type BadStmt struct { - From Pos - To Pos + From source.Pos + To source.Pos } -func (s *BadStmt) stmtNode() {} +func (s *BadStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *BadStmt) Pos() Pos { +func (s *BadStmt) Pos() source.Pos { return s.From } // End returns the position of first character immediately after the node. -func (s *BadStmt) End() Pos { +func (s *BadStmt) End() source.Pos { return s.To } @@ -90,19 +94,19 @@ func (s *BadStmt) String() string { // BlockStmt represents a block statement. type BlockStmt struct { Stmts []Stmt - LBrace Pos - RBrace Pos + LBrace source.Pos + RBrace source.Pos } -func (s *BlockStmt) stmtNode() {} +func (s *BlockStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *BlockStmt) Pos() Pos { +func (s *BlockStmt) Pos() source.Pos { return s.LBrace } // End returns the position of first character immediately after the node. -func (s *BlockStmt) End() Pos { +func (s *BlockStmt) End() source.Pos { return s.RBrace + 1 } @@ -117,24 +121,24 @@ func (s *BlockStmt) String() string { // BranchStmt represents a branch statement. type BranchStmt struct { Token token.Token - TokenPos Pos + TokenPos source.Pos Label *Ident } -func (s *BranchStmt) stmtNode() {} +func (s *BranchStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *BranchStmt) Pos() Pos { +func (s *BranchStmt) Pos() source.Pos { return s.TokenPos } // End returns the position of first character immediately after the node. -func (s *BranchStmt) End() Pos { +func (s *BranchStmt) End() source.Pos { if s.Label != nil { return s.Label.End() } - return Pos(int(s.TokenPos) + len(s.Token.String())) + return source.Pos(int(s.TokenPos) + len(s.Token.String())) } func (s *BranchStmt) String() string { @@ -147,19 +151,19 @@ func (s *BranchStmt) String() string { // EmptyStmt represents an empty statement. type EmptyStmt struct { - Semicolon Pos + Semicolon source.Pos Implicit bool } -func (s *EmptyStmt) stmtNode() {} +func (s *EmptyStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *EmptyStmt) Pos() Pos { +func (s *EmptyStmt) Pos() source.Pos { return s.Semicolon } // End returns the position of first character immediately after the node. -func (s *EmptyStmt) End() Pos { +func (s *EmptyStmt) End() source.Pos { if s.Implicit { return s.Semicolon } @@ -175,15 +179,15 @@ type ExprStmt struct { Expr Expr } -func (s *ExprStmt) stmtNode() {} +func (s *ExprStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *ExprStmt) Pos() Pos { +func (s *ExprStmt) Pos() source.Pos { return s.Expr.Pos() } // End returns the position of first character immediately after the node. -func (s *ExprStmt) End() Pos { +func (s *ExprStmt) End() source.Pos { return s.Expr.End() } @@ -193,52 +197,57 @@ func (s *ExprStmt) String() string { // ForInStmt represents a for-in statement. type ForInStmt struct { - ForPos Pos + ForPos source.Pos Key *Ident Value *Ident Iterable Expr Body *BlockStmt + Else *BlockStmt } -func (s *ForInStmt) stmtNode() {} +func (s *ForInStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *ForInStmt) Pos() Pos { +func (s *ForInStmt) Pos() source.Pos { return s.ForPos } // End returns the position of first character immediately after the node. -func (s *ForInStmt) End() Pos { +func (s *ForInStmt) End() source.Pos { return s.Body.End() } func (s *ForInStmt) String() string { + var str = "for " + s.Key.String() if s.Value != nil { - return "for " + s.Key.String() + ", " + s.Value.String() + - " in " + s.Iterable.String() + " " + s.Body.String() + str += ", " + s.Value.String() } - return "for " + s.Key.String() + " in " + s.Iterable.String() + + str += " in " + s.Iterable.String() + " " + s.Body.String() + if s.Else != nil { + str += " else " + s.Else.String() + } + return str } // ForStmt represents a for statement. type ForStmt struct { - ForPos Pos + ForPos source.Pos Init Stmt Cond Expr Post Stmt Body *BlockStmt } -func (s *ForStmt) stmtNode() {} +func (s *ForStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *ForStmt) Pos() Pos { +func (s *ForStmt) Pos() source.Pos { return s.ForPos } // End returns the position of first character immediately after the node. -func (s *ForStmt) End() Pos { +func (s *ForStmt) End() source.Pos { return s.Body.End() } @@ -254,30 +263,36 @@ func (s *ForStmt) String() string { post = s.Post.String() } + var str = "for " + if init != "" || post != "" { - return "for " + init + " ; " + cond + " ; " + post + s.Body.String() + str += init + " ; " + cond + " ; " + post + } else { + str += cond } - return "for " + cond + s.Body.String() + + str += s.Body.String() + return str } // IfStmt represents an if statement. type IfStmt struct { - IfPos Pos + IfPos source.Pos Init Stmt Cond Expr Body *BlockStmt Else Stmt // else branch; or nil } -func (s *IfStmt) stmtNode() {} +func (s *IfStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *IfStmt) Pos() Pos { +func (s *IfStmt) Pos() source.Pos { return s.IfPos } // End returns the position of first character immediately after the node. -func (s *IfStmt) End() Pos { +func (s *IfStmt) End() source.Pos { if s.Else != nil { return s.Else.End() } @@ -300,19 +315,19 @@ func (s *IfStmt) String() string { type IncDecStmt struct { Expr Expr Token token.Token - TokenPos Pos + TokenPos source.Pos } -func (s *IncDecStmt) stmtNode() {} +func (s *IncDecStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *IncDecStmt) Pos() Pos { +func (s *IncDecStmt) Pos() source.Pos { return s.Expr.Pos() } // End returns the position of first character immediately after the node. -func (s *IncDecStmt) End() Pos { - return Pos(int(s.TokenPos) + 2) +func (s *IncDecStmt) End() source.Pos { + return source.Pos(int(s.TokenPos) + 2) } func (s *IncDecStmt) String() string { @@ -321,19 +336,19 @@ func (s *IncDecStmt) String() string { // ReturnStmt represents a return statement. type ReturnStmt struct { - ReturnPos Pos + ReturnPos source.Pos Result Expr } -func (s *ReturnStmt) stmtNode() {} +func (s *ReturnStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *ReturnStmt) Pos() Pos { +func (s *ReturnStmt) Pos() source.Pos { return s.ReturnPos } // End returns the position of first character immediately after the node. -func (s *ReturnStmt) End() Pos { +func (s *ReturnStmt) End() source.Pos { if s.Result != nil { return s.Result.End() } @@ -349,21 +364,21 @@ func (s *ReturnStmt) String() string { // TryStmt represents an try statement. type TryStmt struct { - TryPos Pos + TryPos source.Pos Body *BlockStmt Catch *CatchStmt // catch branch; or nil Finally *FinallyStmt // finally branch; or nil } -func (s *TryStmt) stmtNode() {} +func (s *TryStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *TryStmt) Pos() Pos { +func (s *TryStmt) Pos() source.Pos { return s.TryPos } // End returns the position of first character immediately after the node. -func (s *TryStmt) End() Pos { +func (s *TryStmt) End() source.Pos { if s.Finally != nil { return s.Finally.End() } @@ -387,20 +402,20 @@ func (s *TryStmt) String() string { // CatchStmt represents an catch statement. type CatchStmt struct { - CatchPos Pos + CatchPos source.Pos Ident *Ident // can be nil if ident is missing Body *BlockStmt } -func (s *CatchStmt) stmtNode() {} +func (s *CatchStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *CatchStmt) Pos() Pos { +func (s *CatchStmt) Pos() source.Pos { return s.CatchPos } // End returns the position of first character immediately after the node. -func (s *CatchStmt) End() Pos { +func (s *CatchStmt) End() source.Pos { return s.Body.End() } @@ -414,19 +429,19 @@ func (s *CatchStmt) String() string { // FinallyStmt represents an finally statement. type FinallyStmt struct { - FinallyPos Pos + FinallyPos source.Pos Body *BlockStmt } -func (s *FinallyStmt) stmtNode() {} +func (s *FinallyStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *FinallyStmt) Pos() Pos { +func (s *FinallyStmt) Pos() source.Pos { return s.FinallyPos } // End returns the position of first character immediately after the node. -func (s *FinallyStmt) End() Pos { +func (s *FinallyStmt) End() source.Pos { return s.Body.End() } @@ -436,19 +451,19 @@ func (s *FinallyStmt) String() string { // ThrowStmt represents an throw statement. type ThrowStmt struct { - ThrowPos Pos + ThrowPos source.Pos Expr Expr } -func (s *ThrowStmt) stmtNode() {} +func (s *ThrowStmt) StmtNode() {} // Pos returns the position of first character belonging to the node. -func (s *ThrowStmt) Pos() Pos { +func (s *ThrowStmt) Pos() source.Pos { return s.ThrowPos } // End returns the position of first character immediately after the node. -func (s *ThrowStmt) End() Pos { +func (s *ThrowStmt) End() source.Pos { return s.Expr.End() } @@ -463,28 +478,41 @@ func (s *ThrowStmt) String() string { // TextStmt represents an TextStmt. type TextStmt struct { Literal string - TextPos Pos + TextPos source.Pos + Data utils.Data +} + +func (e *TextStmt) StmtNode() { } -func (e *TextStmt) stmtNode() { +func (e *TextStmt) ExprNode() { } -func (e *TextStmt) exprNode() { +func (e *TextStmt) TrimLinePrefix(prefix string) { + lines := strings.Split(e.Literal, "\n") + for i, line := range lines { + lines[i] = strings.TrimLeft(line, prefix) + } + e.Literal = strings.Join(lines, "\n") } // Pos returns the position of first character belonging to the node. -func (e *TextStmt) Pos() Pos { +func (e *TextStmt) Pos() source.Pos { return e.TextPos } // End returns the position of first character immediately after the node. -func (e *TextStmt) End() Pos { - return Pos(int(e.TextPos) + len(e.Literal)) +func (e *TextStmt) End() source.Pos { + return source.Pos(int(e.TextPos) + len(e.Literal)) } func (e *TextStmt) String() string { if e != nil { - return "#{= " + strconv.Quote(e.Literal) + " }" + s := "#{= " + strconv.Quote(e.Literal) + if len(e.Data) > 0 { + s += " {" + e.Data.String() + "}" + } + return s + " }" } return nullRep } @@ -492,22 +520,30 @@ func (e *TextStmt) String() string { // ExprToTextStmt represents to text wrapped expression. type ExprToTextStmt struct { Expr Expr - StartLit Literal - EndLit Literal + StartLit ast.Literal + EndLit ast.Literal +} + +func NewExprToTextStmt(expr Expr) *ExprToTextStmt { + return &ExprToTextStmt{ + Expr: expr, + StartLit: ast.Literal{Value: "#{="}, + EndLit: ast.Literal{Value: "}"}, + } } -func (e *ExprToTextStmt) stmtNode() {} +func (e *ExprToTextStmt) StmtNode() {} -func (e *ExprToTextStmt) exprNode() { +func (e *ExprToTextStmt) ExprNode() { } // Pos returns the position of first character belonging to the node. -func (e *ExprToTextStmt) Pos() Pos { +func (e *ExprToTextStmt) Pos() source.Pos { return e.StartLit.Pos } // End returns the position of first character immediately after the node. -func (e *ExprToTextStmt) End() Pos { +func (e *ExprToTextStmt) End() source.Pos { return e.EndLit.Pos } @@ -522,23 +558,73 @@ type ConfigOptions struct { } type ConfigStmt struct { - ConfigPos Pos - EndPos Pos - Literal string + ConfigPos source.Pos + Elements []*KeyValueLit Options ConfigOptions } -func (c *ConfigStmt) Pos() Pos { +func (c *ConfigStmt) Pos() source.Pos { return c.ConfigPos } -func (c *ConfigStmt) End() Pos { - return c.EndPos +func (c *ConfigStmt) End() source.Pos { + if len(c.Elements) == 0 { + return c.ConfigPos + 1 + } + return c.Elements[len(c.Elements)-1].End() } func (c *ConfigStmt) String() string { - return "# gad: " + c.Literal + var elements []string + for _, m := range c.Elements { + elements = append(elements, m.String()) + } + return "# gad: " + strings.Join(elements, ", ") +} + +func (c *ConfigStmt) ParseElements() { + for _, k := range c.Elements { + switch k.Key.String() { + case "mixed": + if k.Value == nil { + c.Options.Mixed = true + } else if b, ok := k.Value.(*BoolLit); ok { + if b.Value { + c.Options.Mixed = true + } else { + c.Options.NoMixed = true + } + } + case "writer": + if k.Value != nil { + c.Options.WriteFunc = k.Value + } + } + } +} + +func (c *ConfigStmt) StmtNode() { +} + +type StmtsExpr struct { + Stmts []Stmt +} + +func (s *StmtsExpr) Pos() source.Pos { + return s.Stmts[0].Pos() +} + +func (s *StmtsExpr) End() source.Pos { + return s.Stmts[len(s.Stmts)-1].End() +} + +func (s *StmtsExpr) String() string { + var str = make([]string, len(s.Stmts)) + for i, stmt := range s.Stmts { + str[i] = fmt.Sprint(stmt) + } + return strings.Join(str, "; ") } -func (c *ConfigStmt) stmtNode() { +func (s *StmtsExpr) ExprNode() { } diff --git a/parser/stmt_decl.go b/parser/node/stmt_decl.go similarity index 83% rename from parser/stmt_decl.go rename to parser/node/stmt_decl.go index 04bc3d3..d92fcec 100644 --- a/parser/stmt_decl.go +++ b/parser/node/stmt_decl.go @@ -6,12 +6,14 @@ // Use of this source code is governed by a MIT License // that can be found in the LICENSE file. -package parser +package node import ( "fmt" "strings" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/source" "github.com/gad-lang/gad/token" ) @@ -22,7 +24,7 @@ type ( // Spec node represents a single (non-parenthesized) variable declaration. // The Spec type stands for any of *ParamSpec or *ValueSpec. Spec interface { - Node + ast.Node specNode() } @@ -47,21 +49,21 @@ type ( ) // Pos returns the position of first character belonging to the spec. -func (s *ParamSpec) Pos() Pos { return s.Ident.Pos() } +func (s *ParamSpec) Pos() source.Pos { return s.Ident.Pos() } // Pos returns the position of first character belonging to the spec. -func (s *NamedParamSpec) Pos() Pos { return s.Ident.Pos() } +func (s *NamedParamSpec) Pos() source.Pos { return s.Ident.Pos() } // Pos returns the position of first character belonging to the spec. -func (s *ValueSpec) Pos() Pos { return s.Idents[0].Pos() } +func (s *ValueSpec) Pos() source.Pos { return s.Idents[0].Pos() } // End returns the position of first character immediately after the spec. -func (s *ParamSpec) End() Pos { +func (s *ParamSpec) End() source.Pos { return s.Ident.End() } // End returns the position of first character immediately after the spec. -func (s *NamedParamSpec) End() Pos { +func (s *NamedParamSpec) End() source.Pos { if s.Value == nil { return s.Ident.End() } @@ -69,7 +71,7 @@ func (s *NamedParamSpec) End() Pos { } // End returns the position of first character immediately after the spec. -func (s *ValueSpec) End() Pos { +func (s *ValueSpec) End() source.Pos { if n := len(s.Values); n > 0 && s.Values[n-1] != nil { return s.Values[n-1].End() } @@ -115,7 +117,7 @@ func (*ValueSpec) specNode() {} // Decl wraps methods for all declaration nodes. type Decl interface { - Node + ast.Node declNode() } @@ -124,13 +126,13 @@ type DeclStmt struct { Decl // *GenDecl with VAR token } -func (*DeclStmt) stmtNode() {} +func (*DeclStmt) StmtNode() {} // A BadDecl node is a placeholder for declarations containing // syntax errors for which no correct declaration nodes can be // created. type BadDecl struct { - From, To Pos // position range of bad declaration + From, To source.Pos // position range of bad declaration } // A GenDecl node (generic declaration node) represents a variable declaration. @@ -140,24 +142,24 @@ type BadDecl struct { // // token.Var *ValueSpec type GenDecl struct { - TokPos Pos // position of Tok + TokPos source.Pos // position of Tok Tok token.Token // Var - Lparen Pos // position of '(', if any + Lparen source.Pos // position of '(', if any Specs []Spec - Rparen Pos // position of ')', if any + Rparen source.Pos // position of ')', if any } // Pos returns the position of first character belonging to the node. -func (d *BadDecl) Pos() Pos { return d.From } +func (d *BadDecl) Pos() source.Pos { return d.From } // Pos returns the position of first character belonging to the node. -func (d *GenDecl) Pos() Pos { return d.TokPos } +func (d *GenDecl) Pos() source.Pos { return d.TokPos } // End returns the position of first character immediately after the node. -func (d *BadDecl) End() Pos { return d.To } +func (d *BadDecl) End() source.Pos { return d.To } // End returns the position of first character immediately after the node. -func (d *GenDecl) End() Pos { +func (d *GenDecl) End() source.Pos { if d.Rparen.IsValid() { return d.Rparen + 1 } diff --git a/parser/parser.go b/parser/parser.go index 7b1d8bb..f36f65e 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -21,6 +21,9 @@ import ( "strconv" "strings" + "github.com/gad-lang/gad/parser/ast" + "github.com/gad-lang/gad/parser/node" + "github.com/gad-lang/gad/parser/source" "github.com/gad-lang/gad/token" "github.com/shopspring/decimal" ) @@ -128,19 +131,25 @@ func (p ErrorList) Err() error { // Parser parses the Tengo source files. It's based on ToInterface's parser // implementation. type Parser struct { - file *SourceFile - errors ErrorList - scanner *Scanner - token Token - prevToken token.Token - exprLevel int // < 0: in control clause, >= 0: in expression - syncPos Pos // last sync position - syncCount int // number of advance calls without progress - trace bool - indent int - mode Mode - traceOut io.Writer - comments []*CommentGroup + File *SourceFile + Errors ErrorList + Scanner ScannerInterface + Token Token + PrevToken Token + ExprLevel int // < 0: in control clause, >= 0: in expression + syncPos source.Pos // last sync position + syncCount int // number of advance calls without progress + Trace bool + indent int + mode Mode + TraceOut io.Writer + comments []*ast.CommentGroup + ParseStmtHandler func() node.Stmt + IgnoreCodeBlockDisabled bool + InCode bool + BlockStart token.Token + BlockEnd token.Token + ScanFunc func() Token } // NewParser creates a Parser. @@ -155,12 +164,35 @@ func NewParserWithMode( trace io.Writer, mode Mode, ) *Parser { - p := &Parser{ - file: file, - trace: trace != nil, - traceOut: trace, - mode: mode, + var m ScanMode + if mode.Has(ParseComments) { + m.Set(ScanComments) + } + if mode.Has(ParseMixed) { + m.Set(Mixed) + } + if mode.Has(ParseConfigDisabled) { + m.Set(ConfigDisabled) } + return NewParserWithArgs(NewScanner(file, src, m), trace, mode) +} + +// NewParserWithArgs creates a Parser with parser mode flags. +func NewParserWithArgs( + scanner ScannerInterface, + trace io.Writer, + mode Mode, +) *Parser { + p := &Parser{ + Scanner: scanner, + File: scanner.SourceFile(), + Trace: trace != nil, + TraceOut: trace, + mode: mode, + BlockStart: token.LBrace, + BlockEnd: token.RBrace, + } + p.ParseStmtHandler = p.DefaultParseStmt var m ScanMode if mode.Has(ParseComments) { m.Set(ScanComments) @@ -171,11 +203,10 @@ func NewParserWithMode( if mode.Has(ParseConfigDisabled) { m.Set(ConfigDisabled) } - p.scanner = NewScanner(p.file, src, - func(pos SourceFilePos, msg string) { - p.errors.Add(pos, msg) - }, m) - p.next() + scanner.ErrorHandler(func(pos SourceFilePos, msg string) { + p.Errors.Add(pos, msg) + }) + p.Next() return p } @@ -188,83 +219,118 @@ func (p *Parser) ParseFile() (file *File, err error) { } } - p.errors.Sort() - err = p.errors.Err() + p.Errors.Sort() + err = p.Errors.Err() + }() + + if p.Trace { + defer untracep(tracep(p, "File")) + } + + if p.Errors.Len() > 0 { + return nil, p.Errors.Err() + } + + stmts, _ := p.ParseStmtList(0) + p.Expect(token.EOF) + if p.Errors.Len() > 0 { + return nil, p.Errors.Err() + } + + file = &File{ + InputFile: p.File, + Stmts: stmts, + Comments: p.comments, + } + return +} + +// ParseFileH parses the source and returns an AST file unit. +func (p *Parser) ParseFileH(listHandler ParseListHandler) (file *File, err error) { + defer func() { + if e := recover(); e != nil { + if _, ok := e.(bailout); !ok { + panic(e) + } + } + + p.Errors.Sort() + err = p.Errors.Err() }() - if p.trace { + if p.Trace { defer untracep(tracep(p, "File")) } - if p.errors.Len() > 0 { - return nil, p.errors.Err() + if p.Errors.Len() > 0 { + return nil, p.Errors.Err() } - stmts, _ := p.parseStmtList(0) - p.expect(token.EOF) - if p.errors.Len() > 0 { - return nil, p.errors.Err() + stmts, _ := listHandler(0) + p.Expect(token.EOF) + if p.Errors.Len() > 0 { + return nil, p.Errors.Err() } file = &File{ - InputFile: p.file, + InputFile: p.File, Stmts: stmts, Comments: p.comments, } return } -func (p *Parser) parseExpr() Expr { - if p.trace { +func (p *Parser) ParseExpr() node.Expr { + if p.Trace { defer untracep(tracep(p, "Expression")) } - expr := p.parseBinaryExpr(token.LowestPrec + 1) + expr := p.ParseBinaryExpr(token.LowestPrec + 1) // ternary conditional expression - if p.token.Token == token.Question { - return p.parseCondExpr(expr) + if p.Token.Token == token.Question { + return p.ParseCondExpr(expr) } return expr } -func (p *Parser) parseBinaryExpr(prec1 int) Expr { - if p.trace { +func (p *Parser) ParseBinaryExpr(prec1 int) node.Expr { + if p.Trace { defer untracep(tracep(p, "BinaryExpression")) } - x := p.parseUnaryExpr() + x := p.ParseUnaryExpr() for { - op, prec := p.token.Token, p.token.Token.Precedence() + op, prec := p.Token.Token, p.Token.Token.Precedence() if prec < prec1 { return x } - pos := p.expect(op) + pos := p.Expect(op) - y := p.parseBinaryExpr(prec + 1) + y := p.ParseBinaryExpr(prec + 1) if op == token.Equal || op == token.NotEqual { - if _, ok := x.(*NilLit); ok { + if _, ok := x.(*node.NilLit); ok { if op == token.Equal { op = token.Null } else { op = token.NotNull } - x = &UnaryExpr{ + x = &node.UnaryExpr{ Expr: y, Token: op, TokenPos: pos, } continue - } else if _, ok := y.(*NilLit); ok { + } else if _, ok := y.(*node.NilLit); ok { if op == token.Equal { op = token.Null } else { op = token.NotNull } - x = &UnaryExpr{ + x = &node.UnaryExpr{ Expr: x, Token: op, TokenPos: pos, @@ -273,7 +339,7 @@ func (p *Parser) parseBinaryExpr(prec1 int) Expr { } } - x = &BinaryExpr{ + x = &node.BinaryExpr{ LHS: x, RHS: y, Token: op, @@ -282,13 +348,13 @@ func (p *Parser) parseBinaryExpr(prec1 int) Expr { } } -func (p *Parser) parseCondExpr(cond Expr) Expr { - questionPos := p.expect(token.Question) - trueExpr := p.parseExpr() - colonPos := p.expect(token.Colon) - falseExpr := p.parseExpr() +func (p *Parser) ParseCondExpr(cond node.Expr) node.Expr { + questionPos := p.Expect(token.Question) + trueExpr := p.ParseExpr() + colonPos := p.Expect(token.Colon) + falseExpr := p.ParseExpr() - return &CondExpr{ + return &node.CondExpr{ Cond: cond, True: trueExpr, False: falseExpr, @@ -297,63 +363,63 @@ func (p *Parser) parseCondExpr(cond Expr) Expr { } } -func (p *Parser) parseUnaryExpr() Expr { - if p.trace { +func (p *Parser) ParseUnaryExpr() node.Expr { + if p.Trace { defer untracep(tracep(p, "UnaryExpression")) } - switch p.token.Token { + switch p.Token.Token { case token.Add, token.Sub, token.Not, token.Xor: - pos, op := p.token.Pos, p.token.Token - p.next() - x := p.parseUnaryExpr() - return &UnaryExpr{ + pos, op := p.Token.Pos, p.Token.Token + p.Next() + x := p.ParseUnaryExpr() + return &node.UnaryExpr{ Token: op, TokenPos: pos, Expr: x, } } - return p.parsePrimaryExpr() + return p.ParsePrimaryExpr() } -func (p *Parser) parsePrimaryExpr() Expr { - if p.trace { +func (p *Parser) ParsePrimaryExpr() node.Expr { + if p.Trace { defer untracep(tracep(p, "PrimaryExpression")) } - x := p.parseOperand() + x := p.ParseOperand() L: for { - switch p.token.Token { + switch p.Token.Token { case token.NullishSelector: - p.next() + p.Next() - switch p.token.Token { + switch p.Token.Token { case token.Ident, token.LParen: - x = p.parseNullishSelector(x) + x = p.ParseNullishSelector(x) default: - pos := p.token.Pos - p.errorExpected(pos, "nullish selector") + pos := p.Token.Pos + p.ErrorExpected(pos, "nullish selector") p.advance(stmtStart) - return &BadExpr{From: pos, To: p.token.Pos} + return &node.BadExpr{From: pos, To: p.Token.Pos} } case token.Period: - p.next() + p.Next() - switch p.token.Token { + switch p.Token.Token { case token.Ident, token.LParen: - x = p.parseSelector(x) + x = p.ParseSelector(x) default: - pos := p.token.Pos - p.errorExpected(pos, "selector") + pos := p.Token.Pos + p.ErrorExpected(pos, "selector") p.advance(stmtStart) - return &BadExpr{From: pos, To: p.token.Pos} + return &node.BadExpr{From: pos, To: p.Token.Pos} } case token.LBrack: - x = p.parseIndexOrSlice(x) + x = p.ParseIndexOrSlice(x) case token.LParen: - x = p.parseCall(x) + x = p.ParseCall(x) default: break L } @@ -361,25 +427,36 @@ L: return x } -func (p *Parser) parseCall(x Expr) *CallExpr { - if p.trace { +func (p *Parser) ParseCall(x node.Expr) *node.CallExpr { + if p.Trace { defer untracep(tracep(p, "Call")) } - lparen := p.expect(token.LParen) - p.exprLevel++ + return &node.CallExpr{ + Func: x, + CallArgs: *p.ParseCallArgs(token.LParen, token.RParen), + } +} + +func (p *Parser) ParseCallArgs(tlparen, trparen token.Token) *node.CallArgs { + if p.Trace { + defer untracep(tracep(p, "CallArgs")) + } + + lparen := p.Expect(tlparen) + p.ExprLevel++ var ( - args CallExprArgs - namedArgs CallExprNamedArgs + args node.CallExprArgs + namedArgs node.CallExprNamedArgs ) - for p.token.Token != token.RParen && p.token.Token != token.EOF && p.token.Token != token.Semicolon { - if p.token.Token == token.Ellipsis { - elipsis := &EllipsisValue{Pos: p.token.Pos} - p.next() - elipsis.Value = p.parseExpr() - if _, ok := elipsis.Value.(*MapLit); ok { + for p.Token.Token != trparen && p.Token.Token != token.EOF && p.Token.Token != token.Semicolon { + if p.Token.Token == token.Ellipsis { + elipsis := &node.EllipsisValue{Pos: p.Token.Pos} + p.Next() + elipsis.Value = p.ParseExpr() + if _, ok := elipsis.Value.(*node.MapLit); ok { namedArgs.Ellipsis = elipsis goto done } else { @@ -387,89 +464,88 @@ func (p *Parser) parseCall(x Expr) *CallExpr { } goto kw } - args.Values = append(args.Values, p.parseExpr()) - switch p.token.Token { + args.Values = append(args.Values, p.ParseExpr()) + switch p.Token.Token { case token.Assign: val := args.Values[len(args.Values)-1] args.Values = args.Values[:len(args.Values)-1] switch t := val.(type) { - case *Ident: - namedArgs.Names = append(namedArgs.Names, NamedArgExpr{Ident: t}) - case *StringLit: - namedArgs.Names = append(namedArgs.Names, NamedArgExpr{String: t}) + case *node.Ident: + namedArgs.Names = append(namedArgs.Names, node.NamedArgExpr{Ident: t}) + case *node.StringLit: + namedArgs.Names = append(namedArgs.Names, node.NamedArgExpr{Lit: t}) default: - p.errorExpected(val.Pos(), "string|ident") + p.ErrorExpected(val.Pos(), "string|ident") } - p.next() - namedArgs.Values = append(namedArgs.Values, p.parseExpr()) + p.Next() + namedArgs.Values = append(namedArgs.Values, p.ParseExpr()) goto kw case token.Semicolon: goto kw } - if !p.atComma("call argument", token.RParen) { + if !p.AtComma("call argument", trparen) { break } - p.next() + p.Next() } kw: - if (p.token.Token == token.Semicolon && p.token.Literal == ";") || - (p.token.Token == token.Comma && (len(namedArgs.Names) == 1 || args.Ellipsis != nil)) { - p.next() + if (p.Token.Token == token.Semicolon && p.Token.Literal == ";") || + (p.Token.Token == token.Comma && (len(namedArgs.Names) == 1 || args.Ellipsis != nil)) { + p.Next() for { - switch p.token.Token { + switch p.Token.Token { case token.Ellipsis: - namedArgs.Ellipsis = &EllipsisValue{Pos: p.token.Pos} - p.next() - namedArgs.Ellipsis.Value = p.parseExpr() + namedArgs.Ellipsis = &node.EllipsisValue{Pos: p.Token.Pos} + p.Next() + namedArgs.Ellipsis.Value = p.ParseExpr() goto done - case token.RParen, token.EOF: + case trparen, token.EOF: goto done default: - expr := p.parsePrimaryExpr() + expr := p.ParsePrimaryExpr() switch t := expr.(type) { - case *Ident: - namedArgs.Names = append(namedArgs.Names, NamedArgExpr{Ident: t}) - case *StringLit: - namedArgs.Names = append(namedArgs.Names, NamedArgExpr{String: t}) - case *CallExpr, *SelectorExpr, *MapLit: - namedArgs.Ellipsis = &EllipsisValue{p.token.Pos, t} - p.expect(token.Ellipsis) - if !p.atComma("call argument", token.RParen) { + case *node.Ident: + namedArgs.Names = append(namedArgs.Names, node.NamedArgExpr{Ident: t}) + case *node.StringLit: + namedArgs.Names = append(namedArgs.Names, node.NamedArgExpr{Lit: t}) + case *node.CallExpr, *node.SelectorExpr, *node.MapLit: + namedArgs.Ellipsis = &node.EllipsisValue{Pos: p.Token.Pos, Value: t} + p.Expect(token.Ellipsis) + if !p.AtComma("call argument", trparen) { goto done } default: - pos := p.token.Pos - p.errorExpected(pos, "string|ident|selector|call") + pos := p.Token.Pos + p.ErrorExpected(pos, "string|ident|selector|call") p.advance(stmtStart) goto done } // check if is flag - switch p.token.Token { - case token.Comma, token.RParen: + switch p.Token.Token { + case token.Comma, trparen: namedArgs.Values = append(namedArgs.Values, nil) // is flag default: - p.expect(token.Assign) - namedArgs.Values = append(namedArgs.Values, p.parseExpr()) + p.Expect(token.Assign) + namedArgs.Values = append(namedArgs.Values, p.ParseExpr()) } - if !p.atComma("call argument", token.RParen) { + if !p.AtComma("call argument", trparen) { break } - p.next() + p.Next() } } } done: - p.exprLevel-- - rparen := p.expect(token.RParen) - return &CallExpr{ - Func: x, + p.ExprLevel-- + rparen := p.Expect(trparen) + return &node.CallArgs{ LParen: lparen, RParen: rparen, Args: args, @@ -477,49 +553,49 @@ done: } } -func (p *Parser) atComma(context string, follow token.Token) bool { - if p.token.Token == token.Comma { +func (p *Parser) AtComma(context string, follow token.Token) bool { + if p.Token.Token == token.Comma { return true } - if p.token.Token != follow { + if p.Token.Token != follow { msg := "missing ','" - if p.token.Token == token.Semicolon && p.token.Literal == "\n" { + if p.Token.Token == token.Semicolon && p.Token.Literal == "\n" { msg += " before newline" } - p.error(p.token.Pos, msg+" in "+context) + p.Error(p.Token.Pos, msg+" in "+context) return true // "insert" comma and continue } return false } -func (p *Parser) parseIndexOrSlice(x Expr) Expr { - if p.trace { +func (p *Parser) ParseIndexOrSlice(x node.Expr) node.Expr { + if p.Trace { defer untracep(tracep(p, "IndexOrSlice")) } - lbrack := p.expect(token.LBrack) - p.exprLevel++ + lbrack := p.Expect(token.LBrack) + p.ExprLevel++ - var index [2]Expr - if p.token.Token != token.Colon { - index[0] = p.parseExpr() + var index [2]node.Expr + if p.Token.Token != token.Colon { + index[0] = p.ParseExpr() } numColons := 0 - if p.token.Token == token.Colon { + if p.Token.Token == token.Colon { numColons++ - p.next() + p.Next() - if p.token.Token != token.RBrack && p.token.Token != token.EOF { - index[1] = p.parseExpr() + if p.Token.Token != token.RBrack && p.Token.Token != token.EOF { + index[1] = p.ParseExpr() } } - p.exprLevel-- - rbrack := p.expect(token.RBrack) + p.ExprLevel-- + rbrack := p.Expect(token.RBrack) if numColons > 0 { // slice expression - return &SliceExpr{ + return &node.SliceExpr{ Expr: x, LBrack: lbrack, RBrack: rbrack, @@ -527,7 +603,7 @@ func (p *Parser) parseIndexOrSlice(x Expr) Expr { High: index[1], } } - return &IndexExpr{ + return &node.IndexExpr{ Expr: x, LBrack: lbrack, RBrack: rbrack, @@ -535,519 +611,533 @@ func (p *Parser) parseIndexOrSlice(x Expr) Expr { } } -func (p *Parser) parseSelector(x Expr) Expr { - if p.trace { +func (p *Parser) ParseSelector(x node.Expr) node.Expr { + if p.Trace { defer untracep(tracep(p, "Selector")) } - var sel Expr - if p.token.Token == token.LParen { - lparen := p.token.Pos - p.next() - sel = p.parseExpr() - rparen := p.expect(token.RParen) - sel = &ParenExpr{sel, lparen, rparen} + var sel node.Expr + if p.Token.Token == token.LParen { + lparen := p.Token.Pos + p.Next() + sel = p.ParseExpr() + rparen := p.Expect(token.RParen) + sel = &node.ParenExpr{Expr: sel, LParen: lparen, RParen: rparen} } else { - ident := p.parseIdent() - sel = &StringLit{ + ident := p.ParseIdent() + sel = &node.StringLit{ Value: ident.Name, ValuePos: ident.NamePos, Literal: ident.Name, } } - return &SelectorExpr{Expr: x, Sel: sel} + return &node.SelectorExpr{Expr: x, Sel: sel} } -func (p *Parser) parseNullishSelector(x Expr) Expr { - if p.trace { +func (p *Parser) ParseNullishSelector(x node.Expr) node.Expr { + if p.Trace { defer untracep(tracep(p, "NullishSelector")) } - var sel Expr - if p.token.Token == token.LParen { - lparen := p.token.Pos - p.next() - sel = p.parseExpr() - rparen := p.expect(token.RParen) - sel = &ParenExpr{sel, lparen, rparen} + var sel node.Expr + if p.Token.Token == token.LParen { + lparen := p.Token.Pos + p.Next() + sel = p.ParseExpr() + rparen := p.Expect(token.RParen) + sel = &node.ParenExpr{Expr: sel, LParen: lparen, RParen: rparen} } else { - ident := p.parseIdent() - sel = &StringLit{ + ident := p.ParseIdent() + sel = &node.StringLit{ Value: ident.Name, ValuePos: ident.NamePos, Literal: ident.Name, } } - return &NullishSelectorExpr{Expr: x, Sel: sel} + return &node.NullishSelectorExpr{Expr: x, Sel: sel} } -func (p *Parser) parsePrimitiveOperand() Expr { - switch p.token.Token { +func (p *Parser) ParseStringLit() *node.StringLit { + v, _ := strconv.Unquote(p.Token.Literal) + x := &node.StringLit{ + Value: v, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, + } + p.Next() + return x +} + +func (p *Parser) ParsePrimitiveOperand() node.Expr { + switch p.Token.Token { case token.Ident: - return p.parseIdent() + return p.ParseIdent() case token.Int: - v, _ := strconv.ParseInt(p.token.Literal, 0, 64) - x := &IntLit{ + v, _ := strconv.ParseInt(p.Token.Literal, 0, 64) + x := &node.IntLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Uint: - v, _ := strconv.ParseUint(strings.TrimSuffix(p.token.Literal, "u"), 0, 64) - x := &UintLit{ + v, _ := strconv.ParseUint(strings.TrimSuffix(p.Token.Literal, "u"), 0, 64) + x := &node.UintLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Float: - v, _ := strconv.ParseFloat(p.token.Literal, 64) - x := &FloatLit{ + v, _ := strconv.ParseFloat(p.Token.Literal, 64) + x := &node.FloatLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Char: - return p.parseCharLit() + return p.ParseCharLit() case token.String: - v, _ := strconv.Unquote(p.token.Literal) - x := &StringLit{ - Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, - } - p.next() - return x + return p.ParseStringLit() case token.True: - x := &BoolLit{ + x := &node.BoolLit{ Value: true, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.False: - x := &BoolLit{ + x := &node.BoolLit{ Value: false, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Nil: - x := &NilLit{TokenPos: p.token.Pos} - p.next() + x := &node.NilLit{TokenPos: p.Token.Pos} + p.Next() return x case token.Callee: - x := &CalleeKeyword{TokenPos: p.token.Pos, Literal: p.token.Literal} - p.next() + x := &node.CalleeKeyword{TokenPos: p.Token.Pos, Literal: p.Token.Literal} + p.Next() return x case token.Args: - x := &ArgsKeyword{TokenPos: p.token.Pos, Literal: p.token.Literal} - p.next() + x := &node.ArgsKeyword{TokenPos: p.Token.Pos, Literal: p.Token.Literal} + p.Next() return x case token.NamedArgs: - x := &NamedArgsKeyword{TokenPos: p.token.Pos, Literal: p.token.Literal} - p.next() + x := &node.NamedArgsKeyword{TokenPos: p.Token.Pos, Literal: p.Token.Literal} + p.Next() return x case token.StdIn: - x := &StdInLit{TokenPos: p.token.Pos} - p.next() + x := &node.StdInLit{TokenPos: p.Token.Pos} + p.Next() return x case token.StdOut: - x := &StdOutLit{TokenPos: p.token.Pos} - p.next() + x := &node.StdOutLit{TokenPos: p.Token.Pos} + p.Next() return x case token.StdErr: - x := &StdErrLit{TokenPos: p.token.Pos} - p.next() + x := &node.StdErrLit{TokenPos: p.Token.Pos} + p.Next() return x } - pos := p.token.Pos - p.errorExpected(pos, "primitive operand") + pos := p.Token.Pos + p.ErrorExpected(pos, "primitive operand") p.advance(stmtStart) - return &BadExpr{From: pos, To: p.token.Pos} + return &node.BadExpr{From: pos, To: p.Token.Pos} } -func (p *Parser) parseOperand() Expr { - if p.trace { +func (p *Parser) ParseOperand() node.Expr { + if p.Trace { defer untracep(tracep(p, "Operand")) } - switch p.token.Token { + switch p.Token.Token { case token.Ident: - return p.parseIdent() + return p.ParseIdent() case token.Int: - v, _ := strconv.ParseInt(p.token.Literal, 0, 64) - x := &IntLit{ + v, _ := strconv.ParseInt(p.Token.Literal, 0, 64) + x := &node.IntLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Uint: - v, _ := strconv.ParseUint(strings.TrimSuffix(p.token.Literal, "u"), 0, 64) - x := &UintLit{ + v, _ := strconv.ParseUint(strings.TrimSuffix(p.Token.Literal, "u"), 0, 64) + x := &node.UintLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Float: - v, _ := strconv.ParseFloat(p.token.Literal, 64) - x := &FloatLit{ + v, _ := strconv.ParseFloat(p.Token.Literal, 64) + x := &node.FloatLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Decimal: - v, err := decimal.NewFromString(strings.TrimSuffix(p.token.Literal, "d")) + v, err := decimal.NewFromString(strings.TrimSuffix(p.Token.Literal, "d")) if err != nil { - p.error(p.token.Pos, err.Error()) + p.Error(p.Token.Pos, err.Error()) } - x := &DecimalLit{ + x := &node.DecimalLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Char: - return p.parseCharLit() + return p.ParseCharLit() case token.String: - v, _ := strconv.Unquote(p.token.Literal) - x := &StringLit{ + v, _ := strconv.Unquote(p.Token.Literal) + x := &node.StringLit{ Value: v, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.True: - x := &BoolLit{ + x := &node.BoolLit{ Value: true, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.False: - x := &BoolLit{ + x := &node.BoolLit{ Value: false, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x case token.Nil: - x := &NilLit{TokenPos: p.token.Pos} - p.next() + x := &node.NilLit{TokenPos: p.Token.Pos} + p.Next() return x case token.StdIn: - x := &StdInLit{TokenPos: p.token.Pos} - p.next() + x := &node.StdInLit{TokenPos: p.Token.Pos} + p.Next() return x case token.StdOut: - x := &StdOutLit{TokenPos: p.token.Pos} - p.next() + x := &node.StdOutLit{TokenPos: p.Token.Pos} + p.Next() return x case token.StdErr: - x := &StdErrLit{TokenPos: p.token.Pos} - p.next() + x := &node.StdErrLit{TokenPos: p.Token.Pos} + p.Next() return x case token.Callee: - x := &CalleeKeyword{TokenPos: p.token.Pos, Literal: p.token.Literal} - p.next() + x := &node.CalleeKeyword{TokenPos: p.Token.Pos, Literal: p.Token.Literal} + p.Next() return x case token.Args: - x := &ArgsKeyword{TokenPos: p.token.Pos, Literal: p.token.Literal} - p.next() + x := &node.ArgsKeyword{TokenPos: p.Token.Pos, Literal: p.Token.Literal} + p.Next() return x case token.NamedArgs: - x := &NamedArgsKeyword{TokenPos: p.token.Pos, Literal: p.token.Literal} - p.next() + x := &node.NamedArgsKeyword{TokenPos: p.Token.Pos, Literal: p.Token.Literal} + p.Next() return x case token.Import: - return p.parseImportExpr() + return p.ParseImportExpr() case token.LParen, token.Begin: - return p.parseParemExpr() + return p.ParseParemExpr() case token.LBrack: // array literal - return p.parseArrayLit() + return p.ParseArrayLit() case token.LBrace: // map literal - return p.parseMapLit() + return p.ParseMapLit() case token.Func: // function literal - return p.parseFuncLit() + return p.ParseFuncLit() case token.Text: - return p.parseTextStmt() + return p.ParseTextStmt() } - pos := p.token.Pos - p.errorExpected(pos, "operand") + pos := p.Token.Pos + p.ErrorExpected(pos, "operand") p.advance(stmtStart) - return &BadExpr{From: pos, To: p.token.Pos} + return &node.BadExpr{From: pos, To: p.Token.Pos} } -func (p *Parser) parseImportExpr() Expr { - pos := p.token.Pos - p.next() - p.expect(token.LParen) - if p.token.Token != token.String { - p.errorExpected(p.token.Pos, "module name") +func (p *Parser) ParseImportExpr() node.Expr { + pos := p.Token.Pos + p.Next() + p.Expect(token.LParen) + if p.Token.Token != token.String { + p.ErrorExpected(p.Token.Pos, "module name") p.advance(stmtStart) - return &BadExpr{From: pos, To: p.token.Pos} + return &node.BadExpr{From: pos, To: p.Token.Pos} } // module name - moduleName, _ := strconv.Unquote(p.token.Literal) - expr := &ImportExpr{ + moduleName, _ := strconv.Unquote(p.Token.Literal) + expr := &node.ImportExpr{ ModuleName: moduleName, Token: token.Import, TokenPos: pos, } - p.next() - p.expect(token.RParen) + p.Next() + p.Expect(token.RParen) return expr } -func (p *Parser) parseParemExpr() Expr { - if p.trace { +func (p *Parser) ParseParemExpr() node.Expr { + if p.Trace { defer untracep(tracep(p, "ParemExpr")) } - lparen := p.token.Pos + lparen := p.Token.Pos end := token.RParen - switch p.token.Token { + switch p.Token.Token { case token.LParen: case token.Begin: end = token.End default: - p.errorExpected(lparen, "'"+token.LParen.String()+"' or '"+token.Begin.String()+"'") + p.ErrorExpected(lparen, "'"+token.LParen.String()+"' or '"+token.Begin.String()+"'") return nil } - p.next() - if end == token.End && p.token.Token == token.Semicolon { - p.next() + p.Next() + if end == token.End && p.Token.Token == token.Semicolon { + p.Next() } - if p.token.Token == token.Semicolon && p.token.Literal == ";" { - return p.parseKeyValueArrayLit(lparen) + if p.Token.Token == token.Semicolon && p.Token.Literal == ";" { + return p.ParseKeyValueArrayLit(lparen) } - p.exprLevel++ - x := p.parseExpr() - p.exprLevel-- - rparen := p.expect(end) + p.ExprLevel++ + x := p.ParseExpr() + p.ExprLevel-- + rparen := p.Expect(end) - return &ParenExpr{ + return &node.ParenExpr{ LParen: lparen, Expr: x, RParen: rparen, } } -func (p *Parser) parseCharLit() Expr { - if n := len(p.token.Literal); n >= 3 { - code, _, _, err := strconv.UnquoteChar(p.token.Literal[1:n-1], '\'') + +func (p *Parser) ParseCharLit() node.Expr { + if n := len(p.Token.Literal); n >= 3 { + code, _, _, err := strconv.UnquoteChar(p.Token.Literal[1:n-1], '\'') if err == nil { - x := &CharLit{ + x := &node.CharLit{ Value: code, - ValuePos: p.token.Pos, - Literal: p.token.Literal, + ValuePos: p.Token.Pos, + Literal: p.Token.Literal, } - p.next() + p.Next() return x } } - pos := p.token.Pos - p.error(pos, "illegal char literal") - p.next() - return &BadExpr{ + pos := p.Token.Pos + p.Error(pos, "illegal char literal") + p.Next() + return &node.BadExpr{ From: pos, - To: p.token.Pos, + To: p.Token.Pos, } } -func (p *Parser) parseFuncLit() Expr { - if p.trace { +func (p *Parser) ParseFuncLit() node.Expr { + if p.Trace { defer untracep(tracep(p, "FuncLit")) } - typ := p.parseFuncType() - p.exprLevel++ - body, closure := p.parseBody() - p.exprLevel-- + typ := p.ParseFuncType() + p.ExprLevel++ + body, closure := p.ParseBody() + p.ExprLevel-- if closure != nil { - return &ClosureLit{ + return &node.ClosureLit{ Type: typ, Body: closure, } } - return &FuncLit{ + return &node.FuncLit{ Type: typ, Body: body, } } -func (p *Parser) parseArrayLit() Expr { - if p.trace { +func (p *Parser) ParseArrayLit() node.Expr { + if p.Trace { defer untracep(tracep(p, "ArrayLit")) } - lbrack := p.expect(token.LBrack) - p.exprLevel++ + lbrack := p.Expect(token.LBrack) + p.ExprLevel++ - var elements []Expr - for p.token.Token != token.RBrack && p.token.Token != token.EOF { - elements = append(elements, p.parseExpr()) + var elements []node.Expr + for p.Token.Token != token.RBrack && p.Token.Token != token.EOF { + elements = append(elements, p.ParseExpr()) - if !p.atComma("array literal", token.RBrack) { + if !p.AtComma("array literal", token.RBrack) { break } - p.next() + p.Next() } - p.exprLevel-- - rbrack := p.expect(token.RBrack) - return &ArrayLit{ + p.ExprLevel-- + rbrack := p.Expect(token.RBrack) + return &node.ArrayLit{ Elements: elements, LBrack: lbrack, RBrack: rbrack, } } -func (p *Parser) parseFuncType() *FuncType { - if p.trace { +func (p *Parser) ParseFuncType() *node.FuncType { + if p.Trace { defer untracep(tracep(p, "FuncType")) } - pos := p.expect(token.Func) - params := p.parseFuncParams() - return &FuncType{ + var ( + pos = p.Expect(token.Func) + ident *node.Ident + ) + + if p.Token.Token == token.Ident { + ident = p.ParseIdent() + } + + params := p.ParseFuncParams() + return &node.FuncType{ FuncPos: pos, + Ident: ident, Params: *params, } } -func (p *Parser) parseFuncParam(prev Spec) (spec Spec) { - p.skipSpace() +func (p *Parser) ParseFuncParam(prev node.Spec) (spec node.Spec) { + p.SkipSpace() var ( - pos = p.token.Pos + pos = p.Token.Pos - ident *Ident + ident *node.Ident variadic bool named bool - value Expr + value node.Expr ) - if p.token.Token == token.Semicolon && p.token.Literal == ";" { - p.next() - p.skipSpace() + if p.Token.Token == token.Semicolon && p.Token.Literal == ";" { + p.Next() + p.SkipSpace() named = true } else if prev != nil { switch t := prev.(type) { - case *NamedParamSpec: + case *node.NamedParamSpec: if t.Value == nil { - p.error(pos, "unexpected func param declaration") - p.expectSemi() + p.Error(pos, "unexpected func param declaration") + p.ExpectSemi() } named = true - case *ParamSpec: + case *node.ParamSpec: if t.Variadic { named = true } } } - if p.token.Token == token.Ident { - ident = p.parseIdent() - p.skipSpace() + if p.Token.Token == token.Ident { + ident = p.ParseIdent() + p.SkipSpace() if named { - p.expect(token.Assign) - value = p.parseExpr() - } else if p.token.Literal == ";" { + p.Expect(token.Assign) + value = p.ParseExpr() + } else if p.Token.Literal == ";" { goto done } - } else if p.token.Token == token.Ellipsis { + } else if p.Token.Token == token.Ellipsis { variadic = true - p.next() - ident = p.parseIdent() - p.skipSpace() + p.Next() + ident = p.ParseIdent() + p.SkipSpace() } - if p.token.Token == token.Comma { - p.next() - p.skipSpace() - } else if p.token.Token == token.Semicolon { - if p.token.Token == token.Assign { + if p.Token.Token == token.Comma { + p.Next() + p.SkipSpace() + } else if p.Token.Token == token.Semicolon { + if p.Token.Token == token.Assign { named = true - p.next() - value = p.parseExpr() - p.skipSpace() - if p.token.Token == token.Comma { - p.next() - p.skipSpace() + p.Next() + value = p.ParseExpr() + p.SkipSpace() + if p.Token.Token == token.Comma { + p.Next() + p.SkipSpace() } } else if !named { - p.expectSemi() + p.ExpectSemi() } } if ident == nil { - p.error(pos, "wrong func params declaration") - p.expectSemi() + p.Error(pos, "wrong func params declaration") + p.ExpectSemi() } if named { if value == nil && !variadic { - p.error(pos, "wrong func params declaration") + p.Error(pos, "wrong func params declaration") } - return &NamedParamSpec{ + return &node.NamedParamSpec{ Ident: ident, Value: value, } } done: - return &ParamSpec{ + return &node.ParamSpec{ Ident: ident, Variadic: variadic, } } -func (p *Parser) parseFuncParams() *FuncParams { - if p.trace { +func (p *Parser) ParseFuncParams() *node.FuncParams { + if p.Trace { defer untracep(tracep(p, "FuncParams")) } var ( - args ArgsList - namedArgs NamedArgsList - lparen = p.token.Pos - spec Spec + args node.ArgsList + namedArgs node.NamedArgsList + lparen = p.Token.Pos + spec node.Spec ) - p.next() + p.Next() - for i := 0; p.token.Token != token.RParen && p.token.Token != token.EOF; i++ { //nolint:predeclared - spec = p.parseFuncParam(spec) - if p, _ := spec.(*ParamSpec); p != nil { + for i := 0; p.Token.Token != token.RParen && p.Token.Token != token.EOF; i++ { //nolint:predeclared + spec = p.ParseFuncParam(spec) + if p, _ := spec.(*node.ParamSpec); p != nil { if p.Variadic { args.Var = p.Ident } else { args.Values = append(args.Values, p.Ident) } } else { - p := spec.(*NamedParamSpec) + p := spec.(*node.NamedParamSpec) if p.Value == nil { namedArgs.Var = p.Ident } else { @@ -1057,9 +1147,9 @@ func (p *Parser) parseFuncParams() *FuncParams { } } - rparen := p.expect(token.RParen) + rparen := p.Expect(token.RParen) - return &FuncParams{ + return &node.FuncParams{ LParen: lparen, RParen: rparen, Args: args, @@ -1067,24 +1157,24 @@ func (p *Parser) parseFuncParams() *FuncParams { } } -func (p *Parser) parseBody() (b *BlockStmt, closure Expr) { - if p.trace { +func (p *Parser) ParseBody() (b *node.BlockStmt, closure node.Expr) { + if p.Trace { defer untracep(tracep(p, "Body")) } - p.skipSpace() + p.SkipSpace() - if p.token.Token == token.Assign { - p.next() - p.expect(token.Greater) + if p.Token.Token == token.Assign { + p.Next() + p.Expect(token.Greater) - if p.token.Token.IsBlockStart() { - closure = &BlockExpr{p.parseBlockStmt()} + if p.Token.Token.IsBlockStart() { + closure = &node.BlockExpr{BlockStmt: p.ParseBlockStmt()} } else { - closure = p.parseExpr() + closure = p.ParseExpr() } } else { - b = p.parseBlockStmt(BlockWrap{ + b = p.ParseBlockStmt(BlockWrap{ Start: token.Do, Ends: []BlockEnd{ {token.End, true}, @@ -1094,27 +1184,27 @@ func (p *Parser) parseBody() (b *BlockStmt, closure Expr) { return } -func (p *Parser) parseStmtList(start token.Token, ends ...BlockWrap) (list []Stmt, end *BlockEnd) { - if p.trace { +func (p *Parser) ParseStmtList(start token.Token, ends ...BlockWrap) (list []node.Stmt, end *BlockEnd) { + if p.Trace { defer untracep(tracep(p, "StatementList")) } - var s Stmt + var s node.Stmt for { - switch p.token.Token { + switch p.Token.Token { case token.EOF, token.RBrace: return case token.Semicolon: - p.next() + p.Next() default: if start != 0 { for _, end_ := range ends { if start == end_.Start { for _, e := range end_.Ends { - if p.token.Token == e.Token { + if p.Token.Token == e.Token { if e.Next { - p.next() + p.Next() } end = &e return @@ -1123,8 +1213,8 @@ func (p *Parser) parseStmtList(start token.Token, ends ...BlockWrap) (list []Stm } } } - if s = p.parseStmt(); s != nil { - if _, ok := s.(*EmptyStmt); ok { + if s = p.ParseStmt(); s != nil { + if _, ok := s.(*node.EmptyStmt); ok { continue } list = append(list, s) @@ -1133,37 +1223,56 @@ func (p *Parser) parseStmtList(start token.Token, ends ...BlockWrap) (list []Stm } } -func (p *Parser) parseIdent() *Ident { - pos := p.token.Pos +func (p *Parser) ParseIdent() *node.Ident { + pos := p.Token.Pos name := "_" - if p.token.Token == token.Ident { - name = p.token.Literal - p.next() + if p.Token.Token == token.Ident { + name = p.Token.Literal + p.Next() } else { - p.expect(token.Ident) + p.Expect(token.Ident) } - return &Ident{ + return &node.Ident{ NamePos: pos, Name: name, } } -func (p *Parser) parseStmt() (stmt Stmt) { - if p.trace { +func (p *Parser) ParseStmt() (stmt node.Stmt) { + if p.Trace { defer untracep(tracep(p, "Statement")) } + tok := p.Token + defer func() { + if p.Token.Token == tok.Token && p.Token.Pos == tok.Pos { + p.Next() + } + }() + + stmt = p.ParseStmtHandler() + + if stmt == nil { + pos := p.Token.Pos + p.ErrorExpected(pos, "statement") + p.advance(stmtStart) + stmt = &node.BadStmt{From: pos, To: p.Token.Pos} + } + return +} + +func (p *Parser) DefaultParseStmt() (stmt node.Stmt) { do: - switch p.token.Token { - case token.Config: - return p.parseConfigStmt() + switch p.Token.Token { + case token.ConfigStart: + return p.ParseConfigStmt() case token.Text: - return p.parseTextStmt() + return p.ParseTextStmt() case token.ToTextBegin: - return p.parseExprToTextStmt() + return p.ParseExprToTextStmt() case token.Var, token.Const, token.Global, token.Param: - return &DeclStmt{Decl: p.parseDecl()} + return &node.DeclStmt{Decl: p.ParseDecl()} case // simple statements token.Func, token.Ident, token.Int, token.Uint, token.Float, token.Char, token.String, token.True, token.False, token.Nil, @@ -1172,153 +1281,145 @@ do: token.Callee, token.Args, token.NamedArgs, token.StdIn, token.StdOut, token.StdErr, token.Then: - s := p.parseSimpleStmt(false) - p.expectSemi() + s := p.ParseSimpleStmt(false) + p.ExpectSemi() return s case token.Return: - return p.parseReturnStmt() + return p.ParseReturnStmt() case token.If: - return p.parseIfStmt() + return p.ParseIfStmt() case token.For: - return p.parseForStmt() + return p.ParseForStmt() case token.Try: - return p.parseTryStmt() + return p.ParseTryStmt() case token.Throw: - return p.parseThrowStmt() + return p.ParseThrowStmt() case token.Break, token.Continue: - return p.parseBranchStmt(p.token.Token) + return p.ParseBranchStmt(p.Token.Token) case token.Semicolon: - p.next() + p.Next() goto do case token.RBrace, token.End: // semicolon may be omitted before a closing "}" - return &EmptyStmt{Semicolon: p.token.Pos, Implicit: true} - default: - pos := p.token.Pos - p.errorExpected(pos, "statement") - p.advance(stmtStart) - return &BadStmt{From: pos, To: p.token.Pos} + return &node.EmptyStmt{Semicolon: p.Token.Pos, Implicit: true} } + + return } -func (p *Parser) parseConfigStmt() (c *ConfigStmt) { - if p.trace { +func (p *Parser) ParseConfigStmt() (c *node.ConfigStmt) { + if p.Trace { defer untracep(tracep(p, "ConfigStmt")) } - c = &ConfigStmt{ - ConfigPos: p.token.Pos, - EndPos: p.token.Data.(Pos), - Literal: p.token.Literal, + + c = &node.ConfigStmt{ + ConfigPos: p.Token.Pos, } - p.next() + p.Next() - testFileSet := NewFileSet() - configLit := "(;" + c.Literal + ")" - testFile := testFileSet.AddFile("config", -1, len(configLit)) - p2 := NewParserWithMode(testFile, []byte(configLit), nil, ParseConfigDisabled) - f, err := p2.ParseFile() - if err != nil { - p2.error(c.ConfigPos, err.Error()) - } else { - cfg := f.Stmts[0].(*ExprStmt).Expr.(*KeyValueArrayLit).Elements - for _, k := range cfg { - switch k.Key.String() { - case "mixed": - if k.Value == nil { - c.Options.Mixed = true - } else if b, ok := k.Value.(*BoolLit); ok { - if b.Value { - c.Options.Mixed = true - } else { - c.Options.NoMixed = true - } - } - case "writer": - if k.Value != nil { - c.Options.WriteFunc = k.Value - } - } - } - } + kva := p.ParseKeyValueArrayLitAt(p.Token.Pos, token.ConfigEnd) + + c.Elements = kva.Elements + c.ParseElements() if c.Options.Mixed { - p.scanner.mode.Set(Mixed) + p.Scanner.SetMode(p.Scanner.Mode() | Mixed) } else if c.Options.NoMixed { - p.scanner.mode.Clear(Mixed) + p.Scanner.SetMode(p.Scanner.Mode() &^ Mixed) } - if !p.scanner.mode.Has(Mixed) { - p.expect(token.Semicolon) - } + p.Expect(token.ConfigEnd) return } -func (p *Parser) parseExprToTextStmt() (ett *ExprToTextStmt) { - if p.trace { +func (p *Parser) ParseExprToTextStmt() (ett *node.ExprToTextStmt) { + if p.Trace { defer untracep(tracep(p, "ExprToText")) } - ett = &ExprToTextStmt{ - StartLit: Literal{p.token.Literal, p.token.Pos}, - } - p.next() - ett.Expr = p.parseExpr() - if p.token.Token == token.ToTextEnd { - ett.EndLit = Literal{p.token.Literal, p.token.Pos} - p.next() + ett = &node.ExprToTextStmt{ + StartLit: ast.Literal{Value: p.Token.Literal, Pos: p.Token.Pos}, + } + if p.Token.Token == token.CodeBegin { + p.Next() + stmts, _ := p.ParseStmtList(token.CodeBegin, BlockWrap{ + Start: token.CodeBegin, + Ends: []BlockEnd{{ + token.CodeEnd, + true, + }}, + }) + if len(stmts) == 1 { + switch t := stmts[0].(type) { + case *node.ExprStmt: + return node.NewExprToTextStmt(t.Expr) + } + } + return node.NewExprToTextStmt(&node.StmtsExpr{Stmts: stmts}) } else { - p.expect(token.ToTextEnd) + p.Next() + ett.Expr = p.ParseExpr() + if p.Token.Token == token.ToTextEnd { + ett.EndLit = ast.Literal{Value: p.Token.Literal, Pos: p.Token.Pos} + p.Next() + } else { + p.Expect(token.ToTextEnd) + } } return } -func (p *Parser) parseTextStmt() (t *TextStmt) { - if p.trace { +func (p *Parser) ParseTextStmt() (t *node.TextStmt) { + if p.Trace { defer untracep(tracep(p, "TextStmt")) } - t = &TextStmt{p.token.Literal, p.token.Pos} - p.next() + t = &node.TextStmt{Literal: p.Token.Literal, TextPos: p.Token.Pos, Data: p.Token.Data} + p.Next() + for p.Token.Token == token.Text { + t.Literal += p.Token.Literal + p.Next() + } return } -func (p *Parser) parseDecl() Decl { - if p.trace { +func (p *Parser) ParseDecl() node.Decl { + if p.Trace { defer untracep(tracep(p, "DeclStmt")) } - switch p.token.Token { + switch p.Token.Token { case token.Global, token.Param: - return p.parseGenDecl(p.token.Token, p.parseParamSpec) + return p.ParseGenDecl(p.Token.Token, p.ParseParamSpec) case token.Var, token.Const: - return p.parseGenDecl(p.token.Token, p.parseValueSpec) + return p.ParseGenDecl(p.Token.Token, p.ParseValueSpec) default: - p.error(p.token.Pos, "only \"param, global, var\" declarations supported") - return &BadDecl{From: p.token.Pos, To: p.token.Pos} + p.Error(p.Token.Pos, "only \"param, global, var\" declarations supported") + return &node.BadDecl{From: p.Token.Pos, To: p.Token.Pos} } } -func (p *Parser) parseGenDecl( +func (p *Parser) ParseGenDecl( keyword token.Token, - fn func(token.Token, bool, []Spec, int) Spec, -) *GenDecl { - if p.trace { + fn func(token.Token, bool, []node.Spec, int) node.Spec, +) *node.GenDecl { + if p.Trace { defer untracep(tracep(p, "GenDecl("+keyword.String()+")")) } - pos := p.expect(keyword) - var lparen, rparen Pos - var list []Spec - if p.token.Token == token.LParen { - lparen = p.token.Pos - p.next() - for i := 0; p.token.Token != token.RParen && p.token.Token != token.EOF; i++ { //nolint:predeclared + pos := p.Expect(keyword) + var lparen, rparen source.Pos + var list []node.Spec + if p.Token.Token == token.LParen { + lparen = p.Token.Pos + p.Next() + for i := 0; p.Token.Token != token.RParen && p.Token.Token != token.EOF; i++ { //nolint:predeclared list = append(list, fn(keyword, true, list, i)) } - rparen = p.expect(token.RParen) - p.expectSemi() + rparen = p.Expect(token.RParen) + p.ExpectSemi() } else { list = append(list, fn(keyword, false, list, 0)) - p.expectSemi() + p.ExpectSemi() } - return &GenDecl{ + return &node.GenDecl{ TokPos: pos, Tok: keyword, Lparen: lparen, @@ -1327,126 +1428,126 @@ func (p *Parser) parseGenDecl( } } -func (p *Parser) parseParamSpec(keyword token.Token, multi bool, prev []Spec, i int) (spec Spec) { - if p.trace { +func (p *Parser) ParseParamSpec(keyword token.Token, multi bool, prev []node.Spec, i int) (spec node.Spec) { + if p.Trace { defer untracep(tracep(p, keyword.String()+"Spec")) } if multi { - p.skipSpace() + p.SkipSpace() } var ( - pos = p.token.Pos + pos = p.Token.Pos - ident *Ident + ident *node.Ident variadic bool named bool - value Expr + value node.Expr ) - if p.token.Token == token.Semicolon && p.token.Literal == ";" { - p.next() + if p.Token.Token == token.Semicolon && p.Token.Literal == ";" { + p.Next() if multi { - p.skipSpace() + p.SkipSpace() } named = true } else if i > 0 { switch t := prev[i-1].(type) { - case *NamedParamSpec: + case *node.NamedParamSpec: if t.Value == nil { - p.error(pos, "unexpected arg declaration") - p.expectSemi() + p.Error(pos, "unexpected arg declaration") + p.ExpectSemi() } named = true - case *ParamSpec: + case *node.ParamSpec: if t.Variadic { named = true } } } - if p.token.Token == token.Ident { - ident = p.parseIdent() - } else if keyword == token.Param && p.token.Token == token.Ellipsis { + if p.Token.Token == token.Ident { + ident = p.ParseIdent() + } else if keyword == token.Param && p.Token.Token == token.Ellipsis { variadic = true - p.next() - ident = p.parseIdent() + p.Next() + ident = p.ParseIdent() if multi { - p.skipSpace() + p.SkipSpace() } } - if multi && p.token.Token == token.Comma { - p.next() - p.skipSpace() + if multi && p.Token.Token == token.Comma { + p.Next() + p.SkipSpace() } else if multi { - if p.token.Token == token.Assign { + if p.Token.Token == token.Assign { named = true - p.next() - value = p.parseExpr() - if p.token.Token == token.Comma || (p.token.Token == token.Semicolon && p.token.Literal == "\n") { - p.next() - p.skipSpace() + p.Next() + value = p.ParseExpr() + if p.Token.Token == token.Comma || (p.Token.Token == token.Semicolon && p.Token.Literal == "\n") { + p.Next() + p.SkipSpace() } } else if !named { - p.expectSemi() + p.ExpectSemi() } } if ident == nil { - p.error(pos, fmt.Sprintf("wrong %s declaration", keyword.String())) - p.expectSemi() + p.Error(pos, fmt.Sprintf("wrong %s declaration", keyword.String())) + p.ExpectSemi() } if named { if value == nil && !variadic { - p.error(pos, fmt.Sprintf("wrong %s declaration", keyword.String())) + p.Error(pos, fmt.Sprintf("wrong %s declaration", keyword.String())) } - return &NamedParamSpec{ + return &node.NamedParamSpec{ Ident: ident, Value: value, } } - return &ParamSpec{ + return &node.ParamSpec{ Ident: ident, Variadic: variadic, } } -func (p *Parser) parseValueSpec(keyword token.Token, multi bool, _ []Spec, i int) Spec { - if p.trace { +func (p *Parser) ParseValueSpec(keyword token.Token, multi bool, _ []node.Spec, i int) node.Spec { + if p.Trace { defer untracep(tracep(p, keyword.String()+"Spec")) } - pos := p.token.Pos - var idents []*Ident - var values []Expr - if p.token.Token == token.Ident { - ident := p.parseIdent() - var expr Expr - if p.token.Token == token.Assign { - p.next() - expr = p.parseExpr() + pos := p.Token.Pos + var idents []*node.Ident + var values []node.Expr + if p.Token.Token == token.Ident { + ident := p.ParseIdent() + var expr node.Expr + if p.Token.Token == token.Assign { + p.Next() + expr = p.ParseExpr() } if keyword == token.Const && expr == nil { if i == 0 { - p.error(p.token.Pos, "missing initializer in const declaration") + p.Error(p.Token.Pos, "missing initializer in const declaration") } } idents = append(idents, ident) values = append(values, expr) - if multi && p.token.Token == token.Comma { - p.next() + if multi && p.Token.Token == token.Comma { + p.Next() } else if multi { - p.expectSemi() + p.ExpectSemi() } } if len(idents) == 0 { - p.error(pos, "wrong var declaration") - p.expectSemi() + p.Error(pos, "wrong var declaration") + p.ExpectSemi() } - spec := &ValueSpec{ + spec := &node.ValueSpec{ Idents: idents, Values: values, Data: i, @@ -1454,53 +1555,96 @@ func (p *Parser) parseValueSpec(keyword token.Token, multi bool, _ []Spec, i int return spec } -func (p *Parser) parseForStmt() Stmt { - if p.trace { +func (p *Parser) ParseForStmt() node.Stmt { + if p.Trace { defer untracep(tracep(p, "ForStmt")) } - pos := p.expect(token.For) + pos := p.Expect(token.For) // for {} - if p.token.Token.IsBlockStart() { - body := p.parseBlockStmt(BlockWrap{token.Do, []BlockEnd{{token.End, true}}}) - p.expectSemi() + if p.Token.Token.IsBlockStart() { + body := p.ParseBlockStmt(BlockWrap{token.Do, []BlockEnd{{token.End, true}}}) + p.ExpectSemi() - return &ForStmt{ + return &node.ForStmt{ ForPos: pos, Body: body, } } - prevLevel := p.exprLevel - p.exprLevel = -1 + prevLevel := p.ExprLevel + p.ExprLevel = -1 - var s1 Stmt - if p.token.Token != token.Semicolon { // skipping init - s1 = p.parseSimpleStmt(true) + var s1 node.Stmt + if p.Token.Token != token.Semicolon { // skipping init + s1 = p.ParseSimpleStmt(true) } // for _ in seq {} or // for value in seq {} or // for key, value in seq {} - if forInStmt, isForIn := s1.(*ForInStmt); isForIn { + if forInStmt, isForIn := s1.(*node.ForInStmt); isForIn { forInStmt.ForPos = pos - p.exprLevel = prevLevel - forInStmt.Body = p.parseBlockStmt(BlockWrap{token.Do, []BlockEnd{{token.End, true}}}) - p.expectSemi() + p.ExprLevel = prevLevel + forInStmt.Body = p.ParseBlockStmt(BlockWrap{token.Do, []BlockEnd{ + {token.End, true}, + {token.Else, false}, + }}) + if p.Token.Token == token.Else { + lbrace := p.Token.Pos + p.Next() + p.SkipSpace() + + switch p.Token.Token { + case token.End: + forInStmt.Else = &node.BlockStmt{LBrace: lbrace, RBrace: p.Token.Pos} + p.Next() + p.ExpectSemi() + case token.LBrace: + forInStmt.Else = p.ParseBlockStmt() + p.ExpectSemi() + case token.Then: + forInStmt.Else = p.ParseBlockStmt(BlockWrap{ + token.Then, + []BlockEnd{ + {token.End, true}, + }, + }) + case token.Colon: + p.Next() + expr := p.ParseExpr() + forInStmt.Else = &node.BlockStmt{ + Stmts: []node.Stmt{&node.ExprStmt{Expr: expr}}, + LBrace: expr.Pos(), + RBrace: expr.End(), + } + p.ExpectSemi() + case token.Semicolon: + p.ExpectSemi() + default: + forInStmt.Else = &node.BlockStmt{LBrace: lbrace, RBrace: p.Token.Pos} + if stmt := p.ParseSimpleStmt(false); stmt != nil { + forInStmt.Else.Stmts = []node.Stmt{stmt} + } + p.ExpectSemi() + } + } else { + p.ExpectSemi() + } return forInStmt } // for init; cond; post {} - var s2, s3 Stmt - if p.token.Token == token.Semicolon { - p.next() - if p.token.Token != token.Semicolon { - s2 = p.parseSimpleStmt(false) // cond + var s2, s3 node.Stmt + if p.Token.Token == token.Semicolon { + p.Next() + if p.Token.Token != token.Semicolon { + s2 = p.ParseSimpleStmt(false) // cond } - p.expect(token.Semicolon) - if !p.token.Token.IsBlockStart() { - s3 = p.parseSimpleStmt(false) // post + p.Expect(token.Semicolon) + if !p.Token.Token.IsBlockStart() { + s3 = p.ParseSimpleStmt(false) // post } } else { // for cond {} @@ -1509,11 +1653,11 @@ func (p *Parser) parseForStmt() Stmt { } // body - p.exprLevel = prevLevel - body := p.parseBlockStmt() - p.expectSemi() - cond := p.makeExpr(s2, "condition expression") - return &ForStmt{ + p.ExprLevel = prevLevel + body := p.ParseBlockStmt() + p.ExpectSemi() + cond := p.MakeExpr(s2, "condition expression") + return &node.ForStmt{ ForPos: pos, Init: s1, Cond: cond, @@ -1522,44 +1666,44 @@ func (p *Parser) parseForStmt() Stmt { } } -func (p *Parser) parseBranchStmt(tok token.Token) Stmt { - if p.trace { +func (p *Parser) ParseBranchStmt(tok token.Token) node.Stmt { + if p.Trace { defer untracep(tracep(p, "BranchStmt")) } - pos := p.expect(tok) + pos := p.Expect(tok) - var label *Ident - if p.token.Token == token.Ident { - label = p.parseIdent() + var label *node.Ident + if p.Token.Token == token.Ident { + label = p.ParseIdent() } - p.expectSemi() - return &BranchStmt{ + p.ExpectSemi() + return &node.BranchStmt{ Token: tok, TokenPos: pos, Label: label, } } -func (p *Parser) parseIfStmt() Stmt { - if p.trace { +func (p *Parser) ParseIfStmt() node.Stmt { + if p.Trace { defer untracep(tracep(p, "IfStmt")) } - pos := p.expect(token.If) - init, cond, starts := p.parseIfHeader() + pos := p.Expect(token.If) + init, cond, starts := p.ParseIfHeader() - var body *BlockStmt - if p.token.Token == token.Colon { - p.next() - expr := p.parseExpr() - body = &BlockStmt{ - Stmts: []Stmt{&ExprStmt{expr}}, + var body *node.BlockStmt + if p.Token.Token == token.Colon { + p.Next() + expr := p.ParseExpr() + body = &node.BlockStmt{ + Stmts: []node.Stmt{&node.ExprStmt{Expr: expr}}, LBrace: expr.Pos(), RBrace: expr.End(), } } else if starts == token.Then { - body = p.parseBlockStmt(BlockWrap{ + body = p.ParseBlockStmt(BlockWrap{ token.Then, []BlockEnd{ {token.End, true}, @@ -1567,21 +1711,22 @@ func (p *Parser) parseIfStmt() Stmt { }, }) } else { - body = p.parseBlockStmt() + body = p.ParseBlockStmt() } - var elseStmt Stmt - if p.token.Token == token.Else { - p.next() + var elseStmt node.Stmt + if p.Token.Token == token.Else { + p.Next() + p.SkipSpace() - switch p.token.Token { + switch p.Token.Token { case token.If: - elseStmt = p.parseIfStmt() - case token.LBrace: - elseStmt = p.parseBlockStmt() - p.expectSemi() + elseStmt = p.ParseIfStmt() + case token.LBrace, p.BlockStart: + elseStmt = p.ParseBlockStmt() + p.ExpectSemi() case token.Then: - elseStmt = p.parseBlockStmt(BlockWrap{ + elseStmt = p.ParseBlockStmt(BlockWrap{ token.Then, []BlockEnd{ {token.End, true}, @@ -1590,22 +1735,27 @@ func (p *Parser) parseIfStmt() Stmt { }, }) case token.Colon: - p.next() - expr := p.parseExpr() - elseStmt = &BlockStmt{ - Stmts: []Stmt{&ExprStmt{expr}}, + p.Next() + expr := p.ParseExpr() + elseStmt = &node.BlockStmt{ + Stmts: []node.Stmt{&node.ExprStmt{Expr: expr}}, LBrace: expr.Pos(), RBrace: expr.End(), } - p.expectSemi() + p.ExpectSemi() default: - p.errorExpected(p.token.Pos, "if or {") - elseStmt = &BadStmt{From: p.token.Pos, To: p.token.Pos} + b := &node.BlockStmt{LBrace: p.Token.Pos, RBrace: p.Token.Pos} + if stmt := p.ParseSimpleStmt(false); stmt != nil { + b.RBrace = p.Token.Pos + b.Stmts = []node.Stmt{stmt} + } + p.ExpectSemi() + elseStmt = b } } else { - p.expectSemi() + p.ExpectSemi() } - return &IfStmt{ + return &node.IfStmt{ IfPos: pos, Init: init, Cond: cond, @@ -1614,12 +1764,12 @@ func (p *Parser) parseIfStmt() Stmt { } } -func (p *Parser) parseTryStmt() Stmt { - if p.trace { +func (p *Parser) ParseTryStmt() node.Stmt { + if p.Trace { defer untracep(tracep(p, "TryStmt")) } - pos := p.expect(token.Try) - body := p.parseBlockStmt(BlockWrap{ + pos := p.Expect(token.Try) + body := p.ParseBlockStmt(BlockWrap{ Start: token.Then, Ends: []BlockEnd{ {token.Catch, false}, @@ -1627,16 +1777,16 @@ func (p *Parser) parseTryStmt() Stmt { {token.End, true}, }, }) - var catchStmt *CatchStmt - var finallyStmt *FinallyStmt - if p.token.Token == token.Catch { - catchStmt = p.parseCatchStmt() + var catchStmt *node.CatchStmt + var finallyStmt *node.FinallyStmt + if p.Token.Token == token.Catch { + catchStmt = p.ParseCatchStmt() } - if p.token.Token == token.Finally || catchStmt == nil { - finallyStmt = p.parseFinallyStmt() + if p.Token.Token == token.Finally || catchStmt == nil { + finallyStmt = p.ParseFinallyStmt() } - p.expectSemi() - return &TryStmt{ + p.ExpectSemi() + return &node.TryStmt{ TryPos: pos, Catch: catchStmt, Finally: finallyStmt, @@ -1644,211 +1794,225 @@ func (p *Parser) parseTryStmt() Stmt { } } -func (p *Parser) parseCatchStmt() *CatchStmt { - if p.trace { +func (p *Parser) ParseCatchStmt() *node.CatchStmt { + if p.Trace { defer untracep(tracep(p, "CatchStmt")) } - pos := p.expect(token.Catch) - var ident *Ident - if p.token.Token == token.Ident { - ident = p.parseIdent() + pos := p.Expect(token.Catch) + var ident *node.Ident + if p.Token.Token == token.Ident { + ident = p.ParseIdent() } - body := p.parseBlockStmt(BlockWrap{ + body := p.ParseBlockStmt(BlockWrap{ token.Then, []BlockEnd{ {token.Finally, false}, {token.End, true}, }, }) - return &CatchStmt{ + return &node.CatchStmt{ CatchPos: pos, Ident: ident, Body: body, } } -func (p *Parser) parseFinallyStmt() *FinallyStmt { - if p.trace { +func (p *Parser) ParseFinallyStmt() *node.FinallyStmt { + if p.Trace { defer untracep(tracep(p, "FinallyStmt")) } - pos := p.expect(token.Finally) - body := p.parseBlockStmt(BlockWrap{ + pos := p.Expect(token.Finally) + body := p.ParseBlockStmt(BlockWrap{ token.Then, []BlockEnd{ {token.End, true}, }, }) - return &FinallyStmt{ + return &node.FinallyStmt{ FinallyPos: pos, Body: body, } } -func (p *Parser) parseThrowStmt() Stmt { - if p.trace { +func (p *Parser) ParseThrowStmt() node.Stmt { + if p.Trace { defer untracep(tracep(p, "ThrowStmt")) } - pos := p.expect(token.Throw) - expr := p.parseExpr() - p.expectSemi() - return &ThrowStmt{ + pos := p.Expect(token.Throw) + expr := p.ParseExpr() + p.ExpectSemi() + return &node.ThrowStmt{ ThrowPos: pos, Expr: expr, } } -func (p *Parser) parseBlockStmt(ends ...BlockWrap) *BlockStmt { - if p.trace { +func (p *Parser) ParseBlockStmt(ends ...BlockWrap) *node.BlockStmt { + if p.Trace { defer untracep(tracep(p, "BlockStmt")) } var ( - lbrace = p.token.Pos - start = p.token.Token + lbrace = p.Token.Pos + start = p.Token.Token ) + if p.BlockStart != 0 { + ends = append(ends, BlockWrap{ + Start: p.BlockStart, + Ends: []BlockEnd{{ + p.BlockEnd, + true, + }}, + }) + } + for _, e := range ends { - if p.token.Token == e.Start { - p.next() + if p.Token.Token == e.Start { + p.Next() goto parse_list } } - p.expect(token.LBrace) + p.Expect(token.LBrace) parse_list: - list, endb := p.parseStmtList(start, ends...) - var rbrace Pos + list, endb := p.ParseStmtList(start, ends...) + var rbrace source.Pos if endb != nil { - rbrace = p.token.Pos + rbrace = p.Token.Pos } else { switch start { case token.Then, token.Do: - rbrace = p.expect(token.End) + if p.Token.Token == token.EOF && p.PrevToken.Token == token.End { + rbrace = p.PrevToken.Pos + } else { + rbrace = p.Expect(token.End) + } default: - rbrace = p.expect(token.RBrace) + rbrace = p.Expect(token.RBrace) } } - return &BlockStmt{ + return &node.BlockStmt{ LBrace: lbrace, RBrace: rbrace, Stmts: list, } } -func (p *Parser) parseIfHeader() (init Stmt, cond Expr, starts token.Token) { - if p.token.Token.IsBlockStart() { - p.error(p.token.Pos, "missing condition in if statement") - cond = &BadExpr{From: p.token.Pos, To: p.token.Pos} +func (p *Parser) ParseIfHeader() (init node.Stmt, cond node.Expr, starts token.Token) { + if p.Token.Token.IsBlockStart() { + p.Error(p.Token.Pos, "missing condition in if statement") + cond = &node.BadExpr{From: p.Token.Pos, To: p.Token.Pos} return } - outer := p.exprLevel - p.exprLevel = -1 - if p.token.Token == token.Semicolon { - p.error(p.token.Pos, "missing init in if statement") + outer := p.ExprLevel + p.ExprLevel = -1 + if p.Token.Token == token.Semicolon { + p.Error(p.Token.Pos, "missing init in if statement") return } - init = p.parseSimpleStmt(false) + init = p.ParseSimpleStmt(false) - var condStmt Stmt - switch p.token.Token { - case token.LBrace, token.Then, token.Colon: + var condStmt node.Stmt + switch p.Token.Token { + case token.LBrace, token.Then, token.Colon, p.BlockStart: condStmt = init init = nil - if p.token.Token == token.Then { + if p.Token.Token == token.Then { starts = token.Then } case token.Semicolon: - p.next() - condStmt = p.parseSimpleStmt(false) - if p.token.Token == token.Then { + p.Next() + condStmt = p.ParseSimpleStmt(false) + if p.Token.Token == token.Then { starts = token.Then } default: - p.error(p.token.Pos, "missing condition in if statement") + p.Error(p.Token.Pos, "missing condition in if statement") } if condStmt != nil { - cond = p.makeExpr(condStmt, "boolean expression") + cond = p.MakeExpr(condStmt, "boolean expression") } if cond == nil { - cond = &BadExpr{From: p.token.Pos, To: p.token.Pos} + cond = &node.BadExpr{From: p.Token.Pos, To: p.Token.Pos} } - p.exprLevel = outer + p.ExprLevel = outer return } -func (p *Parser) makeExpr(s Stmt, want string) Expr { +func (p *Parser) MakeExpr(s node.Stmt, want string) node.Expr { if s == nil { return nil } - if es, isExpr := s.(*ExprStmt); isExpr { + if es, isExpr := s.(*node.ExprStmt); isExpr { return es.Expr } found := "simple statement" - if _, isAss := s.(*AssignStmt); isAss { + if _, isAss := s.(*node.AssignStmt); isAss { found = "assignment" } - p.error(s.Pos(), fmt.Sprintf("expected %s, found %s", want, found)) - return &BadExpr{From: s.Pos(), To: p.safePos(s.End())} + p.Error(s.Pos(), fmt.Sprintf("expected %s, found %s", want, found)) + return &node.BadExpr{From: s.Pos(), To: p.safePos(s.End())} } -func (p *Parser) parseReturnStmt() Stmt { - if p.trace { +func (p *Parser) ParseReturnStmt() node.Stmt { + if p.Trace { defer untracep(tracep(p, "ReturnStmt")) } - pos := p.token.Pos - p.expect(token.Return) + pos := p.Token.Pos + p.Expect(token.Return) - var x Expr - if p.token.Token != token.Semicolon && p.token.Token != token.RBrace { - lbpos := p.token.Pos - x = p.parseExpr() - if p.token.Token != token.Comma { + var x node.Expr + if p.Token.Token != token.Semicolon && p.Token.Token != token.RBrace { + lbpos := p.Token.Pos + x = p.ParseExpr() + if p.Token.Token != token.Comma { goto done } // if the next token is a comma, treat it as multi return so put // expressions into a slice and replace x expression with an ArrayLit. - elements := make([]Expr, 1, 2) + elements := make([]node.Expr, 1, 2) elements[0] = x - for p.token.Token == token.Comma { - p.next() - x = p.parseExpr() + for p.Token.Token == token.Comma { + p.Next() + x = p.ParseExpr() elements = append(elements, x) } - x = &ArrayLit{ + x = &node.ArrayLit{ Elements: elements, LBrack: lbpos, RBrack: x.End(), } } done: - p.expectSemi() - return &ReturnStmt{ + p.ExpectSemi() + return &node.ReturnStmt{ ReturnPos: pos, Result: x, } } -func (p *Parser) parseSimpleStmt(forIn bool) Stmt { - if p.trace { +func (p *Parser) ParseSimpleStmt(forIn bool) node.Stmt { + if p.Trace { defer untracep(tracep(p, "SimpleStmt")) } - x := p.parseExprList() + x := p.ParseExprList() - switch p.token.Token { + switch p.Token.Token { case token.Assign, token.Define: // assignment statement - pos, tok := p.token.Pos, p.token.Token - p.next() - y := p.parseExprList() - return &AssignStmt{ + pos, tok := p.Token.Pos, p.Token.Token + p.Next() + y := p.ParseExprList() + return &node.AssignStmt{ LHS: x, RHS: y, Token: tok, @@ -1856,34 +2020,34 @@ func (p *Parser) parseSimpleStmt(forIn bool) Stmt { } case token.In: if forIn { - p.next() - y := p.parseExpr() + p.Next() + y := p.ParseExpr() - var key, value *Ident + var key, value *node.Ident var ok bool switch len(x) { case 1: - key = &Ident{Name: "_", NamePos: x[0].Pos()} + key = &node.Ident{Name: "_", NamePos: x[0].Pos()} - value, ok = x[0].(*Ident) + value, ok = x[0].(*node.Ident) if !ok { - p.errorExpected(x[0].Pos(), "identifier") - value = &Ident{Name: "_", NamePos: x[0].Pos()} + p.ErrorExpected(x[0].Pos(), "identifier") + value = &node.Ident{Name: "_", NamePos: x[0].Pos()} } case 2: - key, ok = x[0].(*Ident) + key, ok = x[0].(*node.Ident) if !ok { - p.errorExpected(x[0].Pos(), "identifier") - key = &Ident{Name: "_", NamePos: x[0].Pos()} + p.ErrorExpected(x[0].Pos(), "identifier") + key = &node.Ident{Name: "_", NamePos: x[0].Pos()} } - value, ok = x[1].(*Ident) + value, ok = x[1].(*node.Ident) if !ok { - p.errorExpected(x[1].Pos(), "identifier") - value = &Ident{Name: "_", NamePos: x[1].Pos()} + p.ErrorExpected(x[1].Pos(), "identifier") + value = &node.Ident{Name: "_", NamePos: x[1].Pos()} } // TODO: no more than 2 idents } - return &ForInStmt{ + return &node.ForInStmt{ Key: key, Value: value, Iterable: y, @@ -1892,66 +2056,66 @@ func (p *Parser) parseSimpleStmt(forIn bool) Stmt { } if len(x) > 1 { - p.errorExpected(x[0].Pos(), "1 expression") + p.ErrorExpected(x[0].Pos(), "1 expression") // continue with first expression } - switch p.token.Token { + switch p.Token.Token { case token.Define, token.AddAssign, token.SubAssign, token.MulAssign, token.QuoAssign, token.RemAssign, token.AndAssign, token.OrAssign, token.XorAssign, token.ShlAssign, token.ShrAssign, token.AndNotAssign, token.NullichAssign, token.LOrAssign: - pos, tok := p.token.Pos, p.token.Token - p.next() - y := p.parseExpr() - return &AssignStmt{ - LHS: []Expr{x[0]}, - RHS: []Expr{y}, + pos, tok := p.Token.Pos, p.Token.Token + p.Next() + y := p.ParseExpr() + return &node.AssignStmt{ + LHS: []node.Expr{x[0]}, + RHS: []node.Expr{y}, Token: tok, TokenPos: pos, } case token.Inc, token.Dec: // increment or decrement statement - s := &IncDecStmt{Expr: x[0], Token: p.token.Token, TokenPos: p.token.Pos} - p.next() + s := &node.IncDecStmt{Expr: x[0], Token: p.Token.Token, TokenPos: p.Token.Pos} + p.Next() return s } - return &ExprStmt{Expr: x[0]} + return &node.ExprStmt{Expr: x[0]} } -func (p *Parser) parseExprList() (list []Expr) { - if p.trace { +func (p *Parser) ParseExprList() (list []node.Expr) { + if p.Trace { defer untracep(tracep(p, "ExpressionList")) } - list = append(list, p.parseExpr()) - for p.token.Token == token.Comma { - p.next() - list = append(list, p.parseExpr()) + list = append(list, p.ParseExpr()) + for p.Token.Token == token.Comma { + p.Next() + list = append(list, p.ParseExpr()) } return } -func (p *Parser) parseMapElementLit() *MapElementLit { - if p.trace { +func (p *Parser) ParseMapElementLit() *node.MapElementLit { + if p.Trace { defer untracep(tracep(p, "MapElementLit")) } - pos := p.token.Pos + pos := p.Token.Pos name := "_" - if p.token.Token == token.Ident || p.token.Token.IsKeyword() { - name = p.token.Literal - } else if p.token.Token == token.String { - v, _ := strconv.Unquote(p.token.Literal) + if p.Token.Token == token.Ident || p.Token.Token.IsKeyword() { + name = p.Token.Literal + } else if p.Token.Token == token.String { + v, _ := strconv.Unquote(p.Token.Literal) name = v } else { - p.errorExpected(pos, "map key") + p.ErrorExpected(pos, "map key") } - p.next() - colonPos := p.expect(token.Colon) - valueExpr := p.parseExpr() - return &MapElementLit{ + p.Next() + colonPos := p.Expect(token.Colon) + valueExpr := p.ParseExpr() + return &node.MapElementLit{ Key: name, KeyPos: pos, ColonPos: colonPos, @@ -1959,129 +2123,142 @@ func (p *Parser) parseMapElementLit() *MapElementLit { } } -func (p *Parser) parseMapLit() *MapLit { - if p.trace { +func (p *Parser) ParseMapLit() *node.MapLit { + if p.Trace { defer untracep(tracep(p, "MapLit")) } - lbrace := p.expect(token.LBrace) - p.exprLevel++ + lbrace := p.Expect(token.LBrace) + p.ExprLevel++ - var elements []*MapElementLit - for p.token.Token != token.RBrace && p.token.Token != token.EOF { - elements = append(elements, p.parseMapElementLit()) + var elements []*node.MapElementLit + for p.Token.Token != token.RBrace && p.Token.Token != token.EOF { + elements = append(elements, p.ParseMapElementLit()) - if !p.atComma("map literal", token.RBrace) { + if !p.AtComma("map literal", token.RBrace) { break } - p.next() + p.Next() } - p.exprLevel-- - rbrace := p.expect(token.RBrace) - return &MapLit{ + p.ExprLevel-- + rbrace := p.Expect(token.RBrace) + return &node.MapLit{ LBrace: lbrace, RBrace: rbrace, Elements: elements, } } -func (p *Parser) parseKeyValueLit() *KeyValueLit { - if p.trace { +func (p *Parser) ParseKeyValueLit(endToken token.Token) *node.KeyValueLit { + if p.Trace { defer untracep(tracep(p, "KeyValueLit")) } - p.skipSpace() + p.SkipSpace() var ( - pos = p.token.Pos - keyExpr = p.parsePrimitiveOperand() - colonPos Pos - valueExpr Expr + keyExpr = p.ParsePrimitiveOperand() + valueExpr node.Expr ) - p.skipSpace() + p.SkipSpace() - switch p.token.Token { - case token.Comma, token.RParen: + switch p.Token.Token { + case token.Comma, endToken: default: - colonPos = p.expect(token.Assign) - valueExpr = p.parseExpr() - p.skipSpace() + p.Expect(token.Assign) + valueExpr = p.ParseExpr() + p.SkipSpace() } - return &KeyValueLit{ - Key: keyExpr, - KeyPos: pos, - ColonPos: colonPos, - Value: valueExpr, + return &node.KeyValueLit{ + Key: keyExpr, + Value: valueExpr, } } -func (p *Parser) parseKeyValueArrayLit(lbrace Pos) *KeyValueArrayLit { - if p.trace { - defer untracep(tracep(p, "parseKeyValueArrayLit")) - } - - p.exprLevel++ - p.expect(token.Semicolon) +func (p *Parser) ParseKeyValueArrayLitAt(lbrace source.Pos, rbraceToken token.Token) *node.KeyValueArrayLit { + p.ExprLevel++ + var elements []*node.KeyValueLit - var elements []*KeyValueLit - for p.token.Token != token.RParen && p.token.Token != token.EOF { - elements = append(elements, p.parseKeyValueLit()) + for p.Token.Token != rbraceToken && p.Token.Token != token.EOF { + elements = append(elements, p.ParseKeyValueLit(rbraceToken)) - if !p.atComma("keyValueArray literal", token.RParen) { + if !p.AtComma("keyValueArray literal", rbraceToken) { break } - p.next() + p.Next() } - p.exprLevel-- - rbrace := p.expect(token.RParen) - return &KeyValueArrayLit{ + p.ExprLevel-- + + if p.Token.Token != rbraceToken { + p.Expect(rbraceToken) + } + rbrace := p.Token.Pos + return &node.KeyValueArrayLit{ LBrace: lbrace, RBrace: rbrace, Elements: elements, } } -func (p *Parser) expect(token token.Token) Pos { - pos := p.token.Pos +func (p *Parser) ParseKeyValueArrayLit(lbrace source.Pos) *node.KeyValueArrayLit { + if p.Trace { + defer untracep(tracep(p, "ParseKeyValueArrayLit")) + } + + p.Expect(token.Semicolon) + + kva := p.ParseKeyValueArrayLitAt(lbrace, token.RParen) + p.Expect(token.RParen) + return kva +} - if p.token.Token != token { - p.errorExpected(pos, "'"+token.String()+"'") +func (p *Parser) Expect(token token.Token) source.Pos { + pos := p.Token.Pos + + if p.Token.Token != token { + p.ErrorExpected(pos, "'"+token.String()+"'") } - p.next() + p.Next() return pos } -func (p *Parser) expectSemi() { - switch p.token.Token { - case token.RParen, token.RBrace, token.End: +func (p *Parser) ExpectSemi() { + switch p.Token.Token { + case token.RParen, token.RBrace, token.Else, token.CodeEnd: // semicolon is optional before a closing ')' or '}' case token.Comma: // permit a ',' instead of a ';' but complain - p.errorExpected(p.token.Pos, "';'") + p.ErrorExpected(p.Token.Pos, "';'") fallthrough + case token.End: + p.Next() + if p.Token.Token == token.Semicolon { + p.Next() + } case token.Semicolon: - p.next() + p.Next() default: - if p.prevToken == token.End { + switch p.PrevToken.Token { + case token.Else, token.End, p.BlockEnd: return } - p.errorExpected(p.token.Pos, "';'") + p.ErrorExpected(p.Token.Pos, "';'") p.advance(stmtStart) } } func (p *Parser) advance(to map[token.Token]bool) { - for ; p.token.Token != token.EOF; p.next() { - if to[p.token.Token] { - if p.token.Pos == p.syncPos && p.syncCount < 10 { + for ; p.Token.Token != token.EOF; p.Next() { + if to[p.Token.Token] { + if p.Token.Pos == p.syncPos && p.syncCount < 10 { p.syncCount++ return } - if p.token.Pos > p.syncPos { - p.syncPos = p.token.Pos + if p.Token.Pos > p.syncPos { + p.syncPos = p.Token.Pos p.syncCount = 0 return } @@ -2089,11 +2266,11 @@ func (p *Parser) advance(to map[token.Token]bool) { } } -func (p *Parser) error(pos Pos, msg string) { - filePos := p.file.Position(pos) +func (p *Parser) Error(pos source.Pos, msg string) { + filePos := p.File.Position(pos) - n := len(p.errors) - if n > 0 && p.errors[n-1].Pos.Line == filePos.Line { + n := len(p.Errors) + if n > 0 && p.Errors[n-1].Pos.Line == filePos.Line { // discard errors reported on the same line return } @@ -2101,137 +2278,155 @@ func (p *Parser) error(pos Pos, msg string) { // too many errors; terminate early panic(bailout{}) } - p.errors.Add(filePos, msg) + p.Errors.Add(filePos, msg) } -func (p *Parser) errorExpected(pos Pos, msg string) { +func (p *Parser) ErrorExpected(pos source.Pos, msg string) { msg = "expected " + msg - if pos == p.token.Pos { + if pos == p.Token.Pos { // error happened at the current position: provide more specific switch { - case p.token.Token == token.Semicolon && p.token.Literal == "\n": + case p.Token.Token == token.Semicolon && p.Token.Literal == "\n": msg += ", found newline" - case p.token.Token.IsLiteral(): - msg += ", found " + p.token.Literal + case p.Token.Token.IsLiteral(): + msg += ", found " + p.Token.Literal default: - msg += ", found '" + p.token.Token.String() + "'" + msg += ", found '" + p.Token.Token.String() + "'" } } - p.error(pos, msg) + p.Error(pos, msg) } -func (p *Parser) consumeComment() (comment *Comment, endline int) { +func (p *Parser) consumeComment() (comment *ast.Comment, endline int) { // /*-style comments may end on a different line than where they start. // Scan the comment for '\n' chars and adjust endline accordingly. - endline = p.file.Line(p.token.Pos) - if p.token.Literal[1] == '*' { + endline = p.File.Line(p.Token.Pos) + if p.Token.Literal[1] == '*' { // don't use range here - no need to decode Unicode code points - for i := 0; i < len(p.token.Literal); i++ { - if p.token.Literal[i] == '\n' { + for i := 0; i < len(p.Token.Literal); i++ { + if p.Token.Literal[i] == '\n' { endline++ } } } - comment = &Comment{Slash: p.token.Pos, Text: p.token.Literal} + comment = &ast.Comment{Slash: p.Token.Pos, Text: p.Token.Literal} p.next0() return } -func (p *Parser) consumeCommentGroup(n int) (comments *CommentGroup) { - var list []*Comment - endline := p.file.Line(p.token.Pos) - for p.token.Token == token.Comment && p.file.Line(p.token.Pos) <= endline+n { - var comment *Comment +func (p *Parser) consumeCommentGroup(n int) (comments *ast.CommentGroup) { + var list []*ast.Comment + endline := p.File.Line(p.Token.Pos) + for p.Token.Token == token.Comment && p.File.Line(p.Token.Pos) <= endline+n { + var comment *ast.Comment comment, endline = p.consumeComment() list = append(list, comment) } - comments = &CommentGroup{List: list} + comments = &ast.CommentGroup{List: list} p.comments = append(p.comments, comments) return } func (p *Parser) next0() { - if p.trace && p.token.Pos.IsValid() { - s := p.token.Token.String() + if p.Trace && p.Token.Pos.IsValid() { + s := p.Token.Token.String() switch { - case p.token.Token.IsLiteral(): - p.printTrace(s, p.token.Literal) - case p.token.Token.IsOperator(), p.token.Token.IsKeyword(): - p.printTrace(`"` + s + `"`) + case p.Token.Token.IsLiteral(): + p.PrintTrace(s, p.Token.Literal) + case p.Token.Token.IsOperator(), p.Token.Token.IsKeyword(): + p.PrintTrace(`"` + s + `"`) default: - p.printTrace(s) + p.PrintTrace(s) } } - p.token = p.scanner.Scan() + if p.ScanFunc != nil { + p.Token = p.ScanFunc() + } else { + p.Token = p.Scanner.Scan() + } } -func (p *Parser) next() { - prev := p.token.Pos - p.prevToken = p.token.Token +func (p *Parser) Next() { + prev := p.Token.Pos + p.PrevToken = p.Token next: p.next0() - switch p.token.Token { - case token.CodeBegin, token.CodeEnd: + switch p.Token.Token { + case token.CodeBegin: + p.InCode = true + if p.IgnoreCodeBlockDisabled { + return + } goto next + case token.CodeEnd: + p.InCode = false + if p.IgnoreCodeBlockDisabled { + return + } + goto next + case token.Text: + if p.Token.Literal == "" { + goto next + } case token.Comment: - if p.file.Line(p.token.Pos) == p.file.Line(prev) { + if p.File.Line(p.Token.Pos) == p.File.Line(prev) { // line comment of prev token _ = p.consumeCommentGroup(0) } // consume successor comments, if any - for p.token.Token == token.Comment { + for p.Token.Token == token.Comment { // lead comment of next token _ = p.consumeCommentGroup(1) } } } -func (p *Parser) skipSpace() { - for p.token.Token == token.Semicolon && p.token.Literal == "\n" { - p.next() +func (p *Parser) SkipSpace() { + for p.Token.Token == token.Semicolon && p.Token.Literal == "\n" { + p.Next() } } -func (p *Parser) printTrace(a ...any) { +func (p *Parser) PrintTrace(a ...any) { const ( dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " n = len(dots) ) - filePos := p.file.Position(p.token.Pos) - _, _ = fmt.Fprintf(p.traceOut, "%5d: %5d:%3d: ", p.token.Pos, filePos.Line, + filePos := p.File.Position(p.Token.Pos) + _, _ = fmt.Fprintf(p.TraceOut, "%5d: %5d:%3d: ", p.Token.Pos, filePos.Line, filePos.Column) i := 2 * p.indent for i > n { - _, _ = fmt.Fprint(p.traceOut, dots) + _, _ = fmt.Fprint(p.TraceOut, dots) i -= n } - _, _ = fmt.Fprint(p.traceOut, dots[0:i]) - _, _ = fmt.Fprintln(p.traceOut, a...) + _, _ = fmt.Fprint(p.TraceOut, dots[0:i]) + _, _ = fmt.Fprintln(p.TraceOut, a...) } -func (p *Parser) safePos(pos Pos) Pos { - fileBase := p.file.Base - fileSize := p.file.Size +func (p *Parser) safePos(pos source.Pos) source.Pos { + fileBase := p.File.Base + fileSize := p.File.Size if int(pos) < fileBase || int(pos) > fileBase+fileSize { - return Pos(fileBase + fileSize) + return source.Pos(fileBase + fileSize) } return pos } func tracep(p *Parser, msg string) *Parser { - p.printTrace(msg, "(") + p.PrintTrace(msg, "(") p.indent++ return p } func untracep(p *Parser) { p.indent-- - p.printTrace(")") + p.PrintTrace(")") } type BlockEnd struct { diff --git a/parser/parser_handler.go b/parser/parser_handler.go new file mode 100644 index 0000000..a91259c --- /dev/null +++ b/parser/parser_handler.go @@ -0,0 +1,8 @@ +package parser + +import ( + "github.com/gad-lang/gad/parser/node" + "github.com/gad-lang/gad/token" +) + +type ParseListHandler func(start token.Token, ends ...BlockWrap) (list []node.Stmt, end *BlockEnd) diff --git a/parser/parser_test.go b/parser/parser_test.go index 2f110d9..102257c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -5,12 +5,14 @@ import ( "flag" "fmt" "io" - "io/ioutil" "os" "reflect" "strings" "testing" + . "github.com/gad-lang/gad/parser/ast" + . "github.com/gad-lang/gad/parser/node" + . "github.com/gad-lang/gad/parser/source" "github.com/shopspring/decimal" "github.com/stretchr/testify/require" @@ -94,7 +96,7 @@ x := d?.a.b.("c")?.e ?? 5 f, err = os.Open(goldenFile) require.NoError(t, err) } - golden, err := ioutil.ReadAll(f) + golden, err := io.ReadAll(f) require.NoError(t, err) var out bytes.Buffer parse(sample, &out) @@ -105,12 +107,15 @@ x := d?.a.b.("c")?.e ?? 5 } func TestParserMixed(t *testing.T) { - expectParseStringMode(t, ParseMixed, `# gad: writer=myfn; #{- var myfn -} a`, `# gad: writer=myfn; var myfn; #{= "a" }`) + expectParseStringMode(t, ParseMixed, "# gad: writer=myfn\n#{- var myfn -} a", `# gad: writer=myfn; var myfn; #{= "a" }`) + expectParseStringMode(t, ParseMixed, "# gad: writer=myfn\na#{var myfn}b", `# gad: writer=myfn; #{= "a" }; var myfn; #{= "b" }`) + expectParseStringMode(t, ParseMixed, "#{var a}", `var a`) expectParseStringMode(t, ParseMixed, "#{for e in list do}1#{end}", `for _, e in list {#{= "1" }}`) expectParseStringMode(t, ParseMixed, "a #{-= 1 -}\n\tb", `#{= "a" }; #{= 1 }; #{= "b" }`) expectParseStringMode(t, ParseMixed, "#{ a := begin -} 2 #{- end }", `a := (#{= "2" })`) expectParseStringMode(t, ParseMixed, "#{ if 1 then } 2 #{ end }", `if 1 {#{= " 2 " }}`) + expectParseMode(t, ParseMixed, "a #{-= 1 -}\n\tb", func(p pfn) []Stmt { return stmts( text(p(1, 1), "a"), @@ -134,7 +139,6 @@ func TestParserMixed(t *testing.T) { text(p(1, 12), "\n\tb"), ) }) - expectParseMode(t, ParseMixed, "a #{-= 1 -}\n\tb", func(p pfn) []Stmt { return stmts( text(p(1, 1), "a"), @@ -142,7 +146,6 @@ func TestParserMixed(t *testing.T) { text(p(1, 13), "b"), ) }) - expectParseMode(t, ParseMixed, `a#{=1}b`, func(p pfn) []Stmt { return stmts( text(p(1, 1), "a"), @@ -204,9 +207,11 @@ func TestParserMixed(t *testing.T) { expectParseStringMode(t, ParseMixed, "a #{- 1}", `#{= "a" }; 1`) expectParseStringMode(t, ParseMixed, "a\n#{- 1}\tb\n#{-= 2 -}\n\nc", "#{= \"a\" }; 1; #{= \"\\tb\" }; #{= 2 }; #{= \"c\" }") expectParseStringMode(t, ParseMixed, `a#{=1}c#{x := 5}#{=x}`, `#{= "a" }; #{= 1 }; #{= "c" }; x := 5; #{= x }`) - expectParseStringMode(t, ParseMixed, "#{if true then}1#{else if a then}2#{else then}3#{end}", `if true {#{= "1" }} else if a {#{= "2" }} else {#{= "3" }}`) + + expectParseStringMode(t, ParseMixed, "#{if true then}1#{else if a then}2#{else then}3#{fn()}#{end}", `if true {#{= "1" }} else if a {#{= "2" }} else {#{= "3" }; fn()}`) + expectParseStringMode(t, ParseMixed, "#{if true then}1#{else if a then}2#{else}3#{end}", `if true {#{= "1" }} else if a {#{= "2" }} else {#{= "3" }}`) expectParseStringMode(t, ParseMixed, "#{if true then}1#{else if a then}2#{end}", `if true {#{= "1" }} else if a {#{= "2" }}`) - expectParseStringMode(t, ParseMixed, "#{if true then}1#{else then}2#{end}", `if true {#{= "1" }} else {#{= "2" }}`) + expectParseStringMode(t, ParseMixed, "#{if true then}1#{else}2#{end}", `if true {#{= "1" }} else {#{= "2" }}`) } func TestParserError(t *testing.T) { @@ -273,8 +278,8 @@ func TestParseDecl(t *testing.T) { genDecl(token.Param, p(1, 1), p(1, 7), p(1, 31), paramSpec(false, ident("a", p(1, 8))), paramSpec(true, ident("b", p(1, 14))), - nparamSpec(ident("c", p(1, 17)), &IntLit{1, 19, "1"}), - nparamSpec(ident("d", p(1, 22)), &IntLit{2, 24, "2"}), + nparamSpec(ident("c", p(1, 17)), intLit(1, p(1, 19))), + nparamSpec(ident("d", p(1, 22)), intLit(2, p(1, 24))), nparamSpec(ident("e", p(1, 30)), nil), ), ), @@ -284,8 +289,8 @@ func TestParseDecl(t *testing.T) { return stmts( declStmt( genDecl(token.Param, p(1, 1), p(1, 7), p(1, 22), - nparamSpec(ident("c", p(1, 8)), &IntLit{1, 10, "1"}), - nparamSpec(ident("d", p(1, 13)), &IntLit{2, 15, "2"}), + nparamSpec(ident("c", p(1, 8)), intLit(1, p(1, 10))), + nparamSpec(ident("d", p(1, 13)), intLit(2, p(1, 15))), nparamSpec(ident("e", p(1, 21)), nil), ), ), @@ -295,8 +300,8 @@ func TestParseDecl(t *testing.T) { return stmts( declStmt( genDecl(token.Param, p(1, 1), p(1, 7), p(1, 23), - nparamSpec(ident("c", p(1, 9)), &IntLit{1, 11, "1"}), - nparamSpec(ident("d", p(1, 14)), &IntLit{2, 16, "2"}), + nparamSpec(ident("c", p(1, 9)), intLit(1, p(1, 11))), + nparamSpec(ident("d", p(1, 14)), intLit(2, p(1, 16))), nparamSpec(ident("e", p(1, 22)), nil), ), ), @@ -1331,7 +1336,7 @@ func TestParseCallWithNamedArgs(t *testing.T) { ident("add", p(1, 1)), p(1, 4), p(1, 14), callExprNamedArgs(nil, - []NamedArgExpr{{String: stringLit("x", p(1, 5))}, {Ident: ident("y", p(1, 11))}}, + []NamedArgExpr{{Lit: stringLit("x", p(1, 5))}, {Ident: ident("y", p(1, 11))}}, []Expr{intLit(2, p(1, 9)), intLit(3, p(1, 13))}, )))) }) @@ -1485,8 +1490,58 @@ func TestParseForIn(t *testing.T) { p(1, 1))) }) + expectParse(t, "for x in y {} else {}", func(p pfn) []Stmt { + return stmts( + forInStmt( + ident("_", p(1, 5)), + ident("x", p(1, 5)), + ident("y", p(1, 10)), + blockStmt(p(1, 12), p(1, 13)), + p(1, 1), + blockStmt(p(1, 20), p(1, 21)))) + }) + + expectParse(t, "for x in y do x; else 1 end", func(p pfn) []Stmt { + return stmts( + forInStmt( + ident("_", p(1, 5)), + ident("x", p(1, 5)), + ident("y", p(1, 10)), + blockStmt( + p(1, 12), p(1, 18), + exprStmt( + ident("x", p(1, 15)), + ), + ), + p(1, 1), + blockStmt(p(1, 18), p(1, 23), + exprStmt( + intLit(1, p(1, 23)), + ), + ))) + }) + + expectParse(t, "for x in y {} else 1 end", func(p pfn) []Stmt { + return stmts( + forInStmt( + ident("_", p(1, 5)), + ident("x", p(1, 5)), + ident("y", p(1, 10)), + blockStmt(p(1, 12), p(1, 13)), + p(1, 1), + blockStmt(p(1, 15), p(1, 20), + exprStmt( + intLit(1, p(1, 20)), + ), + ))) + }) + expectParseString(t, "for x in y do end", "for _, x in y {}") expectParseString(t, "for x in y do 1 end", "for _, x in y {1}") + expectParseString(t, "for x in y do else end", "for _, x in y {} else {}") + expectParseString(t, "for x in y do 1 else end", "for _, x in y {1} else {}") + expectParseString(t, "for x in y do else end", "for _, x in y {} else {}") + expectParseString(t, "for x in y do 1 else 2 end", "for _, x in y {1} else {2}") expectParseError(t, `for 1 in a {}`) expectParseError(t, `for "" in a {}`) @@ -1661,32 +1716,27 @@ func TestParseClosure(t *testing.T) { ident("c", p(1, 13)), ident("d", p(1, 16))), ), - &BlockExpr{blockStmt(p(1, 22), p(1, 24), - exprStmt(ident("d", p(1, 23))))})), + blockExpr(p(1, 22), p(1, 24), + exprStmt(ident("d", p(1, 23)))))), token.Assign, p(1, 3))) }) } func TestParseFunction(t *testing.T) { - expectParse(t, "a = func(b, c, d) { return d }", func(p pfn) []Stmt { + expectParse(t, "func fn (b) { return d }", func(p pfn) []Stmt { return stmts( - assignStmt( - exprs( - ident("a", p(1, 1))), - exprs( - funcLit( - funcType(p(1, 5), p(1, 9), p(1, 17), - funcArgs(nil, - ident("b", p(1, 10)), - ident("c", p(1, 13)), - ident("d", p(1, 16))), - ), - blockStmt(p(1, 19), p(1, 30), - returnStmt(p(1, 21), ident("d", p(1, 28)))))), - token.Assign, - p(1, 3))) + exprStmt( + funcLit( + funcType(p(1, 5), p(1, 9), p(1, 11), + ident("fn", p(1, 6)), + funcArgs(nil, + ident("b", p(1, 10))), + ), + blockStmt(p(1, 13), p(1, 24), + returnStmt(p(1, 15), ident("d", p(1, 22))))))) }) + expectParse(t, "a = func(b, c, d; e=1, f=2, ...g) { return d }", func(p pfn) []Stmt { return stmts( assignStmt( @@ -2065,8 +2115,8 @@ if a == 5 { expectParseString(t, "if a then end", "if a {}") expectParseString(t, "if a then b end", "if a {b}") expectParseString(t, "if true; a then b end", "if true; a {b}") - expectParseString(t, "if a then b; else then c end", "if a {b} else {c}") - expectParseString(t, "if a then b; else if 1 then 2; else then c end", "if a {b} else if 1 {2} else {c}") + expectParseString(t, "if a then b; else c end", "if a {b} else {c}") + expectParseString(t, "if a then b; else if 1 then 2; else c end", "if a {b} else if 1 {2} else {c}") expectParseError(t, `if {}`) expectParseError(t, `if a == b { } else a != b { }`) @@ -2673,37 +2723,35 @@ func TestParseString(t *testing.T) { func TestParseConfig(t *testing.T) { expectParse(t, `# gad: mixed -y -#{b}`, func(p pfn) []Stmt { +# gad: mixed=false +a`, func(p pfn) []Stmt { return stmts( - config(p(1, 1), p(1, 12), ConfigOptions{Mixed: true}), - text(p(2, 1), "y\n"), - exprStmt(ident("b", p(3, 3))), + config(p(1, 1), kv(ident("mixed", p(1, 8)))), + config(p(2, 1), kv(ident("mixed", p(2, 8)), boolLit(false, p(2, 14)))), + exprStmt(ident("a", p(3, 1))), ) }) expectParse(t, `# gad: mixed -# gad: mixed=false -a`, func(p pfn) []Stmt { +y +#{b}`, func(p pfn) []Stmt { return stmts( - config(p(1, 1), p(1, 12), ConfigOptions{Mixed: true}), - config(p(2, 1), p(2, 18), ConfigOptions{NoMixed: true}), - exprStmt(ident("a", p(3, 1))), + config(p(1, 1), kv(ident("mixed", p(1, 8)))), + text(p(2, 1), "y\n"), + exprStmt(ident("b", p(3, 3))), ) }) - expectParse(t, `# gad: mixed a #{b}`, func(p pfn) []Stmt { return stmts( - config(p(1, 1), p(1, 12), ConfigOptions{Mixed: true}), + config(p(1, 1), kv(ident("mixed", p(1, 8)))), text(p(2, 1), "a\n"), exprStmt(ident("b", p(3, 3))), ) }) - expectParse(t, `# gad: mixed`, func(p pfn) []Stmt { return stmts( - config(p(1, 1), p(1, 12), ConfigOptions{Mixed: true})) + config(p(1, 1), kv(ident("mixed", p(1, 8))))) }) } @@ -2832,7 +2880,7 @@ func expectParseMode(t *testing.T, mode Mode, input string, fn expectedFn) { var ok bool defer func() { if !ok { - // print trace + // print Trace tr := &parseTracer{} p := NewParser(testFile, []byte(input), tr) actual, _ := p.ParseFile() @@ -2866,7 +2914,7 @@ func expectParseError(t *testing.T, input string) { var ok bool defer func() { if !ok { - // print trace + // print Trace tr := &parseTracer{} p := NewParser(testFile, []byte(input), tr) _, _ = p.ParseFile() @@ -2888,7 +2936,7 @@ func expectParseStringMode(t *testing.T, mode Mode, input, expected string) { var ok bool defer func() { if !ok { - // print trace + // print Trace tr := &parseTracer{} _, _ = parseSource("test", []byte(input), tr, mode) t.Logf("Trace:\n%s", strings.Join(tr.out, "")) @@ -2977,10 +3025,14 @@ func forInStmt( seq Expr, body *BlockStmt, pos Pos, + elseb ...*BlockStmt, ) *ForInStmt { - return &ForInStmt{ + f := &ForInStmt{ Key: key, Value: value, Iterable: seq, Body: body, ForPos: pos, } + for _, f.Else = range elseb { + } + return f } func breakStmt(pos Pos) *BranchStmt { @@ -3056,6 +3108,8 @@ func funcType(pos, lparen, rparen Pos, v ...any) *FuncType { f.Params.Args = t case NamedArgsList: f.Params.NamedArgs = t + case *Ident: + f.Ident = t } } return f @@ -3073,6 +3127,10 @@ func blockStmt(lbrace, rbrace Pos, list ...Stmt) *BlockStmt { return &BlockStmt{Stmts: list, LBrace: lbrace, RBrace: rbrace} } +func blockExpr(lbrace, rbrace Pos, list ...Stmt) *BlockExpr { + return &BlockExpr{BlockStmt: &BlockStmt{Stmts: list, LBrace: lbrace, RBrace: rbrace}} +} + func ident(name string, pos Pos) *Ident { return &Ident{Name: name, NamePos: pos} } @@ -3082,22 +3140,32 @@ func text(pos Pos, lit string) *TextStmt { } func toText(start, end Literal, expr Expr) *ExprToTextStmt { - return &ExprToTextStmt{expr, start, end} + return &ExprToTextStmt{Expr: expr, StartLit: start, EndLit: end} } func lit(value string, pos Pos) Literal { - return Literal{value, pos} + return Literal{Value: value, Pos: pos} +} + +func kv(key Expr, value ...Expr) *KeyValueLit { + kv := &KeyValueLit{Key: key} + for _, expr := range value { + kv.Value = expr + } + return kv } -func config(start, end Pos, opts ConfigOptions) *ConfigStmt { - return &ConfigStmt{ConfigPos: start, EndPos: end, Options: opts} +func config(start Pos, opts ...*KeyValueLit) *ConfigStmt { + c := &ConfigStmt{ConfigPos: start, Elements: opts} + c.ParseElements() + return c } func nullishSelector( sel, expr Expr, ) *NullishSelectorExpr { - return &NullishSelectorExpr{sel, expr} + return &NullishSelectorExpr{Expr: sel, Sel: expr} } func binaryExpr( @@ -3164,15 +3232,15 @@ func arrayLit(lbracket, rbracket Pos, list ...Expr) *ArrayLit { } func caleeKw(pos Pos) *CalleeKeyword { - return &CalleeKeyword{pos, token.Callee.String()} + return &CalleeKeyword{TokenPos: pos, Literal: token.Callee.String()} } func argsKw(pos Pos) *ArgsKeyword { - return &ArgsKeyword{pos, token.Args.String()} + return &ArgsKeyword{TokenPos: pos, Literal: token.Args.String()} } func nargsKw(pos Pos) *NamedArgsKeyword { - return &NamedArgsKeyword{pos, token.NamedArgs.String()} + return &NamedArgsKeyword{TokenPos: pos, Literal: token.NamedArgs.String()} } func mapElementLit( @@ -3210,7 +3278,7 @@ func callExpr( lparen, rparen Pos, args ...any, ) (ce *CallExpr) { - ce = &CallExpr{Func: f, LParen: lparen, RParen: rparen} + ce = &CallExpr{Func: f, CallArgs: CallArgs{LParen: lparen, RParen: rparen}} for _, v := range args { switch t := v.(type) { case CallExprArgs: @@ -3223,7 +3291,7 @@ func callExpr( } func ellipsis(pos Pos, value Expr) *EllipsisValue { - return &EllipsisValue{pos, value} + return &EllipsisValue{Pos: pos, Value: value} } func callExprArgs( @@ -3321,7 +3389,7 @@ func equalStmt(t *testing.T, expected, actual Stmt) { } require.Equal(t, expectedSpec.Ident, actualSpec.Ident) if expectedSpec.Value != nil || actualSpec.Value != nil { - require.Equal(t, expectedSpec.Value, actualSpec.Value) + equalExpr(t, expectedSpec.Value, actualSpec.Value) } case *ValueSpec: actualSpec, ok := actSpec.(*ValueSpec) @@ -3385,6 +3453,8 @@ func equalStmt(t *testing.T, expected, actual Stmt) { actual.(*ForInStmt).Body) require.Equal(t, expected.ForPos, actual.(*ForInStmt).ForPos) + equalStmt(t, expected.Else, + actual.(*ForInStmt).Else) case *ReturnStmt: equalExpr(t, expected.Result, actual.(*ReturnStmt).Result) @@ -3416,10 +3486,13 @@ func equalStmt(t *testing.T, expected, actual Stmt) { case *ConfigStmt: require.Equal(t, expected.ConfigPos, actual.(*ConfigStmt).ConfigPos) - require.Equal(t, expected.EndPos, - actual.(*ConfigStmt).EndPos) require.Equal(t, expected.Options, actual.(*ConfigStmt).Options) + require.Equal(t, len(expected.Elements), + len(actual.(*ConfigStmt).Elements)) + for i, e := range expected.Elements { + equalExpr(t, e, actual.(*ConfigStmt).Elements[i]) + } default: panic(fmt.Errorf("unknown type: %T", expected)) } @@ -3629,6 +3702,11 @@ func equalExpr(t *testing.T, expected, actual Expr) { case *BlockExpr: equalStmt(t, expected.BlockStmt, actual.(*BlockExpr).BlockStmt) + case *KeyValueLit: + equalExpr(t, expected.Key, + actual.(*KeyValueLit).Key) + equalExpr(t, expected.Value, + actual.(*KeyValueLit).Value) default: panic(fmt.Errorf("unknown type: %T", expected)) } @@ -3657,7 +3735,7 @@ func equalNamedArgsNames(t *testing.T, expected, actual []NamedArgExpr) { require.Equal(t, len(expected), len(actual)) for i := 0; i < len(expected); i++ { equalExpr(t, expected[i].Ident, actual[i].Ident) - equalExpr(t, expected[i].String, actual[i].String) + equalExpr(t, expected[i].Lit, actual[i].Lit) } } diff --git a/parser/scanner.go b/parser/scanner.go index 6f3b21a..0cfc7c0 100644 --- a/parser/scanner.go +++ b/parser/scanner.go @@ -16,14 +16,18 @@ package parser import ( "fmt" + "reflect" + "strconv" "unicode" "unicode/utf8" + "github.com/gad-lang/gad/parser/source" + "github.com/gad-lang/gad/parser/utils" "github.com/gad-lang/gad/token" ) -// byte order mark -const bom = 0xFEFF +// BOM byte order mark +const BOM = 0xFEFF // ScanMode represents a scanner mode. type ScanMode uint8 @@ -67,172 +71,366 @@ const ( // ScannerErrorHandler is an error handler for the scanner. type ScannerErrorHandler func(pos SourceFilePos, msg string) -type Token struct { - Pos Pos - Token token.Token - Literal string - InsertSemi bool - Data any +type ScannerInterface interface { + Scan() (t Token) + Mode() ScanMode + SetMode(m ScanMode) + SourceFile() *SourceFile + Source() []byte + ErrorHandler(h ...ScannerErrorHandler) } -var _ fmt.Stringer = Token{} +type TokenPool []*Token -func (t Token) String() string { - return t.Token.String() + ": " + t.Literal +func (p *TokenPool) Shift() (t *Token) { + t = (*p)[0] + *p = (*p)[1:] + return +} + +func (p TokenPool) Last() (t *Token) { + return p[len(p)-1] +} + +func (p TokenPool) Empty() bool { + return len(p) == 0 +} + +func (p *TokenPool) Add(t ...*Token) { + *p = append(*p, t...) +} + +func (p *TokenPool) Semi() { + *p = append(*p, &Token{Token: token.Semicolon, Literal: ";"}) +} + +type NextHandlers struct { + LineEndHandlers []func() + PostLineEndHandlers []func() + EOFHandlers []func(s *Scanner) +} + +func (h *NextHandlers) LineEndHandler(f func()) { + h.LineEndHandlers = append(h.LineEndHandlers, f) +} + +func (h *NextHandlers) CallLineEndHandlers() { + for _, handler := range h.LineEndHandlers { + handler() + } + h.LineEndHandlers = nil +} + +func (h *NextHandlers) PostLineEndHandler(f func()) { + h.PostLineEndHandlers = append(h.PostLineEndHandlers, f) +} + +func (h *NextHandlers) CallPostLineEndHandlers() { + for _, handler := range h.PostLineEndHandlers { + handler() + } + h.PostLineEndHandlers = nil +} + +func (h *NextHandlers) EOFHandler(f func(*Scanner)) { + h.EOFHandlers = append(h.EOFHandlers, f) +} + +func (h *NextHandlers) CallEOFHandlers(s *Scanner) { + handlers := h.EOFHandlers + h.EOFHandlers = nil + + for _, handler := range handlers { + handler(s) + } +} + +func (s *Handlers) TokenHandler(f func(t *Token)) { + s.TokenHandlers = append(s.TokenHandlers, f) +} + +func (s *Handlers) CallTokenHandlers(t *Token) { + if t.handled { + return + } + t.handled = true + for _, handler := range s.TokenHandlers { + handler(t) + } +} + +type TokenHandler func(t *Token) + +type TokenHandlers []TokenHandler + +func (th *TokenHandlers) Remove(h TokenHandler) { + addr := reflect.ValueOf(h).Pointer() + for i, handler := range *th { + if reflect.ValueOf(handler).Pointer() == addr { + defer func() { + *th = append((*th)[:i], (*th)[i+1:]...) + }() + break + } + } +} + +type Handlers struct { + NextHandlers + ScanHandler func(ch rune) (t Token, insertSemi, ok bool) + TokenHandlers TokenHandlers } // Scanner reads the Gad source text. It's based on ToInterface's scanner // implementation. type Scanner struct { - file *SourceFile // source file handle - src []byte // source - ch rune // current character - offset int // character offset - readOffset int // reading offset (position after current character) - lineOffset int // current line offset - insertSemi bool // insert a semicolon before next newline - errorHandler ScannerErrorHandler // error reporting; or nil - errorCount int // number of errors encountered - mode ScanMode - inCode bool - toText bool - braceCount int - tokenPool []Token - textTrimLeft bool + Handlers + + File *SourceFile // source file handle + Src []byte // source + Ch rune // current character + Offset int // character offset + ReadOffset int // reading offset (position after current character) + lineOffset int // current line offset + InsertSemi bool // insert a semicolon before next newline + errorHandler []ScannerErrorHandler // error reporting; or nil + errorCount int // number of errors encountered + mode ScanMode + InCode bool + InCodeBrace int + ToText bool + BraceCount int + BreacksCount int + ParenCount int + TokenPool TokenPool + TextTrimLeft bool + SkipWhitespaceFunc func(s *Scanner) + NewLineEscape func() bool + NewLineEscaped bool + HandleMixed func(textStart *int, rt func() *Token) + EOF *Token } // NewScanner creates a Scanner. func NewScanner( file *SourceFile, src []byte, - errorHandler ScannerErrorHandler, mode ScanMode, ) *Scanner { if file.Size != len(src) { - panic(fmt.Sprintf("file size (%d) does not match src len (%d)", + panic(fmt.Sprintf("file size (%d) does not match Src len (%d)", file.Size, len(src))) } s := &Scanner{ - file: file, - src: src, - errorHandler: errorHandler, - ch: ' ', - mode: mode, + File: file, + Src: src, + Ch: ' ', + mode: mode, + } + + s.SkipWhitespaceFunc = func(s *Scanner) { + for s.Ch == ' ' || s.Ch == '\t' || s.Ch == '\n' && !s.InsertSemi { + s.Next() + } } - s.next() - if s.ch == bom { - s.next() // ignore BOM at file beginning + s.Next() + if s.Ch == BOM { + s.Next() // ignore BOM at file beginning } return s } +func (s *Scanner) SkipWhitespace() { + s.SkipWhitespaceFunc(s) +} + +func (s *Scanner) List() (ret []Token) { + var t Token + for { + t = s.Scan() + if t.Token == token.EOF { + return + } + ret = append(ret, t) + } +} + +func (s *Scanner) ErrorHandler(h ...ScannerErrorHandler) { + s.errorHandler = append(s.errorHandler, h...) +} + +func (s *Scanner) SourceFile() *SourceFile { + return s.File +} + +func (s *Scanner) Source() []byte { + return s.Src +} + // ErrorCount returns the number of errors. func (s *Scanner) ErrorCount() int { return s.errorCount } -func (s *Scanner) AddNextToken(n ...Token) { - s.tokenPool = append(s.tokenPool, n...) +func (s *Scanner) AddNextToken(n ...Token) (r *Token) { + for _, t := range n { + t2 := t + r = s.AddNextTokenPtr(&t2) + } + return +} + +func (s *Scanner) AddNextTokenPtr(n ...*Token) (r *Token) { + var newN []*Token + for _, t := range n { + if t.Prev != nil { + for _, p := range t.Prev { + p2 := p + newN = append(newN, &p2) + } + } + t.Prev = nil + newN = append(newN, t) + } + n = newN + for i := range n { + if n[i].Token == token.EOF { + if l := len(s.TokenPool); l > 0 { + if s.TokenPool[l-1].Token == token.EOF { + if i == 0 { + r = n[i] + } + n = n[:i] + if len(n) == 0 { + return + } + break + } + } + } + s.CallTokenHandlers(n[i]) + } + s.TokenPool.Add(n...) + return n[len(n)-1] +} + +func (s *Scanner) Mode() ScanMode { + return s.mode +} + +func (s *Scanner) ModeP() *ScanMode { + return &s.mode +} + +func (s *Scanner) SetMode(m ScanMode) { + s.mode = m } -// Scan returns a token, token literal and its position. func (s *Scanner) Scan() (t Token) { - if len(s.tokenPool) > 0 { - t = s.tokenPool[0] - s.tokenPool = s.tokenPool[1:] - return + if !s.TokenPool.Empty() { + return *s.TokenPool.Shift() + } else if s.EOF != nil { + return *s.EOF + } + + t = s.ScanNow() + if t.Token == token.EOF { + s.EOF = &t + s.CallEOFHandlers(s) + return t + } + s.AddNextToken(t) + return *s.TokenPool.Shift() +} + +func (s Scanner) PeekScan() (t Token) { + return s.ScanNow() +} + +// ScanNow returns a token, token literal and its position. +func (s *Scanner) ScanNow() (t Token) { + t.Pos = s.File.FileSetPos(s.Offset) + + if s.Ch == -1 { + if s.InsertSemi { + s.InsertSemi = false // EOF consumed + t.Literal = "\n" + t.Token = token.Semicolon + return t + } + return Token{Token: token.EOF, Pos: t.Pos} } - t.Pos = s.file.FileSetPos(s.offset) + if s.mode.Has(Mixed) && !s.InCode && s.Ch != -1 { + start := s.Offset + readText := func() { + t.Token = token.Text + t.Pos = s.File.FileSetPos(start) - if s.mode.Has(Mixed) && !s.inCode && s.ch != -1 { - start := s.offset + if s.Offset > start { + t.Literal = string(s.Src[start:s.Offset]) + if s.TextTrimLeft { + t.Literal = TrimSpace(true, false, t.Literal) + } + } + s.TextTrimLeft = false + } for { var scape bool - switch s.ch { + switch s.Ch { case '\\': if scape { scape = false } case -1: - t.Token = token.Text - t.Literal = string(s.src[start:s.offset]) - if s.textTrimLeft { - t.Literal = trimSpace(true, false, t.Literal) - } - s.textTrimLeft = false - s.tokenPool = append(s.tokenPool, Token{Pos: s.file.FileSetPos(s.offset), Token: token.EOF}) - return + readText() + return t case '#': if !scape { - if s.peek() == '{' { - var ( - end = s.offset - lit = "#{" - ) - s.inCode = true - s.next() - s.next() - s.braceCount++ - - t.Literal = string(s.src[start:end]) - t.Token = token.Text - - switch s.ch { - case '-': - s.nextNoSpace() - t.Literal = trimSpace(s.textTrimLeft, true, t.Literal) - } - - s.textTrimLeft = false - - if s.ch == '=' { - s.toText = true - s.nextNoSpace() - next := Token{Token: token.ToTextBegin, Pos: s.file.FileSetPos(end), Literal: lit + "="} - if t.Literal == "" { - t = next - } else { - s.AddNextToken(next) - } - } else { - next := Token{Token: token.CodeBegin, Pos: s.file.FileSetPos(end), Literal: lit} - if t.Literal == "" { - t = next - } else { - s.AddNextToken(next) - } - } - return + if s.Peek() == '{' { + readText() + return s.ScanCodeBlock(&t) } - if !s.mode.Has(ConfigDisabled) && string(s.peekNoSpaceN(4)) == "gad:" { - t, ok := s.scanConfig() - if ok { - return t + if !s.mode.Has(ConfigDisabled) { + // at line start + if s.Offset == 0 || s.Src[s.Offset-1] == '\n' { + if l := s.PeekNoSingleSpaceEq("gad:", 0); l > 0 { + return s.scanConfig(t.Pos, l+1) + } } } goto do } } - s.next() + if s.HandleMixed != nil { + s.HandleMixed(&start, func() *Token { + readText() + return &t + }) + if s.Ch == -1 { + s.Ch = -1 + return + } + } + s.Next() } } do: - s.skipWhitespace() - t.Pos = s.file.FileSetPos(s.offset) + s.SkipWhitespace() + t.Pos = s.File.FileSetPos(s.Offset) insertSemi := false // determine token value - switch ch := s.ch; { - case isLetter(ch): - t.Literal = s.scanIdentifier() + switch ch := s.Ch; { + case utils.IsLetter(ch): + t.Literal = s.ScanIdentifier() t.Token = token.Lookup(t.Literal) switch t.Token { case token.Ident, token.Break, token.Continue, token.Return, @@ -243,65 +441,66 @@ do: } case '0' <= ch && ch <= '9': insertSemi = true - t.Token, t.Literal = s.scanNumber(false) + t.Token, t.Literal = s.ScanNumber(false) default: - s.next() // always make progress + s.Next() // always make progress switch ch { case -1: // EOF - if s.insertSemi { - s.insertSemi = false // EOF consumed - t.Data = "\n" + if s.InsertSemi { + s.InsertSemi = false // EOF consumed + t.Literal = "\n" t.Token = token.Semicolon return } t.Token = token.EOF + s.CallEOFHandlers(s) case '\n': - // we only reach here if s.insertSemi was set in the first place - s.insertSemi = false // newline consumed + // we only reach here if s.InsertSemi was set in the first place + s.InsertSemi = false // newline consumed t.Literal = "\n" t.Token = token.Semicolon return case '"': insertSemi = true t.Token = token.String - t.Literal = s.scanString() + t.Literal = s.ScanString() case '\'': insertSemi = true t.Token = token.Char - t.Literal = s.scanRune() + t.Literal = s.ScanRune() case '`': insertSemi = true t.Token = token.String - t.Literal = s.scanRawString() + t.Literal = s.ScanRawString() case ':': - t.Token = s.switch2(token.Colon, token.Define) + t.Token = s.Switch2(token.Colon, token.Define) case '.': - if '0' <= s.ch && s.ch <= '9' { + if '0' <= s.Ch && s.Ch <= '9' { insertSemi = true - t.Token, t.Literal = s.scanNumber(true) + t.Token, t.Literal = s.ScanNumber(true) } else { t.Token = token.Period - if s.ch == '.' && s.peek() == '.' { - s.next() - s.next() // consume last '.' + if s.Ch == '.' && s.Peek() == '.' { + s.Next() + s.Next() // consume last '.' t.Token = token.Ellipsis } } case ',': t.Token = token.Comma case '?': - switch s.ch { + switch s.Ch { case '.': - s.next() + s.Next() t.Token = token.NullishSelector case '?': - if s.peek() == '=' { - s.next() - s.next() + if s.Peek() == '=' { + s.Next() + s.Next() t.Token = token.NullichAssign } else { - s.next() + s.Next() t.Token = token.NullichCoalesce } default: @@ -312,36 +511,44 @@ do: t.Literal = ";" case '(': t.Token = token.LParen + s.ParenCount++ case ')': insertSemi = true t.Token = token.RParen + s.ParenCount-- case '[': t.Token = token.LBrack + s.BreacksCount++ case ']': insertSemi = true t.Token = token.RBrack + s.BreacksCount-- case '{': t.Token = token.LBrace - if s.inCode { - s.braceCount++ - } + s.BraceCount++ case '}': insertSemi = true t.Token = token.RBrace - if s.inCode { - s.braceCount-- - if s.braceCount == 0 { - s.inCode = false + s.BraceCount-- + if s.InCode { + if s.InCodeBrace == s.BraceCount { + s.InCodeBrace = 0 + s.InCode = false insertSemi = false t.Token = token.Semicolon t.Literal = "\n" - if s.toText { + if s.ToText { t.Token = token.ToTextEnd - s.toText = false + s.ToText = false t.Literal = "}" - t.Data = s.textTrimLeft + if s.TextTrimLeft { + t.Set("trim_left_space", s.TextTrimLeft) + } } else { - next := Token{Token: token.CodeEnd, Literal: "}", Pos: t.Pos, Data: s.textTrimLeft} + next := Token{Token: token.CodeEnd, Literal: "}", Pos: t.Pos} + if s.TextTrimLeft { + next.Set("trim_left_space", s.TextTrimLeft) + } if !s.mode.Has(DontInsertSemis) { s.AddNextToken(t) t = next @@ -352,160 +559,298 @@ do: } } case '+': - t.Token = s.switch3(token.Add, token.AddAssign, '+', token.Inc) + t.Token = s.Switch3(token.Add, token.AddAssign, '+', token.Inc) if t.Token == token.Inc { insertSemi = true } case '-': - if s.ch == '}' { - s.textTrimLeft = true + if s.Ch == '}' { + s.TextTrimLeft = true goto do } - t.Token = s.switch3(token.Sub, token.SubAssign, '-', token.Dec) + t.Token = s.Switch3(token.Sub, token.SubAssign, '-', token.Dec) if t.Token == token.Dec { insertSemi = true } case '*': - t.Token = s.switch2(token.Mul, token.MulAssign) + t.Token = s.Switch2(token.Mul, token.MulAssign) case '/': - if s.ch == '/' || s.ch == '*' { + if s.Ch == '/' || s.Ch == '*' { // comment - if s.insertSemi && s.findLineEnd() { + if s.InsertSemi && s.FindLineEnd() { // reset position to the beginning of the comment - s.ch = '/' - s.offset = s.file.Offset(t.Pos) - s.readOffset = s.offset + 1 - s.insertSemi = false // newline consumed - t.Data = "\n" + s.Ch = '/' + s.Offset = s.File.Offset(t.Pos) + s.ReadOffset = s.Offset + 1 + s.InsertSemi = false // newline consumed + t.Literal = "\n" t.Token = token.Semicolon return } - comment := s.scanComment() + comment := s.ScanComment() if !s.mode.Has(ScanComments) { // skip comment - s.insertSemi = false // newline consumed + s.InsertSemi = false // newline consumed return s.Scan() } t.Token = token.Comment t.Literal = comment } else { - t.Token = s.switch2(token.Quo, token.QuoAssign) + t.Token = s.Switch2(token.Quo, token.QuoAssign) } case '%': - t.Token = s.switch2(token.Rem, token.RemAssign) + t.Token = s.Switch2(token.Rem, token.RemAssign) case '^': - t.Token = s.switch2(token.Xor, token.XorAssign) + t.Token = s.Switch2(token.Xor, token.XorAssign) case '<': - t.Token = s.switch4(token.Less, token.LessEq, '<', + t.Token = s.Switch4(token.Less, token.LessEq, '<', token.Shl, token.ShlAssign) case '>': - t.Token = s.switch4(token.Greater, token.GreaterEq, '>', + t.Token = s.Switch4(token.Greater, token.GreaterEq, '>', token.Shr, token.ShrAssign) case '=': - t.Token = s.switch2(token.Assign, token.Equal) + t.Token = s.Switch2(token.Assign, token.Equal) case '!': - t.Token = s.switch2(token.Not, token.NotEqual) + t.Token = s.Switch2(token.Not, token.NotEqual) case '&': - if s.ch == '^' { - s.next() - t.Token = s.switch2(token.AndNot, token.AndNotAssign) + if s.Ch == '^' { + s.Next() + t.Token = s.Switch2(token.AndNot, token.AndNotAssign) } else { - t.Token = s.switch3(token.And, token.AndAssign, '&', token.LAnd) + t.Token = s.Switch3(token.And, token.AndAssign, '&', token.LAnd) } case '|': - if s.ch == '=' { - s.next() + if s.Ch == '=' { + s.Next() t.Token = token.OrAssign - } else if s.ch == '|' { - if s.peek() == '=' { - s.next() - s.next() + } else if s.Ch == '|' { + if s.Peek() == '=' { + s.Next() + s.Next() t.Token = token.LOrAssign } else { - s.next() + s.Next() t.Token = token.LOr } } else { t.Token = token.Or } case '#': - if !s.mode.Has(ConfigDisabled) && string(s.peekNoSpaceN(4)) == "gad:" { - t, ok := s.scanConfig() - if ok { - return t + if !s.mode.Has(ConfigDisabled) { + // at line start + if s.Offset == 1 || s.Src[s.Offset-2] == '\n' { + if l := s.PeekNoSingleSpaceEq("gad:", 0); l > 0 { + return s.scanConfig(t.Pos, l+1) + } } } fallthrough default: // next reports unexpected BOMs - don't repeat - if ch != bom { - s.error(s.file.Offset(t.Pos), + if ch != BOM { + if s.ScanHandler != nil { + var ok bool + if t, insertSemi, ok = s.ScanHandler(ch); ok { + goto done + } + } + s.Error(s.File.Offset(t.Pos), fmt.Sprintf("illegal character %#U", ch)) } - insertSemi = s.insertSemi // preserve insertSemi info + insertSemi = s.InsertSemi // preserve InsertSemi info t.Token = token.Illegal t.Literal = string(ch) } } +done: if !s.mode.Has(DontInsertSemis) { - s.insertSemi = insertSemi + s.InsertSemi = insertSemi } return } -func (s *Scanner) next() { - if s.readOffset < len(s.src) { - s.offset = s.readOffset - if s.ch == '\n' { - s.lineOffset = s.offset - s.file.AddLine(s.offset) +func (s *Scanner) NextC(count int) { + for i := 0; i < count; i++ { + s.Next() + } +} + +func (s *Scanner) Skip(str string) { + for _, r := range str { + if s.Ch != r { + break + } + s.Next() + } +} + +func (s *Scanner) NextTo(v string) { + for _, r := range v { + s.Expect(r, "next to: "+strconv.Quote(v)) + s.Next() + } +} + +func (s *Scanner) Next() { + var newLineEscape bool +next: + if s.ReadOffset < len(s.Src) { + s.Offset = s.ReadOffset + + if s.Ch == '\n' { + s.lineOffset = s.Offset + s.File.AddLine(s.Offset) + if s.NewLineEscaped { + s.NewLineEscaped = false + } else { + s.CallLineEndHandlers() + } + + defer s.CallPostLineEndHandlers() } - r, w := rune(s.src[s.readOffset]), 1 + + r, w := rune(s.Src[s.ReadOffset]), 1 switch { case r == 0: - s.error(s.offset, "illegal character NUL") + s.Error(s.Offset, "illegal character NUL") case r >= utf8.RuneSelf: // not ASCII - r, w = utf8.DecodeRune(s.src[s.readOffset:]) + r, w = utf8.DecodeRune(s.Src[s.ReadOffset:]) if r == utf8.RuneError && w == 1 { - s.error(s.offset, "illegal UTF-8 encoding") - } else if r == bom && s.offset > 0 { - s.error(s.offset, "illegal byte order mark") + s.Error(s.Offset, "illegal UTF-8 encoding") + } else if r == BOM && s.Offset > 0 { + s.Error(s.Offset, "illegal byte order mark") + } + } + s.ReadOffset += w + s.Ch = r + + if s.Ch == '\\' && s.Peek() == '\n' { + newLineEscape = s.NewLineEscape != nil && s.NewLineEscape() + if newLineEscape { + goto next + } + } else if s.Ch == '\n' { + s.NewLineEscaped = newLineEscape + if newLineEscape { + s.SkipWhitespace() } } - s.readOffset += w - s.ch = r } else { - s.offset = len(s.src) - if s.ch == '\n' { - s.lineOffset = s.offset - s.file.AddLine(s.offset) + s.Offset = len(s.Src) + if s.Ch == '\n' { + s.lineOffset = s.Offset + s.File.AddLine(s.Offset) + s.CallLineEndHandlers() } - s.ch = -1 // eof + s.Ch = -1 // EOF } } -func (s *Scanner) nextNoSpace() { - s.next() - s.skipWhitespace() +func (s *Scanner) PeekAtEndLine() (start, end int) { + start = s.Offset + end = s.Offset + for end < len(s.Src) { + switch s.Src[end] { + case '\n': + if s.Src[end-1] != '\\' { + return + } + } + end++ + } + return } -func (s *Scanner) peek() byte { - if s.readOffset < len(s.src) { - return s.src[s.readOffset] +func (s *Scanner) NextNoSpace() { + s.Next() + s.SkipWhitespace() +} + +func (s *Scanner) Peek() byte { + if s.ReadOffset < len(s.Src) { + return s.Src[s.ReadOffset] } return 0 } -func (s *Scanner) peekNoSpaceN(n int) []byte { - off := s.readOffset - for off < len(s.src) { - switch s.src[off] { - case ' ', '\r', '\n', '\t': +func (s *Scanner) NextPosOf(b byte) (end int) { + end = s.Offset + 1 + + var escape bool + for end < len(s.Src) { + switch s.Src[end] { + case '\\': + escape = !escape + case b: + if !escape { + return + } + } + end++ + } + return end +} + +func (s *Scanner) ReadAt(b rune) []byte { + var ( + start = s.Offset + end = s.Offset + ) + + var escape bool + for end < len(s.Src) { + if s.Ch == -1 { + return nil + } + + if s.Ch == '\\' { + escape = !escape + } + if s.Ch == b && !escape { + s.Next() + break + } + s.Next() + } + return s.Src[start:end] +} + +func (s *Scanner) PeekNoSpace() byte { + offs := s.ReadOffset + for offs < len(s.Src) { + switch s.Src[offs] { + case ' ', '\n', '\t': + offs++ + default: + return s.Src[offs] + } + } + return 0 +} + +func (s *Scanner) PeekInlineNoSpace() byte { + offs := s.ReadOffset + for offs < len(s.Src) { + switch s.Src[offs] { + case ' ', '\t': + offs++ + default: + return s.Src[offs] + } + } + return 0 +} + +func (s *Scanner) PeekNoSpaceN(n int) []byte { + off := s.ReadOffset + for off < len(s.Src) { + switch s.Src[off] { + case ' ', '\n', '\t': off++ default: - part := s.src[off:] + part := s.Src[off:] if len(part) >= n { return part[:n] } @@ -515,178 +860,216 @@ func (s *Scanner) peekNoSpaceN(n int) []byte { return nil } -func (s *Scanner) error(offset int, msg string) { - if s.errorHandler != nil { - s.errorHandler(s.file.Position(s.file.FileSetPos(offset)), msg) +func (s *Scanner) PeekN(n int) []byte { + if (s.ReadOffset + n) < len(s.Src) { + return s.Src[s.ReadOffset : s.ReadOffset+n] } - s.errorCount++ + return nil } -func (s *Scanner) scanConfig() (t Token, ok bool) { - off, roff, loff := s.offset, s.readOffset, s.lineOffset - pos := s.file.FileSetPos(off - 1) - p := s.file.position(pos) - if p.Column != 1 { - return +func (s *Scanner) PeekEq(str string) bool { + b := s.PeekN(len(str)) + if b != nil { + return string(b) == str } - s.nextNoSpace() - if isLetter(s.ch) { - name := s.scanIdentifier() - if name == "gad" { - s.skipWhitespace() - if s.ch == ':' { - ok = true - s.nextNoSpace() - var ( - start = s.offset - end int - semi string - ) - - cfg_line: - for { - switch s.ch { - case '\n', ';', -1: - end = s.offset - 1 - if s.src[end] == '\r' { - end-- - } - t.Data = s.file.FileSetPos(end) - if s.ch != -1 { - semi = string(s.ch) - } - s.next() - break cfg_line - } - s.next() + return true +} + +func (s *Scanner) PeekNoSpaceEq(to string, skip int) bool { + off := s.ReadOffset + skip + for off < len(s.Src) { + switch s.Src[off] { + case ' ', '\n', '\t': + off++ + default: + n := len(to) + if (off + n) <= len(s.Src) { + b := s.Src[off : off+n] + return to == string(b) + } + return false + } + } + + return false +} + +func (s *Scanner) PeekNoSingleSpaceEq(to string, skip int) (length int) { + off := s.ReadOffset + skip + for off < len(s.Src) { + switch s.Src[off] { + case ' ', '\t': + off++ + default: + n := len(to) + if (off + n) <= len(s.Src) { + b := s.Src[off : off+n] + if to == string(b) { + return off + n - s.ReadOffset } + return 0 + } + return 0 + } + } - t.Literal = string(s.src[start : end+1]) - t.Token = token.Config - t.Pos = pos - s.tokenPool = append(s.tokenPool, Token{Pos: s.file.FileSetPos(end), Token: token.Semicolon, Literal: semi}) - return + return 0 +} + +func (s *Scanner) PeekIdentEq(to string, skip int) bool { + off := s.ReadOffset + skip + for off < len(s.Src) { + switch s.Src[off] { + case ' ', '\t': + off++ + default: + n := len(to) + if (off + n) <= len(s.Src) { + b := s.Src[off : off+n] + return to == string(b) } + return false } } - s.offset, s.readOffset, s.lineOffset = off, roff, loff + + return false +} + +func (s *Scanner) Error(offset int, msg string) { + pos := s.File.Position(s.File.FileSetPos(offset)) + for _, h := range s.errorHandler { + h(pos, msg) + } + s.errorCount++ +} + +func (s *Scanner) Expect(ch rune, msg string) bool { + if s.Ch != ch { + s.ExpectError(msg + fmt.Sprintf(", but got %s", string(s.Ch))) + s.Ch = -1 + return false + } + return true +} + +func (s *Scanner) ExpectError(msg string) { + s.Error(s.Offset, "Expect: "+msg) +} + +func (s *Scanner) scanConfig(pos source.Pos, skip int) (t Token) { + s.NextC(skip) + eol := s.NextPosOf('\n') + s2 := *s + s2.Src = s2.Src[:eol] + s2.mode = 0 + s2.NextNoSpace() + s2.TokenPool = nil + t.Token = token.ConfigEnd + t.Pos = s.File.FileSetPos(eol) + t.Prev = append([]Token{{ + Token: token.ConfigStart, + Pos: pos, + }}, s2.List()...) + s.NextC(eol - s.Offset + 1) return } -func (s *Scanner) scanComment() string { +func (s *Scanner) ScanComment() string { // initial '/' already consumed; s.ch == '/' || s.ch == '*' - offs := s.offset - 1 // position of initial '/' - var numCR int + offs := s.Offset - 1 // position of initial '/' - if s.ch == '/' { + if s.Ch == '/' { // -style comment // (the final '\n' is not considered part of the comment) - s.next() - for s.ch != '\n' && s.ch >= 0 { - if s.ch == '\r' { - numCR++ - } - s.next() + s.Next() + for s.Ch != '\n' && s.Ch >= 0 { + s.Next() } goto exit } /*-style comment */ - s.next() - for s.ch >= 0 { - ch := s.ch - if ch == '\r' { - numCR++ - } - s.next() - if ch == '*' && s.ch == '/' { - s.next() + s.Next() + for s.Ch >= 0 { + ch := s.Ch + s.Next() + if ch == '*' && s.Ch == '/' { + s.Next() goto exit } } - s.error(offs, "comment not terminated") + s.Error(offs, "comment not terminated") exit: - lit := s.src[offs:s.offset] - - // On Windows, a (//-comment) line may end in "\r\n". - // Remove the final '\r' before analyzing the text for line directives (matching the compiler). - // Remove any other '\r' afterwards (matching the pre-existing behavior of the scanner). - if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' { - lit = lit[:len(lit)-1] - numCR-- - } - if numCR > 0 { - lit = StripCR(lit, lit[1] == '*') - } + lit := s.Src[offs:s.Offset] return string(lit) } -func (s *Scanner) findLineEnd() bool { +func (s *Scanner) FindLineEnd() bool { // initial '/' already consumed defer func(offs int) { - // reset scanner state to where it was upon calling findLineEnd - s.ch = '/' - s.offset = offs - s.readOffset = offs + 1 - s.next() // consume initial '/' again - }(s.offset - 1) + // reset scanner state to where it was upon calling FindLineEnd + s.Ch = '/' + s.Offset = offs + s.ReadOffset = offs + 1 + s.Next() // consume initial '/' again + }(s.Offset - 1) // read ahead until a newline, EOF, or non-comment tok is found - for s.ch == '/' || s.ch == '*' { - if s.ch == '/' { + for s.Ch == '/' || s.Ch == '*' { + if s.Ch == '/' { // -style comment always contains a newline return true } /*-style comment: look for newline */ - s.next() - for s.ch >= 0 { - ch := s.ch + s.Next() + for s.Ch >= 0 { + ch := s.Ch if ch == '\n' { return true } - s.next() - if ch == '*' && s.ch == '/' { - s.next() + s.Next() + if ch == '*' && s.Ch == '/' { + s.Next() break } } - s.skipWhitespace() // s.insertSemi is set - if s.ch < 0 || s.ch == '\n' { + s.SkipWhitespace() // s.InsertSemi is set + if s.Ch < 0 || s.Ch == '\n' { return true } - if s.ch != '/' { + if s.Ch != '/' { // non-comment Token return false } - s.next() // consume '/' + s.Next() // consume '/' } return false } -func (s *Scanner) scanIdentifier() string { - offs := s.offset - for isLetter(s.ch) || isDigit(s.ch) { - s.next() +func (s *Scanner) ScanIdentifier() string { + offs := s.Offset + for utils.IsLetter(s.Ch) || utils.IsDigit(s.Ch) { + s.Next() } - return string(s.src[offs:s.offset]) + return string(s.Src[offs:s.Offset]) } func (s *Scanner) scanMantissa(base int) { - for digitVal(s.ch) < base { - s.next() + for utils.DigitVal(s.Ch) < base { + s.Next() } } -func (s *Scanner) scanNumber(seenDecimalPoint bool) (tok token.Token, lit string) { - // digitVal(s.ch) < 10 - offs := s.offset +func (s *Scanner) ScanNumber(seenDecimalPoint bool) (tok token.Token, lit string) { + // DigitVal(s.ch) < 10 + offs := s.Offset tok = token.Int defer func() { - lit = string(s.src[offs:s.offset]) + lit = string(s.Src[offs:s.Offset]) }() if seenDecimalPoint { @@ -698,40 +1081,40 @@ func (s *Scanner) scanNumber(seenDecimalPoint bool) (tok token.Token, lit string goto exponent } - if s.ch == '0' { + if s.Ch == '0' { // int or float - offs := s.offset - s.next() - if s.ch == 'x' || s.ch == 'X' { + offs := s.Offset + s.Next() + if s.Ch == 'x' || s.Ch == 'X' { // hexadecimal int - s.next() + s.Next() s.scanMantissa(16) - if s.offset-offs <= 2 { + if s.Offset-offs <= 2 { // only scanned "0x" or "0X" - s.error(offs, "illegal hexadecimal number") + s.Error(offs, "illegal hexadecimal number") } } else { // octal int or float seenDecimalDigit := false s.scanMantissa(8) - if s.ch == '8' || s.ch == '9' { + if s.Ch == '8' || s.Ch == '9' { // illegal octal int or float seenDecimalDigit = true s.scanMantissa(10) } - if s.ch == '.' || s.ch == 'e' || s.ch == 'E' || s.ch == 'i' { + if s.Ch == '.' || s.Ch == 'e' || s.Ch == 'E' || s.Ch == 'i' { goto fraction } // octal int if seenDecimalDigit { - s.error(offs, "illegal octal number") + s.Error(offs, "illegal octal number") } // check if unsigned - if s.ch == 'u' { - s.next() + if s.Ch == 'u' { + s.Next() tok = token.Uint - } else if s.ch == 'd' { - s.next() + } else if s.Ch == 'd' { + s.Next() tok = token.Decimal } } @@ -741,114 +1124,114 @@ func (s *Scanner) scanNumber(seenDecimalPoint bool) (tok token.Token, lit string s.scanMantissa(10) // check if unsigned - if s.ch == 'u' { - s.next() + if s.Ch == 'u' { + s.Next() tok = token.Uint - } else if s.ch == 'd' { - s.next() + } else if s.Ch == 'd' { + s.Next() tok = token.Decimal } fraction: - if s.ch == '.' { + if s.Ch == '.' { tok = token.Float - s.next() + s.Next() s.scanMantissa(10) } exponent: - if s.ch == 'e' || s.ch == 'E' { + if s.Ch == 'e' || s.Ch == 'E' { if tok != token.Decimal { tok = token.Float } - s.next() - if s.ch == '-' || s.ch == '+' { - s.next() + s.Next() + if s.Ch == '-' || s.Ch == '+' { + s.Next() } - if digitVal(s.ch) < 10 { + if utils.DigitVal(s.Ch) < 10 { s.scanMantissa(10) } else { - s.error(offs, "illegal floating-point exponent") + s.Error(offs, "illegal floating-point exponent") } } - if s.ch == 'd' && tok != token.Decimal && tok != token.Uint { + if s.Ch == 'd' && tok != token.Decimal && tok != token.Uint { tok = token.Decimal - s.next() + s.Next() } return } func (s *Scanner) scanEscape(quote rune) bool { - offs := s.offset + offs := s.Offset var n int var base, max uint32 - switch s.ch { + switch s.Ch { case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote: - s.next() + s.Next() return true case '0', '1', '2', '3', '4', '5', '6', '7': n, base, max = 3, 8, 255 case 'x': - s.next() + s.Next() n, base, max = 2, 16, 255 case 'u': - s.next() + s.Next() n, base, max = 4, 16, unicode.MaxRune case 'U': - s.next() + s.Next() n, base, max = 8, 16, unicode.MaxRune default: msg := "unknown escape sequence" - if s.ch < 0 { + if s.Ch < 0 { msg = "escape sequence not terminated" } - s.error(offs, msg) + s.Error(offs, msg) return false } var x uint32 for n > 0 { - d := uint32(digitVal(s.ch)) + d := uint32(utils.DigitVal(s.Ch)) if d >= base { msg := fmt.Sprintf( - "illegal character %#U in escape sequence", s.ch) - if s.ch < 0 { + "illegal character %#U in escape sequence", s.Ch) + if s.Ch < 0 { msg = "escape sequence not terminated" } - s.error(s.offset, msg) + s.Error(s.Offset, msg) return false } x = x*base + d - s.next() + s.Next() n-- } if x > max || 0xD800 <= x && x < 0xE000 { - s.error(offs, "escape sequence is invalid Unicode code point") + s.Error(offs, "escape sequence is invalid Unicode code point") return false } return true } -func (s *Scanner) scanRune() string { - offs := s.offset - 1 // '\'' opening already consumed +func (s *Scanner) ScanRune() string { + offs := s.Offset - 1 // '\'' opening already consumed valid := true n := 0 for { - ch := s.ch + ch := s.Ch if ch == '\n' || ch < 0 { // only report error if we don't have one already if valid { - s.error(offs, "rune literal not terminated") + s.Error(offs, "rune literal not terminated") valid = false } break } - s.next() + s.Next() if ch == '\'' { break } @@ -862,21 +1245,21 @@ func (s *Scanner) scanRune() string { } if valid && n != 1 { - s.error(offs, "illegal rune literal") + s.Error(offs, "illegal rune literal") } - return string(s.src[offs:s.offset]) + return string(s.Src[offs:s.Offset]) } -func (s *Scanner) scanString() string { - offs := s.offset - 1 // '"' opening already consumed +func (s *Scanner) ScanString() string { + offs := s.Offset - 1 // '"' opening already consumed for { - ch := s.ch + ch := s.Ch if ch == '\n' || ch < 0 { - s.error(offs, "string literal not terminated") + s.Error(offs, "string literal not terminated") break } - s.next() + s.Next() if ch == '"' { break } @@ -884,36 +1267,27 @@ func (s *Scanner) scanString() string { s.scanEscape('"') } } - return string(s.src[offs:s.offset]) + return string(s.Src[offs:s.Offset]) } -func (s *Scanner) scanRawString() string { - offs := s.offset - 1 // '`' opening already consumed +func (s *Scanner) ScanRawString() string { + offs := s.Offset - 1 // '`' opening already consumed - hasCR := false for { - ch := s.ch + ch := s.Ch if ch < 0 { - s.error(offs, "raw string literal not terminated") + s.Error(offs, "raw string literal not terminated") break } - s.next() + s.Next() if ch == '`' { break } - - if ch == '\r' { - hasCR = true - } } - lit := s.src[offs:s.offset] - if hasCR { - lit = StripCR(lit, false) - } - return string(lit) + return string(s.Src[offs:s.Offset]) } // StripCR removes carriage return characters. @@ -934,50 +1308,43 @@ func StripCR(b []byte, comment bool) []byte { return c[:i] } -func (s *Scanner) skipWhitespace() { - for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !s.insertSemi || - s.ch == '\r' { - s.next() - } -} - -func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token { - if s.ch == '=' { - s.next() +func (s *Scanner) Switch2(tok0, tok1 token.Token) token.Token { + if s.Ch == '=' { + s.Next() return tok1 } return tok0 } -func (s *Scanner) switch3( +func (s *Scanner) Switch3( tok0, tok1 token.Token, ch2 rune, tok2 token.Token, ) token.Token { - if s.ch == '=' { - s.next() + if s.Ch == '=' { + s.Next() return tok1 } - if s.ch == ch2 { - s.next() + if s.Ch == ch2 { + s.Next() return tok2 } return tok0 } -func (s *Scanner) switch4( +func (s *Scanner) Switch4( tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token, ) token.Token { - if s.ch == '=' { - s.next() + if s.Ch == '=' { + s.Next() return tok1 } - if s.ch == ch2 { - s.next() - if s.ch == '=' { - s.next() + if s.Ch == ch2 { + s.Next() + if s.Ch == '=' { + s.Next() return tok3 } return tok2 @@ -985,24 +1352,46 @@ func (s *Scanner) switch4( return tok0 } -func isLetter(ch rune) bool { - return ch == '$' || 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || - ch >= utf8.RuneSelf && unicode.IsLetter(ch) -} +func (s *Scanner) ScanCodeBlock(leftText *Token) (code Token) { + var ( + end = s.Offset + lit = "#{" + ) + s.InCode = true + s.InCodeBrace = s.BraceCount + s.Next() + s.Next() + s.BraceCount++ + + if leftText != nil { + switch s.Ch { + case '-': + s.NextNoSpace() + leftText.Literal = TrimSpace(s.TextTrimLeft, true, leftText.Literal) + } + if leftText.Literal == "" { + leftText = nil + } + } -func isDigit(ch rune) bool { - return '0' <= ch && ch <= '9' || - ch >= utf8.RuneSelf && unicode.IsDigit(ch) -} + s.TextTrimLeft = false -func digitVal(ch rune) int { - switch { - case '0' <= ch && ch <= '9': - return int(ch - '0') - case 'a' <= ch && ch <= 'f': - return int(ch - 'a' + 10) - case 'A' <= ch && ch <= 'F': - return int(ch - 'A' + 10) + code = Token{ + Token: token.CodeBegin, + Pos: s.File.FileSetPos(end), + Literal: lit, + } + + if s.Ch == '=' { + s.ToText = true + s.NextNoSpace() + code.Literal += "=" + code.Token = token.ToTextBegin } - return 16 // larger than any legal digit val + + if leftText != nil { + code.Prev = append(code.Prev, *leftText) + } + + return } diff --git a/parser/scanner_test.go b/parser/scanner_test.go index 99c1062..01e2698 100644 --- a/parser/scanner_test.go +++ b/parser/scanner_test.go @@ -44,10 +44,10 @@ func TestScanner_Scan(t *testing.T) { }{ {token.Comment, "/* a comment */"}, {token.Comment, "// a comment \n"}, - {token.Comment, "/*\r*/"}, - {token.Comment, "/**\r/*/"}, - {token.Comment, "/**\r\r/*/"}, - {token.Comment, "//\r\n"}, + {token.Comment, "/*\n*/"}, + {token.Comment, "/**\n/*/"}, + {token.Comment, "/**\n\n/*/"}, + {token.Comment, "//\n"}, {token.Ident, "foobar"}, {token.Ident, "a۰۱۸"}, {token.Ident, "foo६४"}, @@ -95,8 +95,8 @@ func TestScanner_Scan(t *testing.T) { bar` + "`", }, - {token.String, "`\r`"}, - {token.String, "`foo\r\nbar`"}, + {token.String, "`\n`"}, + {token.String, "`foo\nbar`"}, {token.Add, "+"}, {token.Sub, "-"}, {token.Mul, "*"}, @@ -308,8 +308,8 @@ func scanExpect( s := parser.NewScanner( testFile, []byte(input), - func(_ parser.SourceFilePos, msg string) { require.Fail(t, msg) }, mode) + s.ErrorHandler(func(_ parser.SourceFilePos, msg string) { require.Fail(t, msg) }) for idx, e := range expected { tok := s.Scan() diff --git a/parser/pos.go b/parser/source/pos.go similarity index 97% rename from parser/pos.go rename to parser/source/pos.go index 3d7d9ad..e19dca1 100644 --- a/parser/pos.go +++ b/parser/source/pos.go @@ -10,7 +10,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE.golang file. -package parser +package source // Pos represents a position in the file set. type Pos int diff --git a/parser/source_file.go b/parser/source_file.go index 0834b46..3617679 100644 --- a/parser/source_file.go +++ b/parser/source_file.go @@ -15,6 +15,8 @@ package parser import ( "fmt" "sort" + + "github.com/gad-lang/gad/parser/source" ) // SourceFilePos represents a position information in the file. @@ -77,6 +79,7 @@ func (s *SourceFileSet) AddFile(filename string, base, size int) *SourceFile { if base < s.Base || size < 0 { panic("illegal base or size") } + f := &SourceFile{ set: s, Name: filename, @@ -98,24 +101,24 @@ func (s *SourceFileSet) AddFile(filename string, base, size int) *SourceFile { // File returns the file that contains the position p. If no such file is // found (for instance for p == NoPos), the result is nil. -func (s *SourceFileSet) File(p Pos) (f *SourceFile) { - if p != NoPos { +func (s *SourceFileSet) File(p source.Pos) (f *SourceFile) { + if p != source.NoPos { f = s.file(p) } return } // Position converts a SourcePos p in the fileset into a SourceFilePos value. -func (s *SourceFileSet) Position(p Pos) (pos SourceFilePos) { - if p != NoPos { +func (s *SourceFileSet) Position(p source.Pos) (pos SourceFilePos) { + if p != source.NoPos { if f := s.file(p); f != nil { - return f.position(p) + return f.SafePosition(p) } } return } -func (s *SourceFileSet) file(p Pos) *SourceFile { +func (s *SourceFileSet) file(p source.Pos) *SourceFile { // common case: p is in last file f := s.LastFile if f != nil && f.Base <= int(p) && int(p) <= f.Base+f.Size { @@ -166,33 +169,49 @@ func (f *SourceFile) LineCount() int { // AddLine adds a new line. func (f *SourceFile) AddLine(offset int) { - i := len(f.Lines) - if (i == 0 || f.Lines[i-1] < offset) && offset < f.Size { + if offset >= f.Size { + return + } + + lc := len(f.Lines) + if lc == 0 { + f.Lines = append(f.Lines, offset) + } else { + for i := lc; i > 0; i-- { + if off := f.Lines[i-1]; off == offset { + return + } else if off > offset { + f.Lines = append(f.Lines, -1) + copy(f.Lines[i:], f.Lines[i-1:]) + f.Lines[i-1] = offset + return + } + } f.Lines = append(f.Lines, offset) } } // LineStart returns the position of the first character in the line. -func (f *SourceFile) LineStart(line int) Pos { +func (f *SourceFile) LineStart(line int) source.Pos { if line < 1 { panic("illegal line number (line numbering starts at 1)") } if line > len(f.Lines) { panic("illegal line number") } - return Pos(f.Base + f.Lines[line-1]) + return source.Pos(f.Base + f.Lines[line-1]) } // FileSetPos returns the position in the file set. -func (f *SourceFile) FileSetPos(offset int) Pos { +func (f *SourceFile) FileSetPos(offset int) source.Pos { if offset > f.Size { panic("illegal file offset") } - return Pos(f.Base + offset) + return source.Pos(f.Base + offset) } // Offset translates the file set position into the file offset. -func (f *SourceFile) Offset(p Pos) int { +func (f *SourceFile) Offset(p source.Pos) int { if int(p) < f.Base || int(p) > f.Base+f.Size { panic("illegal SourcePos value") } @@ -200,29 +219,29 @@ func (f *SourceFile) Offset(p Pos) int { } // Line returns the line of given position. -func (f *SourceFile) Line(p Pos) int { +func (f *SourceFile) Line(p source.Pos) int { return f.Position(p).Line } // Position translates the file set position into the file position. -func (f *SourceFile) Position(p Pos) (pos SourceFilePos) { - if p != NoPos { +func (f *SourceFile) Position(p source.Pos) (pos SourceFilePos) { + if p != source.NoPos { if int(p) < f.Base || int(p) > f.Base+f.Size { panic("illegal SourcePos value") } - pos = f.position(p) + pos = f.SafePosition(p) } return } -func (f *SourceFile) position(p Pos) (pos SourceFilePos) { +func (f *SourceFile) SafePosition(p source.Pos) (pos SourceFilePos) { offset := int(p) - f.Base pos.Offset = offset - pos.Filename, pos.Line, pos.Column = f.unpack(offset) + pos.Filename, pos.Line, pos.Column = f.Unpack(offset) return } -func (f *SourceFile) unpack(offset int) (filename string, line, column int) { +func (f *SourceFile) Unpack(offset int) (filename string, line, column int) { filename = f.Name if i := searchInts(f.Lines, offset); i >= 0 { line, column = i+1, offset-f.Lines[i]+1 @@ -230,6 +249,51 @@ func (f *SourceFile) unpack(offset int) (filename string, line, column int) { return } +func (f *SourceFile) LineIndexOf(p source.Pos) int { + l := len(f.Lines) + for i := l; i > 0; i-- { + p2 := f.Lines[i-1] + if p2 <= int(p) { + return i - 1 + } + } + return 0 +} + +func (f *SourceFile) LinePos(p source.Pos) source.Pos { + l := len(f.Lines) + for i := l; i > 0; i-- { + p2 := f.Lines[i-1] + if p2 <= int(p) { + if i == l { + // last line, first column + return source.Pos(f.Lines[i-1] + 1) + } + return source.Pos(f.Lines[i-1]) + } + } + return p +} + +func (f *SourceFile) NextLinePos(p source.Pos) source.Pos { + l := len(f.Lines) + for i := l; i > 0; i-- { + p2 := f.Lines[i-1] + if p2 <= int(p) { + if i == l { + // last line, first column + c1 := source.Pos(f.Lines[i-1] + 1) + if p <= c1 { + c1-- + } + return c1 + } + return source.Pos(f.Lines[i-1]) + } + } + return p +} + func searchInts(a []int, x int) int { // This function body is a manually inlined version of: // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 diff --git a/parser/testdata/trace.golden b/parser/testdata/trace.golden index 933d88c..626c242 100644 --- a/parser/testdata/trace.golden +++ b/parser/testdata/trace.golden @@ -349,8 +349,10 @@ 184: 19: 4: . . . . . . . . . . . . . . . . . . . . . . IDENT x 185: 19: 5: . . . . . . . . . . . . . . . . . . . . . ) 185: 19: 5: . . . . . . . . . . . . . . . . . . . . . Call ( - 185: 19: 5: . . . . . . . . . . . . . . . . . . . . . . "(" - 186: 19: 6: . . . . . . . . . . . . . . . . . . . . . . ")" + 185: 19: 5: . . . . . . . . . . . . . . . . . . . . . . CallArgs ( + 185: 19: 5: . . . . . . . . . . . . . . . . . . . . . . . "(" + 186: 19: 6: . . . . . . . . . . . . . . . . . . . . . . . ")" + 187: 19: 7: . . . . . . . . . . . . . . . . . . . . . . ) 187: 19: 7: . . . . . . . . . . . . . . . . . . . . . ) 187: 19: 7: . . . . . . . . . . . . . . . . . . . . ) 187: 19: 7: . . . . . . . . . . . . . . . . . . . ) @@ -380,19 +382,21 @@ 207: 21: 4: . . . . . . . . . . . . . . . . . . . . . . . IDENT println 214: 21: 11: . . . . . . . . . . . . . . . . . . . . . . ) 214: 21: 11: . . . . . . . . . . . . . . . . . . . . . . Call ( - 214: 21: 11: . . . . . . . . . . . . . . . . . . . . . . . "(" - 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . Expression ( - 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . BinaryExpression ( - 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . UnaryExpression ( - 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . PrimaryExpression ( - 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . . Operand ( - 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . . . IDENT err + 214: 21: 11: . . . . . . . . . . . . . . . . . . . . . . . CallArgs ( + 214: 21: 11: . . . . . . . . . . . . . . . . . . . . . . . . "(" + 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . Expression ( + 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . BinaryExpression ( + 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . UnaryExpression ( + 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . . PrimaryExpression ( + 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . . . Operand ( + 215: 21: 12: . . . . . . . . . . . . . . . . . . . . . . . . . . . . . IDENT err + 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . . . . . . ) 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . . . . . ) 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . . . . ) 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . . . ) 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . . ) - 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . ) - 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . ")" + 218: 21: 15: . . . . . . . . . . . . . . . . . . . . . . . . ")" + 219: 21: 16: . . . . . . . . . . . . . . . . . . . . . . . ) 219: 21: 16: . . . . . . . . . . . . . . . . . . . . . . ) 219: 21: 16: . . . . . . . . . . . . . . . . . . . . . ) 219: 21: 16: . . . . . . . . . . . . . . . . . . . . ) @@ -605,31 +609,33 @@ 320: 31: 2: . . . . . . . . . . . . . . IDENT println 327: 31: 9: . . . . . . . . . . . . . ) 327: 31: 9: . . . . . . . . . . . . . Call ( - 327: 31: 9: . . . . . . . . . . . . . . "(" - 328: 31: 10: . . . . . . . . . . . . . . Expression ( - 328: 31: 10: . . . . . . . . . . . . . . . BinaryExpression ( - 328: 31: 10: . . . . . . . . . . . . . . . . UnaryExpression ( - 328: 31: 10: . . . . . . . . . . . . . . . . . PrimaryExpression ( - 328: 31: 10: . . . . . . . . . . . . . . . . . . Operand ( - 328: 31: 10: . . . . . . . . . . . . . . . . . . . IDENT k + 327: 31: 9: . . . . . . . . . . . . . . CallArgs ( + 327: 31: 9: . . . . . . . . . . . . . . . "(" + 328: 31: 10: . . . . . . . . . . . . . . . Expression ( + 328: 31: 10: . . . . . . . . . . . . . . . . BinaryExpression ( + 328: 31: 10: . . . . . . . . . . . . . . . . . UnaryExpression ( + 328: 31: 10: . . . . . . . . . . . . . . . . . . PrimaryExpression ( + 328: 31: 10: . . . . . . . . . . . . . . . . . . . Operand ( + 328: 31: 10: . . . . . . . . . . . . . . . . . . . . IDENT k + 329: 31: 11: . . . . . . . . . . . . . . . . . . . ) 329: 31: 11: . . . . . . . . . . . . . . . . . . ) 329: 31: 11: . . . . . . . . . . . . . . . . . ) 329: 31: 11: . . . . . . . . . . . . . . . . ) 329: 31: 11: . . . . . . . . . . . . . . . ) - 329: 31: 11: . . . . . . . . . . . . . . ) - 329: 31: 11: . . . . . . . . . . . . . . "," - 331: 31: 13: . . . . . . . . . . . . . . Expression ( - 331: 31: 13: . . . . . . . . . . . . . . . BinaryExpression ( - 331: 31: 13: . . . . . . . . . . . . . . . . UnaryExpression ( - 331: 31: 13: . . . . . . . . . . . . . . . . . PrimaryExpression ( - 331: 31: 13: . . . . . . . . . . . . . . . . . . Operand ( - 331: 31: 13: . . . . . . . . . . . . . . . . . . . IDENT v + 329: 31: 11: . . . . . . . . . . . . . . . "," + 331: 31: 13: . . . . . . . . . . . . . . . Expression ( + 331: 31: 13: . . . . . . . . . . . . . . . . BinaryExpression ( + 331: 31: 13: . . . . . . . . . . . . . . . . . UnaryExpression ( + 331: 31: 13: . . . . . . . . . . . . . . . . . . PrimaryExpression ( + 331: 31: 13: . . . . . . . . . . . . . . . . . . . Operand ( + 331: 31: 13: . . . . . . . . . . . . . . . . . . . . IDENT v + 332: 31: 14: . . . . . . . . . . . . . . . . . . . ) 332: 31: 14: . . . . . . . . . . . . . . . . . . ) 332: 31: 14: . . . . . . . . . . . . . . . . . ) 332: 31: 14: . . . . . . . . . . . . . . . . ) 332: 31: 14: . . . . . . . . . . . . . . . ) - 332: 31: 14: . . . . . . . . . . . . . . ) - 332: 31: 14: . . . . . . . . . . . . . . ")" + 332: 31: 14: . . . . . . . . . . . . . . . ")" + 333: 31: 15: . . . . . . . . . . . . . . ) 333: 31: 15: . . . . . . . . . . . . . ) 333: 31: 15: . . . . . . . . . . . . ) 333: 31: 15: . . . . . . . . . . . ) @@ -702,19 +708,21 @@ 361: 34: 12: . . . . . . . . . . . . . . . . . . . . . IDENT error 366: 34: 17: . . . . . . . . . . . . . . . . . . . . ) 366: 34: 17: . . . . . . . . . . . . . . . . . . . . Call ( - 366: 34: 17: . . . . . . . . . . . . . . . . . . . . . "(" - 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . Expression ( - 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . BinaryExpression ( - 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . UnaryExpression ( - 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . PrimaryExpression ( - 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . . Operand ( - 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . . . STRING "err" + 366: 34: 17: . . . . . . . . . . . . . . . . . . . . . CallArgs ( + 366: 34: 17: . . . . . . . . . . . . . . . . . . . . . . "(" + 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . Expression ( + 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . BinaryExpression ( + 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . UnaryExpression ( + 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . . PrimaryExpression ( + 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . . . Operand ( + 367: 34: 18: . . . . . . . . . . . . . . . . . . . . . . . . . . . STRING "err" + 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . . . . . . ) 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . . . . . ) 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . . . . ) 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . . . ) 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . . ) - 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . ) - 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . ")" + 372: 34: 23: . . . . . . . . . . . . . . . . . . . . . . ")" + 373: 34: 24: . . . . . . . . . . . . . . . . . . . . . ) 373: 34: 24: . . . . . . . . . . . . . . . . . . . . ) 373: 34: 24: . . . . . . . . . . . . . . . . . . . ) 373: 34: 24: . . . . . . . . . . . . . . . . . . ) @@ -774,8 +782,10 @@ 386: 36: 11: . . . . . . . . . . IDENT f 387: 36: 12: . . . . . . . . . ) 387: 36: 12: . . . . . . . . . Call ( - 387: 36: 12: . . . . . . . . . . "(" - 388: 36: 13: . . . . . . . . . . ")" + 387: 36: 12: . . . . . . . . . . CallArgs ( + 387: 36: 12: . . . . . . . . . . . "(" + 388: 36: 13: . . . . . . . . . . . ")" + 389: 36: 14: . . . . . . . . . . ) 389: 36: 14: . . . . . . . . . ) 389: 36: 14: . . . . . . . . ) 389: 36: 14: . . . . . . . ) @@ -1108,8 +1118,10 @@ 513: 45: 6: . . . . . . . . . . IDENT Now 516: 45: 9: . . . . . . . . . ) 516: 45: 9: . . . . . . . . . Call ( - 516: 45: 9: . . . . . . . . . . "(" - 517: 45: 10: . . . . . . . . . . ")" + 516: 45: 9: . . . . . . . . . . CallArgs ( + 516: 45: 9: . . . . . . . . . . . "(" + 517: 45: 10: . . . . . . . . . . . ")" + 519: 45: 12: . . . . . . . . . . ) 519: 45: 12: . . . . . . . . . ) 519: 45: 12: . . . . . . . . ) 519: 45: 12: . . . . . . . ) diff --git a/parser/token.go b/parser/token.go new file mode 100644 index 0000000..148aa7c --- /dev/null +++ b/parser/token.go @@ -0,0 +1,34 @@ +package parser + +import ( + "fmt" + "strings" + + "github.com/gad-lang/gad/parser/source" + "github.com/gad-lang/gad/parser/utils" + "github.com/gad-lang/gad/token" +) + +type Token struct { + Pos source.Pos + Token token.Token + Literal string + InsertSemi bool + handled bool + Prev []Token + utils.Data +} + +var _ fmt.Stringer = Token{} + +func (t Token) String() string { + return t.Token.String() + ": " + t.Literal +} + +func (t *Token) LiteralRemoveLinePrefix(prefix string) { + lines := strings.Split(t.Literal, "\n") + for i, line := range lines { + lines[i] = strings.TrimPrefix(line, prefix) + } + t.Literal = strings.Join(lines, "\n") +} diff --git a/parser/utils.go b/parser/utils.go index eabc054..503203a 100644 --- a/parser/utils.go +++ b/parser/utils.go @@ -10,7 +10,7 @@ var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} // TrimSpace returns a slice of the string s, with all leading // and trailing white space removed, as defined by Unicode. -func trimSpace(left, right bool, s string) string { +func TrimSpace(left, right bool, s string) string { start := 0 if left { // Fast path for ASCII: look for the first ASCII non-space byte diff --git a/parser/utils/data.go b/parser/utils/data.go new file mode 100644 index 0000000..01febae --- /dev/null +++ b/parser/utils/data.go @@ -0,0 +1,40 @@ +package utils + +import ( + "fmt" + "sort" + "strings" +) + +type Data map[any]any + +func (m *Data) Set(key, value any) { + if *m == nil { + *m = map[any]any{} + } + (*m)[key] = value +} + +func (m Data) GetOk(key any) (v any, ok bool) { + v, ok = m[key] + return +} + +func (m Data) Get(key any) (v any) { + return m[key] +} + +func (m Data) String() string { + if m == nil { + return "" + } + var s []string + for k, v := range m { + if v == nil { + continue + } + s = append(s, fmt.Sprintf("%v: %v", k, v)) + } + sort.Strings(s) + return strings.Join(s, ", ") +} diff --git a/parser/utils/utils.go b/parser/utils/utils.go new file mode 100644 index 0000000..bad14e1 --- /dev/null +++ b/parser/utils/utils.go @@ -0,0 +1,36 @@ +package utils + +import ( + "unicode" + "unicode/utf8" +) + +func IsLetter(ch rune) bool { + return ch == '$' || 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || + ch >= utf8.RuneSelf && unicode.IsLetter(ch) +} + +func IsDigit(ch rune) bool { + return '0' <= ch && ch <= '9' || + ch >= utf8.RuneSelf && unicode.IsDigit(ch) +} + +func IsSpace(ch rune) bool { + return ch == '\n' || IsSingleSpace(ch) +} + +func IsSingleSpace(ch rune) bool { + return ch == ' ' || ch == '\t' +} + +func DigitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= ch && ch <= 'f': + return int(ch - 'a' + 10) + case 'A' <= ch && ch <= 'F': + return int(ch - 'A' + 10) + } + return 16 // larger than any legal digit val +} diff --git a/stdlib/json/encode.go b/stdlib/json/encode.go index b68d6d7..f9694cd 100644 --- a/stdlib/json/encode.go +++ b/stdlib/json/encode.go @@ -37,7 +37,7 @@ func Marshal(v gad.Object) ([]byte, error) { return buf, nil } -// MarshalIndent is like Marshal but applies Indent to format the output. +// MarshalIndent is like Marshal but applies IndentCount to format the output. // Each JSON element in the output will begin on a new line beginning with prefix // followed by one or more copies of indent according to the indentation nesting. func MarshalIndent(v gad.Object, prefix, indent string) ([]byte, error) { diff --git a/stdlib/json/module.go b/stdlib/json/module.go index e11cd6e..fe606e6 100644 --- a/stdlib/json/module.go +++ b/stdlib/json/module.go @@ -25,16 +25,16 @@ var Module = map[string]gad.Object{ }, // gad:doc // MarshalIndent(v any, prefix string, indent string) -> bytes - // MarshalIndent is like Marshal but applies Indent to format the output. + // MarshalIndent is like Marshal but applies IndentCount to format the output. "MarshalIndent": &gad.Function{ Name: "MarshalIndent", Value: stdlib.FuncPOssRO(marshalIndentFunc), }, // gad:doc - // Indent(src bytes, prefix string, indent string) -> bytes + // IndentCount(src bytes, prefix string, indent string) -> bytes // Returns indented form of the JSON-encoded src or error. - "Indent": &gad.Function{ - Name: "Indent", + "IndentCount": &gad.Function{ + Name: "IndentCount", Value: stdlib.FuncPb2ssRO(indentFunc), }, // gad:doc diff --git a/stdlib/json/module_test.go b/stdlib/json/module_test.go index c07cb05..20c6bea 100644 --- a/stdlib/json/module_test.go +++ b/stdlib/json/module_test.go @@ -77,8 +77,8 @@ func TestScript(t *testing.T) { expectRun(t, catchf(`string(json.Marshal({_: 1, k2:[3,true,"a"]}))`), nil, String(`{"_":1,"k2":[3,true,"a"]}`)) - expectRun(t, catchf(`json.Indent()`), nil, errnarg(3, 0)) - expectRun(t, catchf(`string(json.Indent("[1,2]", "", " "))`), nil, String("[\n 1,\n 2\n]")) + expectRun(t, catchf(`json.IndentCount()`), nil, errnarg(3, 0)) + expectRun(t, catchf(`string(json.IndentCount("[1,2]", "", " "))`), nil, String("[\n 1,\n 2\n]")) expectRun(t, catchf(`json.MarshalIndent()`), nil, errnarg(3, 0)) expectRun(t, catchf(`string(json.MarshalIndent({a: 1, b: [2, true, "<"]},"", " "))`), @@ -122,7 +122,7 @@ func TestScript(t *testing.T) { expectRun(t, catchf(`string(json.Unmarshal(bytes(0)))`), nil, String(`error: invalid character '\x00' looking for beginning of value`)) - expectRun(t, catchf(`string(json.Indent(bytes(0), "", " "))`), + expectRun(t, catchf(`string(json.IndentCount(bytes(0), "", " "))`), nil, String(`error: invalid character '\x00' looking for beginning of value`)) expectRun(t, catchf(`string(json.Compact(bytes(0), true))`), nil, String(`error: invalid character '\x00' looking for beginning of value`)) diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index f44734a..59836bb 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -49,7 +49,7 @@ package stdlib // //gad:callable func(p []byte, b bool) (ret gad.Object) -// json module Indent +// json module IndentCount // //gad:callable func(p []byte, s1 string, s2 string) (ret gad.Object) diff --git a/stringw/writer.go b/stringw/writer.go new file mode 100644 index 0000000..e108244 --- /dev/null +++ b/stringw/writer.go @@ -0,0 +1,56 @@ +package stringw + +import ( + "fmt" + "io" + "strings" +) + +type StringWriter interface { + io.Writer + io.StringWriter +} + +type StringerTo interface { + StringTo(w StringWriter) +} + +func ToStringW(w StringWriter, v any) { + switch t := v.(type) { + case io.WriterTo: + t.WriteTo(w) + case StringerTo: + t.StringTo(w) + default: + fmt.Fprint(w, v) + } +} + +func ToString(v any) string { + var s strings.Builder + ToStringW(&s, v) + return s.String() +} + +func ToStringSlice[T any](w StringWriter, sep string, s []T) { + l := len(s) + if l == 0 { + return + } + for i := 0; i < l-1; i++ { + ToStringW(w, s[i]) + w.WriteString(sep) + } + ToStringW(w, s[l-1]) +} + +func Each[T any](s []T, do func(e any, last bool)) { + l := len(s) + if l == 0 { + return + } + for i := 0; i < l-1; i++ { + do(s[i], false) + } + do(s[l-1], true) +} diff --git a/token/token.go b/token/token.go index 0084e07..48ef4be 100644 --- a/token/token.go +++ b/token/token.go @@ -16,7 +16,8 @@ const ( Illegal Token = iota EOF Comment - Config + ConfigStart + ConfigEnd Text ToTextBegin ToTextEnd @@ -123,7 +124,7 @@ const ( var tokens = [...]string{ Illegal: "ILLEGAL", EOF: "EOF", - Config: "CONFIG", + ConfigStart: "CONFIG", Comment: "COMMENT", Ident: "IDENT", Int: "INT", @@ -140,6 +141,8 @@ var tokens = [...]string{ StdErr: "STDERR", CodeBegin: "CODEBEGIN", CodeEnd: "CODEEND", + ToTextBegin: "TOTEXTBEGIN", + ToTextEnd: "TOTEXTEND", Add: "+", Sub: "-", Mul: "*", diff --git a/vm.go b/vm.go index 5f1027a..f3aa3e7 100644 --- a/vm.go +++ b/vm.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/source" "github.com/gad-lang/gad/token" ) @@ -1080,9 +1081,9 @@ func (vm *VM) newErrorFromError(err error) *RuntimeError { return vm.newError(&Error{Message: err.Error(), Cause: err}) } -func (vm *VM) getSourcePos() parser.Pos { +func (vm *VM) getSourcePos() source.Pos { if vm.curFrame == nil || vm.curFrame.fn == nil { - return parser.NoPos + return source.NoPos } return vm.curFrame.fn.SourcePos(vm.ip) } @@ -1187,9 +1188,9 @@ type frame struct { namedArgs *NamedArgs } -func getFrameSourcePos(frame *frame) parser.Pos { +func getFrameSourcePos(frame *frame) source.Pos { if frame == nil || frame.fn == nil { - return parser.NoPos + return source.NoPos } return frame.fn.SourcePos(frame.ip + 1) } diff --git a/vm_err_test.go b/vm_err_test.go index 2d45f6f..f157bdb 100644 --- a/vm_err_test.go +++ b/vm_err_test.go @@ -5,7 +5,7 @@ import ( "errors" "testing" - "github.com/gad-lang/gad/parser" + "github.com/gad-lang/gad/parser/source" "github.com/stretchr/testify/require" . "github.com/gad-lang/gad" @@ -105,7 +105,7 @@ func TestVMErrorHandlers(t *testing.T) { require.Equal(t, "", invOpErr.Err.Message) require.Nil(t, invOpErr.Err.Cause) require.Equal(t, 1, len(invOpErr.Trace)) - require.Equal(t, parser.Pos(1), invOpErr.Trace[0]) + require.Equal(t, source.Pos(1), invOpErr.Trace[0]) expectErrIs(t, `try { throw WrongNumArgumentsError } catch err { throw err }`, newOpts().Skip2Pass(), ErrWrongNumArguments) @@ -118,8 +118,8 @@ func TestVMErrorHandlers(t *testing.T) { require.Equal(t, "x", errZeroDiv.Err.Message) require.Equal(t, ErrZeroDivision, errZeroDiv.Err.Cause) require.Equal(t, 2, len(errZeroDiv.Trace)) - require.Equal(t, parser.Pos(7), errZeroDiv.Trace[0]) - require.Equal(t, parser.Pos(54), errZeroDiv.Trace[1]) + require.Equal(t, source.Pos(7), errZeroDiv.Trace[0]) + require.Equal(t, source.Pos(54), errZeroDiv.Trace[1]) errZeroDiv = nil expectErrAs(t, `func(x) { return 1/x }(0)`, newOpts().Skip2Pass(), &errZeroDiv, nil) @@ -127,8 +127,8 @@ func TestVMErrorHandlers(t *testing.T) { require.Equal(t, "", errZeroDiv.Err.Message) require.Equal(t, nil, errZeroDiv.Err.Cause) require.Equal(t, 2, len(errZeroDiv.Trace)) - require.Equal(t, parser.Pos(18), errZeroDiv.Trace[0]) - require.Equal(t, parser.Pos(1), errZeroDiv.Trace[1]) + require.Equal(t, source.Pos(18), errZeroDiv.Trace[0]) + require.Equal(t, source.Pos(1), errZeroDiv.Trace[1]) errZeroDiv = nil expectErrAs(t, `1/0`, newOpts().Skip2Pass(), &errZeroDiv, nil) @@ -136,7 +136,7 @@ func TestVMErrorHandlers(t *testing.T) { require.Equal(t, "", errZeroDiv.Err.Message) require.Equal(t, nil, errZeroDiv.Err.Cause) require.Equal(t, 1, len(errZeroDiv.Trace)) - require.Equal(t, parser.Pos(1), errZeroDiv.Trace[0]) + require.Equal(t, source.Pos(1), errZeroDiv.Trace[0]) } func TestVMNoPanic(t *testing.T) { diff --git a/vm_loop.go b/vm_loop.go index c94c8e2..0fe6cd5 100644 --- a/vm_loop.go +++ b/vm_loop.go @@ -504,6 +504,19 @@ VMLoop: iterator := vm.stack[vm.sp-1] hasMore := iterator.(Iterator).Next() vm.stack[vm.sp-1] = Bool(hasMore) + case OpIterNextElse: + iterator := vm.stack[vm.sp-1] + truePos := int(vm.curInsts[vm.ip+2]) | int(vm.curInsts[vm.ip+1])<<8 + falsePos := int(vm.curInsts[vm.ip+4]) | int(vm.curInsts[vm.ip+3])<<8 + vm.ip += 4 + hasMore := iterator.(Iterator).Next() + vm.stack[vm.sp-1] = Bool(hasMore) + + if hasMore { + vm.ip = truePos - 1 + } else { + vm.ip = falsePos - 1 + } case OpIterKey: iterator := vm.stack[vm.sp-1] val := iterator.(Iterator).Key() diff --git a/vm_test.go b/vm_test.go index 3f3f6de..8c90cd1 100644 --- a/vm_test.go +++ b/vm_test.go @@ -1716,6 +1716,27 @@ func TestVMForIn(t *testing.T) { nil, String("abde")) expectErrIs(t, `a := 1; for k,v in a {}`, nil, ErrNotIterable) + + // with else + expectRun(t, `var r = ""; for x in [] { r += string(x) } else { r += "@"}; r+="#"; return r`, nil, String("@#")) + expectRun(t, `var r = ""; for x in [1] { r += string(x) } else { r += "@"}; r+="#"; return r`, nil, String("1#")) + expectRun(t, `var r = ""; for x in [1,2] { r += string(x) } else { r += "@"}; r+="#"; return r`, nil, String("12#")) + expectRun(t, `var r = (;); + for k, v in bytes("abc") { + r = append(r, keyValue(k, char(v))) + } else { + r = append(r, keyValue("else", true)) + }; + r = append(r, keyValue("done", true)) + return string(r)`, nil, String("(;0=a, 1=b, 2=c, done)")) + expectRun(t, `var r = (;); + for k, v in bytes("") { + r = append(r, keyValue(k, char(v))) + } else { + r = append(r, keyValue("else", true)) + }; + r = append(r, keyValue("done", true)) + return string(r)`, nil, String("(;else, done)")) } func TestFor(t *testing.T) { @@ -3405,6 +3426,7 @@ func TestVMTailCallFreeVars(t *testing.T) { func TestVMCall(t *testing.T) { expectRun(t, `f := func() {}; return f()`, nil, Nil) + expectRun(t, `func f (a) { return a; }; return f(1)`, nil, Int(1)) expectRun(t, `f := func(a) { return a; }; return f(1)`, nil, Int(1)) expectRun(t, `f := func(a, b) { return [a, b]; }; return f(1, 2)`, nil, Array{Int(1), Int(2)}) expectErrIs(t, `f := func() { return; }; return f(1)`, nil, ErrWrongNumArguments)