Skip to content

Commit

Permalink
parser: fix reporting of errors and parsing of decimals with leading …
Browse files Browse the repository at this point in the history 
…zeros
  • Loading branch information
@ncw
ncw committed Jun 4, 2015
1 parent 60d12b3 commit 927e70d
Show file tree
Hide file tree
Showing 8 changed files with 408 additions and 318 deletions.
2 changes: 0 additions & 2 deletions notes.txt
View file
Expand Up @@ -9,8 +9,6 @@ Limitations
* string keys only in dictionaries
* ints only 64 bit

FIXME parsing 00007 works fine whereas it should be throwing an error

FIXME interesting crash with compiling `int("42"), sausage=11)`
Compile error: SystemError: [interface conversion: *ast.Call is not ast.SetCtxer: missing method SetCtx]

Expand Down
12 changes: 6 additions & 6 deletions parser/grammar.y
View file
Expand Up @@ -1335,7 +1335,7 @@ comp_op:
| LTGT
{
// panic("FIXME no coverage")
yylex.Error("Invalid syntax")
yylex.(*yyLex).SyntaxError("Invalid syntax")
}
| PLINGEQ
{
Expand Down Expand Up @@ -1495,14 +1495,14 @@ strings:
case py.String:
$$ = a + b
default:
yylex.Error("SyntaxError: cannot mix string and nonstring literals")
yylex.(*yyLex).SyntaxError("cannot mix string and nonstring literals")
}
case py.Bytes:
switch b := $2.(type) {
case py.Bytes:
$$ = append(a, b...)
default:
yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
yylex.(*yyLex).SyntaxError("cannot mix bytes and nonbytes literals")
}
}
}
Expand Down Expand Up @@ -1823,7 +1823,7 @@ arglist:
call := $1
call.Starargs = $3
if len($4.Args) != 0 {
yylex.Error("SyntaxError: only named arguments may follow *expression")
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
}
call.Keywords = append(call.Keywords, $4.Keywords...)
$$ = call
Expand All @@ -1834,7 +1834,7 @@ arglist:
call.Starargs = $3
call.Kwargs = $7
if len($4.Args) != 0 {
yylex.Error("SyntaxError: only named arguments may follow *expression")
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
}
call.Keywords = append(call.Keywords, $4.Keywords...)
$$ = call
Expand Down Expand Up @@ -1868,7 +1868,7 @@ argument:
if name, ok := test.(*ast.Name); ok {
$$.Keywords = []*ast.Keyword{&ast.Keyword{Pos: name.Pos, Arg: name.Id, Value: $3}}
} else {
yylex.Error("SyntaxError: keyword can't be an expression")
yylex.(*yyLex).SyntaxError("keyword can't be an expression")
}
}

Expand Down
574 changes: 292 additions & 282 deletions parser/grammar_data_test.go
View file

Large diffs are not rendered by default.

32 changes: 26 additions & 6 deletions parser/grammar_test.go
View file
Expand Up @@ -7,24 +7,44 @@ import (
"testing"

"github.com/ncw/gpython/ast"
"github.com/ncw/gpython/py"
)

var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4")

// FIXME test pos is correct

// FIXME add tests to test the error cases

func TestGrammar(t *testing.T) {
SetDebug(*debugLevel)
for _, test := range grammarTestData {
Ast, err := ParseString(test.in, test.mode)
if err != nil {
t.Errorf("Parse(%q) returned error: %v", test.in, err)
if test.exceptionType == nil {
t.Errorf("%s: Got exception %v when not expecting one", test.in, err)
return
} else if exc, ok := err.(*py.Exception); !ok {
t.Errorf("%s: Got non python exception %T %v", test.in, err, err)
return
} else if exc.Type() != test.exceptionType {
t.Errorf("%s: want exception type %v got %v", test.in, test.exceptionType, exc.Type())
return
} else if exc.Type() != test.exceptionType {
t.Errorf("%s: want exception type %v got %v", test.in, test.exceptionType, exc.Type())
return
} else {
msg := string(exc.Args.(py.Tuple)[0].(py.String))
if msg != test.errString {
t.Errorf("%s: want exception text %q got %q", test.in, test.errString, msg)
}
}
} else {
out := ast.Dump(Ast)
if out != test.out {
t.Errorf("Parse(%q)\nwant> %q\n got> %q\n", test.in, test.out, out)
if test.exceptionType != nil {
t.Errorf("%s: expecting exception %q", test.in, test.errString)
} else {
out := ast.Dump(Ast)
if out != test.out {
t.Errorf("Parse(%q)\nwant> %q\n got> %q\n", test.in, test.out, out)
}
}
}
}
Expand Down
35 changes: 22 additions & 13 deletions parser/lexer.go
View file
Expand Up @@ -90,7 +90,7 @@ func (x *yyLex) refill() {
x.eof = true
default:
x.eof = true
x.Errorf("Error reading input: %v", err)
x.SyntaxErrorf("Error reading input: %v", err)
}
// If this is exec input, add a newline to the end of the
// string if there isn't one already.
Expand Down Expand Up @@ -129,7 +129,7 @@ func countIndent(s string) int {
// a b
indent += tabSize - (indent & (tabSize - 1))
default:
panic("bad indent")
panic(py.ExceptionNewf(py.IndentationError, "unexpected indent"))
}

}
Expand Down Expand Up @@ -399,7 +399,7 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
goto foundIndent
}
}
x.Error("Inconsistent indent")
x.SyntaxError("Inconsistent indent")
return eof
foundIndent:
x.indentStack = x.indentStack[:len(x.indentStack)-1]
Expand Down Expand Up @@ -489,7 +489,7 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
}

// Nothing we recognise found
x.Error("invalid syntax")
x.SyntaxError("invalid syntax")
return eof
case checkEof:
if x.eof {
Expand Down Expand Up @@ -683,8 +683,9 @@ isNumber:
value = py.Complex(complex(0, f))
} else {
// Discard numbers with leading 0 except all 0s
if illegalDecimalInteger.FindString(x.line) != "" {
x.Error("illegal decimal with leading zero")
if illegalDecimalInteger.FindString(s) != "" {
// FIXME where is this error going in the grammar?
x.SyntaxError("illegal decimal with leading zero")
return eofError, nil
}
value, err = py.IntFromString(s, 10)
Expand Down Expand Up @@ -806,12 +807,12 @@ found:
}
}
if !multiLineString {
x.Errorf("Unterminated %sx%s string", stringEnd, stringEnd)
x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
return eofError, nil
}
readMore:
if x.eof {
x.Errorf("Unterminated %sx%s string", stringEnd, stringEnd)
x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
return eofError, nil
}
x.refill()
Expand All @@ -821,7 +822,7 @@ foundEndOfString:
var err error
buf, err = DecodeEscape(buf, byteString)
if err != nil {
x.Errorf("Decode error: %v", err)
x.SyntaxErrorf("Decode error: %v", err)
return eofError, nil
}
}
Expand All @@ -834,23 +835,31 @@ foundEndOfString:
// The parser calls this method on a parse error.
func (x *yyLex) Error(s string) {
x.error = true
x.errorString = s
if yyDebug >= 1 {
log.Printf("Parse error: %s", s)
log.Printf("Parse buffer %q", x.line)
log.Printf("State %#v", x)
}
}

// The parser calls this method on a parse error.
func (x *yyLex) SyntaxError(s string) {
x.errorString = s
x.Error(s)
}

// Call this to write formatted errors
func (x *yyLex) Errorf(format string, a ...interface{}) {
x.Error(fmt.Sprintf(format, a...))
func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) {
x.SyntaxError(fmt.Sprintf(format, a...))
}

// Returns an python error for the current yyLex
func (x *yyLex) ErrorReturn() error {
if x.error {
return py.ExceptionNewf(py.SyntaxError, "Syntax Error: %s", x.errorString)
if x.errorString == "" {
x.errorString = "invalid syntax"
}
return py.ExceptionNewf(py.SyntaxError, "%s", x.errorString)
}
return nil
}
Expand Down
27 changes: 26 additions & 1 deletion parser/lexer_test.go
View file
Expand Up @@ -233,6 +233,31 @@ func TestLex(t *testing.T) {
{DEDENT, nil},
{ENDMARKER, nil},
}},
{"1", "", "eval", LexTokens{
{EVAL_INPUT, nil},
{NUMBER, py.Int(1)},
{ENDMARKER, nil},
}},
{"01", "illegal decimal with leading zero", "eval", LexTokens{
{EVAL_INPUT, nil},
}},
{"1", "", "exec", LexTokens{
{FILE_INPUT, nil},
{NUMBER, py.Int(1)},
{NEWLINE, nil},
{ENDMARKER, nil},
}},
{"1 2 3", "", "exec", LexTokens{
{FILE_INPUT, nil},
{NUMBER, py.Int(1)},
{NUMBER, py.Int(2)},
{NUMBER, py.Int(3)},
{NEWLINE, nil},
{ENDMARKER, nil},
}},
{"01", "illegal decimal with leading zero", "exec", LexTokens{
{FILE_INPUT, nil},
}},
{"1\n 2\n 3\n4\n", "", "exec", LexTokens{
{FILE_INPUT, nil},
{NUMBER, py.Int(1)},
Expand Down Expand Up @@ -343,7 +368,7 @@ func TestLex(t *testing.T) {
errString = err.Error()
}
if test.errString != "" {
test.errString = "SyntaxError: [Syntax Error: " + test.errString + "]"
test.errString = "SyntaxError: [" + test.errString + "]"
}
if errString != test.errString || !lts.Eq(test.lts) {
t.Errorf("Lex(%q) expecting (%v, %q) got (%v, %q)", test.in, test.lts, test.errString, lts, errString)
Expand Down
32 changes: 30 additions & 2 deletions parser/make_grammar_test.py
View file
Expand Up @@ -24,6 +24,9 @@
("b'abc'", "eval"),
("b'abc' b'''123'''", "eval"),
("1234", "eval"),
("01234", "eval", SyntaxError, "illegal decimal with leading zero"),
("1234d", "eval", SyntaxError, "invalid syntax"),
("1234d", "exec", SyntaxError),
("0x1234", "eval"),
("12.34", "eval"),
("1,", "eval"),
Expand Down Expand Up @@ -359,6 +362,7 @@

# Assign
("a = b", "exec"),
("a = 007", "exec", SyntaxError, "illegal decimal with leading zero"),
("a = b = c", "exec"),
("a, b = 1, 2", "exec"),
("a, b = c, d = 1, 2", "exec"),
Expand Down Expand Up @@ -466,13 +470,37 @@ def main():
package parser
import (
"github.com/ncw/gpython/py"
)
var grammarTestData = []struct {
in string
mode string
out string
exceptionType *py.Type
errString string
}{"""]
for source, mode in inp:
out.append('{"%s", "%s", "%s"},' % (escape(source), mode, escape(dump(source, mode))))
for x in inp:
source, mode = x[:2]
if len(x) > 2:
exc = x[2]
errString = (x[3] if len(x) > 3 else "")
try:
dump(source, mode)
except exc as e:
error = e.msg
else:
raise ValueError("Expecting exception %s" % exc)
if errString != "":
error = errString # override error string
dmp = ""
exc_name = "py.%s" % exc.__name__
else:
dmp = dump(source, mode)
exc_name = "nil"
error = ""
out.append('{"%s", "%s", "%s", %s, "%s"},' % (escape(source), mode, escape(dmp), exc_name, escape(error)))
out.append("}")
print("Writing %s" % path)
with open(path, "w") as f:
Expand Down
12 changes: 6 additions & 6 deletions parser/y.go
View file
Expand Up @@ -2063,7 +2063,7 @@ yydefault:
//line grammar.y:1336
{
// panic("FIXME no coverage")
yylex.Error("Invalid syntax")
yylex.(*yyLex).SyntaxError("Invalid syntax")
}
case 209:
//line grammar.y:1341
Expand Down Expand Up @@ -2234,14 +2234,14 @@ yydefault:
case py.String:
yyVAL.obj = a + b
default:
yylex.Error("SyntaxError: cannot mix string and nonstring literals")
yylex.(*yyLex).SyntaxError("cannot mix string and nonstring literals")
}
case py.Bytes:
switch b := yyS[yypt-0].obj.(type) {
case py.Bytes:
yyVAL.obj = append(a, b...)
default:
yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
yylex.(*yyLex).SyntaxError("cannot mix bytes and nonbytes literals")
}
}
}
Expand Down Expand Up @@ -2582,7 +2582,7 @@ yydefault:
call := yyS[yypt-3].call
call.Starargs = yyS[yypt-1].expr
if len(yyS[yypt-0].call.Args) != 0 {
yylex.Error("SyntaxError: only named arguments may follow *expression")
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
}
call.Keywords = append(call.Keywords, yyS[yypt-0].call.Keywords...)
yyVAL.call = call
Expand All @@ -2594,7 +2594,7 @@ yydefault:
call.Starargs = yyS[yypt-4].expr
call.Kwargs = yyS[yypt-0].expr
if len(yyS[yypt-3].call.Args) != 0 {
yylex.Error("SyntaxError: only named arguments may follow *expression")
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
}
call.Keywords = append(call.Keywords, yyS[yypt-3].call.Keywords...)
yyVAL.call = call
Expand Down Expand Up @@ -2628,7 +2628,7 @@ yydefault:
if name, ok := test.(*ast.Name); ok {
yyVAL.call.Keywords = []*ast.Keyword{&ast.Keyword{Pos: name.Pos, Arg: name.Id, Value: yyS[yypt-0].expr}}
} else {
yylex.Error("SyntaxError: keyword can't be an expression")
yylex.(*yyLex).SyntaxError("keyword can't be an expression")
}
}
case 303:
Expand Down

0 comments on commit 927e70d

Please sign in to comment.