Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

improve loop checks #208

Merged
merged 2 commits into from Oct 6, 2020
Merged

improve loop checks #208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
df42ded
improve loop checks
ZacLN Sep 30, 2020
e2236fb
missed a file...
ZacLN Sep 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 4 additions & 9 deletions src/CSTParser.jl
View file
Open in desktop
Expand Up @@ -215,12 +215,8 @@ function parse(ps::ParseState, cont=false)
push!(top, mLITERAL(ps.nt.startbyte, ps.nt.startbyte, "", Tokens.NOTHING))
end

safetytrip = 0
prevpos = position(ps)
while kindof(ps.nt) !== Tokens.ENDMARKER
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
curr_line = ps.nt.startpos[1]
ret = parse_doc(ps)
if _continue_doc_parse(ps, ret)
Expand All @@ -237,6 +233,7 @@ function parse(ps::ParseState, cont=false)
push!(top, ret)
end
last_line = curr_line
prevpos = loop_check(ps, prevpos)
end
else
if kindof(ps.nt) === Tokens.WHITESPACE || kindof(ps.nt) === Tokens.COMMENT
Expand All @@ -252,14 +249,12 @@ function parse(ps::ParseState, cont=false)
if kindof(ps.ws) == SemiColonWS
top = EXPR(TopLevel, EXPR[top])
safetytrip = 0
prevpos = position(ps)
while kindof(ps.ws) == SemiColonWS && ps.nt.startpos[1] == last_line && kindof(ps.nt) != Tokens.ENDMARKER
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
ret = parse_doc(ps)
push!(top, ret)
last_line = ps.nt.startpos[1]
prevpos = loop_check(ps, prevpos)
end
end
else
Expand Down
68 changes: 22 additions & 46 deletions src/components/internals.jl
View file
Open in desktop
Expand Up @@ -5,13 +5,8 @@ Continue parsing statements until an element of `closers` is hit (usually
`end`). Statements are grouped in a `Block` EXPR.
"""
function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.END,), docable=false)
safetytrip = 0
prevpos = position(ps)
while kindof(ps.nt) ∉ closers # loop until an expected closer is hit
safetytrip += 1
if safetytrip > 10_000
# Not needed, we take a take a token or break the loop for each branch.
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
if kindof(ps.nt) ∈ term_c # error handling if an unexpected closer is hit
if kindof(ps.nt) === Tokens.ENDMARKER
break
Expand All @@ -32,6 +27,7 @@ function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.E
end
push!(ret, a)
end
prevpos = loop_check(ps, prevpos)
end
return ret
end
Expand Down Expand Up @@ -70,15 +66,12 @@ function parse_iterators(ps::ParseState, allowfilter=false)
arg = parse_iterator(ps)
if iscomma(ps.nt) # we've hit a comma separated list of iterators.
arg = EXPR(Block, EXPR[arg])
safetytrip = 0
prevpos = position(ps)
while iscomma(ps.nt)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
accept_comma(ps, arg)
nextarg = parse_iterator(ps)
push!(arg, nextarg)
prevpos = loop_check(ps, prevpos)
end
end

Expand Down Expand Up @@ -147,8 +140,8 @@ Parses a comma separated list, optionally allowing for conversion of
assignment (`=`) expressions to `Kw`.
"""
function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=false, istuple=false)
prevpos = position(ps)
@nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !closer(ps)
starting_offset = ps.t.startbyte
a = parse_expression(ps)
if kw && _do_kw_convert(ps, a)
a = _kw_convert(a)
Expand All @@ -159,10 +152,7 @@ function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=fals
else# if kindof(ps.ws) == SemiColonWS
break
end
if ps.t.startbyte <= starting_offset
# We've not progressed over the course of a loop.
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
prevpos = loop_check(ps, prevpos)
end
if istuple && length(args) > 2
block = false
Expand All @@ -179,14 +169,11 @@ function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=fals
a = @nocloser ps :newline @closer ps :comma @nocloser ps :inwhere parse_expression(ps)
if block && !(length(args) == 1 && ispunctuation(args[1])) && !is_splat(last(args)) && !(istuple && iscomma(ps.nt))
args1 = EXPR[pop!(args), a]
safetytrip = 0
prevpos = position(ps)
@nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
a = parse_expression(ps)
push!(args1, a)
prevpos = loop_check(ps, prevpos)
end
body = EXPR(Block, args1)
push!(args, body)
Expand All @@ -206,12 +193,8 @@ Parses parameter arguments for a function call (e.g. following a semicolon).
"""
function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR}=EXPR[]; usekw=true)
isfirst = isempty(args1)
safetytrip = 0
prevpos = position(ps)
@nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !isfirst || (@nocloser ps :semicolon !closer(ps))
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
if isfirst
a = parse_expression(ps)
else
Expand All @@ -232,6 +215,11 @@ function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR
if kindof(ps.ws) == SemiColonWS
parse_parameters(ps, args1; usekw=usekw)
end
if isfirst
prevpos = loop_check(ps, prevpos)
else
prevpos = position(ps)
end
isfirst = true
end
if !isempty(args1)
Expand All @@ -256,15 +244,12 @@ function parse_macrocall(ps::ParseState)

# Handle cases with @ at start of dotted expressions
if kindof(ps.nt) === Tokens.DOT && isemptyws(ps.ws)
safetytrip = 0
prevpos = position(ps)
while kindof(ps.nt) === Tokens.DOT
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
op = mOPERATOR(next(ps))
nextarg = mIDENTIFIER(next(ps))
mname = mBinaryOpCall(mname, op, EXPR(Quotenode, EXPR[nextarg]))
prevpos = loop_check(ps, prevpos)
end
end

Expand All @@ -275,12 +260,8 @@ function parse_macrocall(ps::ParseState)
else
args = EXPR[mname]
insquare = ps.closer.insquare
safetytrip = 0
prevpos = position(ps)
@default ps while !closer(ps)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
if insquare
a = @closer ps :insquare @closer ps :inmacro @closer ps :ws @closer ps :wsop parse_expression(ps)
else
Expand All @@ -290,6 +271,7 @@ function parse_macrocall(ps::ParseState)
if insquare && kindof(ps.nt) === Tokens.FOR
break
end
prevpos = loop_check(ps, prevpos)
end
return EXPR(MacroCall, args)
end
Expand Down Expand Up @@ -325,12 +307,8 @@ Helper function for parsing import/using statements.
function parse_dot_mod(ps::ParseState, is_colon=false)
args = EXPR[]

safetytrip = 0
prevpos = position(ps)
while kindof(ps.nt) === Tokens.DOT || kindof(ps.nt) === Tokens.DDOT || kindof(ps.nt) === Tokens.DDDOT
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
d = mOPERATOR(next(ps))
trailing_ws = d.fullspan - d.span
if is_dot(d)
Expand All @@ -343,6 +321,7 @@ function parse_dot_mod(ps::ParseState, is_colon=false)
push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
end
prevpos = loop_check(ps, prevpos)
end

# import/export ..
Expand All @@ -353,12 +332,8 @@ function parse_dot_mod(ps::ParseState, is_colon=false)
# end
# end

safetytrip = 0
prevpos = position(ps)
while true
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
if kindof(ps.nt) === Tokens.AT_SIGN
at = mPUNCTUATION(next(ps))
a = INSTANCE(next(ps))
Expand Down Expand Up @@ -388,6 +363,7 @@ function parse_dot_mod(ps::ParseState, is_colon=false)
else
break
end
prevpos = loop_check(ps, prevpos)
end
args
end
42 changes: 12 additions & 30 deletions src/components/keywords.jl
View file
Open in desktop
Expand Up @@ -164,27 +164,21 @@ function parse_imports(ps::ParseState)

arg = parse_dot_mod(ps, true)
append!(ret, arg)
safetytrip = 0
prevpos = position(ps)
while iscomma(ps.nt)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
accept_comma(ps, ret)
arg = parse_dot_mod(ps, true)
append!(ret, arg)
prevpos = loop_check(ps, prevpos)
end
else
ret = EXPR(kwt, vcat(kw, arg))
safetytrip = 0
prevpos = position(ps)
while iscomma(ps.nt)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
accept_comma(ps, ret)
arg = parse_dot_mod(ps)
append!(ret, arg)
prevpos = loop_check(ps, prevpos)
end
end

Expand All @@ -195,15 +189,12 @@ function parse_export(ps::ParseState)
args = EXPR[mKEYWORD(ps)]
append!(args, parse_dot_mod(ps))

safetytrip = 0
prevpos = position(ps)
while iscomma(ps.nt)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
push!(args, mPUNCTUATION(next(ps)))
arg = parse_dot_mod(ps)[1]
push!(args, arg)
prevpos = loop_check(ps, prevpos)
end

return EXPR(Export, args)
Expand All @@ -225,13 +216,10 @@ function parse_blockexpr_sig(ps::ParseState, head)
if convertsigtotuple(sig)
sig = EXPR(TupleH, sig.args)
end
safetytrip = 0
prevpos = position(ps)
while kindof(ps.nt) === Tokens.WHERE && kindof(ps.ws) != Tokens.NEWLINE_WS
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
sig = @closer ps :inwhere @closer ps :ws parse_operator_where(ps, sig, INSTANCE(next(ps)), false)
prevpos = loop_check(ps, prevpos)
end
return sig
elseif head === Let
Expand All @@ -241,35 +229,29 @@ function parse_blockexpr_sig(ps::ParseState, head)
arg = @closer ps :comma @closer ps :ws parse_expression(ps)
if iscomma(ps.nt) || !(is_wrapped_assignment(arg) || isidentifier(arg))
arg = EXPR(Block, EXPR[arg])
safetytrip = 0
prevpos = position(ps)
while iscomma(ps.nt)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
accept_comma(ps, arg)
startbyte = ps.nt.startbyte
nextarg = @closer ps :comma @closer ps :ws parse_expression(ps)
push!(arg, nextarg)
prevpos = loop_check(ps, prevpos)
end
end
return arg
end
elseif head === Do
sig = EXPR(TupleH, EXPR[])
safetytrip = 0
prevpos = position(ps)
@closer ps :comma @closer ps :block while !closer(ps)
safetytrip += 1
if safetytrip > 10_000
throw(CSTInfiniteLoop("Infinite loop at $ps"))
end
@closer ps :ws a = parse_expression(ps)
push!(sig, a)
if kindof(ps.nt) === Tokens.COMMA
accept_comma(ps, sig)
elseif @closer ps :ws closer(ps)
break
end
prevpos = loop_check(ps, prevpos)
end
return sig
elseif head === ModuleH || head === BareModule
Expand Down