Skip to content

Fixes for obscure function definition forms #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 13, 2022
Merged

Fixes for obscure function definition forms #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions src/parse_stream.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,14 +536,14 @@ function first_child_position(stream::ParseStream, pos::ParseStreamPosition)
c = 0
@assert pos.range_index > 0
parent = stream.ranges[pos.range_index]
i = pos.range_index-1
while i >= 1
if stream.ranges[i].first_token >= parent.first_token &&
(c == 0 || stream.ranges[i].first_token < stream.ranges[c].first_token) &&
!is_trivia(stream.ranges[i])
for i = pos.range_index-1:-1:1
if stream.ranges[i].first_token < parent.first_token
break
end
if (c == 0 || stream.ranges[i].first_token < stream.ranges[c].first_token) &&
!is_trivia(stream.ranges[i])
c = i
end
i -= 1
end

# Find first nontrivia token
Expand All @@ -558,7 +558,8 @@ function first_child_position(stream::ParseStream, pos::ParseStreamPosition)
if c != 0
if t != 0
if stream.ranges[c].first_token > t
return ParseStreamPosition(t, c-1)
# Need a child index strictly before `t`. `c=0` works.
return ParseStreamPosition(t, 0)
else
return ParseStreamPosition(stream.ranges[c].last_token, c)
end
Expand Down
47 changes: 25 additions & 22 deletions src/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,10 +308,11 @@ function was_eventually_call(ps::ParseState)
stream = ps.stream
p = position(ps)
while true
kb = peek_behind(stream, p).kind
if kb == K"call"
b = peek_behind(stream, p)
if b.kind == K"call"
return true
elseif kb == K"where" || kb == K"::"
elseif b.kind == K"where" || (b.kind == K"::" &&
has_flags(b.flags, INFIX_FLAG))
p = first_child_position(ps, p)
else
return false
Expand Down Expand Up @@ -1353,20 +1354,20 @@ function parse_factor_after(ps::ParseState)
end

# Parse type declarations and lambda syntax
# a::b ==> (:: a b)
# a::b ==> (::-i a b)
# a->b ==> (-> a b)
#
# flisp: parse-decl-with-initial-ex
function parse_decl_with_initial_ex(ps::ParseState, mark)
while peek(ps) == K"::"
# a::b::c ==> (:: (:: a b) c)
# a::b::c ==> (::-i (::-i a b) c)
bump(ps, TRIVIA_FLAG)
parse_where(ps, parse_call)
emit(ps, mark, K"::")
emit(ps, mark, K"::", INFIX_FLAG)
end
if peek(ps) == K"->"
# x -> y ==> (-> x y)
# a::b->c ==> (-> (:: a b) c)
# a::b->c ==> (-> (::-i a b) c)
bump(ps, TRIVIA_FLAG)
# -> is unusual: it binds tightly on the left and loosely on the right.
parse_eq_star(ps)
Expand All @@ -1393,7 +1394,7 @@ end
# parse syntactic unary operators
#
# &a ==> (& a)
# ::a ==> (:: a)
# ::a ==> (::-pre a)
# $a ==> ($ a)
#
# flisp: parse-unary-prefix
Expand All @@ -1418,7 +1419,9 @@ function parse_unary_prefix(ps::ParseState)
# $&a ==> ($ (& a))
parse_unary_prefix(ps)
end
emit(ps, mark, k)
# Only need PREFIX_OP_FLAG for ::
f = k == K"::" ? PREFIX_OP_FLAG : EMPTY_FLAGS
emit(ps, mark, k, f)
end
else
# .&(x,y) ==> (call .& x y)
Expand All @@ -1431,7 +1434,7 @@ function parse_identifier_or_interpolate(ps::ParseState)
mark = position(ps)
parse_unary_prefix(ps)
b = peek_behind(ps)
# export (x::T) ==> (export (error (:: x T)))
# export (x::T) ==> (export (error (::-i x T)))
# export outer ==> (export outer)
# export ($f) ==> (export ($ f))
ok = (b.is_leaf && (b.kind == K"Identifier" || is_operator(b.kind))) ||
Expand Down Expand Up @@ -1792,7 +1795,7 @@ function parse_resword(ps::ParseState)
m = position(ps)
n_subexprs = parse_comma_separated(ps, parse_eq_star)
kb = peek_behind(ps).kind
# let x::1 ; end ==> (let (block (:: x 1)) (block))
# let x::1 ; end ==> (let (block (::-i x 1)) (block))
# let x ; end ==> (let (block x) (block))
# let x=1,y=2 ; end ==> (let (block (= x 1) (= y 2) (block)))
# let x+=1 ; end ==> (let (block (+= x 1)) (block))
Expand Down Expand Up @@ -1897,7 +1900,7 @@ function parse_resword(ps::ParseState)
bump_closing_token(ps, K"end")
emit(ps, mark, K"abstract")
elseif word in KSet"struct mutable"
# struct A <: B \n a::X \n end ==> (struct false (<: A B) (block (:: a X)))
# struct A <: B \n a::X \n end ==> (struct false (<: A B) (block (::-i a X)))
# struct A \n a \n b \n end ==> (struct false A (block a b))
#v1.7: struct A const a end ==> (struct false A (block (error (const a))))
#v1.8: struct A const a end ==> (struct false A (block (const a)))
Expand Down Expand Up @@ -2104,7 +2107,7 @@ function parse_function_signature(ps::ParseState, is_function::Bool)
parsed_call = opts.parsed_call
if is_anon_func
# function (x) body end ==> (function (tuple x) (block body))
# function (x::f()) end ==> (function (tuple (:: x (call f))) (block))
# function (x::f()) end ==> (function (tuple (::-i x (call f))) (block))
# function (x,y) end ==> (function (tuple x y) (block))
# function (x=1) end ==> (function (tuple (= x 1)) (block))
# function (;x=1) end ==> (function (tuple (parameters (= x 1))) (block))
Expand All @@ -2117,8 +2120,8 @@ function parse_function_signature(ps::ParseState, is_function::Bool)
else
# function (A).f() end ==> (function (call (. A (quote f))) (block))
# function (:)() end ==> (function (call :) (block))
# function (x::T)() end ==> (function (call (:: x T)) (block))
# function (::T)() end ==> (function (call (:: T)) (block))
# function (x::T)() end ==> (function (call (::-i x T)) (block))
# function (::T)() end ==> (function (call (::-pre T)) (block))
end
else
parse_unary_prefix(ps)
Expand All @@ -2134,7 +2137,7 @@ function parse_function_signature(ps::ParseState, is_function::Bool)
# function \n f() end ==> (function (call f) (block))
# function $f() end ==> (function (call ($ f)) (block))
# function (:)() end ==> (function (call :) (block))
# function (::Type{T})(x) end ==> (function (call (:: (curly Type T)) x) (block))
# function (::Type{T})(x) end ==> (function (call (::-pre (curly Type T)) x) (block))
end
end
end
Expand All @@ -2155,28 +2158,28 @@ function parse_function_signature(ps::ParseState, is_function::Bool)
end
if is_function && peek(ps) == K"::"
# Function return type
# function f()::T end ==> (function (:: (call f) T) (block))
# function f()::g(T) end ==> (function (:: (call f) (call g T)) (block))
# function f()::T end ==> (function (::-i (call f) T) (block))
# function f()::g(T) end ==> (function (::-i (call f) (call g T)) (block))
bump(ps, TRIVIA_FLAG)
parse_call(ps)
emit(ps, mark, K"::")
emit(ps, mark, K"::", INFIX_FLAG)
end
if peek(ps) == K"where"
# Function signature where syntax
# function f() where {T} end ==> (function (where (call f) T) (block))
# function f() where T end ==> (function (where (call f) T) (block))
parse_where_chain(ps, mark)
end
# function f()::S where T end ==> (function (where (:: (call f) S) T) (block))
# function f()::S where T end ==> (function (where (::-i (call f) S) T) (block))
#
# Ugly cases for compat where extra parentheses existed and we've
# already parsed at least the call part of the signature
#
# function (f() where T) end ==> (function (where (call f) T) (block))
# function (f()) where T end ==> (function (where (call f) T) (block))
# function (f() where T) where U end ==> (function (where (where (call f) T) U) (block))
# function (f()::S) end ==> (function (:: (call f) S) (block))
# function ((f()::S) where T) end ==> (function (where (:: (call f) S) T) (block))
# function (f()::S) end ==> (function (::-i (call f) S) (block))
# function ((f()::S) where T) end ==> (function (where (::-i (call f) S) T) (block))
#
# TODO: Warn for use of parens? The precedence of `::` and
# `where` don't work inside parens so this is a bit of a syntax
Expand Down
44 changes: 23 additions & 21 deletions test/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,8 @@ tests = [
],
JuliaSyntax.parse_unary => [
":T" => "(quote T)"
"in::T" => "(:: in T)"
"isa::T" => "(:: isa T)"
"in::T" => "(::-i in T)"
"isa::T" => "(::-i isa T)"
"-2^x" => "(call-pre - (call-i 2 ^ x))"
"-2[1, 3]" => "(call-pre - (ref 2 1 3))"
# signed literals
Expand All @@ -197,7 +197,7 @@ tests = [
# Standalone non-dotted operators
"+)" => "+"
# Call with type parameters or non-unary prefix call
"+{T}(x::T)" => "(call (curly + T) (:: x T))"
"+{T}(x::T)" => "(call (curly + T) (::-i x T))"
"*(x)" => "(call * x)"
".*(x)" => "(call .* x)"
# Prefix function calls for operators which are both binary and unary
Expand Down Expand Up @@ -240,19 +240,19 @@ tests = [
"x^y" => "(call-i x ^ y)"
"x^y^z" => "(call-i x ^ (call-i y ^ z))"
"x .^ y" => "(dotcall-i x ^ y)"
"begin x end::T" => "(:: (block x) T)"
"begin x end::T" => "(::-i (block x) T)"
# parse_decl_with_initial_ex
"a::b" => "(:: a b)"
"a::b" => "(::-i a b)"
"a->b" => "(-> a b)"
"a::b::c" => "(:: (:: a b) c)"
"a::b->c" => "(-> (:: a b) c)"
"a::b::c" => "(::-i (::-i a b) c)"
"a::b->c" => "(-> (::-i a b) c)"
],
JuliaSyntax.parse_unary_subtype => [
"<: )" => "<:"
"<: \n" => "<:"
"<: =" => "<:"
"<:{T}(x::T)" => "(call (curly <: T) (:: x T))"
"<:(x::T)" => "(<:-pre (:: x T))"
"<:{T}(x::T)" => "(call (curly <: T) (::-i x T))"
"<:(x::T)" => "(<:-pre (::-i x T))"
"<: x" => "(<:-pre x)"
"<: A where B" => "(<:-pre (where A B))"
# Really for parse_where
Expand All @@ -268,7 +268,7 @@ tests = [
"&)" => "&"
"\$\n" => "\$"
"&a" => "(& a)"
"::a" => "(:: a)"
"::a" => "(::-pre a)"
"\$a" => "(\$ a)"
"\$\$a" => "(\$ (\$ a))"
],
Expand Down Expand Up @@ -417,7 +417,7 @@ tests = [
"let x=1\n end" => "(let (block (= x 1)) (block))" => Expr(:let, Expr(:(=), :x, 1), Expr(:block))
"let x=1 ; end" => "(let (block (= x 1)) (block))" => Expr(:let, Expr(:(=), :x, 1), Expr(:block))
"let x ; end" => "(let (block x) (block))" => Expr(:let, :x, Expr(:block))
"let x::1 ; end" => "(let (block (:: x 1)) (block))" => Expr(:let, Expr(:(::), :x, 1), Expr(:block))
"let x::1 ; end" => "(let (block (::-i x 1)) (block))" => Expr(:let, Expr(:(::), :x, 1), Expr(:block))
"let x=1,y=2 end" => "(let (block (= x 1) (= y 2)) (block))" => Expr(:let, Expr(:block, Expr(:(=), :x, 1), Expr(:(=), :y, 2)), Expr(:block))
"let x+=1 ; end" => "(let (block (+= x 1)) (block))" => Expr(:let, Expr(:block, Expr(:+=, :x, 1)), Expr(:block))
"let ; end" => "(let (block) (block))" => Expr(:let, Expr(:block), Expr(:block))
Expand All @@ -436,7 +436,7 @@ tests = [
"primitive type A \$N end" => "(primitive A (\$ N))"
"primitive type A <: B \n 8 \n end" => "(primitive (<: A B) 8)"
# struct
"struct A <: B \n a::X \n end" => "(struct false (<: A B) (block (:: a X)))" => Expr(:struct, false, Expr(:<:, :A, :B), Expr(:block, Expr(:(::), :a, :X)))
"struct A <: B \n a::X \n end" => "(struct false (<: A B) (block (::-i a X)))" => Expr(:struct, false, Expr(:<:, :A, :B), Expr(:block, Expr(:(::), :a, :X)))
"struct A \n a \n b \n end" => "(struct false A (block a b))" => Expr(:struct, false, :A, Expr(:block, :a, :b))
"mutable struct A end" => "(struct true A (block))"
((v=v"1.8",), "struct A const a end") => "(struct false A (block (const a)))" => Expr(:struct, false, :A, Expr(:block, Expr(:const, :a)))
Expand Down Expand Up @@ -466,7 +466,7 @@ tests = [
"export +, ==" => "(export + ==)" => Expr(:export, :+, :(==))
"export \n a" => "(export a)" => Expr(:export, :a)
"export \$a, \$(a*b)" => "(export (\$ a) (\$ (call-i a * b)))" => Expr(:export, Expr(:$, :a), Expr(:$, Expr(:call, :*, :a, :b)))
"export (x::T)" => "(export (error (:: x T)))"
"export (x::T)" => "(export (error (::-i x T)))"
"export outer" => "(export outer)" => Expr(:export, :outer)
"export (\$f)" => "(export (\$ f))" => Expr(:export, Expr(:$, :f))
],
Expand Down Expand Up @@ -517,15 +517,17 @@ tests = [
"function ()(x) end" => "(function (call (tuple) x) (block))"
"function (A).f() end" => "(function (call (. A (quote f))) (block))"
"function (:)() end" => "(function (call :) (block))"
"function (x::T)() end"=> "(function (call (:: x T)) (block))"
"function (::T)() end" => "(function (call (:: T)) (block))"
"function (x::T)() end"=> "(function (call (::-i x T)) (block))"
"function (::g(x))() end" => "(function (call (::-pre (call g x))) (block))"
"function (f::T{g(i)})() end" => "(function (call (::-i f (curly T (call g i)))) (block))"
"function (::T)() end" => "(function (call (::-pre T)) (block))"
"function begin() end" => "(function (call (error begin)) (block))"
"function f() end" => "(function (call f) (block))"
"function type() end" => "(function (call type) (block))"
"function \n f() end" => "(function (call f) (block))"
"function \$f() end" => "(function (call (\$ f)) (block))"
"function (:)() end" => "(function (call :) (block))"
"function (::Type{T})(x) end" => "(function (call (:: (curly Type T)) x) (block))"
"function (::Type{T})(x) end" => "(function (call (::-pre (curly Type T)) x) (block))"
# Function/macro definition with no methods
"function f end" => "(function f)"
"function f \n\n end" => "(function f)"
Expand All @@ -536,18 +538,18 @@ tests = [
"function f{T}() end" => "(function (call (curly f T)) (block))"
"function A.f() end" => "(function (call (. A (quote f))) (block))"
"function f body end" => "(function (error f) (block body))"
"function f()::T end" => "(function (:: (call f) T) (block))"
"function f()::g(T) end" => "(function (:: (call f) (call g T)) (block))"
"function f()::T end" => "(function (::-i (call f) T) (block))"
"function f()::g(T) end" => "(function (::-i (call f) (call g T)) (block))"
"function f() where {T} end" => "(function (where (call f) T) (block))"
"function f() where T end" => "(function (where (call f) T) (block))"
"function f()::S where T end" => "(function (where (:: (call f) S) T) (block))"
"function f()::S where T end" => "(function (where (::-i (call f) S) T) (block))"
# Ugly cases for compat where extra parentheses existed and we've
# already parsed at least the call part of the signature
"function (f() where T) end" => "(function (where (call f) T) (block))" => Expr(:function, Expr(:where, Expr(:call, :f), :T), Expr(:block))
"function (f()) where T end" => "(function (where (call f) T) (block))"
"function (f() where T) where U end" => "(function (where (where (call f) T) U) (block))"
"function (f()::S) end"=> "(function (:: (call f) S) (block))" => Expr(:function, Expr(:(::), Expr(:call, :f), :S), Expr(:block))
"function ((f()::S) where T) end" => "(function (where (:: (call f) S) T) (block))"
"function (f()::S) end"=> "(function (::-i (call f) S) (block))" => Expr(:function, Expr(:(::), Expr(:call, :f), :S), Expr(:block))
"function ((f()::S) where T) end" => "(function (where (::-i (call f) S) T) (block))"
# body
"function f() \n a \n b end" => "(function (call f) (block a b))"
"function f() end" => "(function (call f) (block))"
Expand Down