diff --git a/README.md b/README.md index c968b916..ca5f6906 100644 --- a/README.md +++ b/README.md @@ -50,15 +50,16 @@ First, a source-ordered AST with `SyntaxNode` (`call-i` in the dump here means the `call` has the infix `-i` flag): ```julia -julia> using JuliaSyntax: JuliaSyntax, SyntaxNode, GreenNode +julia> using JuliaSyntax -julia> JuliaSyntax.parse(SyntaxNode, "(x + y)*z", filename="foo.jl") +julia> parsestmt(SyntaxNode, "(x + y)*z", filename="foo.jl") line:col│ tree │ file_name 1:1 │[call-i] │foo.jl - 1:2 │ [call-i] - 1:2 │ x - 1:4 │ + - 1:6 │ y + 1:1 │ [parens] + 1:2 │ [call-i] + 1:2 │ x + 1:4 │ + + 1:6 │ y 1:8 │ * 1:9 │ z ``` @@ -71,16 +72,17 @@ representation, despite being important for parsing. ```julia julia> text = "(x + y)*z" - greentree = JuliaSyntax.parse(GreenNode, text) + greentree = parsestmt(JuliaSyntax.GreenNode, text) 1:9 │[call] - 1:1 │ ( - 2:6 │ [call] - 2:2 │ Identifier ✔ - 3:3 │ Whitespace - 4:4 │ + ✔ - 5:5 │ Whitespace - 6:6 │ Identifier ✔ - 7:7 │ ) + 1:7 │ [parens] + 1:1 │ ( + 2:6 │ [call] + 2:2 │ Identifier ✔ + 3:3 │ Whitespace + 4:4 │ + ✔ + 5:5 │ Whitespace + 6:6 │ Identifier ✔ + 7:7 │ ) 8:8 │ * ✔ 9:9 │ Identifier ✔ ``` @@ -91,14 +93,15 @@ supplying the source text string: ```julia julia> show(stdout, MIME"text/plain"(), greentree, text) 1:9 │[call] - 1:1 │ ( "(" - 2:6 │ [call] - 2:2 │ Identifier ✔ "x" - 3:3 │ Whitespace " " - 4:4 │ + ✔ "+" - 5:5 │ Whitespace " " - 6:6 │ Identifier ✔ "y" - 7:7 │ ) ")" + 1:7 │ [parens] + 1:1 │ ( "(" + 2:6 │ [call] + 2:2 │ Identifier ✔ "x" + 3:3 │ Whitespace " " + 4:4 │ + ✔ "+" + 5:5 │ Whitespace " " + 6:6 │ Identifier ✔ "y" + 7:7 │ ) ")" 8:8 │ * ✔ "*" 9:9 │ Identifier ✔ "z" ``` @@ -106,7 +109,7 @@ julia> show(stdout, MIME"text/plain"(), greentree, text) Julia `Expr` can also be produced: ```julia -julia> JuliaSyntax.parse(Expr, "(x + y)*z") +julia> JuliaSyntax.parsestmt(Expr, "(x + y)*z") :((x + y) * z) ``` diff --git a/src/JuliaSyntax.jl b/src/JuliaSyntax.jl index 76b68675..3f1ad27a 100644 --- a/src/JuliaSyntax.jl +++ b/src/JuliaSyntax.jl @@ -1,5 +1,23 @@ module JuliaSyntax +# Conservative list of exports - only export the most common/useful things +# here. + +# Parsing. See also +# parse!(), ParseStream +export parsestmt, parseall, parseatom +# Tokenization +export tokenize, Token, untokenize +# Source file handling. See also +# highlight() sourcetext() source_line() source_location() +export SourceFile +# Expression heads/kinds. See also +# flags() and related predicates. +export @K_str, kind, head +# Syntax tree types. See also +# GreenNode +export SyntaxNode + # Helper utilities include("utils.jl") @@ -26,4 +44,5 @@ include("expr.jl") # Hooks to integrate the parser with Base include("hooks.jl") include("precompile.jl") + end diff --git a/src/hooks.jl b/src/hooks.jl index 47d7dc47..6805bb11 100644 --- a/src/hooks.jl +++ b/src/hooks.jl @@ -122,7 +122,7 @@ end # Debug log file for dumping parsed code const _debug_log = Ref{Union{Nothing,IO}}(nothing) -function _core_parser_hook(code, filename, lineno, offset, options) +function _core_parser_hook(code, filename::String, lineno::Int, offset::Int, options::Symbol) try # TODO: Check that we do all this input wrangling without copying the # code buffer @@ -144,8 +144,7 @@ function _core_parser_hook(code, filename, lineno, offset, options) seek(io, offset) stream = ParseStream(io) - rule = options === :all ? :toplevel : options - if rule === :statement || rule === :atom + if options === :statement || options === :atom # To copy the flisp parser driver: # * Parsing atoms consumes leading trivia # * Parsing statements consumes leading+trailing trivia @@ -157,8 +156,8 @@ function _core_parser_hook(code, filename, lineno, offset, options) return Core.svec(nothing, last_byte(stream)) end end - parse!(stream; rule=rule) - if rule === :statement + parse!(stream; rule=options) + if options === :statement bump_trivia(stream) end @@ -342,7 +341,7 @@ function _fl_parse_string(text::AbstractString, filename::AbstractString, ex, offset+1 end -# Convenience functions to mirror `JuliaSyntax.parse(Expr, ...)` in simple cases. +# Convenience functions to mirror `JuliaSyntax.parsestmt(Expr, ...)` in simple cases. fl_parse(::Type{Expr}, args...; kws...) = fl_parse(args...; kws...) fl_parseall(::Type{Expr}, args...; kws...) = fl_parseall(args...; kws...) diff --git a/src/parser_api.jl b/src/parser_api.jl index a22a6a25..bc45d22e 100644 --- a/src/parser_api.jl +++ b/src/parser_api.jl @@ -28,21 +28,25 @@ Base.display_error(io::IO, err::ParseError, bt) = Base.showerror(io, err, bt) """ - parse!(stream::ParseStream; rule=:toplevel) + parse!(stream::ParseStream; rule=:all) Parse Julia source code from a [`ParseStream`](@ref) object. Output tree data structures may be extracted from `stream` with the [`build_tree`](@ref) function. `rule` may be any of -* `:toplevel` (default) — parse a whole "file" of top level statements. In this +* `:all` (default) — parse a whole "file" of top level statements. In this mode, the parser expects to fully consume the input. * `:statement` — parse a single statement, or statements separated by semicolons. * `:atom` — parse a single syntax "atom": a literal, identifier, or parenthesized expression. """ -function parse!(stream::ParseStream; rule::Symbol=:toplevel) +function parse!(stream::ParseStream; rule::Symbol=:all) + if rule == :toplevel + Base.depwarn("Use of rule == :toplevel in parse!() is deprecated. use `rule=:all` instead.", :parse!) + rule = :all + end ps = ParseState(stream) - if rule === :toplevel + if rule === :all parse_toplevel(ps) elseif rule === :statement parse_stmts(ps) @@ -56,14 +60,14 @@ function parse!(stream::ParseStream; rule::Symbol=:toplevel) end """ - parse!(TreeType, io::IO; rule=:toplevel, version=VERSION) + parse!(TreeType, io::IO; rule=:all, version=VERSION) Parse Julia source code from a seekable `IO` object. The output is a tuple `(tree, diagnostics)`. When `parse!` returns, the stream `io` is positioned directly after the last byte which was consumed during parsing. """ function parse!(::Type{TreeType}, io::IO; - rule::Symbol=:toplevel, version=VERSION, kws...) where {TreeType} + rule::Symbol=:all, version=VERSION, kws...) where {TreeType} stream = ParseStream(io; version=version) parse!(stream; rule=rule) tree = build_tree(TreeType, stream; kws...) @@ -75,7 +79,7 @@ function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version= ignore_trivia=true, filename=nothing, first_line=1, ignore_errors=false, ignore_warnings=ignore_errors) where {T} stream = ParseStream(text, index; version=version) - if ignore_trivia && rule != :toplevel + if ignore_trivia && rule != :all bump_trivia(stream, skip_newlines=true) empty!(stream) end @@ -100,19 +104,22 @@ function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version= end _parse_docs = """ - parse(TreeType, text, [index]; - version=VERSION, - ignore_trivia=true, - filename=nothing, - ignore_errors=false, - ignore_warnings=ignore_errors) - - # Or, with the same arguments + # Parse a single expression/statement + parsestmt(TreeType, text, [index]; + version=VERSION, + ignore_trivia=true, + filename=nothing, + ignore_errors=false, + ignore_warnings=ignore_errors) + + # Parse all statements at top level (file scope) parseall(...) + + # Parse a single syntax atom parseatom(...) Parse Julia source code string `text` into a data structure of type `TreeType`. -`parse` parses a single Julia statement, `parseall` parses top level statements +`parsestmt` parses a single Julia statement, `parseall` parses top level statements at file scope and `parseatom` parses a single Julia identifier or other "syntax atom". @@ -136,16 +143,17 @@ parsing. To avoid exceptions due to warnings, use `ignore_warnings=true`. To also avoid exceptions due to errors, use `ignore_errors=true`. """ -parse(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:statement, true, T, text; kws...)[1] -parseall(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:toplevel, true, T, text; kws...)[1] -parseatom(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:atom, true, T, text; kws...)[1] +"$_parse_docs" +parsestmt(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:statement, true, T, text; kws...)[1] -@eval @doc $_parse_docs parse -@eval @doc $_parse_docs parseall -@eval @doc $_parse_docs parseatom +"$_parse_docs" +parseall(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:all, true, T, text; kws...)[1] -parse(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:statement, false, T, text, index; kws...) -parseall(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:toplevel, false, T, text, index; kws...) +"$_parse_docs" +parseatom(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:atom, true, T, text; kws...)[1] + +parsestmt(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:statement, false, T, text, index; kws...) +parseall(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:all, false, T, text, index; kws...) parseatom(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:atom, false, T, text, index; kws...) #------------------------------------------------------------------------------- @@ -178,7 +186,7 @@ This interface works on UTF-8 encoded string or buffer data only. """ function tokenize(text) ps = ParseStream(text) - parse!(ps, rule=:toplevel) + parse!(ps, rule=:all) ts = ps.tokens output_tokens = Token[] for i = 2:length(ts) @@ -198,3 +206,5 @@ end function untokenize(token::Token, text::Vector{UInt8}) text[token.range] end + +@deprecate parse parsestmt diff --git a/src/source_files.jl b/src/source_files.jl index 40214f37..203523e3 100644 --- a/src/source_files.jl +++ b/src/source_files.jl @@ -36,18 +36,18 @@ function SourceFile(; filename, kwargs...) end # Get line number of the given byte within the code -function source_line_index(source::SourceFile, byte_index) +function _source_line_index(source::SourceFile, byte_index) lineidx = searchsortedlast(source.line_starts, byte_index) return (lineidx < lastindex(source.line_starts)) ? lineidx : lineidx-1 end _source_line(source::SourceFile, lineidx) = lineidx + source.first_line - 1 -source_line(source::SourceFile, byte_index) = _source_line(source, source_line_index(source, byte_index)) +source_line(source::SourceFile, byte_index) = _source_line(source, _source_line_index(source, byte_index)) """ Get line number and character within the line at the given byte index. """ function source_location(source::SourceFile, byte_index) - lineidx = source_line_index(source, byte_index) + lineidx = _source_line_index(source, byte_index) i = source.line_starts[lineidx] column = 1 while i < byte_index @@ -63,7 +63,7 @@ Get byte range of the source line at byte_index, buffered by """ function source_line_range(source::SourceFile, byte_index; context_lines_before=0, context_lines_after=0) - lineidx = source_line_index(source, byte_index) + lineidx = _source_line_index(source, byte_index) fbyte = source.line_starts[max(lineidx-context_lines_before, 1)] lbyte = source.line_starts[min(lineidx+1+context_lines_after, end)] - 1 fbyte,lbyte diff --git a/test/benchmark.jl b/test/benchmark.jl index b7dc4e01..0cae3e07 100644 --- a/test/benchmark.jl +++ b/test/benchmark.jl @@ -15,7 +15,7 @@ end all_base_code = concat_base() -b_ParseStream = @benchmark JuliaSyntax.parse!(JuliaSyntax.ParseStream(all_base_code), rule=:toplevel) +b_ParseStream = @benchmark JuliaSyntax.parse!(JuliaSyntax.ParseStream(all_base_code), rule=:all) b_GreenNode = @benchmark JuliaSyntax.parseall(JuliaSyntax.GreenNode, all_base_code) b_SyntaxNode = @benchmark JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, all_base_code) b_Expr = @benchmark JuliaSyntax.parseall(Expr, all_base_code) @@ -30,5 +30,5 @@ b_Expr = @benchmark JuliaSyntax.parseall(Expr, all_base_code) # Allocs.clear() # stream = JuliaSyntax.ParseStream(text); # JuliaSyntax.peek(stream); -# Allocs.@profile sample_rate=1 JuliaSyntax.parse(stream) +# Allocs.@profile sample_rate=1 JuliaSyntax.parsestmt(stream) # PProf.Allocs.pprof() diff --git a/test/expr.jl b/test/expr.jl index 39d7edd4..31d466cc 100644 --- a/test/expr.jl +++ b/test/expr.jl @@ -10,7 +10,7 @@ @testset "Line numbers" begin @testset "Blocks" begin - @test parse(Expr, "begin a\nb\n\nc\nend") == + @test parsestmt(Expr, "begin a\nb\n\nc\nend") == Expr(:block, LineNumberNode(1), :a, @@ -19,7 +19,7 @@ LineNumberNode(4), :c, ) - @test parse(Expr, "begin end") == + @test parsestmt(Expr, "begin end") == Expr(:block, LineNumberNode(1) ) @@ -32,7 +32,7 @@ :b, ) - @test parse(Expr, "module A\n\nbody\nend") == + @test parsestmt(Expr, "module A\n\nbody\nend") == Expr(:module, true, :A, @@ -45,7 +45,7 @@ end @testset "Function definition lines" begin - @test parse(Expr, "function f()\na\n\nb\nend") == + @test parsestmt(Expr, "function f()\na\n\nb\nend") == Expr(:function, Expr(:call, :f), Expr(:block, @@ -56,7 +56,7 @@ :b, ) ) - @test parse(Expr, "f() = 1") == + @test parsestmt(Expr, "f() = 1") == Expr(:(=), Expr(:call, :f), Expr(:block, @@ -66,14 +66,14 @@ ) # function/macro without methods - @test parse(Expr, "function f end") == + @test parsestmt(Expr, "function f end") == Expr(:function, :f) - @test parse(Expr, "macro f end") == + @test parsestmt(Expr, "macro f end") == Expr(:macro, :f) end @testset "elseif" begin - @test parse(Expr, "if a\nb\nelseif c\n d\nend") == + @test parsestmt(Expr, "if a\nb\nelseif c\n d\nend") == Expr(:if, :a, Expr(:block, @@ -91,7 +91,7 @@ end @testset "No line numbers in for/let bindings" begin - @test parse(Expr, "for i=is, j=js\nbody\nend") == + @test parsestmt(Expr, "for i=is, j=js\nbody\nend") == Expr(:for, Expr(:block, Expr(:(=), :i, :is), @@ -102,7 +102,7 @@ :body ) ) - @test parse(Expr, "let i=is, j=js\nbody\nend") == + @test parsestmt(Expr, "let i=is, j=js\nbody\nend") == Expr(:let, Expr(:block, Expr(:(=), :i, :is), @@ -118,7 +118,7 @@ @testset "Short form function line numbers" begin # A block is added to hold the line number node - @test parse(Expr, "f() = xs") == + @test parsestmt(Expr, "f() = xs") == Expr(:(=), Expr(:call, :f), Expr(:block, @@ -126,7 +126,7 @@ :xs)) # flisp parser quirk: In a for loop the block is not added, despite # this defining a short-form function. - @test parse(Expr, "for f() = xs\nend") == + @test parsestmt(Expr, "for f() = xs\nend") == Expr(:for, Expr(:(=), Expr(:call, :f), :xs), Expr(:block, @@ -135,7 +135,7 @@ end @testset "Long form anonymous functions" begin - @test parse(Expr, "function (xs...)\nbody end") == + @test parsestmt(Expr, "function (xs...)\nbody end") == Expr(:function, Expr(:..., :xs), Expr(:block, @@ -146,25 +146,25 @@ @testset "String conversions" begin # String unwrapping / wrapping - @test parse(Expr, "\"str\"") == "str" - @test parse(Expr, "\"\$(\"str\")\"") == + @test parsestmt(Expr, "\"str\"") == "str" + @test parsestmt(Expr, "\"\$(\"str\")\"") == Expr(:string, Expr(:string, "str")) # Concatenation of string chunks in triple quoted cases - @test parse(Expr, "```\n a\n b```") == + @test parsestmt(Expr, "```\n a\n b```") == Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), LineNumberNode(1), "a\nb") - @test parse(Expr, "\"\"\"\n a\n \$x\n b\n c\"\"\"") == + @test parsestmt(Expr, "\"\"\"\n a\n \$x\n b\n c\"\"\"") == Expr(:string, "a\n", :x, "\nb\nc") end @testset "Char conversions" begin - @test parse(Expr, "'a'") == 'a' - @test parse(Expr, "'α'") == 'α' - @test parse(Expr, "'\\xce\\xb1'") == 'α' + @test parsestmt(Expr, "'a'") == 'a' + @test parsestmt(Expr, "'α'") == 'α' + @test parsestmt(Expr, "'\\xce\\xb1'") == 'α' end @testset "do block conversion" begin - @test parse(Expr, "f(x) do y\n body end") == + @test parsestmt(Expr, "f(x) do y\n body end") == Expr(:do, Expr(:call, :f, :x), Expr(:->, Expr(:tuple, :y), Expr(:block, @@ -174,29 +174,29 @@ @testset "= to Expr(:kw) conversion" begin # Call - @test parse(Expr, "f(a=1)") == + @test parsestmt(Expr, "f(a=1)") == Expr(:call, :f, Expr(:kw, :a, 1)) - @test parse(Expr, "f(; b=2)") == + @test parsestmt(Expr, "f(; b=2)") == Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2))) - @test parse(Expr, "f(a=1; b=2)") == + @test parsestmt(Expr, "f(a=1; b=2)") == Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1)) # Infix call = is not :kw - @test parse(Expr, "(x=1) != 2") == + @test parsestmt(Expr, "(x=1) != 2") == Expr(:call, :!=, Expr(:(=), :x, 1), 2) # Dotcall - @test parse(Expr, "f.(a=1; b=2)") == + @test parsestmt(Expr, "f.(a=1; b=2)") == Expr(:., :f, Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))) # Named tuples - @test parse(Expr, "(a=1,)") == + @test parsestmt(Expr, "(a=1,)") == Expr(:tuple, Expr(:(=), :a, 1)) - @test parse(Expr, "(a=1,; b=2)") == + @test parsestmt(Expr, "(a=1,; b=2)") == Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:(=), :a, 1)) - @test parse(Expr, "(a=1,; b=2; c=3)") == + @test parsestmt(Expr, "(a=1,; b=2; c=3)") == Expr(:tuple, Expr(:parameters, Expr(:parameters, Expr(:kw, :c, 3)), @@ -204,99 +204,99 @@ Expr(:(=), :a, 1)) # ref - @test parse(Expr, "x[i=j]") == + @test parsestmt(Expr, "x[i=j]") == Expr(:ref, :x, Expr(:kw, :i, :j)) - @test parse(Expr, "(i=j)[x]") == + @test parsestmt(Expr, "(i=j)[x]") == Expr(:ref, Expr(:(=), :i, :j), :x) - @test parse(Expr, "x[a, b; i=j]") == + @test parsestmt(Expr, "x[a, b; i=j]") == Expr(:ref, :x, Expr(:parameters, Expr(:(=), :i, :j)), :a, :b) # curly - @test parse(Expr, "(i=j){x}") == + @test parsestmt(Expr, "(i=j){x}") == Expr(:curly, Expr(:(=), :i, :j), :x) - @test parse(Expr, "x{a, b; i=j}") == + @test parsestmt(Expr, "x{a, b; i=j}") == Expr(:curly, :x, Expr(:parameters, Expr(:(=), :i, :j)), :a, :b) # vect - @test parse(Expr, "[a=1,; b=2]") == + @test parsestmt(Expr, "[a=1,; b=2]") == Expr(:vect, Expr(:parameters, Expr(:(=), :b, 2)), Expr(:(=), :a, 1)) # braces - @test parse(Expr, "{a=1,; b=2}") == + @test parsestmt(Expr, "{a=1,; b=2}") == Expr(:braces, Expr(:parameters, Expr(:(=), :b, 2)), Expr(:(=), :a, 1)) # dotted = is not :kw - @test parse(Expr, "f(a .= 1)") == + @test parsestmt(Expr, "f(a .= 1)") == Expr(:call, :f, Expr(:.=, :a, 1)) # = inside parens in calls and tuples # (TODO: we should warn for these cases.) - @test parse(Expr, "f(((a = 1)))") == + @test parsestmt(Expr, "f(((a = 1)))") == Expr(:call, :f, Expr(:kw, :a, 1)) - @test parse(Expr, "(((a = 1)),)") == + @test parsestmt(Expr, "(((a = 1)),)") == Expr(:tuple, Expr(:(=), :a, 1)) - @test parse(Expr, "(;((a = 1)),)") == + @test parsestmt(Expr, "(;((a = 1)),)") == Expr(:tuple, Expr(:parameters, Expr(:kw, :a, 1))) end @testset "dotcall" begin - @test parse(Expr, "f.(x,y)") == Expr(:., :f, Expr(:tuple, :x, :y)) - @test parse(Expr, "f.(x=1)") == Expr(:., :f, Expr(:tuple, Expr(:kw, :x, 1))) - @test parse(Expr, "x .+ y") == Expr(:call, Symbol(".+"), :x, :y) - @test parse(Expr, "(x=1) .+ y") == Expr(:call, Symbol(".+"), Expr(:(=), :x, 1), :y) - @test parse(Expr, "a .< b .< c") == Expr(:comparison, :a, Symbol(".<"), + @test parsestmt(Expr, "f.(x,y)") == Expr(:., :f, Expr(:tuple, :x, :y)) + @test parsestmt(Expr, "f.(x=1)") == Expr(:., :f, Expr(:tuple, Expr(:kw, :x, 1))) + @test parsestmt(Expr, "x .+ y") == Expr(:call, Symbol(".+"), :x, :y) + @test parsestmt(Expr, "(x=1) .+ y") == Expr(:call, Symbol(".+"), Expr(:(=), :x, 1), :y) + @test parsestmt(Expr, "a .< b .< c") == Expr(:comparison, :a, Symbol(".<"), :b, Symbol(".<"), :c) - @test parse(Expr, ".*(x)") == Expr(:call, Symbol(".*"), :x) - @test parse(Expr, ".+(x)") == Expr(:call, Symbol(".+"), :x) - @test parse(Expr, ".+x") == Expr(:call, Symbol(".+"), :x) + @test parsestmt(Expr, ".*(x)") == Expr(:call, Symbol(".*"), :x) + @test parsestmt(Expr, ".+(x)") == Expr(:call, Symbol(".+"), :x) + @test parsestmt(Expr, ".+x") == Expr(:call, Symbol(".+"), :x) end @testset "where" begin - @test parse(Expr, "A where {X, Y; Z}") == Expr(:where, :A, Expr(:parameters, :Z), :X, :Y) + @test parsestmt(Expr, "A where {X, Y; Z}") == Expr(:where, :A, Expr(:parameters, :Z), :X, :Y) end @testset "macrocall" begin # line numbers - @test parse(Expr, "@m\n") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1)) - @test parse(Expr, "\n@m") == Expr(:macrocall, Symbol("@m"), LineNumberNode(2)) + @test parsestmt(Expr, "@m\n") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1)) + @test parsestmt(Expr, "\n@m") == Expr(:macrocall, Symbol("@m"), LineNumberNode(2)) # parameters - @test parse(Expr, "@m(x; a)") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1), + @test parsestmt(Expr, "@m(x; a)") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1), Expr(:parameters, :a), :x) - @test parse(Expr, "@m(a=1; b=2)") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1), + @test parsestmt(Expr, "@m(a=1; b=2)") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1), Expr(:parameters, Expr(:kw, :b, 2)), Expr(:(=), :a, 1)) # @__dot__ - @test parse(Expr, "@.") == Expr(:macrocall, Symbol("@__dot__"), LineNumberNode(1)) - @test parse(Expr, "using A: @.") == Expr(:using, Expr(Symbol(":"), Expr(:., :A), Expr(:., Symbol("@__dot__")))) + @test parsestmt(Expr, "@.") == Expr(:macrocall, Symbol("@__dot__"), LineNumberNode(1)) + @test parsestmt(Expr, "using A: @.") == Expr(:using, Expr(Symbol(":"), Expr(:., :A), Expr(:., Symbol("@__dot__")))) end @testset "try" begin - @test parse(Expr, "try x catch e; y end") == + @test parsestmt(Expr, "try x catch e; y end") == Expr(:try, Expr(:block, LineNumberNode(1), :x), :e, Expr(:block, LineNumberNode(1), :y)) - @test parse(Expr, "try x finally y end") == + @test parsestmt(Expr, "try x finally y end") == Expr(:try, Expr(:block, LineNumberNode(1), :x), false, false, Expr(:block, LineNumberNode(1), :y)) - @test parse(Expr, "try x catch e; y finally z end") == + @test parsestmt(Expr, "try x catch e; y finally z end") == Expr(:try, Expr(:block, LineNumberNode(1), :x), :e, Expr(:block, LineNumberNode(1), :y), Expr(:block, LineNumberNode(1), :z)) - @test parse(Expr, "try x catch e; y else z end", version=v"1.8") == + @test parsestmt(Expr, "try x catch e; y else z end", version=v"1.8") == Expr(:try, Expr(:block, LineNumberNode(1), :x), :e, Expr(:block, LineNumberNode(1), :y), false, Expr(:block, LineNumberNode(1), :z)) - @test parse(Expr, "try x catch e; y else z finally w end", version=v"1.8") == + @test parsestmt(Expr, "try x catch e; y else z finally w end", version=v"1.8") == Expr(:try, Expr(:block, LineNumberNode(1), :x), :e, @@ -304,14 +304,14 @@ Expr(:block, LineNumberNode(1), :w), Expr(:block, LineNumberNode(1), :z)) # finally before catch - @test parse(Expr, "try x finally y catch e z end", ignore_warnings=true) == + @test parsestmt(Expr, "try x finally y catch e z end", ignore_warnings=true) == Expr(:try, Expr(:block, LineNumberNode(1), :x), :e, Expr(:block, LineNumberNode(1), :z), Expr(:block, LineNumberNode(1), :y)) # empty recovery - @test parse(Expr, "try x end", ignore_errors=true) == + @test parsestmt(Expr, "try x end", ignore_errors=true) == Expr(:try, Expr(:block, LineNumberNode(1), :x), false, false, @@ -319,58 +319,58 @@ end @testset "juxtapose" begin - @test parse(Expr, "2x") == Expr(:call, :*, 2, :x) - @test parse(Expr, "(2)(3)x") == Expr(:call, :*, 2, 3, :x) + @test parsestmt(Expr, "2x") == Expr(:call, :*, 2, :x) + @test parsestmt(Expr, "(2)(3)x") == Expr(:call, :*, 2, 3, :x) end @testset "Core.@doc" begin - @test parse(Expr, "\"x\" f") == + @test parsestmt(Expr, "\"x\" f") == Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(1), "x", :f) - @test parse(Expr, "\n\"x\" f") == + @test parsestmt(Expr, "\n\"x\" f") == Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(2), "x", :f) end @testset "return" begin - @test parse(Expr, "return x") == Expr(:return, :x) - @test parse(Expr, "return") == Expr(:return, nothing) + @test parsestmt(Expr, "return x") == Expr(:return, :x) + @test parsestmt(Expr, "return") == Expr(:return, nothing) end @testset "struct" begin - @test parse(Expr, "struct A end") == + @test parsestmt(Expr, "struct A end") == Expr(:struct, false, :A, Expr(:block, LineNumberNode(1))) - @test parse(Expr, "mutable struct A end") == + @test parsestmt(Expr, "mutable struct A end") == Expr(:struct, true, :A, Expr(:block, LineNumberNode(1))) end @testset "module" begin - @test parse(Expr, "module A end") == + @test parsestmt(Expr, "module A end") == Expr(:module, true, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1))) - @test parse(Expr, "baremodule A end") == + @test parsestmt(Expr, "baremodule A end") == Expr(:module, false, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1))) end @testset "errors" begin - @test parse(Expr, "--", ignore_errors=true) == + @test parsestmt(Expr, "--", ignore_errors=true) == Expr(:error, "invalid operator: `--`") @test parseall(Expr, "a b", ignore_errors=true) == Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(1), Expr(:error, :b)) - @test parse(Expr, "(x", ignore_errors=true) == + @test parsestmt(Expr, "(x", ignore_errors=true) == Expr(:block, :x, Expr(:error)) end @testset "import" begin - @test parse(Expr, "import A.(:b).:c: x.:z", ignore_warnings=true) == + @test parsestmt(Expr, "import A.(:b).:c: x.:z", ignore_warnings=true) == Expr(:import, Expr(Symbol(":"), Expr(:., :A, :b, :c), Expr(:., :x, :z))) # Stupid parens and quotes in import paths - @test parse(Expr, "import A.:+", ignore_warnings=true) == + @test parsestmt(Expr, "import A.:+", ignore_warnings=true) == Expr(:import, Expr(:., :A, :+)) - @test parse(Expr, "import A.(:+)", ignore_warnings=true) == + @test parsestmt(Expr, "import A.(:+)", ignore_warnings=true) == Expr(:import, Expr(:., :A, :+)) - @test parse(Expr, "import A.:(+)", ignore_warnings=true) == + @test parsestmt(Expr, "import A.:(+)", ignore_warnings=true) == Expr(:import, Expr(:., :A, :+)) - @test parse(Expr, "import A.:(+) as y", ignore_warnings=true, version=v"1.6") == + @test parsestmt(Expr, "import A.:(+) as y", ignore_warnings=true, version=v"1.6") == Expr(:import, Expr(:as, Expr(:., :A, :+), :y)) end end diff --git a/test/parser_api.jl b/test/parser_api.jl index f237f7ac..bac33c10 100644 --- a/test/parser_api.jl +++ b/test/parser_api.jl @@ -1,34 +1,44 @@ @testset "parser API" begin @testset "parse with String input" begin - @test parse(Expr, " x ") == :x + @test parsestmt(Expr, " x ") == :x @test JuliaSyntax.remove_linenums!(parseall(Expr, " x ")) == Expr(:toplevel, :x) @test parseatom(Expr, " x ") == :x @test parseatom(Expr, "(x)") == :x # SubString - @test parse(Expr, SubString("x+y")) == :(x+y) - @test parse(Expr, SubString("α+x")) == :(α+x) + @test parsestmt(Expr, SubString("x+y")) == :(x+y) + @test parsestmt(Expr, SubString("α+x")) == :(α+x) @test parseatom(Expr, SubString("x+y",3,3)) == :y # Exceptions due to extra trailing syntax @test_throws JuliaSyntax.ParseError parseatom(Expr, "x+y") - @test_throws JuliaSyntax.ParseError parse(Expr, "x+y\nz") + @test_throws JuliaSyntax.ParseError parsestmt(Expr, "x+y\nz") # ignore_warnings flag - @test_throws JuliaSyntax.ParseError parse(Expr, "import . .A") - @test parse(Expr, "import . .A", ignore_warnings=true) == :(import ..A) + @test_throws JuliaSyntax.ParseError parsestmt(Expr, "import . .A") + @test parsestmt(Expr, "import . .A", ignore_warnings=true) == :(import ..A) # version selection - @test_throws JuliaSyntax.ParseError parse(Expr, "[a ;; b]", version=v"1.6") - @test parse(Expr, "[a ;; b]", version=v"1.7") == Expr(:ncat, 2, :a, :b) + @test_throws JuliaSyntax.ParseError parsestmt(Expr, "[a ;; b]", version=v"1.6") + @test parsestmt(Expr, "[a ;; b]", version=v"1.7") == Expr(:ncat, 2, :a, :b) # filename - @test JuliaSyntax.parse(Expr, "begin\na\nend", filename="foo.jl", first_line=55) == + @test parsestmt(Expr, "begin\na\nend", filename="foo.jl", first_line=55) == Expr(:block, LineNumberNode(56, Symbol("foo.jl")), :a) # ignore_trivia @test parseatom(Expr, " x ", ignore_trivia=true) == :x @test_throws JuliaSyntax.ParseError parseatom(Expr, " x ", ignore_trivia=false) + + # Top level parsing + @test parseall(Expr, "a\nb") == + Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(2), :b) + @test parseall(Expr, "a\nb #==#") == + Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(2), :b) + @test parseall(Expr, "#==#\na\nb") == + Expr(:toplevel, LineNumberNode(2), :a, LineNumberNode(3), :b) + @test parseall(Expr, "a\nb\n#==#") == + Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(2), :b) end @testset "IO input" begin @@ -67,18 +77,18 @@ @test JuliaSyntax.remove_linenums!(ex) == Expr(:toplevel, :(x+y), :z) @test pos == 6 end - @test parse(Expr, "x+y\nz", 1) == (:(x+y), 4) + @test parsestmt(Expr, "x+y\nz", 1) == (:(x+y), 4) @test parseatom(Expr, "x+y\nz", 1) == (:x, 2) @test parseatom(Expr, "x+y\nz", 5) == (:z, 6) # SubString - @test parse(Expr, SubString("α+x\ny"), 1) == (:(α+x), 5) + @test parsestmt(Expr, SubString("α+x\ny"), 1) == (:(α+x), 5) @test parseatom(Expr, SubString("x+y"), 1) == (:x, 2) @test parseatom(Expr, SubString("x+y"), 3) == (:y, 4) end @testset "error/warning handling" begin - parseshow(s;kws...) = sprint(show, MIME("text/x.sexpression"), parse(SyntaxNode, s; kws...)) + parseshow(s;kws...) = sprint(show, MIME("text/x.sexpression"), parsestmt(SyntaxNode, s; kws...)) @test_throws JuliaSyntax.ParseError parseshow("try finally catch ex end") @test parseshow("try finally catch ex end", ignore_warnings=true) == "(try (block) (finally (block)) (catch ex (block)))" @@ -97,7 +107,7 @@ end @testset "ParseError printing" begin try - JuliaSyntax.parse(JuliaSyntax.SyntaxNode, "a -- b -- c", filename="somefile.jl") + JuliaSyntax.parsestmt(JuliaSyntax.SyntaxNode, "a -- b -- c", filename="somefile.jl") @assert false "error should be thrown" catch exc @test exc isa JuliaSyntax.ParseError diff --git a/test/syntax_tree.jl b/test/syntax_tree.jl index c6dd6585..c0cbf547 100644 --- a/test/syntax_tree.jl +++ b/test/syntax_tree.jl @@ -1,7 +1,7 @@ @testset "SyntaxNode" begin # Child access tt = "a*b + c" - t = parse(SyntaxNode, tt) + t = parsestmt(SyntaxNode, tt) @test sourcetext(child(t, 1)) == "a*b" @test sourcetext(child(t, 1, 1)) == "a" @@ -37,22 +37,22 @@ @test occursin("immutable", e.msg) && occursin("SyntaxData", e.msg) # copy - t = parse(SyntaxNode, "a*b + c") + t = parsestmt(SyntaxNode, "a*b + c") ct = copy(t) ct.data = nothing @test ct.data === nothing && t.data !== nothing @test child(ct, 1).parent === ct @test child(ct, 1) !== child(t, 1) - node = parse(SyntaxNode, "f()") - push!(node, parse(SyntaxNode, "x")) + node = parsestmt(SyntaxNode, "f()") + push!(node, parsestmt(SyntaxNode, "x")) @test length(children(node)) == 2 - node[2] = parse(SyntaxNode, "y") + node[2] = parsestmt(SyntaxNode, "y") @test sourcetext(child(node, 2)) == "y" end @testset "SyntaxNode pretty printing" begin - t = parse(SyntaxNode, "f(a*b,\n c)", filename="foo.jl") + t = parsestmt(SyntaxNode, "f(a*b,\n c)", filename="foo.jl") @test sprint(show, MIME("text/plain"), t) == """ line:col│ tree │ file_name 1:1 │[call] │foo.jl diff --git a/test/test_utils.jl b/test/test_utils.jl index 36fc22cd..2d76cf0b 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -9,7 +9,7 @@ using .JuliaSyntax: SourceFile, source_location, parse!, - parse, + parsestmt, parseall, parseatom, build_tree, @@ -71,7 +71,7 @@ end # Parse text with JuliaSyntax vs reference parser and show a textural diff of # the resulting expressions function parse_diff(text, showfunc=dump) - ex = parse(Expr, text, filename="none") + ex = parsestmt(Expr, text, filename="none") fl_ex = fl_parse(text) show_expr_text_diff(stdout, showfunc, ex, fl_ex) end