From b46edda75ab29fae880fdeb1b0c043d25d9a97b3 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 18 Mar 2023 12:16:07 -0500 Subject: [PATCH 01/10] Also match argnames to validate methods While working on TypedSyntax.jl it became apparent that CodeTracking sometimes returns spurious results. At least some of these arise from the recent support of anonymous functions, #102, which might in retrospect have been ill-considered. Rather than back that change out, this adopts a different resolution: validate the hits more carefully. The primary mechanism introduced here is to match not just the function name, but also the argument names. This can work even for anonymous functions, so we do not need to drop support for them. This also adds quite a few new tests. These additions would have passed before, but they proved valuable to ensure that the new argname-matching works sufficiently well. On TypedSyntax's "exhaustive.jl" test, this brings the number of failed cases (specifically, the `badmis`) from either 460 or 94 (depending on whether you include a few fixes in TypedSyntax) to just 2. --- Project.toml | 2 +- README.md | 5 ++ src/CodeTracking.jl | 5 +- src/utils.jl | 144 ++++++++++++++++++++++++++++++++++++++------ test/runtests.jl | 60 +++++++++++++----- test/script.jl | 26 ++++++++ 6 files changed, 205 insertions(+), 37 deletions(-) diff --git a/Project.toml b/Project.toml index 583ce36..cf2d323 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "CodeTracking" uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" authors = ["Tim Holy "] -version = "1.2.2" +version = "1.3.0" [deps] InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" diff --git a/README.md b/README.md index a3a9c81..a599614 100644 --- a/README.md +++ b/README.md @@ -143,3 +143,8 @@ file/line info in the method itself if Revise isn't running.) CodeTracking is perhaps best thought of as the "query" part of Revise.jl, providing a lightweight and stable API for gaining access to information it maintains internally. + +## Limitations (without Revise) + +- parsing sometimes starts on the wrong line. Line numbers are determined by counting `'\n'` in the source file, without parsing the contents. Consequently quoted- or in-code `'\n'` can mess up CodeTracking's notion of line numbering +- default constructor methods for `struct`s are not found diff --git a/src/CodeTracking.jl b/src/CodeTracking.jl index b60e65c..a158eb7 100644 --- a/src/CodeTracking.jl +++ b/src/CodeTracking.jl @@ -243,6 +243,7 @@ function definition(::Type{String}, method::Method) src === nothing && return nothing src = replace(src, "\r"=>"") # Step forward to the definition of this method, keeping track of positions of newlines + # Issue: in-code `'\n'`. To fix, presumably we'd have to parse the entire file. eol = isequal('\n') linestarts = Int[] istart = 1 @@ -253,14 +254,14 @@ function definition(::Type{String}, method::Method) # Parse the function definition (hoping that we've found the right location to start) ex, iend = Meta.parse(src, istart; raise=false) iend = prevind(src, iend) - if isfuncexpr(ex, methodname) + if is_func_expr(ex, method) iend = min(iend, lastindex(src)) return clean_source(src[istart:iend]), line end # The function declaration was presumably on a previous line lineindex = lastindex(linestarts) linestop = max(0, lineindex - 20) - while !isfuncexpr(ex, methodname) && lineindex > linestop + while !is_func_expr(ex, method) && lineindex > linestop istart = linestarts[lineindex] try ex, iend = Meta.parse(src, istart) diff --git a/src/utils.jl b/src/utils.jl index 81d1b8c..c85ad48 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,12 +1,11 @@ # This should stay as the first method because it's used in a test # (or change the test) -function checkname(fdef::Expr, name) - fproto = fdef.args[1] - (fdef.head === :where || fdef.head == :(::)) && return checkname(fproto, name) +function checkname(fdef::Expr, name) # this is now unused fdef.head === :call || return false + fproto = fdef.args[1] if fproto isa Expr - fproto.head == :(::) && return last(fproto.args) == name - fproto.head == :curly && return fproto.args[1] === name + fproto.head == :(::) && return last(fproto.args) === name # (obj::MyCallable)(x) = ... + fproto.head == :curly && return fproto.args[1] === name # MyType{T}(x) = ... # A metaprogramming-generated function fproto.head === :$ && return true # uncheckable, let's assume all is well # Is the check below redundant? @@ -17,31 +16,110 @@ function checkname(fdef::Expr, name) isa(fproto, Symbol) || isa(fproto, QuoteNode) || isa(fproto, Expr) || return false return checkname(fproto, name) end -checkname(fname::Symbol, name::Symbol) = begin - fname === name && return true - startswith(string(name), string('#', fname, '#')) && return true - string(name) == string(fname, "##kw") && return true - match(r"^#\d+$", string(name)) !== nothing && return true # support `f = x -> 2x` - return false + +function get_call_expr(@nospecialize(ex)) + while isa(ex, Expr) && ex.head ∈ (:where, :(::)) + ex = ex.args[1] + end + isexpr(ex, :call) && return ex + return nothing end -checkname(fname::Symbol, ::Nothing) = true -checkname(fname::QuoteNode, name) = checkname(fname.value, name) -function isfuncexpr(ex, name=nothing) +function get_func_expr(@nospecialize(ex)) + isa(ex, Expr) || return ex # Strip any macros that wrap the method definition - if ex isa Expr && ex.head === :toplevel + while isa(ex, Expr) && ex.head ∈ (:toplevel, :macrocall) + ex.head == :macrocall && length(ex.args) < 3 && return ex ex = ex.args[end] end - while ex isa Expr && ex.head === :macrocall && length(ex.args) >= 3 - ex = ex.args[end] + isa(ex, Expr) || return ex + if ex.head == :(=) && length(ex.args) == 2 + child1, child2 = ex.args + isexpr(get_call_expr(child1), :call) && return ex + isexpr(child2, :(->)) && return child2 end + return ex +end + +function is_func_expr(@nospecialize(ex)) isa(ex, Expr) || return false - if ex.head === :function || ex.head === :(=) - return checkname(ex.args[1], name) + ex.head ∈ (:function, :(->)) && return true + if ex.head == :(=) && length(ex.args) == 2 + child1 = ex.args[1] + isexpr(get_call_expr(child1), :call) && return true end return false end +function is_func_expr(@nospecialize(ex), name::Symbol) + ex = get_func_expr(ex) + is_func_expr(ex) || return false + return checkname(get_call_expr(ex.args[1]), name) +end + +function is_func_expr(@nospecialize(ex), meth::Method) + @show ex + ex = get_func_expr(ex) + is_func_expr(ex) || return false + if ex.head == :(->) + exargs = ex.args[1] + if isexpr(exargs, :tuple) + exargs = exargs.args + elseif (isa(exargs, Expr) && exargs.head ∈ (:(::), :.)) || isa(exargs, Symbol) + exargs = [exargs] + elseif isa(exargs, Expr) + return false + end + else + callex = get_call_expr(ex.args[1]) + isexpr(callex, :call) || return false + fname = callex.args[1] + if isexpr(fname, :curly) # where clause + fname = fname.args[1] + end + if isexpr(fname, :., 2) # module-qualified + fname = fname.args[2] + @assert isa(fname, QuoteNode) + fname = fname.value + end + if isexpr(fname, :(::)) + fname = fname.args[end] + end + if !(isa(fname, Symbol) && is_gensym(fname)) && !isexpr(fname, :$) + # match the function name + fname === strip_gensym(meth.name) || return false + end + exargs = callex.args[2:end] + end + # match the argnames + if !isempty(exargs) && isexpr(first(exargs), :parameters) + popfirst!(exargs) # don't match kwargs + end + margs = Base.method_argnames(meth) + @show exargs margs + if is_kw_call(meth) + margs = margs[findlast(==(Symbol("")), margs)+1:end] + end + for (arg, marg) in zip(exargs, margs[2:end]) + aname = get_argname(arg) + aname === marg || (aname === Symbol("#unused#") && marg === Symbol("")) || return false + end + return true # this will match any fcn `() -> ...`, but file/line is the only thing we have +end + +function get_argname(@nospecialize(ex)) + isa(ex, Symbol) && return ex + isexpr(ex, :(::), 2) && return get_argname(ex.args[1]) # type-asserted + isexpr(ex, :(::), 1) && return Symbol("#unused#") # nameless args (e.g., `::Type{String}`) + isexpr(ex, :kw) && return get_argname(ex.args[1]) # default value + isexpr(ex, :(=)) && return get_argname(ex.args[1]) # default value inside `@nospecialize` + isexpr(ex, :macrocall) && return get_argname(ex.args[end]) # @nospecialize + isexpr(ex, :...) && return get_argname(only(ex.args)) # varargs + isexpr(ex, :tuple) && return Symbol("") # tuple-destructuring + dump(ex) + error("unexpected argument ", ex) +end + function linerange(def::Expr) start, haslinestart = findline(def, identity) stop, haslinestop = findline(def, Iterators.reverse) @@ -70,6 +148,34 @@ Base.convert(::Type{LineNumberNode}, lin::LineInfoNode) = LineNumberNode(lin.lin # This regex matches the pseudo-file name of a REPL history entry. const rREPL = r"^REPL\[(\d+)\]$" +# Match anonymous function names +const rexfanon = r"^#\d+$" +# Match kwfunc method names +const rexkwfunc = r"^#.*##kw$" + +is_gensym(s::Symbol) = is_gensym(string(s)) +is_gensym(str::AbstractString) = startswith(str, '#') + +strip_gensym(s::Symbol) = strip_gensym(string(s)) +function strip_gensym(str::AbstractString) + if startswith(str, '#') + idx = findnext('#', str, 2) + if idx !== nothing + return Symbol(str[2:idx-1]) + end + end + endswith(str, "##kw") && return Symbol(str[1:end-4]) + return Symbol(str) +end + +if isdefined(Core, :kwcall) + is_kw_call(m::Method) = Base.unwrap_unionall(m.sig).parameters[1] === typeof(Core.kwcall) +else + function is_kw_call(m::Method) + T = Base.unwrap_unionall(m.sig).parameters[1] + return match(rexkwfunc, string(T.name.name)) !== nothing + end +end """ src = src_from_file_or_REPL(origin::AbstractString, repl = Base.active_repl) diff --git a/test/runtests.jl b/test/runtests.jl index e75cb94..d407c5a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -136,14 +136,12 @@ isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.j # Issues raised in #48 m = @which(sum([1]; dims=1)) + def = definition(String, m) + @test isa(def[1], AbstractString) if !isdefined(Main, :Revise) - def = definition(String, m) - @test def === nothing || isa(def[1], AbstractString) def = definition(Expr, m) @test def === nothing || isa(def, Expr) else - def = definition(String, m) - @test isa(def[1], AbstractString) def = definition(Expr, m) @test isa(def, Expr) end @@ -178,15 +176,31 @@ isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.j src, line = definition(String, m) @test occursin("x^3", src) @test line == 52 + m = only(methods(f80_2)) + src, line = definition(String, m) + @test occursin("x*y", src) + @test line == 53 # Issue #103 if isdefined(Base, Symbol("@assume_effects")) m = only(methods(pow103)) src, line = definition(String, m) @test occursin("res *= x", src) - @test line == 57 + @test line == 58 end + # @eval-ed methods + m = which(mysin, (Real,)) + src, line = definition(String, m) + @test occursin("xf", src) + @test line == 85 + + # unnamed arguments + m = which(unnamedarg, (Type{String}, Any)) + src, line = definition(String, m) + @test occursin("string(x)", src) + @test line == 93 + # Invalidation-insulating methods used by Revise and perhaps others d = IdDict{Union{String,Symbol},Union{Function,Vector{Function}}}() CodeTracking.invoked_setindex!(d, sin, "sin") @@ -282,15 +296,9 @@ end end @testset "kwargs methods" begin - m = nothing - for i in 1:30 - s = Symbol("#func_2nd_kwarg#$i") - if isdefined(Main, s) - m = @eval $s - end - end - m === nothing && error("couldn't find keyword function") - body, loc = CodeTracking.definition(String, first(methods(m))) + mdirect = only(methods(func_2nd_kwarg)) + fbody = Base.bodyfunction(mdirect) + body, loc = CodeTracking.definition(String, first(methods(fbody))) @test loc == 28 @test body == "func_2nd_kwarg(; kw=2) = true" end @@ -319,7 +327,6 @@ struct Functor end @test body == "(::Functor)(x, y) = x+y" end -if v"1.6" <= VERSION @testset "kwfuncs" begin body, _ = CodeTracking.definition(String, @which fkw(; x=1)) @test body == """ @@ -327,4 +334,27 @@ if v"1.6" <= VERSION x end""" end + +@testset "Decorated args" begin + body, _ = CodeTracking.definition(String, which(nospec, (Any,))) + @test body == "nospec(@nospecialize(x)) = 2x" + body, _ = CodeTracking.definition(String, which(nospec2, (Vector,))) + @test body == "nospec2(@nospecialize(x::AbstractVecOrMat)) = first(x)" + body, _ = CodeTracking.definition(String, which(nospec3, (Symbol,))) + @test body == "nospec3(name::Symbol, @nospecialize(arg=nothing)) = name" + body, _ = CodeTracking.definition(String, which(nospec3, (Symbol, String))) + @test body == "nospec3(name::Symbol, @nospecialize(arg=nothing)) = name" + body, _ = CodeTracking.definition(String, which(withva, (Char,))) + @test body == "withva(a...) = length(a)" + body, _ = CodeTracking.definition(String, which(hasdefault, (Int,))) + @test body == "hasdefault(xd, yd=2) = xd + yd" + body, _ = CodeTracking.definition(String, which(hasdefault, (Int, Float32))) + @test body == "hasdefault(xd, yd=2) = xd + yd" + body, _ = CodeTracking.definition(String, which(hasdefaulttypearg, (Type{Float32},))) + @test body == "hasdefaulttypearg(::Type{T}=Rational{Int}) where T = zero(T)" +end + +@testset "tuple-destructured args" begin + body, _ = CodeTracking.definition(String, which(diffminmax, (Any,))) + @test body == "diffminmax((min, max)) = max - min" end diff --git a/test/script.jl b/test/script.jl index 5b9a677..b1c3504 100644 --- a/test/script.jl +++ b/test/script.jl @@ -50,6 +50,7 @@ end # Issue #80 f80 = x -> 2 * x^3 + 1 +f80_2 = (x, y) -> x*y # Issue #103 if isdefined(Base, Symbol("@assume_effects")) @@ -76,3 +77,28 @@ end LikeNamedTuple() = LikeNamedTuple{(),Tuple{}}(()) LikeNamedTuple{names}(args::Tuple) where {names} = LikeNamedTuple{names,typeof(args)}(args) + +# Test @eval-ed methods +# This is taken from the definition of `sin(::Int)` in Base, copied here for testing purposes +# in case the implementation changes +for f in (:mysin,) + @eval function ($f)(x::Real) + xf = float(x) + x === xf && throw(MethodError($f, (x,))) + return ($f)(xf) + end +end +mysin(x::AbstractFloat) = sin(x) + +unnamedarg(::Type{String}, x) = string(x) + +# "decorated" args +nospec(@nospecialize(x)) = 2x +nospec2(@nospecialize(x::AbstractVecOrMat)) = first(x) +nospec3(name::Symbol, @nospecialize(arg=nothing)) = name +withva(a...) = length(a) +hasdefault(xd, yd=2) = xd + yd +hasdefaulttypearg(::Type{T}=Rational{Int}) where T = zero(T) + +# tuple-destructuring +diffminmax((min, max)) = max - min From 60b19bd9f0335a3f1282fb3366957527e0b4d420 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 16:41:29 -0500 Subject: [PATCH 02/10] remove printing --- src/utils.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index c85ad48..4d0bd9d 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -58,7 +58,6 @@ function is_func_expr(@nospecialize(ex), name::Symbol) end function is_func_expr(@nospecialize(ex), meth::Method) - @show ex ex = get_func_expr(ex) is_func_expr(ex) || return false if ex.head == :(->) @@ -96,7 +95,6 @@ function is_func_expr(@nospecialize(ex), meth::Method) popfirst!(exargs) # don't match kwargs end margs = Base.method_argnames(meth) - @show exargs margs if is_kw_call(meth) margs = margs[findlast(==(Symbol("")), margs)+1:end] end From d9cd380bd9188992e4c73a415f4cb77dcaec12d1 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 17:39:51 -0500 Subject: [PATCH 03/10] Fix handling of kw body functions --- src/utils.jl | 14 ++++++++++++-- test/runtests.jl | 2 +- test/script.jl | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 4d0bd9d..34c00a8 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -60,6 +60,7 @@ end function is_func_expr(@nospecialize(ex), meth::Method) ex = get_func_expr(ex) is_func_expr(ex) || return false + fname = nothing if ex.head == :(->) exargs = ex.args[1] if isexpr(exargs, :tuple) @@ -95,8 +96,8 @@ function is_func_expr(@nospecialize(ex), meth::Method) popfirst!(exargs) # don't match kwargs end margs = Base.method_argnames(meth) - if is_kw_call(meth) - margs = margs[findlast(==(Symbol("")), margs)+1:end] + if is_kw_call(meth) || is_body_fcn(meth, fname) + margs = margs[findlast(==(Symbol("")), margs):end] end for (arg, marg) in zip(exargs, margs[2:end]) aname = get_argname(arg) @@ -175,6 +176,15 @@ else end end +is_body_fcn(m::Method, basename::Symbol) = match(Regex("^#$basename#\\d+\$"), string(m.name)) !== nothing +function is_body_fcn(m::Method, basename::Expr) + basename.head == :. || return false + bn = basename.args[end] + @assert isa(bn, QuoteNode) + return is_body_fcn(m, bn.value) +end +is_body_fcn(m::Method, ::Nothing) = false + """ src = src_from_file_or_REPL(origin::AbstractString, repl = Base.active_repl) diff --git a/test/runtests.jl b/test/runtests.jl index d407c5a..c241e04 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -300,7 +300,7 @@ end fbody = Base.bodyfunction(mdirect) body, loc = CodeTracking.definition(String, first(methods(fbody))) @test loc == 28 - @test body == "func_2nd_kwarg(; kw=2) = true" + @test body == "func_2nd_kwarg(a, b; kw=2) = true" end @testset "method extensions" begin diff --git a/test/script.jl b/test/script.jl index b1c3504..8755659 100644 --- a/test/script.jl +++ b/test/script.jl @@ -25,7 +25,7 @@ function f50() # issue #50 end func_1st_nokwarg() = true -func_2nd_kwarg(; kw=2) = true +func_2nd_kwarg(a, b; kw=2) = true module Foo module Bar From 97d84efec08a27fed2d1d4de4c0558c76859d034 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 20:26:36 -0500 Subject: [PATCH 04/10] More robust handling for kw* methods Extracting the name from the source-text is problematic for methods that are defined in `@eval`. This reworks recognition of kw & kwbody methods to rely only on the signature. --- src/utils.jl | 44 +++++++++++++++++++++++++++++++++++--------- test/runtests.jl | 8 ++++++++ test/script.jl | 4 ++-- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 34c00a8..4a00f98 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -96,8 +96,9 @@ function is_func_expr(@nospecialize(ex), meth::Method) popfirst!(exargs) # don't match kwargs end margs = Base.method_argnames(meth) - if is_kw_call(meth) || is_body_fcn(meth, fname) - margs = margs[findlast(==(Symbol("")), margs):end] + _, idx = kwmethod_basename(meth) + if idx > 0 + margs = margs[idx:end] end for (arg, marg) in zip(exargs, margs[2:end]) aname = get_argname(arg) @@ -176,14 +177,39 @@ else end end -is_body_fcn(m::Method, basename::Symbol) = match(Regex("^#$basename#\\d+\$"), string(m.name)) !== nothing -function is_body_fcn(m::Method, basename::Expr) - basename.head == :. || return false - bn = basename.args[end] - @assert isa(bn, QuoteNode) - return is_body_fcn(m, bn.value) +# is_body_fcn(m::Method, basename::Symbol) = match(Regex("^#$basename#\\d+\$"), string(m.name)) !== nothing +# function is_body_fcn(m::Method, basename::Expr) +# basename.head == :. || return false +# return is_body_fcn(m, get_basename(basename)) +# end +# is_body_fcn(m::Method, ::Nothing) = false +# function get_basename(basename::Expr) +# bn = basename.args[end] +# @assert isa(bn, QuoteNode) +# return is_body_fcn(m, bn.value) +# end + +function kwmethod_basename(meth::Method) + name = meth.name + mtch = match(r"^#+(.*)#", string(name)) + name = mtch === nothing ? name : Symbol(only(mtch.captures)) + ftypname = Symbol(string('#', name)) + idx = findfirst(Base.unwrap_unionall(meth.sig).parameters) do @nospecialize(T) + if isa(T, DataType) + Tname = T.name.name + if Tname === :Type + p1 = Base.unwrap_unionall(T.parameters[1]) + Tname = isa(p1, DataType) ? p1.name.name : + isa(p1, TypeVar) ? p1.name : error("unexpected type ", typeof(p1), "for ", meth) + return Tname == name + end + return ftypname === Tname + end + false + end + idx === nothing && return name, 0 + return name, idx end -is_body_fcn(m::Method, ::Nothing) = false """ src = src_from_file_or_REPL(origin::AbstractString, repl = Base.active_repl) diff --git a/test/runtests.jl b/test/runtests.jl index c241e04..5e1fa8a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -194,6 +194,14 @@ isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.j src, line = definition(String, m) @test occursin("xf", src) @test line == 85 + m = only(methods(Base.bodyfunction(m))) + src, line = definition(String, m) + @test occursin("xf", src) + @test line == 85 + m = @which mysin(0.5; return_zero=true) + src, line = definition(String, m) + @test occursin("xf", src) + @test line == 85 # unnamed arguments m = which(unnamedarg, (Type{String}, Any)) diff --git a/test/script.jl b/test/script.jl index 8755659..68cfd82 100644 --- a/test/script.jl +++ b/test/script.jl @@ -80,9 +80,9 @@ LikeNamedTuple{names}(args::Tuple) where {names} = LikeNamedTuple{names,typeof(a # Test @eval-ed methods # This is taken from the definition of `sin(::Int)` in Base, copied here for testing purposes -# in case the implementation changes +# in case the implementation changes. Also added a kw. for f in (:mysin,) - @eval function ($f)(x::Real) + @eval function ($f)(x::Real; return_zero::Bool=false) xf = float(x) x === xf && throw(MethodError($f, (x,))) return ($f)(xf) From bc4664140f6b0d29d7316065eedf8e5fe5845e20 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 20:58:39 -0500 Subject: [PATCH 05/10] Support '_' args --- src/utils.jl | 1 + test/runtests.jl | 4 ++++ test/script.jl | 8 ++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 4a00f98..fe76a53 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -102,6 +102,7 @@ function is_func_expr(@nospecialize(ex), meth::Method) end for (arg, marg) in zip(exargs, margs[2:end]) aname = get_argname(arg) + aname === :_ && continue aname === marg || (aname === Symbol("#unused#") && marg === Symbol("")) || return false end return true # this will match any fcn `() -> ...`, but file/line is the only thing we have diff --git a/test/runtests.jl b/test/runtests.jl index 5e1fa8a..d389155 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -208,6 +208,10 @@ isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.j src, line = definition(String, m) @test occursin("string(x)", src) @test line == 93 + m = which(mypush!, (Nowhere, Any)) + src, line = definition(String, m) + @test occursin("::Nowhere", src) + @test line == 108 # Invalidation-insulating methods used by Revise and perhaps others d = IdDict{Union{String,Symbol},Union{Function,Vector{Function}}}() diff --git a/test/script.jl b/test/script.jl index 68cfd82..8f4325f 100644 --- a/test/script.jl +++ b/test/script.jl @@ -80,7 +80,7 @@ LikeNamedTuple{names}(args::Tuple) where {names} = LikeNamedTuple{names,typeof(a # Test @eval-ed methods # This is taken from the definition of `sin(::Int)` in Base, copied here for testing purposes -# in case the implementation changes. Also added a kw. +# in case the implementation changes. Also added a (useless) kw. for f in (:mysin,) @eval function ($f)(x::Real; return_zero::Bool=false) xf = float(x) @@ -90,7 +90,7 @@ for f in (:mysin,) end mysin(x::AbstractFloat) = sin(x) -unnamedarg(::Type{String}, x) = string(x) +unnamedarg(::Type{String}, x) = string(x) # see more unnamed on line 108 # "decorated" args nospec(@nospecialize(x)) = 2x @@ -102,3 +102,7 @@ hasdefaulttypearg(::Type{T}=Rational{Int}) where T = zero(T) # tuple-destructuring diffminmax((min, max)) = max - min + +# _ args +struct Nowhere end +mypush!(::Nowhere, _) = nothing From 6460f90b939d6595127ddf2611c07a7cf128c9b9 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 21:11:17 -0500 Subject: [PATCH 06/10] Strip `global` annotations --- src/CodeTracking.jl | 7 ++----- src/utils.jl | 2 +- test/runtests.jl | 6 ++++++ test/script.jl | 5 +++++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/CodeTracking.jl b/src/CodeTracking.jl index a158eb7..e530bb8 100644 --- a/src/CodeTracking.jl +++ b/src/CodeTracking.jl @@ -254,11 +254,8 @@ function definition(::Type{String}, method::Method) # Parse the function definition (hoping that we've found the right location to start) ex, iend = Meta.parse(src, istart; raise=false) iend = prevind(src, iend) - if is_func_expr(ex, method) - iend = min(iend, lastindex(src)) - return clean_source(src[istart:iend]), line - end - # The function declaration was presumably on a previous line + # The function declaration may have been on a previous line, + # allow some slop lineindex = lastindex(linestarts) linestop = max(0, lineindex - 20) while !is_func_expr(ex, method) && lineindex > linestop diff --git a/src/utils.jl b/src/utils.jl index fe76a53..6da3c19 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -28,7 +28,7 @@ end function get_func_expr(@nospecialize(ex)) isa(ex, Expr) || return ex # Strip any macros that wrap the method definition - while isa(ex, Expr) && ex.head ∈ (:toplevel, :macrocall) + while isa(ex, Expr) && ex.head ∈ (:toplevel, :macrocall, :global, :local) ex.head == :macrocall && length(ex.args) < 3 && return ex ex = ex.args[end] end diff --git a/test/runtests.jl b/test/runtests.jl index d389155..72ff0c0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -213,6 +213,12 @@ isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.j @test occursin("::Nowhere", src) @test line == 108 + # global annotations + m = which(inlet, (Any,)) + src, line = definition(String, m) + @test occursin("inlet(x)", src) + @test line == 112 + # Invalidation-insulating methods used by Revise and perhaps others d = IdDict{Union{String,Symbol},Union{Function,Vector{Function}}}() CodeTracking.invoked_setindex!(d, sin, "sin") diff --git a/test/script.jl b/test/script.jl index 8f4325f..530bd1d 100644 --- a/test/script.jl +++ b/test/script.jl @@ -106,3 +106,8 @@ diffminmax((min, max)) = max - min # _ args struct Nowhere end mypush!(::Nowhere, _) = nothing + +# global +let + global inlet(x) = x^2 +end From 68ff832b99c82538d14a0d52a293ab22793d3240 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 21:45:32 -0500 Subject: [PATCH 07/10] Improve support for Callables & constructors --- src/utils.jl | 37 +++++++++++++++++++++++++++---------- test/runtests.jl | 18 ++++++++++++++++++ test/script.jl | 9 +++++++++ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 6da3c19..cde8d23 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -74,18 +74,35 @@ function is_func_expr(@nospecialize(ex), meth::Method) callex = get_call_expr(ex.args[1]) isexpr(callex, :call) || return false fname = callex.args[1] - if isexpr(fname, :curly) # where clause - fname = fname.args[1] - end - if isexpr(fname, :., 2) # module-qualified - fname = fname.args[2] - @assert isa(fname, QuoteNode) - fname = fname.value - end - if isexpr(fname, :(::)) - fname = fname.args[end] + modified = true + while modified + modified = false + if isexpr(fname, :curly) # where clause + fname = fname.args[1] + modified = true + end + if isexpr(fname, :., 2) # module-qualified + fname = fname.args[2] + @assert isa(fname, QuoteNode) + fname = fname.value + modified = true + end + if isexpr(fname, :(::)) + fname = fname.args[end] + modified = true + end end if !(isa(fname, Symbol) && is_gensym(fname)) && !isexpr(fname, :$) + if fname === :Type && isexpr(ex.args[1], :where) && isexpr(callex.args[1], :(::)) && isexpr(callex.args[1].args[end], :curly) + Tsym = callex.args[1].args[end].args[2] + for wheretyp in ex.args[1].args[2:end] + @assert isexpr(wheretyp, :(<:)) + if Tsym == wheretyp.args[1] + fname = wheretyp.args[2] + break + end + end + end # match the function name fname === strip_gensym(meth.name) || return false end diff --git a/test/runtests.jl b/test/runtests.jl index 72ff0c0..3b0daf4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -219,6 +219,24 @@ isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.j @test occursin("inlet(x)", src) @test line == 112 + # Callables + gg = Gaussian(1.0) + m = @which gg(2) + src, line = definition(String, m) + @test occursin("::Gaussian)(x)", src) + @test line == 119 + invt = Invert() + m = @which invt([false, true]) + src, line = definition(String, m) + @test occursin("::Invert)(v", src) + @test line == 121 + + # Constructor with `where` + m = @which Invert((false, true)) + src, line = definition(String, m) + @test occursin("(::Type{T})(itr) where {T<:Invert}", src) + @test line == 122 + # Invalidation-insulating methods used by Revise and perhaps others d = IdDict{Union{String,Symbol},Union{Function,Vector{Function}}}() CodeTracking.invoked_setindex!(d, sin, "sin") diff --git a/test/script.jl b/test/script.jl index 530bd1d..9bc9acd 100644 --- a/test/script.jl +++ b/test/script.jl @@ -111,3 +111,12 @@ mypush!(::Nowhere, _) = nothing let global inlet(x) = x^2 end + +# Callables +struct Gaussian + σ::Float64 +end +(g::Gaussian)(x) = exp(-x^2 / (2*g.σ^2)) / (sqrt(2*π)*g.σ) +struct Invert end +(::Invert)(v::AbstractVector{Bool}) = (!).(v) +(::Type{T})(itr) where {T<:Invert} = [!x for x in itr] From 20c6356528c0f653b42f9ac68d4b365ea9aacf15 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 19 Mar 2023 22:13:35 -0500 Subject: [PATCH 08/10] Fix `kwmethod_basename` on older Julia --- src/utils.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils.jl b/src/utils.jl index cde8d23..c132b9e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -209,7 +209,11 @@ end function kwmethod_basename(meth::Method) name = meth.name - mtch = match(r"^#+(.*)#", string(name)) + sname = string(name) + mtch = match(r"^(.*)##kw$", sname) + if mtch === nothing + mtch = match(r"^#+(.*)#", sname) + end name = mtch === nothing ? name : Symbol(only(mtch.captures)) ftypname = Symbol(string('#', name)) idx = findfirst(Base.unwrap_unionall(meth.sig).parameters) do @nospecialize(T) From 54a830b18345a207c4258d53b6026743a43ba301 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 20 Mar 2023 23:39:13 -0500 Subject: [PATCH 09/10] Fix string-indexing bug --- src/CodeTracking.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CodeTracking.jl b/src/CodeTracking.jl index e530bb8..f4eb683 100644 --- a/src/CodeTracking.jl +++ b/src/CodeTracking.jl @@ -268,7 +268,7 @@ function definition(::Type{String}, method::Method) line -= 1 end lineindex <= linestop && return nothing - return clean_source(src[istart:iend-1]), line + return clean_source(src[istart:prevind(src, iend)]), line end function clean_source(src) From 91d9837848dd135451d6b40567ea2bb9dcf084e5 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 21 Mar 2023 01:25:05 -0500 Subject: [PATCH 10/10] Fix REPL tests --- src/CodeTracking.jl | 2 +- test/runtests.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CodeTracking.jl b/src/CodeTracking.jl index f4eb683..7ac8205 100644 --- a/src/CodeTracking.jl +++ b/src/CodeTracking.jl @@ -251,9 +251,9 @@ function definition(::Type{String}, method::Method) push!(linestarts, istart) istart = findnext(eol, src, istart) + 1 end + push!(linestarts, length(src) + 1) # Parse the function definition (hoping that we've found the right location to start) ex, iend = Meta.parse(src, istart; raise=false) - iend = prevind(src, iend) # The function declaration may have been on a previous line, # allow some slop lineindex = lastindex(linestarts) diff --git a/test/runtests.jl b/test/runtests.jl index 3b0daf4..16e6a5d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -286,7 +286,7 @@ end end push!(hp.history, fstr) m = first(methods(f)) - @test definition(String, first(methods(f))) == (fstr, 1) + @test definition(String, m) == (fstr, 1) @test !isempty(signatures_at(String(m.file), m.line)) histidx += 1