From 31a8fd9514670b5fad826582fb217abccab492a9 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Tue, 11 Nov 2025 08:53:26 -0500 Subject: [PATCH 1/9] add `@testinferred` --- src/TestExtras.jl | 3 ++ src/testinferred.jl | 96 +++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 49 +++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 src/testinferred.jl diff --git a/src/TestExtras.jl b/src/TestExtras.jl index 93dedef..da2dd71 100644 --- a/src/TestExtras.jl +++ b/src/TestExtras.jl @@ -3,9 +3,11 @@ module TestExtras export @constinferred, @constinferred_broken export @timedtestset export @include +export @testinferred export ConstInferred include("constinferred.jl") +include("testinferred.jl") include("includemacro.jl") if VERSION >= v"1.8" @@ -15,6 +17,7 @@ else end using .ConstInferred: @constinferred, @constinferred_broken +using .TestInferred: @testinferred using .TimedTests: @timedtestset end diff --git a/src/testinferred.jl b/src/testinferred.jl new file mode 100644 index 0000000..c4c9ff4 --- /dev/null +++ b/src/testinferred.jl @@ -0,0 +1,96 @@ +module TestInferred + +export @testinferred + +using InteractiveUtils: gen_call_with_extracted_types +using Test: @test + +""" + @testinferred [AllowedType] f(x) + +Tests that the call expression `f(x)` returns a value of the same type inferred by the compiler. +It is useful to check for type stability. +This is similar to `Test.@inferred`, but instead of throwing an error, a `@test` is added. + +Optionally, `AllowedType` relaxes the test, by making it pass when either the type of `f(x)` matches the inferred type modulo `AllowedType`, or when the return type is a subtype of `AllowedType`. +This is useful when testing type stability of functions returning a small union such as `Union{Nothing, T}` or `Union{Missing, T}`. +""" +macro testinferred(ex) + return _inferred(ex, __module__) +end + +macro testinferred(allow, ex) + return _inferred(ex, __module__, allow) +end + +# helper functions +_args_and_call((args..., f)...; kwargs...) = (args, kwargs, f(args...; kwargs...)) +_materialize_broadcasted(f, args...) = Broadcast.materialize(Broadcast.broadcasted(f, args...)) +@static if isdefined(Base, :typesplit) + const typesplit = Base.typesplit +else + Base.@pure function typesplit(@nospecialize(a), @nospecialize(b)) + a <: b && return Base.Bottom + isa(a, Union) && return Union{typesplit(a.a, b), typesplit(a.b, b)} + return a + end +end + +# ensure backwards compatibility - have to use `return_types` which can return a vector of types +@static if isdefined(Base, :infer_return_type) + infer_return_type(args...) = Base.infer_return_type(args...) +else + function infer_return_type(args...) + inftypes = Base.return_types(args...) + @assert length(inftypes) == 1 + return only(inftypes) + end +end + +function _inferred(ex, mod, allow = :(Union{})) + if Meta.isexpr(ex, :ref) + ex = Expr(:call, :getindex, ex.args...) + end + Meta.isexpr(ex, :call)|| error("@testinferred requires a call expression") + + # handle broadcasting expressions + farg = ex.args[1] + if isa(farg, Symbol) && farg !== :.. && first(string(farg)) == '.' + farg = Symbol(string(farg)[2:end]) + ex = Expr(:call, GlobalRef(@__MODULE__, :_materialize_broadcasted), farg, ex.args[2:end]...) + end + + result = let ex = ex + quote + let allow = $(esc(allow)) + allow isa Type || throw(ArgumentError("@testinferred requires a type as second argument")) + $( + if any(@nospecialize(a) -> (Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex.args) + # Has keywords + # Create the call expression with escaped user expressions + call_expr = :($(esc(ex.args[1]))(args...; kwargs...)) + quote + args, kwargs, result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) + # wrap in dummy hygienic-scope to work around scoping issues with `call_expr` already having `esc` on the necessary parts + inftype = $(Expr(:var"hygienic-scope", gen_call_with_extracted_types(mod, infer_return_type, call_expr), @__MODULE__)) + end + else + # No keywords + quote + args = ($([esc(ex.args[i]) for i in 2:length(ex.args)]...),) + result = $(esc(ex.args[1]))(args...) + inftype = $(GlobalRef(@__MODULE__, :infer_return_type))($(esc(ex.args[1])), Base.typesof(args...)) + end + end + ) + rettype = result isa Type ? Type{result} : typeof(result) + @test (rettype <: allow || rettype == typesplit(inftype, allow)) + result + end + end + end + + return Base.remove_linenums!(result) +end + +end diff --git a/test/runtests.jl b/test/runtests.jl index 37cf0dd..d502aa7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -64,6 +64,55 @@ h25835(; x=1, y=1) = x isa Int ? x * y : (rand(Bool) ? 1.0 : 1) @test @constinferred(Union{Float64,Int64}, h25835(x=1.0, y=1.0)) == 1 end +# @testinferred +# ------------- +# testset to record failed tests without actually making them fail +mutable struct NoThrowTestSet <: Test.AbstractTestSet + results::Vector + NoThrowTestSet(desc) = new([]) +end +Test.record(ts::NoThrowTestSet, t::Test.Result) = (push!(ts.results, t); t) +Test.finish(ts::NoThrowTestSet) = ts.results + +struct SillyArray <: AbstractArray{Float64, 1} end +Base.getindex(::SillyArray, i) = rand() > 0.5 ? 0 : false + +uninferrable_function(i) = (1, "1")[i] +uninferrable_small_union(i) = (1, nothing)[i] + +inferrable_kwtest(x; y = 1) = 2x +uninferrable_kwtest(x; y = 1) = 2x + y + +@timedtestset "testinferred" begin + # function only ran once + global inferred_test_global = 0 + @testinferred inferred_test_function() + @test inferred_test_global == 1 + + @test (@testinferred (1:3)[2]) == 2 + + @testinferred Nothing uninferrable_small_union(1) + @testinferred Nothing uninferrable_small_union(2) + + @test (@testinferred inferrable_kwtest(1)) == 2 + @test (@testinferred inferrable_kwtest(1; y = 1)) == 2 + @test (@testinferred uninferrable_kwtest(1)) == 3 + @test (@testinferred uninferrable_kwtest(1; y = 2)) == 4 + + @test_throws ArgumentError (@testinferred(nothing, uninferrable_small_union(1))) +end + +let fails = @testset NoThrowTestSet begin + @testinferred SillyArray()[2] + @testinferred uninferrable_function(1) + @testinferred uninferrable_small_union(1) + @testinferred Missing uninferrable_small_union(1) + end + for fail in fails + @test fail isa Test.Fail + end +end + @timedtestset "@test" begin @test true @test 1 == 1 From 451195194d2fbfc9a04b756fb67ee06e438fa08c Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Tue, 11 Nov 2025 09:19:07 -0500 Subject: [PATCH 2/9] bump v0.3.2 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9423a90..47cff1b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TestExtras" uuid = "5ed8adda-3752-4e41-b88a-e8b09835ee3a" authors = ["Jutho Haegeman and contributors"] -version = "0.3.1" +version = "0.3.2" [deps] InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" From a21a4b1c0dedcace1d52dd43cb80fb629d794403 Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Wed, 19 Nov 2025 16:33:15 +0100 Subject: [PATCH 3/9] some reorganisation --- src/TestExtras.jl | 1 + src/constinferred.jl | 22 +++------------------- src/testinferred.jl | 30 ++++-------------------------- src/utilities.jl | 27 +++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 45 deletions(-) create mode 100644 src/utilities.jl diff --git a/src/TestExtras.jl b/src/TestExtras.jl index da2dd71..94dd896 100644 --- a/src/TestExtras.jl +++ b/src/TestExtras.jl @@ -6,6 +6,7 @@ export @include export @testinferred export ConstInferred +include("utilities.jl") include("constinferred.jl") include("testinferred.jl") include("includemacro.jl") diff --git a/src/constinferred.jl b/src/constinferred.jl index 602d8e3..549c7d3 100644 --- a/src/constinferred.jl +++ b/src/constinferred.jl @@ -3,32 +3,16 @@ export @constinferred, @constinferred_broken const ConstantValue = Union{Number,Char,QuoteNode} -using InteractiveUtils: gen_call_with_extracted_types using Test using Test: Returned, Threw -@static if isdefined(Base, :typesplit) - const typesplit = Base.typesplit -else - Base.@pure function typesplit(@nospecialize(a), @nospecialize(b)) - if a <: b - return Base.Bottom - end - if isa(a, Union) - return Union{typesplit(a.a, b), - typesplit(a.b, b)} - end - return a - end -end +using ..Utilities: typesplit, materialize_broadcasted + _enabled = Ref(true) enable() = (_enabled[] = true; return nothing) disable() = (_enabled[] = false; return nothing) -function _materialize_broadcasted(f, args...) - return Broadcast.materialize(Broadcast.broadcasted(f, args...)) -end """ @constinferred [AllowedType] f(x) @@ -72,7 +56,7 @@ function _constinferred(ex, mod, src, test_f, allow=:(Union{})) farg = ex.args[1] if isa(farg, Symbol) && first(string(farg)) == '.' farg = Symbol(string(farg)[2:end]) - ex = Expr(:call, GlobalRef(Test, :_materialize_broadcasted), + ex = Expr(:call, GlobalRef(@__MODULE__, :materialize_broadcasted), farg, ex.args[2:end]...) end pre = quote diff --git a/src/testinferred.jl b/src/testinferred.jl index c4c9ff4..6ffa82b 100644 --- a/src/testinferred.jl +++ b/src/testinferred.jl @@ -4,6 +4,8 @@ export @testinferred using InteractiveUtils: gen_call_with_extracted_types using Test: @test +using ..Utilities: args_and_call, materialize_broadcasted, typesplit, infer_return_type +using InteractiveUtils: gen_call_with_extracted_types """ @testinferred [AllowedType] f(x) @@ -23,30 +25,6 @@ macro testinferred(allow, ex) return _inferred(ex, __module__, allow) end -# helper functions -_args_and_call((args..., f)...; kwargs...) = (args, kwargs, f(args...; kwargs...)) -_materialize_broadcasted(f, args...) = Broadcast.materialize(Broadcast.broadcasted(f, args...)) -@static if isdefined(Base, :typesplit) - const typesplit = Base.typesplit -else - Base.@pure function typesplit(@nospecialize(a), @nospecialize(b)) - a <: b && return Base.Bottom - isa(a, Union) && return Union{typesplit(a.a, b), typesplit(a.b, b)} - return a - end -end - -# ensure backwards compatibility - have to use `return_types` which can return a vector of types -@static if isdefined(Base, :infer_return_type) - infer_return_type(args...) = Base.infer_return_type(args...) -else - function infer_return_type(args...) - inftypes = Base.return_types(args...) - @assert length(inftypes) == 1 - return only(inftypes) - end -end - function _inferred(ex, mod, allow = :(Union{})) if Meta.isexpr(ex, :ref) ex = Expr(:call, :getindex, ex.args...) @@ -57,7 +35,7 @@ function _inferred(ex, mod, allow = :(Union{})) farg = ex.args[1] if isa(farg, Symbol) && farg !== :.. && first(string(farg)) == '.' farg = Symbol(string(farg)[2:end]) - ex = Expr(:call, GlobalRef(@__MODULE__, :_materialize_broadcasted), farg, ex.args[2:end]...) + ex = Expr(:call, GlobalRef(@__MODULE__, :materialize_broadcasted), farg, ex.args[2:end]...) end result = let ex = ex @@ -70,7 +48,7 @@ function _inferred(ex, mod, allow = :(Union{})) # Create the call expression with escaped user expressions call_expr = :($(esc(ex.args[1]))(args...; kwargs...)) quote - args, kwargs, result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) + args, kwargs, result = $(esc(Expr(:call, args_and_call, ex.args[2:end]..., ex.args[1]))) # wrap in dummy hygienic-scope to work around scoping issues with `call_expr` already having `esc` on the necessary parts inftype = $(Expr(:var"hygienic-scope", gen_call_with_extracted_types(mod, infer_return_type, call_expr), @__MODULE__)) end diff --git a/src/utilities.jl b/src/utilities.jl new file mode 100644 index 0000000..c6ee475 --- /dev/null +++ b/src/utilities.jl @@ -0,0 +1,27 @@ +module Utilities + +args_and_call((args..., f)...; kwargs...) = (args, kwargs, f(args...; kwargs...)) + +materialize_broadcasted(f, args...) = Broadcast.materialize(Broadcast.broadcasted(f, args...)) + +@static if isdefined(Base, :typesplit) + const typesplit = Base.typesplit +else + Base.@pure function typesplit(@nospecialize(a), @nospecialize(b)) + a <: b && return Base.Bottom + isa(a, Union) && return Union{typesplit(a.a, b), typesplit(a.b, b)} + return a + end +end + +# ensure backwards compatibility - have to use `return_types` which can return a vector of types +@static if isdefined(Base, :infer_return_type) + infer_return_type(args...) = Base.infer_return_type(args...) +else + function infer_return_type(args...) + inftypes = Base.return_types(args...) + return only(inftypes) + end +end + +end \ No newline at end of file From 3feccd56e9434d62885c8d8810528df2cd81163e Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Fri, 21 Nov 2025 00:51:29 +0100 Subject: [PATCH 4/9] complete redesign --- src/TestExtras.jl | 7 +- src/constinferred.jl | 174 ----------------- src/testinferred.jl | 443 ++++++++++++++++++++++++++++++++++++++----- src/utilities.jl | 27 --- 4 files changed, 396 insertions(+), 255 deletions(-) delete mode 100644 src/constinferred.jl delete mode 100644 src/utilities.jl diff --git a/src/TestExtras.jl b/src/TestExtras.jl index 94dd896..eebadde 100644 --- a/src/TestExtras.jl +++ b/src/TestExtras.jl @@ -1,13 +1,11 @@ module TestExtras +export @testinferred, @testinferred_broken export @constinferred, @constinferred_broken export @timedtestset export @include -export @testinferred export ConstInferred -include("utilities.jl") -include("constinferred.jl") include("testinferred.jl") include("includemacro.jl") @@ -17,8 +15,7 @@ else include("timedtest.jl") end -using .ConstInferred: @constinferred, @constinferred_broken -using .TestInferred: @testinferred +using .TestInferred: @constinferred, @constinferred_broken, @testinferred, @testinferred_broken using .TimedTests: @timedtestset end diff --git a/src/constinferred.jl b/src/constinferred.jl deleted file mode 100644 index 549c7d3..0000000 --- a/src/constinferred.jl +++ /dev/null @@ -1,174 +0,0 @@ -module ConstInferred -export @constinferred, @constinferred_broken - -const ConstantValue = Union{Number,Char,QuoteNode} - -using Test -using Test: Returned, Threw - -using ..Utilities: typesplit, materialize_broadcasted - - -_enabled = Ref(true) -enable() = (_enabled[] = true; return nothing) -disable() = (_enabled[] = false; return nothing) - - -""" - @constinferred [AllowedType] f(x) -Tests that the call expression `f(x)` returns a value of the same type inferred by the -compiler. It is useful to check for type stability. Similar to `Test.@inferred`, but -differs in two important ways. - -Firstly, `@constinferred` tries to test type stability under constant propagation by -testing type stability of a new function, where all arguments or keyword arguments of -the original function `f` that have a constant value (in the call expression) of type -`Union{Number,Char,Symbol}` are hard coded. - -If you want to test for constant propagation in a variable which is not hard-coded in -the call expression, you can interpolate it into the expression. - -Secondly, @constinferred returns the value if type inference succeeds, like `@inferred`, -but used the `Test.@test` mechanism and shows up as an actual test error when type -inference fails. -``` -""" -macro constinferred(ex) - return _constinferred(ex, __module__, __source__, Test.do_test) -end -macro constinferred(allow, ex) - return _constinferred(ex, __module__, __source__, Test.do_test, allow) -end - -macro constinferred_broken(ex) - return _constinferred(ex, __module__, __source__, Test.do_broken_test) -end - -macro constinferred_broken(allow, ex) - return _constinferred(ex, __module__, __source__, Test.do_broken_test, allow) -end - -function _constinferred(ex, mod, src, test_f, allow=:(Union{})) - if Meta.isexpr(ex, :ref) - ex = Expr(:call, :getindex, ex.args...) - end - Meta.isexpr(ex, :call) || error("@constinferred requires a call expression") - farg = ex.args[1] - if isa(farg, Symbol) && first(string(farg)) == '.' - farg = Symbol(string(farg)[2:end]) - ex = Expr(:call, GlobalRef(@__MODULE__, :materialize_broadcasted), - farg, ex.args[2:end]...) - end - pre = quote - $(esc(allow)) isa Type || - throw(ArgumentError("@constinferred requires a type as second argument")) - end - if length(ex.args) > 1 && Meta.isexpr(ex.args[2], :parameters) - kwargs = ex.args[2].args - args = ex.args[3:end] - elseif length(ex.args) > 1 && Meta.isexpr(ex.args[2], :kw) - kwargs = ex.args[2:end] - args = Any[] - else - kwargs = Any[] - args = ex.args[2:end] - end - newf = gensym() - rightargs = Any[] - rightkwargs = Any[] - leftargs = Any[] - callargs = Any[] - quoteargs = Any[] - for x in args - if x isa ConstantValue - push!(rightargs, x) - elseif Meta.isexpr(x, :$) - s = gensym() - push!(rightargs, s) - push!(quoteargs, Expr(:(=), s, x)) - else - s = gensym() - push!(leftargs, s) - if Meta.isexpr(x, :...) - push!(rightargs, Expr(:..., s)) - push!(callargs, Expr(:tuple, esc(x))) - else - push!(rightargs, s) - push!(callargs, esc(x)) - end - end - end - for x in kwargs - if x isa Expr && x.head == :kw - xkey = x.args[1] - xval = x.args[2] - elseif x isa Symbol - xkey = x - xval = x - else - return Expr(:call, :error, - "syntax: invalid keyword argument syntax \"$x\" at $src") - end - if xval isa ConstantValue - push!(rightkwargs, x) - elseif Meta.isexpr(xval, :$) - s = gensym() - push!(rightkwargs, Expr(:kw, xkey, s)) - push!(quoteargs, Expr(:(=), s, xval)) - else - s = gensym() - push!(rightkwargs, Expr(:kw, xkey, s)) - push!(leftargs, s) - push!(callargs, esc(xval)) - end - end - f = Expr(:$, ex.args[1]) - - fundefhead = Expr(:tuple, leftargs...) - fundefbody = Expr(:block, quoteargs..., - isempty(kwargs) ? - Expr(:call, f, rightargs...) : - Expr(:call, f, Expr(:parameters, rightkwargs...), rightargs...)) - fundefex = esc(Expr(:quote, Expr(:(=), newf, Expr(:->, fundefhead, fundefbody)))) - - inftypes = gensym() - rettype = gensym() - result = esc(gensym()) - newfcall = Expr(:., mod, QuoteNode(newf)) - main = quote - callargs = ($(callargs...),) - $result = $newfcall(callargs...) - $(esc(inftypes)) = Base.return_types($newfcall, Base.typesof(callargs...)) - $(esc(rettype)) = $result isa Type ? Type{$result} : typeof($result) - end - orig_ex = Expr(:inert, Expr(:macrocall, Symbol("@constinferred"), nothing, ex)) - post = quote - if $(_enabled[]) - if length($(esc(inftypes))) > 1 - testresult = Threw(ArgumentError("more than one inferred type"), - Base.catch_stack(), $(QuoteNode(src))) - else - v = $(esc(rettype)) <: $(esc(allow)) || - $(esc(rettype)) == typesplit($(esc(inftypes))[1], $(esc(allow))) - testresult = Returned(v, - Expr(:call, :!=, $(esc(rettype)), - $(esc(inftypes))[1]), $(QuoteNode(src))) - end - - $test_f(testresult, $orig_ex) - end - $result - end - latestworld = isdefined(Core, :var"@latestworld") ? :(@Core.latestworld) : nothing - finalex = Base.remove_linenums!(quote - $pre - Core.eval($mod, $fundefex) - $latestworld - let - $main - $post - end - end) - return finalex -end -end diff --git a/src/testinferred.jl b/src/testinferred.jl index 6ffa82b..74f48f9 100644 --- a/src/testinferred.jl +++ b/src/testinferred.jl @@ -1,74 +1,419 @@ module TestInferred +export @testinferred, @testinferred_broken +export @constinferred, @constinferred_broken -export @testinferred +const ConstantValue = Union{Number, Char, QuoteNode} + +using Test +using Test: Returned, Threw, do_test, do_broken_test using InteractiveUtils: gen_call_with_extracted_types -using Test: @test -using ..Utilities: args_and_call, materialize_broadcasted, typesplit, infer_return_type -using InteractiveUtils: gen_call_with_extracted_types +_enabled = Ref(true) +enable() = (_enabled[] = true; return nothing) +disable() = (_enabled[] = false; return nothing) + +# Some utility functions +function materialize_broadcasted(f, args...) + return Broadcast.materialize(Broadcast.broadcasted(f, args...)) +end + +# this function extracts the parts of a type `a` that are not subtypes of `b`, +# which makes sense only for `a` being a Union type. +@static if isdefined(Base, :typesplit) + const typesplit = Base.typesplit +else + Base.@pure function typesplit(@nospecialize(a), @nospecialize(b)) + a <: b && return Base.Bottom + isa(a, Union) && return Union{typesplit(a.a, b),typesplit(a.b, b)} + return a + end +end + +# ensure backwards compatibility - have to use `return_types` which can return a vector of types +@static if isdefined(Base, :infer_return_type) + infer_return_type(args...) = Base.infer_return_type(args...) +else + function infer_return_type(args...) + inftypes = Base.return_types(args...) + return only(inftypes) + end +end + +function parsekwargs(macroname, kwargexprs...) + kwargs = Any[] + for arg in kwargexprs + if !Meta.isexpr(arg, :(=)) + error("$macroname: invalid expression for keyword argument $arg") + end + key = arg.args[1] + if !(key ∈ (:broken, :constprop)) + error("$macroname: unknown keyword argument \"$(key)\"") + end + val = arg.args[2] + if key == :constprop && !(val isa Bool) + error("$macroname: only `true` or `false` allowed for value of keyword argument `constprop`") + end + push!(kwargs, key => val) + end + return kwargs +end + +# macro definitions """ - @testinferred [AllowedType] f(x) + @testinferred [AllowedType] f(x) [constprop=true|false] [broken=true|false] + @testinferred_broken [AllowedType] f(x) [constprop=true|false] + @constinferred [AllowedType] f(x) [broken=true|false] + @constinferred_broken [AllowedType] f(x) + +Tests that the call expression `f(x)` returns a value of the same type inferred by the +compiler. This is useful to test for type stability. It is similar to `Test.@inferred`, +but in contrast to `Test.@inferred` the result of `f(x)` is always returned and +the success or failure of type inference is reported to the passed and failed test count. + +`f(x)` can be any call expression, including broadcasting expressions. + +Optionally, `AllowedType` relaxes the test, by making it pass when either the type of `f(x)` +matches the inferred type modulo `AllowedType`, or when the return type is a subtype of +`AllowedType`. This is useful when testing type stability of functions returning a small +union such as `Union{Nothing, T}` or `Union{Missing, T}`. + +Furthermore, the keyword argument `constprop` can be used to enable constant propagation +while testing for type inferrability. Constant propagation is applied for all arguments +and keyword arguments that have a constant value (in the call expression) of type +`Union{Number,Char,Symbol}`. If you want to test for constant propagation in a variable +which is not hard-coded in the call expression, you can interpolate it into the expression. +Note that `constprop` is `false` by default, and can only have an explicit `true` or `false` +value. + +!!! note + Interpolating values into the call expression is only possible with `constprop = true`. + +!!! warning + With `constprop = true`, a new temporary function is created, which is not possible + within the scope of another function. + +Finally, the keyword argument `broken` can be used to test that type inference fails. Here, +the value of `broken` can be a general expression that evaluates to `true` or `false`. + +Alternatively to the keyword argument `constprop=true`, you can use the `@constinferred` +macro, which has constant propagation enabled by default. Similarly, you can use the +macros `@testinferred_broken` and `@constinferred_broken` to test for broken type inference. + +```jldoctest +julia> f(a) = a < 10 ? missing : 1.0 +f (generic function with 1 method) + +julia> @testinferred f(2) +Test Failed at REPL[54]:1 + Expression: @testinferred f(2) + Evaluated: Missing != Union{Missing, Float64} + +ERROR: There was an error during testing + +julia> @constinferred f(2) # with constant propagation enabled +missing + +julia> x = 2; @testinferred f(x) +Test Failed at REPL[55]:1 + Expression: @testinferred f(x) + Evaluated: Missing != Union{Missing, Float64} + +ERROR: There was an error during testing + +julia> x = 2; @constinferred f(x) +Test Failed at REPL[57]:1 + Expression: @constinferred f(x) + Evaluated: Missing != Union{Missing, Float64} + +ERROR: There was an error during testing + +julia> x = 2; @constinferred f(\$x) +missing + +julia> x = 2; @testinferred_broken f(x) +missing + +julia> broken = true; @testinferred f(x) broken = broken +missing + +julia> x = 2; @constinferred_broken f(x) +missing + +julia> @testinferred Missing f(2) +missing -Tests that the call expression `f(x)` returns a value of the same type inferred by the compiler. -It is useful to check for type stability. -This is similar to `Test.@inferred`, but instead of throwing an error, a `@test` is added. +julia> h() = (@testinferred_broken f(2)); h() +missing -Optionally, `AllowedType` relaxes the test, by making it pass when either the type of `f(x)` matches the inferred type modulo `AllowedType`, or when the return type is a subtype of `AllowedType`. -This is useful when testing type stability of functions returning a small union such as `Union{Nothing, T}` or `Union{Missing, T}`. +ERROR: There was an error during testing + +julia> h() = (@constinferred_broken f(2)); h() +ERROR: syntax: World age increment not at top level +Stacktrace: + [1] top-level scope +``` """ -macro testinferred(ex) - return _inferred(ex, __module__) +macro testinferred(args...) + orig_ex = Expr(:inert, Expr(:macrocall, Symbol("@testinferred"), nothing, args...)) + kwargstart = something(findfirst(x -> Meta.isexpr(x, :(=)), args), length(args) + 1) + if kwargstart > 3 + error("@testinferred: invalid expression") + end + kwargs = parsekwargs("@testinferred", args[kwargstart:end]...) + if kwargstart == 3 + allow = args[1] + ex = args[2] + return _testinferred(ex, orig_ex, __module__, __source__, allow; kwargs...) + else + ex = args[1] + return _testinferred(ex, orig_ex, __module__, __source__; kwargs...) + end end - -macro testinferred(allow, ex) - return _inferred(ex, __module__, allow) +macro testinferred_broken(args...) + orig_ex = Expr(:inert, Expr(:macrocall, Symbol("@testinferred_broken"), nothing, args...)) + kwargstart = something(findfirst(x -> Meta.isexpr(x, :(=)), args), length(args) + 1) + if kwargstart > 3 + error("@testinferred_broken: invalid expression") + end + kwargs = parsekwargs("@testinferred_broken", Expr(:(=), :broken, true), args[kwargstart:end]...) + if kwargstart == 3 + allow = args[1] + ex = args[2] + return _testinferred(ex, orig_ex, __module__, __source__, allow; kwargs...) + else + ex = args[1] + return _testinferred(ex, orig_ex, __module__, __source__; kwargs...) + end +end +macro constinferred(args...) + orig_ex = Expr(:inert, Expr(:macrocall, Symbol("@constinferred"), nothing, args...)) + kwargstart = something(findfirst(x -> Meta.isexpr(x, :(=)), args), length(args) + 1) + if kwargstart > 3 + error("@constinferred: invalid expression") + end + kwargs = parsekwargs("@constinferred", Expr(:(=), :constprop, true), args[kwargstart:end]...) + if kwargstart == 3 + allow = args[1] + ex = args[2] + return _testinferred(ex, orig_ex, __module__, __source__, allow; kwargs...) + else + ex = args[1] + return _testinferred(ex, orig_ex, __module__, __source__; kwargs...) + end +end +macro constinferred_broken(args...) + orig_ex = Expr(:inert, Expr(:macrocall, Symbol("@constinferred_broken"), nothing, args...)) + kwargstart = something(findfirst(x -> Meta.isexpr(x, :(=)), args), length(args) + 1) + if kwargstart > 3 + error("@constinferred_broken: invalid expression") + end + kwargs = parsekwargs("@constinferred_broken", Expr(:(=), :broken, true), Expr(:(=), :constprop, true), args[kwargstart:end]...) + if kwargstart == 3 + allow = args[1] + ex = args[2] + return _testinferred(ex, orig_ex, __module__, __source__, allow; kwargs...) + else + ex = args[1] + return _testinferred(ex, orig_ex, __module__, __source__; kwargs...) + end end -function _inferred(ex, mod, allow = :(Union{})) +function _testinferred(ex, orig_ex, mod, src, allow = :(Union{}); constprop = false, broken = false) if Meta.isexpr(ex, :ref) ex = Expr(:call, :getindex, ex.args...) end - Meta.isexpr(ex, :call)|| error("@testinferred requires a call expression") - - # handle broadcasting expressions + Meta.isexpr(ex, :call) || error("@constinferred requires a call expression") farg = ex.args[1] - if isa(farg, Symbol) && farg !== :.. && first(string(farg)) == '.' + if isa(farg, Symbol) && first(string(farg)) == '.' farg = Symbol(string(farg)[2:end]) - ex = Expr(:call, GlobalRef(@__MODULE__, :materialize_broadcasted), farg, ex.args[2:end]...) - end - - result = let ex = ex - quote - let allow = $(esc(allow)) - allow isa Type || throw(ArgumentError("@testinferred requires a type as second argument")) - $( - if any(@nospecialize(a) -> (Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex.args) - # Has keywords - # Create the call expression with escaped user expressions - call_expr = :($(esc(ex.args[1]))(args...; kwargs...)) - quote - args, kwargs, result = $(esc(Expr(:call, args_and_call, ex.args[2:end]..., ex.args[1]))) - # wrap in dummy hygienic-scope to work around scoping issues with `call_expr` already having `esc` on the necessary parts - inftype = $(Expr(:var"hygienic-scope", gen_call_with_extracted_types(mod, infer_return_type, call_expr), @__MODULE__)) - end - else - # No keywords - quote - args = ($([esc(ex.args[i]) for i in 2:length(ex.args)]...),) - result = $(esc(ex.args[1]))(args...) - inftype = $(GlobalRef(@__MODULE__, :infer_return_type))($(esc(ex.args[1])), Base.typesof(args...)) - end - end + ex = Expr( + :call, GlobalRef(@__MODULE__, :materialize_broadcasted), + farg, ex.args[2:end]... + ) + end + pre1 = quote + $(esc(allow)) isa Type || + throw(ArgumentError("@constinferred requires a type as second argument")) + end + # extract args and kwargs + if length(ex.args) > 1 && Meta.isexpr(ex.args[2], :parameters) + kwargs = ex.args[2].args + args = ex.args[3:end] + elseif length(ex.args) > 1 && Meta.isexpr(ex.args[2], :kw) + kwargs = ex.args[2:end] + args = Any[] + else + kwargs = Any[] + args = ex.args[2:end] + end + + if constprop + callexpr, pre2 = make_callexpr_constprop(ex.args[1], args, kwargs, mod) + else + callexpr, pre2 = make_callexpr(ex.args[1], args, kwargs, mod) + end + + inftype = esc(gensym()) + rettype = esc(gensym()) + result = esc(gensym()) + main = quote + $result = $callexpr + if $(_enabled[]) + $inftype = $( + Expr( + :var"hygienic-scope", + gen_call_with_extracted_types( + mod, + infer_return_type, + callexpr + ), + @__MODULE__ ) - rettype = result isa Type ? Type{result} : typeof(result) - @test (rettype <: allow || rettype == typesplit(inftype, allow)) - result + ) + $rettype = $result isa Type ? Type{$result} : typeof($result) + v = $rettype <: $(esc(allow)) || + $rettype == typesplit($inftype, $(esc(allow))) + testresult = Returned( + v, Expr(:call, :!=, $rettype, $inftype), + $(QuoteNode(src)) + ) + if $(esc(broken)) + $(Test.do_broken_test)(testresult, $orig_ex) + else + $(Test.do_test)(testresult, $orig_ex) end end + $result end + finalex = quote + $pre1 + $pre2 + let + $main + end + end + return Base.remove_linenums!(finalex) +end + +function make_callexpr_constprop(f, args, kwargs, mod) + newf = gensym() + rightargs = Any[] + rightkwargs = Any[] + leftargs = Any[] + callargs = Any[] + quoteargs = Any[] + for x in args + if x isa ConstantValue + push!(rightargs, x) + elseif Meta.isexpr(x, :$) + s = gensym() + push!(rightargs, s) + push!(quoteargs, Expr(:(=), s, x)) + else + s = gensym() + push!(leftargs, s) + if Meta.isexpr(x, :...) + push!(rightargs, Expr(:..., s)) + push!(callargs, Expr(:tuple, esc(x))) + else + push!(rightargs, s) + push!(callargs, esc(x)) + end + end + end + for x in kwargs + if x isa Expr && x.head == :kw + xkey = x.args[1] + xval = x.args[2] + elseif x isa Symbol + xkey = x + xval = x + else + return Expr( + :call, :error, + "syntax: invalid keyword argument syntax \"$x\" at $src" + ) + end + if xval isa ConstantValue + push!(rightkwargs, x) + elseif Meta.isexpr(xval, :$) + s = gensym() + push!(rightkwargs, Expr(:kw, xkey, s)) + push!(quoteargs, Expr(:(=), s, xval)) + else + s = gensym() + push!(rightkwargs, Expr(:kw, xkey, s)) + push!(leftargs, s) + push!(callargs, esc(xval)) + end + end + farg = Expr(:$, f) + fundefhead = Expr(:tuple, leftargs...) + fundefbody = Expr( + :block, quoteargs..., + isempty(kwargs) ? + Expr(:call, farg, rightargs...) : + Expr(:call, farg, Expr(:parameters, rightkwargs...), rightargs...) + ) + fundefex = esc(Expr(:quote, Expr(:(=), newf, Expr(:->, fundefhead, fundefbody)))) + # call expression + callex = Expr(:call, Expr(:., mod, QuoteNode(newf)), callargs...) + latestworld = isdefined(Core, :var"@latestworld") ? :(Core.@latestworld) : nothing + pre = quote + Core.eval($mod, $fundefex) + $latestworld + end + return callex, pre +end - return Base.remove_linenums!(result) +function make_callexpr(f, args, kwargs, mod) + callargs = Any[] + callkwargs = Any[] + preargs = Any[] + for x in args + if x isa ConstantValue + push!(callargs, x) + elseif x isa Symbol + push!(callargs, esc(x)) + elseif Meta.isexpr(x, :$) + error("value interpolation with `\$` is not supported in @constinferred without constant propagation") + else + s = gensym() + if Meta.isexpr(x, :...) + push!(callargs, Expr(:..., s)) + push!(preargs, Expr(:(=), s, Expr(:tuple, esc(x)))) + else + push!(callargs, s) + push!(preargs, Expr(:(=), s, esc(x))) + end + end + end + for x in kwargs + if x isa Expr && x.head == :kw + xkey = x.args[1] + xval = x.args[2] + elseif x isa Symbol + xkey = x + xval = x + else + error("syntax: invalid keyword argument syntax \"$x\" at $src") + end + if xval isa Symbol || xval isa ConstantValue + push!(callkwargs, x) + elseif Meta.isexpr(xval, :$) + error("value interpolation with `\$` is not supported in @constinferred without constant propagation") + else + s = gensym() + push!(callkwargs, Expr(:kw, xkey, s)) + push!(preargs, Expr(:(=), s, esc(xval))) + end + end + pre = Expr(:block, preargs...) + callexpr = isempty(kwargs) ? + Expr(:call, esc(f), callargs...) : + Expr(:call, esc(f), Expr(:parameters, callkwargs...), callargs...) + return callexpr, pre end end diff --git a/src/utilities.jl b/src/utilities.jl deleted file mode 100644 index c6ee475..0000000 --- a/src/utilities.jl +++ /dev/null @@ -1,27 +0,0 @@ -module Utilities - -args_and_call((args..., f)...; kwargs...) = (args, kwargs, f(args...; kwargs...)) - -materialize_broadcasted(f, args...) = Broadcast.materialize(Broadcast.broadcasted(f, args...)) - -@static if isdefined(Base, :typesplit) - const typesplit = Base.typesplit -else - Base.@pure function typesplit(@nospecialize(a), @nospecialize(b)) - a <: b && return Base.Bottom - isa(a, Union) && return Union{typesplit(a.a, b), typesplit(a.b, b)} - return a - end -end - -# ensure backwards compatibility - have to use `return_types` which can return a vector of types -@static if isdefined(Base, :infer_return_type) - infer_return_type(args...) = Base.infer_return_type(args...) -else - function infer_return_type(args...) - inftypes = Base.return_types(args...) - return only(inftypes) - end -end - -end \ No newline at end of file From 3dea8780b20f8822c4f5ee5015792adb0590d357 Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Sat, 29 Nov 2025 01:10:15 +0100 Subject: [PATCH 5/9] some more tests and coverage --- test/runtests.jl | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index d502aa7..c0ba028 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,14 +10,25 @@ using Test end @constinferred mysqrt(+3) - @constinferred mysqrt(-3) + @testinferred mysqrt(-3) constprop = true + @testinferred mysqrt(-3) constprop = false broken = true + brokenval = true + @testinferred mysqrt(-3) constprop = false broken = brokenval + @testinferred mysqrt(-3) constprop = false broken = VERSION > v"1" for x in -1.5:0.5:+1.5 @constinferred mysqrt($x) - @constinferred mysqrt($(rand() < 0 ? x : -x)) + @testinferred mysqrt($(rand() < 0 ? x : -x)) constprop = true end + constprop = false + @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) constprop = constprop) + @test_throws ErrorException @macroexpand(@constinferred mysqrt(-3) constprop = false broken = true x = 6) + @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) constprop = VERSION > v"1") + @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) this_is_not_a_keyword = true) + @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) this_is_not_valid) + @constinferred Nothing iterate(1:5) - @constinferred Nothing iterate(1:-1) + @testinferred Nothing iterate(1:-1) constprop = true @constinferred Tuple{Int,Int} iterate(1:-1) x = (2, 3) @@ -33,6 +44,11 @@ using Test @constinferred_broken mysqrt(x; complex=true) complex = false @constinferred_broken mysqrt(x; complex) + @constinferred_broken mysqrt(x; complex = complex) + @constinferred_broken mysqrt(x; complex = VERSION < v"1") + @constinferred mysqrt(x; complex) broken = true + @testinferred mysqrt(x; complex) broken = false + @testinferred mysqrt(x) broken = true end # ensure constinferred only evaluates argument once From 0030561c01511cd231ec0ee862d7c43f8879d34c Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Sat, 29 Nov 2025 22:55:07 +0100 Subject: [PATCH 6/9] final fix --- src/testinferred.jl | 4 +++- test/runtests.jl | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/testinferred.jl b/src/testinferred.jl index 74f48f9..11b3647 100644 --- a/src/testinferred.jl +++ b/src/testinferred.jl @@ -399,8 +399,10 @@ function make_callexpr(f, args, kwargs, mod) else error("syntax: invalid keyword argument syntax \"$x\" at $src") end - if xval isa Symbol || xval isa ConstantValue + if xval isa ConstantValue push!(callkwargs, x) + elseif x isa Symbol + push!(callkwargs, esc(x)) elseif Meta.isexpr(xval, :$) error("value interpolation with `\$` is not supported in @constinferred without constant propagation") else diff --git a/test/runtests.jl b/test/runtests.jl index c0ba028..ceb01a0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,7 +2,7 @@ using TestExtras using Test @timedtestset "constinferred tests" begin - function mysqrt(x; complex=true) + function mysqrt(x; complex::Bool=true) return x >= 0 ? sqrt(x) : (complex ? im * sqrt(-x) : throw(DomainError(x, @@ -47,7 +47,7 @@ using Test @constinferred_broken mysqrt(x; complex = complex) @constinferred_broken mysqrt(x; complex = VERSION < v"1") @constinferred mysqrt(x; complex) broken = true - @testinferred mysqrt(x; complex) broken = false + @testinferred mysqrt(x; complex) broken = true @testinferred mysqrt(x) broken = true end From b10570c65bd55859ed6f8ac30bcee1e41b9a7740 Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Sat, 29 Nov 2025 22:59:53 +0100 Subject: [PATCH 7/9] update version and readme --- Project.toml | 2 +- README.md | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Project.toml b/Project.toml index 47cff1b..fc5624d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TestExtras" uuid = "5ed8adda-3752-4e41-b88a-e8b09835ee3a" authors = ["Jutho Haegeman and contributors"] -version = "0.3.2" +version = "0.3.3" [deps] InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" diff --git a/README.md b/README.md index 1f288b9..67a2176 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,16 @@ This package adds useful additions to the functionality provided by `Test`, the Julia standard library for writing tests. -# What's new in version 0.2 +# What's new in version 0.3.3 -* The way in which `@constinferred` defines a new function has been changed so as to avoid - "method overwrite" warnings in Julia v1.10 and higher. -* Since Julia v1.8, the default `Test.@testset` prints elapsed timings of tests, thereby - replicating the behaviour of `@timedtestset` and making the latter superfluous. However, - `@timedtestset` and its associated test set type `TimedTestSet` remains available in - TestExtras.jl in order to support older Julia versions. It is now a literal copy of the - `DefaultTestSet` implementation as in the `Test` standard library of Julia 1.8 and thus - also supports the `verbose` keyword. +* Introduction of a `@testinferred` macro, that unlike `@constinferred` can be used inside + functions, but does not apply constant propagation. However, unlike `Test.@inferred`, it + does contribute to the test results. # Short description of the package -The first feature of TestExtras.jl is a macro `@constinferred`, which is a replacement of -`Test.@inferred` but with two major differences. +The first feature of TestExtras.jl are the macros `@testinferred` and `@constinferred`, +which are a replacement of `Test.@inferred` but with two major differences. 1. Unlike `Test.@inferred`, the comparison between the actual and inferred runtype is a proper test which contributes to the total number of passed or failed tests. @@ -39,6 +34,12 @@ The first feature of TestExtras.jl is a macro `@constinferred`, which is a repl in the test, for example because you want to loop over possible values. In that case, you can interpolate the value into the `@constinferred` expression. + However, because this macro works by defining a new function, it cannot be used inside + other functions. Therefore, `@testinferred` is provided as a variant that works inside + functions, but without constant propagation. In fact, `@constinferred f(args...)` is + equivalent to `@testinferred f(args...) constprop=true`, where the optional keyword argument + `constprop` (default `false`) controls whether constant propagation is applied. + Some example is probably more insightful. We define a new function `mysqrt` that is type-unstable with respect to real values of the argument `x`, at least if the keyword argument `complex = true`. From ff9d79a61b555cce209f58732e573df62a8afc59 Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Sat, 29 Nov 2025 23:06:18 +0100 Subject: [PATCH 8/9] final test correction --- test/runtests.jl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index ceb01a0..8d514b5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -21,11 +21,16 @@ using Test end constprop = false - @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) constprop = constprop) - @test_throws ErrorException @macroexpand(@constinferred mysqrt(-3) constprop = false broken = true x = 6) - @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) constprop = VERSION > v"1") - @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) this_is_not_a_keyword = true) - @test_throws ErrorException @macroexpand(@testinferred mysqrt(-3) this_is_not_valid) + errortype = @static if VERSION < v"1.7" + LoadError + else + ErrorException + end + @test_throws errortype @macroexpand(@testinferred mysqrt(-3) constprop = constprop) + @test_throws errortype @macroexpand(@constinferred mysqrt(-3) constprop = false broken = true x = 6) + @test_throws errortype @macroexpand(@testinferred mysqrt(-3) constprop = VERSION > v"1") + @test_throws errortype @macroexpand(@testinferred mysqrt(-3) this_is_not_a_keyword = true) + @test_throws errortype @macroexpand(@testinferred mysqrt(-3) this_is_not_valid) @constinferred Nothing iterate(1:5) @testinferred Nothing iterate(1:-1) constprop = true From 23ce71b2a4e56b83b3f845915cd2ca1ee557de29 Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Sat, 29 Nov 2025 23:34:21 +0100 Subject: [PATCH 9/9] one more improvement --- src/testinferred.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/testinferred.jl b/src/testinferred.jl index 11b3647..0108696 100644 --- a/src/testinferred.jl +++ b/src/testinferred.jl @@ -400,9 +400,9 @@ function make_callexpr(f, args, kwargs, mod) error("syntax: invalid keyword argument syntax \"$x\" at $src") end if xval isa ConstantValue - push!(callkwargs, x) + push!(callkwargs, Expr(:kw, xkey, xval)) elseif x isa Symbol - push!(callkwargs, esc(x)) + push!(callkwargs, Expr(:kw, xkey, esc(xval))) elseif Meta.isexpr(xval, :$) error("value interpolation with `\$` is not supported in @constinferred without constant propagation") else