Skip to content

Commit

Permalink
enhance the effectiveness of the test cases introduced in #53300
Browse files Browse the repository at this point in the history
While experimenting with precompilation for external absints on builds
just after #53300 was merged, I found that the test case for
`CustomAbstractInterpreterCaching2.jl` fails if the test case for
`CustomAbstractInterpreterCaching1.jl` isn't run in the same session
beforehand. That is probably because of the previous lack of support
for proper `CodeInstance` caching. To address this, I've changed the
tests to run in separate processes in this commit. Note that it appears
that a recent refactor concerning `CodeInstance` might have resolved
this issue, so the new test cases runs successfully on master. However,
I suspect the fix hasn't been applied to v1.11 yet, we would need more
research.
  • Loading branch information
aviatesk committed Mar 14, 2024
1 parent 5023ee2 commit bdb3966
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 182 deletions.
189 changes: 7 additions & 182 deletions test/precompile.jl
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

original_depot_path = copy(Base.DEPOT_PATH)
original_load_path = copy(Base.LOAD_PATH)

using Test, Distributed, Random, Logging
using REPL # doc lookup function

include("precompile_utils.jl")

Foo_module = :Foo4b3a94a1a081a8cb
Foo2_module = :F2oo4b3a94a1a081a8cb
FooBase_module = :FooBase4b3a94a1a081a8cb
Expand All @@ -16,37 +15,6 @@ FooBase_module = :FooBase4b3a94a1a081a8cb
end
using .ConflictingBindings

function precompile_test_harness(@nospecialize(f), testset::String)
@testset "$testset" begin
precompile_test_harness(f, true)
end
end
function precompile_test_harness(@nospecialize(f), separate::Bool)
load_path = mktempdir()
load_cache_path = separate ? mktempdir() : load_path
try
pushfirst!(LOAD_PATH, load_path)
pushfirst!(DEPOT_PATH, load_cache_path)
f(load_path)
finally
try
rm(load_path, force=true, recursive=true)
catch err
@show err
end
if separate
try
rm(load_cache_path, force=true, recursive=true)
catch err
@show err
end
end
filter!(()(load_path), LOAD_PATH)
separate && filter!(()(load_cache_path), DEPOT_PATH)
end
nothing
end

# method root provenance

rootid(m::Module) = Base.module_build_id(Base.parentmodule(m)) % UInt64
Expand Down Expand Up @@ -1713,150 +1681,10 @@ precompile_test_harness("issue #46296") do load_path
(@eval (using CodeInstancePrecompile))
end

let newinterp_path = abspath("compiler/newinterp.jl")
precompile_test_harness("AbstractInterpreter caching") do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

write(joinpath(load_path, "CustomAbstractInterpreterCaching.jl"), :(module CustomAbstractInterpreterCaching
import SimpleModule: basic_caller, basic_callee

module Custom
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching"))
@eval let
using CustomAbstractInterpreterCaching
cache_owner = Core.Compiler.cache_owner(
CustomAbstractInterpreterCaching.Custom.PrecompileInterpreter())
let m = only(methods(CustomAbstractInterpreterCaching.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi in Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end

write(joinpath(load_path, "CustomAbstractInterpreterCaching2.jl"), :(module CustomAbstractInterpreterCaching2
import SimpleModule: basic_caller, basic_callee

module Custom
const CC = Core.Compiler
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
struct CustomData
inferred
CustomData(@nospecialize inferred) = new(inferred)
end
function CC.transform_result_for_cache(interp::PrecompileInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
return CustomData(inferred_result)
end
function CC.src_inlining_policy(interp::PrecompileInterpreter, @nospecialize(src),
@nospecialize(info::CC.CallInfo), stmt_flag::UInt32)
if src isa CustomData
src = src.inferred
end
return @invoke CC.src_inlining_policy(interp::CC.AbstractInterpreter, src::Any,
info::CC.CallInfo, stmt_flag::UInt32)
end
CC.retrieve_ir_for_inlining(cached_result::Core.CodeInstance, src::CustomData) =
CC.retrieve_ir_for_inlining(cached_result, src.inferred)
CC.retrieve_ir_for_inlining(mi::Core.MethodInstance, src::CustomData, preserve_local_sources::Bool) =
CC.retrieve_ir_for_inlining(mi, src.inferred, preserve_local_sources)
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching2"))
@eval let
using CustomAbstractInterpreterCaching2
cache_owner = Core.Compiler.cache_owner(
CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter())
let m = only(methods(CustomAbstractInterpreterCaching2.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi = Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end
@testset "Precompile external abstract interpreter" begin
dir = @__DIR__
@test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint1.jl`; dir); stdout, stderr))
@test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint2.jl`; dir); stdout, stderr))
end

precompile_test_harness("Recursive types") do load_path
Expand Down Expand Up @@ -2098,7 +1926,4 @@ precompile_test_harness("Test flags") do load_path
@test !Base.isprecompiled(id, ;flags=current_flags)
end

empty!(Base.DEPOT_PATH)
append!(Base.DEPOT_PATH, original_depot_path)
empty!(Base.LOAD_PATH)
append!(Base.LOAD_PATH, original_load_path)
finish_precompile_test!()
73 changes: 73 additions & 0 deletions test/precompile_absint1.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test

include("precompile_utils.jl")

precompile_test_harness() do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

newinterp_path = abspath("compiler/newinterp.jl")
write(joinpath(load_path, "TestAbsIntPrecompile1.jl"), :(module TestAbsIntPrecompile1
import SimpleModule: basic_caller, basic_callee

module Custom
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("TestAbsIntPrecompile1"))

@eval let
using TestAbsIntPrecompile1
cache_owner = Core.Compiler.cache_owner(
TestAbsIntPrecompile1.Custom.PrecompileInterpreter())
let m = only(methods(TestAbsIntPrecompile1.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi in Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end

finish_precompile_test!()
96 changes: 96 additions & 0 deletions test/precompile_absint2.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test

include("precompile_utils.jl")

precompile_test_harness() do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

newinterp_path = abspath("compiler/newinterp.jl")
write(joinpath(load_path, "TestAbsIntPrecompile2.jl"), :(module TestAbsIntPrecompile2
import SimpleModule: basic_caller, basic_callee

module Custom
const CC = Core.Compiler
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
struct CustomData
inferred
CustomData(@nospecialize inferred) = new(inferred)
end
function CC.transform_result_for_cache(interp::PrecompileInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
return CustomData(inferred_result)
end
function CC.src_inlining_policy(interp::PrecompileInterpreter, @nospecialize(src),
@nospecialize(info::CC.CallInfo), stmt_flag::UInt32)
if src isa CustomData
src = src.inferred
end
return @invoke CC.src_inlining_policy(interp::CC.AbstractInterpreter, src::Any,
info::CC.CallInfo, stmt_flag::UInt32)
end
CC.retrieve_ir_for_inlining(cached_result::Core.CodeInstance, src::CustomData) =
CC.retrieve_ir_for_inlining(cached_result, src.inferred)
CC.retrieve_ir_for_inlining(mi::Core.MethodInstance, src::CustomData, preserve_local_sources::Bool) =
CC.retrieve_ir_for_inlining(mi, src.inferred, preserve_local_sources)
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("TestAbsIntPrecompile2"))

@eval let
using TestAbsIntPrecompile2
cache_owner = Core.Compiler.cache_owner(
TestAbsIntPrecompile2.Custom.PrecompileInterpreter())
let m = only(methods(TestAbsIntPrecompile2.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi = Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end

finish_precompile_test!()
Loading

0 comments on commit bdb3966

Please sign in to comment.