From 638117d01007be274a61becb3a7613a168c75893 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 15 Oct 2025 21:22:38 +0200 Subject: [PATCH 1/5] Support custom records --- Project.toml | 2 +- src/ParallelTestRunner.jl | 25 ++++++----- test/runtests.jl | 91 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/Project.toml b/Project.toml index 6c83409..20ac4d7 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ParallelTestRunner" uuid = "d3525ed8-44d0-4b2c-a655-542cee43accc" authors = ["Valentin Churavy "] -version = "1.0.2" +version = "1.1.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 074e161..965ed4c 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -1,6 +1,7 @@ module ParallelTestRunner export runtests, addworkers, addworker +public extract_flag! using Distributed using Dates @@ -100,7 +101,7 @@ struct TestIOContext rss_align::Int end -function test_IOContext(::Type{TestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int) +function test_IOContext(::Type{<:AbstractTestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int) elapsed_align = textwidth("Time (s)") gc_align = textwidth("GC (s)") percent_align = textwidth("GC %") @@ -115,7 +116,7 @@ function test_IOContext(::Type{TestRecord}, stdout::IO, stderr::IO, lock::Reentr ) end -function print_header(::Type{TestRecord}, ctx::TestIOContext, testgroupheader, workerheader) +function print_header(::Type{<:AbstractTestRecord}, ctx::TestIOContext, testgroupheader, workerheader) lock(ctx.lock) try printstyled(ctx.stdout, " "^(ctx.name_align + textwidth(testgroupheader) - 3), " │ ") @@ -129,7 +130,7 @@ function print_header(::Type{TestRecord}, ctx::TestIOContext, testgroupheader, w end end -function print_test_started(::Type{TestRecord}, wrkr, test, ctx::TestIOContext) +function print_test_started(::Type{<:AbstractTestRecord}, wrkr, test, ctx::TestIOContext) lock(ctx.lock) try printstyled(ctx.stdout, test, lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │", color = :white) @@ -143,7 +144,7 @@ function print_test_started(::Type{TestRecord}, wrkr, test, ctx::TestIOContext) end end -function print_test_finished(record::TestRecord, wrkr, test, ctx::TestIOContext) +function print_test_finished(record::AbstractTestRecord, wrkr, test, ctx::TestIOContext) lock(ctx.lock) try printstyled(ctx.stdout, test, color = :white) @@ -158,7 +159,7 @@ function print_test_finished(record::TestRecord, wrkr, test, ctx::TestIOContext) alloc_str = @sprintf("%5.2f", record.bytes / 2^20) printstyled(ctx.stdout, lpad(alloc_str, ctx.alloc_align, " "), " │ ", color = :white) - rss_str = @sprintf("%5.2f", record.rss / 2^20) + rss_str = @sprintf("%5.2f", memory_usage(record) / 2^20) printstyled(ctx.stdout, lpad(rss_str, ctx.rss_align, " "), " │\n", color = :white) flush(ctx.stdout) @@ -167,7 +168,7 @@ function print_test_finished(record::TestRecord, wrkr, test, ctx::TestIOContext) end end -function print_test_failed(record::TestRecord, wrkr, test, ctx::TestIOContext) +function print_test_failed(record::AbstractTestRecord, wrkr, test, ctx::TestIOContext) lock(ctx.lock) try printstyled(ctx.stderr, test, color = :red) @@ -193,7 +194,7 @@ function print_test_failed(record::TestRecord, wrkr, test, ctx::TestIOContext) end end -function print_test_crashed(::Type{TestRecord}, wrkr, test, ctx::TestIOContext) +function print_test_crashed(::Type{<:AbstractTestRecord}, wrkr, test, ctx::TestIOContext) lock(ctx.lock) try printstyled(ctx.stderr, test, color = :red) @@ -212,9 +213,9 @@ end # # entry point -# +# -function runtest(::Type{TestRecord}, f, name, init_code, color) +function runtest(::Type{TestRecord}, f, name, init_code, color, custom_args) function inner() # generate a temporary module to execute the tests in mod = @eval(Main, module $(gensym(name)) end) @@ -466,7 +467,8 @@ Workers are automatically recycled when they exceed memory limits to prevent out issues during long test runs. The memory limit is set based on system architecture. """ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = TestRecord, - custom_tests::Dict{String, Expr}=Dict{String, Expr}(), init_code = :(), + custom_tests::Dict{String, Expr}=Dict{String, Expr}(), init_code = :(), + custom_record_init = :(), custom_args = (;), test_worker = Returns(nothing), stdout = Base.stdout, stderr = Base.stderr) # # set-up @@ -789,8 +791,9 @@ function runtests(mod::Module, ARGS; test_filter = Returns(true), RecordType = T put!(printer_channel, (:started, test, wrkr)) result = try Distributed.remotecall_eval(Main, wrkr, :(import ParallelTestRunner)) + custom_record_init != :() && Distributed.remotecall_eval(Main, wrkr, custom_record_init) remotecall_fetch(runtest, wrkr, RecordType, test_runners[test], test, - init_code, io_ctx.color) + init_code, io_ctx.color, custom_args) catch ex if isa(ex, InterruptException) # the worker got interrupted, signal other tasks to stop diff --git a/test/runtests.jl b/test/runtests.jl index 35912e2..f116b28 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -75,6 +75,97 @@ end @test contains(str, "SUCCESS") end +@testset "custom testrecord" begin + custom_record_init = quote + struct CustomTestRecord <: ParallelTestRunner.AbstractTestRecord + # TODO: Would it be better to wrap "ParallelTestRunner.TestRecord " + value::Any # AbstractTestSet or TestSetException + output::String # captured stdout/stderr + + # stats + time::Float64 + bytes::UInt64 + gctime::Float64 + rss::UInt64 + end + function ParallelTestRunner.memory_usage(rec::CustomTestRecord) + return rec.rss + end + function ParallelTestRunner.test_IOContext(::Type{CustomTestRecord}, args...) + return ParallelTestRunner.test_IOContext(ParallelTestRunner.TestRecord, args...) + end + function ParallelTestRunner.runtest(::Type{CustomTestRecord}, f, name, init_code, color, (; say_hello)) + function inner() + # generate a temporary module to execute the tests in + mod = @eval(Main, module $(gensym(name)) end) + @eval(mod, import ParallelTestRunner: Test, Random) + @eval(mod, using .Test, .Random) + + Core.eval(mod, init_code) + + data = @eval mod begin + GC.gc(true) + Random.seed!(1) + + mktemp() do path, io + stats = redirect_stdio(stdout=io, stderr=io) do + @timed try + if say_hello + println("Hello from test '$name'") + end + @testset $name begin + $f + end + catch err + isa(err, Test.TestSetException) || rethrow() + + # return the error to package it into a TestRecord + err + end + end + close(io) + output = read(path, String) + (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) + + end + end + + # process results + rss = Sys.maxrss() + record = TestRecord(data..., rss) + + GC.gc(true) + return record + end + + @static if VERSION >= v"1.13.0-DEV.1044" + @with Test.TESTSET_PRINT_ENABLE => false begin + inner() + end + else + old_print_setting = Test.TESTSET_PRINT_ENABLE[] + Test.TESTSET_PRINT_ENABLE[] = false + try + inner() + finally + Test.TESTSET_PRINT_ENABLE[] = old_print_setting + end + end + end + end # quote + eval(custom_record_init) + + io = IOBuffer() + + runtests(ParallelTestRunner, ["--verbose"]; custom_record_init, custom_args=(; say_hello=true), stdout=io, stderr=io) + str = String(take!(io)) + + @test contains(str, r"basic .+ started at") + @test contains(str, r"Hello from test 'basic'") + @test contains(str, "SUCCESS") +end + + @testset "failing test" begin custom_tests = Dict( "failing test" => quote From bb47b349f2876c01236e95593da3e5efe012b10f Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 15 Oct 2025 21:27:40 +0200 Subject: [PATCH 2/5] fixup! Support custom records --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index f116b28..48d714e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -157,7 +157,7 @@ end io = IOBuffer() - runtests(ParallelTestRunner, ["--verbose"]; custom_record_init, custom_args=(; say_hello=true), stdout=io, stderr=io) + runtests(ParallelTestRunner, ["--verbose"]; custom_record_init, RecordType=CustomTestRecord, custom_args=(; say_hello=true), stdout=io, stderr=io) str = String(take!(io)) @test contains(str, r"basic .+ started at") From ed34667cf9dcde7531fef6d1b219673bdb258f69 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 15 Oct 2025 23:35:03 +0200 Subject: [PATCH 3/5] fix test for CustomTestRecord --- test/runtests.jl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 48d714e..28d3069 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -77,6 +77,7 @@ end @testset "custom testrecord" begin custom_record_init = quote + import ParallelTestRunner: Test struct CustomTestRecord <: ParallelTestRunner.AbstractTestRecord # TODO: Would it be better to wrap "ParallelTestRunner.TestRecord " value::Any # AbstractTestSet or TestSetException @@ -91,13 +92,13 @@ end function ParallelTestRunner.memory_usage(rec::CustomTestRecord) return rec.rss end - function ParallelTestRunner.test_IOContext(::Type{CustomTestRecord}, args...) - return ParallelTestRunner.test_IOContext(ParallelTestRunner.TestRecord, args...) + function ParallelTestRunner.test_IOContext(::Type{CustomTestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int64) + return ParallelTestRunner.test_IOContext(ParallelTestRunner.TestRecord, stdout, stderr, lock, name_align) end function ParallelTestRunner.runtest(::Type{CustomTestRecord}, f, name, init_code, color, (; say_hello)) function inner() # generate a temporary module to execute the tests in - mod = @eval(Main, module $(gensym(name)) end) + mod = Core.eval(Main, Expr(:module, true, gensym(name), Expr(:block))) @eval(mod, import ParallelTestRunner: Test, Random) @eval(mod, using .Test, .Random) @@ -110,11 +111,12 @@ end mktemp() do path, io stats = redirect_stdio(stdout=io, stderr=io) do @timed try - if say_hello - println("Hello from test '$name'") + # Since we are in a double quote we need to use this form to escape `$` + if $(Expr(:$, :say_hello)) + println("Hello from test '" * $(Expr(:$, :name)) * "'") end - @testset $name begin - $f + @testset $(Expr(:$, :name)) begin + $(Expr(:$, :f)) end catch err isa(err, Test.TestSetException) || rethrow() @@ -132,7 +134,7 @@ end # process results rss = Sys.maxrss() - record = TestRecord(data..., rss) + record = CustomTestRecord(data..., rss) GC.gc(true) return record From f45b75749d21c3ec8b9e85b2be571c0152412861 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 15 Oct 2025 23:37:13 +0200 Subject: [PATCH 4/5] public ... --- src/ParallelTestRunner.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 965ed4c..6931cea 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -1,7 +1,7 @@ module ParallelTestRunner export runtests, addworkers, addworker -public extract_flag! +# public extract_flag! using Distributed using Dates From b4bdff520df58609435781c53e265a18f3033b6a Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 16 Oct 2025 03:29:57 +0200 Subject: [PATCH 5/5] refactor inner --- src/ParallelTestRunner.jl | 73 ++++++++++++++++++----------------- test/runtests.jl | 81 ++++++++++++++------------------------- 2 files changed, 66 insertions(+), 88 deletions(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 6931cea..5cf6da3 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -1,7 +1,9 @@ module ParallelTestRunner export runtests, addworkers, addworker -# public extract_flag! +if VERSION >= v"1.11.0-DEV.469" + eval(Meta.parse("public extract_flag!")) +end using Distributed using Dates @@ -215,56 +217,55 @@ end # entry point # -function runtest(::Type{TestRecord}, f, name, init_code, color, custom_args) - function inner() - # generate a temporary module to execute the tests in - mod = @eval(Main, module $(gensym(name)) end) - @eval(mod, import ParallelTestRunner: Test, Random) - @eval(mod, using .Test, .Random) - - Core.eval(mod, init_code) - - data = @eval mod begin - GC.gc(true) - Random.seed!(1) - - mktemp() do path, io - stats = redirect_stdio(stdout=io, stderr=io) do - @timed try - @testset $name begin - $f - end - catch err - isa(err, Test.TestSetException) || rethrow() +function execute(::Type{TestRecord}, mod, f, name, color, custom_args)::TestRecord + data = @eval mod begin + GC.gc(true) + Random.seed!(1) - # return the error to package it into a TestRecord - err + mktemp() do path, io + stats = redirect_stdio(stdout=io, stderr=io) do + @timed try + @testset $name begin + $f end - end - close(io) - output = read(path, String) - (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) + catch err + isa(err, Test.TestSetException) || rethrow() + # return the error to package it into a TestRecord + err + end end + close(io) + output = read(path, String) + (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) end + end - # process results - rss = Sys.maxrss() - record = TestRecord(data..., rss) + # process results + rss = Sys.maxrss() + record = TestRecord(data..., rss) - GC.gc(true) - return record - end + GC.gc(true) + return record +end + +function runtest(RecordType::Type{<:AbstractTestRecord}, f, name, init_code, color, custom_args) + # generate a temporary module to execute the tests in + mod = Core.eval(Main, Expr(:module, true, gensym(name), Expr(:block))) + @eval(mod, import ParallelTestRunner: Test, Random) + @eval(mod, using .Test, .Random) + + Core.eval(mod, init_code) @static if VERSION >= v"1.13.0-DEV.1044" @with Test.TESTSET_PRINT_ENABLE => false begin - inner() + execute(RecordType, mod, f, name, color, custom_args) end else old_print_setting = Test.TESTSET_PRINT_ENABLE[] Test.TESTSET_PRINT_ENABLE[] = false try - inner() + execute(RecordType, mod, f, name, color, custom_args) finally Test.TESTSET_PRINT_ENABLE[] = old_print_setting end diff --git a/test/runtests.jl b/test/runtests.jl index 28d3069..3facc15 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -95,64 +95,41 @@ end function ParallelTestRunner.test_IOContext(::Type{CustomTestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int64) return ParallelTestRunner.test_IOContext(ParallelTestRunner.TestRecord, stdout, stderr, lock, name_align) end - function ParallelTestRunner.runtest(::Type{CustomTestRecord}, f, name, init_code, color, (; say_hello)) - function inner() - # generate a temporary module to execute the tests in - mod = Core.eval(Main, Expr(:module, true, gensym(name), Expr(:block))) - @eval(mod, import ParallelTestRunner: Test, Random) - @eval(mod, using .Test, .Random) - - Core.eval(mod, init_code) - - data = @eval mod begin - GC.gc(true) - Random.seed!(1) - - mktemp() do path, io - stats = redirect_stdio(stdout=io, stderr=io) do - @timed try - # Since we are in a double quote we need to use this form to escape `$` - if $(Expr(:$, :say_hello)) - println("Hello from test '" * $(Expr(:$, :name)) * "'") - end - @testset $(Expr(:$, :name)) begin - $(Expr(:$, :f)) - end - catch err - isa(err, Test.TestSetException) || rethrow() - - # return the error to package it into a TestRecord - err + function ParallelTestRunner.execute(::Type{CustomTestRecord}, mod, f, name, color, (; say_hello)) + data = @eval mod begin + GC.gc(true) + Random.seed!(1) + + mktemp() do path, io + stats = redirect_stdio(stdout=io, stderr=io) do + @timed try + # Since we are in a double quote we need to use this form to escape `$` + if $(Expr(:$, :say_hello)) + println("Hello from test '" * $(Expr(:$, :name)) * "'") end - end - close(io) - output = read(path, String) - (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) + @testset $(Expr(:$, :name)) begin + $(Expr(:$, :f)) + end + catch err + isa(err, Test.TestSetException) || rethrow() + # return the error to package it into a TestRecord + err + end end - end - - # process results - rss = Sys.maxrss() - record = CustomTestRecord(data..., rss) + close(io) + output = read(path, String) + (; testset=stats.value, output, stats.time, stats.bytes, stats.gctime) - GC.gc(true) - return record - end - - @static if VERSION >= v"1.13.0-DEV.1044" - @with Test.TESTSET_PRINT_ENABLE => false begin - inner() - end - else - old_print_setting = Test.TESTSET_PRINT_ENABLE[] - Test.TESTSET_PRINT_ENABLE[] = false - try - inner() - finally - Test.TESTSET_PRINT_ENABLE[] = old_print_setting end end + + # process results + rss = Sys.maxrss() + record = CustomTestRecord(data..., rss) + + GC.gc(true) + return record end end # quote eval(custom_record_init)