Merge pull request #24 from JuliaDebug/teh/limit

Run Julia tests with resource limits, with worker process, and collate the results

.gitignore

@@ -5,4 +5,5 @@ expected.out
 failed.out
 src/builtins.jl
 deps/build.log
-docs/build/
+docs/build/
+test/results.md

Manifest.toml

@@ -3,10 +3,24 @@
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
 [[InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

Project.toml

@@ -6,7 +6,9 @@ version = "0.1.1"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [extras]
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Test", "Distributed", "Random"]

src/JuliaInterpreter.jl

@@ -151,6 +151,7 @@ function show_stackloc(io::IO, stack, frame, pc=frame.pc[])
     end
     println(io, indent, frame.code.scope, ", pc = ", convert(Int, pc))
 end
+show_stackloc(stack, frame, pc=frame.pc[]) = show_stackloc(stderr, stack, frame, pc)
 
 function moduleof(x)
     if isa(x, JuliaStackFrame)

src/interpret.jl

@@ -157,7 +157,7 @@ function evaluate_foreigncall!(stack, frame::JuliaStackFrame, call_expr::Expr, p
     return Core.eval(moduleof(frame), Expr(:foreigncall, args...))
 end
 
-function evaluate_call!(::Compiled, frame::JuliaStackFrame, call_expr::Expr, pc)
+function evaluate_call!(::Compiled, frame::JuliaStackFrame, call_expr::Expr, pc;  #=unused=# exec!::Function=finish_and_return!)
     ret = maybe_evaluate_builtin(frame, call_expr)
     isa(ret, Some{Any}) && return ret.value
     fargs = collect_args(frame, call_expr)
@@ -172,7 +172,7 @@ function evaluate_call!(::Compiled, frame::JuliaStackFrame, call_expr::Expr, pc)
     return ret
 end
 
-function evaluate_call!(stack, frame::JuliaStackFrame, call_expr::Expr, pc)
+function evaluate_call!(stack, frame::JuliaStackFrame, call_expr::Expr, pc; exec!::Function=finish_and_return!)
     ret = maybe_evaluate_builtin(frame, call_expr)
     isa(ret, Some{Any}) && return ret.value
     fargs = collect_args(frame, call_expr)
@@ -193,7 +193,7 @@ function evaluate_call!(stack, frame::JuliaStackFrame, call_expr::Expr, pc)
     frame.pc[] = pc  # to mark position in the frame (e.g., if we hit breakpoint or error)
     push!(stack, frame)
     newframe = build_frame(framecode, fargs, lenv)
-    ret = finish_and_return!(stack, newframe)
+    ret = exec!(stack, newframe)
     pop!(stack)
     push!(junk, newframe)  # rather than going through GC, just re-use it
     return ret
@@ -333,6 +333,8 @@ end
 
 function _step_expr!(stack, frame, @nospecialize(node), pc::JuliaProgramCounter, istoplevel::Bool)
     local rhs
+    # show_stackloc(stack, frame, pc)
+    # @show node
     try
         if isa(node, Expr)
             if node.head == :(=)
@@ -419,20 +421,7 @@ function _step_expr!(stack, frame, @nospecialize(node), pc::JuliaProgramCounter,
             rhs = @lookup(frame, node)
         end
     catch err
-        # Check for world age errors, which generally indicate a failure to go back to toplevel
-        if isa(err, MethodError)
-            is_arg_types = isa(err.args, DataType)
-            arg_types = is_arg_types ? err.args : Base.typesof(err.args...)
-            if (err.world != typemax(UInt) &&
-                hasmethod(err.f, arg_types) &&
-                !hasmethod(err.f, arg_types, world = err.world))
-                @warn "likely failure to return to toplevel, try Base.invokelatest"
-                rethrow(err)
-            end
-        end
-        isempty(frame.exception_frames) && rethrow(err)
-        frame.last_exception[] = err
-        return JuliaProgramCounter(frame.exception_frames[end])
+        return handle_err(frame, err)
     end
     if isassign(frame, pc)
         if !@isdefined(rhs)
@@ -461,6 +450,23 @@ function step_expr!(stack, frame, istoplevel::Bool=false)
     frame.pc[] = pc
 end
 
+function handle_err(frame, err)
+    # Check for world age errors, which generally indicate a failure to go back to toplevel
+    if isa(err, MethodError)
+        is_arg_types = isa(err.args, DataType)
+        arg_types = is_arg_types ? err.args : Base.typesof(err.args...)
+        if (err.world != typemax(UInt) &&
+            hasmethod(err.f, arg_types) &&
+            !hasmethod(err.f, arg_types, world = err.world))
+            @warn "likely failure to return to toplevel, try Base.invokelatest"
+            rethrow(err)
+        end
+    end
+    isempty(frame.exception_frames) && rethrow(err)
+    frame.last_exception[] = err
+    return JuliaProgramCounter(frame.exception_frames[end])
+end
+
 """
     pc = finish!(stack, frame, pc=frame.pc[])
 
@@ -491,7 +497,7 @@ Optionally supply the starting `pc`, if you don't want to start at the current l
 function finish_and_return!(stack, frame, pc::JuliaProgramCounter=frame.pc[], istoplevel::Bool=false)
     pc = finish!(stack, frame, pc, istoplevel)
     node = pc_expr(frame, pc)
-    isexpr(node, :return) || error("unexpected node ", node)
+    isexpr(node, :return) || error("unexpected return statement ", node)
     return @lookup(frame, (node::Expr).args[1])
 end
 finish_and_return!(stack, frame, istoplevel::Bool) = finish_and_return!(stack, frame, frame.pc[], istoplevel)

test/juliatests.jl

@@ -1,5 +1,7 @@
 using JuliaInterpreter
-using Test, Random
+using Test, Random, InteractiveUtils, Distributed
+
+# Much of this file is taken from Julia's test/runtests.jl file.
 
 if !isdefined(Main, :read_and_parse)
     include("utils.jl")
@@ -10,67 +12,114 @@ const testdir = joinpath(juliadir, "test")
 if isdir(testdir)
     include(joinpath(testdir, "choosetests.jl"))
 else
-    @warn "Julia's test/ directory not found, skipping Julia tests"
+    @error "Julia's test/ directory not found, can't run Julia tests"
 end
 
-module JuliaTests
-using Test
+function test_path(test)
+    t = split(test, '/')
+    if t[1] in STDLIBS
+        if length(t) == 2
+            return joinpath(STDLIB_DIR, t[1], "test", t[2])
+        else
+            return joinpath(STDLIB_DIR, t[1], "test", "runtests")
+        end
+    else
+        return joinpath(testdir, test)
+    end
 end
 
-@testset "Julia tests" begin
-    # To do this efficiently, certain methods must be run in Compiled mode
-    cm = JuliaInterpreter.compiled_methods
-    empty!(cm)
-    push!(cm, which(Test.eval_test, Tuple{Expr, Expr, LineNumberNode}))
-    push!(cm, which(Test.finish, Tuple{Test.DefaultTestSet}))
-    push!(cm, which(Test.get_testset, Tuple{}))
-    push!(cm, which(Test.push_testset, Tuple{Test.AbstractTestSet}))
-    push!(cm, which(Test.pop_testset, Tuple{}))
-    push!(cm, which(Random.seed!, Tuple{Union{Integer,Vector{UInt32}}}))
-    push!(cm, which(copy!, Tuple{Random.MersenneTwister, Random.MersenneTwister}))
-    push!(cm, which(copy, Tuple{Random.MersenneTwister}))
-    push!(cm, which(Base.include, Tuple{Module, String}))
-    push!(cm, which(Base.show_backtrace, Tuple{IO, Vector}))
-    push!(cm, which(Base.show_backtrace, Tuple{IO, Vector{Any}}))
+nstmts = 10^4  # very quick, aborts a lot
+i = 1
+while i <= length(ARGS)
+    global i
+    a = ARGS[i]
+    if a == "--nstmts"
+        global nstmts = parse(Int, ARGS[i+1])
+        deleteat!(ARGS, i:i+1)
+    else
+        i += 1
+    end
+end
+
+tests, _, exit_on_error, seed = choosetests(ARGS)
+
+function spin_up_workers(n)
+    procs = addprocs(n)
+    @sync begin
+        @async for p in procs
+            remotecall_wait(include, p, "utils.jl")
+            remotecall_wait(configure_test, p)
+        end
+    end
+    return procs
+end
 
-    stack = JuliaStackFrame[]
-    function runtest(frame)
-        empty!(stack)
-        # empty!(JuliaInterpreter.framedict)
-        # empty!(JuliaInterpreter.genframedict)
-        return JuliaInterpreter.finish_and_return!(stack, frame, true)
+# Really, we're just going to skip all the tests that run on node1
+const node1_tests = String[]
+function move_to_node1(t)
+    if t in tests
+        splice!(tests, findfirst(isequal(t), tests))
+        push!(node1_tests, t)
     end
-    function dotest!(test)
-        println("Working on ", test, "...")
-        fullpath = joinpath(testdir, test)*".jl"
-        ex = read_and_parse(fullpath)
-        # so `include` works properly, we have to set up the relative path
-        oldpath = current_task().storage[:SOURCE_PATH]
-        if isexpr(ex, :error)
-            @error "error parsing $test: $ex"
-        else
-            try
-                current_task().storage[:SOURCE_PATH] = fullpath
-                lower_incrementally(runtest, JuliaTests, ex)
-                # Core.eval(JuliaTests, ex)
-                println("Finished ", test)
-            finally
-                current_task().storage[:SOURCE_PATH] = oldpath
+    nothing
+end
+move_to_node1("precompile")
+move_to_node1("SharedArrays")
+move_to_node1("stress")
+move_to_node1("Distributed")
+
+@testset "Julia tests" begin
+    nworkers = min(Sys.CPU_THREADS, length(tests))
+    println("Using $nworkers workers")
+    procs = spin_up_workers(nworkers)
+    results = Dict{String,Any}()
+    tests0 = copy(tests)
+    @sync begin
+        for p in procs
+            @async begin
+                while length(tests) > 0
+                    test = popfirst!(tests)
+                    local resp
+                    wrkr = p
+                    fullpath = test_path(test*".jl")
+                    try
+                        resp = remotecall_fetch(dotest, p, test, fullpath, nstmts)
+                    catch e
+                        isa(e, InterruptException) && return
+                        resp = e
+                        if isa(e, ProcessExitedException)
+                            p = spin_up_workers(1)[1]
+                        end
+                    end
+                    results[test] = resp
+                    if resp isa Exception && exit_on_error
+                        skipped = length(tests)
+                        empty!(tests)
+                    end
+                end
             end
         end
     end
-    if isdir(testdir)
-        tests, _ = choosetests()
-        delayed = []
-        for test in tests
-            if startswith(test, "compiler") || test == "subarray"
-                push!(delayed, test)
+
+    open("results.md", "w") do io
+        versioninfo(io)
+        println(io, "Maximum number of statements per lowered expression: ", nstmts)
+        println(io)
+        println(io, "| Test file | Passes | Fails | Errors | Broken | Aborted blocks |")
+        println(io, "| --------- | ------:| -----:| ------:| ------:| --------------:|")
+        for test in tests0
+            result = results[test]
+            if isa(result, Tuple{Test.AbstractTestSet, Vector})
+                ts, aborts = result
+                passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = Test.get_test_counts(ts)
+                naborts = length(aborts)
+                println(io, "| ", test, " | ", passes+c_passes, " | ", fails+c_fails, " | ", errors+c_errors, " | ", broken+c_broken, " | ", naborts, " |")
+            elseif isa(result, ProcessExitedException)
+                println(io, "| ", test, " | ☠️ | ☠️ | ☠️ | ☠️ | ☠️ |")
             else
-                dotest!(test)
+                println(test, " => ", result)
+                println(io, "| ", test, " | X | X | X | X | X |")
             end
         end
-        for test in delayed
-            dotest!(failed, test)
-        end
     end
 end