Add exitcodes and xfail options

src/core.jl

@@ -25,6 +25,9 @@ See also [`Run.test`](@ref) and [`Run.docs`](@ref).
 - `check_bounds::Union{Nothing, Bool} = nothing`: Control
   `--check-bounds` option.  `nothing` means to inherit the option
   specified for the current Julia session.
+- `xfail::bool = false`: If failure is expected.
+- `exitcodes::AbstractVector{<:Integer} = xfail ? [1] : [0]`: List of
+  allowed exit codes.
 - Other keywords are passed to `Run.prepare_test`.
 """
 script
@@ -135,9 +138,41 @@ function Base.show(io::IO, ::MIME"text/plain", result::Result)
     if exitcode !== 0
         print(io, " ")
         printstyled(io, "(exit code: ", exitcode, ")"; color=:red)
+        println(io)
+        println(io, "Command: ", Cmd(result.proc.cmd.exec))
+        print(io, "Environment variables:")
+        _printenv(io, something(result.proc.cmd.env, ENV))
     end
 end
 
+function _printenv(io, env)
+    for nv in env
+        if nv isa AbstractString
+            name, value = split(nv, "=", limit = 2)
+        else
+            name, value = nv
+        end
+        # regex taken from `versioninfo`:
+        if startswith(name, "JULIA") || occursin(r"PATH|FLAG|^TERM$|HOME", name)
+            println(io)
+            print(io, "  ", name, " = ", value)
+        end
+    end
+end
+
+struct Failed <: Exception
+    result::Result
+end
+
+function Base.show(io::IO, ::MIME"text/plain", failed::Failed)
+    print(io, "Failed ")
+    show(io, MIME"text/plain"(), failed.result)
+end
+
+function Base.showerror(io::IO, failed::Failed)
+    show(io, MIME"text/plain"(), failed)
+end
+
 function prepare(projectpath; precompile=true, parentproject=nothing)
     projectpath = dirname(existingproject(projectpath))
     code = prepare_code
@@ -188,6 +223,8 @@ function script(
     compiled_modules = nothing,
     precompile = (compiled_modules != false),
     parentproject = nothing,
+    xfail::Bool = false,
+    exitcodes::AbstractVector{<:Integer} = xfail ? [1] : [0],
     kwargs...,
 )
     if get(ENV, "CI", "false") == "true"
@@ -226,7 +263,13 @@ function script(
         end
         @info "Running $script"
         cmd = setenv(`$(_julia_cmd()) $julia_options $script`, env)
-        return Result("run finished", run(cmd))
+        proc = run(ignorestatus(cmd))
+        result = Result("run finished", proc)
+        if proc.exitcode in exitcodes
+            return result
+        else
+            throw(Failed(result))
+        end
     end
 end
 # Note: Copying toml files to make it work nicely with running script

test/fail/fail.jl

@@ -0,0 +1 @@
+exit(1)

test/pass/pass.jl

@@ -0,0 +1 @@
+exit(0)

test/runtests.jl

@@ -1,7 +1,55 @@
 using Run
+using Run: Result, Failed
 using Pkg
 using Test
 
+pass_script = joinpath(@__DIR__, "pass", "pass.jl")
+fail_script = joinpath(@__DIR__, "fail", "fail.jl")
+
+@testset "xfail" begin
+    @testset "true pass" begin
+        result = nothing
+        @test begin
+            result = Run.test(pass_script)
+        end isa Result
+        @test result.proc.exitcode == 0
+        @test sprint(show, "text/plain", result) isa String
+    end
+    @testset "true failure" begin
+        err = nothing
+        @test try
+            Run.test(fail_script)
+            false
+        catch err
+            true
+        end
+        @test err.result.proc.exitcode == 1
+        @test sprint(show, "text/plain", err) isa String
+    end
+
+    @testset "expected failure" begin
+        result = nothing
+        @test begin
+            result = Run.test(fail_script; xfail = true)
+        end isa Result
+        @test result.proc.exitcode == 1
+        @test sprint(show, "text/plain", result) isa String
+        @test sprint(showerror, result) isa String
+    end
+    @testset "unexpected pass" begin
+        err = nothing
+        @test try
+            Run.test(pass_script; xfail = true)
+            false
+        catch err
+            true
+        end
+        @test err.result.proc.exitcode == 0
+        @test sprint(show, "text/plain", err) isa String
+        @test sprint(showerror, err) isa String
+    end
+end
+
 @testset "smoke test" begin
     withenv("DOCUMENTER_KEY" => nothing) do
         @test Run.docs(joinpath(@__DIR__, "..", "docs")) isa Any