diff --git a/docs/src/index.md b/docs/src/index.md
index beca059d..ae6d42eb 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -58,15 +58,12 @@ Keep this in mind when testing discontinuous rules for functions like [ReLU](htt
 ```jldoctest ex; output = false
 using ChainRulesTestUtils
 
-test_frule(two2three, 3.33, -7.77)
+test_frule(two2three, 3.33, -7.77);
+
 # output
-Test Summary:                    | Pass  Total
-Tuple{Float64,Float64,Float64}.1 |    1      1
-Test Summary:                    | Pass  Total
-Tuple{Float64,Float64,Float64}.2 |    1      1
-Test Summary:                    | Pass  Total
-Tuple{Float64,Float64,Float64}.3 |    1      1
-Test Passed
+Test Summary:                          | Pass  Total
+test_frule: two2three at (3.33, -7.77) |    5      5
+Test.DefaultTestSet("test_frule: two2three at (3.33, -7.77)", Any[Test.DefaultTestSet("Tuple{Float64,Float64,Float64}.1", Any[], 1, false), Test.DefaultTestSet("Tuple{Float64,Float64,Float64}.2", Any[], 1, false), Test.DefaultTestSet("Tuple{Float64,Float64,Float64}.3", Any[], 1, false)], 2, false)
 ```
 
 ### Testing the `rrule`
@@ -75,11 +72,12 @@ Test Passed
 The call will test the `rrule` for function `f` at the point `x`, and similarly to `frule` some rules should be tested at multiple points in the domain.
 
 ```jldoctest ex; output = false
-test_rrule(two2three, 3.33, -7.77)
+test_rrule(two2three, 3.33, -7.77);
+
 # output
-Test Summary:                      |
-Don't thunk only non_zero argument | No tests
-Test.DefaultTestSet("Don't thunk only non_zero argument", Any[], 0, false)
+Test Summary:                          | Pass  Total
+test_rrule: two2three at (3.33, -7.77) |    6      6
+Test.DefaultTestSet("test_rrule: two2three at (3.33, -7.77)", Any[Test.DefaultTestSet("Don't thunk only non_zero argument", Any[], 0, false)], 6, false)
 ```
 
 ## Scalar example
@@ -104,18 +102,15 @@ with the `frule` and `rrule` defined with the help of `@scalar_rule` macro
 `test_scalar` function is provided to test both the `frule` and the `rrule` with a single
 call.
 ```jldoctest ex; output = false
-test_scalar(relu, 0.5)
-test_scalar(relu, -0.5)
+test_scalar(relu, 0.5);
+test_scalar(relu, -0.5);
 
 # output
-Test Summary:                 | Pass  Total
-relu at 0.5, with tangent 1.0 |    3      3
-Test Summary:                   | Pass  Total
-relu at 0.5, with cotangent 1.0 |    4      4
-Test Summary:                  | Pass  Total
-relu at -0.5, with tangent 1.0 |    3      3
-Test Summary:                    | Pass  Total
-relu at -0.5, with cotangent 1.0 |    4      4
+Test Summary:            | Pass  Total
+test_scalar: relu at 0.5 |    7      7
+Test Summary:             | Pass  Total
+test_scalar: relu at -0.5 |    7      7
+Test.DefaultTestSet("test_scalar: relu at -0.5", Any[Test.DefaultTestSet("with tangent 1.0", Any[Test.DefaultTestSet("test_frule: relu at (ChainRulesTestUtils.PrimalAndTangent{Float64,Float64}(-0.5, 1.0),)", Any[], 3, false)], 0, false), Test.DefaultTestSet("with cotangent 1.0", Any[Test.DefaultTestSet("test_rrule: relu at (ChainRulesTestUtils.PrimalAndTangent{Float64,Float64}(-0.5, 1.0),)", Any[Test.DefaultTestSet("Don't thunk only non_zero argument", Any[], 0, false)], 4, false)], 0, false)], 0, false)
 ```
 
 ## Specifying Tangents
diff --git a/src/testers.jl b/src/testers.jl
index f9eaf49c..11900975 100644
--- a/src/testers.jl
+++ b/src/testers.jl
@@ -17,51 +17,53 @@ function test_scalar(f, z; rtol=1e-9, atol=1e-9, fdm=_fdm, fkwargs=NamedTuple(),
     rule_test_kwargs = (; rtol=rtol, atol=atol, fdm=fdm, fkwargs=fkwargs, check_inferred=check_inferred, kwargs...)
     isapprox_kwargs = (; rtol=rtol, atol=atol, kwargs...)
 
-    _ensure_not_running_on_functor(f, "test_scalar")
-    # z = x + im * y
-    # Ω = u(x, y) + im * v(x, y)
-    Ω = f(z; fkwargs...)
-
-    # test jacobian using forward mode
-    Δx = one(z)
-    @testset "$f at $z, with tangent $Δx" begin
-        # check ∂u_∂x and (if Ω is complex) ∂v_∂x via forward mode
-        test_frule(f, z ⊢ Δx; rule_test_kwargs...)
-        if z isa Complex
-            # check that same tangent is produced for tangent 1.0 and 1.0 + 0.0im
-            _, real_tangent = frule((Zero(), real(Δx)), f, z; fkwargs...)
-            _, embedded_tangent = frule((Zero(), Δx), f, z; fkwargs...)
-            check_equal(real_tangent, embedded_tangent; isapprox_kwargs...)
+    @testset "test_scalar: $f at $z" begin
+        _ensure_not_running_on_functor(f, "test_scalar")
+        # z = x + im * y
+        # Ω = u(x, y) + im * v(x, y)
+        Ω = f(z; fkwargs...)
+
+        # test jacobian using forward mode
+        Δx = one(z)
+        @testset "with tangent $Δx" begin
+            # check ∂u_∂x and (if Ω is complex) ∂v_∂x via forward mode
+            test_frule(f, z ⊢ Δx; rule_test_kwargs...)
+            if z isa Complex
+                # check that same tangent is produced for tangent 1.0 and 1.0 + 0.0im
+                _, real_tangent = frule((Zero(), real(Δx)), f, z; fkwargs...)
+                _, embedded_tangent = frule((Zero(), Δx), f, z; fkwargs...)
+                check_equal(real_tangent, embedded_tangent; isapprox_kwargs...)
+            end
         end
-    end
-    if z isa Complex
-        Δy = one(z) * im
-        @testset "$f at $z, with tangent $Δy" begin
-            # check ∂u_∂y and (if Ω is complex) ∂v_∂y via forward mode
-            test_frule(f, z ⊢ Δy; rule_test_kwargs...)
+        if z isa Complex
+            Δy = one(z) * im
+            @testset "with tangent $Δy" begin
+                # check ∂u_∂y and (if Ω is complex) ∂v_∂y via forward mode
+                test_frule(f, z ⊢ Δy; rule_test_kwargs...)
+            end
         end
-    end
 
-    # test jacobian transpose using reverse mode
-    Δu = one(Ω)
-    @testset "$f at $z, with cotangent $Δu" begin
-        # check ∂u_∂x and (if z is complex) ∂u_∂y via reverse mode
-        test_rrule(f, z ⊢ Δx; output_tangent=Δu, rule_test_kwargs...)
-        if Ω isa Complex
-            # check that same cotangent is produced for cotangent 1.0 and 1.0 + 0.0im
-            _, back = rrule(f, z)
-            _, real_cotangent = back(real(Δu))
-            _, embedded_cotangent = back(Δu)
-            check_equal(real_cotangent, embedded_cotangent; isapprox_kwargs...)
+        # test jacobian transpose using reverse mode
+        Δu = one(Ω)
+        @testset "with cotangent $Δu" begin
+            # check ∂u_∂x and (if z is complex) ∂u_∂y via reverse mode
+            test_rrule(f, z ⊢ Δx; output_tangent=Δu, rule_test_kwargs...)
+            if Ω isa Complex
+                # check that same cotangent is produced for cotangent 1.0 and 1.0 + 0.0im
+                _, back = rrule(f, z)
+                _, real_cotangent = back(real(Δu))
+                _, embedded_cotangent = back(Δu)
+                check_equal(real_cotangent, embedded_cotangent; isapprox_kwargs...)
+            end
         end
-    end
-    if Ω isa Complex
-        Δv = one(Ω) * im
-        @testset "$f at $z, with cotangent $Δv" begin
-            # check ∂v_∂x and (if z is complex) ∂v_∂y via reverse mode
-            test_rrule(f, z ⊢ Δx; output_tangent=Δv,  rule_test_kwargs...)
+        if Ω isa Complex
+            Δv = one(Ω) * im
+            @testset "with cotangent $Δv" begin
+                # check ∂v_∂x and (if z is complex) ∂v_∂y via reverse mode
+                test_rrule(f, z ⊢ Δx; output_tangent=Δv,  rule_test_kwargs...)
+            end
         end
-    end
+    end  # top-level testset
 end
 
 
@@ -96,28 +98,30 @@ function test_frule(
     # To simplify some of the calls we make later lets group the kwargs for reuse
     isapprox_kwargs = (; rtol=rtol, atol=atol, kwargs...)
 
-    _ensure_not_running_on_functor(f, "test_frule")
+    @testset "test_frule: $f at $inputs" begin
+        _ensure_not_running_on_functor(f, "test_frule")
 
-    xẋs = auto_primal_and_tangent.(inputs)
-    xs = primal.(xẋs)
-    ẋs = tangent.(xẋs)
-    if check_inferred && _is_inferrable(f, deepcopy(xs)...; deepcopy(fkwargs)...)
-        _test_inferred(frule, (NO_FIELDS, deepcopy(ẋs)...), f, deepcopy(xs)...; deepcopy(fkwargs)...)
-    end
-    res = frule((NO_FIELDS, deepcopy(ẋs)...), f, deepcopy(xs)...; deepcopy(fkwargs)...)
-    res === nothing && throw(MethodError(frule, typeof((f, xs...))))
-    res isa Tuple || error("The frule should return (y, ∂y), not $res.")
-    Ω_ad, dΩ_ad = res
-    Ω = f(deepcopy(xs)...; deepcopy(fkwargs)...)
-    check_equal(Ω_ad, Ω; isapprox_kwargs...)
-
-    ẋs_is_ignored = ẋs .== nothing
-    # Correctness testing via finite differencing.
-    dΩ_fd = _make_jvp_call(fdm, (xs...) -> f(deepcopy(xs)...; deepcopy(fkwargs)...), Ω, xs, ẋs, ẋs_is_ignored)
-    check_equal(dΩ_ad, dΩ_fd; isapprox_kwargs...)
-
-    acc = output_tangent isa Auto ? rand_tangent(Ω) : output_tangent
-    _check_add!!_behaviour(acc, dΩ_ad; rtol=rtol, atol=atol, kwargs...)
+        xẋs = auto_primal_and_tangent.(inputs)
+        xs = primal.(xẋs)
+        ẋs = tangent.(xẋs)
+        if check_inferred && _is_inferrable(f, deepcopy(xs)...; deepcopy(fkwargs)...)
+            _test_inferred(frule, (NO_FIELDS, deepcopy(ẋs)...), f, deepcopy(xs)...; deepcopy(fkwargs)...)
+        end
+        res = frule((NO_FIELDS, deepcopy(ẋs)...), f, deepcopy(xs)...; deepcopy(fkwargs)...)
+        res === nothing && throw(MethodError(frule, typeof((f, xs...))))
+        res isa Tuple || error("The frule should return (y, ∂y), not $res.")
+        Ω_ad, dΩ_ad = res
+        Ω = f(deepcopy(xs)...; deepcopy(fkwargs)...)
+        check_equal(Ω_ad, Ω; isapprox_kwargs...)
+
+        ẋs_is_ignored = ẋs .== nothing
+        # Correctness testing via finite differencing.
+        dΩ_fd = _make_jvp_call(fdm, (xs...) -> f(deepcopy(xs)...; deepcopy(fkwargs)...), Ω, xs, ẋs, ẋs_is_ignored)
+        check_equal(dΩ_ad, dΩ_fd; isapprox_kwargs...)
+
+        acc = output_tangent isa Auto ? rand_tangent(Ω) : output_tangent
+        _check_add!!_behaviour(acc, dΩ_ad; rtol=rtol, atol=atol, kwargs...)
+    end  # top-level testset
 end
 
 
@@ -152,47 +156,49 @@ function test_rrule(
     # To simplify some of the calls we make later lets group the kwargs for reuse
     isapprox_kwargs = (; rtol=rtol, atol=atol, kwargs...)
 
-    _ensure_not_running_on_functor(f, "test_rrule")
+    @testset "test_rrule: $f at $inputs" begin
+        _ensure_not_running_on_functor(f, "test_rrule")
 
-    # Check correctness of evaluation.
-    xx̄s = auto_primal_and_tangent.(inputs)
-    xs = primal.(xx̄s)
-    accumulated_x̄ = tangent.(xx̄s)
-    if check_inferred && _is_inferrable(f, xs...; fkwargs...)
-        _test_inferred(rrule, f, xs...; fkwargs...)
-    end
-    res = rrule(f, xs...; fkwargs...)
-    res === nothing && throw(MethodError(rrule, typeof((f, xs...))))
-    y_ad, pullback = res
-    y = f(xs...; fkwargs...)
-    check_equal(y_ad, y; isapprox_kwargs...)  # make sure primal is correct
-
-    ȳ = output_tangent isa Auto ? rand_tangent(y) : output_tangent
-
-    check_inferred && _test_inferred(pullback, ȳ)
-    ∂s = pullback(ȳ)
-    ∂s isa Tuple || error("The pullback must return (∂self, ∂args...), not $∂s.")
-    ∂self = ∂s[1]
-    x̄s_ad = ∂s[2:end]
-    @test ∂self === NO_FIELDS  # No internal fields
-
-    # Correctness testing via finite differencing.
-    x̄s_is_dne = accumulated_x̄ .== nothing
-    x̄s_fd = _make_j′vp_call(fdm, (xs...) -> f(xs...; fkwargs...), ȳ, xs, x̄s_is_dne)
-    for (accumulated_x̄, x̄_ad, x̄_fd) in zip(accumulated_x̄, x̄s_ad, x̄s_fd)
-        if accumulated_x̄ === nothing  # then we marked this argument as not differentiable
-            @assert x̄_fd === nothing  # this is how `_make_j′vp_call` works
-            @test x̄_ad isa DoesNotExist  # we said it wasn't differentiable.
-        else
-            x̄_ad isa AbstractThunk && check_inferred && _test_inferred(unthunk, x̄_ad)
-
-            # The main test of the actual deriviative being correct:
-            check_equal(x̄_ad, x̄_fd; isapprox_kwargs...)
-            _check_add!!_behaviour(accumulated_x̄, x̄_ad; isapprox_kwargs...)
+        # Check correctness of evaluation.
+        xx̄s = auto_primal_and_tangent.(inputs)
+        xs = primal.(xx̄s)
+        accumulated_x̄ = tangent.(xx̄s)
+        if check_inferred && _is_inferrable(f, xs...; fkwargs...)
+            _test_inferred(rrule, f, xs...; fkwargs...)
+        end
+        res = rrule(f, xs...; fkwargs...)
+        res === nothing && throw(MethodError(rrule, typeof((f, xs...))))
+        y_ad, pullback = res
+        y = f(xs...; fkwargs...)
+        check_equal(y_ad, y; isapprox_kwargs...)  # make sure primal is correct
+
+        ȳ = output_tangent isa Auto ? rand_tangent(y) : output_tangent
+
+        check_inferred && _test_inferred(pullback, ȳ)
+        ∂s = pullback(ȳ)
+        ∂s isa Tuple || error("The pullback must return (∂self, ∂args...), not $∂s.")
+        ∂self = ∂s[1]
+        x̄s_ad = ∂s[2:end]
+        @test ∂self === NO_FIELDS  # No internal fields
+
+        # Correctness testing via finite differencing.
+        x̄s_is_dne = accumulated_x̄ .== nothing
+        x̄s_fd = _make_j′vp_call(fdm, (xs...) -> f(xs...; fkwargs...), ȳ, xs, x̄s_is_dne)
+        for (accumulated_x̄, x̄_ad, x̄_fd) in zip(accumulated_x̄, x̄s_ad, x̄s_fd)
+            if accumulated_x̄ === nothing  # then we marked this argument as not differentiable
+                @assert x̄_fd === nothing  # this is how `_make_j′vp_call` works
+                @test x̄_ad isa DoesNotExist  # we said it wasn't differentiable.
+            else
+                x̄_ad isa AbstractThunk && check_inferred && _test_inferred(unthunk, x̄_ad)
+
+                # The main test of the actual deriviative being correct:
+                check_equal(x̄_ad, x̄_fd; isapprox_kwargs...)
+                _check_add!!_behaviour(accumulated_x̄, x̄_ad; isapprox_kwargs...)
+            end
         end
-    end
 
-    check_thunking_is_appropriate(x̄s_ad)
+        check_thunking_is_appropriate(x̄s_ad)
+    end  # top-level testset
 end
 
 function check_thunking_is_appropriate(x̄s)
diff --git a/test/deprecated.jl b/test/deprecated.jl
index ff0a5b4a..137f4cee 100644
--- a/test/deprecated.jl
+++ b/test/deprecated.jl
@@ -162,7 +162,10 @@ end
 
         # we defined these functions at top of file to throw errors unless we pass `err=false`
         @test_throws ErrorException futestkws(randn())
-        @test_throws ErrorException test_scalar(futestkws, randn())
+        @test errors(
+            ()->test_scalar(futestkws, randn()),
+            "futestkws_err",
+        )
         @test_throws ErrorException frule((nothing, randn()), futestkws, randn())
         @test_throws ErrorException rrule(futestkws, randn())
 
diff --git a/test/meta_testing_tools.jl b/test/meta_testing_tools.jl
index 0bc5055b..809ea673 100644
--- a/test/meta_testing_tools.jl
+++ b/test/meta_testing_tools.jl
@@ -2,53 +2,57 @@
 # if they were less nasty in implementation we might consider moving them to a package
 # MetaTesting.jl
 
-# need to bring this into scope explictly so can use in @testset nonpassing_results
-using Test: DefaultTestSet
-
 """
-    nonpassing_results(f)
+    EncasedTestSet(desc, results) <: AbstractTestset
 
-`f` should be a function that takes no argument, and calls some code that used `@test`.
-Invoking it via `nonpassing_results(f)` will prevent those `@test` being added to the
-current testset, and will return a collection of all nonpassing test results.
+A custom testset that encases all test results within, not letting them out.
+It doesn't let anything propagate up to the parent testset
+(or to the top-level fallback testset, which throws an error on any non-passing result).
+Not passes, not failures, not even errors.
+
+
+This is useful for being able to observe the testsets results programatically;
+without them triggering actual passes/failures/errors.
 """
-function nonpassing_results(f)
-    mute() do
-        nonpasses = []
-        # Specify testset type incase parent testset is some other typer
-        @testset DefaultTestSet "nonpassing internal" begin
-            f()
-            ts = Test.get_testset()  # this is the current testset "nonpassing internal"
-            nonpasses = _extract_nonpasses(ts)
-            # Prevent the failure being recorded in parent testset.
-            empty!(ts.results)
-            ts.anynonpass = false
-        end
-        # Note: we allow the "nonpassing internal" testset to still be pushed as an empty
-        # passing testset in its parent testset. We could remove that if we wanted
-        return nonpasses
+struct EncasedTestSet <: Test.AbstractTestSet
+    description::String
+    results::Vector{Any}
+end
+EncasedTestSet(desc) = EncasedTestSet(desc, [])
+
+Test.record(ts::EncasedTestSet, t) = (push!(ts.results, t); t)
+
+function Test.finish(ts::EncasedTestSet)
+    if Test.get_testset_depth() != 0
+        # Attach this test set to the parent test set *if* it is also a NonPassingTestset
+        # Otherwise don't as we don't want to push the errors and failures further up.
+        parent_ts = Test.get_testset()
+        parent_ts isa EncasedTestSet && Test.record(parent_ts, ts)
+        return ts
     end
+    return ts
 end
 
+
 """
-    mute(f)
+    nonpassing_results(f)
 
-Calls `f()` silencing stdout.
+`f` should be a function that takes no argument, and calls some code that used `@test`.
+Invoking it via `nonpassing_results(f)` will prevent those `@test` being added to the
+current testset, and will return a collection of all nonpassing test results.
 """
-function mute(f)
-    # TODO: once we are on Julia 1.6 this can be change to just use
-    # `redirect_stdout(devnull)` See: https://github.com/JuliaLang/julia/pull/36146
-    mktemp() do path, tempio
-        redirect_stdout(tempio) do
-            f()
-        end
+function nonpassing_results(f)
+    # Specify testset type to hijack system
+    ts = @testset EncasedTestSet "nonpassing internal" begin
+        f()
     end
+    return _extract_nonpasses(ts)
 end
 
 "extracts as flat collection of failures from a (potential nested) testset"
 _extract_nonpasses(x::Test.Result) = [x,]
 _extract_nonpasses(x::Test.Pass) = Test.Result[]
-_extract_nonpasses(ts::Test.DefaultTestSet) = _extract_nonpasses(ts.results)
+_extract_nonpasses(ts::EncasedTestSet) = _extract_nonpasses(ts.results)
 function _extract_nonpasses(xs::Vector)
     if isempty(xs)
         return Test.Result[]
@@ -71,7 +75,6 @@ function fails(f)
         did_fail |= result isa Test.Fail
         if result isa Test.Error
             # Log a error message, with original backtrace
-            show(result)
             # Sadly we can't throw the original exception as it is only stored as a String
             error("Error occurred during `fails`")
         end
@@ -79,6 +82,28 @@ function fails(f)
     return did_fail
 end
 
+"""
+    errors(f, msg_pattern="")
+
+Returns true if at least 1 error is recorded into a testset
+with a failure matching the given pattern.
+
+`f` should be a function that takes no argument, and calls some code that uses `@testset`.
+`msg_pattern` is a regex or a string, that should be contained in the error message.
+If nothing is passed then it default to the empty string, which matches any error message.
+
+If a test fails (rather than passing or erroring) then `errors` will throw an error.
+"""
+function errors(f, msg_pattern="")
+    results = nonpassing_results(f)
+
+    for result in results
+        result isa Test.Fail && error("Test actually failed (nor errored): \n $result")
+        result isa Test.Error && occursin(msg_pattern, result.value) && return true
+    end
+    return false  # no matching error occured
+end
+
 #Meta Meta tests
 @testset "meta_testing_tools.jl" begin
     @testset "Checking for non-passes" begin
@@ -110,6 +135,29 @@ end
             @test fails[1].orig_expr == :(false==true)
             @test fails[2].orig_expr == :(true==false)
         end
+
+
+        @testset "Single Error" begin
+            bads = nonpassing_results(()->error("noo"))
+            @test length(bads) === 1
+            @test bads[1] isa Test.Error
+        end
+
+        @testset "Single Test Erroring" begin
+            bads = nonpassing_results(()->@test error("nooo"))
+            @test length(bads) === 1
+            @test bads[1] isa Test.Error
+        end
+
+        @testset "Single Testset Erroring" begin
+            bads = nonpassing_results() do
+                @testset "inner" begin
+                    error("noo")
+                end
+            end
+            @test length(bads) === 1
+            @test bads[1] isa Test.Error
+        end
     end
 
     @testset "fails" begin
@@ -125,8 +173,24 @@ end
             end
         end
 
-        @test_throws Exception mute() do  # mute it so we don't see the reprinted error.
-            fails(()->@test error("Bad"))
+        @test_throws ErrorException fails(()->@test error("Bad"))
+    end
+
+
+    @testset "errors" begin
+        @test !errors(()->@test true)
+        @test errors(()->error("nooo"))
+        @test errors(()->error("nooo"), "noo")
+        @test !errors(()->error("nooo"), "ok")
+
+        @test errors() do
+            @testset "eg" begin
+                @test true
+                error("nooo")
+                @test true
+            end
         end
+
+        @test_throws ErrorException errors(()->@test false)
     end
 end
diff --git a/test/testers.jl b/test/testers.jl
index 9c7691c8..66b22d8f 100644
--- a/test/testers.jl
+++ b/test/testers.jl
@@ -1,7 +1,7 @@
 # For some reason if these aren't defined here, then they are interpreted as closures
-futestkws(x; err = true) = err ? error() : x
+futestkws(x; err = true) = err ? error("futestkws_err") : x
 
-fbtestkws(x, y; err = true) = err ? error() : x
+fbtestkws(x, y; err = true) = err ? error("fbtestkws_err") : x
 
 sinconj(x) = sin(x)
 
@@ -126,11 +126,16 @@ end
             end
 
             test_frule(f_noninferrable_frule, 2.0; check_inferred = false)
-            @test_throws ErrorException test_frule(f_noninferrable_frule, 2.0)
+            @test errors(
+                ()->test_frule(f_noninferrable_frule, 2.0),
+                "does not match inferred return type"
+            )
+
             test_scalar(f_noninferrable_frule, 2.0; check_inferred = false)
-            # `fails` plucks out `ErrorException` raised by `@inferred` from nested `TestSet`
-            # `mute` silences the printed error
-            @test_throws ErrorException mute(() -> fails(() -> test_scalar(f_noninferrable_frule, 2.0)))
+            @test errors(
+                ()->test_scalar(f_noninferrable_frule, 2.0),
+                "does not match inferred return type"
+            )
         end
 
         @testset "check not inferred in rrule" begin
@@ -145,11 +150,16 @@ end
             end
 
             test_rrule(f_noninferrable_rrule, 2.0; check_inferred = false)
-            @test_throws ErrorException test_rrule(f_noninferrable_rrule, 2.0)
+            @test errors(
+                ()->test_rrule(f_noninferrable_rrule, 2.0),
+                "does not match inferred return type"
+            )
+
             test_scalar(f_noninferrable_rrule, 2.0; check_inferred = false)
-            # `fails` plucks out `ErrorException` raised by `@inferred` from nested `TestSet`
-            # `mute` silences the printed error
-            @test_throws ErrorException mute(() -> fails(() -> test_scalar(f_noninferrable_rrule, 2.0)))
+            @test errors(
+                ()->test_scalar(f_noninferrable_rrule, 2.0),
+                "does not match inferred return type"
+            )
         end
 
         @testset "check not inferred in pullback" begin
@@ -158,7 +168,10 @@ end
                 return x, f_noninferrable_pullback_pullback
             end
             test_rrule(f_noninferrable_pullback, 2.0; check_inferred = false)
-            @test_throws ErrorException test_rrule(f_noninferrable_pullback, 2.0)
+            @test errors(
+                ()->test_rrule(f_noninferrable_pullback, 2.0),
+                "does not match inferred return type"
+            )
         end
 
         @testset "check not inferred in thunk" begin
@@ -170,7 +183,10 @@ end
                 return x + y, f_noninferrable_thunk_pullback
             end
             test_rrule(f_noninferrable_thunk, 2.0, 3.0; check_inferred = false)
-            @test_throws ErrorException test_rrule(f_noninferrable_thunk, 2.0, 3.0)
+            @test errors(
+                ()->test_rrule(f_noninferrable_thunk, 2.0, 3.0),
+                "does not match inferred return type"
+            )
         end
 
         @testset "check non-inferrable primal still passes if pullback inferrable" begin
@@ -303,7 +319,7 @@ end
 
         # we defined these functions at top of file to throw errors unless we pass `err=false`
         @test_throws ErrorException futestkws(randn())
-        @test_throws ErrorException test_scalar(futestkws, randn())
+        @test errors(()->test_scalar(futestkws, randn()), "futestkws_err")
         @test_throws ErrorException frule((nothing, randn()), futestkws, randn())
         @test_throws ErrorException rrule(futestkws, randn())