diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb
index e44c2c0f..3ecd592f 100644
--- a/test/benchmark_runner_test.rb
+++ b/test/benchmark_runner_test.rb
@@ -91,12 +91,18 @@
     end
 
     it 'exits when file does not exist' do
-      assert_raises(SystemExit) { BenchmarkRunner.expand_pre_init('/nonexistent/file.rb') }
+      out = capture_io do
+        assert_raises(SystemExit) { BenchmarkRunner.expand_pre_init('/nonexistent/file.rb') }
+      end
+      assert_includes out, "--with-pre-init called with non-existent file!\n"
     end
 
     it 'exits when path is a directory' do
       Dir.mktmpdir do |dir|
-        assert_raises(SystemExit) { BenchmarkRunner.expand_pre_init(dir) }
+        out = capture_io do
+          assert_raises(SystemExit) { BenchmarkRunner.expand_pre_init(dir) }
+        end
+        assert_includes out, "--with-pre-init called with a directory, please pass a .rb file\n"
       end
     end
   end