test_runner: enable testing panics in mainTerminal

lib/std/child_process.zig

@@ -251,6 +251,9 @@ pub const ChildProcess = struct {
     }
 
     /// Blocks until child process terminates and then cleans up all resources.
+    /// In case of error, the caller is responsible to clean up the ressources
+    /// via calling `self.cleanupStreams()`.
+    /// TODO: This describes the current state. Is this intended?
     pub fn wait(self: *ChildProcess) !Term {
         const term = if (builtin.os.tag == .windows)
             try self.waitWindows()
@@ -323,7 +326,7 @@ pub const ChildProcess = struct {
     };
 
     /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns.
-    /// If it succeeds, the caller owns result.stdout and result.stderr memory.
+    /// If spawning succeeds, then the caller owns result.stdout and result.stderr memory.
     pub fn exec(args: struct {
         allocator: mem.Allocator,
         argv: []const []const u8,
@@ -426,7 +429,7 @@ pub const ChildProcess = struct {
         self.term = self.cleanupAfterWait(status);
     }
 
-    fn cleanupStreams(self: *ChildProcess) void {
+    pub fn cleanupStreams(self: *ChildProcess) void {
         if (self.stdin) |*stdin| {
             stdin.close();
             self.stdin = null;

lib/std/std.zig

@@ -292,6 +292,9 @@ pub const options = struct {
         options_override.side_channels_mitigations
     else
         crypto.default_side_channels_mitigations;
+
+    /// Default thread-local storage panic message size used for panic tests.
+    pub const testing_max_panic_msg_size = 100;
 };
 
 // This forces the start.zig file to be imported, and the comptime logic inside that

lib/std/testing.zig

@@ -935,6 +935,37 @@ test "expectEqualDeep composite type" {
     }
 }
 
+pub const can_panic_test = builtin.is_test and !builtin.single_threaded and std.process.can_spawn;
+
+/// Static storage to support user-generated panic messages
+/// Parent process writes these, returns to the test execution loop and spawns,
+/// child process ignores these.
+const TestFn_iT = if (can_panic_test) ?[std.options.testing_max_panic_msg_size:0]u8 else void;
+pub threadlocal var panic_msg: TestFn_iT = if (can_panic_test) null else {};
+
+/// Distinguishes between parent and child, if panics are tested for.
+/// TODO: is_panic_parentproc and panic_msg feels like it belongs into test api to
+/// allow implementations providing their own way to prevent the necessity to use tls.
+pub var is_panic_parentproc: if (can_panic_test) bool else void = if (can_panic_test) true else {};
+
+/// To be used for panic tests after test block declaration.
+pub fn spawnExpectPanic(msg: []const u8) error{ SpawnZigTest, SkipZigTest }!void {
+    std.debug.assert(can_panic_test); // Caller is responsible to check.
+    if (is_panic_parentproc) {
+        if (panic_msg == null) {
+            panic_msg = .{undefined} ** std.options.testing_max_panic_msg_size;
+            @memcpy(panic_msg.?[0..msg.len], msg); // Message must be persistent, not stack-local.
+            panic_msg.?[msg.len] = 0; // 0-sentinel for the len without separate field
+            return error.SpawnZigTest; // Test will be run in separate process
+        } else {
+            @panic("std.testing.panic_msg must be only used in spawnExpectPanic");
+        }
+    } else {
+        std.debug.assert(panic_msg == null);
+        // panic runner continues running the test block
+    }
+}
+
 fn printIndicatorLine(source: []const u8, indicator_index: usize) void {
     const line_begin_index = if (std.mem.lastIndexOfScalar(u8, source[0..indicator_index], '\n')) |line_begin|
         line_begin + 1

lib/std/zig/Server.zig

@@ -42,7 +42,7 @@ pub const Message = struct {
     ///   - 0 means not async
     /// * expected_panic_msg: [tests_len]u32,
     ///   - null-terminated string_bytes index
-    ///   - 0 means does not expect pani
+    ///   - 0 means does not expect panic
     /// * string_bytes: [string_bytes_len]u8,
     pub const TestMetadata = extern struct {
         string_bytes_len: u32,

lib/test_runner.zig

@@ -12,6 +12,21 @@ var log_err_count: usize = 0;
 var cmdline_buffer: [4096]u8 = undefined;
 var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);
 
+const Mode = enum {
+    listen,
+    terminal,
+    panic_test,
+};
+
+fn callError(args: [][:0]u8) noreturn {
+    std.debug.print("invalid cli arguments:\n", .{});
+    for (args) |arg| {
+        std.debug.print("{s} ", .{arg});
+    }
+    std.debug.print("\n", .{});
+    @panic("call error");
+}
+
 pub fn main() void {
     if (builtin.zig_backend == .stage2_aarch64) {
         return mainSimple() catch @panic("test failure");
@@ -20,20 +35,33 @@ pub fn main() void {
     const args = std.process.argsAlloc(fba.allocator()) catch
         @panic("unable to parse command line args");
 
-    var listen = false;
+    var i: u32 = 1;
+    var test_i: usize = undefined;
+    var mode: Mode = .terminal;
 
-    for (args[1..]) |arg| {
-        if (std.mem.eql(u8, arg, "--listen=-")) {
-            listen = true;
+    while (i < args.len) : (i += 1) {
+        if (std.mem.eql(u8, args[i], "--listen=-")) {
+            mode = .listen;
+        } else if (std.mem.eql(u8, args[i], "--test_panic_index")) {
+            i += 1;
+            if (i < args.len) {
+                test_i = std.fmt.parseInt(usize, args[i], 10) catch {
+                    callError(args);
+                };
+                mode = .panic_test;
+                if (std.testing.can_panic_test) std.testing.is_panic_parentproc = false;
+            } else {
+                callError(args);
+            }
         } else {
-            @panic("unrecognized command line argument");
+            callError(args);
         }
     }
 
-    if (listen) {
-        return mainServer() catch @panic("internal test runner failure");
-    } else {
-        return mainTerminal();
+    switch (mode) {
+        .listen => return mainServer() catch @panic("internal test runner failure"),
+        .terminal => return mainTerminal(args),
+        .panic_test => return panicTest(test_i),
     }
 }
 
@@ -130,7 +158,10 @@ fn mainServer() !void {
     }
 }
 
-fn mainTerminal() void {
+fn mainTerminal(args: [][:0]const u8) void {
+    var test_i_buf: [20]u8 = undefined;
+    // TODO make environment buffer size configurable and use a sane default
+    // Tradeoff: waste stack space or allocate on every panic test
     const test_fn_list = builtin.test_functions;
     var ok_count: usize = 0;
     var skip_count: usize = 0;
@@ -146,7 +177,6 @@ fn mainTerminal() void {
     // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly
     // ignores the alignment of the slice.
     async_frame_buffer = &[_]u8{};
-
     var leaks: usize = 0;
     for (test_fn_list, 0..) |test_fn, i| {
         std.testing.allocator_instance = .{};
@@ -189,9 +219,133 @@ fn mainTerminal() void {
                 progress.log("SKIP\n", .{});
                 test_node.end();
             },
+            error.SpawnZigTest => {
+                if (!std.testing.can_panic_test)
+                    @panic("Found error.SpawnZigTest without panic test capabilities.");
+                if (std.testing.panic_msg == null)
+                    @panic("Panic test expects `panic_msg` to be set. Use std.testing.spawnExpectPanic().");
+
+                const test_i_written = std.fmt.bufPrint(&test_i_buf, "{d}", .{i}) catch unreachable;
+                var child_proc = std.ChildProcess.init(
+                    &.{ args[0], "--test_panic_index", test_i_written },
+                    std.testing.allocator,
+                );
+                progress.log("SPAWN '{s} {s} {s}'\n", .{ args[0], "--test_panic_index", test_i_written });
+
+                child_proc.stdin_behavior = .Ignore;
+                child_proc.stdout_behavior = .Pipe;
+                child_proc.stderr_behavior = .Pipe;
+                child_proc.spawn() catch |spawn_err| {
+                    progress.log("FAIL spawn ({s})\n", .{@errorName(spawn_err)});
+                    fail_count += 1;
+                    test_node.end();
+                    continue;
+                };
+
+                var stdout = std.ArrayList(u8).init(std.testing.allocator);
+                defer stdout.deinit();
+                var stderr = std.ArrayList(u8).init(std.testing.allocator);
+                defer stderr.deinit();
+                // child_process.zig: max_output_bytes: usize = 50 * 1024,
+                child_proc.collectOutput(&stdout, &stderr, 50 * 1024) catch |collect_err| {
+                    progress.log("FAIL collect ({s})\n", .{@errorName(collect_err)});
+                    fail_count += 1;
+                    test_node.end();
+                    continue;
+                };
+                const term = child_proc.wait() catch |wait_err| {
+                    child_proc.cleanupStreams();
+                    progress.log("FAIL wait_error (exit_status: {d})\n", .{@errorName(wait_err)});
+                    fail_count += 1;
+                    test_node.end();
+                    continue;
+                };
+                switch (term) {
+                    .Exited => |code| {
+                        progress.log("FAIL term exited, (exit_status: {})\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                        fail_count += 1;
+                        test_node.end();
+                        continue;
+                    },
+                    .Signal => |code| {
+                        progress.log("Signal: {d}\n", .{code});
+                        // assert: panic message format: 'XYZ thread thread_id panic: msg'
+                        // Any signal can be returned on panic, if a custom signal
+                        // or panic handler was installed as part of the unit test.
+                        var pos_eol: usize = 0;
+                        var found_eol: bool = false;
+                        while (pos_eol < stderr.items.len) : (pos_eol += 1) {
+                            if (stderr.items[pos_eol] == '\n') {
+                                found_eol = true;
+                                break;
+                            }
+                        }
+
+                        if (!found_eol) {
+                            progress.log("FAIL no end of line in panic format, Signal: {d}\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                            fail_count += 1;
+                            test_node.end();
+                            continue;
+                        }
+
+                        var it = std.mem.tokenize(u8, stderr.items[0..pos_eol], " ");
+                        var parsed_panic_msg = false;
+                        while (it.next()) |word| { // 'thread thread_id panic: msg'
+                            if (!std.mem.eql(u8, word, "thread")) continue;
+                            const thread_id = it.next();
+                            if (thread_id == null) continue;
+                            _ = std.fmt.parseInt(u64, thread_id.?, 10) catch continue;
+                            const panic_txt = it.next();
+                            if (panic_txt == null) continue;
+                            if (!std.mem.eql(u8, panic_txt.?, "panic:")) continue;
+                            const panic_msg = it.next();
+                            if (panic_msg == null) continue;
+                            const panic_msg_start = it.index - panic_msg.?.len;
+                            const len_exp_panic_msg = std.mem.len(@as([*:0]u8, std.testing.panic_msg.?[0..]));
+                            const expected_panic_msg = std.testing.panic_msg.?[0..len_exp_panic_msg];
+                            const panic_msg_end = panic_msg_start + expected_panic_msg.len;
+                            if (panic_msg_end > pos_eol) break;
+
+                            parsed_panic_msg = true;
+                            const current_panic_msg = stderr.items[panic_msg_start..panic_msg_end];
+
+                            if (!std.mem.eql(u8, "SKIP (async test)", current_panic_msg) and !std.mem.eql(u8, expected_panic_msg, current_panic_msg)) {
+                                progress.log("FAIL expected_panic_msg: '{s}', got: '{s}', Signal: {d}\n", .{ expected_panic_msg, current_panic_msg, code });
+                                std.testing.panic_msg = null;
+                                fail_count += 1;
+                                test_node.end();
+                                break;
+                            }
+                            std.testing.panic_msg = null;
+                            ok_count += 1;
+                            test_node.end();
+                            if (!have_tty) std.debug.print("OK\n", .{});
+                            break;
+                        }
+                        if (!parsed_panic_msg) {
+                            progress.log("FAIL panic_msg format expected 'XYZ thread thread_id panic: msg'\nstdout: ({s})\nstderr: ({s})\n", .{ stdout.items, stderr.items });
+                            fail_count += 1;
+                            test_node.end();
+                            continue;
+                        }
+                    },
+                    .Stopped => |code| {
+                        fail_count += 1;
+                        progress.log("FAIL stopped, status: ({d})\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                        test_node.end();
+                        continue;
+                    },
+                    .Unknown => |code| {
+                        fail_count += 1;
+                        progress.log("FAIL unknown, status: ({d})\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                        test_node.end();
+                        continue;
+                    },
+                }
+            },
             else => {
                 fail_count += 1;
-                progress.log("FAIL ({s})\n", .{@errorName(err)});
+                progress.log("FAIL unexpected error ({s})\n", .{@errorName(err)});
                 if (@errorReturnTrace()) |trace| {
                     std.debug.dumpStackTrace(trace.*);
                 }
@@ -216,6 +370,45 @@ fn mainTerminal() void {
     }
 }
 
+fn panicTest(test_i: usize) void {
+    const test_fn_list = builtin.test_functions;
+    var async_frame_buffer: []align(builtin.target.stackAlignment()) u8 = undefined;
+    // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly
+    // ignores the alignment of the slice.
+    async_frame_buffer = &[_]u8{};
+    {
+        std.testing.allocator_instance = .{};
+        // custom panic handler to restore to save state and prevent memory
+        // leakage is out of scope, so ignore memory leaks
+        defer {
+            if (std.testing.allocator_instance.deinit() == .leak) {
+                @panic("internal test runner memory leak");
+            }
+        }
+        std.testing.log_level = .warn;
+        const result = if (test_fn_list[test_i].async_frame_size) |size| switch (std.options.io_mode) {
+            .evented => blk: {
+                if (async_frame_buffer.len < size) {
+                    std.heap.page_allocator.free(async_frame_buffer);
+                    async_frame_buffer = std.heap.page_allocator.alignedAlloc(u8, std.Target.stack_align, size) catch @panic("out of memory");
+                }
+                const casted_fn = @as(fn () callconv(.Async) anyerror!void, @ptrCast(test_fn_list[test_i].func));
+                break :blk await @asyncCall(async_frame_buffer, {}, casted_fn, .{});
+            },
+            .blocking => @panic("SKIP (async test)"),
+        } else test_fn_list[test_i].func();
+
+        if (result) {
+            std.os.exit(0);
+        } else |err| {
+            std.debug.print("FAIL unexpected error ({s})\n", .{@errorName(err)});
+            if (@errorReturnTrace()) |trace| {
+                std.debug.dumpStackTrace(trace.*);
+            }
+        }
+    }
+}
+
 pub fn log(
     comptime message_level: std.log.Level,
     comptime scope: @Type(.EnumLiteral),