ci: don't special case benchmark testing

As per O(1) build files, benchmark is not special enough to warrant a
dedicate space in the yamls. Move it to the integration tests. Even in
debug, it's fast enough to run.

The release mode is exercised by the devhub.

.github/workflows/linux.yml

@@ -128,14 +128,6 @@ jobs:
   #         name: coverage-diff.html
   #         path: coverage-diff.html
 
-  benchmark:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v4
-      - run: ./scripts/install_zig.sh
-      - run: ./zig/zig build -Drelease -Dconfig=production run -- benchmark --transfer-count=4000
-
   aof:
     runs-on: ubuntu-latest
     steps:

.github/workflows/macos.yml

@@ -18,17 +18,6 @@ jobs:
       - run: ./zig/zig build test
       - run: ./scripts/install.sh
 
-  benchmark:
-    timeout-minutes: 10
-    strategy:
-      matrix:
-        target: [macos-13]
-    runs-on: ${{ matrix.target }}
-    steps:
-      - uses: actions/checkout@v4
-      - run: ./scripts/install_zig.sh
-      - run: ./zig/zig build -Drelease -Dconfig=production run -- benchmark --transfer-count=4000
-
   c_sample:
     strategy:
       matrix:

.github/workflows/windows.yml

@@ -4,13 +4,6 @@ on:
   workflow_call:
 
 jobs:
-  benchmark:
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: .\scripts\install_zig.bat
-      - run:  .\zig\zig build -Drelease -Dconfig=production run -- benchmark --transfer-count=4000
-
   c_sample:
     runs-on: windows-latest
     steps:

src/integration_tests.zig

@@ -144,3 +144,15 @@ test "repl integration" {
         \\
     ));
 }
+
+test "benchmark smoke" {
+    const shell = try Shell.create(std.testing.allocator);
+    defer shell.destroy();
+
+    const tigerbeetle = try tigerbeetle_exe(shell);
+    const status_ok = try shell.exec_status_ok(
+        "{tigerbeetle} benchmark --transfer-count=4000",
+        .{ .tigerbeetle = tigerbeetle },
+    );
+    try std.testing.expect(status_ok);
+}

src/shell.zig

@@ -388,7 +388,7 @@ pub fn exec_options(
     }
 }
 
-/// Returns `true` if the command executed successfully with a zero exit code and an empty stderr.
+/// Returns `true` if the command executed successfully with a zero exit code.
 ///
 /// One intended use-case is sanity-checking that an executable is present, by running
 /// `my-tool --version`.
@@ -411,7 +411,7 @@ pub fn exec_status_ok(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype
     defer shell.gpa.free(res.stdout);
 
     return switch (res.term) {
-        .Exited => |code| code == 0 and res.stderr.len == 0,
+        .Exited => |code| code == 0,
         else => false,
     };
 }

src/tigerbeetle/benchmark_driver.zig

@@ -22,11 +22,16 @@ const benchmark_load = @import("./benchmark_load.zig");
 
 const log = std.log;
 
-// Note: we intentionally don't use a temporary directory for this data file, and instead just put
-// it into CWD, as performance of TigerBeetle very much depends on a specific file system.
-const data_file = "0_0.tigerbeetle.benchmark";
-
 pub fn main(allocator: std.mem.Allocator, args: *const cli.Command.Benchmark) !void {
+    // Note: we intentionally don't use a temporary directory for this data file, and instead just
+    // put it into CWD, as performance of TigerBeetle very much depends on a specific file system.
+    const data_file = data_file: {
+        var random_bytes: [4]u8 = undefined;
+        std.crypto.random.bytes(&random_bytes);
+        const random_suffix: [8]u8 = std.fmt.bytesToHex(random_bytes, .lower);
+        break :data_file "0_0-" ++  random_suffix ++ ".tigerbeetle.benchmark";
+    };
+
     var data_file_created = false;
     defer {
         if (data_file_created) {