diff --git a/src/clients/dotnet/ci.zig b/src/clients/dotnet/ci.zig index 1922d108dc..8fccb61372 100644 --- a/src/clients/dotnet/ci.zig +++ b/src/clients/dotnet/ci.zig @@ -9,6 +9,7 @@ const Shell = @import("../../shell.zig"); const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig"); pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { + assert(shell.file_exists("TigerBeetle.sln")); try shell.exec("dotnet format --verify-no-changes", .{}); // Unit tests. @@ -24,13 +25,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { // Integration tests. inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| { - var sample_dir = try shell.project_root.openDir( - "src/clients/dotnet/samples/" ++ sample, - .{}, - ); - defer sample_dir.close(); - - try sample_dir.setAsCwd(); + try shell.pushd("./samples/" ++ sample); + defer shell.popd(); var tmp_beetle = try TmpTigerBeetle.init(gpa, .{}); defer tmp_beetle.deinit(gpa); @@ -41,11 +37,6 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { // Container smoke tests. if (builtin.target.os.tag == .linux) { - var client_dir = try shell.project_root.openDir("src/clients/dotnet/", .{}); - defer client_dir.close(); - - try client_dir.setAsCwd(); - // Here, we want to check that our package does not break horrible on upstream containers // due to missing runtime dependencies, mismatched glibc ABI and similar issues. // diff --git a/src/clients/go/ci.zig b/src/clients/go/ci.zig index 2b1d361f7a..156b883104 100644 --- a/src/clients/go/ci.zig +++ b/src/clients/go/ci.zig @@ -9,6 +9,8 @@ const Shell = @import("../../shell.zig"); const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig"); pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { + assert(shell.file_exists("go.mod")); + // No unit tests for Go :-( // `go build` won't compile the native library automatically, we need to do that ourselves. @@ -31,10 +33,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { }; inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| { - var sample_dir = try shell.project_root.openDir("src/clients/go/samples/" ++ sample, .{}); - defer sample_dir.close(); - - try sample_dir.setAsCwd(); + try shell.pushd("./samples/" ++ sample); + defer shell.popd(); var tmp_beetle = try TmpTigerBeetle.init(gpa, .{}); defer tmp_beetle.deinit(gpa); diff --git a/src/clients/java/ci.zig b/src/clients/java/ci.zig index c83b665949..09f3d715c2 100644 --- a/src/clients/java/ci.zig +++ b/src/clients/java/ci.zig @@ -8,6 +8,8 @@ const Shell = @import("../../shell.zig"); const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig"); pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { + assert(shell.file_exists("pom.xml")); + // Java's maven doesn't support a separate test command, or a way to add dependency on a // project (as opposed to a compiled jar file). // @@ -17,10 +19,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { try shell.exec("mvn --batch-mode --file pom.xml --quiet install", .{}); inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| { - var sample_dir = try shell.project_root.openDir("src/clients/java/samples/" ++ sample, .{}); - defer sample_dir.close(); - - try sample_dir.setAsCwd(); + try shell.pushd("./samples/" ++ sample); + defer shell.popd(); var tmp_beetle = try TmpTigerBeetle.init(gpa, .{}); defer tmp_beetle.deinit(gpa); diff --git a/src/clients/node/ci.zig b/src/clients/node/ci.zig index eb4f231d99..188d6473d8 100644 --- a/src/clients/node/ci.zig +++ b/src/clients/node/ci.zig @@ -9,6 +9,8 @@ const Shell = @import("../../shell.zig"); const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig"); pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { + assert(shell.file_exists("package.json")); + // We have some unit-tests for node, but they are likely bitrotted, as they are not run on CI. // Integration tests. @@ -18,10 +20,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { try shell.exec("npm pack --quiet", .{}); inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| { - var sample_dir = try shell.project_root.openDir("src/clients/node/samples/" ++ sample, .{}); - defer sample_dir.close(); - - try sample_dir.setAsCwd(); + try shell.pushd("./samples/" ++ sample); + defer shell.popd(); var tmp_beetle = try TmpTigerBeetle.init(gpa, .{}); defer tmp_beetle.deinit(gpa); @@ -33,10 +33,6 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void { // Container smoke tests. if (builtin.target.os.tag == .linux) { - var client_dir = try shell.project_root.openDir("src/clients/node/", .{}); - defer client_dir.close(); - - try client_dir.setAsCwd(); // Installing node through . diff --git a/src/scripts/ci.zig b/src/scripts/ci.zig index bfaba2c9a6..33465833f9 100644 --- a/src/scripts/ci.zig +++ b/src/scripts/ci.zig @@ -60,15 +60,12 @@ pub fn main() !void { var section = try shell.open_section(@tagName(language) ++ " ci"); defer section.close(); - var client_src_dir = try shell.project_root.openDir( - "src/clients/" ++ @tagName(language), - .{}, - ); - defer client_src_dir.close(); + { + try shell.pushd("./src/clients/" ++ @tagName(language)); + defer shell.popd(); - try client_src_dir.setAsCwd(); - - try ci.tests(shell, gpa); + try ci.tests(shell, gpa); + } // Piggy back on node client testing to verify our docs, as we use node to generate // them anyway. @@ -86,13 +83,8 @@ pub fn main() !void { } fn build_docs(shell: *Shell) !void { - var docs_dir = try shell.project_root.openDir( - "src/docs_website", - .{}, - ); - defer docs_dir.close(); - - try docs_dir.setAsCwd(); + try shell.pushd("./src/docs_website"); + defer shell.popd(); try shell.exec("npm install", .{}); try shell.exec("npm run build", .{}); diff --git a/src/scripts/release.zig b/src/scripts/release.zig index b87c11cd69..ff114ec18f 100644 --- a/src/scripts/release.zig +++ b/src/scripts/release.zig @@ -132,8 +132,6 @@ fn build_tigerbeetle(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !vo var section = try shell.open_section("build tigerbeetle"); defer section.close(); - try shell.project_root.setAsCwd(); - const llvm_lipo = for (@as([2][]const u8, .{ "llvm-lipo-16", "llvm-lipo" })) |llvm_lipo| { if (shell.exec_stdout("{llvm_lipo} -version", .{ .llvm_lipo = llvm_lipo, @@ -220,10 +218,8 @@ fn build_dotnet(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { var section = try shell.open_section("build dotnet"); defer section.close(); - var client_src_dir = try shell.project_root.openDir("src/clients/dotnet", .{}); - defer client_src_dir.close(); - - try client_src_dir.setAsCwd(); + try shell.pushd("./src/clients/dotnet"); + defer shell.popd(); const dotnet_version = shell.exec_stdout("dotnet --version", .{}) catch { fatal("can't find dotnet", .{}); @@ -236,7 +232,7 @@ fn build_dotnet(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { , .{ .version = info.version }); try Shell.copy_path( - client_src_dir, + shell.cwd, try shell.print("TigerBeetle/bin/Release/tigerbeetle.{s}.nupkg", .{info.version}), dist_dir, try shell.print("tigerbeetle.{s}.nupkg", .{info.version}), @@ -247,10 +243,8 @@ fn build_go(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { var section = try shell.open_section("build go"); defer section.close(); - var client_src_dir = try shell.project_root.openDir("src/clients/go", .{}); - defer client_src_dir.close(); - - try client_src_dir.setAsCwd(); + try shell.pushd("./src/clients/go"); + defer shell.popd(); try shell.zig("build go_client -Doptimize=ReleaseSafe -Dconfig=production", .{}); @@ -259,7 +253,7 @@ fn build_go(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { var copied_count: u32 = 0; while (files_lines.next()) |file| { assert(file.len > 3); - try Shell.copy_path(client_src_dir, file, dist_dir, file); + try Shell.copy_path(shell.cwd, file, dist_dir, file); copied_count += 1; } assert(copied_count >= 10); @@ -267,7 +261,7 @@ fn build_go(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { const native_files = try shell.find(.{ .where = &.{"."}, .extensions = &.{ ".a", ".lib" } }); copied_count = 0; for (native_files) |native_file| { - try Shell.copy_path(client_src_dir, native_file, dist_dir, native_file); + try Shell.copy_path(shell.cwd, native_file, dist_dir, native_file); copied_count += 1; } // 5 = 3 + 2 @@ -291,18 +285,16 @@ fn build_java(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { var section = try shell.open_section("build java"); defer section.close(); - var client_src_dir = try shell.project_root.openDir("src/clients/java", .{}); - defer client_src_dir.close(); - - try client_src_dir.setAsCwd(); + try shell.pushd("./src/clients/java"); + defer shell.popd(); const java_version = shell.exec_stdout("java --version", .{}) catch { fatal("can't find java", .{}); }; log.info("java version {s}", .{java_version}); - try backup_create(client_src_dir, "pom.xml"); - defer backup_restore(client_src_dir, "pom.xml"); + try backup_create(shell.cwd, "pom.xml"); + defer backup_restore(shell.cwd, "pom.xml"); try shell.exec( \\mvn --batch-mode --quiet --file pom.xml @@ -316,7 +308,7 @@ fn build_java(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { , .{}); try Shell.copy_path( - client_src_dir, + shell.cwd, try shell.print("target/tigerbeetle-java-{s}.jar", .{info.version}), dist_dir, try shell.print("tigerbeetle-java-{s}.jar", .{info.version}), @@ -327,28 +319,26 @@ fn build_node(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void { var section = try shell.open_section("build node"); defer section.close(); - var client_src_dir = try shell.project_root.openDir("src/clients/node", .{}); - defer client_src_dir.close(); - - try client_src_dir.setAsCwd(); + try shell.pushd("./src/clients/node"); + defer shell.popd(); const node_version = shell.exec_stdout("node --version", .{}) catch { fatal("can't find nodejs", .{}); }; log.info("node version {s}", .{node_version}); - try backup_create(client_src_dir, "package.json"); - defer backup_restore(client_src_dir, "package.json"); + try backup_create(shell.cwd, "package.json"); + defer backup_restore(shell.cwd, "package.json"); - try backup_create(client_src_dir, "package-lock.json"); - defer backup_restore(client_src_dir, "package-lock.json"); + try backup_create(shell.cwd, "package-lock.json"); + defer backup_restore(shell.cwd, "package-lock.json"); try shell.exec("npm version --no-git-tag-version {version}", .{ .version = info.version }); try shell.exec("npm install", .{}); try shell.exec("npm pack --quiet", .{}); try Shell.copy_path( - client_src_dir, + shell.cwd, try shell.print("tigerbeetle-node-{s}.tgz", .{info.version}), dist_dir, try shell.print("tigerbeetle-node-{s}.tgz", .{info.version}), @@ -359,7 +349,6 @@ fn publish(shell: *Shell, languages: LanguageSet, info: VersionInfo) !void { var section = try shell.open_section("publish all"); defer section.close(); - try shell.project_root.setAsCwd(); assert(try shell.dir_exists("dist")); if (languages.contains(.zig)) { @@ -447,7 +436,6 @@ fn publish(shell: *Shell, languages: LanguageSet, info: VersionInfo) !void { } if (languages.contains(.zig)) { - try shell.project_root.setAsCwd(); try shell.exec( \\gh release edit --draft=false \\ {tag} @@ -490,7 +478,6 @@ fn publish_dotnet(shell: *Shell, info: VersionInfo) !void { var section = try shell.open_section("publish dotnet"); defer section.close(); - try shell.project_root.setAsCwd(); assert(try shell.dir_exists("dist/dotnet")); const nuget_key = try shell.env_get("NUGET_KEY"); @@ -509,7 +496,6 @@ fn publish_go(shell: *Shell, info: VersionInfo) !void { var section = try shell.open_section("publish go"); defer section.close(); - try shell.project_root.setAsCwd(); assert(try shell.dir_exists("dist/go")); const token = try shell.env_get("TIGERBEETLE_GO_PAT"); @@ -538,10 +524,8 @@ fn publish_go(shell: *Shell, info: VersionInfo) !void { ); } - var tigerbeetle_go_dir = try shell.project_root.openDir("tigerbeetle-go", .{}); - defer tigerbeetle_go_dir.close(); - - try tigerbeetle_go_dir.setAsCwd(); + try shell.pushd("tigerbeetle-go"); + defer shell.popd(); try shell.exec("git add .", .{}); // Native libraries are ignored in this repository, but we want to push them to the @@ -571,7 +555,6 @@ fn publish_java(shell: *Shell, info: VersionInfo) !void { var section = try shell.open_section("publish java"); defer section.close(); - try shell.project_root.setAsCwd(); assert(try shell.dir_exists("dist/java")); // These variables don't have a special meaning in maven, and instead are a part of @@ -611,7 +594,6 @@ fn publish_node(shell: *Shell, info: VersionInfo) !void { var section = try shell.open_section("publish node"); defer section.close(); - try shell.project_root.setAsCwd(); assert(try shell.dir_exists("dist/node")); // `NODE_AUTH_TOKEN` env var doesn't have a special meaning in npm. It does have special meaning @@ -630,7 +612,6 @@ fn publish_docker(shell: *Shell, info: VersionInfo) !void { var section = try shell.open_section("publish docker"); defer section.close(); - try shell.project_root.setAsCwd(); assert(try shell.dir_exists("dist/tigerbeetle")); try shell.exec( @@ -693,15 +674,12 @@ fn publish_docs(shell: *Shell, info: VersionInfo) !void { defer section.close(); { - var docs_dir = try shell.project_root.openDir("src/docs_website", .{}); - defer docs_dir.close(); - - try docs_dir.setAsCwd(); + try shell.pushd("src/docs_website"); + defer shell.popd(); try shell.exec("npm install", .{}); try shell.exec("npm run build", .{}); } - try shell.project_root.setAsCwd(); const token = try shell.env_get("TIGERBEETLE_DOCS_PAT"); try shell.exec( @@ -729,10 +707,8 @@ fn publish_docs(shell: *Shell, info: VersionInfo) !void { ); } - var tigerbeetle_docs_dir = try shell.project_root.openDir("tigerbeetle-docs", .{}); - defer tigerbeetle_docs_dir.close(); - - try tigerbeetle_docs_dir.setAsCwd(); + try shell.pushd("tigerebeetle-docs"); + defer shell.popd(); try shell.exec("git add .", .{}); try shell.env.put("GIT_AUTHOR_NAME", "TigerBeetle Bot"); diff --git a/src/shell.zig b/src/shell.zig index 722f3877d5..aebcd057ee 100644 --- a/src/shell.zig +++ b/src/shell.zig @@ -21,6 +21,8 @@ const assert = std.debug.assert; const Shell = @This(); +const cwd_stack_max = 16; + /// For internal use by the `Shell` itself. gpa: std.mem.Allocator, @@ -31,13 +33,19 @@ arena: std.heap.ArenaAllocator, /// Root directory of this repository. /// -/// Prefer this to `std.fs.cwd()` if possible. -/// /// This is initialized when a shell is created. It would be more flexible to lazily initialize this /// on the first access, but, given that we always use `Shell` in the context of our repository, /// eager initialization is more ergonomic. project_root: std.fs.Dir, +/// Shell's logical cwd which is used for all functions in this file. It might be different from +/// `std.fs.cwd()` and is set to `project_root` on init. +cwd: std.fs.Dir, + +// Stack of working directories backing pushd/popd. +cwd_stack: [cwd_stack_max]std.fs.Dir, +cwd_stack_count: usize, + env: std.process.EnvMap, /// True if the process is run in CI (the CI env var is set) @@ -50,6 +58,9 @@ pub fn create(gpa: std.mem.Allocator) !*Shell { var project_root = try discover_project_root(); errdefer project_root.close(); + var cwd = try project_root.openDir(".", .{}); + errdefer cwd.close(); + var env = try std.process.getEnvMap(gpa); errdefer env.deinit(); @@ -62,6 +73,9 @@ pub fn create(gpa: std.mem.Allocator) !*Shell { .gpa = gpa, .arena = arena, .project_root = project_root, + .cwd = cwd, + .cwd_stack = undefined, + .cwd_stack_count = 0, .env = env, .ci = ci, }; @@ -72,7 +86,11 @@ pub fn create(gpa: std.mem.Allocator) !*Shell { pub fn destroy(shell: *Shell) void { const gpa = shell.gpa; + assert(shell.cwd_stack_count == 0); // pushd not paired by popd + shell.env.deinit(); + shell.cwd.close(); + shell.project_root.close(); shell.arena.deinit(); gpa.destroy(shell); } @@ -153,6 +171,8 @@ const Section = struct { } }; +/// Convenience string formatting function which uses shell's arena and doesn't require +/// freeing the resulting string. pub fn print(shell: *Shell, comptime fmt: []const u8, fmt_args: anytype) ![]const u8 { return std.fmt.allocPrint(shell.arena.allocator(), fmt, fmt_args); } @@ -165,11 +185,41 @@ pub fn env_get(shell: *Shell, var_name: []const u8) ![]const u8 { return try std.process.getEnvVarOwned(shell.arena.allocator(), var_name); } +/// Change `shell`'s working directory. It *must* be followed by +/// +/// defer shell.popd(); +/// +/// to restore the previous directory back. +pub fn pushd(shell: *Shell, path: []const u8) !void { + assert(shell.cwd_stack_count < cwd_stack_max); + assert(path[0] == '.'); // allow only explicitly relative paths + + const cwd_new = try shell.cwd.openDir(path, .{}); + + shell.cwd_stack[shell.cwd_stack_count] = shell.cwd; + shell.cwd_stack_count += 1; + shell.cwd = cwd_new; +} + +pub fn popd(shell: *Shell) void { + shell.cwd.close(); + shell.cwd_stack_count -= 1; + shell.cwd = shell.cwd_stack[shell.cwd_stack_count]; +} + /// Checks if the path exists and is a directory. +/// +/// Note: this api is prone to TOCTOU and exists primarily for assertions. pub fn dir_exists(shell: *Shell, path: []const u8) !bool { - _ = shell; + return subdir_exists(shell.cwd, path); +} - return subdir_exists(std.fs.cwd(), path); +/// Checks if the path exists and is a file. +/// +/// Note: this api is prone to TOCTOU and exists primarily for assertions. +pub fn file_exists(shell: *Shell, path: []const u8) bool { + const stat = shell.cwd.statFile(path) catch return false; + return stat.kind == .file; } fn subdir_exists(dir: std.fs.Dir, path: []const u8) !bool { @@ -206,9 +256,8 @@ pub fn find(shell: *Shell, options: FindOptions) ![]const []const u8 { var result = std.ArrayList([]const u8).init(shell.arena.allocator()); - const cwd = std.fs.cwd(); for (options.where) |base_path| { - var base_dir = try cwd.openIterableDir(base_path, .{}); + var base_dir = try shell.cwd.openIterableDir(base_path, .{}); defer base_dir.close(); var walker = try base_dir.walk(shell.gpa); @@ -277,7 +326,12 @@ pub fn exec(shell: Shell, comptime cmd: []const u8, cmd_args: anytype) !void { try expand_argv(&argv, cmd, cmd_args); + // TODO(Zig): use cwd_dir once that is available https://github.com/ziglang/zig/issues/5190 + var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; + const cwd_path = try shell.cwd.realpath(".", &buffer); + var child = std.ChildProcess.init(argv.slice(), shell.gpa); + child.cwd = cwd_path; child.env_map = &shell.env; child.stdin_behavior = .Ignore; child.stdout_behavior = .Inherit; @@ -302,9 +356,14 @@ pub fn exec_status_ok(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype try expand_argv(&argv, cmd, cmd_args); + // TODO(Zig): use cwd_dir once that is available https://github.com/ziglang/zig/issues/5190 + var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; + const cwd_path = try shell.cwd.realpath(".", &buffer); + const res = std.ChildProcess.exec(.{ .allocator = shell.gpa, .argv = argv.slice(), + .cwd = cwd_path, }) catch return false; defer shell.gpa.free(res.stderr); defer shell.gpa.free(res.stdout); @@ -326,9 +385,14 @@ pub fn exec_stdout(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype) ! try expand_argv(&argv, cmd, cmd_args); + // TODO(Zig): use cwd_dir once that is available https://github.com/ziglang/zig/issues/5190 + var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; + const cwd_path = try shell.cwd.realpath(".", &buffer); + const child = try std.ChildProcess.exec(.{ .allocator = shell.arena.allocator(), .argv = argv.slice(), + .cwd = cwd_path, }); defer shell.arena.allocator().free(child.stderr); errdefer { @@ -360,7 +424,12 @@ pub fn spawn(shell: Shell, comptime cmd: []const u8, cmd_args: anytype) !std.Chi try expand_argv(&argv, cmd, cmd_args); + // TODO(Zig): use cwd_dir once that is available https://github.com/ziglang/zig/issues/5190 + var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; + const cwd_path = try shell.cwd.realpath(".", &buffer); + var child = std.ChildProcess.init(argv.slice(), shell.gpa); + child.cwd = cwd_path; child.stdin_behavior = .Ignore; child.stdout_behavior = .Pipe; child.stderr_behavior = .Pipe; @@ -386,7 +455,12 @@ pub fn zig(shell: Shell, comptime cmd: []const u8, cmd_args: anytype) !void { try argv.append_new_arg(zig_exe); try expand_argv(&argv, cmd, cmd_args); + // TODO(Zig): use cwd_dir once that is available https://github.com/ziglang/zig/issues/5190 + var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined; + const cwd_path = try shell.cwd.realpath(".", &buffer); + var child = std.ChildProcess.init(argv.slice(), shell.gpa); + child.cwd = cwd_path; child.stdin_behavior = .Ignore; child.stdout_behavior = .Inherit; child.stderr_behavior = .Inherit;