Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: more robust handling of directories in shell.zig #1275

Merged
merged 2 commits into from
Nov 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 3 additions & 12 deletions src/clients/dotnet/ci.zig
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const Shell = @import("../../shell.zig");
const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig");

pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
assert(shell.file_exists("TigerBeetle.sln"));
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fun fact: I've added these asserts as a drive by change, because why not, and they've already caught a bug in this PR!

try shell.exec("dotnet format --verify-no-changes", .{});

// Unit tests.
Expand All @@ -24,13 +25,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {

// Integration tests.
inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| {
var sample_dir = try shell.project_root.openDir(
"src/clients/dotnet/samples/" ++ sample,
.{},
);
defer sample_dir.close();

try sample_dir.setAsCwd();
try shell.pushd("./samples/" ++ sample);
defer shell.popd();

var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
defer tmp_beetle.deinit(gpa);
Expand All @@ -41,11 +37,6 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {

// Container smoke tests.
if (builtin.target.os.tag == .linux) {
var client_dir = try shell.project_root.openDir("src/clients/dotnet/", .{});
defer client_dir.close();

try client_dir.setAsCwd();

// Here, we want to check that our package does not break horrible on upstream containers
// due to missing runtime dependencies, mismatched glibc ABI and similar issues.
//
Expand Down
8 changes: 4 additions & 4 deletions src/clients/go/ci.zig
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ const Shell = @import("../../shell.zig");
const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig");

pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
assert(shell.file_exists("go.mod"));

// No unit tests for Go :-(

// `go build` won't compile the native library automatically, we need to do that ourselves.
Expand All @@ -31,10 +33,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
};

inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| {
var sample_dir = try shell.project_root.openDir("src/clients/go/samples/" ++ sample, .{});
defer sample_dir.close();

try sample_dir.setAsCwd();
try shell.pushd("./samples/" ++ sample);
defer shell.popd();

var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
defer tmp_beetle.deinit(gpa);
Expand Down
8 changes: 4 additions & 4 deletions src/clients/java/ci.zig
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ const Shell = @import("../../shell.zig");
const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig");

pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
assert(shell.file_exists("pom.xml"));

// Java's maven doesn't support a separate test command, or a way to add dependency on a
// project (as opposed to a compiled jar file).
//
Expand All @@ -17,10 +19,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
try shell.exec("mvn --batch-mode --file pom.xml --quiet install", .{});

inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| {
var sample_dir = try shell.project_root.openDir("src/clients/java/samples/" ++ sample, .{});
defer sample_dir.close();

try sample_dir.setAsCwd();
try shell.pushd("./samples/" ++ sample);
defer shell.popd();

var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
defer tmp_beetle.deinit(gpa);
Expand Down
12 changes: 4 additions & 8 deletions src/clients/node/ci.zig
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ const Shell = @import("../../shell.zig");
const TmpTigerBeetle = @import("../../testing/tmp_tigerbeetle.zig");

pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
assert(shell.file_exists("package.json"));

// We have some unit-tests for node, but they are likely bitrotted, as they are not run on CI.

// Integration tests.
Expand All @@ -18,10 +20,8 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
try shell.exec("npm pack --quiet", .{});

inline for (.{ "basic", "two-phase", "two-phase-many" }) |sample| {
var sample_dir = try shell.project_root.openDir("src/clients/node/samples/" ++ sample, .{});
defer sample_dir.close();

try sample_dir.setAsCwd();
try shell.pushd("./samples/" ++ sample);
defer shell.popd();

var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
defer tmp_beetle.deinit(gpa);
Expand All @@ -33,10 +33,6 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {

// Container smoke tests.
if (builtin.target.os.tag == .linux) {
var client_dir = try shell.project_root.openDir("src/clients/node/", .{});
defer client_dir.close();

try client_dir.setAsCwd();

// Installing node through <https://github.com/nodesource/distributions>.

Expand Down
22 changes: 7 additions & 15 deletions src/scripts/ci.zig
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,12 @@ pub fn main() !void {
var section = try shell.open_section(@tagName(language) ++ " ci");
defer section.close();

var client_src_dir = try shell.project_root.openDir(
"src/clients/" ++ @tagName(language),
.{},
);
defer client_src_dir.close();
{
try shell.pushd("./src/clients/" ++ @tagName(language));
defer shell.popd();

try client_src_dir.setAsCwd();

try ci.tests(shell, gpa);
try ci.tests(shell, gpa);
}

// Piggy back on node client testing to verify our docs, as we use node to generate
// them anyway.
Expand All @@ -86,13 +83,8 @@ pub fn main() !void {
}

fn build_docs(shell: *Shell) !void {
var docs_dir = try shell.project_root.openDir(
"src/docs_website",
.{},
);
defer docs_dir.close();

try docs_dir.setAsCwd();
try shell.pushd("./src/docs_website");
defer shell.popd();

try shell.exec("npm install", .{});
try shell.exec("npm run build", .{});
Expand Down
74 changes: 25 additions & 49 deletions src/scripts/release.zig
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,6 @@ fn build_tigerbeetle(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !vo
var section = try shell.open_section("build tigerbeetle");
defer section.close();

try shell.project_root.setAsCwd();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before, we were defensively setting up cwd, because it could have been changed from under our feet. With pushd/popd we can rather assume that the cwd is reasonable, and just assert that when appropriate.


const llvm_lipo = for (@as([2][]const u8, .{ "llvm-lipo-16", "llvm-lipo" })) |llvm_lipo| {
if (shell.exec_stdout("{llvm_lipo} -version", .{
.llvm_lipo = llvm_lipo,
Expand Down Expand Up @@ -220,10 +218,8 @@ fn build_dotnet(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
var section = try shell.open_section("build dotnet");
defer section.close();

var client_src_dir = try shell.project_root.openDir("src/clients/dotnet", .{});
defer client_src_dir.close();

try client_src_dir.setAsCwd();
try shell.pushd("./src/clients/dotnet");
defer shell.popd();

const dotnet_version = shell.exec_stdout("dotnet --version", .{}) catch {
fatal("can't find dotnet", .{});
Expand All @@ -236,7 +232,7 @@ fn build_dotnet(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
, .{ .version = info.version });

try Shell.copy_path(
client_src_dir,
shell.cwd,
try shell.print("TigerBeetle/bin/Release/tigerbeetle.{s}.nupkg", .{info.version}),
dist_dir,
try shell.print("tigerbeetle.{s}.nupkg", .{info.version}),
Expand All @@ -247,10 +243,8 @@ fn build_go(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
var section = try shell.open_section("build go");
defer section.close();

var client_src_dir = try shell.project_root.openDir("src/clients/go", .{});
defer client_src_dir.close();

try client_src_dir.setAsCwd();
try shell.pushd("./src/clients/go");
defer shell.popd();

try shell.zig("build go_client -Doptimize=ReleaseSafe -Dconfig=production", .{});

Expand All @@ -259,15 +253,15 @@ fn build_go(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
var copied_count: u32 = 0;
while (files_lines.next()) |file| {
assert(file.len > 3);
try Shell.copy_path(client_src_dir, file, dist_dir, file);
try Shell.copy_path(shell.cwd, file, dist_dir, file);
copied_count += 1;
}
assert(copied_count >= 10);

const native_files = try shell.find(.{ .where = &.{"."}, .extensions = &.{ ".a", ".lib" } });
copied_count = 0;
for (native_files) |native_file| {
try Shell.copy_path(client_src_dir, native_file, dist_dir, native_file);
try Shell.copy_path(shell.cwd, native_file, dist_dir, native_file);
copied_count += 1;
}
// 5 = 3 + 2
Expand All @@ -291,18 +285,16 @@ fn build_java(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
var section = try shell.open_section("build java");
defer section.close();

var client_src_dir = try shell.project_root.openDir("src/clients/java", .{});
defer client_src_dir.close();

try client_src_dir.setAsCwd();
try shell.pushd("./src/clients/java");
defer shell.popd();

const java_version = shell.exec_stdout("java --version", .{}) catch {
fatal("can't find java", .{});
};
log.info("java version {s}", .{java_version});

try backup_create(client_src_dir, "pom.xml");
defer backup_restore(client_src_dir, "pom.xml");
try backup_create(shell.cwd, "pom.xml");
defer backup_restore(shell.cwd, "pom.xml");

try shell.exec(
\\mvn --batch-mode --quiet --file pom.xml
Expand All @@ -316,7 +308,7 @@ fn build_java(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
, .{});

try Shell.copy_path(
client_src_dir,
shell.cwd,
try shell.print("target/tigerbeetle-java-{s}.jar", .{info.version}),
dist_dir,
try shell.print("tigerbeetle-java-{s}.jar", .{info.version}),
Expand All @@ -327,28 +319,26 @@ fn build_node(shell: *Shell, info: VersionInfo, dist_dir: std.fs.Dir) !void {
var section = try shell.open_section("build node");
defer section.close();

var client_src_dir = try shell.project_root.openDir("src/clients/node", .{});
defer client_src_dir.close();

try client_src_dir.setAsCwd();
try shell.pushd("./src/clients/node");
defer shell.popd();

const node_version = shell.exec_stdout("node --version", .{}) catch {
fatal("can't find nodejs", .{});
};
log.info("node version {s}", .{node_version});

try backup_create(client_src_dir, "package.json");
defer backup_restore(client_src_dir, "package.json");
try backup_create(shell.cwd, "package.json");
defer backup_restore(shell.cwd, "package.json");

try backup_create(client_src_dir, "package-lock.json");
defer backup_restore(client_src_dir, "package-lock.json");
try backup_create(shell.cwd, "package-lock.json");
defer backup_restore(shell.cwd, "package-lock.json");

try shell.exec("npm version --no-git-tag-version {version}", .{ .version = info.version });
try shell.exec("npm install", .{});
try shell.exec("npm pack --quiet", .{});

try Shell.copy_path(
client_src_dir,
shell.cwd,
try shell.print("tigerbeetle-node-{s}.tgz", .{info.version}),
dist_dir,
try shell.print("tigerbeetle-node-{s}.tgz", .{info.version}),
Expand All @@ -359,7 +349,6 @@ fn publish(shell: *Shell, languages: LanguageSet, info: VersionInfo) !void {
var section = try shell.open_section("publish all");
defer section.close();

try shell.project_root.setAsCwd();
assert(try shell.dir_exists("dist"));

if (languages.contains(.zig)) {
Expand Down Expand Up @@ -447,7 +436,6 @@ fn publish(shell: *Shell, languages: LanguageSet, info: VersionInfo) !void {
}

if (languages.contains(.zig)) {
try shell.project_root.setAsCwd();
try shell.exec(
\\gh release edit --draft=false
\\ {tag}
Expand Down Expand Up @@ -490,7 +478,6 @@ fn publish_dotnet(shell: *Shell, info: VersionInfo) !void {
var section = try shell.open_section("publish dotnet");
defer section.close();

try shell.project_root.setAsCwd();
assert(try shell.dir_exists("dist/dotnet"));

const nuget_key = try shell.env_get("NUGET_KEY");
Expand All @@ -509,7 +496,6 @@ fn publish_go(shell: *Shell, info: VersionInfo) !void {
var section = try shell.open_section("publish go");
defer section.close();

try shell.project_root.setAsCwd();
assert(try shell.dir_exists("dist/go"));

const token = try shell.env_get("TIGERBEETLE_GO_PAT");
Expand Down Expand Up @@ -538,10 +524,8 @@ fn publish_go(shell: *Shell, info: VersionInfo) !void {
);
}

var tigerbeetle_go_dir = try shell.project_root.openDir("tigerbeetle-go", .{});
defer tigerbeetle_go_dir.close();

try tigerbeetle_go_dir.setAsCwd();
try shell.pushd("tigerbeetle-go");
defer shell.popd();

try shell.exec("git add .", .{});
// Native libraries are ignored in this repository, but we want to push them to the
Expand Down Expand Up @@ -571,7 +555,6 @@ fn publish_java(shell: *Shell, info: VersionInfo) !void {
var section = try shell.open_section("publish java");
defer section.close();

try shell.project_root.setAsCwd();
assert(try shell.dir_exists("dist/java"));

// These variables don't have a special meaning in maven, and instead are a part of
Expand Down Expand Up @@ -611,7 +594,6 @@ fn publish_node(shell: *Shell, info: VersionInfo) !void {
var section = try shell.open_section("publish node");
defer section.close();

try shell.project_root.setAsCwd();
assert(try shell.dir_exists("dist/node"));

// `NODE_AUTH_TOKEN` env var doesn't have a special meaning in npm. It does have special meaning
Expand All @@ -630,7 +612,6 @@ fn publish_docker(shell: *Shell, info: VersionInfo) !void {
var section = try shell.open_section("publish docker");
defer section.close();

try shell.project_root.setAsCwd();
assert(try shell.dir_exists("dist/tigerbeetle"));

try shell.exec(
Expand Down Expand Up @@ -693,15 +674,12 @@ fn publish_docs(shell: *Shell, info: VersionInfo) !void {
defer section.close();

{
var docs_dir = try shell.project_root.openDir("src/docs_website", .{});
defer docs_dir.close();

try docs_dir.setAsCwd();
try shell.pushd("src/docs_website");
defer shell.popd();

try shell.exec("npm install", .{});
try shell.exec("npm run build", .{});
}
try shell.project_root.setAsCwd();

const token = try shell.env_get("TIGERBEETLE_DOCS_PAT");
try shell.exec(
Expand Down Expand Up @@ -729,10 +707,8 @@ fn publish_docs(shell: *Shell, info: VersionInfo) !void {
);
}

var tigerbeetle_docs_dir = try shell.project_root.openDir("tigerbeetle-docs", .{});
defer tigerbeetle_docs_dir.close();

try tigerbeetle_docs_dir.setAsCwd();
try shell.pushd("tigerebeetle-docs");
defer shell.popd();

try shell.exec("git add .", .{});
try shell.env.put("GIT_AUTHOR_NAME", "TigerBeetle Bot");
Expand Down