ci: implement release verification

This adds last-mile testing of the thing that was actually released
(GitHub release, nmp package, etc). This is too late to _prevent_
issues, but helps to discover things otherwise impossible to discover
(eg, some files were lost during upload to GitHub), and also helps with
finding divergence of upstream dependencies (e.g., if something breaks
on a new version of node js).

.github/workflows/release_validate.yml

@@ -1,8 +1,13 @@
 name: "Release (validate)"
 
 on:
+  workflow_run:
+    workflows: ["Release"]
+    types:
+      - completed
+
   schedule:
-    - cron: 0 0 0 1 1 ? 1970 # Never for now, planned to run every six hours
+    - cron: 0 */6 * * * # Every six hours
 
 jobs:
   validate:
@@ -17,10 +22,7 @@ jobs:
 
       - run: ./scripts/install_zig.sh
 
-      - run: ./zig/zig build dotnet_client:ci -- --release-validate
-      - run: ./zig/zig build go_client:ci     -- --release-validate
-      - run: ./zig/zig build java_client:ci   -- --release-validate
-      - run: ./zig/zig build node_client:ci   -- --release-validate
+      - run: ./zig/zig build ci -- --verify-release
 
   alert_failure:
     runs-on: ubuntu-latest

src/clients/dotnet/ci.zig

@@ -94,17 +94,26 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
     }
 }
 
-pub fn verify_release(shell: *Shell, gpa: std.mem.Allocator, tmp_dir: std.fs.Dir) !void {
-    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
+pub fn validate_release(shell: *Shell, gpa: std.mem.Allocator, options: struct {
+    version: []const u8,
+    tigerbeetle: []const u8,
+}) !void {
+    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{
+        .prebuilt = options.tigerbeetle,
+    });
     defer tmp_beetle.deinit(gpa);
 
+    try shell.env.put("TB_ADDRESS", tmp_beetle.port_str.slice());
+
     try shell.exec("dotnet new console", .{});
-    try shell.exec("dotnet add package tigerbeetle", .{});
+    try shell.exec("dotnet add package tigerbeetle --version {version}", .{
+        .version = options.version,
+    });
 
     try Shell.copy_path(
         shell.project_root,
         "src/clients/dotnet/samples/basic/Program.cs",
-        tmp_dir,
+        std.fs.cwd(),
         "Program.cs",
     );
     try shell.exec("dotnet run", .{});

src/clients/go/ci.zig

@@ -47,17 +47,26 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
     }
 }
 
-pub fn verify_release(shell: *Shell, gpa: std.mem.Allocator, tmp_dir: std.fs.Dir) !void {
-    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
+pub fn validate_release(shell: *Shell, gpa: std.mem.Allocator, options: struct {
+    version: []const u8,
+    tigerbeetle: []const u8,
+}) !void {
+    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{
+        .prebuilt = options.tigerbeetle,
+    });
     defer tmp_beetle.deinit(gpa);
 
+    try shell.env.put("TB_ADDRESS", tmp_beetle.port_str.slice());
+
     try shell.exec("go mod init tbtest", .{});
-    try shell.exec("go get github.com/tigerbeetle/tigerbeetle-go", .{});
+    try shell.exec("go get github.com/tigerbeetle/tigerbeetle-go@v{version}", .{
+        .version = options.version,
+    });
 
     try Shell.copy_path(
         shell.project_root,
         "src/clients/go/samples/basic/main.go",
-        tmp_dir,
+        std.fs.cwd(),
         "main.go",
     );
     const zig_exe = try shell.project_root.realpathAlloc(

src/clients/java/ci.zig

@@ -33,11 +33,72 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
     }
 }
 
-pub fn verify_release(shell: *Shell, gpa: std.mem.Allocator, tmp_dir: std.fs.Dir) !void {
-    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
+pub fn validate_release(shell: *Shell, gpa: std.mem.Allocator, options: struct {
+    version: []const u8,
+    tigerbeetle: []const u8,
+}) !void {
+    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{
+        .prebuilt = options.tigerbeetle,
+    });
     defer tmp_beetle.deinit(gpa);
 
-    // TODO
-    _ = shell;
-    _ = tmp_dir;
+    try shell.env.put("TB_ADDRESS", tmp_beetle.port_str.slice());
+
+    try std.fs.cwd().writeFile(
+        "pom.xml",
+        \\<project>
+        \\  <modelVersion>4.0.0</modelVersion>
+        \\  <groupId>com.tigerbeetle</groupId>
+        \\  <artifactId>samples</artifactId>
+        \\  <version>1.0-SNAPSHOT</version>
+        \\
+        \\  <properties>
+        \\    <maven.compiler.source>11</maven.compiler.source>
+        \\    <maven.compiler.target>11</maven.compiler.target>
+        \\  </properties>
+        \\
+        \\  <build>
+        \\    <plugins>
+        \\      <plugin>
+        \\        <groupId>org.apache.maven.plugins</groupId>
+        \\        <artifactId>maven-compiler-plugin</artifactId>
+        \\        <version>3.8.1</version>
+        \\        <configuration>
+        \\          <compilerArgs>
+        \\            <arg>-Xlint:all,-options,-path</arg>
+        \\          </compilerArgs>
+        \\        </configuration>
+        \\      </plugin>
+        \\
+        \\      <plugin>
+        \\        <groupId>org.codehaus.mojo</groupId>
+        \\        <artifactId>exec-maven-plugin</artifactId>
+        \\        <version>1.6.0</version>
+        \\        <configuration>
+        \\          <mainClass>com.tigerbeetle.samples.Main</mainClass>
+        \\        </configuration>
+        \\      </plugin>
+        \\    </plugins>
+        \\  </build>
+        \\
+        \\  <dependencies>
+        \\    <dependency>
+        \\      <groupId>com.tigerbeetle</groupId>
+        \\      <artifactId>tigerbeetle-java</artifactId>
+        \\      <version>0.14.158</version>
+        \\    </dependency>
+        \\  </dependencies>
+        \\</project>
+        ,
+    );
+
+    try Shell.copy_path(
+        shell.project_root,
+        "src/clients/java/samples/basic/src/main/java/Main.java",
+        std.fs.cwd(),
+        "src/main/java/Main.java",
+    );
+
+    try shell.exec("mvn package", .{});
+    try shell.exec("mvn exec:java", .{});
 }

src/clients/node/ci.zig

@@ -99,16 +99,25 @@ pub fn tests(shell: *Shell, gpa: std.mem.Allocator) !void {
     }
 }
 
-pub fn verify_release(shell: *Shell, gpa: std.mem.Allocator, tmp_dir: std.fs.Dir) !void {
-    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{});
+pub fn validate_release(shell: *Shell, gpa: std.mem.Allocator, options: struct {
+    version: []const u8,
+    tigerbeetle: []const u8,
+}) !void {
+    var tmp_beetle = try TmpTigerBeetle.init(gpa, .{
+        .prebuilt = options.tigerbeetle,
+    });
     defer tmp_beetle.deinit(gpa);
 
-    try shell.exec("npm install tigerbeetle-node", .{});
+    try shell.env.put("TB_ADDRESS", tmp_beetle.port_str.slice());
+
+    try shell.exec("npm install tigerbeetle-node@{version}", .{
+        .version = options.version,
+    });
 
     try Shell.copy_path(
         shell.project_root,
         "src/clients/node/samples/basic/main.js",
-        tmp_dir,
+        std.fs.cwd(),
         "main.js",
     );
     try shell.exec("node main.js", .{});

src/scripts/ci.zig

@@ -8,6 +8,7 @@ const builtin = @import("builtin");
 const log = std.log;
 const assert = std.debug.assert;
 
+const stdx = @import("../stdx.zig");
 const flags = @import("../flags.zig");
 const fatal = flags.fatal;
 const Shell = @import("../shell.zig");
@@ -23,7 +24,7 @@ const LanguageCI = .{
 
 const CliArgs = struct {
     language: ?Language = null,
-    verify_release: bool = false,
+    validate_release: bool = false,
 };
 
 pub fn main() !void {
@@ -46,39 +47,38 @@ pub fn main() !void {
     assert(args.skip());
     const cli_args = flags.parse_flags(&args, CliArgs);
 
+    if (cli_args.validate_release) {
+        try validate_release(shell, gpa, cli_args.language);
+    } else {
+        try run_tests(shell, gpa, cli_args.language);
+    }
+}
+
+fn run_tests(shell: *Shell, gpa: std.mem.Allocator, language_requested: ?Language) !void {
     inline for (comptime std.enums.values(Language)) |language| {
-        if (cli_args.language == language or cli_args.language == null) {
+        if (language_requested == language or language_requested == null) {
             const ci = @field(LanguageCI, @tagName(language));
-            if (cli_args.verify_release) {
-                var tmp_dir = std.testing.tmpDir(.{});
-                defer tmp_dir.cleanup();
-
-                try tmp_dir.dir.setAsCwd();
-
-                try ci.verify_release(shell, gpa, tmp_dir.dir);
-            } else {
-                var section = try shell.open_section(@tagName(language) ++ " ci");
-                defer section.close();
-
-                var client_src_dir = try shell.project_root.openDir(
-                    "src/clients/" ++ @tagName(language),
-                    .{},
-                );
-                defer client_src_dir.close();
-
-                try client_src_dir.setAsCwd();
-
-                try ci.tests(shell, gpa);
-
-                // Piggy back on node client testing to verify our docs, as we use node to generate
-                // them anyway.
-                if (language == .node and builtin.os.tag == .linux) {
-                    const node_version = try shell.exec_stdout("node --version", .{});
-                    if (std.mem.startsWith(u8, node_version, "v14")) {
-                        log.warn("skip building documentation on old Node.js", .{});
-                    } else {
-                        try build_docs(shell);
-                    }
+            var section = try shell.open_section(@tagName(language) ++ " ci");
+            defer section.close();
+
+            var client_src_dir = try shell.project_root.openDir(
+                "src/clients/" ++ @tagName(language),
+                .{},
+            );
+            defer client_src_dir.close();
+
+            try client_src_dir.setAsCwd();
+
+            try ci.tests(shell, gpa);
+
+            // Piggy back on node client testing to verify our docs, as we use node to generate
+            // them anyway.
+            if (language == .node and builtin.os.tag == .linux) {
+                const node_version = try shell.exec_stdout("node --version", .{});
+                if (std.mem.startsWith(u8, node_version, "v14")) {
+                    log.warn("skip building documentation on old Node.js", .{});
+                } else {
+                    try build_docs(shell);
                 }
             }
         }
@@ -97,3 +97,44 @@ fn build_docs(shell: *Shell) !void {
     try shell.exec("npm install", .{});
     try shell.exec("npm run build", .{});
 }
+
+fn validate_release(shell: *Shell, gpa: std.mem.Allocator, language_requested: ?Language) !void {
+    var tmp_dir = std.testing.tmpDir(.{});
+    defer tmp_dir.cleanup();
+
+    try tmp_dir.dir.setAsCwd();
+
+    const release_info = try shell.exec_stdout(
+        "gh release list --repo tigerbeetle/tigerbeetle --limit 1",
+        .{},
+    );
+    const tag = stdx.cut(release_info, "\t").?.prefix;
+    log.info("validateing release {s}", .{tag});
+
+    try shell.exec("gh release download {tag}", .{ .tag = tag });
+
+    if (builtin.os.tag != .linux) {
+        log.warn("skip release verification for platforms other than Linux", .{});
+    }
+
+    try shell.exec("unzip tigerbeetle-x86_64-linux.zip", .{});
+    const version = try shell.exec_stdout("./tigerbeetle version --verbose", .{});
+    assert(std.mem.indexOf(u8, version, tag) != null);
+    assert(std.mem.indexOf(u8, version, "ReleaseSafe") != null);
+
+    const tigerbeetle_absolute_path = try std.fs.cwd().realpathAlloc(gpa, "tigerbeetle");
+    defer gpa.free(tigerbeetle_absolute_path);
+
+    inline for (comptime std.enums.values(Language)) |language| {
+        // TODO: these two don't work yet
+        if (language == .go or language == .dotnet) continue;
+
+        if (language_requested == language or language_requested == null) {
+            const ci = @field(LanguageCI, @tagName(language));
+            try ci.validate_release(shell, gpa, .{
+                .tigerbeetle = tigerbeetle_absolute_path,
+                .version = tag,
+            });
+        }
+    }
+}

src/testing/tmp_tigerbeetle.zig

@@ -33,24 +33,36 @@ stderr_reader_thread: std.Thread,
 
 pub fn init(
     gpa: std.mem.Allocator,
-    options: struct { echo: bool = true },
+    options: struct {
+        echo: bool = true,
+        prebuilt: ?[]const u8 = null,
+    },
 ) !TmpTigerBeetle {
     const shell = try Shell.create(gpa);
     defer shell.destroy();
 
-    const tigerbeetle_exe = comptime "tigerbeetle" ++ builtin.target.exeFileExt();
+    var from_source_path: ?[]const u8 = null;
+    defer if (from_source_path) |path| gpa.free(path);
 
-    // If tigerbeetle binary does not exist yet, build it.
-    // TODO: just run `zig build run` unconditionally here, when that doesn't do spurious rebuilds.
-    _ = shell.project_root.statFile(tigerbeetle_exe) catch {
-        log.info("building TigerBeetle", .{});
-        try shell.zig("build", .{});
+    if (options.prebuilt == null) {
+        const tigerbeetle_exe = comptime "tigerbeetle" ++ builtin.target.exeFileExt();
 
-        _ = try shell.project_root.statFile(tigerbeetle_exe);
-    };
+        // If tigerbeetle binary does not exist yet, build it.
+        //
+        // TODO: just run `zig build run` unconditionally here, when that doesn't do spurious
+        // rebuilds.
+        _ = shell.project_root.statFile(tigerbeetle_exe) catch {
+            log.info("building TigerBeetle", .{});
+            try shell.zig("build", .{});
+
+            _ = try shell.project_root.statFile(tigerbeetle_exe);
+        };
+
+        from_source_path = try shell.project_root.realpathAlloc(gpa, tigerbeetle_exe);
+    }
 
-    const tigerbeetle: []const u8 = try shell.project_root.realpathAlloc(gpa, tigerbeetle_exe);
-    defer gpa.free(tigerbeetle);
+    const tigerbeetle: []const u8 = options.prebuilt orelse from_source_path.?;
+    assert(std.fs.path.isAbsolute(tigerbeetle));
 
     var tmp_dir = std.testing.tmpDir(.{});
     errdefer tmp_dir.cleanup();