From 80c5799b253bc112c7f9694619ea283c9f0e5c97 Mon Sep 17 00:00:00 2001 From: Alex Kladov Date: Wed, 20 Mar 2024 17:02:58 +0000 Subject: [PATCH] devhub: unify devhub and nyrkio metrics format The two are isomorphic, and nyrkio's one is better in details. I will manually update existing data in the devhubdb repo after this merges. --- src/devhub/devhub.js | 40 +++++++++++++-------------- src/devhub/index.html | 2 +- src/scripts/devhub.zig | 62 +++++++++++++++--------------------------- 3 files changed, 43 insertions(+), 61 deletions(-) diff --git a/src/devhub/devhub.js b/src/devhub/devhub.js index 89e051b769..5df7c76c8d 100644 --- a/src/devhub/devhub.js +++ b/src/devhub/devhub.js @@ -8,8 +8,8 @@ // - no TypeScript, no build step async function main() { - const runs = await fetchData(); - const series = runsToSeries(runs); + const batches = await fetchData(); + const series = batchesToSeries(batches); plotSeries(series, document.querySelector("#charts")); const releaseManager = getReleaseManager(); @@ -63,32 +63,32 @@ async function fetchData() { // form a single array which is what we want to plot. // // This doesn't depend on particular plotting library though. -function runsToSeries(runs) { +function batchesToSeries(batches) { const result = new Map(); - for (const run of runs) { - for (const measurement of run.measurements) { - if (!result.has(measurement.label)) { - result.set(measurement.label, { - label: measurement.label, + for (const batch of batches) { + for (const metric of batch.metrics) { + if (!result.has(metric.name)) { + result.set(metric.label, { + name: metric.name, unit: undefined, value: [], - revision: [], + git_commit: [], timestamp: [], }); } - const series = result.get(measurement.label); - assert(series.label == measurement.label); + const series = result.get(metric.name); + assert(series.name == metric.name); if (series.unit) { - assert(series.unit == measurement.unit); + assert(series.unit == metric.unit); } else { - series.unit = measurement.unit; + series.unit = metric.unit; } - series.value.push(measurement.value); - series.revision.push(run.revision); - series.timestamp.push(run.timestamp); + series.value.push(metric.value); + series.git_commit.push(batch.attributes.git_commit); + series.timestamp.push(batch.timestamp); } } return result.values(); @@ -99,7 +99,7 @@ function plotSeries(seriesList, rootNode) { for (const series of seriesList) { let options = { title: { - text: series.label, + text: series.name, }, chart: { type: "line", @@ -108,7 +108,7 @@ function plotSeries(seriesList, rootNode) { dataPointSelection: (event, chartContext, { dataPointIndex }) => { window.open( "https://github.com/tigerbeetle/tigerbeetle/commit/" + - series.revision[dataPointIndex], + series.git_commit[dataPointIndex], ); }, }, @@ -117,7 +117,7 @@ function plotSeries(seriesList, rootNode) { size: 4, }, series: [{ - name: series.label, + name: series.name, data: series.value, }], xaxis: { @@ -137,7 +137,7 @@ function plotSeries(seriesList, rootNode) { const timestamp = new Date(series.timestamp[dataPointIndex] * 1000); const formattedDate = timestamp.toLocaleString(); return `
${ - series.revision[dataPointIndex] + series.git_commit[dataPointIndex] }
${formattedDate}
`; }, }, diff --git a/src/devhub/index.html b/src/devhub/index.html index 53282e16dd..bfb952ef99 100644 --- a/src/devhub/index.html +++ b/src/devhub/index.html @@ -26,7 +26,7 @@

Release

-

Release Metrics

+

Metrics (on Nyrkiö)

diff --git a/src/scripts/devhub.zig b/src/scripts/devhub.zig index 0d2b944e65..66f5b66bd0 100644 --- a/src/scripts/devhub.zig +++ b/src/scripts/devhub.zig @@ -38,29 +38,14 @@ pub fn main(shell: *Shell, gpa: std.mem.Allocator, cli_args: CliArgs) !void { const query_p99_ms = try get_measurement(benchmark_result, "query latency p99", "ms"); const rss_bytes = try get_measurement(benchmark_result, "rss", "bytes"); - try upload_run(shell, Run{ - // Use commit timestamp, rather wall clock time here. That way, it is possible to re-bench - // mark the entire history while getting a comparable time series. - .timestamp = commit_timestamp, - .revision = cli_args.sha, - .measurements = &[_]Measurement{ - .{ .label = "build time", .value = build_time_ms, .unit = "ms" }, - .{ .label = "executable size", .value = executable_size_bytes, .unit = "bytes" }, - .{ .label = "TPS", .value = tps, .unit = "count" }, - .{ .label = "batch p99", .value = batch_p99_ms, .unit = "ms" }, - .{ .label = "query p99", .value = query_p99_ms, .unit = "ms" }, - .{ .label = "RSS", .value = rss_bytes, .unit = "bytes" }, - }, - }); - - upload_nyrkio(shell, NyrkioRun{ + const batch = MetricBatch{ .timestamp = commit_timestamp, .attributes = .{ .git_repo = "https://github.com/tigerbeetle/tigerbeetle", .git_commit = cli_args.sha, .branch = "main", }, - .metrics = &[_]NyrkioRun.Metric{ + .metrics = &[_]Metric{ .{ .name = "build time", .value = build_time_ms, .unit = "ms" }, .{ .name = "executable size", .value = executable_size_bytes, .unit = "bytes" }, .{ .name = "TPS", .value = tps, .unit = "count" }, @@ -68,7 +53,11 @@ pub fn main(shell: *Shell, gpa: std.mem.Allocator, cli_args: CliArgs) !void { .{ .name = "query p99", .value = query_p99_ms, .unit = "ms" }, .{ .name = "RSS", .value = rss_bytes, .unit = "bytes" }, }, - }) catch |err| { + }; + + try upload_run(shell, &batch); + + upload_nyrkio(shell, &batch) catch |err| { log.err("failed to upload Nyrkiö metrics: {}", .{err}); }; } @@ -88,19 +77,7 @@ fn get_measurement( return try std.fmt.parseInt(u64, cut.prefix, 10); } -const Measurement = struct { - label: []const u8, - value: u64, - unit: []const u8, -}; - -const Run = struct { - timestamp: u64, - revision: []const u8, - measurements: []const Measurement, -}; - -fn upload_run(shell: *Shell, run: Run) !void { +fn upload_run(shell: *Shell, batch: *const MetricBatch) !void { const token = try shell.env_get("DEVHUBDB_PAT"); try shell.exec( \\git clone --depth 1 @@ -120,7 +97,7 @@ fn upload_run(shell: *Shell, run: Run) !void { defer file.close(); try file.seekFromEnd(0); - try std.json.stringify(run, .{}, file.writer()); + try std.json.stringify(batch, .{}, file.writer()); try file.writeAll("\n"); } @@ -130,12 +107,13 @@ fn upload_run(shell: *Shell, run: Run) !void { try shell.exec("git push", .{}); } -const NyrkioRun = struct { - const Metric = struct { - name: []const u8, - unit: []const u8, - value: u64, - }; +const Metric = struct { + name: []const u8, + unit: []const u8, + value: u64, +}; + +const MetricBatch = struct { timestamp: u64, metrics: []const Metric, attributes: struct { @@ -145,9 +123,13 @@ const NyrkioRun = struct { }, }; -fn upload_nyrkio(shell: *Shell, run: NyrkioRun) !void { +fn upload_nyrkio(shell: *Shell, batch: *const MetricBatch) !void { const token = try shell.env_get("NYRKIO_TOKEN"); - const payload = try std.json.stringifyAlloc(shell.arena.allocator(), [_]NyrkioRun{run}, .{}); + const payload = try std.json.stringifyAlloc( + shell.arena.allocator(), + [_]*const MetricBatch{batch}, // Nyrkiö needs an _array_ of batches. + .{}, + ); try shell.exec( \\curl -s -X POST --fail-with-body \\ -H {content_type}