feat: collect benchmark data into 'benchmarks' branch

.github/workflows/build.yml

@@ -103,6 +103,31 @@ jobs:
                         ./Artifacts/*
                     if-no-files-found: ignore
 
+    publish-benchmark-report:
+        name: "Publish Benchmark Report"
+        needs: [ benchmarks ]
+        if: github.ref == 'refs/heads/main'
+        runs-on: ubuntu-latest
+        permissions:
+            actions: read
+            contents: write
+        env:
+            DOTNET_NOLOGO: true
+        steps:
+            -   uses: actions/checkout@v6
+                with:
+                    fetch-depth: 0
+            -   name: Setup .NET SDKs
+                uses: actions/setup-dotnet@v5
+                with:
+                    dotnet-version: |
+                        10.0.x
+            -   name: Publish benchmark report
+                run: ./build.sh PublishBenchmarkReport
+                env:
+                    GithubToken: ${{ secrets.GITHUB_TOKEN }}
+                    WorkflowRunId: ${{ github.run_id }}
+
     mutation-tests:
         name: "Mutation tests"
         runs-on: ubuntu-latest

.nuke/build.schema.json

@@ -42,6 +42,7 @@
         "MutationTestExecution",
         "MutationTests",
         "Pack",
+        "PublishBenchmarkReport",
         "Restore",
         "UnitTests",
         "UpdateReadme"

Pipeline/Build.Benchmarks.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Nuke.Common;
 using Nuke.Common.IO;
+using Nuke.Common.Tooling;
 using Nuke.Common.Tools.DotNet;
+using Nuke.Common.Tools.Git;
 using Octokit;
 using Serilog;
 using static Nuke.Common.Tools.DotNet.DotNetTasks;
@@ -15,6 +18,11 @@ namespace Build;
 
 partial class Build
 {
+	private const string BenchmarkBranch = "benchmarks";
+	private const string BenchmarkDataPath = "Docs/pages/static/js/data.js";
+	private const string BenchmarkLimitedDataPath = "Docs/pages/static/js/limited-data.js";
+	private const int BenchmarkLimit = 50;
+
 	[Parameter("Filter for BenchmarkDotNet - Default is '*'")] readonly string BenchmarkFilter = "*";
 
 	Target BenchmarkDotNet => _ => _
@@ -126,6 +134,108 @@ partial class Build
 		.DependsOn(BenchmarkDotNet)
 		.DependsOn(BenchmarkResult);
 
+	Target PublishBenchmarkReport => _ => _
+		.Description("Aggregates BenchmarkDotNet JSON results from the matrix Benchmarks-* artifacts of the " +
+		             "current workflow run, then commits an updated data.js / limited-data.js to the " +
+		             $"'{BenchmarkBranch}' branch.")
+		.Requires(() => GithubToken)
+		.Executes(async () =>
+		{
+			await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken);
+
+			List<string> benchmarkReports = LoadBenchmarkJsonReports(ArtifactsDirectory / "Benchmarks" / "results");
+			if (benchmarkReports.Count == 0)
+			{
+				Log.Warning("Skip benchmark report: no benchmark JSON reports found.");
+				return;
+			}
+
+			PageBenchmarkReportGenerator.CommitInfo commitInfo = ReadCurrentCommitInfo();
+			Log.Information(
+				"Appending benchmark data for commit {Sha} ({Author}, {Date}): {Message}",
+				commitInfo.Sha, commitInfo.Author, commitInfo.Date, commitInfo.Message);
+
+			BuildExtensions.GithubFile dataFile =
+				await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken);
+			BuildExtensions.GithubFile limitedFile =
+				await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken);
+
+			(string updated, string limited) = PageBenchmarkReportGenerator.Append(
+				commitInfo,
+				dataFile.Content,
+				benchmarkReports,
+				BenchmarkLimit);
+
+			if (string.IsNullOrWhiteSpace(updated))
+			{
+				Log.Information("No changes to publish (commit already recorded).");
+				return;
+			}
+
+			string commitMessage =
+				$"Update benchmark for {commitInfo.Sha.Substring(0, 8)}: {commitInfo.Message} by {commitInfo.Author}";
+			await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, updated,
+				dataFile.Sha, GithubToken);
+			await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage,
+				limited, limitedFile.Sha, GithubToken);
+		});
+
+	private static List<string> LoadBenchmarkJsonReports(AbsolutePath resultsDirectory)
+	{
+		List<string> reports = new();
+		if (!Directory.Exists(resultsDirectory))
+		{
+			return reports;
+		}
+
+		foreach (string file in Directory.GetFiles(resultsDirectory, "*-report-full-compressed.json"))
+		{
+			reports.Add(File.ReadAllText(file));
+		}
+
+		return reports;
+	}
+
+	private static PageBenchmarkReportGenerator.CommitInfo ReadCurrentCommitInfo()
+	{
+		Output[] lines = GitTasks.Git("log -1").ToArray();
+		string commitId = null, author = null, date = null, message = null;
+		foreach (string line in lines.Select(x => x.Text))
+		{
+			if (commitId == null && line.StartsWith("commit "))
+			{
+				commitId = line.Substring("commit ".Length).Substring(0, 40);
+				continue;
+			}
+
+			if (author == null && line.StartsWith("Author: "))
+			{
+				author = line.Substring("Author: ".Length);
+				int index = author.IndexOf(" <", StringComparison.Ordinal);
+				if (index > 0)
+				{
+					author = author.Substring(0, index);
+				}
+
+				continue;
+			}
+
+			if (date == null && line.StartsWith("Date:   "))
+			{
+				date = line.Substring("Date:   ".Length);
+				continue;
+			}
+
+			if (commitId != null && author != null && date != null && !string.IsNullOrWhiteSpace(line))
+			{
+				message = line.Trim();
+				break;
+			}
+		}
+
+		return new PageBenchmarkReportGenerator.CommitInfo(commitId, author, date, message);
+	}
+
 	async Task<string> DownloadBaselineBenchmarks(AbsolutePath baselineDirectory)
 	{
 		long[] candidateRunIds = await BuildExtensions.FindRecentSuccessfulRunIds("build.yml", "main", 10, GithubToken);

Pipeline/BuildExtensions.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Nuke.Common.CI.GitHubActions;
@@ -163,4 +164,78 @@ private static async Task DownloadArtifactsFromRun(string runId, Func<string, bo
 			Log.Error($"Could not parse JSON: {e.Message}\n{responseContent}");
 		}
 	}
+
+	/// <summary>
+	///     Reads a file from the given <paramref name="branch" /> using the GitHub contents API.
+	///     Returns <see langword="null" /> if the file does not exist on that branch.
+	/// </summary>
+	public static async Task<GithubFile> ReadBranchFileAsync(string path, string branch, string githubToken)
+	{
+		using HttpClient client = CreateGithubClient(githubToken);
+		HttpResponseMessage response = await client.GetAsync(
+			$"{RepositoryApiBaseUrl}/contents/{path}?ref={Uri.EscapeDataString(branch)}");
+		if (!response.IsSuccessStatusCode)
+		{
+			return null;
+		}
+
+		string responseContent = await response.Content.ReadAsStringAsync();
+		using JsonDocument document = JsonDocument.Parse(responseContent);
+		string sha = document.RootElement.GetProperty("sha").GetString();
+		string downloadUrl =
+			$"https://raw.githubusercontent.com/{Owner}/{Repo}/refs/heads/{branch}/{path}";
+		using HttpClient rawClient = new();
+		rawClient.DefaultRequestHeaders.UserAgent.ParseAdd(Repo);
+		string content = await rawClient.GetStringAsync(downloadUrl);
+		return new GithubFile(content, sha);
+	}
+
+	/// <summary>
+	///     Writes a file to the given <paramref name="branch" /> using the GitHub contents API.
+	/// </summary>
+	public static async Task WriteBranchFileAsync(string path, string branch, string commitMessage, string content,
+		string existingSha, string githubToken)
+	{
+		using HttpClient client = CreateGithubClient(githubToken);
+		GithubUpdateFile body = new(commitMessage, Base64Encode(content), existingSha, branch);
+		HttpResponseMessage response = await client.PutAsync(
+			$"{RepositoryApiBaseUrl}/contents/{path}",
+			new StringContent(JsonSerializer.Serialize(body), Encoding.UTF8, "application/json"));
+		if (response.IsSuccessStatusCode)
+		{
+			Log.Information("Updated {Path} on branch '{Branch}'.", path, branch);
+		}
+		else
+		{
+			string responseContent = await response.Content.ReadAsStringAsync();
+			throw new InvalidOperationException(
+				$"Could not update '{path}' on branch '{branch}': {responseContent}");
+		}
+	}
+
+	private static HttpClient CreateGithubClient(string githubToken)
+	{
+		HttpClient client = new();
+		client.DefaultRequestHeaders.UserAgent.ParseAdd(Repo);
+		if (!string.IsNullOrEmpty(githubToken))
+		{
+			client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", githubToken);
+		}
+
+		return client;
+	}
+
+	private static string Base64Encode(string plainText)
+	{
+		byte[] plainTextBytes = Encoding.UTF8.GetBytes(plainText);
+		return Convert.ToBase64String(plainTextBytes);
+	}
+
+	public record GithubFile(string Content, string Sha);
+
+	// ReSharper disable InconsistentNaming
+	// ReSharper disable NotAccessedPositionalProperty.Local
+	private record GithubUpdateFile(string message, string content, string sha, string branch);
+	// ReSharper restore NotAccessedPositionalProperty.Local
+	// ReSharper restore InconsistentNaming
 }