diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ffcba82..89add602 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,6 +103,31 @@ jobs: ./Artifacts/* if-no-files-found: ignore + publish-benchmark-report: + name: "Publish Benchmark Report" + needs: [ benchmarks ] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + actions: read + contents: write + env: + DOTNET_NOLOGO: true + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Setup .NET SDKs + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 10.0.x + - name: Publish benchmark report + run: ./build.sh PublishBenchmarkReport + env: + GithubToken: ${{ secrets.GITHUB_TOKEN }} + WorkflowRunId: ${{ github.run_id }} + mutation-tests: name: "Mutation tests" runs-on: ubuntu-latest diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index b86fdf90..42be27ab 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -42,6 +42,7 @@ "MutationTestExecution", "MutationTests", "Pack", + "PublishBenchmarkReport", "Restore", "UnitTests", "UpdateReadme" diff --git a/Pipeline/Build.Benchmarks.cs b/Pipeline/Build.Benchmarks.cs index 852e5e19..f70e688e 100644 --- a/Pipeline/Build.Benchmarks.cs +++ b/Pipeline/Build.Benchmarks.cs @@ -1,10 +1,13 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using Nuke.Common; using Nuke.Common.IO; +using Nuke.Common.Tooling; using Nuke.Common.Tools.DotNet; +using Nuke.Common.Tools.Git; using Octokit; using Serilog; using static Nuke.Common.Tools.DotNet.DotNetTasks; @@ -15,6 +18,11 @@ namespace Build; partial class Build { + private const string BenchmarkBranch = "benchmarks"; + private const string BenchmarkDataPath = "Docs/pages/static/js/data.js"; + private const string BenchmarkLimitedDataPath = "Docs/pages/static/js/limited-data.js"; + private const int BenchmarkLimit = 50; + [Parameter("Filter for BenchmarkDotNet - Default is '*'")] readonly string BenchmarkFilter = "*"; Target BenchmarkDotNet => _ => _ @@ -126,6 +134,108 @@ partial class Build .DependsOn(BenchmarkDotNet) .DependsOn(BenchmarkResult); + Target PublishBenchmarkReport => _ => _ + .Description("Aggregates BenchmarkDotNet JSON results from the matrix Benchmarks-* artifacts of the " + + "current workflow run, then commits an updated data.js / limited-data.js to the " + + $"'{BenchmarkBranch}' branch.") + .Requires(() => GithubToken) + .Executes(async () => + { + await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken); + + List benchmarkReports = LoadBenchmarkJsonReports(ArtifactsDirectory / "Benchmarks" / "results"); + if (benchmarkReports.Count == 0) + { + Log.Warning("Skip benchmark report: no benchmark JSON reports found."); + return; + } + + PageBenchmarkReportGenerator.CommitInfo commitInfo = ReadCurrentCommitInfo(); + Log.Information( + "Appending benchmark data for commit {Sha} ({Author}, {Date}): {Message}", + commitInfo.Sha, commitInfo.Author, commitInfo.Date, commitInfo.Message); + + BuildExtensions.GithubFile dataFile = + await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken); + BuildExtensions.GithubFile limitedFile = + await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken); + + (string updated, string limited) = PageBenchmarkReportGenerator.Append( + commitInfo, + dataFile.Content, + benchmarkReports, + BenchmarkLimit); + + if (string.IsNullOrWhiteSpace(updated)) + { + Log.Information("No changes to publish (commit already recorded)."); + return; + } + + string commitMessage = + $"Update benchmark for {commitInfo.Sha.Substring(0, 8)}: {commitInfo.Message} by {commitInfo.Author}"; + await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, updated, + dataFile.Sha, GithubToken); + await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage, + limited, limitedFile.Sha, GithubToken); + }); + + private static List LoadBenchmarkJsonReports(AbsolutePath resultsDirectory) + { + List reports = new(); + if (!Directory.Exists(resultsDirectory)) + { + return reports; + } + + foreach (string file in Directory.GetFiles(resultsDirectory, "*-report-full-compressed.json")) + { + reports.Add(File.ReadAllText(file)); + } + + return reports; + } + + private static PageBenchmarkReportGenerator.CommitInfo ReadCurrentCommitInfo() + { + Output[] lines = GitTasks.Git("log -1").ToArray(); + string commitId = null, author = null, date = null, message = null; + foreach (string line in lines.Select(x => x.Text)) + { + if (commitId == null && line.StartsWith("commit ")) + { + commitId = line.Substring("commit ".Length).Substring(0, 40); + continue; + } + + if (author == null && line.StartsWith("Author: ")) + { + author = line.Substring("Author: ".Length); + int index = author.IndexOf(" <", StringComparison.Ordinal); + if (index > 0) + { + author = author.Substring(0, index); + } + + continue; + } + + if (date == null && line.StartsWith("Date: ")) + { + date = line.Substring("Date: ".Length); + continue; + } + + if (commitId != null && author != null && date != null && !string.IsNullOrWhiteSpace(line)) + { + message = line.Trim(); + break; + } + } + + return new PageBenchmarkReportGenerator.CommitInfo(commitId, author, date, message); + } + async Task DownloadBaselineBenchmarks(AbsolutePath baselineDirectory) { long[] candidateRunIds = await BuildExtensions.FindRecentSuccessfulRunIds("build.yml", "main", 10, GithubToken); diff --git a/Pipeline/BuildExtensions.cs b/Pipeline/BuildExtensions.cs index cfc0e060..7e6f762e 100644 --- a/Pipeline/BuildExtensions.cs +++ b/Pipeline/BuildExtensions.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Text; using System.Text.Json; using System.Threading.Tasks; using Nuke.Common.CI.GitHubActions; @@ -163,4 +164,78 @@ private static async Task DownloadArtifactsFromRun(string runId, Func + /// Reads a file from the given using the GitHub contents API. + /// Returns if the file does not exist on that branch. + /// + public static async Task ReadBranchFileAsync(string path, string branch, string githubToken) + { + using HttpClient client = CreateGithubClient(githubToken); + HttpResponseMessage response = await client.GetAsync( + $"{RepositoryApiBaseUrl}/contents/{path}?ref={Uri.EscapeDataString(branch)}"); + if (!response.IsSuccessStatusCode) + { + return null; + } + + string responseContent = await response.Content.ReadAsStringAsync(); + using JsonDocument document = JsonDocument.Parse(responseContent); + string sha = document.RootElement.GetProperty("sha").GetString(); + string downloadUrl = + $"https://raw.githubusercontent.com/{Owner}/{Repo}/refs/heads/{branch}/{path}"; + using HttpClient rawClient = new(); + rawClient.DefaultRequestHeaders.UserAgent.ParseAdd(Repo); + string content = await rawClient.GetStringAsync(downloadUrl); + return new GithubFile(content, sha); + } + + /// + /// Writes a file to the given using the GitHub contents API. + /// + public static async Task WriteBranchFileAsync(string path, string branch, string commitMessage, string content, + string existingSha, string githubToken) + { + using HttpClient client = CreateGithubClient(githubToken); + GithubUpdateFile body = new(commitMessage, Base64Encode(content), existingSha, branch); + HttpResponseMessage response = await client.PutAsync( + $"{RepositoryApiBaseUrl}/contents/{path}", + new StringContent(JsonSerializer.Serialize(body), Encoding.UTF8, "application/json")); + if (response.IsSuccessStatusCode) + { + Log.Information("Updated {Path} on branch '{Branch}'.", path, branch); + } + else + { + string responseContent = await response.Content.ReadAsStringAsync(); + throw new InvalidOperationException( + $"Could not update '{path}' on branch '{branch}': {responseContent}"); + } + } + + private static HttpClient CreateGithubClient(string githubToken) + { + HttpClient client = new(); + client.DefaultRequestHeaders.UserAgent.ParseAdd(Repo); + if (!string.IsNullOrEmpty(githubToken)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", githubToken); + } + + return client; + } + + private static string Base64Encode(string plainText) + { + byte[] plainTextBytes = Encoding.UTF8.GetBytes(plainText); + return Convert.ToBase64String(plainTextBytes); + } + + public record GithubFile(string Content, string Sha); + + // ReSharper disable InconsistentNaming + // ReSharper disable NotAccessedPositionalProperty.Local + private record GithubUpdateFile(string message, string content, string sha, string branch); + // ReSharper restore NotAccessedPositionalProperty.Local + // ReSharper restore InconsistentNaming } diff --git a/Pipeline/PageBenchmarkReportGenerator.cs b/Pipeline/PageBenchmarkReportGenerator.cs new file mode 100644 index 00000000..9b090db6 --- /dev/null +++ b/Pipeline/PageBenchmarkReportGenerator.cs @@ -0,0 +1,306 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using Serilog; + +// ReSharper disable CollectionNeverQueried.Local +// ReSharper disable UnusedMember.Local +// ReSharper disable UnusedAutoPropertyAccessor.Local +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Local + +namespace Build; + +/// +/// Aggregates BenchmarkDotNet JSON reports into the data.js / limited-data.js files consumed +/// by the documentation site (). +/// +/// +/// The structure intentionally mirrors aweXpect's PageBenchmarkReportGenerator so the same docs-site +/// renderer can pick up Mockolate's data without per-project tweaks. The differences are: +/// +/// The set of compared libraries (Mockolate plus Moq/NSubstitute/FakeItEasy/TUnitMocks/Imposter). +/// Chart keys include BenchmarkDotNet parameter sets (e.g. Method (N=1)) so that each +/// [Params] combination becomes its own chart. +/// +/// +public class PageBenchmarkReportGenerator +{ + private const string FilePrefix = "window.BENCHMARK_DATA = "; + + internal const string BaselineLibrary = "Mockolate"; + + private static readonly JsonSerializerOptions BenchmarkSerializerOptions = new() + { + WriteIndented = true, + }; + + public static (string FullContent, string LimitedContent) Append(CommitInfo commitInfo, + string currentFileContent, List benchmarkReportsContents, int limit) + { + if (!currentFileContent.StartsWith(FilePrefix)) + { + throw new NotSupportedException($"The benchmark data file is incorrect (does not start with {FilePrefix})"); + } + + PageReportData pageReport = + JsonSerializer.Deserialize(currentFileContent.Substring(FilePrefix.Length)); + + if (pageReport.Values.Any(r => r.Commits.Any(c => c.Sha == commitInfo.Sha))) + { + Log.Warning( + "The benchmark already has data for {Sha}: {Message} by {Author} on {Date}", + commitInfo.Sha, commitInfo.Message, commitInfo.Author, commitInfo.Date); + return (null, null); + } + + Log.Debug( + "Updating benchmark report for {Sha}: {Message} by {Author} on {Date}", + commitInfo.Sha, commitInfo.Message, commitInfo.Author, commitInfo.Date); + + foreach (string benchmarkReportContent in benchmarkReportsContents) + { + BenchmarkReport benchmarkReport = JsonSerializer.Deserialize(benchmarkReportContent); + if (!pageReport.Append(commitInfo, benchmarkReport)) + { + throw new NotSupportedException("The new benchmark data is incorrect"); + } + } + + string newFileContent = + $"{FilePrefix}{JsonSerializer.Serialize(pageReport, BenchmarkSerializerOptions)}"; + string limitedFileContent = + $"{FilePrefix}{JsonSerializer.Serialize(pageReport.Limit(limit), BenchmarkSerializerOptions)}"; + return (newFileContent, limitedFileContent); + } + + internal sealed class PageReportData : Dictionary + { + public bool Append(CommitInfo commitInfo, BenchmarkReport benchmarkReport) + { + HashSet chartsTouchedByBaseline = new(); + foreach (BenchmarkReport.Benchmark benchmark in benchmarkReport.Benchmarks) + { + if (!Append(commitInfo, benchmark, chartsTouchedByBaseline)) + { + return false; + } + } + + return true; + } + + private bool Append(CommitInfo commitInfo, BenchmarkReport.Benchmark benchmark, + HashSet chartsTouchedByBaseline) + { + if (!ParseMethod(benchmark.Method, out string scenario, out string library)) + { + return false; + } + + if (!IsIncluded(library)) + { + return true; + } + + string chartKey = BuildChartKey(scenario, benchmark.Parameters); + if (!TryGetValue(chartKey, out PageReport pageReport)) + { + pageReport = new PageReport(); + this[chartKey] = pageReport; + } + + if (library == BaselineLibrary && chartsTouchedByBaseline.Add(chartKey)) + { + pageReport.Commits.Add(commitInfo); + pageReport.Labels.Add(commitInfo.Sha.Substring(0, 8)); + } + + AppendTimeDataset(benchmark, pageReport, library); + AppendMemoryDataset(benchmark, pageReport, library); + + return true; + } + + private static string BuildChartKey(string scenario, string parameters) + => string.IsNullOrWhiteSpace(parameters) ? scenario : $"{scenario} ({parameters})"; + + private static void AppendMemoryDataset(BenchmarkReport.Benchmark benchmark, PageReport pageReport, + string library) + { + PageReport.Dataset memoryDataset = pageReport.Datasets.FirstOrDefault(x + => x.Label.StartsWith(library, StringComparison.OrdinalIgnoreCase) && x.YAxisId == "y1"); + if (memoryDataset == null) + { + memoryDataset = new PageReport.Dataset + { + Label = $"{library} memory", + Unit = "b", + PointStyle = "triangle", + BorderDash = [5, 5,], + YAxisId = "y1", + BackgroundColor = GetColor(library), + BorderColor = GetColor(library), + Data = new List(), + }; + pageReport.Datasets.Add(memoryDataset); + } + + memoryDataset.Data.Add(benchmark.Metrics + .Where(x => x.Descriptor.Id == "Allocated Memory") + .Select(x => x.Value) + .FirstOrDefault(double.NaN)); + } + + private static void AppendTimeDataset(BenchmarkReport.Benchmark benchmark, PageReport pageReport, string library) + { + PageReport.Dataset timeDataset = pageReport.Datasets.FirstOrDefault(x + => x.Label.StartsWith(library, StringComparison.OrdinalIgnoreCase) && x.YAxisId == "y"); + if (timeDataset == null) + { + timeDataset = new PageReport.Dataset + { + Label = $"{library} time", + Unit = "ns", + PointStyle = "circle", + YAxisId = "y", + BackgroundColor = GetColor(library), + BorderColor = GetColor(library), + Data = new List(), + }; + pageReport.Datasets.Add(timeDataset); + } + + timeDataset.Data.Add(benchmark.Statistics.Mean); + } + + private static bool IsIncluded(string library) + => library is "Mockolate" or "Moq" or "NSubstitute" or "FakeItEasy" or "TUnitMocks" or "Imposter"; + + private static string GetColor(string library) + => library switch + { + "Mockolate" => "#63A2AC", + "Moq" => "#A052B0", + "NSubstitute" => "#5E2750", + "FakeItEasy" => "#4A6FA5", + "TUnitMocks" => "#FF8C00", + "Imposter" => "#E84393", + _ => "#888888", + }; + + private static bool ParseMethod(string method, out string scenario, out string library) + { + int index = method.LastIndexOf('_'); + if (index <= 0) + { + scenario = null; + library = null; + return false; + } + + scenario = method.Substring(0, index); + library = method.Substring(index + 1); + return true; + } + + public PageReportData Limit(int limit) + { + PageReportData pageReportData = new(); + foreach ((string key, PageReport pageReport) in this) + { + pageReportData[key] = pageReport.Limit(limit); + } + + return pageReportData; + } + } + + public class CommitInfo(string sha, string author, string date, string message) + { + [JsonPropertyName("sha")] public string Sha { get; } = sha; + [JsonPropertyName("author")] public string Author { get; } = author; + [JsonPropertyName("date")] public string Date { get; } = date; + [JsonPropertyName("message")] public string Message { get; } = message; + } + + internal sealed class PageReport + { + [JsonPropertyName("commits")] public List Commits { get; init; } = new(); + [JsonPropertyName("labels")] public List Labels { get; init; } = new(); + + [JsonPropertyName("datasets")] public List Datasets { get; init; } = new(); + + public PageReport Limit(int limit) + => new() + { + Commits = Commits.TakeLast(limit).ToList(), + Labels = Labels.TakeLast(limit).ToList(), + Datasets = Datasets.Select(dataset => dataset.Limit(limit)).ToList(), + }; + + public class Dataset + { + [JsonPropertyName("label")] public string Label { get; init; } + [JsonPropertyName("unit")] public string Unit { get; set; } + + [JsonPropertyName("data")] public List Data { get; init; } + + [JsonPropertyName("borderColor")] public string BorderColor { get; set; } + + [JsonPropertyName("backgroundColor")] public string BackgroundColor { get; set; } + + [JsonPropertyName("yAxisID")] public string YAxisId { get; init; } + + [JsonPropertyName("borderDash")] public int[] BorderDash { get; set; } = []; + + [JsonPropertyName("pointStyle")] public string PointStyle { get; set; } + + public Dataset Limit(int limit) + => new() + { + Label = Label, + Unit = Unit, + Data = Data.TakeLast(limit).ToList(), + BorderColor = BorderColor, + BackgroundColor = BackgroundColor, + YAxisId = YAxisId, + BorderDash = BorderDash, + PointStyle = PointStyle, + }; + } + } + + internal sealed class BenchmarkReport + { + public Benchmark[] Benchmarks { get; init; } + + public class Benchmark + { + public string Type { get; init; } + public string Method { get; init; } + public string Parameters { get; init; } + public BenchmarkStatistics Statistics { get; init; } + public BenchmarkMetrics[] Metrics { get; init; } + } + + public class BenchmarkStatistics + { + public double Mean { get; init; } + } + + public class BenchmarkMetrics + { + public double Value { get; init; } + public BenchmarkMetricDescriptor Descriptor { get; init; } + } + + public class BenchmarkMetricDescriptor + { + public string Id { get; init; } + public string DisplayName { get; init; } + public string Unit { get; init; } + } + } +}