From 6633417275134382dbee1d7f6cedf3216fefb74d Mon Sep 17 00:00:00 2001 From: Donnie Goodson <49205731+donnie-msft@users.noreply.github.com> Date: Fri, 28 Jun 2024 12:41:49 -0700 Subject: [PATCH 1/2] Point CI Reliability report tool to NuGet.Client CI pipeline --- .../CiReliability/CiReliabilityReport.cs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs b/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs index a00fc66..d936b76 100644 --- a/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs +++ b/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs @@ -54,6 +54,12 @@ private FileStream OpenOutputFile(string outFile) private async Task GetDataAsync(string sprintName) { + string organizationName = "dnceng-public"; + string projectId = "cbb18261-c48f-4abb-8651-8cdcb5474649"; + string definitionId = "289"; + string sourceBranch = "refs/heads/dev"; + string reason = "individualCI"; + TextWriter? log; if (Console.IsOutputRedirected) { @@ -71,11 +77,11 @@ private async Task GetDataAsync(string sprintName) string failedBuildsQuery = $@"let start = startofday(datetime(""{startOfSprint.ToString("yyyy-MM-dd")}"")); let end = endofday(datetime(""{endOfSprint.ToString("yyyy-MM-dd")}"")); let nugetBuilds = Build -| where OrganizationName == 'devdiv' and ProjectId == '0bdbc590-a062-4c3f-b0f6-9383f67865ee' and DefinitionId == 8118 and FinishTime between (start..end) and SourceBranch == 'refs/heads/dev' and Reason == 'schedule'; +| where OrganizationName == '{organizationName}' and ProjectId == '{projectId}' and DefinitionId == {definitionId} and FinishTime between (start..end) and SourceBranch == '{sourceBranch}' and Reason == '{reason}'; let sprintBuilds = nugetBuilds | project BuildId; let previousAttempts = BuildTimelineRecord -| where OrganizationName == 'devdiv' and ProjectId == '0bdbc590-a062-4c3f-b0f6-9383f67865ee' and BuildId in (sprintBuilds) +| where OrganizationName == '{organizationName}' and ProjectId == '{projectId}' and BuildId in (sprintBuilds) | summarize PreviousAttempts=countif(PreviousAttempts !in ('', '[]')) by BuildId | where PreviousAttempts > 0 | project BuildId; @@ -86,7 +92,7 @@ private async Task GetDataAsync(string sprintName) string buildCountQuery = $@"let start = startofday(datetime(""{startOfSprint.ToString("yyyy-MM-dd")}"")); let end = endofday(datetime(""{endOfSprint.ToString("yyyy-MM-dd")}"")); Build -| where OrganizationName == 'devdiv' and ProjectId == '0bdbc590-a062-4c3f-b0f6-9383f67865ee' and DefinitionId == 8118 and FinishTime between (start..end) and SourceBranch == 'refs/heads/dev' +| where OrganizationName == '{organizationName}' and ProjectId == '{projectId}' and DefinitionId == {definitionId} and FinishTime between (start..end) and SourceBranch == '{sourceBranch}' | summarize count()"; var connectionBuilder = new KustoConnectionStringBuilder("https://1es.kusto.windows.net/", "AzureDevOps") @@ -102,6 +108,7 @@ private async Task GetDataAsync(string sprintName) using (var client = KustoClientFactory.CreateCslQueryProvider(connectionBuilder)) { + log?.WriteLine("Query arguments: organizationName=" + organizationName + " | " + "projectId=" + projectId + " | " + "definitionId =" + definitionId + " | " + "sourceBranch=" + sourceBranch + " | " + "reason=" + reason); log?.WriteLine($"Querying builds from {startOfSprint:yyyy-MM-dd} to {endOfSprint:yyyy-MM-dd}"); var (failedBuilds, trackingIssues) = await GetFailedBuilds(client, crp, failedBuildsQuery, log); @@ -198,7 +205,7 @@ private async Task GetBuildCount(ICslQueryProvider client, ClientRequestPro List> rows = new(); var query = @"BuildTimelineRecord -| where OrganizationName == 'devdiv' and ProjectId == '0bdbc590-a062-4c3f-b0f6-9383f67865ee' and BuildId == " + buildId; +| where OrganizationName == 'dnceng-public' and ProjectId == 'cbb18261-c48f-4abb-8651-8cdcb5474649' and BuildId == " + buildId; using (var result = await client.ExecuteQueryAsync("AzureDevOps", query, crp)) { while (result.Read()) @@ -306,7 +313,7 @@ private void Output(ReportData data, FileStream outputFileStream) using var sw = new StreamWriter(outputFileStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); sw.WriteLine("# NuGet.Client CI Reliability " + data.SprintName); sw.WriteLine(); - sw.WriteLine("[NuGet.Client-PR dev branch builds](https://dev.azure.com/devdiv/DevDiv/_build?definitionId=8118&branchFilter=101196%2C101196%2C101196%2C101196%2C101196)"); + sw.WriteLine("[NuGet.Client CI dev branch builds](https://dev.azure.com/dnceng-public/public/_build?definitionId=289&branchFilter=86197%2C86197%2C86197)"); sw.WriteLine(); sw.WriteLine("|Total Builds|Failed Builds|Reliability|Reliability Ignoring Apex|"); sw.WriteLine("|:--:|:--:|:--:|:--:|"); @@ -339,11 +346,11 @@ private void Output(ReportData data, FileStream outputFileStream) { if (build.Details.Count > 1) { - sw.WriteLine($" {build.Number}"); + sw.WriteLine($" {build.Number}"); } else { - sw.WriteLine($" {build.Number}"); + sw.WriteLine($" {build.Number}"); } } sw.WriteLine($" {build.Details[i].Job}"); From 710cacdebb57784368bd59c587b70d29bb38f0db Mon Sep 17 00:00:00 2001 From: Donnie Goodson <49205731+donnie-msft@users.noreply.github.com> Date: Wed, 3 Jul 2024 13:41:20 -0700 Subject: [PATCH 2/2] Support multiple pipelines in Reliability Report --- .../CiReliability/CiReliabilityReport.cs | 144 ++++++++++++------ .../Reports/CiReliability/PipelineData.cs | 15 ++ .../Reports/CiReliability/ReportData.cs | 2 + 3 files changed, 113 insertions(+), 48 deletions(-) create mode 100644 GithubIssueTagger/Reports/CiReliability/PipelineData.cs diff --git a/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs b/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs index d936b76..451bd0c 100644 --- a/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs +++ b/GithubIssueTagger/Reports/CiReliability/CiReliabilityReport.cs @@ -26,9 +26,38 @@ public async Task RunAsync(string sprintName, string outFile) { using FileStream fileStream = OpenOutputFile(outFile); - ReportData data = await GetDataAsync(sprintName); + PipelineData buildPipelineData = new PipelineData() + { + PipelineName = "NuGet.Client-PrivateDev", + BranchFilterQueryString = "101196%2C101196%2C101196%2C101196%2C101196", + DatabaseName = "AzureDevOps", + OrganizationName = "devdiv", + ProjectId = "0bdbc590-a062-4c3f-b0f6-9383f67865ee", + ProjectName = "DevDiv", + DefinitionId = "8118", + SourceBranch = "refs/heads/dev", + Reason = "schedule", + }; + + ReportData buildReportData = await GetDataAsync(sprintName, queryName: "dev branch builds", buildPipelineData); - Output(data, fileStream); + PipelineData funcUnitTestPipelineData = new PipelineData() + { + PipelineName = "NuGet.Client CI", + DatabaseName = "AzureDevOps", + BranchFilterQueryString = "86197%2C86197%2C86197", + OrganizationName = "dnceng-public", + ProjectId = "cbb18261-c48f-4abb-8651-8cdcb5474649", + ProjectName = "public", + DefinitionId = "289", + SourceBranch = "refs/heads/dev", + Reason = "individualCI", + }; + + ReportData funcUnitTestReportData = await GetDataAsync(sprintName, queryName: "dev branch Functional & Unit tests", funcUnitTestPipelineData); + + var data = new Tuple[] { new(buildPipelineData, buildReportData), new(funcUnitTestPipelineData, funcUnitTestReportData) }; + Output(data, fileStream, reportSprintName: sprintName); } private FileStream OpenOutputFile(string outFile) @@ -52,14 +81,8 @@ private FileStream OpenOutputFile(string outFile) return fileStream; } - private async Task GetDataAsync(string sprintName) + private async Task GetDataAsync(string sprintName, string queryName, PipelineData pipelineData) { - string organizationName = "dnceng-public"; - string projectId = "cbb18261-c48f-4abb-8651-8cdcb5474649"; - string definitionId = "289"; - string sourceBranch = "refs/heads/dev"; - string reason = "individualCI"; - TextWriter? log; if (Console.IsOutputRedirected) { @@ -77,11 +100,11 @@ private async Task GetDataAsync(string sprintName) string failedBuildsQuery = $@"let start = startofday(datetime(""{startOfSprint.ToString("yyyy-MM-dd")}"")); let end = endofday(datetime(""{endOfSprint.ToString("yyyy-MM-dd")}"")); let nugetBuilds = Build -| where OrganizationName == '{organizationName}' and ProjectId == '{projectId}' and DefinitionId == {definitionId} and FinishTime between (start..end) and SourceBranch == '{sourceBranch}' and Reason == '{reason}'; +| where OrganizationName == '{pipelineData.OrganizationName}' and ProjectId == '{pipelineData.ProjectId}' and DefinitionId == {pipelineData.DefinitionId} and FinishTime between (start..end) and SourceBranch == '{pipelineData.SourceBranch}' and Reason == '{pipelineData.Reason}'; let sprintBuilds = nugetBuilds | project BuildId; let previousAttempts = BuildTimelineRecord -| where OrganizationName == '{organizationName}' and ProjectId == '{projectId}' and BuildId in (sprintBuilds) +| where OrganizationName == '{pipelineData.OrganizationName}' and ProjectId == '{pipelineData.ProjectId}' and BuildId in (sprintBuilds) | summarize PreviousAttempts=countif(PreviousAttempts !in ('', '[]')) by BuildId | where PreviousAttempts > 0 | project BuildId; @@ -92,10 +115,10 @@ private async Task GetDataAsync(string sprintName) string buildCountQuery = $@"let start = startofday(datetime(""{startOfSprint.ToString("yyyy-MM-dd")}"")); let end = endofday(datetime(""{endOfSprint.ToString("yyyy-MM-dd")}"")); Build -| where OrganizationName == '{organizationName}' and ProjectId == '{projectId}' and DefinitionId == {definitionId} and FinishTime between (start..end) and SourceBranch == '{sourceBranch}' +| where OrganizationName == '{pipelineData.OrganizationName}' and ProjectId == '{pipelineData.ProjectId}' and DefinitionId == {pipelineData.DefinitionId} and FinishTime between (start..end) and SourceBranch == '{pipelineData.SourceBranch}' | summarize count()"; - var connectionBuilder = new KustoConnectionStringBuilder("https://1es.kusto.windows.net/", "AzureDevOps") + var connectionBuilder = new KustoConnectionStringBuilder("https://1es.kusto.windows.net/", pipelineData.DatabaseName) { FederatedSecurity = true }; @@ -108,16 +131,18 @@ private async Task GetDataAsync(string sprintName) using (var client = KustoClientFactory.CreateCslQueryProvider(connectionBuilder)) { - log?.WriteLine("Query arguments: organizationName=" + organizationName + " | " + "projectId=" + projectId + " | " + "definitionId =" + definitionId + " | " + "sourceBranch=" + sourceBranch + " | " + "reason=" + reason); + log?.WriteLine("Query arguments: pipelineName =" + pipelineData.PipelineName + " | " + "organizationName = " + pipelineData.OrganizationName + " | " + "projectId=" + pipelineData.ProjectId + " | " + "definitionId =" + + pipelineData.DefinitionId + " | " + "sourceBranch=" + pipelineData.SourceBranch + " | " + "reason=" + pipelineData.Reason); log?.WriteLine($"Querying builds from {startOfSprint:yyyy-MM-dd} to {endOfSprint:yyyy-MM-dd}"); - var (failedBuilds, trackingIssues) = await GetFailedBuilds(client, crp, failedBuildsQuery, log); + var (failedBuilds, trackingIssues) = await GetFailedBuilds(client, crp, failedBuildsQuery, pipelineData, log); log?.WriteLine("Querying total builds in sprint"); - int totalBuilds = await GetBuildCount(client, crp, buildCountQuery); + int totalBuilds = await GetBuildCount(client, crp, buildCountQuery, pipelineData); data = new ReportData() { SprintName = sprintName, + QueryName = queryName, KustoQuery = failedBuildsQuery, FailedBuilds = failedBuilds, TrackingIssues = trackingIssues, @@ -128,9 +153,9 @@ private async Task GetDataAsync(string sprintName) return data; } - private async Task GetBuildCount(ICslQueryProvider client, ClientRequestProperties crp, string query) + private async Task GetBuildCount(ICslQueryProvider client, ClientRequestProperties crp, string query, PipelineData pipelineData) { - using var result = await client.ExecuteQueryAsync("AzureDevOps", query, crp); + using var result = await client.ExecuteQueryAsync(pipelineData.DatabaseName, query, crp); if (!result.Read()) { @@ -148,12 +173,13 @@ private async Task GetBuildCount(ICslQueryProvider client, ClientRequestPro ICslQueryProvider client, ClientRequestProperties crp, string query, + PipelineData pipelineData, TextWriter? log) { List failedBuilds = new(); Dictionary trackingIssues = new(); - var result = await client.ExecuteQueryAsync("AzureDevOps", query, crp); + var result = await client.ExecuteQueryAsync(pipelineData.DatabaseName, query, crp); int buildIdColumn = result.GetOrdinal("BuildId"); int buildNumberColumn = result.GetOrdinal("BuildNumber"); @@ -175,7 +201,7 @@ private async Task GetBuildCount(ICslQueryProvider client, ClientRequestPro { log?.WriteLine($"Checking failed build {i + 1}/{failedBuilds.Count}"); - var (details, tracking) = await GetFailedBuildDetails(failedBuilds[i].Id, client, crp); + var (details, tracking) = await GetFailedBuildDetails(failedBuilds[i].Id, client, crp, pipelineData); foreach (var kvp in tracking) { @@ -197,16 +223,17 @@ private async Task GetBuildCount(ICslQueryProvider client, ClientRequestPro private async Task<(IReadOnlyList details, IReadOnlyDictionary tracking)> GetFailedBuildDetails( long buildId, ICslQueryProvider client, - ClientRequestProperties crp) + ClientRequestProperties crp, + PipelineData pipelineData) { List details = new(); Dictionary trackingIssues = new(); List> rows = new(); - var query = @"BuildTimelineRecord -| where OrganizationName == 'dnceng-public' and ProjectId == 'cbb18261-c48f-4abb-8651-8cdcb5474649' and BuildId == " + buildId; - using (var result = await client.ExecuteQueryAsync("AzureDevOps", query, crp)) + var query = $@"BuildTimelineRecord +| where OrganizationName == '{pipelineData.OrganizationName}' and ProjectId == '{pipelineData.ProjectId}' and BuildId == {buildId}"; + using (var result = await client.ExecuteQueryAsync(pipelineData.DatabaseName, query, crp)) { while (result.Read()) { @@ -300,26 +327,57 @@ bool IsFailedTask(Dictionary timelineEntry, string parentId) } } - private void Output(ReportData data, FileStream outputFileStream) + private void Output(Tuple[] pipelineReportDataList, FileStream outputFileStream, string reportSprintName) { - if (data == null) { throw new ArgumentNullException(nameof(data)); } if (outputFileStream is null || !outputFileStream.CanRead || !outputFileStream.CanWrite) { throw new ArgumentException(paramName: nameof(outputFileStream), message: "Cannot read and write to output file"); } - if (data.FailedBuilds == null) { throw new ArgumentException(paramName: nameof(data.FailedBuilds), message: "data.FailedBuilds must not be null"); } + using var sw = new StreamWriter(outputFileStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); - float reliability = (data.TotalBuilds - data.FailedBuilds.Count) * 100.0f / data.TotalBuilds; - int failedBuildsOnlyBecauseOfApex = data.FailedBuilds.Where(b => b.Details?.Count == 1 && b.Details[0].Job == "Apex Test Execution").Count(); - float reliabilityIgnoringApex = (data.TotalBuilds - data.FailedBuilds.Count + failedBuildsOnlyBecauseOfApex) * 100.0f / data.TotalBuilds; + sw.WriteLine($"# NuGet Client CI Reliability " + reportSprintName); - using var sw = new StreamWriter(outputFileStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); - sw.WriteLine("# NuGet.Client CI Reliability " + data.SprintName); + foreach (var pipelineReportData in pipelineReportDataList) + { + var pipelineData = pipelineReportData.Item1; + var data = pipelineReportData.Item2; + if (data == null) { throw new ArgumentNullException(nameof(data)); } + if (data.FailedBuilds == null) { throw new ArgumentException(paramName: nameof(data.FailedBuilds), message: "data.FailedBuilds must not be null"); } + + float reliability = (data.TotalBuilds - data.FailedBuilds.Count) * 100.0f / data.TotalBuilds; + int failedBuildsOnlyBecauseOfApex = data.FailedBuilds.Where(b => b.Details?.Count == 1 && b.Details[0].Job == "Apex Test Execution").Count(); + float reliabilityIgnoringApex = (data.TotalBuilds - data.FailedBuilds.Count + failedBuildsOnlyBecauseOfApex) * 100.0f / data.TotalBuilds; + + OutputPipelineData(pipelineData, data, reliability, reliabilityIgnoringApex, sw); + + sw.WriteLine(); + sw.WriteLine(); + sw.WriteLine("### Tracking"); + if (data.TrackingIssues.Count == 0) + { + sw.WriteLine("No tracking issues"); + } + else + { + foreach (var kvp in data.TrackingIssues) + { + sw.WriteLine(); + sw.WriteLine($"- {kvp.Key}"); + sw.WriteLine(); + sw.WriteLine(kvp.Value); + } + } + } + } + + private static void OutputPipelineData(PipelineData pipelineData, ReportData data, float reliability, float reliabilityIgnoringApex, StreamWriter sw) + { + sw.WriteLine($"## {pipelineData.PipelineName}"); sw.WriteLine(); - sw.WriteLine("[NuGet.Client CI dev branch builds](https://dev.azure.com/dnceng-public/public/_build?definitionId=289&branchFilter=86197%2C86197%2C86197)"); + sw.WriteLine($"[{pipelineData.PipelineName} {data.QueryName}](https://dev.azure.com/{pipelineData.OrganizationName}/{pipelineData.ProjectName}/_build?definitionId={pipelineData.DefinitionId}&branchFilter={pipelineData.BranchFilterQueryString})"); sw.WriteLine(); sw.WriteLine("|Total Builds|Failed Builds|Reliability|Reliability Ignoring Apex|"); sw.WriteLine("|:--:|:--:|:--:|:--:|"); sw.WriteLine($"|{data.TotalBuilds}|{data.FailedBuilds.Count}|{reliability:f1}%|{reliabilityIgnoringApex:f1}%|"); sw.WriteLine(); - sw.WriteLine("## Failed Builds"); + sw.WriteLine("### Failed Builds"); sw.WriteLine(); sw.WriteLine("**Note:**: Includes builds that succeeded on retry, so first attempt failed"); sw.WriteLine(); @@ -346,11 +404,11 @@ private void Output(ReportData data, FileStream outputFileStream) { if (build.Details.Count > 1) { - sw.WriteLine($" {build.Number}"); + sw.WriteLine($" {build.Number}"); } else { - sw.WriteLine($" {build.Number}"); + sw.WriteLine($" {build.Number}"); } } sw.WriteLine($" {build.Details[i].Job}"); @@ -360,16 +418,6 @@ private void Output(ReportData data, FileStream outputFileStream) } } sw.WriteLine(""); - sw.WriteLine(); - sw.WriteLine("### Tracking"); - - foreach (var kvp in data.TrackingIssues) - { - sw.WriteLine(); - sw.WriteLine($"- {kvp.Key}"); - sw.WriteLine(); - sw.WriteLine(kvp.Value); - } } private class CiReliabilityCommandFactory : ICommandFactory @@ -396,7 +444,7 @@ public Command CreateCommand(Type type, GitHubPatBinder patBinder) return command; } - public async Task RunAsync(string sprint, string outfile) + public async Task RunAsync(string sprint, string outFile) { var serviceProvider = new ServiceCollection() .AddGithubIssueTagger() @@ -406,7 +454,7 @@ public async Task RunAsync(string sprint, string outfile) using (scopeFactory.CreateScope()) { var report = serviceProvider.GetRequiredService(); - await report.RunAsync(sprint, outfile); + await report.RunAsync(sprint, outFile); } } } diff --git a/GithubIssueTagger/Reports/CiReliability/PipelineData.cs b/GithubIssueTagger/Reports/CiReliability/PipelineData.cs new file mode 100644 index 0000000..4addba2 --- /dev/null +++ b/GithubIssueTagger/Reports/CiReliability/PipelineData.cs @@ -0,0 +1,15 @@ +namespace GithubIssueTagger.Reports.CiReliability +{ + internal struct PipelineData + { + public string? PipelineName { get; init; } + public string? BranchFilterQueryString { get; init; } + public string? DatabaseName { get; init; } + public string? OrganizationName { get; init; } + public string? ProjectId { get; init; } + public string? ProjectName { get; init; } + public string? DefinitionId { get; init; } + public string? SourceBranch { get; init; } + public string? Reason { get; init; } + } +} diff --git a/GithubIssueTagger/Reports/CiReliability/ReportData.cs b/GithubIssueTagger/Reports/CiReliability/ReportData.cs index 5b7d22d..f9644a0 100644 --- a/GithubIssueTagger/Reports/CiReliability/ReportData.cs +++ b/GithubIssueTagger/Reports/CiReliability/ReportData.cs @@ -6,6 +6,8 @@ internal class ReportData { public string? SprintName { get; init; } + public string? QueryName { get; init; } + public string? KustoQuery { get; init; } public required IReadOnlyList FailedBuilds { get; init; }