diff --git a/TestFx.slnx b/TestFx.slnx index f009643b2b..01bd46c018 100644 --- a/TestFx.slnx +++ b/TestFx.slnx @@ -41,6 +41,7 @@ + diff --git a/docs/images/html-report/duplicate-uid.png b/docs/images/html-report/duplicate-uid.png new file mode 100644 index 0000000000..2012283b44 Binary files /dev/null and b/docs/images/html-report/duplicate-uid.png differ diff --git a/docs/images/html-report/group-by-class.png b/docs/images/html-report/group-by-class.png new file mode 100644 index 0000000000..fa14c9f2e4 Binary files /dev/null and b/docs/images/html-report/group-by-class.png differ diff --git a/docs/images/html-report/group-by-duration.png b/docs/images/html-report/group-by-duration.png new file mode 100644 index 0000000000..01e796fb00 Binary files /dev/null and b/docs/images/html-report/group-by-duration.png differ diff --git a/docs/images/html-report/group-by-trait-dark.png b/docs/images/html-report/group-by-trait-dark.png new file mode 100644 index 0000000000..3964e7e9d2 Binary files /dev/null and b/docs/images/html-report/group-by-trait-dark.png differ diff --git a/docs/images/html-report/overview.png b/docs/images/html-report/overview.png new file mode 100644 index 0000000000..e6bbc39c9e Binary files /dev/null and b/docs/images/html-report/overview.png differ diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/BannedSymbols.txt b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/BannedSymbols.txt new file mode 100644 index 0000000000..64ef236c50 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/BannedSymbols.txt @@ -0,0 +1,9 @@ +P:System.DateTime.Now; Use 'IClock' instead +P:System.DateTime.UtcNow; Use 'IClock' instead +M:System.Threading.Tasks.Task.Run(System.Action); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Threading.Tasks.Task[]); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Collections.Generic.IEnumerable{System.Threading.Tasks.Task}); Use 'ITask' instead +M:System.String.IsNullOrEmpty(System.String); Use 'RoslynString.IsNullOrEmpty' instead +M:System.String.IsNullOrWhiteSpace(System.String); Use 'RoslynString.IsNullOrWhiteSpace' instead +M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead +M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/CapturedTestResult.cs b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/CapturedTestResult.cs new file mode 100644 index 0000000000..49df78bdd0 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/CapturedTestResult.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.HtmlReport; + +// Minimal capped-size projection of a TestNodeUpdateMessage. The consumer projects +// each message into this DTO immediately so that we don't retain entire test nodes +// (and their potentially huge stdout/stderr/stack trace strings) in memory for the +// whole session. All variable-length text fields are already truncated at this point +// so the engine doesn't need to truncate again. +internal sealed class CapturedTestResult +{ + public required string Uid { get; init; } + + public required string DisplayName { get; init; } + + public required string Outcome { get; init; } + + public required TimeSpan Duration { get; init; } + + public DateTimeOffset? StartTime { get; init; } + + public DateTimeOffset? EndTime { get; init; } + + public string? ClassName { get; init; } + + public string? MethodName { get; init; } + + public string? ErrorMessage { get; init; } + + public string? ExceptionType { get; init; } + + public string? StackTrace { get; init; } + + public string? StandardOutput { get; init; } + + public string? StandardError { get; init; } + + public IReadOnlyList>? Traits { get; init; } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/HtmlReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/HtmlReportEngine.cs new file mode 100644 index 0000000000..3985538152 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/HtmlReportEngine.cs @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.HtmlReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.HtmlReport; + +internal sealed class HtmlReportEngine +{ + private const string TemplateResourceName = "Microsoft.Testing.Extensions.HtmlReport.Templates.report-template.html"; + private const string DataPlaceholder = "/*__MTP_DATA__*/null"; + private const string GeneratorVersionPlaceholder = "__MTP_GENERATOR_VERSION__"; + + private readonly IFileSystem _fileSystem; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly IEnvironment _environment; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IConfiguration _configuration; + private readonly IClock _clock; + private readonly ITestFramework _testFramework; + private readonly DateTimeOffset _testStartTime; + private readonly int _exitCode; + private readonly CancellationToken _cancellationToken; + + public HtmlReportEngine( + IFileSystem fileSystem, + ITestApplicationModuleInfo testApplicationModuleInfo, + IEnvironment environment, + ICommandLineOptions commandLineOptions, + IConfiguration configuration, + IClock clock, + ITestFramework testFramework, + DateTimeOffset testStartTime, + int exitCode, + CancellationToken cancellationToken) + { + _fileSystem = fileSystem; + _testApplicationModuleInfo = testApplicationModuleInfo; + _environment = environment; + _commandLineOptions = commandLineOptions; + _configuration = configuration; + _clock = clock; + _testFramework = testFramework; + _testStartTime = testStartTime; + _exitCode = exitCode; + _cancellationToken = cancellationToken; + } + + public Task<(string FileName, string? Warning)> GenerateReportAsync(CapturedTestResult[] results) + => GenerateReportCoreAsync(results, _clock.UtcNow); + + private async Task<(string FileName, string? Warning)> GenerateReportCoreAsync(CapturedTestResult[] results, DateTimeOffset finishTime) + { + _cancellationToken.ThrowIfCancellationRequested(); + + bool fileNameExplicitlyProvided = _commandLineOptions.TryGetOptionArgumentList( + HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName, + out string[]? providedFileName); + + string fileName = fileNameExplicitlyProvided + ? providedFileName![0] + : BuildDefaultFileName(finishTime); + + string outputDirectory = _configuration.GetTestResultDirectory(); + string finalPath = Path.Combine(outputDirectory, fileName); + + string template = LoadTemplate(); + string json = BuildJson(results, finishTime); + + string html = template + .Replace(GeneratorVersionPlaceholder, ExtensionVersion.DefaultSemVer) + .Replace(DataPlaceholder, json); + + byte[] bytes = Encoding.UTF8.GetBytes(html); + + return await WriteWithRetryAsync(finalPath, bytes, fileNameExplicitlyProvided).ConfigureAwait(false); + } + + private async Task<(string FileName, string? Warning)> WriteWithRetryAsync(string finalPath, byte[] bytes, bool fileNameExplicitlyProvided) + { + // Explicit file names: use FileMode.Create (overwrite). Default-generated file + // names: use FileMode.CreateNew but retry with disambiguating suffixes when the + // file already exists, so concurrent runs (or two runs within the same second + // sharing the result directory) don't fail with IOException. + if (fileNameExplicitlyProvided) + { + bool willOverwrite = _fileSystem.ExistFile(finalPath); + await WriteAsync(finalPath, FileMode.Create, bytes).ConfigureAwait(false); + return ( + finalPath, + willOverwrite + ? string.Format(CultureInfo.InvariantCulture, ExtensionResources.HtmlReportFileExistsAndWillBeOverwritten, finalPath) + : null); + } + + DateTimeOffset firstTry = _clock.UtcNow; + string directory = Path.GetDirectoryName(finalPath) ?? string.Empty; + string baseName = Path.GetFileNameWithoutExtension(finalPath); + string extension = Path.GetExtension(finalPath); + string candidate = finalPath; + int attempt = 0; + + while (true) + { + _cancellationToken.ThrowIfCancellationRequested(); + + try + { + await WriteAsync(candidate, FileMode.CreateNew, bytes).ConfigureAwait(false); + return (candidate, null); + } + catch (IOException) + { + if (_clock.UtcNow - firstTry > TimeSpan.FromSeconds(5)) + { + throw; + } + + attempt++; + candidate = Path.Combine(directory, $"{baseName}_{attempt}{extension}"); + } + } + } + + private async Task WriteAsync(string path, FileMode mode, byte[] bytes) + { + // Note that we need to dispose the IFileStream, not the inner stream. + // IFileStream implementations will be responsible to dispose their inner stream. + using IFileStream stream = _fileSystem.NewFileStream(path, mode); +#if NETCOREAPP + await stream.Stream.WriteAsync(bytes.AsMemory(), _cancellationToken).ConfigureAwait(false); +#else + await stream.Stream.WriteAsync(bytes, 0, bytes.Length, _cancellationToken).ConfigureAwait(false); +#endif + } + + private string BuildDefaultFileName(DateTimeOffset finishTime) + { + string user = _environment.GetEnvironmentVariable("UserName") + ?? _environment.GetEnvironmentVariable("USER") + ?? "user"; + string raw = $"{user}_{_environment.MachineName}_{finishTime:yyyy-MM-dd_HH_mm_ss}.html"; + return ReplaceInvalidFileNameChars(raw); + } + + private static string ReplaceInvalidFileNameChars(string fileName) + { + var sb = new StringBuilder(fileName.Length); + char[] invalid = Path.GetInvalidFileNameChars(); + foreach (char c in fileName) + { + sb.Append(Array.IndexOf(invalid, c) >= 0 ? '_' : c); + } + + return sb.ToString(); + } + + private static string LoadTemplate() + { + Assembly assembly = typeof(HtmlReportEngine).Assembly; + using Stream stream = assembly.GetManifestResourceStream(TemplateResourceName) + ?? throw ApplicationStateGuard.Unreachable(); + using var reader = new StreamReader(stream, Encoding.UTF8); + return reader.ReadToEnd(); + } + + private string BuildJson(CapturedTestResult[] results, DateTimeOffset finishTime) + { + int passed = 0; + int failed = 0; + int skipped = 0; + int timedout = 0; + int errored = 0; + TimeSpan totalDuration = TimeSpan.Zero; + + // First pass: count how many entries each UID is going to produce so we can + // annotate rows that share a UID with "attemptIndex"/"attemptOf". This lets the + // UI surface frameworks that emit multiple terminal results for the same UID + // (parameterized rows, in-process retries, broken UID generators, etc.) without + // silently dropping any data. + Dictionary countByUid = []; + foreach (CapturedTestResult r in results) + { + countByUid[r.Uid] = countByUid.TryGetValue(r.Uid, out int existing) ? existing + 1 : 1; + } + + Dictionary emittedByUid = []; + + StringBuilder sb = new(8 * 1024); + sb.Append('{'); + AppendStringPair(sb, "schemaVersion", "1"); + sb.Append(','); + AppendStringPair(sb, "generator", "Microsoft.Testing.Extensions.HtmlReport"); + sb.Append(','); + AppendStringPair(sb, "generatorVersion", ExtensionVersion.DefaultSemVer); + sb.Append(','); + AppendStringPair(sb, "testApplication", _testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); + sb.Append(','); + AppendStringPair(sb, "machineName", _environment.MachineName); + sb.Append(','); + AppendStringPair(sb, "userName", _environment.GetEnvironmentVariable("UserName") ?? _environment.GetEnvironmentVariable("USER") ?? string.Empty); + sb.Append(','); + AppendStringPair(sb, "framework", _testFramework.DisplayName); + sb.Append(','); + AppendStringPair(sb, "frameworkUid", _testFramework.Uid); + sb.Append(','); + AppendStringPair(sb, "frameworkVersion", _testFramework.Version); + sb.Append(','); + AppendStringPair(sb, "startTime", _testStartTime.ToString("O", CultureInfo.InvariantCulture)); + sb.Append(','); + AppendStringPair(sb, "endTime", finishTime.ToString("O", CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "exitCode", _exitCode.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendKey(sb, "tests"); + sb.Append('['); + + bool first = true; + for (int i = 0; i < results.Length; i++) + { + CapturedTestResult r = results[i]; + if (!first) + { + sb.Append(','); + } + + first = false; + + CountOutcome(r.Outcome, ref passed, ref failed, ref skipped, ref timedout, ref errored); + totalDuration += r.Duration; + + int attemptOf = countByUid[r.Uid]; + int attemptIndex = emittedByUid.TryGetValue(r.Uid, out int alreadyEmitted) ? alreadyEmitted + 1 : 1; + emittedByUid[r.Uid] = attemptIndex; + + sb.Append('{'); + AppendNumberPair(sb, "rowKey", i.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendStringPair(sb, "uid", r.Uid); + sb.Append(','); + AppendStringPair(sb, "displayName", r.DisplayName); + sb.Append(','); + AppendStringPair(sb, "outcome", r.Outcome); + sb.Append(','); + AppendNumberPair(sb, "durationMs", r.Duration.TotalMilliseconds.ToString("F3", CultureInfo.InvariantCulture)); + + if (attemptOf > 1) + { + sb.Append(','); + AppendNumberPair(sb, "attemptIndex", attemptIndex.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "attemptOf", attemptOf.ToString(CultureInfo.InvariantCulture)); + } + + if (r.StartTime is { } startTime) + { + sb.Append(','); + AppendStringPair(sb, "startTime", startTime.ToString("O", CultureInfo.InvariantCulture)); + } + + if (r.EndTime is { } endTime) + { + sb.Append(','); + AppendStringPair(sb, "endTime", endTime.ToString("O", CultureInfo.InvariantCulture)); + } + + if (r.ClassName is not null) + { + sb.Append(','); + AppendStringPair(sb, "className", r.ClassName); + } + + if (r.MethodName is not null) + { + sb.Append(','); + AppendStringPair(sb, "methodName", r.MethodName); + } + + if (r.Traits is { Count: > 0 }) + { + sb.Append(','); + AppendKey(sb, "traits"); + sb.Append('['); + for (int t = 0; t < r.Traits.Count; t++) + { + if (t > 0) + { + sb.Append(','); + } + + KeyValuePair trait = r.Traits[t]; + sb.Append('{'); + AppendStringPair(sb, "key", trait.Key); + sb.Append(','); + AppendStringPair(sb, "value", trait.Value); + sb.Append('}'); + } + + sb.Append(']'); + } + + if (r.ErrorMessage is not null) + { + sb.Append(','); + AppendStringPair(sb, "errorMessage", r.ErrorMessage); + } + + if (r.ExceptionType is not null) + { + sb.Append(','); + AppendStringPair(sb, "exceptionType", r.ExceptionType); + } + + if (r.StackTrace is not null) + { + sb.Append(','); + AppendStringPair(sb, "stackTrace", r.StackTrace); + } + + if (r.StandardOutput is not null) + { + sb.Append(','); + AppendStringPair(sb, "standardOutput", r.StandardOutput); + } + + if (r.StandardError is not null) + { + sb.Append(','); + AppendStringPair(sb, "standardError", r.StandardError); + } + + sb.Append('}'); + } + + sb.Append("],"); + + AppendKey(sb, "summary"); + sb.Append('{'); + AppendNumberPair(sb, "total", (passed + failed + skipped + timedout + errored).ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "passed", passed.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "failed", failed.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "skipped", skipped.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "timedOut", timedout.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "errored", errored.ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + AppendNumberPair(sb, "totalDurationMs", totalDuration.TotalMilliseconds.ToString("F3", CultureInfo.InvariantCulture)); + sb.Append('}'); + + sb.Append('}'); + + return sb.ToString(); + } + + private static void CountOutcome(string outcome, ref int passed, ref int failed, ref int skipped, ref int timedout, ref int errored) + { + switch (outcome) + { + case "passed": passed++; break; + case "failed": failed++; break; + case "skipped": skipped++; break; + case "timedOut": timedout++; break; + case "errored": errored++; break; + default: throw ApplicationStateGuard.Unreachable(); + } + } + + // ------------------------------------------------------------------ + // Minimal HTML-safe JSON writer. + // We can't use System.Text.Json because the extension targets + // netstandard2.0 (where it isn't part of the platform's reference set). + // The output is a plain JSON document but every character that could + // close the embedding + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/TestResultCapture.cs b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/TestResultCapture.cs new file mode 100644 index 0000000000..5928a4bf54 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/TestResultCapture.cs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Messages; + +namespace Microsoft.Testing.Extensions.HtmlReport; + +// Projects a TestNodeUpdateMessage into a capped CapturedTestResult so that the +// generator does not retain entire test node payloads (and their potentially huge +// stdout/stderr/stack traces) in memory for the whole session. +internal static class TestResultCapture +{ + internal const int MaxStandardStreamLength = 32 * 1024; + internal const int MaxStackTraceLength = 32 * 1024; + internal const int MaxMessageLength = 16 * 1024; + + public static CapturedTestResult? TryCapture(TestNode node) + { + TestNodeStateProperty? state = node.Properties.SingleOrDefault(); + if (state is null or DiscoveredTestNodeStateProperty or InProgressTestNodeStateProperty) + { + return null; + } + + string outcome = ClassifyOutcome(state); + + TimingProperty? timing = node.Properties.SingleOrDefault(); + TimeSpan duration = timing?.GlobalTiming.Duration ?? TimeSpan.Zero; + + (string? className, string? methodName) = GetClassAndMethodName(node); + + string? errorMessage = state.Explanation; + string? stackTrace = null; + string? exceptionType = null; + Exception? exception = state switch + { + FailedTestNodeStateProperty f => f.Exception, + ErrorTestNodeStateProperty e => e.Exception, + TimeoutTestNodeStateProperty t => t.Exception, +#pragma warning disable CS0618, MTP0001 // CancelledTestNodeStateProperty is obsolete + CancelledTestNodeStateProperty c => c.Exception, +#pragma warning restore CS0618, MTP0001 + _ => null, + }; + + if (exception is not null) + { + errorMessage ??= exception.Message; + stackTrace = exception.StackTrace; + exceptionType = exception.GetType().FullName; + } + + string? stdout = node.Properties.SingleOrDefault()?.StandardOutput; + string? stderr = node.Properties.SingleOrDefault()?.StandardError; + + // Collect traits without using LINQ to avoid an enumerator allocation per node. + List>? traits = null; + foreach (IProperty p in node.Properties) + { + if (p is TestMetadataProperty meta) + { + traits ??= []; + traits.Add(new KeyValuePair(meta.Key, meta.Value)); + } + } + + return new CapturedTestResult + { + Uid = node.Uid.Value, + DisplayName = node.DisplayName, + Outcome = outcome, + Duration = duration, + StartTime = timing?.GlobalTiming.StartTime, + EndTime = timing?.GlobalTiming.EndTime, + ClassName = className, + MethodName = methodName, + ErrorMessage = Truncate(errorMessage, MaxMessageLength), + ExceptionType = exceptionType, + StackTrace = Truncate(stackTrace, MaxStackTraceLength), + StandardOutput = Truncate(stdout, MaxStandardStreamLength), + StandardError = Truncate(stderr, MaxStandardStreamLength), + Traits = traits, + }; + } + + private static string ClassifyOutcome(TestNodeStateProperty state) + => state switch + { + PassedTestNodeStateProperty => "passed", + SkippedTestNodeStateProperty => "skipped", + TimeoutTestNodeStateProperty => "timedOut", + ErrorTestNodeStateProperty => "errored", + FailedTestNodeStateProperty => "failed", + _ when Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties, state.GetType()) >= 0 + => "failed", + _ => throw ApplicationStateGuard.Unreachable(), + }; + + private static (string? ClassName, string? MethodName) GetClassAndMethodName(TestNode node) + { + TestMethodIdentifierProperty? identifier = node.Properties.SingleOrDefault(); + if (identifier is null) + { + return (null, null); + } + + string className = RoslynString.IsNullOrEmpty(identifier.Namespace) + ? identifier.TypeName + : $"{identifier.Namespace}.{identifier.TypeName}"; + + return (className, identifier.MethodName); + } + + internal static string? Truncate(string? value, int maxLength) + => value is null || value.Length <= maxLength + ? value + : value.Substring(0, maxLength) + + $"\n…[truncated, original length: {value.Length.ToString(CultureInfo.InvariantCulture)}]"; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/TestingPlatformBuilderHook.cs b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/TestingPlatformBuilderHook.cs new file mode 100644 index 0000000000..64552ccde9 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/TestingPlatformBuilderHook.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Builder; + +namespace Microsoft.Testing.Extensions.HtmlReport; + +/// +/// This class is used by Microsoft.Testing.Platform.MSBuild to hook into the Testing Platform Builder +/// to add HTML report support. +/// +public static class TestingPlatformBuilderHook +{ + /// + /// Adds HTML report support to the Testing Platform Builder. + /// + /// The test application builder. + /// The command line arguments. + public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, string[] _) + => testApplicationBuilder.AddHtmlReportProvider(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/build/Microsoft.Testing.Extensions.HtmlReport.props b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/build/Microsoft.Testing.Extensions.HtmlReport.props new file mode 100644 index 0000000000..c55f4029a6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/build/Microsoft.Testing.Extensions.HtmlReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/buildMultiTargeting/Microsoft.Testing.Extensions.HtmlReport.props b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/buildMultiTargeting/Microsoft.Testing.Extensions.HtmlReport.props new file mode 100644 index 0000000000..45a5901b17 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/buildMultiTargeting/Microsoft.Testing.Extensions.HtmlReport.props @@ -0,0 +1,13 @@ + + + + + + Microsoft.Testing.Extensions.HtmlReport + Microsoft.Testing.Extensions.HtmlReport.TestingPlatformBuilderHook + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/buildTransitive/Microsoft.Testing.Extensions.HtmlReport.props b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/buildTransitive/Microsoft.Testing.Extensions.HtmlReport.props new file mode 100644 index 0000000000..c55f4029a6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/buildTransitive/Microsoft.Testing.Extensions.HtmlReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 9aabda5ddc..b73ddfff2f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -45,6 +45,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 7449a51387..6bc7acb6f3 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -100,6 +100,10 @@ Disable reporting progress to screen. --output Output verbosity when reporting tests. Valid values are 'Normal', 'Detailed'. Default is 'Normal'. + --report-html + Enable generating an HTML report + --report-html-filename + The name of the generated HTML report --report-trx Enable generating TRX report --report-trx-filename @@ -312,6 +316,19 @@ Default is 30m. Description: Specify the type of the dump. Valid values are 'Mini', 'Heap', 'Triage', 'None' (only available in .NET 6+) or 'Full'. Default type is 'Full' + HtmlReportGeneratorCommandLine + Name: HTML report generator + Version: * + Description: Produce a self-contained HTML report for the current test session + Options: + --report-html + Arity: 0 + Hidden: False + Description: Enable generating an HTML report + --report-html-filename + Arity: 1 + Hidden: False + Description: The name of the generated HTML report MSBuildCommandLineProvider Name: MSBuildCommandLineProvider Version: * @@ -434,6 +451,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HtmlReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HtmlReportTests.cs new file mode 100644 index 0000000000..9190e5a412 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HtmlReportTests.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +[TestClass] +public class HtmlReportTests : AcceptanceTestBase +{ + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Html_WhenReportHtmlIsNotSpecified_HtmlReportIsNotGenerated(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + // The HTML report is published as an in-process artifact; check the correct block. + string outputPattern = """ + In process file artifacts produced: + - .+?\.html +"""; + testHostResult.AssertOutputDoesNotMatchRegex(outputPattern); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Html_WhenReportHtmlIsSpecified_HtmlReportIsGeneratedInDefaultLocation(string tfm) + { + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string htmlPathPattern = Path.Combine(testResultsPath, ".+?\\.html").Replace(@"\", @"\\"); + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-html", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string outputPattern = $""" + In process file artifacts produced: + - {htmlPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Match match = Regex.Match(testHostResult.StandardOutput, htmlPathPattern); + Assert.IsTrue(match.Success, $"HTML report path not found in output:\n{testHostResult.StandardOutput}"); + + string htmlContent = File.ReadAllText(match.Value); + Assert.Contains("", htmlContent, "Generated file does not appear to be a valid HTML report."); + Assert.Contains("id=\"mtp-data\"", htmlContent, "Generated HTML report does not contain embedded JSON data."); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Html_WhenReportHtmlFilenameIsSpecified_HtmlReportIsGeneratedWithThatName(string tfm) + { + const string customFileName = "my-custom-report.html"; + string testResultsPath = Path.Combine(AssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string customFilePath = Path.Combine(testResultsPath, customFileName); + string expectedFilePath = Regex.Escape(customFilePath); + + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-html --report-html-filename {customFileName}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string outputPattern = $""" + In process file artifacts produced: + - {expectedFilePath} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + + Assert.IsTrue( + File.Exists(customFilePath), + $"Expected custom HTML report file '{customFileName}' was not found in '{testResultsPath}'."); + + string htmlContent = File.ReadAllText(customFilePath); + Assert.Contains("", htmlContent, "Generated file does not appear to be a valid HTML report."); + Assert.Contains("id=\"mtp-data\"", htmlContent, "Generated HTML report does not contain embedded JSON data."); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Html_WhenReportHtmlFilenameContainsPath_ErrorIsDisplayed(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--report-html --report-html-filename {Path.Combine("subdir", "report.html")}", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("file name argument must not contain a path or invalid characters"); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Html_WhenReportHtmlFilenameIsSpecifiedWithoutReportHtml_ErrorIsDisplayed(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--report-html-filename report.html", + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--report-html-filename' requires '--report-html' to be enabled"); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string AssetName = "HtmlReportTest"; + + private const string TestCode = """ +#file HtmlReportTest.csproj + + + $TargetFrameworks$ + enable + enable + Exe + preview + + + + + + +#file Program.cs +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework( + sp => new TestFrameworkCapabilities(), + (_, __) => new DummyTestFramework()); + builder.AddHtmlReportProvider(); + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DummyTestFramework : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DummyTestFramework); + public string Description => nameof(DummyTestFramework); + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( + context.Request.Session.SessionUid, + new TestNode() + { + Uid = "test-1", + DisplayName = "PassingTest", + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + })); + context.Complete(); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + TestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } + + public TestContext TestContext { get; set; } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs index a069f5d16b..233905b5df 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs @@ -25,6 +25,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis var testHost = TestInfrastructure.TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, rid: RID, verb: verb, buildConfiguration: compilationMode); TestHostResult testHostResult = await testHost.ExecuteAsync("--help", cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContains("--crashdump"); + testHostResult.AssertOutputContains("--report-html"); testHostResult.AssertOutputContains("--report-trx"); testHostResult.AssertOutputContains("--retry-failed-tests"); testHostResult.AssertOutputContains("--hangdump"); @@ -37,6 +38,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis Assert.Contains("Microsoft.Testing.Extensions.CrashDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HangDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HotReload.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); + Assert.Contains("Microsoft.Testing.Extensions.HtmlReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.Retry.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.Telemetry.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.TrxReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); @@ -70,6 +72,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis + diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/ExtensionVersionTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/ExtensionVersionTests.cs index 029448a60b..7516764a4d 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/ExtensionVersionTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/ExtensionVersionTests.cs @@ -46,6 +46,13 @@ public void TrxReportGeneratorCommandLine_UsesItsOwnAssemblyVersion() AssertVersionMatchesAssembly(provider.Version, typeof(TrxReportGeneratorCommandLine).Assembly); } + [TestMethod] + public void HtmlReportGeneratorCommandLine_UsesItsOwnAssemblyVersion() + { + var provider = new Microsoft.Testing.Extensions.HtmlReport.HtmlReportGeneratorCommandLine(); + AssertVersionMatchesAssembly(provider.Version, typeof(Microsoft.Testing.Extensions.HtmlReport.HtmlReportGeneratorCommandLine).Assembly); + } + private static void AssertVersionMatchesAssembly(string reportedVersion, Assembly extensionAssembly) { Assert.IsFalse(string.IsNullOrEmpty(reportedVersion), "Reported version should not be null or empty."); diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HtmlReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HtmlReportEngineTests.cs new file mode 100644 index 0000000000..1f9a993ae3 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HtmlReportEngineTests.cs @@ -0,0 +1,395 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.HtmlReport; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Services; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public class HtmlReportEngineTests +{ + private readonly Mock _environmentMock = new(); + private readonly Mock _commandLineOptionsMock = new(); + private readonly Mock _configurationMock = new(); + private readonly Mock _clockMock = new(); + private readonly Mock _testFrameworkMock = new(); + private readonly Mock _testApplicationModuleInfoMock = new(); + private readonly Mock _fileSystem = new(); + + [TestMethod] + public async Task GenerateReportAsync_WritesValidHtml_WithEmbeddedJson() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("p1", "Passing test", "passed"), + Captured("f1", "Failing test", "failed", errorMessage: "expected 1, got 2"), + Captured("s1", "Skipped test", "skipped", errorMessage: "not relevant"), + ]; + + (string fileName, string? warning) = await engine.GenerateReportAsync(tests); + + Assert.IsNotNull(fileName); + Assert.IsNull(warning); + string html = memoryStream.GetUtf8Content(); + Assert.Contains("", html); + Assert.Contains("id=\"mtp-data\"", html); + Assert.Contains("\"passed\":1", html); + Assert.Contains("\"failed\":1", html); + Assert.Contains("\"skipped\":1", html); + } + + [TestMethod] + public async Task GenerateReportAsync_EscapesScriptInjection_InDisplayName() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + const string Hostile = "evil"; + CapturedTestResult[] tests = + [ + Captured("hostile", Hostile, "failed", errorMessage: Hostile), + ]; + + (string fileName, _) = await engine.GenerateReportAsync(tests); + + Assert.IsNotNull(fileName); + string html = memoryStream.GetUtf8Content(); + + // The literal hostile sequence MUST NOT appear in the HTML — it must be escaped to + // \u003C / \u003E / \u0026 etc. so the browser parser cannot escape the JSON island. + Assert.DoesNotContain("', "\\u003E")] + [DataRow('&', "\\u0026")] + [DataRow('\'', "\\u0027")] + public async Task GenerateReportAsync_EscapesHtmlUnsafeCharacters_AsUnicode(char raw, string expected) + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + string display = "x" + raw + "y"; + CapturedTestResult[] tests = [Captured("u", display, "passed")]; + + await engine.GenerateReportAsync(tests); + + string html = memoryStream.GetUtf8Content(); + Assert.Contains("x" + expected + "y", html); + Assert.DoesNotContain("\"x" + raw + "y\"", html); + } + + [TestMethod] + public async Task GenerateReportAsync_EscapesLineSeparators_U2028_AndU2029() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + string display = "line1\u2028line2\u2029line3"; + CapturedTestResult[] tests = [Captured("ls", display, "passed")]; + + await engine.GenerateReportAsync(tests); + + string html = memoryStream.GetUtf8Content(); + Assert.Contains("line1\\u2028line2\\u2029line3", html); + Assert.DoesNotContain("\u2028", html); + Assert.DoesNotContain("\u2029", html); + } + + [TestMethod] + public void TestResultCapture_Truncates_OverLength_StandardOutput_AtBoundary() + { + string huge = new('a', TestResultCapture.MaxStandardStreamLength + 7); + + var bag = new PropertyBag(PassedTestNodeStateProperty.CachedInstance); + bag.Add(new StandardOutputProperty(huge)); + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = bag }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.IsNotNull(result); + Assert.IsNotNull(result.StandardOutput); + Assert.StartsWith(new string('a', TestResultCapture.MaxStandardStreamLength), result.StandardOutput!); + Assert.Contains("[truncated, original length:", result.StandardOutput); + Assert.Contains((TestResultCapture.MaxStandardStreamLength + 7).ToString(CultureInfo.InvariantCulture), result.StandardOutput); + } + + [TestMethod] + public void TestResultCapture_Does_Not_Truncate_When_Exactly_At_MaxLength() + { + string atMax = new('a', TestResultCapture.MaxStandardStreamLength); + + var bag = new PropertyBag(PassedTestNodeStateProperty.CachedInstance); + bag.Add(new StandardOutputProperty(atMax)); + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = bag }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual(atMax, result.StandardOutput); + } + + [TestMethod] + public void TestResultCapture_Returns_Null_For_NonTerminalStates() + { + TestNode discovered = new() { Uid = "a", DisplayName = "x", Properties = new(DiscoveredTestNodeStateProperty.CachedInstance) }; + TestNode inProgress = new() { Uid = "b", DisplayName = "y", Properties = new(InProgressTestNodeStateProperty.CachedInstance) }; + + Assert.IsNull(TestResultCapture.TryCapture(discovered)); + Assert.IsNull(TestResultCapture.TryCapture(inProgress)); + } + + [TestMethod] + [DataRow("passed", typeof(PassedTestNodeStateProperty))] + [DataRow("skipped", typeof(SkippedTestNodeStateProperty))] + [DataRow("failed", typeof(FailedTestNodeStateProperty))] + [DataRow("errored", typeof(ErrorTestNodeStateProperty))] + [DataRow("timedOut", typeof(TimeoutTestNodeStateProperty))] + public void TestResultCapture_ClassifiesEveryWellKnownTerminalOutcome(string expected, Type stateType) + { + TestNodeStateProperty state = stateType switch + { + Type t when t == typeof(PassedTestNodeStateProperty) => PassedTestNodeStateProperty.CachedInstance, + Type t when t == typeof(SkippedTestNodeStateProperty) => SkippedTestNodeStateProperty.CachedInstance, + Type t when t == typeof(FailedTestNodeStateProperty) => new FailedTestNodeStateProperty("x"), + Type t when t == typeof(ErrorTestNodeStateProperty) => new ErrorTestNodeStateProperty("x"), + Type t when t == typeof(TimeoutTestNodeStateProperty) => new TimeoutTestNodeStateProperty("x"), + _ => throw new InvalidOperationException(), + }; + + TestNode node = new() { Uid = "id", DisplayName = "T", Properties = new(state) }; + + CapturedTestResult result = TestResultCapture.TryCapture(node)!; + + Assert.AreEqual(expected, result.Outcome); + } + + [TestMethod] + public async Task GenerateReportAsync_CountsAllOutcomeKindsSeparately() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("p1", "Passed", "passed"), + Captured("f1", "Failed", "failed"), + Captured("s1", "Skipped", "skipped"), + Captured("e1", "Errored", "errored"), + Captured("t1", "Timed out", "timedOut"), + ]; + + await engine.GenerateReportAsync(tests); + + string html = memoryStream.GetUtf8Content(); + Assert.Contains("\"total\":5", html); + Assert.Contains("\"passed\":1", html); + Assert.Contains("\"failed\":1", html); + Assert.Contains("\"skipped\":1", html); + Assert.Contains("\"errored\":1", html); + Assert.Contains("\"timedOut\":1", html); + } + + [TestMethod] + public async Task GenerateReportAsync_PreservesAllResultsForDuplicateUids_AndAnnotatesAttemptOf() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("dup", "Row A", "failed", errorMessage: "first failure"), + Captured("dup", "Row B", "failed", errorMessage: "second failure"), + Captured("dup", "Row C", "passed"), + Captured("unique", "Solo", "passed"), + ]; + + await engine.GenerateReportAsync(tests); + + string html = memoryStream.GetUtf8Content(); + + // All three rows for the same UID must appear in the report. + Assert.Contains("\"displayName\":\"Row A\"", html); + Assert.Contains("\"displayName\":\"Row B\"", html); + Assert.Contains("\"displayName\":\"Row C\"", html); + + // Each duplicate row carries attemptIndex/attemptOf annotation. + Assert.Contains("\"attemptIndex\":1,\"attemptOf\":3", html); + Assert.Contains("\"attemptIndex\":2,\"attemptOf\":3", html); + Assert.Contains("\"attemptIndex\":3,\"attemptOf\":3", html); + + // Counts reflect every observation, not just unique UIDs. + Assert.Contains("\"total\":4", html); + Assert.Contains("\"failed\":2", html); + Assert.Contains("\"passed\":2", html); + + // The unique UID row does not get an attempts annotation. + int soloIdx = html.IndexOf("\"displayName\":\"Solo\"", StringComparison.Ordinal); + Assert.IsGreaterThanOrEqualTo(0, soloIdx); + string soloFragment = html.Substring(soloIdx, Math.Min(400, html.Length - soloIdx)); + Assert.DoesNotContain("\"attemptOf\"", soloFragment, "Unique UIDs should not carry attemptOf annotation."); + } + + [TestMethod] + public async Task GenerateReportAsync_EmitsStableRowKey_Per_Result() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + Captured("a", "A", "passed"), + Captured("a", "A2", "passed"), + Captured("b", "B", "passed"), + ]; + + await engine.GenerateReportAsync(tests); + + string html = memoryStream.GetUtf8Content(); + // The engine must emit a unique row key per result (used by the UI for expand + // state), independent of UID, so multiple rows sharing the same UID never + // collide and a UID like "a#1" can never collide with a derived key. + Assert.Contains("\"rowKey\":0", html); + Assert.Contains("\"rowKey\":1", html); + Assert.Contains("\"rowKey\":2", html); + } + + [TestMethod] + public async Task GenerateReportAsync_IncludesTraits() + { + using var memoryStream = new MemoryFileStream(); + HtmlReportEngine engine = CreateEngine(memoryStream); + CapturedTestResult[] tests = + [ + new CapturedTestResult + { + Uid = "id", + DisplayName = "T", + Outcome = "passed", + Duration = TimeSpan.Zero, + Traits = + [ + new KeyValuePair("Category", "FastTest"), + new KeyValuePair("Owner", "alice"), + ], + }, + ]; + + await engine.GenerateReportAsync(tests); + + string html = memoryStream.GetUtf8Content(); + Assert.Contains("\"traits\":[", html); + Assert.Contains("\"key\":\"Category\"", html); + Assert.Contains("\"value\":\"FastTest\"", html); + Assert.Contains("\"key\":\"Owner\"", html); + Assert.Contains("\"value\":\"alice\"", html); + } + + [TestMethod] + public async Task GenerateReportAsync_AppendsDisambiguatingSuffix_When_DefaultFileExists() + { + // Set up file system: pretend the default file already exists, then succeed on + // the second name. The engine must retry rather than throwing IOException. + var bytesSeen = new List(); + int callCount = 0; + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.CreateNew)) + .Returns((path, _) => + { + callCount++; + bytesSeen.Add(path); + return callCount == 1 + ? throw new IOException("file exists") + : new MemoryFileStream(); + }); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("M"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("u"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("app"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("F"); + _ = _clockMock.SetupGet(_ => _.UtcNow).Returns(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + + var engine = new HtmlReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + + (string finalPath, _) = await engine.GenerateReportAsync([Captured("a", "A", "passed")]); + + Assert.AreEqual(2, callCount); + Assert.AreEqual(bytesSeen[1], finalPath); + Assert.Contains("_1.html", finalPath); + } + + private static CapturedTestResult Captured(string uid, string name, string outcome, + TimeSpan? duration = null, string? errorMessage = null) + => new() + { + Uid = uid, + DisplayName = name, + Outcome = outcome, + Duration = duration ?? TimeSpan.Zero, + ErrorMessage = errorMessage, + }; + + private HtmlReportEngine CreateEngine(MemoryFileStream stream) + { + _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); + _ = _fileSystem.Setup(x => x.NewFileStream(It.IsAny(), It.IsAny())).Returns(stream); + + _ = _configurationMock.SetupGet(_ => _[It.IsAny()]).Returns(string.Empty); + _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("MachineName"); + _ = _environmentMock.Setup(_ => _.GetEnvironmentVariable(It.IsAny())).Returns("user"); + _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("TestAppPath"); + _ = _testFrameworkMock.SetupGet(_ => _.Uid).Returns("fake-uid"); + _ = _testFrameworkMock.SetupGet(_ => _.Version).Returns("0.0.0"); + _ = _testFrameworkMock.SetupGet(_ => _.DisplayName).Returns("Fake"); + + return new HtmlReportEngine( + _fileSystem.Object, + _testApplicationModuleInfoMock.Object, + _environmentMock.Object, + _commandLineOptionsMock.Object, + _configurationMock.Object, + _clockMock.Object, + _testFrameworkMock.Object, + DateTimeOffset.UtcNow, + 0, + CancellationToken.None); + } + + internal sealed class MemoryFileStream : IFileStream + { + public MemoryFileStream() => Stream = new MemoryStream(); + + public MemoryStream Stream { get; } + + Stream IFileStream.Stream => Stream; + + string IFileStream.Name => string.Empty; + + public string GetUtf8Content() => Encoding.UTF8.GetString(Stream.ToArray()); + + void IDisposable.Dispose() => Stream.Dispose(); + +#if NETCOREAPP + ValueTask IAsyncDisposable.DisposeAsync() => Stream.DisposeAsync(); +#endif + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HtmlReportGeneratorCommandLineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HtmlReportGeneratorCommandLineTests.cs new file mode 100644 index 0000000000..1cace2d6f0 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HtmlReportGeneratorCommandLineTests.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.HtmlReport; +using Microsoft.Testing.Extensions.UnitTests.Helpers; +using Microsoft.Testing.Platform.CommandLine; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class HtmlReportGeneratorCommandLineTests +{ + [TestMethod] + public async Task IsValid_If_PureHtmlFileName_Is_Provided() + { + var provider = new HtmlReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, ["report.html"]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [TestMethod] + [DataRow("report.txt")] // wrong extension + [DataRow("report")] // no extension + [DataRow("REPORT.HTM")] // wrong extension (htm vs html) + public async Task IsInvalid_If_FileName_Does_Not_End_With_Html(string fileName) + { + var provider = new HtmlReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(HtmlReport.Resources.ExtensionResources.HtmlReportFileNameExtensionIsNotHtml, result.ErrorMessage); + } + + [TestMethod] + [DataRow("sub/report.html")] + [DataRow("sub\\report.html")] + [DataRow("..\\report.html")] + [DataRow("../report.html")] + [DataRow("..report.html")] // contains ".." + [DataRow("C:report.html")] // drive letter + [DataRow(" report.html")] // leading whitespace + [DataRow("report.html ")] // trailing whitespace + public async Task IsInvalid_If_FileName_Contains_Path_Or_Invalid_Chars(string fileName) + { + var provider = new HtmlReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(HtmlReport.Resources.ExtensionResources.HtmlReportFileNameShouldNotContainPath, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_FileName_Provided_Without_HtmlReport_Flag() + { + var provider = new HtmlReportGeneratorCommandLine(); + var options = new Dictionary + { + [HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName] = ["report.html"], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(HtmlReport.Resources.ExtensionResources.HtmlReportFileNameRequiresHtmlReport, result.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_If_HtmlReport_Used_With_DiscoverTests() + { + var provider = new HtmlReportGeneratorCommandLine(); + var options = new Dictionary + { + [HtmlReportGeneratorCommandLine.HtmlReportOptionName] = [], + [PlatformCommandLineProvider.DiscoverTestsOptionKey] = [], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(HtmlReport.Resources.ExtensionResources.HtmlReportIsNotValidForDiscovery, result.ErrorMessage); + } + + [TestMethod] + public async Task IsValid_When_HtmlReport_Used_Alone() + { + var provider = new HtmlReportGeneratorCommandLine(); + var options = new Dictionary + { + [HtmlReportGeneratorCommandLine.HtmlReportOptionName] = [], + }; + + ValidationResult result = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + } + + [TestMethod] + [DataRow("report*.html")] // * is Windows-invalid even though Linux allows it + [DataRow("report?.html")] // ? same + [DataRow("report\".html")] + [DataRow("report<.html")] + [DataRow("report>.html")] + [DataRow("report|.html")] + public async Task IsInvalid_When_FileName_Contains_WindowsInvalidChars_OnAnyOS(string fileName) + { + var provider = new HtmlReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(HtmlReport.Resources.ExtensionResources.HtmlReportFileNameShouldNotContainPath, result.ErrorMessage); + } + + [TestMethod] + [DataRow("CON.html")] + [DataRow("con.html")] // case insensitive + [DataRow("NUL.html")] + [DataRow("PRN.html")] + [DataRow("AUX.html")] + [DataRow("COM1.html")] + [DataRow("LPT9.html")] + public async Task IsInvalid_When_FileName_Is_Reserved_Windows_Device_Name(string fileName) + { + var provider = new HtmlReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, [fileName]).ConfigureAwait(false); + + Assert.IsFalse(result.IsValid); + Assert.AreEqual(HtmlReport.Resources.ExtensionResources.HtmlReportFileNameShouldNotContainPath, result.ErrorMessage); + } + + [TestMethod] + public async Task IsValid_When_FileName_Starts_With_Reserved_Name_But_Has_Extra_Chars() + { + // "CONfig.html" is not a reserved device name (only the bare "CON" base name is). + var provider = new HtmlReportGeneratorCommandLine(); + Platform.Extensions.CommandLine.CommandLineOption option = provider.GetCommandLineOptions() + .First(x => x.Name == HtmlReportGeneratorCommandLine.HtmlReportFileNameOptionName); + + ValidationResult result = await provider.ValidateOptionArgumentsAsync(option, ["CONfig.html"]).ConfigureAwait(false); + + Assert.IsTrue(result.IsValid); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj index d2b52fa0ac..4809c8f492 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj @@ -24,6 +24,9 @@ TargetFramework=netstandard2.0 + + TargetFramework=netstandard2.0 + TargetFramework=netstandard2.0