diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs index dfcce5307f..c94fa1d9eb 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Xml.Linq; + using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; @@ -89,3 +91,117 @@ public void FailingTest() public TestContext TestContext { get; set; } = null!; } + +[TestClass] +public sealed class TrxReportDataDrivenOutputTests : AcceptanceTestBase +{ + /// + /// Regression test for https://github.com/microsoft/testfx/issues/7908. + /// Verifies that data-driven test output does not accumulate across data rows in the TRX file. + /// Each data row's StdOut in the TRX should contain only that row's output, not output from previous rows. + /// + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(string tfm) + { + string fileName = Guid.NewGuid().ToString("N"); + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); + string trxContent = File.ReadAllText(trxFile); + + XNamespace ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + var trxDoc = XDocument.Parse(trxContent); + var results = trxDoc.Descendants(ns + "UnitTestResult").ToList(); + + // We have 3 data rows, each writing a unique marker like "UNIQUE_ROW_0_MARKER", "UNIQUE_ROW_1_MARKER", "UNIQUE_ROW_2_MARKER" + Assert.IsGreaterThanOrEqualTo(3, results.Count, $"Expected at least 3 test results but found {results.Count}. TRX content:\n{trxContent}"); + + int resultsWithOutput = 0; + foreach (XElement result in results) + { + string? stdOut = result.Descendants(ns + "StdOut").FirstOrDefault()?.Value; + if (stdOut is null) + { + continue; + } + + resultsWithOutput++; + + // Count how many unique row markers appear in this single result's StdOut. + // Each result should contain exactly ONE marker (its own row's output). + int markerCount = 0; + for (int i = 0; i < 3; i++) + { + if (stdOut.Contains($"UNIQUE_ROW_{i}_MARKER")) + { + markerCount++; + } + } + + Assert.AreEqual( + 1, + markerCount, + $"Test result '{result.Attribute("testName")?.Value}' contains output from {markerCount} data rows. " + + $"Each row should only contain its own output. StdOut:\n{stdOut}"); + } + + Assert.IsGreaterThanOrEqualTo(3, resultsWithOutput, $"Expected at least 3 test results to have StdOut output but only {resultsWithOutput} did. TRX content:\n{trxContent}"); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string ProjectName = "MSTestTrxDataDriven"; + + public string TargetAssetPath => GetAssetPath(ProjectName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + private const string SourceCode = """ +#file MSTestTrxDataDriven.csproj + + + + Exe + true + $TargetFrameworks$ + latest + + + + + + + + + +#file UnitTest1.cs +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestTrxDataDriven; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + [DataRow(0)] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTestWithOutput(int row) + { + Console.WriteLine($"UNIQUE_ROW_{row}_MARKER"); + } +} +"""; + } + + public TestContext TestContext { get; set; } = null!; +} diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs index e2d249bdea..b3618c7991 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs @@ -314,6 +314,72 @@ public async Task TestMethodInfoInvokeShouldClearTestContextMessagesAfterReporti result.TestContextMessages!.Contains("SeaShore").Should().BeTrue(); } + public async Task TestMethodInfoInvokeShouldClearStdOutAfterReporting() + { + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleOut("output1"); + + var method = new TestMethodInfo( + _methodInfo, + _testClassInfo) + { + TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000), + Executor = _testMethodAttribute, + }; + + TestResult result1 = await method.InvokeAsync(null); + result1.LogOutput.Should().Contain("output1"); + + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleOut("output2"); + TestResult result2 = await method.InvokeAsync(null); + + result2.LogOutput.Should().Contain("output2"); + result2.LogOutput.Should().NotContain("output1", "StdOut should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests"); + } + + public async Task TestMethodInfoInvokeShouldClearStdErrAfterReporting() + { + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleErr("error1"); + + var method = new TestMethodInfo( + _methodInfo, + _testClassInfo) + { + TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000), + Executor = _testMethodAttribute, + }; + + TestResult result1 = await method.InvokeAsync(null); + result1.LogError.Should().Contain("error1"); + + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleErr("error2"); + TestResult result2 = await method.InvokeAsync(null); + + result2.LogError.Should().Contain("error2"); + result2.LogError.Should().NotContain("error1", "StdErr should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests"); + } + + public async Task TestMethodInfoInvokeShouldClearDebugTraceAfterReporting() + { + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteTrace("trace1"); + + var method = new TestMethodInfo( + _methodInfo, + _testClassInfo) + { + TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000), + Executor = _testMethodAttribute, + }; + + TestResult result1 = await method.InvokeAsync(null); + result1.DebugTrace.Should().Contain("trace1"); + + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteTrace("trace2"); + TestResult result2 = await method.InvokeAsync(null); + + result2.DebugTrace.Should().Contain("trace2"); + result2.DebugTrace.Should().NotContain("trace1", "DebugTrace should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests"); + } + public async Task Invoke_WhenTestMethodThrowsMissingMethodException_TestOutcomeIsFailedAndExceptionIsPreserved() { DummyTestClass.TestMethodBody = _ =>