From 6806b9971c3edfb3a8dcecff3e5bb6912b3f2756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 29 Apr 2026 07:13:38 +0200 Subject: [PATCH 1/4] Fix O(n^2) output accumulation in data-driven tests StdOut, StdErr, and Trace string builders in TestContextImplementation were not cleared between data-driven test iterations, causing each subsequent iteration's result to include all previous iterations' output. This caused TRX files to grow quadratically with the number of data rows, leading to OutOfMemoryException when parsing the TRX file. The fix adds GetAndClear variants for Out/Err/Trace (matching the existing GetAndClearDiagnosticMessages pattern) and uses them in TestMethodInfo.InvokeAsync. Fixes #7908 --- .../Execution/TestMethodInfo.cs | 6 +- .../Services/TestContextImplementation.cs | 21 ++++++ .../Execution/TestMethodInfoTests.cs | 66 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.cs index ee240aab96..11c44cdc44 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.cs @@ -148,9 +148,9 @@ public virtual async Task InvokeAsync(object?[]? arguments) if (result != null) { var testContextImpl = TestContext as TestContextImplementation; - result.LogOutput = testContextImpl?.GetOut(); - result.LogError = testContextImpl?.GetErr(); - result.DebugTrace = testContextImpl?.GetTrace(); + result.LogOutput = testContextImpl?.GetAndClearOut(); + result.LogError = testContextImpl?.GetAndClearErr(); + result.DebugTrace = testContextImpl?.GetAndClearTrace(); result.TestContextMessages = TestContext?.GetAndClearDiagnosticMessages(); result.ResultFiles = TestContext?.GetResultFiles(); result.Duration = watch.Elapsed; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Services/TestContextImplementation.cs b/src/Adapter/MSTestAdapter.PlatformServices/Services/TestContextImplementation.cs index 78bce06425..1f33d95ef2 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Services/TestContextImplementation.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Services/TestContextImplementation.cs @@ -383,9 +383,30 @@ private SynchronizedStringBuilder GetTestContextMessagesStringBuilder() internal string? GetOut() => _stdOutStringBuilder?.ToString(); + internal string? GetAndClearOut() + { + string? result = _stdOutStringBuilder?.ToString(); + _stdOutStringBuilder?.Clear(); + return result; + } + internal string? GetErr() => _stdErrStringBuilder?.ToString(); + internal string? GetAndClearErr() + { + string? result = _stdErrStringBuilder?.ToString(); + _stdErrStringBuilder?.Clear(); + return result; + } + internal string? GetTrace() => _traceStringBuilder?.ToString(); + + internal string? GetAndClearTrace() + { + string? result = _traceStringBuilder?.ToString(); + _traceStringBuilder?.Clear(); + return result; + } } diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs index 98e17cfcb3..6bb10b358b 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs @@ -314,6 +314,72 @@ public async Task TestMethodInfoInvokeShouldClearTestContextMessagesAfterReporti result.TestContextMessages!.Contains("SeaShore").Should().BeTrue(); } + public async Task TestMethodInfoInvokeShouldClearStdOutAfterReporting() + { + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleOut("output1"); + + var method = new TestMethodInfo( + _methodInfo, + _testClassInfo) + { + TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000), + Executor = _testMethodAttribute, + }; + + TestResult result1 = await method.InvokeAsync(null); + result1.LogOutput.Should().Contain("output1"); + + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleOut("output2"); + TestResult result2 = await method.InvokeAsync(null); + + result2.LogOutput.Should().Contain("output2"); + result2.LogOutput.Should().NotContain("output1", "StdOut should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests"); + } + + public async Task TestMethodInfoInvokeShouldClearStdErrAfterReporting() + { + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleErr("error1"); + + var method = new TestMethodInfo( + _methodInfo, + _testClassInfo) + { + TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000), + Executor = _testMethodAttribute, + }; + + TestResult result1 = await method.InvokeAsync(null); + result1.LogError.Should().Contain("error1"); + + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleErr("error2"); + TestResult result2 = await method.InvokeAsync(null); + + result2.LogError.Should().Contain("error2"); + result2.LogError.Should().NotContain("error1", "StdErr should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests"); + } + + public async Task TestMethodInfoInvokeShouldClearDebugTraceAfterReporting() + { + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteTrace("trace1"); + + var method = new TestMethodInfo( + _methodInfo, + _testClassInfo) + { + TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000), + Executor = _testMethodAttribute, + }; + + TestResult result1 = await method.InvokeAsync(null); + result1.DebugTrace.Should().Contain("trace1"); + + DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteTrace("trace2"); + TestResult result2 = await method.InvokeAsync(null); + + result2.DebugTrace.Should().Contain("trace2"); + result2.DebugTrace.Should().NotContain("trace1", "DebugTrace should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests"); + } + public async Task Invoke_WhenTestMethodThrowsMissingMethodException_TestOutcomeIsFailedAndExceptionIsPreserved() { DummyTestClass.TestMethodBody = _ => From facdab3adda9c7e4ccdcbbf06d3f0991d97f4794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 29 Apr 2026 07:38:28 +0200 Subject: [PATCH 2/4] Add acceptance test for data-driven TRX output accumulation Adds a refactoring-resilient acceptance test that runs an actual MSTest data-driven test with Console.WriteLine output across 3 DataRow iterations, then parses the TRX file and verifies each UnitTestResult's StdOut contains only its own row's output marker, not markers from other rows. --- .../TrxReportTests.cs | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs index 5453af5894..283baa9ac7 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Xml.Linq; + using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; @@ -89,3 +91,112 @@ public void FailingTest() public TestContext TestContext { get; set; } = null!; } + +[TestClass] +public sealed class TrxReportDataDrivenOutputTests : AcceptanceTestBase +{ + /// + /// Regression test for https://github.com/microsoft/testfx/issues/7908. + /// Verifies that data-driven test output does not accumulate across data rows in the TRX file. + /// Each data row's StdOut in the TRX should contain only that row's output, not output from previous rows. + /// + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(string tfm) + { + string fileName = Guid.NewGuid().ToString("N"); + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); + string trxContent = File.ReadAllText(trxFile); + + XNamespace ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + var trxDoc = XDocument.Parse(trxContent); + var results = trxDoc.Descendants(ns + "UnitTestResult").ToList(); + + // We have 3 data rows, each writing a unique marker like "UNIQUE_ROW_0_MARKER", "UNIQUE_ROW_1_MARKER", "UNIQUE_ROW_2_MARKER" + Assert.IsGreaterThanOrEqualTo(results.Count, 3, $"Expected at least 3 test results but found {results.Count}. TRX content:\n{trxContent}"); + + foreach (XElement result in results) + { + string? stdOut = result.Descendants(ns + "StdOut").FirstOrDefault()?.Value; + if (stdOut is null) + { + continue; + } + + // Count how many unique row markers appear in this single result's StdOut. + // Each result should contain exactly ONE marker (its own row's output). + int markerCount = 0; + for (int i = 0; i < 3; i++) + { + if (stdOut.Contains($"UNIQUE_ROW_{i}_MARKER")) + { + markerCount++; + } + } + + Assert.IsLessThanOrEqualTo( + markerCount, + 1, + $"Test result '{result.Attribute("testName")?.Value}' contains output from {markerCount} data rows. " + + $"Each row should only contain its own output. StdOut:\n{stdOut}"); + } + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + public const string ProjectName = "MSTestTrxDataDriven"; + + public string TargetAssetPath => GetAssetPath(ProjectName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + private const string SourceCode = """ +#file MSTestTrxDataDriven.csproj + + + + Exe + true + $TargetFrameworks$ + latest + + + + + + + + + +#file UnitTest1.cs +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestTrxDataDriven; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + [DataRow(0)] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTestWithOutput(int row) + { + Console.WriteLine($"UNIQUE_ROW_{row}_MARKER"); + } +} +"""; + } + + public TestContext TestContext { get; set; } = null!; +} From 28103bba200cb85e07951a7bc507a5a00684a2b1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 29 Apr 2026 09:13:14 +0000 Subject: [PATCH 3/4] Fix test assertions: correct argument order and add positive StdOut presence check Agent-Logs-Url: https://github.com/microsoft/testfx/sessions/4ed836b7-5892-4268-939a-3b930b923ff8 Co-authored-by: Evangelink <11340282+Evangelink@users.noreply.github.com> --- .../TrxReportTests.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs index 283baa9ac7..6fdb768d66 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs @@ -118,8 +118,9 @@ public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(str var results = trxDoc.Descendants(ns + "UnitTestResult").ToList(); // We have 3 data rows, each writing a unique marker like "UNIQUE_ROW_0_MARKER", "UNIQUE_ROW_1_MARKER", "UNIQUE_ROW_2_MARKER" - Assert.IsGreaterThanOrEqualTo(results.Count, 3, $"Expected at least 3 test results but found {results.Count}. TRX content:\n{trxContent}"); + Assert.IsGreaterThanOrEqualTo(3, results.Count, $"Expected at least 3 test results but found {results.Count}. TRX content:\n{trxContent}"); + int resultsWithOutput = 0; foreach (XElement result in results) { string? stdOut = result.Descendants(ns + "StdOut").FirstOrDefault()?.Value; @@ -128,6 +129,8 @@ public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(str continue; } + resultsWithOutput++; + // Count how many unique row markers appear in this single result's StdOut. // Each result should contain exactly ONE marker (its own row's output). int markerCount = 0; @@ -139,12 +142,14 @@ public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(str } } - Assert.IsLessThanOrEqualTo( - markerCount, + Assert.AreEqual( 1, + markerCount, $"Test result '{result.Attribute("testName")?.Value}' contains output from {markerCount} data rows. " + $"Each row should only contain its own output. StdOut:\n{stdOut}"); } + + Assert.IsGreaterThanOrEqualTo(3, resultsWithOutput, $"Expected at least 3 test results to have StdOut output but only {resultsWithOutput} did. TRX content:\n{trxContent}"); } public sealed class TestAssetFixture() : TestAssetFixtureBase() From ccb4fc25cce8d46ee5919601ae9c6b2f85dbb0cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Thu, 30 Apr 2026 15:46:32 +0200 Subject: [PATCH 4/4] Fix ExitCodes.Success -> ExitCode.Success compile error --- .../MSTest.Acceptance.IntegrationTests/TrxReportTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs index ac53ecefae..c94fa1d9eb 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TrxReportTests.cs @@ -108,7 +108,7 @@ public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(str var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx", cancellationToken: TestContext.CancellationToken); - testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCode.Success); string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); string trxContent = File.ReadAllText(trxFile);