Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Xml.Linq;

using Microsoft.Testing.Platform.Acceptance.IntegrationTests;
using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;
using Microsoft.Testing.Platform.Helpers;
Expand Down Expand Up @@ -89,3 +91,117 @@ public void FailingTest()

public TestContext TestContext { get; set; } = null!;
}

[TestClass]
public sealed class TrxReportDataDrivenOutputTests : AcceptanceTestBase<TrxReportDataDrivenOutputTests.TestAssetFixture>
{
/// <summary>
/// Regression test for https://github.com/microsoft/testfx/issues/7908.
/// Verifies that data-driven test output does not accumulate across data rows in the TRX file.
/// Each data row's StdOut in the TRX should contain only that row's output, not output from previous rows.
/// </summary>
[TestMethod]
[DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(string tfm)
{
string fileName = Guid.NewGuid().ToString("N");
var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm);
TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx", cancellationToken: TestContext.CancellationToken);

testHostResult.AssertExitCodeIs(ExitCode.Success);

string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single();
string trxContent = File.ReadAllText(trxFile);

XNamespace ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010";
var trxDoc = XDocument.Parse(trxContent);
var results = trxDoc.Descendants(ns + "UnitTestResult").ToList();

// We have 3 data rows, each writing a unique marker like "UNIQUE_ROW_0_MARKER", "UNIQUE_ROW_1_MARKER", "UNIQUE_ROW_2_MARKER"
Assert.IsGreaterThanOrEqualTo(3, results.Count, $"Expected at least 3 test results but found {results.Count}. TRX content:\n{trxContent}");

int resultsWithOutput = 0;
foreach (XElement result in results)
{
string? stdOut = result.Descendants(ns + "StdOut").FirstOrDefault()?.Value;
if (stdOut is null)
{
continue;
}
Comment on lines +124 to +130
Copy link

Copilot AI Apr 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The XML parsing loop skips UnitTestResults without a <StdOut> element (if (stdOut is null) continue;). Given the test’s summary says each data row’s StdOut should contain its marker, this can allow a false-positive pass if StdOut is missing/empty. Consider asserting that the expected markers are present (e.g., total markers across all results equals 3, or each of the 3 data-row results has exactly one marker) so the test can’t silently succeed when output isn’t written.

Copilot uses AI. Check for mistakes.

resultsWithOutput++;

// Count how many unique row markers appear in this single result's StdOut.
// Each result should contain exactly ONE marker (its own row's output).
int markerCount = 0;
for (int i = 0; i < 3; i++)
{
if (stdOut.Contains($"UNIQUE_ROW_{i}_MARKER"))
{
markerCount++;
}
Comment thread
Evangelink marked this conversation as resolved.
}

Assert.AreEqual(
1,
markerCount,
$"Test result '{result.Attribute("testName")?.Value}' contains output from {markerCount} data rows. " +
$"Each row should only contain its own output. StdOut:\n{stdOut}");
}

Assert.IsGreaterThanOrEqualTo(3, resultsWithOutput, $"Expected at least 3 test results to have StdOut output but only {resultsWithOutput} did. TRX content:\n{trxContent}");
}
Comment thread
Evangelink marked this conversation as resolved.

public sealed class TestAssetFixture() : TestAssetFixtureBase()
{
public const string ProjectName = "MSTestTrxDataDriven";

public string TargetAssetPath => GetAssetPath(ProjectName);

public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName,
SourceCode
.PatchTargetFrameworks(TargetFrameworks.All)
.PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)
.PatchCodeWithReplace("$MSTestVersion$", MSTestVersion));

private const string SourceCode = """
#file MSTestTrxDataDriven.csproj
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<EnableMSTestRunner>true</EnableMSTestRunner>
<TargetFrameworks>$TargetFrameworks$</TargetFrameworks>
<LangVersion>latest</LangVersion>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Testing.Platform" Version="$MicrosoftTestingPlatformVersion$" />
<PackageReference Include="MSTest" Version="$MSTestVersion$" />
</ItemGroup>

</Project>

#file UnitTest1.cs
using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace MSTestTrxDataDriven;

[TestClass]
public class UnitTest1
{
[TestMethod]
[DataRow(0)]
[DataRow(1)]
[DataRow(2)]
public void DataDrivenTestWithOutput(int row)
{
Console.WriteLine($"UNIQUE_ROW_{row}_MARKER");
}
}
""";
}

public TestContext TestContext { get; set; } = null!;
}
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,72 @@ public async Task TestMethodInfoInvokeShouldClearTestContextMessagesAfterReporti
result.TestContextMessages!.Contains("SeaShore").Should().BeTrue();
}

public async Task TestMethodInfoInvokeShouldClearStdOutAfterReporting()
{
DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleOut("output1");

var method = new TestMethodInfo(
_methodInfo,
_testClassInfo)
{
TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000),
Executor = _testMethodAttribute,
};

TestResult result1 = await method.InvokeAsync(null);
result1.LogOutput.Should().Contain("output1");

DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleOut("output2");
TestResult result2 = await method.InvokeAsync(null);

result2.LogOutput.Should().Contain("output2");
result2.LogOutput.Should().NotContain("output1", "StdOut should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests");
}

public async Task TestMethodInfoInvokeShouldClearStdErrAfterReporting()
{
DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleErr("error1");

var method = new TestMethodInfo(
_methodInfo,
_testClassInfo)
{
TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000),
Executor = _testMethodAttribute,
};

TestResult result1 = await method.InvokeAsync(null);
result1.LogError.Should().Contain("error1");

DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteConsoleErr("error2");
TestResult result2 = await method.InvokeAsync(null);

result2.LogError.Should().Contain("error2");
result2.LogError.Should().NotContain("error1", "StdErr should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests");
}

public async Task TestMethodInfoInvokeShouldClearDebugTraceAfterReporting()
{
DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteTrace("trace1");

var method = new TestMethodInfo(
_methodInfo,
_testClassInfo)
{
TimeoutInfo = TimeoutInfo.FromTimeout(3600 * 1000),
Executor = _testMethodAttribute,
};

TestResult result1 = await method.InvokeAsync(null);
result1.DebugTrace.Should().Contain("trace1");

DummyTestClass.TestMethodBody = o => _testContextImplementation.WriteTrace("trace2");
TestResult result2 = await method.InvokeAsync(null);

result2.DebugTrace.Should().Contain("trace2");
result2.DebugTrace.Should().NotContain("trace1", "DebugTrace should be cleared between invocations to prevent O(n^2) output accumulation in data-driven tests");
}

public async Task Invoke_WhenTestMethodThrowsMissingMethodException_TestOutcomeIsFailedAndExceptionIsPreserved()
{
DummyTestClass.TestMethodBody = _ =>
Expand Down
Loading