Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,11 @@ public async Task<int> OrchestrateTestHostExecutionAsync(CancellationToken cance
RemoveOption(finalArguments, TreeNodeFilterCommandLineOptionsProvider.TreenodeFilter);
RemoveOption(finalArguments, PlatformCommandLineProvider.FilterUidOptionKey);

// Strip --minimum-expected-tests on retry attempts: a retry only re-runs the previously
// failed tests, so propagating the original threshold (computed against the full test
// set) would always trip the policy and fail the run. See issue #5639.
RemoveOption(finalArguments, PlatformCommandLineProvider.MinimumExpectedTestsOptionKey);

// The RSP parser (ResponseFileHelper.SplitCommandLine) strips all '"' characters
// from tokens, so UIDs containing literal '"' (e.g. parameterized tests with
// string arguments that include double quotes) cannot safely round-trip through
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,37 @@ public async Task RetryFailedTests_WithPreexistingTreeNodeFilter_ReplacesFilterO
Assert.DoesNotContain("TestMethod3", trxContent);
}

[TestMethod]
[DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
public async Task RetryFailedTests_WithMinimumExpectedTests_StripsThresholdOnRetry(string tfm)
{
// Regression test for https://github.com/microsoft/testfx/issues/5639.
// --minimum-expected-tests must be honored on the first attempt but stripped from retry-attempt
// arguments. Without the strip, the retry (which only re-runs the previously failed tests) would
// always trip the policy and exit with code 9 (MinimumExpectedTestsPolicyViolation) even though
// the full first-attempt run satisfied the threshold.
var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm);
string resultDirectory = Path.Combine(testHost.DirectoryName, Guid.NewGuid().ToString("N"));

// METHOD1=1 causes TestMethod1 to fail on the first attempt and pass on the second. The asset has
// 3 tests in total. With --minimum-expected-tests 3, the first attempt runs all 3 (so the policy
// is satisfied) and the retry attempt runs only TestMethod1 (so without the fix the policy would
// fail with "tests ran 1, minimum expected 3").
TestHostResult testHostResult = await testHost.ExecuteAsync(
$"--retry-failed-tests 3 --minimum-expected-tests 3 --results-directory {resultDirectory}",
new()
{
{ EnvironmentVariableConstants.TESTINGPLATFORM_TELEMETRY_OPTOUT, "1" },
{ "METHOD1", "1" },
{ "RESULTDIR", resultDirectory },
},
cancellationToken: TestContext.CancellationToken);

testHostResult.AssertExitCodeIs(ExitCode.Success);
testHostResult.AssertOutputContains("Tests suite completed successfully in 2 attempts");
testHostResult.AssertOutputDoesNotContain("Minimum expected tests policy violation");
}

public sealed class TestAssetFixture() : TestAssetFixtureBase()
{
public string TargetAssetPath => GetAssetPath(AssetName);
Expand Down
Loading