Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refreshing tests #3023

Closed
wants to merge 6 commits into from
Closed

Refreshing tests #3023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
51430c5
removing ignore attribute and removing ordered tests
Sanan07 Aug 31, 2021
3eeb70d
Merge branch 'main' of https://github.com/microsoft/vstest into dev/s…
Sanan07 Aug 31, 2021
276dfb1
updating externals
Sanan07 Aug 31, 2021
cc8ccff
remove more generic and ordered tests
Sanan07 Aug 31, 2021
cf65466
removing mstestv1 project
Sanan07 Aug 31, 2021
4c8f5bd
removing tests which used old mstestv1 project
Sanan07 Aug 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/build/TestPlatform.Dependencies.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
<JsonNetVersion>9.0.1</JsonNetVersion>
<MoqVersion>4.8.3</MoqVersion>

<TestPlatformExternalsVersion>17.0.0-previews-4-31622-033</TestPlatformExternalsVersion>
<TestPlatformExternalsVersion>17.0.0-previews-4-31630-230</TestPlatformExternalsVersion>
<!-- <TestPlatformMSDiaVersion>$(TestPlatformExternalsVersion)</TestPlatformMSDiaVersion> -->
<TestPlatformMSDiaVersion>17.0.0-previews-3-31515-332</TestPlatformMSDiaVersion>

Expand Down
2 changes: 1 addition & 1 deletion scripts/verify-nupkgs.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ function Verify-Nuget-Packages($packageDirectory, $version)
$expectedNumOfFiles = @{
"Microsoft.CodeCoverage" = 59;
"Microsoft.NET.Test.Sdk" = 27;
"Microsoft.TestPlatform" = 486;
"Microsoft.TestPlatform" = 484;
"Microsoft.TestPlatform.Build" = 21;
"Microsoft.TestPlatform.CLI" = 367;
"Microsoft.TestPlatform.Extensions.TrxLogger" = 35;
Expand Down
2 changes: 0 additions & 2 deletions src/Microsoft.TestPlatform.CrossPlatEngine/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ public class Constants
internal static readonly IList<string> DefaultAdapters = new ReadOnlyCollection<string>(new List<string>
{
"executor://CodedWebTestAdapter/v1",
"executor://GenericTestAdapter/v1",
"executor://OrderedTestAdapter/v1",
"executor://MSTestAdapter/v1",
"executor://WebTestAdapter/v1",
"executor://CppUnitTestExecutor/v1"
Expand Down
2 changes: 0 additions & 2 deletions src/package/nuspec/Microsoft.TestPlatform.nuspec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,6 @@
<file src="net451\$Runtime$\Extensions\Microsoft.TestPlatform.TestHostRuntimeProvider.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.TestPlatform.TestHostRuntimeProvider.dll" />
<file src="net451\$Runtime$\Extensions\VideoRecorder\Microsoft.VisualStudio.QualityTools.VideoRecorderEngine.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\VideoRecorder\Microsoft.VisualStudio.QualityTools.VideoRecorderEngine.dll" />
<file src="net451\$Runtime$\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.CodedWebTestAdapter.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.CodedWebTestAdapter.dll" />
<file src="net451\$Runtime$\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.GenericTestAdapter.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.GenericTestAdapter.dll" />
<file src="net451\$Runtime$\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.OrderedTestAdapter.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.OrderedTestAdapter.dll" />
<file src="net451\$Runtime$\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.TmiAdapter.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.TmiAdapter.dll" />
<file src="net451\$Runtime$\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.dll" />
<file src="net451\$Runtime$\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.dll" target="tools\net451\Common7\IDE\Extensions\TestPlatform\Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.dll" />
Expand Down
2 changes: 0 additions & 2 deletions src/package/sign/sign.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@

<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestTools.DataCollection.MediaRecorder.Model.dll" />
<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorderCollector.dll" />
<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.GenericTestAdapter.dll" />
<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.OrderedTestAdapter.dll" />
<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.CodedWebTestAdapter.dll" />
<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.WebTestAdapter.dll" />
<AssembliesToSign Include="$(ArtifactsDirectory)Extensions\Microsoft.VisualStudio.TestPlatform.Extensions.TmiAdapter.dll" />
Expand Down
1 change: 0 additions & 1 deletion test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ public class CUITTest : AcceptanceTestBase
{
[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource]
public void CUITRunAllTests(RunnerInfo runnerInfo)
{
Expand Down
2 changes: 0 additions & 2 deletions test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ public void CPPRunAllTestExecutionPlatformx64(RunnerInfo runnerInfo)

[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource]
public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo)
{
Expand All @@ -76,7 +75,6 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo)

[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource]
public void CodedWebTestRunAllTests(RunnerInfo runnerInfo)
{
Expand Down
12 changes: 0 additions & 12 deletions test/Microsoft.TestPlatform.AcceptanceTests/ListExtensionsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,6 @@ public void ListDiscoverersShouldShowInboxDiscoverers(RunnerInfo runnerInfo)
if (this.IsDesktopRunner())
{
this.StdOutputContains("executor://codedwebtestadapter/v1");
this.StdOutputContains("executor://generictestadapter/v1");
this.StdOutputContains(".generictest");
this.StdOutputContains("executor://orderedtestadapter/v1");
this.StdOutputContains(".orderedtest");
this.StdOutputContains("executor://mstestadapter/v1");
this.StdOutputContains("executor://webtestadapter/v1");
this.StdOutputContains(".Webtest");
Expand All @@ -34,10 +30,6 @@ public void ListDiscoverersShouldShowInboxDiscoverers(RunnerInfo runnerInfo)
{
// There are no inbox adapters for dotnet core
this.StdOutputDoesNotContains("executor://codedwebtestadapter/v1");
this.StdOutputDoesNotContains("executor://generictestadapter/v1");
this.StdOutputDoesNotContains(".generictest");
this.StdOutputDoesNotContains("executor://orderedtestadapter/v1");
this.StdOutputDoesNotContains(".orderedtest");
this.StdOutputDoesNotContains("executor://mstestadapter/v1");
this.StdOutputDoesNotContains("executor://webtestadapter/v1");
this.StdOutputDoesNotContains(".Webtest");
Expand All @@ -56,8 +48,6 @@ public void ListExecutorsShouldShowInboxExecutors(RunnerInfo runnerInfo)
if (this.IsDesktopRunner())
{
this.StdOutputContains("executor://CodedWebTestAdapter/v1");
this.StdOutputContains("executor://GenericTestAdapter/v1");
this.StdOutputContains("executor://OrderedTestAdapter/v1");
this.StdOutputContains("executor://MSTestAdapter/v1");
this.StdOutputContains("executor://WebTestAdapter/v1");
this.StdOutputContains("executor://CppUnitTestExecutor/v1");
Expand All @@ -67,8 +57,6 @@ public void ListExecutorsShouldShowInboxExecutors(RunnerInfo runnerInfo)
{
// There are no inbox adapters for dotnet core
this.StdOutputDoesNotContains("executor://CodedWebTestAdapter/v1");
this.StdOutputDoesNotContains("executor://GenericTestAdapter/v1");
this.StdOutputDoesNotContains("executor://OrderedTestAdapter/v1");
this.StdOutputDoesNotContains("executor://MSTestAdapter/v1");
this.StdOutputDoesNotContains("executor://WebTestAdapter/v1");
this.StdOutputDoesNotContains("executor://CppUnitTestExecutor/v1");
Expand Down
60 changes: 0 additions & 60 deletions test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs
View file
Open in desktop

This file was deleted.

4 changes: 0 additions & 4 deletions test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,6 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo)

[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)]
public void LegacySettingsWithScripts(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -367,7 +366,6 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo)

[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)]
public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -405,7 +403,6 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo)

[TestMethod]
[TestCategory("Windows")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)]
public void LegacySettingsTestTimeout(RunnerInfo runnerInfo)
{
Expand Down Expand Up @@ -434,7 +431,6 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo)

[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource(inIsolation: true, useCoreRunner: false)]
public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo)
{
Expand Down
68 changes: 0 additions & 68 deletions test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,73 +167,5 @@ public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerI
this.ValidateSummaryStatus(1, 1, 1);
TryRemoveDirectory(resultsDir);
}

/// <summary>
/// Discover tests using mstest v1 adapter with test case filters.
/// </summary>
[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo)
{
if (runnerInfo.RunnerFramework.StartsWith("netcoreapp"))
{
Assert.Inconclusive("Mstest v1 tests not supported with .Netcore runner.");
return;
}

AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();

var arguments = PrepareArguments(
this.testEnvironment.GetTestAsset("MstestV1UnitTestProject.dll"),
this.GetTestAdapterPath(),
string.Empty, this.FrameworkArgValue,
runnerInfo.InIsolationValue, resultsDirectory: resultsDir);
arguments = string.Concat(arguments, " /listtests /TestCaseFilter:\"(TestCategory!=CategoryA&Priority!=3)\"");

this.InvokeVsTest(arguments);
var listOfTests = new string[] {"MstestV1UnitTestProject.UnitTest1.PassingTest1", "MstestV1UnitTestProject.UnitTest1.PassingTest2",
"MstestV1UnitTestProject.UnitTest1.FailingTest2", "MstestV1UnitTestProject.UnitTest1.SkippingTest" };
var listOfNotDiscoveredTests = new string[] { "MstestV1UnitTestProject.UnitTest1.FailingTest1" };
this.ValidateDiscoveredTests(listOfTests);
this.ValidateTestsNotDiscovered(listOfNotDiscoveredTests);
TryRemoveDirectory(resultsDir);
}

/// <summary>
/// Discover tests using tmi adapter with test case filters.
/// </summary>
[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource]
public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo)
{
if (runnerInfo.RunnerFramework.StartsWith("netcoreapp"))
{
Assert.Inconclusive("Tmi tests not supported with .Netcore runner.");
return;
}

AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();

string testAssemblyPath = this.testEnvironment.GetTestAsset("MstestV1UnitTestProject.dll");
var arguments = PrepareArguments(
testAssemblyPath,
this.GetTestAdapterPath(),
string.Empty, this.FrameworkArgValue,
runnerInfo.InIsolationValue, resultsDirectory: resultsDir);
string testSettingsPath = Path.Combine(Path.GetDirectoryName(testAssemblyPath), "MstestV1UnitTestProjectTestSettings.testsettings");
arguments = string.Concat(arguments, " /listtests /TestCaseFilter:PassingTest /settings:", testSettingsPath);

this.InvokeVsTest(arguments);
var listOfTests = new string[] { "MstestV1UnitTestProject.UnitTest1.PassingTest1", "MstestV1UnitTestProject.UnitTest1.PassingTest2" };
var listOfNotDiscoveredTests = new string[] { "MstestV1UnitTestProject.UnitTest1.FailingTest1", "MstestV1UnitTestProject.UnitTest1.FailingTest2", "MstestV1UnitTestProject.UnitTest1.SkippingTest" };
this.ValidateDiscoveredTests(listOfTests);
this.ValidateTestsNotDiscovered(listOfNotDiscoveredTests);
TryRemoveDirectory(resultsDir);
}
}
}
1 change: 0 additions & 1 deletion test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ public void CleanUp()

[TestMethod]
[TestCategory("Windows-Review")]
[Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
[NetFullTargetFrameworkDataSource(useCoreRunner: false)]
[NetCoreTargetFrameworkDataSource(useCoreRunner: false)]
public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo)
Expand Down
33 changes: 0 additions & 33 deletions ...Platform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,39 +104,6 @@ public void RunTestsWithRunSettingsWithParallel(RunnerInfo runnerInfo)
$"Number of '{ string.Join(", ", testHostNames) }' process created, expected: {expectedNumOfProcessCreated} actual: {numOfProcessCreatedTask.Result.Count} ({ string.Join(", ", numOfProcessCreatedTask.Result) })");
}

[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
public void RunTestsWithTestSettings(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
this.ExecuteNotSupportedRunnerFrameworkTests(runnerInfo.RunnerFramework, Netcoreapp, Message);
this.Setup();

var testsettingsFile = Path.Combine(Path.GetTempPath(), "tempsettings.testsettings");
string testSettingsXml = @"<?xml version=""1.0"" encoding=""utf-8""?><TestSettings></TestSettings>";

File.WriteAllText(testsettingsFile, testSettingsXml, Encoding.UTF8);
var runSettings = $"<RunSettings><RunConfiguration><TargetFrameworkVersion>{FrameworkArgValue}</TargetFrameworkVersion></RunConfiguration><MSTest><SettingsFile>" + testsettingsFile + "</SettingsFile></MSTest></RunSettings>";
var sources = new List<string>
{
this.GetAssetFullPath("MstestV1UnitTestProject.dll")
};

this.vstestConsoleWrapper.RunTests(
sources,
runSettings,
this.runEventHandler);

// Assert
Assert.AreEqual(5, this.runEventHandler.TestResults.Count);
Assert.AreEqual(2, this.runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed));
Assert.AreEqual(2, this.runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed));
Assert.AreEqual(1, this.runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped));

File.Delete(testsettingsFile);
}

[TestMethod]
[NetFullTargetFrameworkDataSource]
[NetCoreTargetFrameworkDataSource]
Expand Down