Move xUnit tests in new folder (#8356)

## Motivation I have a PR where there are many new xUnit tests. It would also be useful to create new xUnit tests for public APIs. The number of xUnit tests will increase and their ordering is required. ## PR Summary - Move C# xUnit tests in new folder. This allows to put new xUnit tests in directory structure in accordance with directory structure where cs files are. - Use an xUnit TestCaseOrderer attribute to sequentially process tests for `powershell.config.json`. - Update README.md - A race condition was fixed which allowed to run all XUnit tests in single batch job.
PowerShell · Dec 13, 2018 · a9f106d · a9f106d
1 parent cd8319a
commit a9f106d
Show file tree

Hide file tree

Showing 20 changed files with 122 additions and 65 deletions.
diff --git a/build.psm1 b/build.psm1
@@ -1369,7 +1369,6 @@ function Test-PSPesterResults
 
 function Start-PSxUnit {
     [CmdletBinding()]param(
-        [string] $SequentialTestResultsFile = "SequentialXUnitResults.xml",
         [string] $ParallelTestResultsFile = "ParallelXUnitResults.xml"
     )
 
@@ -1382,10 +1381,9 @@ function Start-PSxUnit {
     }
 
     try {
-        Push-Location $PSScriptRoot/test/csharp
+        Push-Location $PSScriptRoot/test/xUnit
 
         # Path manipulation to obtain test project output directory
-        dotnet restore
 
         if(-not $Environment.IsWindows)
         {
@@ -1417,30 +1415,22 @@ function Start-PSxUnit {
             }
         }
 
-        # Run sequential tests first, and then run the tests that can execute in parallel
-        if (Test-Path $SequentialTestResultsFile) {
-            Remove-Item $SequentialTestResultsFile -Force -ErrorAction SilentlyContinue
-        }
-        dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Sequential -p:ParallelizeTestCollections=false --test-adapter-path:. "--logger:xunit;LogFilePath=$SequentialTestResultsFile"
-        Publish-TestResults -Path $SequentialTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential'
+        dotnet build --configuration $Options.configuration 
 
-        $extraParams = @()
+        if (Test-Path $ParallelTestResultsFile) {
+            Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue
+        }
 
         # we are having intermittent issues on macOS with these tests failing.
         # VSTS has suggested forcing them to be sequential
         if($env:TF_BUILD -and $IsMacOS)
         {
             Write-Log 'Forcing parallel xunit tests to run sequentially.'
-            $extraParams += @(
-                '-parallel'
-                'none'
-            )
+            dotnet test -p:ParallelizeTestCollections=false --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile"
+        } else {
+            dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile"
         }
 
-        if (Test-Path $ParallelTestResultsFile) {
-            Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue
-        }
-        dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Parallel --no-build  --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile"
         Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel'
     }
     finally {

diff --git a/test/README.md b/test/README.md
@@ -4,8 +4,7 @@ Testing
 The tests are organized by testing language. Thus Pester tests, which
 are written in the PowerShell language, are in
 [./powershell](./powershell) and xUnit tests, written in C#, are in
-[./csharp](./csharp). The sanity tests for the Full .NET build of
-PowerShell are in [./fullclr](./fullclr), and the third-party
+[./xUnit](./xUnit). The third-party
 [shebang][] test is in [./shebang](./shebang).
 
 [shebang]: https://en.wikipedia.org/wiki/Shebang_(Unix)
diff --git a/test/csharp/README.md b/test/csharp/README.md
diff --git a/test/xUnit/Asserts/PriorityAttribute.cs b/test/xUnit/Asserts/PriorityAttribute.cs
@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public class TestPriorityAttribute : Attribute
+{
+    public TestPriorityAttribute(int priority)
+    {
+        Priority = priority;
+    }
+
+    public int Priority { get; private set; }
+}
diff --git a/test/xUnit/Asserts/PriorityOrderer.cs b/test/xUnit/Asserts/PriorityOrderer.cs
@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestOrder.TestCaseOrdering
+{
+    public class PriorityOrderer : ITestCaseOrderer
+    {
+        public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases) where TTestCase : ITestCase
+        {
+            var sortedMethods = new SortedDictionary<int, List<TTestCase>>();
+
+            foreach (TTestCase testCase in testCases)
+            {
+                int priority = 0;
+
+                foreach (IAttributeInfo attr in testCase.TestMethod.Method.GetCustomAttributes((typeof(TestPriorityAttribute).AssemblyQualifiedName)))
+                    priority = attr.GetNamedArgument<int>("Priority");
+
+                GetOrCreate(sortedMethods, priority).Add(testCase);
+            }
+
+            foreach (var list in sortedMethods.Keys.Select(priority => sortedMethods[priority]))
+            {
+                list.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.TestMethod.Method.Name, y.TestMethod.Method.Name));
+                foreach (TTestCase testCase in list)
+                    yield return testCase;
+            }
+        }
+
+        static TValue GetOrCreate<TKey, TValue>(IDictionary<TKey, TValue> dictionary, TKey key) where TValue : new()
+        {
+            TValue result;
+
+            if (dictionary.TryGetValue(key, out result)) return result;
+
+            result = new TValue();
+            dictionary[key] = result;
+
+            return result;
+        }
+    }
+}
diff --git a/test/xUnit/README.md b/test/xUnit/README.md
@@ -0,0 +1,29 @@
+# xUnit Tests
+
+The folder contains xUnit tests for PowerShell Core project.
+
+## Running xUnit Tests
+
+Go to the top level of the PowerShell repository and run full set of tests:
+`Start-PSxUnit` inside a self-hosted copy of PowerShell.
+
+Go to the test project folder and run `dotnet test -c Release`.
+
+Use [`filter`](xunit-filter) parameter to run only needed tests:
+```powershell
+dotnet test -c Release --filter "FullyQualifiedName~UnitTest1   # Runs tests which have UnitTest1 in FullyQualifiedName
+dotnet test --filter Name~TestMethod1   # Runs tests whose name contains TestMethod1
+```
+
+## Creating xUnit Tests
+
+Keep the folder structure that is for Pester [../../test/powershell](../../test/powershell) and C# files [../../src](../../src).
+
+Use namespace names started with `PSTests`.
+```c#
+namespace PSTests.YourNameSpace
+{
+}
+```
+
+[xunit-filter]: https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests
diff --git a/test/csharp/test_Binders.cs → test/xUnit/csharp/test_Binders.cs b/test/csharp/test_Binders.cs → test/xUnit/csharp/test_Binders.cs
diff --git a/test/csharp/test_CorePsPlatform.cs → test/xUnit/csharp/test_CorePsPlatform.cs b/test/csharp/test_CorePsPlatform.cs → test/xUnit/csharp/test_CorePsPlatform.cs
diff --git a/test/csharp/test_ExtensionMethods.cs → test/xUnit/csharp/test_ExtensionMethods.cs b/test/csharp/test_ExtensionMethods.cs → test/xUnit/csharp/test_ExtensionMethods.cs
diff --git a/test/csharp/test_FileSystemProvider.cs → test/xUnit/csharp/test_FileSystemProvider.cs b/test/csharp/test_FileSystemProvider.cs → test/xUnit/csharp/test_FileSystemProvider.cs
diff --git a/test/csharp/test_MshSnapinInfo.cs → test/xUnit/csharp/test_MshSnapinInfo.cs b/test/csharp/test_MshSnapinInfo.cs → test/xUnit/csharp/test_MshSnapinInfo.cs
diff --git a/test/csharp/test_PSConfiguration.cs → test/xUnit/csharp/test_PSConfiguration.cs b/test/csharp/test_PSConfiguration.cs → test/xUnit/csharp/test_PSConfiguration.cs
@@ -12,6 +12,7 @@
 
 namespace PSTests.Sequential
 {
+    [TestCaseOrderer("TestOrder.TestCaseOrdering.PriorityOrderer", "powershell-tests")]
     public class PowerShellPolicyFixture : IDisposable
     {
         private const string configFileName = "powershell.config.json";
@@ -344,7 +345,7 @@ public PowerShellPolicyTests(PowerShellPolicyFixture fixture)
             this.fixture = fixture;
         }
 
-        [Fact]
+        [Fact, TestPriority(1)]
         public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotEmpty()
         {
             fixture.SetupConfigFile1();
@@ -358,7 +359,7 @@ public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotEmpty()
             fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(2)]
         public void PowerShellConfig_GetPowerShellPolicies_EmptyUserConfig()
         {
             fixture.SetupConfigFile2();
@@ -371,7 +372,7 @@ public void PowerShellConfig_GetPowerShellPolicies_EmptyUserConfig()
             fixture.CompareTwoPolicies(sysPolicies, fixture.SystemWidePolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(3)]
         public void PowerShellConfig_GetPowerShellPolicies_EmptySystemConfig()
         {
             fixture.SetupConfigFile3();
@@ -384,7 +385,7 @@ public void PowerShellConfig_GetPowerShellPolicies_EmptySystemConfig()
             fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(4)]
         public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesEmpty()
         {
             fixture.SetupConfigFile4();
@@ -395,7 +396,7 @@ public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesEmpty()
             Assert.Null(userPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(5)]
         public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotExist()
         {
             fixture.CleanupConfigFiles();
@@ -406,7 +407,7 @@ public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotExist()
             Assert.Null(userPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(6)]
         public void Utils_GetPolicySetting_BothConfigFilesNotEmpty()
         {
             fixture.SetupConfigFile1();
@@ -504,7 +505,7 @@ public void Utils_GetPolicySetting_BothConfigFilesNotEmpty()
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration);
         }
 
-        [Fact]
+        [Fact, TestPriority(7)]
         public void Utils_GetPolicySetting_EmptyUserConfig()
         {
             fixture.SetupConfigFile2();
@@ -602,7 +603,7 @@ public void Utils_GetPolicySetting_EmptyUserConfig()
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration);
         }
 
-        [Fact]
+        [Fact, TestPriority(8)]
         public void Utils_GetPolicySetting_EmptySystemConfig()
         {
             fixture.SetupConfigFile3();
@@ -701,7 +702,7 @@ public void Utils_GetPolicySetting_EmptySystemConfig()
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null);
         }
 
-        [Fact]
+        [Fact, TestPriority(9)]
         public void Utils_GetPolicySetting_BothConfigFilesEmpty()
         {
             fixture.SetupConfigFile4();
@@ -800,7 +801,7 @@ public void Utils_GetPolicySetting_BothConfigFilesEmpty()
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null);
         }
 
-        [Fact]
+        [Fact, TestPriority(10)]
         public void Utils_GetPolicySetting_BothConfigFilesNotExist()
         {
             fixture.CleanupConfigFiles();

diff --git a/test/csharp/test_PSVersionInfo.cs → test/xUnit/csharp/test_PSVersionInfo.cs b/test/csharp/test_PSVersionInfo.cs → test/xUnit/csharp/test_PSVersionInfo.cs
diff --git a/test/csharp/test_Runspace.cs → test/xUnit/csharp/test_Runspace.cs b/test/csharp/test_Runspace.cs → test/xUnit/csharp/test_Runspace.cs
@@ -66,7 +66,8 @@ public void TestRunspaceWithPowerShell()
         [Fact]
         public void TestRunspaceWithPowerShellAndInitialSessionState()
         {
-            InitialSessionState iss = InitialSessionState.CreateDefault2();
+            // CreateDefault2 is intentional.
+            InitialSessionState iss = InitialSessionState.CreateDefault();
 
             // NOTE: instantiate custom host myHost for the next line to capture stdout and stderr output
             //       in addition to just the PSObjects
@@ -90,9 +91,11 @@ public void TestRunspaceWithPowerShellAndInitialSessionState()
                         ++objCount;
                         Assert.NotNull(result);
                     }
+
                     Assert.Equal(count, objCount);
-                    powerShell.Dispose();
                 }
+
+                runspace.Close();
             }
         }
     }

diff --git a/test/csharp/test_SecuritySupport.cs → test/xUnit/csharp/test_SecuritySupport.cs b/test/csharp/test_SecuritySupport.cs → test/xUnit/csharp/test_SecuritySupport.cs
diff --git a/test/csharp/test_SessionState.cs → test/xUnit/csharp/test_SessionState.cs b/test/csharp/test_SessionState.cs → test/xUnit/csharp/test_SessionState.cs
diff --git a/test/csharp/test_Utils.cs → test/xUnit/csharp/test_Utils.cs b/test/csharp/test_Utils.cs → test/xUnit/csharp/test_Utils.cs
diff --git a/test/csharp/csharp.tests.csproj → test/xUnit/xUnit.tests.csproj b/test/csharp/csharp.tests.csproj → test/xUnit/xUnit.tests.csproj
@@ -5,6 +5,7 @@
   <PropertyGroup>
     <Description>PowerShell xUnit Tests</Description>
     <AssemblyName>powershell-tests</AssemblyName>
+    <GenerateProgramFile>true</GenerateProgramFile>
     <RuntimeIdentifiers>win7-x86;win7-x64;osx-x64;linux-x64</RuntimeIdentifiers>
   </PropertyGroup>
 
@@ -26,5 +27,4 @@
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
     <PackageReference Include="XunitXml.TestLogger" Version="2.0.0" />
   </ItemGroup>
-
 </Project>
diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1
@@ -369,7 +369,6 @@ function Invoke-AppVeyorTest
     Write-Host -Foreground Green 'Run CoreCLR tests'
     $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml"
     $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml"
-    $SequentialXUnitTestResultsFile = "$pwd\SequentialXUnitTestResults.xml"
     $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml"
     if(!(Test-Path "$env:CoreOutput\pwsh.exe"))
     {
@@ -443,19 +442,13 @@ function Invoke-AppVeyorTest
         Write-Host -Foreground Green 'Upload CoreCLR Admin test results'
         Update-AppVeyorTestResults -resultsFile $testResultsAdminFile
 
-        Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile
+        Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile
         Write-Host -ForegroundColor Green 'Uploading PSxUnit test results'
-        Update-AppVeyorTestResults -resultsFile $SequentialXUnitTestResultsFile
         Update-AppVeyorTestResults -resultsFile $ParallelXUnitTestResultsFile
 
         # Fail the build, if tests failed
         Test-PSPesterResults -TestResultsFile $testResultsAdminFile
-        @(
-            $SequentialXUnitTestResultsFile,
-            $ParallelXUnitTestResultsFile
-        ) | ForEach-Object {
-            Test-XUnitTestResults -TestResultsFile $_
-        }
+        Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
 
         # Run tests with specified experimental features enabled
         foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {

diff --git a/tools/travis.ps1 b/tools/travis.ps1
@@ -330,12 +330,11 @@ elseif($Stage -eq 'Build')
     }
 
     try {
-        $SequentialXUnitTestResultsFile = "$pwd/SequentialXUnitTestResults.xml"
         $ParallelXUnitTestResultsFile = "$pwd/ParallelXUnitTestResults.xml"
 
-        Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile
+        Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile
         # If there are failures, Test-XUnitTestResults throws
-        $SequentialXUnitTestResultsFile, $ParallelXUnitTestResultsFile | ForEach-Object { Test-XUnitTestResults -TestResultsFile $_ }
+        Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
     }
     catch {
         $result = "FAIL"