Update test framework and tests to support 4x version of Pester (#6064)

* Test fixes and changes needed to support Pester 4.0.8

* Replace 'Should Contain' with new 'Should FileContentMatch' assertion

Explicitly check for string creation with write-output

* Use the current version of pester and install it in modulesDir

* Simplify logic for relative path test.

Multiple '..' is not needed for a relative path, a single one will do. Also, on multi-drive systems using split-path -noqualifier will probably do the wrong thing with regard to constructing a correct path.
Remove extraneous Should Not Throw test, if this throws, the test will fail, we don't need to explicitly assert the not throw

* In some environments it is possible that computer name is 'localhost', so that should be allowed

* [feature] Add link for migrating tests from Pester v3 to v4

Fix up capitalization and white space issues
Change one test to check FullyQualifiedErrorId rather than just `Should Throw`

* [feature] update invoke-item test to handle the case where multiple notepad processes are running

* Fix spelling issue with Pester 4x, calling it Pester 4 should be sufficient

build.psm1

@@ -662,7 +662,8 @@ function Restore-PSModuleToBuild
 
     if($CI.IsPresent)
     {
-        Restore-PSPester -Destination $modulesDir
+        # take the latest version of pester and install it so it may be used
+        Save-Module -Path $modulesDir -Name Pester
     }
 }
 
@@ -987,7 +988,7 @@ function Start-PSPester {
         Write-Warning @"
 Pester module not found.
 Restore the module to '$Pester' by running:
-    Restore-PSPester
+    Save-Module Pester -Path '$binDir/Modules'
 "@
         return;
     }

docs/testing-guidelines/WritingPesterTests.md

@@ -1,7 +1,11 @@
 ### Writing Pester Tests
-Note that this document does not replace the documents found in the [Pester](https://github.com/pester/pester "Pester") project. This is just
-some quick tips and suggestions for creating Pester tests for this project. The Pester community is vibrant and active, if you have questions
-about Pester or creating tests, the [Pester Wiki](https://github.com/pester/pester/wiki) has a lot of great information.
+Note that this document does not replace the documents found in the [Pester](https://github.com/pester/pester "Pester") project.
+This is just some quick tips and suggestions for creating Pester tests for this project.
+The Pester community is vibrant and active, if you have questions about Pester or creating tests, the [Pester Wiki](https://github.com/pester/pester/wiki) has a lot of great information.
+As of January 2018, PowerShell Core is using Pester version 4 which has some changes from earlier versions.
+See [Migrating from Pester 3 to Pester 4](https://github.com/pester/Pester/wiki/Migrating-from-Pester-3-to-Pester-4) for more information.
+
+
 
 When creating tests, keep the following in mind:
 * Tests should not be overly complicated and test too many things
@@ -16,7 +20,7 @@ Here's the simplest of tests
 Describe "A variable can be assigned and retrieved" {
     It "Create a variable and make sure its value is correct" {
        $a = 1
-       $a | Should be 1
+       $a | Should Be 1
    }
 }
 ```
@@ -27,7 +31,7 @@ If you need to do type checking, that can be done as well
 Describe "One is really one" {
     It "Compare 1 to 1" {
        $a = 1
-       $a | Should be 1
+       $a | Should Be 1
     }
     It "1 is really an int" {
        $i = 1
@@ -42,7 +46,7 @@ alternatively, you could do the following:
 Describe "One is really one" {
     It "Compare 1 to 1" {
        $a = 1
-       $a | Should be 1
+       $a | Should Be 1
     }
     It "1 is really an int" {
        $i = 1
@@ -136,7 +140,7 @@ $testCases = @(
 Describe "A test" {
     It "<a> -xor <b> should be <expectedresult>" -testcase $testcases {
         param ($a, $b, $ExpectedResult)
-        $a -xor $b | Should be $ExpectedResult
+        $a -xor $b | Should Be $ExpectedResult
     }
 }
 ```
@@ -151,7 +155,7 @@ The following example illustrates simple use:
 Context "Get-Random is not random" {
         Mock Get-Random { return 3 }
         It "Get-Random returns 3" {
-            Get-Random | Should be 3
+            Get-Random | Should Be 3
         }
     }
 ```

test/powershell/Modules/Microsoft.PowerShell.Core/Import-Module.Tests.ps1

@@ -42,7 +42,7 @@
 
     It "should only load the specified version" {
         Import-Module TestModule -RequiredVersion 1.1
-        (Get-Module TestModule).Version | Should Be "1.1"
+        (Get-Module TestModule).Version | Should BeIn "1.1"
     }
 }
 
@@ -192,7 +192,7 @@ Describe "Import-Module should be case insensitive" -Tags 'CI' {
         Import-Module testMODULE
         $m = Get-Module TESTmodule
         $m | Should BeOfType "System.Management.Automation.PSModuleInfo"
-        $m.Name | Should Be "TESTMODULE"
+        $m.Name | Should BeIn "TESTMODULE"
         mytest | Should BeExactly "hello"
         Remove-Module TestModule
         Get-Module tESTmODULE | Should BeNullOrEmpty

test/powershell/Modules/Microsoft.PowerShell.Core/Out-Default.Tests.ps1

@@ -15,7 +15,7 @@ Describe "Out-Default Tests" -tag CI {
 "@
 
         & $powershell -noprofile -command $script | Should BeExactly 'bye'
-        "TestDrive:\transcript.txt" | Should Contain 'hello'
+        "TestDrive:\transcript.txt" | Should FileContentMatch 'hello'
     }
 
     It "Out-Default reverts transcription state when used more than once in a pipeline" {

test/powershell/Modules/Microsoft.PowerShell.Management/Clear-Content.Tests.ps1

@@ -64,7 +64,7 @@ Describe "Clear-Content cmdlet tests" -Tags "CI" {
     # we could suppress the WhatIf output here if we use the testhost, but it's not necessary
     It "The filesystem provider supports should process" -skip:(!$IsWindows) {
       clear-content TESTDRIVE:\$file2 -WhatIf
-      "TESTDRIVE:\$file2" | should contain "This is content"
+      "TESTDRIVE:\$file2" | should FileContentMatch "This is content"
     }
 
     It "The filesystem provider should support ShouldProcess (reference ProviderSupportsShouldProcess member)" {

test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1

@@ -378,8 +378,7 @@ Describe "Handling of globbing patterns" -Tags "CI" {
         BeforeEach {
             $file = New-Item -ItemType File -Path $filePath -Force
         }
-        AfterEach
-        {
+        AfterEach {
             Remove-Item -Force -Recurse -Path $dirPath -ErrorAction SilentlyContinue
             Remove-Item -Force -LiteralPath $newPath -ErrorAction SilentlyContinue
         }
@@ -1392,7 +1391,7 @@ Describe "UNC paths" -Tags 'CI' {
             $testPath = Join-Path "\\localhost" $systemDrive
             & $cmdlet $testPath
             Get-Location | Should BeExactly "Microsoft.PowerShell.Core\FileSystem::$testPath"
-            $children = { Get-ChildItem -ErrorAction Stop } | Should Not Throw
+            $children = Get-ChildItem -ErrorAction Stop
             $children.Count | Should BeGreaterThan 0
         }
         finally {

test/powershell/Modules/Microsoft.PowerShell.Management/Move-Item.Tests.ps1

@@ -9,7 +9,7 @@ Describe "Move-Item tests" -Tag "CI" {
         Move-Item $source $target
         test-path $source | Should be $false
         test-path $target | Should be $true
-        "$target" | Should ContainExactly "This is content"
+        "$target" | Should FileContentMatchExactly "This is content"
     }
 
     Context "Move-Item with filters" {
@@ -49,7 +49,7 @@ Describe "Move-Item tests" -Tag "CI" {
             $newBarPath | Should Exist
             $booPath | Should Exist
             $fooPath | Should Exist
-            $newBarPath | Should ContainExactly $barContent
+            $newBarPath | Should FileContentMatchExactly $barContent
         }
         It "Can move to different directory, filtered with -Exclude" {
             Move-Item -Path $filePath -Destination $moveToPath -Exclude "b*" -ErrorVariable e -ErrorAction SilentlyContinue
@@ -58,7 +58,7 @@ Describe "Move-Item tests" -Tag "CI" {
             $newFooPath | Should Exist
             $booPath | Should Exist
             $barPath | Should Exist
-            $newFooPath | Should ContainExactly $fooContent
+            $newFooPath | Should FileContentMatchExactly $fooContent
         }
         It "Can move to different directory, filtered with -Filter" {
             Move-Item -Path $filePath -Destination $moveToPath -Filter "bo*" -ErrorVariable e -ErrorAction SilentlyContinue
@@ -67,7 +67,7 @@ Describe "Move-Item tests" -Tag "CI" {
             $newBooPath | Should Exist
             $barPath | Should Exist
             $fooPath | Should Exist
-            $newBooPath | Should ContainExactly $booContent
+            $newBooPath | Should FileContentMatchExactly $booContent
         }
 
         It "Can rename via move, filtered with -Include" {
@@ -77,7 +77,7 @@ Describe "Move-Item tests" -Tag "CI" {
             $barPath | Should Not Exist
             $booPath | Should Exist
             $fooPath | Should Exist
-            $renameToPath | Should ContainExactly $barContent
+            $renameToPath | Should FileContentMatchExactly $barContent
         }
         It "Can rename via move, filtered with -Exclude" {
             Move-Item -Path $filePath -Destination $renameToPath -Exclude "b*" -ErrorVariable e -ErrorAction SilentlyContinue
@@ -86,7 +86,7 @@ Describe "Move-Item tests" -Tag "CI" {
             $fooPath | Should Not Exist
             $booPath | Should Exist
             $barPath | Should Exist
-            $renameToPath | Should ContainExactly $fooContent
+            $renameToPath | Should FileContentMatchExactly $fooContent
         }
         It "Can rename via move, filtered with -Filter" {
             Move-Item -Path $filePath -Destination $renameToPath -Filter "bo*" -ErrorVariable e -ErrorAction SilentlyContinue
@@ -95,7 +95,7 @@ Describe "Move-Item tests" -Tag "CI" {
             $booPath | Should Not Exist
             $fooPath | Should Exist
             $barPath | Should Exist
-            $renameToPath | Should ContainExactly $booContent
+            $renameToPath | Should FileContentMatchExactly $booContent
         }
     }
 }

test/powershell/Modules/Microsoft.PowerShell.Management/Rename-Item.Tests.ps1

@@ -9,6 +9,6 @@ Describe "Rename-Item tests" -Tag "CI" {
         Rename-Item $source $target
         test-path $source | Should be $false
         test-path $target | Should be $true
-        "$target" | Should ContainExactly "This is content"
+        "$target" | Should FileContentMatchExactly "This is content"
     }
 }

test/powershell/Modules/Microsoft.PowerShell.Management/TimeZone.Tests.ps1

@@ -63,15 +63,15 @@ Describe "Get-Timezone test cases" -Tags "CI" {
     It "Call with ListAvailable switch returns a list containing TimeZoneInfo.Local" {
         $observedIdList = Get-TimeZone -ListAvailable | Select-Object -ExpandProperty BaseUtcOffset
         $oneExpectedOffset = ([System.TimeZoneInfo]::Local).BaseUtcOffset
-        $observedIdList -eq $oneExpectedOffset | Should Be $oneExpectedOffset
+        $oneExpectedOffset | Should BeIn $observedIdList
     }
 
     ## The local time zone could be set to UTC or GMT*. In this case, the .NET API returns the region ID
     ## and not UTC. To avoid a string matching error, we compare the BaseUtcOffset instead.
     It "Call with ListAvailable switch returns a list containing one returned by Get-TimeZone" {
         $observedIdList = Get-TimeZone -ListAvailable | Select-Object -ExpandProperty BaseUtcOffset
         $oneExpectedOffset = (Get-TimeZone).BaseUtcOffset
-        $observedIdList -eq $oneExpectedOffset | Should Be $oneExpectedOffset
+        $oneExpectedOffset | Should BeIn $observedIdList
     }
 
     It "Call Get-TimeZone using ID param and single item" {

test/powershell/Modules/Microsoft.PowerShell.Utility/ConvertFrom-StringData.Tests.ps1

@@ -65,8 +65,9 @@ bazz = 2
     It "Should work for multiple lines" {
 	{ ConvertFrom-StringData -StringData $sampleData } | Should Not Throw
 
-	$(ConvertFrom-StringData -StringData $sampleData).Keys   | Should Be "foo", "bar", "bazz"
+    # keys are not order guaranteed
+	$(ConvertFrom-StringData -StringData $sampleData).Keys   | Should BeIn @("foo", "bar", "bazz")
 
-	$(ConvertFrom-StringData -StringData $sampleData).Values | Should Be "0","1","2"
+	$(ConvertFrom-StringData -StringData $sampleData).Values | Should BeIn @("0","1","2")
     }
 }

test/powershell/Modules/Microsoft.PowerShell.Utility/ConvertTo-Html.Tests.ps1

@@ -138,7 +138,9 @@ After the object
 
     It "Test ConvertTo-HTML meta with invalid properties should throw warning" {
         $parms = @{"authors"="John Doe";"keywords"="PowerShell,PSv6"}
-        ($customObject | ConvertTo-HTML -Meta $parms 3>&1) -match $parms["authors"]  | Should Be $true
+        # make this a string, rather than an array of string so match will behave
+        [string]$observedProperties = $customObject | ConvertTo-HTML -Meta $parms 3>&1
+        $observedProperties | Should Match $parms["authors"]
     }
 
     It "Test ConvertTo-HTML charset"{

test/powershell/Modules/Microsoft.PowerShell.Utility/Export-Alias.Tests.ps1

@@ -63,46 +63,46 @@ Describe "Export-Alias DRT Unit Tests" -Tags "CI" {
 
 	It "Export-Alias for Default"{
 		Export-Alias $fulltestpath abcd01 -passthru
-		$fulltestpath| Should ContainExactly '"abcd01","efgh01","","None"'
+		$fulltestpath| Should FileContentMatchExactly '"abcd01","efgh01","","None"'
     }
 
 	It "Export-Alias As CSV"{
 		Export-Alias $fulltestpath abcd01 -As CSV -passthru
-		$fulltestpath| Should ContainExactly '"abcd01","efgh01","","None"'
+		$fulltestpath| Should FileContentMatchExactly '"abcd01","efgh01","","None"'
     }
 
 	It "Export-Alias As CSV With Description"{
 		Export-Alias $fulltestpath abcd01 -As CSV -description "My Aliases" -passthru
-		$fulltestpath| Should ContainExactly '"abcd01","efgh01","","None"'
-		$fulltestpath| Should ContainExactly "My Aliases"
+		$fulltestpath| Should FileContentMatchExactly '"abcd01","efgh01","","None"'
+		$fulltestpath| Should FileContentMatchExactly "My Aliases"
     }
 
 	It "Export-Alias As CSV With Multiline Description"{
 		Export-Alias $fulltestpath abcd01 -As CSV -description "My Aliases\nYour Aliases\nEveryones Aliases" -passthru
-		$fulltestpath| Should ContainExactly '"abcd01","efgh01","","None"'
-		$fulltestpath| Should ContainExactly "My Aliases"
-		$fulltestpath| Should ContainExactly "Your Aliases"
-		$fulltestpath| Should ContainExactly "Everyones Aliases"
+		$fulltestpath| Should FileContentMatchExactly '"abcd01","efgh01","","None"'
+		$fulltestpath| Should FileContentMatchExactly "My Aliases"
+		$fulltestpath| Should FileContentMatchExactly "Your Aliases"
+		$fulltestpath| Should FileContentMatchExactly "Everyones Aliases"
     }
 
 	It "Export-Alias As Script"{
 		Export-Alias $fulltestpath abcd01 -As Script -passthru
-		$fulltestpath| Should ContainExactly 'set-alias -Name:"abcd01" -Value:"efgh01" -Description:"" -Option:"None"'
+		$fulltestpath| Should FileContentMatchExactly 'set-alias -Name:"abcd01" -Value:"efgh01" -Description:"" -Option:"None"'
     }
 
 	It "Export-Alias As Script With Multiline Description"{
 		Export-Alias $fulltestpath abcd01 -As Script -description "My Aliases\nYour Aliases\nEveryones Aliases" -passthru
-		$fulltestpath| Should ContainExactly 'set-alias -Name:"abcd01" -Value:"efgh01" -Description:"" -Option:"None"'
-		$fulltestpath| Should ContainExactly "My Aliases"
-		$fulltestpath| Should ContainExactly "Your Aliases"
-		$fulltestpath| Should ContainExactly "Everyones Aliases"
+		$fulltestpath| Should FileContentMatchExactly 'set-alias -Name:"abcd01" -Value:"efgh01" -Description:"" -Option:"None"'
+		$fulltestpath| Should FileContentMatchExactly "My Aliases"
+		$fulltestpath| Should FileContentMatchExactly "Your Aliases"
+		$fulltestpath| Should FileContentMatchExactly "Everyones Aliases"
     }
 
 	It "Export-Alias for Force Test"{
 		Export-Alias $fulltestpath abcd01
 		Export-Alias $fulltestpath abcd02 -force
-		$fulltestpath| Should Not ContainExactly '"abcd01","efgh01","","None"'
-		$fulltestpath| Should ContainExactly '"abcd02","efgh02","","None"'
+		$fulltestpath| Should Not FileContentMatchExactly '"abcd01","efgh01","","None"'
+		$fulltestpath| Should FileContentMatchExactly '"abcd02","efgh02","","None"'
     }
 
 	It "Export-Alias for Force ReadOnly Test" {
@@ -124,9 +124,9 @@ Describe "Export-Alias DRT Unit Tests" -Tags "CI" {
 			$_.FullyQualifiedErrorId | Should be "FileOpenFailure,Microsoft.PowerShell.Commands.ExportAliasCommand"
 		}
 		Export-Alias $fulltestpath abcd03 -force
-		$fulltestpath| Should Not ContainExactly '"abcd01","efgh01","","None"'
-		$fulltestpath| Should Not ContainExactly '"abcd02","efgh02","","None"'
-		$fulltestpath| Should ContainExactly '"abcd03","efgh03","","None"'
+		$fulltestpath| Should Not FileContentMatchExactly '"abcd01","efgh01","","None"'
+		$fulltestpath| Should Not FileContentMatchExactly '"abcd02","efgh02","","None"'
+		$fulltestpath| Should FileContentMatchExactly '"abcd03","efgh03","","None"'
 
 		if ( $IsWindows )
 		{

test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Variable.Tests.ps1

@@ -145,7 +145,8 @@ Describe "Get-Variable" -Tags "CI" {
 	It "Should not be able to clear a global scope variable using the local switch" {
 	    New-Variable globalVar -Value 1 -Scope global -Force
 
-	    Get-Variable -Name globalVar -Scope local -ErrorAction SilentlyContinue | Should Throw
+	    Get-Variable -Name globalVar -Scope local -ErrorAction SilentlyContinue -ErrorVariable removeGlobalAsLocal
+	    $removeGlobalAsLocal.FullyQualifiedErrorId | Should Be "VariableNotFound,Microsoft.PowerShell.Commands.GetVariableCommand"
 	}
 
 	It "Should be able to get a global variable when there's one in the script scope" {

test/powershell/Modules/Microsoft.PowerShell.Utility/Invoke-Item.Tests.ps1

@@ -67,7 +67,8 @@ Describe "Invoke-Item basic tests" -Tags "Feature" {
                 Get-Process -Name $notepadProcessName | Stop-Process -Force
                 Invoke-Item -Path $notepad
                 $notepadProcess = Get-Process -Name $notepadProcessName
-                $notepadProcess.Name | Should Be $notepadProcessName
+                # we need BeIn because multiple notepad processes could be running
+                $notepadProcess.Name | Should BeIn $notepadProcessName
                 Stop-Process -InputObject $notepadProcess
             }
         } else {

test/powershell/Modules/Microsoft.PowerShell.Utility/New-Variable.Tests.ps1

@@ -85,8 +85,9 @@ Describe "New-Variable" -Tags "CI" {
     }
 
     It "Should not be able to set the name of a new variable to that of an old variable within same scope when the Force switch is missing" {
-	New-Variable var1
-	(New-Variable var1 -ErrorAction SilentlyContinue) | Should Throw
+        New-Variable var1
+        New-Variable var1 -ErrorAction SilentlyContinue -ErrorVariable newWhenExists
+        $newWhenExists.FullyQualifiedErrorId | Should Be "VariableAlreadyExists,Microsoft.PowerShell.Commands.NewVariableCommand"
     }
 
     It "Should change the value of an already existing variable using the Force switch" {

test/powershell/Modules/Microsoft.PowerShell.Utility/Out-File.Tests.ps1

@@ -11,8 +11,8 @@ Describe "Out-File DRT Unit Tests" -Tags "CI" {
         $tempFile = Join-Path -Path $TestDrive -ChildPath "outfileAppendTest.txt"
         { 'This is first line.' | out-file $tempFile } | Should Not Throw
         { 'This is second line.' | out-file -append $tempFile } | Should Not Throw
-        $tempFile |Should Contain "first"
-        $tempFile |Should Contain "second"
+        $tempFile |Should FileContentMatch "first"
+        $tempFile |Should FileContentMatch "second"
         Remove-Item $tempFile -Force
     }
 }

test/powershell/Modules/Microsoft.PowerShell.Utility/Remove-Event.Tests.ps1

@@ -12,27 +12,27 @@ Describe "Remove-Event" -Tags "CI" {
 
 	It "Should remove an event given a sourceidentifier" {
 	    { Remove-Event -sourceidentifier PesterTimer }
-	    { Get-Event -ErrorAction SilentlyContinue | Should Not Contain PesterTimer }
+	    { Get-Event -ErrorAction SilentlyContinue | Should Not FileMatchContent PesterTimer }
 	}
 
 	It "Should remove an event given an event identifier" {
 	    { $events = Get-Event -sourceidentifier PesterTimer }
 	    { $events = $events.EventIdentifier }
 	    { Remove-Event -EventIdentifier $events }
 	    { $events = Get-Event -ErrorAction SilentlyContinue}
-	    { $events.SourceIdentifier | Should Not Contain "PesterTimer" }
+	    { $events.SourceIdentifier | Should Not FileMatchContent "PesterTimer" }
 	}
 
 	It "Should be able to remove an event given a pipe from Get-Event" {
 	    { Get-Event -sourceidentifier PesterTimer | Remove-Event }
-	    { Get-Event -ErrorAction SilentlyContinue | Should Not Contain "PesterTimer" }
+	    { Get-Event -ErrorAction SilentlyContinue | Should Not FileMatchContent "PesterTimer" }
 
 	}
 
 	It "Should NOT remove an event given the whatif flag" {
 	    { Remove-Event -sourceidentifier PesterTimer -whatif }
 	    { $events = Get-Event }
-	    { $events.SourceIdentifier  | Should Contain "PesterTimer" }
+	    { $events.SourceIdentifier  | Should FileContentMatch "PesterTimer" }
 	}
     }
 }