From 3df40810c5212e458551fa021ca89abb789de0b6 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Thu, 3 Nov 2016 13:31:52 -0700 Subject: [PATCH 1/7] added get-pspesterfailures and format-pspesterfailure to make it easier to extract the failures --- build.psm1 | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/build.psm1 b/build.psm1 index 544cfd44170..2f60b7e7aef 100644 --- a/build.psm1 +++ b/build.psm1 @@ -627,6 +627,87 @@ function Publish-PSTestTools { } +function Format-PSPesterFailure { + [CmdletBinding(DefaultParameterSetName="file")] + param( + [Parameter(ParameterSetName="file")] + [string]$NUnitLog = "pester-tests.xml", + + [Parameter(ParameterSetName="object",ValueFromPipeline=$true)] + [PSCustomObject]$PesterFailure + ) + + begin + { + if ($PSBoundParameters.Count -eq 0 -or $PSBoundParameters.ContainsKey("NUnitLog")) + { + $PesterFailure = Get-PSPesterFailures -NUnitLog $NUnitLog + } + [int]$totalFailures = 0 + [string]$currentSuiteName = [String]::Empty + } + + process + { + foreach ($Failure in $PesterFailure) + { + if (!$Failure.PSTypeNames.Contains("Pester.Failure")) + { + throw "Only [Pester.Failure] objects supported" + } + + if ($Failure.SuiteName -ne $currentSuiteName) + { + $currentSuiteName = $Failure.SuiteName + Write-Host $currentSuiteName -ForegroundColor Magenta + } + Write-Host "`n`t$($Failure.TestDescription)" -ForegroundColor Green + Write-Host "`t`t$($Failure.FailMessage)" -ForegroundColor Red + Write-Host "`t`t$($Failure.Failstack)" -ForegroundColor Gray + $totalFailures++ + } + } + + end + { + Write-Host "Failures: $totalFailures" -ForegroundColor Yellow + } +} + +function Get-PSPesterFailures { + param( + [string]$NUnitLog = "pester-tests.xml" + ) + + function SelectNodes ($xml, [string] $xpath) + { + if ($psversiontable.PSEdition -eq 'Desktop') + { + $xml.SelectNodes($xpath) + } + else + { + [System.Xml.XmlDocumentXPathExtensions]::SelectNodes($xml, $xpath) + } + } + + $outputxml = [xml](get-content $NUnitLog) + $failureSuites = SelectNodes -xml $outputxml -xpath "/test-results/test-suite/results/test-suite[@result='Failure']" + + foreach ($failureSuite in $failureSuites) + { + $failureTests = SelectNodes -xml $failureSuite -xpath "results/test-case[@result='Failure']" + foreach ($failureTest in $failureTests) + { + $totalFailures++ + $failure = New-Object -TypeName PSCustomObject -Property @{SuiteName=$failureSuite.Name;TestDescription=$failureTest.Description; + FailMessage=$failureTest.failure.message;FailStack=$failuretest.failure.'stack-trace'} + $failure.PSTypeNames.Insert(0,"Pester.Failure") + $failure + } + } +} + function Start-PSPester { [CmdletBinding()] param( From 1cf3d74b079b0f3f3bda5371dfb65b46dd1bc529 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Thu, 3 Nov 2016 13:37:53 -0700 Subject: [PATCH 2/7] fixed formatting --- build.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.psm1 b/build.psm1 index 2f60b7e7aef..f2715b9b5e6 100644 --- a/build.psm1 +++ b/build.psm1 @@ -661,9 +661,9 @@ function Format-PSPesterFailure { $currentSuiteName = $Failure.SuiteName Write-Host $currentSuiteName -ForegroundColor Magenta } - Write-Host "`n`t$($Failure.TestDescription)" -ForegroundColor Green - Write-Host "`t`t$($Failure.FailMessage)" -ForegroundColor Red - Write-Host "`t`t$($Failure.Failstack)" -ForegroundColor Gray + Write-Host "$($Failure.TestDescription)" -ForegroundColor Green + Write-Host "$($Failure.FailMessage)" -ForegroundColor Red + Write-Host "$($Failure.Failstack)" -ForegroundColor Gray $totalFailures++ } } From 97f9765dcff35a19c94c6983cb7c62bd72cc65bf Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Thu, 3 Nov 2016 13:40:30 -0700 Subject: [PATCH 3/7] made cmdlet singular --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index f2715b9b5e6..b6cdc1a8f7a 100644 --- a/build.psm1 +++ b/build.psm1 @@ -674,7 +674,7 @@ function Format-PSPesterFailure { } } -function Get-PSPesterFailures { +function Get-PSPesterFailure { param( [string]$NUnitLog = "pester-tests.xml" ) From 613be777dc146fee0b44878fb97cc822b1fc4ae6 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Mon, 7 Nov 2016 11:23:29 -0800 Subject: [PATCH 4/7] fixed call to get-pspesterfailure --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index b6cdc1a8f7a..c5f907aa362 100644 --- a/build.psm1 +++ b/build.psm1 @@ -641,7 +641,7 @@ function Format-PSPesterFailure { { if ($PSBoundParameters.Count -eq 0 -or $PSBoundParameters.ContainsKey("NUnitLog")) { - $PesterFailure = Get-PSPesterFailures -NUnitLog $NUnitLog + $PesterFailure = Get-PSPesterFailure -NUnitLog $NUnitLog } [int]$totalFailures = 0 [string]$currentSuiteName = [String]::Empty From 5e1d48ae813fed2b709262759c26776ed7ca2bb1 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Tue, 8 Nov 2016 09:41:03 -0800 Subject: [PATCH 5/7] Added to test docs new cmdlets --- docs/testing-guidelines/testing-guidelines.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/testing-guidelines/testing-guidelines.md b/docs/testing-guidelines/testing-guidelines.md index d2d4d18ede1..697e195ee13 100755 --- a/docs/testing-guidelines/testing-guidelines.md +++ b/docs/testing-guidelines/testing-guidelines.md @@ -84,18 +84,29 @@ When running tests in this way, be sure that you have started PowerShell with `- environment is not the default or has any customization. For example, to run all the Pester tests for CI (assuming you are at the root of the PowerShell repo): -``` +```PowerShell Import-Module ./build.psm1 Start-PSPester ``` If you wish to run specific tests, that is possible as well: -``` +```PowerShell Start-PSPester -Directory test/powershell/engine/Api ``` Or a specific Pester test file: -``` +```PowerShell Start-PSPester -Directory test/powershell/engine/Api -Test XmlAdapter.Tests.Api ``` +If you added a `Feature` test and not a `CI` test, then you would specify the tag: +```PowerShell +Start-PSPester -Path ./myTest.ps1 -Tag Feature +``` + +### How to deal with failures? +As part of your PR, you must ensure all existing tests pass. +If you see any failures from `Start-PSPester` (summary reported at end), then you must investigate them and fix your PR so those tests pass or fix the test if the test case is wrong. +Two helper functions are part of the build.psm1 module to help with that: +* `Get-PSPesterFailure` will parse the NUnit test result log and return PowerShell objects for each failure so you can do additional filtering, sorting, grouping, etc... +* `Format-PSPesterFailure` will call `Get-PSPesterFailure` if no parameters are provided and show just the failures at the console simliar to what Pester displays ### What happens after your PR? When your PR has successfully passed the CI test gates, your changes will be used to create PowerShell binaries which can be run From 40ce3d63f488e93134cac7bbb2782c5204fd5cf5 Mon Sep 17 00:00:00 2001 From: Jason Shirk Date: Tue, 8 Nov 2016 11:40:03 -0800 Subject: [PATCH 6/7] Update testing-guidelines.md Fix spelling error. --- docs/testing-guidelines/testing-guidelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing-guidelines/testing-guidelines.md b/docs/testing-guidelines/testing-guidelines.md index 697e195ee13..c05b9c68042 100755 --- a/docs/testing-guidelines/testing-guidelines.md +++ b/docs/testing-guidelines/testing-guidelines.md @@ -106,7 +106,7 @@ As part of your PR, you must ensure all existing tests pass. If you see any failures from `Start-PSPester` (summary reported at end), then you must investigate them and fix your PR so those tests pass or fix the test if the test case is wrong. Two helper functions are part of the build.psm1 module to help with that: * `Get-PSPesterFailure` will parse the NUnit test result log and return PowerShell objects for each failure so you can do additional filtering, sorting, grouping, etc... -* `Format-PSPesterFailure` will call `Get-PSPesterFailure` if no parameters are provided and show just the failures at the console simliar to what Pester displays +* `Format-PSPesterFailure` will call `Get-PSPesterFailure` if no parameters are provided and show just the failures at the console similar to what Pester displays ### What happens after your PR? When your PR has successfully passed the CI test gates, your changes will be used to create PowerShell binaries which can be run From 6795cc1923399cdb0963b6291c0903c9021f24d0 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Mon, 14 Nov 2016 11:17:15 -0800 Subject: [PATCH 7/7] Update testing-guidelines.md added sample to using failure cmdlets --- docs/testing-guidelines/testing-guidelines.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/testing-guidelines/testing-guidelines.md b/docs/testing-guidelines/testing-guidelines.md index c05b9c68042..34e70ad17f9 100755 --- a/docs/testing-guidelines/testing-guidelines.md +++ b/docs/testing-guidelines/testing-guidelines.md @@ -108,6 +108,11 @@ Two helper functions are part of the build.psm1 module to help with that: * `Get-PSPesterFailure` will parse the NUnit test result log and return PowerShell objects for each failure so you can do additional filtering, sorting, grouping, etc... * `Format-PSPesterFailure` will call `Get-PSPesterFailure` if no parameters are provided and show just the failures at the console similar to what Pester displays +```PowerShell +Start-PSPester # summary shows failures +Format-PSPesterFailure +``` + ### What happens after your PR? When your PR has successfully passed the CI test gates, your changes will be used to create PowerShell binaries which can be run in Microsoft's internal test frameworks.