Refactor Invoke-Pester and validate configuration early

src/Format.ps1

@@ -202,17 +202,24 @@ function Format-Type ([Type]$Value) {
     [string]$Value
 }
 
-function Join-And ($Items, $Threshold = 2) {
-
+function Join-With ($Items, $Threshold = 2, $Separator = ', ', $LastSeparator = ' and ') {
     if ($null -eq $items -or $items.count -lt $Threshold) {
-        $items -join ', '
+        $items -join $Separator
     }
     else {
         $c = $items.count
-        ($items[0..($c - 2)] -join ', ') + ' and ' + $items[-1]
+        ($items[0..($c - 2)] -join $Separator) + $LastSeparator + $items[-1]
     }
 }
 
+function Join-And ($Items, $Threshold = 2) {
+    Join-With -Items $Items -Threshold $Threshold -Separator ', ' -LastSeparator ' and '
+}
+
+function Join-Or ($Items, $Threshold = 2) {
+    Join-With -Items $Items -Threshold $Threshold -Separator ', ' -LastSeparator ' or '
+}
+
 function Add-SpaceToNonEmptyString ([string]$Value) {
     if ($Value) {
         " $Value"

src/Pester.Runtime.ps1

@@ -1182,6 +1182,17 @@ function Run-Test {
 
         $result
     }
+
+    $steps = $state.Plugin.RunEnd
+    if ($null -ne $steps -and 0 -lt @($steps).Count) {
+        Invoke-PluginStep -Plugins $state.Plugin -Step RunEnd -Context @{
+            Blocks                   = $Block
+            Configuration            = $state.PluginConfiguration
+            Data                     = $state.PluginData
+            WriteDebugMessages       = $PesterPreference.Debug.WriteDebugMessages.Value
+            Write_PesterDebugMessage = if ($PesterPreference.Debug.WriteDebugMessages.Value) { $script:SafeCommands['Write-PesterDebugMessage'] }
+        } -ThrowOnFailure
+    }
 }
 
 function Invoke-PluginStep {
@@ -1292,21 +1303,20 @@ function Assert-Success {
     $anyFailed = $false
     $err = ""
     foreach ($r in $InvocationResult) {
+        $rc++
         $ec = 0
         if ($null -ne $r.ErrorRecord -and $r.ErrorRecord.Length -gt 0) {
-            $err += "Result $($rc++):"
             $anyFailed = $true
             foreach ($e in $r.ErrorRecord) {
-                $err += "Error $($ec++):"
+                $err += "$([Environment]::NewLine)Result $rc - Error $((++$ec)):"
                 $err += & $SafeCommands["Out-String"] -InputObject $e
                 $err += & $SafeCommands["Out-String"] -InputObject $e.ScriptStackTrace
             }
         }
     }
 
     if ($anyFailed) {
-        $Message = $Message + ":`n$err"
-        Write-PesterHostMessage -ForegroundColor Red $Message
+        $Message = $Message + ":$err"
         throw $Message
     }
 }

src/Pester.Utility.ps1

@@ -391,3 +391,14 @@ function IsPSEnumerable($Object) {
     $enumerator = [System.Management.Automation.LanguagePrimitives]::GetEnumerator($Object)
     return $null -ne $enumerator
 }
+
+function Get-StringOptionErrorMessage {
+    param (
+        [Parameter(Mandatory)]
+        [string] $OptionPath,
+        [string[]] $SupportedValues = @(),
+        [string] $Value
+    )
+    $supportedValuesString = Join-Or ($SupportedValues -replace '^|$', "'")
+    return "$OptionPath must be $supportedValuesString, but it was '$Value'. Please review your configuration."
+}

src/functions/Coverage.Plugin.ps1

@@ -1,5 +1,62 @@
 function Get-CoveragePlugin {
-    New-PluginObject -Name "Coverage" -RunStart {
+    # Validate configuration
+    Resolve-CodeCoverageConfiguration
+
+    $p = @{
+        Name = 'Coverage'
+    }
+
+    $p.Start = {
+        param($Context)
+
+        $paths = @(if (0 -lt $PesterPreference.CodeCoverage.Path.Value.Count) {
+                $PesterPreference.CodeCoverage.Path.Value
+            }
+            else {
+                # no paths specific to CodeCoverage were provided, resolve them from
+                # tests by using the whole directory in which the test or the
+                # provided directory. We might need another option to disable this convention.
+                @(foreach ($p in $PesterPreference.Run.Path.Value) {
+                        # this is a bit ugly, but the logic here is
+                        # that we check if the path exists,
+                        # and if it does and is a file then we return the
+                        # parent directory, otherwise we got a directory
+                        # and return just it
+                        $i = & $SafeCommands['Get-Item'] $p
+                        if ($i.PSIsContainer) {
+                            & $SafeCommands['Join-Path'] $i.FullName "*"
+                        }
+                        else {
+                            & $SafeCommands['Join-Path'] $i.Directory.FullName "*"
+                        }
+                    })
+            })
+
+        $outputPath = if ([IO.Path]::IsPathRooted($PesterPreference.CodeCoverage.OutputPath.Value)) {
+            $PesterPreference.CodeCoverage.OutputPath.Value
+        }
+        else {
+            & $SafeCommands['Join-Path'] $pwd.Path $PesterPreference.CodeCoverage.OutputPath.Value
+        }
+
+        $CodeCoverage = @{
+            Enabled                 = $PesterPreference.CodeCoverage.Enabled.Value
+            OutputFormat            = $PesterPreference.CodeCoverage.OutputFormat.Value
+            OutputPath              = $outputPath
+            OutputEncoding          = $PesterPreference.CodeCoverage.OutputEncoding.Value
+            ExcludeTests            = $PesterPreference.CodeCoverage.ExcludeTests.Value
+            Path                    = @($paths)
+            RecursePaths            = $PesterPreference.CodeCoverage.RecursePaths.Value
+            TestExtension           = $PesterPreference.Run.TestExtension.Value
+            UseSingleHitBreakpoints = $PesterPreference.CodeCoverage.SingleHitBreakpoints.Value
+            UseBreakpoints          = $PesterPreference.CodeCoverage.UseBreakpoints.Value
+        }
+
+        # Save PluginConfiguration for Coverage
+        $Context.Configuration['Coverage'] = $CodeCoverage
+    }
+
+    $p.RunStart = {
         param($Context)
 
         $sw = [System.Diagnostics.Stopwatch]::StartNew()
@@ -47,16 +104,18 @@
         if ($PesterPreference.Output.Verbosity.Value -in "Detailed", "Diagnostic") {
             Write-PesterHostMessage -ForegroundColor Magenta "Code Coverage preparation finished after $($sw.ElapsedMilliseconds) ms."
         }
-    } -End {
+    }
+
+    $p.RunEnd = {
         param($Context)
 
         $config = $Context.Configuration['Coverage']
 
-        if (-not $Context.TestRun.PluginData.ContainsKey("Coverage")) {
+        if (-not $Context.Data.ContainsKey("Coverage")) {
             return
         }
 
-        $coverageData = $Context.TestRun.PluginData.Coverage
+        $coverageData = $Context.Data.Coverage
 
         if (-not $config.UseBreakpoints) {
             Stop-TraceScript -Patched $coverageData.Patched
@@ -68,4 +127,97 @@
             Exit-CoverageAnalysis -CommandCoverage $coverageData.CommandCoverage
         }
     }
+
+    $p.End = {
+        param($Context)
+
+        $run = $Context.TestRun
+
+        if ($PesterPreference.Output.Verbosity.Value -ne "None") {
+            $sw = [Diagnostics.Stopwatch]::StartNew()
+            Write-PesterHostMessage -ForegroundColor Magenta "Processing code coverage result."
+        }
+
+        $breakpoints = @($run.PluginData.Coverage.CommandCoverage)
+        $measure = if (-not $PesterPreference.CodeCoverage.UseBreakpoints.Value) { @($run.PluginData.Coverage.Tracer.Hits) }
+        $coverageReport = Get-CoverageReport -CommandCoverage $breakpoints -Measure $measure
+        $totalMilliseconds = $run.Duration.TotalMilliseconds
+
+        $configuration = $run.PluginConfiguration.Coverage
+
+        if ($configuration.OutputFormat -in 'JaCoCo', 'CoverageGutters') {
+            [xml] $jaCoCoReport = [xml] (Get-JaCoCoReportXml -CommandCoverage $breakpoints -TotalMilliseconds $totalMilliseconds -CoverageReport $coverageReport -Format $configuration.OutputFormat)
+        }
+        else {
+            throw "CodeCoverage.CoverageFormat '$($configuration.OutputFormat)' is not valid, please review your configuration."
+        }
+
+        $settings = [Xml.XmlWriterSettings] @{
+            Indent              = $true
+            NewLineOnAttributes = $false
+        }
+
+        $stringWriter = $null
+        $xmlWriter = $null
+        try {
+            $stringWriter = [Pester.Factory]::CreateStringWriter()
+            $xmlWriter = [Xml.XmlWriter]::Create($stringWriter, $settings)
+
+            $jaCocoReport.WriteContentTo($xmlWriter)
+
+            $xmlWriter.Flush()
+            $stringWriter.Flush()
+        }
+        finally {
+            if ($null -ne $xmlWriter) {
+                try {
+                    $xmlWriter.Close()
+                }
+                catch {
+                }
+            }
+            if ($null -ne $stringWriter) {
+                try {
+                    $stringWriter.Close()
+                }
+                catch {
+                }
+            }
+        }
+
+        $resolvedPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($PesterPreference.CodeCoverage.OutputPath.Value)
+        if (-not (& $SafeCommands['Test-Path'] $resolvedPath)) {
+            $dir = & $SafeCommands['Split-Path'] $resolvedPath
+            $null = & $SafeCommands['New-Item'] $dir -Force -ItemType Container
+        }
+
+        $stringWriter.ToString() | & $SafeCommands['Out-File'] $resolvedPath -Encoding $PesterPreference.CodeCoverage.OutputEncoding.Value -Force
+        if ($PesterPreference.Output.Verbosity.Value -in 'Detailed', 'Diagnostic') {
+            Write-PesterHostMessage -ForegroundColor Magenta "Code Coverage result processed in $($sw.ElapsedMilliseconds) ms."
+        }
+        $reportText = Write-CoverageReport $coverageReport
+
+        $coverage = [Pester.CodeCoverage]::Create()
+        $coverage.CoverageReport = $reportText
+        $coverage.CoveragePercent = $coverageReport.CoveragePercent
+        $coverage.CommandsAnalyzedCount = $coverageReport.NumberOfCommandsAnalyzed
+        $coverage.CommandsExecutedCount = $coverageReport.NumberOfCommandsExecuted
+        $coverage.CommandsMissedCount = $coverageReport.NumberOfCommandsMissed
+        $coverage.FilesAnalyzedCount = $coverageReport.NumberOfFilesAnalyzed
+        $coverage.CommandsMissed = $coverageReport.MissedCommands
+        $coverage.CommandsExecuted = $coverageReport.HitCommands
+        $coverage.FilesAnalyzed = $coverageReport.AnalyzedFiles
+        $coverage.CoveragePercentTarget = $PesterPreference.CodeCoverage.CoveragePercentTarget.Value
+
+        $run.CodeCoverage = $coverage
+    }
+
+    New-PluginObject @p
+}
+
+function Resolve-CodeCoverageConfiguration {
+    $supportedFormats = 'JaCoCo', 'CoverageGutters'
+    if ($PesterPreference.CodeCoverage.OutputFormat.Value -notin $supportedFormats) {
+        throw (Get-StringOptionErrorMessage -OptionPath 'CodeCoverage.OutputFormat' -SupportedValues $supportedFormats -Value $PesterPreference.CodeCoverage.OutputFormat.Value)
+    }
 }

src/functions/Get-SkipRemainingOnFailurePlugin.ps1

@@ -9,16 +9,27 @@ function New-SkippedTestMessage {
     "Skipped due to previous failure at '$($Test.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'"
 }
 
+function Resolve-SkipRemainingOnFailureConfiguration {
+    $supportedValues = 'None', 'Block', 'Container', 'Run'
+    if ($PesterPreference.Run.SkipRemainingOnFailure.Value -notin $supportedValues) {
+        throw (Get-StringOptionErrorMessage -OptionPath 'Run.SkipRemainingOnFailure' -SupportedValues $supportedValues -Value $PesterPreference.Run.SkipRemainingOnFailure.Value)
+    }
+}
+
+
 function Get-SkipRemainingOnFailurePlugin {
+    # Validate configuration
+    Resolve-SkipRemainingOnFailureConfiguration
+
+    # Create plugin
     $p = @{
-        Name = "SkipRemainingOnFailure"
+        Name = 'SkipRemainingOnFailure'
     }
 
-    if ($PesterPreference.Output.Verbosity.Value -in 'Detailed', 'Diagnostic') {
-        $p.Start = {
-            param ($Context)
-            $Context.Configuration.SkipRemainingOnFailureCount = 0
-        }
+    $p.Start = {
+        param ($Context)
+
+        $Context.Configuration.SkipRemainingOnFailureCount = 0
     }
 
     if ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Block') {