Limit output information for failed tests (#1975)

* Added ShowStackTrace property to Output section * Added ShowStackTrace parameter to Write-ErrorToScreen and refactored some long lines * ShowStackTrace doesn't limit ScriptStackTrace * Add ShowStackTrace output option to every call to Write-ErrorToScreen * Added configuration tests and made slight improvements in naming of existing tests * Use string builder to create the error message * Using empty string instead of new() for StringBuilder * Refactored format logic into separate Format-ErrorMessage function * Move first line of trace into Trace property * Fixing up tests * Added tests for Format-ErrorMessage and Write-ErrorToScreen * Removing unnecessary escaping and typos * Converted ShowStackTrace to StackTraceVerbosity option to support multiple options * Changed code to use StackTraceVerbosity to filter output * Set default for StackTraceVerbosity parameter * Updated configuration tests * Modified Output tests to test new option * Add warning message and override for Debug.ShowFullErrors * Removed warning and put deprecation notice in configuration * Added more configuration tests * Set Debug.ShowFullErrors to false prevent override for other tests * Use BeforeEach to simplify output tests
pester · Jun 14, 2021 · 989dac0 · 989dac0
1 parent 23f9492
commit 989dac0
Show file tree

Hide file tree

Showing 9 changed files with 432 additions and 53 deletions.
diff --git a/src/Main.ps1 b/src/Main.ps1
@@ -890,7 +890,8 @@ function Invoke-Pester {
                 if ($PSBoundParameters.ContainsKey('Configuration')) {
                     # Advanced configuration used, merging to get new reference
                     [PesterConfiguration] $PesterPreference = [PesterConfiguration]::Merge([PesterConfiguration]::Default, $Configuration)
-                } else {
+                }
+                else {
                     [PesterConfiguration] $PesterPreference = $Configuration
                 }
             }
@@ -921,6 +922,10 @@ function Invoke-Pester {
                 $PesterPreference.Debug.WriteDebugMessagesFrom = $PesterPreference.Debug.WriteDebugMessagesFrom.Value + @($missingCategories)
             }
 
+            if ($PesterPreference.Debug.ShowFullErrors.Value) {
+                $PesterPreference.Output.StackTraceVerbosity = "Full"
+            }
+
             $plugins +=
             @(
                 # decorator plugin needs to be added after output
@@ -1153,7 +1158,7 @@ function Invoke-Pester {
 
         }
         catch {
-            Write-ErrorToScreen $_ -Throw:$PesterPreference.Run.Throw.Value
+            Write-ErrorToScreen $_ -Throw:$PesterPreference.Run.Throw.Value -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
             if ($PesterPreference.Run.Exit.Value) {
                 exit -1
             }

diff --git a/src/Pester.RSpec.ps1 b/src/Pester.RSpec.ps1
@@ -377,7 +377,7 @@ function New-PesterConfiguration {
       Default value: 'Stop'
 
     Debug:
-      ShowFullErrors: Show full errors including Pester internal stack.
+      ShowFullErrors: Show full errors including Pester internal stack. This property is deprecated, and if set to true it will override Output.StackTraceVerbosity to 'Full'.
       Default value: $false
 
       WriteDebugMessages: Write Debug messages to screen.
@@ -395,6 +395,9 @@ function New-PesterConfiguration {
     Output:
       Verbosity: The verbosity of output, options are None, Normal, Detailed and Diagnostic.
       Default value: 'Normal'
+
+      StackTraceVerbosity: The verbosity of stacktrace output, options are None, FirstLine, Filtered and Full.
+      Default value: 'Filtered'
     ```
 
     .PARAMETER Hashtable

diff --git a/src/csharp/Pester/DebugConfiguration.cs b/src/csharp/Pester/DebugConfiguration.cs
@@ -30,7 +30,7 @@ public static DebugConfiguration ShallowClone(DebugConfiguration configuration)
 
         public DebugConfiguration() : base("Debug configuration for Pester. ⚠ Use at your own risk!")
         {
-            ShowFullErrors = new BoolOption("Show full errors including Pester internal stack.", false);
+            ShowFullErrors = new BoolOption("Show full errors including Pester internal stack. This property is deprecated, and if set to true it will override Output.StackTraceVerbosity to 'Full'.", false);
             WriteDebugMessages = new BoolOption("Write Debug messages to screen.", false);
             WriteDebugMessagesFrom = new StringArrayOption("Write Debug messages from a given source, WriteDebugMessages must be set to true for this to work. You can use like wildcards to get messages from multiple sources, as well as * to get everything.", new string[] { "Discovery", "Skip", "Mock", "CodeCoverage" });
             ShowNavigationMarkers = new BoolOption("Write paths after every block and test, for easy navigation in VSCode.", false);

diff --git a/src/csharp/Pester/OutputConfiguration.cs b/src/csharp/Pester/OutputConfiguration.cs
@@ -23,6 +23,7 @@ namespace Pester
     public class OutputConfiguration : ConfigurationSection
     {
         private StringOption _verbosity;
+        private StringOption _stackTraceVerbosity;
 
         public static OutputConfiguration Default { get { return new OutputConfiguration(); } }
         public static OutputConfiguration ShallowClone(OutputConfiguration configuration)
@@ -35,12 +36,14 @@ public OutputConfiguration(IDictionary configuration) : this()
             if (configuration != null)
             {
                 Verbosity = configuration.GetObjectOrNull<string>("Verbosity") ?? Verbosity;
+                StackTraceVerbosity = configuration.GetObjectOrNull<string>("StackTraceVerbosity") ?? StackTraceVerbosity;
             }
         }
 
         public OutputConfiguration() : base("Output configuration")
         {
             Verbosity = new StringOption("The verbosity of output, options are None, Normal, Detailed and Diagnostic.", "Normal");
+            StackTraceVerbosity = new StringOption("The verbosity of stacktrace output, options are None, FirstLine, Filtered and Full.", "Filtered");
         }
 
         public StringOption Verbosity
@@ -59,6 +62,22 @@ public StringOption Verbosity
             }
         }
 
+        public StringOption StackTraceVerbosity
+        {
+            get { return _stackTraceVerbosity; }
+            set
+            {
+                if (_stackTraceVerbosity == null)
+                {
+                    _stackTraceVerbosity = value;
+                }
+                else
+                {
+                    _stackTraceVerbosity = new StringOption(_stackTraceVerbosity, value?.Value);
+                }
+            }
+        }
+
         private string FixMinimal(string value)
         {
             return string.Equals(value, "Minimal", StringComparison.OrdinalIgnoreCase) ? "Normal" : value;

diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1
@@ -293,8 +293,8 @@ function Write-PesterReport {
         & $SafeCommands['Write-Host'] ("Container failed: {0}" -f $RunResult.FailedContainersCount) -Foreground $ReportTheme.Fail
         & $SafeCommands['Write-Host'] ($cs -join [Environment]::NewLine) -Foreground $ReportTheme.Fail
     }
-        # & $SafeCommands['Write-Host'] ($ReportStrings.TestsPending -f $RunResult.PendingCount) -Foreground $Pending -NoNewLine
-        # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive
+    # & $SafeCommands['Write-Host'] ($ReportStrings.TestsPending -f $RunResult.PendingCount) -Foreground $Pending -NoNewLine
+    # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive
     # }
 }
 
@@ -398,7 +398,7 @@ function ConvertTo-FailureLines {
         [array]::Reverse($exceptionLines)
         $lines.Message += $exceptionLines
         if ($ErrorRecord.FullyQualifiedErrorId -eq 'PesterAssertionFailed') {
-            $lines.Message += "at $($ErrorRecord.TargetObject.LineText.Trim()), $($ErrorRecord.TargetObject.File):$($ErrorRecord.TargetObject.Line)".Split([string[]]($([System.Environment]::NewLine), "`n"), [System.StringSplitOptions]::RemoveEmptyEntries)
+            $lines.Trace += "at $($ErrorRecord.TargetObject.LineText.Trim()), $($ErrorRecord.TargetObject.File):$($ErrorRecord.TargetObject.Line)".Split([string[]]($([System.Environment]::NewLine), "`n"), [System.StringSplitOptions]::RemoveEmptyEntries)
         }
 
         if ( -not ($ErrorRecord | & $SafeCommands['Get-Member'] -Name ScriptStackTrace) ) {
@@ -417,7 +417,7 @@ function ConvertTo-FailureLines {
             $traceLines = $ErrorRecord.ScriptStackTrace.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries)
         }
 
-        if ($ForceFullError -or $PesterPreference.Debug.ShowFullErrors.Value) {
+        if ($ForceFullError -or $PesterPreference.Debug.ShowFullErrors.Value -or $PesterPreference.Output.StackTraceVerbosity.Value -eq 'Full') {
             $lines.Trace += $traceLines
         }
         else {
@@ -514,7 +514,7 @@ function Get-WriteScreenPlugin ($Verbosity) {
             }
 
             & $SafeCommands["Write-Host"] -ForegroundColor $ReportTheme.Fail "[-] Discovery in $($path) failed with:"
-            Write-ErrorToScreen $Context.Block.ErrorRecord
+            Write-ErrorToScreen $Context.Block.ErrorRecord -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
         }
     }
 
@@ -574,7 +574,7 @@ function Get-WriteScreenPlugin ($Verbosity) {
 
         if ($Context.Result.ErrorRecord.Count -gt 0) {
             & $SafeCommands["Write-Host"] -ForegroundColor $ReportTheme.Fail "[-] $($Context.Result.Item) failed with:"
-            Write-ErrorToScreen $Context.Result.ErrorRecord
+            Write-ErrorToScreen $Context.Result.ErrorRecord -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
         }
 
         if ('Normal' -eq $PesterPreference.Output.Verbosity.Value) {
@@ -636,6 +636,10 @@ function Get-WriteScreenPlugin ($Verbosity) {
             throw "Unsupported level out output '$($PesterPreference.Output.Verbosity.Value)'"
         }
 
+        if ($PesterPreference.Output.StackTraceVerbosity.Value -notin 'None', 'FirstLine', 'Filtered', 'Full') {
+            throw "Unsupported level of stacktrace output '$($PesterPreference.Output.StackTraceVerbosity.Value)'"
+        }
+
         $humanTime = "$(Get-HumanTime ($_test.Duration)) ($(Get-HumanTime $_test.UserDuration)|$(Get-HumanTime $_test.FrameworkDuration))"
 
         if ($PesterPreference.Debug.ShowNavigationMarkers.Value) {
@@ -661,16 +665,16 @@ function Get-WriteScreenPlugin ($Verbosity) {
                         & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
                         & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
 
-                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailDetail $($e.DisplayStackTrace -replace '(?m)^',$error_margin)
-                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailDetail $($e.DisplayErrorMessage -replace '(?m)^',$error_margin)
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailDetail $($e.DisplayStackTrace -replace '(?m)^', $error_margin)
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailDetail $($e.DisplayErrorMessage -replace '(?m)^', $error_margin)
                     }
 
                 }
                 else {
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
 
-                    Write-ErrorToScreen $_test.ErrorRecord -ErrorMargin $error_margin
+                    Write-ErrorToScreen $_test.ErrorRecord -ErrorMargin $error_margin -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
                 }
                 break
             }
@@ -757,7 +761,7 @@ function Get-WriteScreenPlugin ($Verbosity) {
 
         foreach ($e in $Context.Block.ErrorRecord) { ConvertTo-FailureLines $e }
         & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.BlockFail "[-] $($Context.Block.FrameworkData.CommandUsed) $($Context.Block.Path -join ".") failed"
-        Write-ErrorToScreen $Context.Block.ErrorRecord $error_margin
+        Write-ErrorToScreen $Context.Block.ErrorRecord $error_margin -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
     }
 
     $p.End = {
@@ -769,13 +773,13 @@ function Get-WriteScreenPlugin ($Verbosity) {
     New-PluginObject @p
 }
 
-function Write-ErrorToScreen {
+function Format-ErrorMessage {
     [CmdletBinding()]
     param (
         [Parameter(Mandatory)]
         $Err,
         [string] $ErrorMargin,
-        [switch] $Throw
+        [string] $StackTraceVerbosity = [PesterConfiguration]::Default.Output.StackTraceVerbosity.Value
     )
 
     $multipleErrors = 1 -lt $Err.Count
@@ -784,21 +788,79 @@ function Write-ErrorToScreen {
     $out = if ($multipleErrors) {
         $c = 0
         $(foreach ($e in $Err) {
-            $isFormattedError = $null -ne $e.DisplayErrorMessage
-            "[$(($c++))] $(if ($isFormattedError){ $e.DisplayErrorMessage } else { $e.Exception })$(if ($null -ne $e.DisplayStackTrace) {"$([Environment]::NewLine)$($e.DisplayStackTrace)"})"
-        }) -join [Environment]::NewLine
+                $errorMessageSb = [System.Text.StringBuilder]""
+
+                if ($null -ne $e.DisplayErrorMessage) {
+                    [void]$errorMessageSb.Append("[$(($c++))] $($e.DisplayErrorMessage)")
+                }
+                else {
+                    [void]$errorMessageSb.Append("[$(($c++))] $($e.Exception)")
+                }
+
+                if ($null -ne $e.DisplayStackTrace -and $StackTraceVerbosity -ne 'None') {
+                    $stackTraceLines = $e.DisplayStackTrace -split [Environment]::NewLine
+
+                    if ($StackTraceVerbosity -eq 'FirstLine') {
+                        [void]$errorMessageSb.Append([Environment]::NewLine + $stackTraceLines[0])
+                    }
+                    else {
+                        [void]$errorMessageSb.Append([Environment]::NewLine + $e.DisplayStackTrace)
+                    }
+                }
+
+                $errorMessageSb.ToString()
+            }) -join [Environment]::NewLine
     }
     else {
-        $isFormattedError = $null -ne $Err.DisplayErrorMessage
-        "$(if ($isFormattedError){ $Err.DisplayErrorMessage } else { $Err.Exception })$(if ($isFormattedError) { if ($null -ne $Err.DisplayStackTrace) {"$([Environment]::NewLine)$($Err.DisplayStackTrace)"}} else { if  ($null -ne $Err.ScriptStackTrace) {"$([Environment]::NewLine)$($Err.ScriptStackTrace)"}})"
+        $errorMessageSb = [System.Text.StringBuilder]""
+
+        if ($null -ne $Err.DisplayErrorMessage) {
+            [void]$errorMessageSb.Append($Err.DisplayErrorMessage)
+
+            if ($null -ne $Err.DisplayStackTrace -and $StackTraceVerbosity -ne 'None') {
+                $stackTraceLines = $Err.DisplayStackTrace -split [Environment]::NewLine
+
+                if ($StackTraceVerbosity -eq 'FirstLine') {
+                    [void]$errorMessageSb.Append([Environment]::NewLine + $stackTraceLines[0])
+                }
+                else {
+                    [void]$errorMessageSb.Append([Environment]::NewLine + $Err.DisplayStackTrace)
+                }
+            }
+        }
+        else {
+            [void]$errorMessageSb.Append($Err.Exception.ToString())
+
+            if ($null -ne $Err.ScriptStackTrace) {
+                [void]$errorMessageSb.Append([Environment]::NewLine + $Err.ScriptStackTrace)
+            }
+        }
+
+        $errorMessageSb.ToString()
     }
 
     $withMargin = ($out -split [Environment]::NewLine) -replace '(?m)^', $ErrorMargin -join [Environment]::NewLine
+
+    return $withMargin
+}
+
+function Write-ErrorToScreen {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        $Err,
+        [string] $ErrorMargin,
+        [switch] $Throw,
+        [string] $StackTraceVerbosity = [PesterConfiguration]::Default.Output.StackTraceVerbosity.Value
+    )
+
+    $errorMessage = Format-ErrorMessage -Err $Err -ErrorMargin:$ErrorMargin -StackTraceVerbosity:$StackTraceVerbosity
+
     if ($Throw) {
-        throw $withMargin
+        throw $errorMessage
     }
     else {
-        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$withMargin"
+        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$errorMessage"
     }
 }