Azure Devops & Github Actions error logging (#1996)

* Added CIFormat output option * Added CI detection * Also check Auto is set has been set manually * Added Write-CIErrorMessage function to format different CIs * Added CIFormat detection in Output.ps1 * Renamed Write-CIErrorMessage to Write-CIErrorToScreen * Remove nullorempty check for header * Updated documentation * Formatting error alignment for github actions * Created Format-CIErrorMessage function to format errors + added tests * Write-CIErrorToScreen now just responsible for writing * Fix formatting * Adding configuration tests to catch issues * Set both TF_BUILD and GITHUB_ACTIONS variables in each test to avoid conflict with auto * Set CIFormat to None for RSpec tests that fail on purpose * Disabling missing Rspec test that fails on purpose * Removed overriding of CIFormat when env variables don't exist * Simplified option documentation * Added CI formatting to Main.ps1 * Simplified Format-CIErrorMessage by putting github actions messages in expandable group * Add title to expandable group * Fixing up tests and updating docs * Trim GHA message whitespace * Use try/finally when executing tests to ensure previous env variables are always set even after failure
pester · Jun 24, 2021 · 465544d · 465544d
1 parent 570d3b6
commit 465544d
Show file tree

Hide file tree

Showing 11 changed files with 551 additions and 15 deletions.
diff --git a/src/Main.ps1 b/src/Main.ps1
@@ -899,6 +899,24 @@ function Invoke-Pester {
                 [PesterConfiguration] $PesterPreference = [PesterConfiguration]::Merge($callerPreference, $Configuration)
             }
 
+            if ($PesterPreference.Output.CIFormat.Value -eq 'Auto') {
+
+                # Variable is set to 'True' if the script is being run by a Azure Devops build task. https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml
+                # Do not fix this to check for boolean value, the value is set to literal string 'True'
+                if ($env:TF_BUILD -eq 'True') {
+                    $PesterPreference.Output.CIFormat = 'AzureDevops'
+                }
+                # Variable is set to 'True' if the script is being run by a Github Actions workflow. https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables
+                # Do not fix this to check for boolean value, the value is set to literal string 'True'
+                elseif ($env:GITHUB_ACTIONS -eq 'True') {
+                    $PesterPreference.Output.CIFormat = 'GithubActions'
+                }
+
+                else {
+                    $PesterPreference.Output.CIFormat = 'None'
+                }
+            }
+
             & $SafeCommands['Get-Variable'] 'Configuration' -Scope Local | Remove-Variable
 
             # $sessionState = Set-SessionStateHint -PassThru  -Hint "Caller - Captured in Invoke-Pester" -SessionState $PSCmdlet.SessionState
@@ -1167,7 +1185,19 @@ function Invoke-Pester {
 
         }
         catch {
-            Write-ErrorToScreen $_ -Throw:$PesterPreference.Run.Throw.Value -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
+            $formatErrorParams = @{
+                Err                 = $_
+                StackTraceVerbosity = $PesterPreference.Output.StackTraceVerbosity.Value
+            }
+
+            if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
+                $errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
+                Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorMessage[0] -Message $errorMessage[1..($errorMessage.Count - 1)]
+            }
+            else {
+                Write-ErrorToScreen @formatErrorParams -Throw:$PesterPreference.Run.Throw.Value
+            }
+
             if ($PesterPreference.Run.Exit.Value) {
                 exit -1
             }

diff --git a/src/Pester.RSpec.ps1 b/src/Pester.RSpec.ps1
@@ -399,6 +399,9 @@ function New-PesterConfiguration {
       StackTraceVerbosity: The verbosity of stacktrace output, options are None, FirstLine, Filtered and Full.
       Default value: 'Filtered'
 
+      CIFormat: The CI format of error output in build logs, options are None, Auto, AzureDevops and GithubActions.
+      Default value: 'Auto'
+
     TestDrive:
       Enabled: Enable TestDrive.
       Default value: $true

diff --git a/src/csharp/Pester/OutputConfiguration.cs b/src/csharp/Pester/OutputConfiguration.cs
@@ -24,6 +24,7 @@ public class OutputConfiguration : ConfigurationSection
     {
         private StringOption _verbosity;
         private StringOption _stackTraceVerbosity;
+        private StringOption _ciFormat;
 
         public static OutputConfiguration Default { get { return new OutputConfiguration(); } }
         public static OutputConfiguration ShallowClone(OutputConfiguration configuration)
@@ -37,13 +38,15 @@ public OutputConfiguration(IDictionary configuration) : this()
             {
                 Verbosity = configuration.GetObjectOrNull<string>("Verbosity") ?? Verbosity;
                 StackTraceVerbosity = configuration.GetObjectOrNull<string>("StackTraceVerbosity") ?? StackTraceVerbosity;
+                CIFormat = configuration.GetObjectOrNull<string>("CIFormat") ?? CIFormat;
             }
         }
 
         public OutputConfiguration() : base("Output configuration")
         {
             Verbosity = new StringOption("The verbosity of output, options are None, Normal, Detailed and Diagnostic.", "Normal");
             StackTraceVerbosity = new StringOption("The verbosity of stacktrace output, options are None, FirstLine, Filtered and Full.", "Filtered");
+            CIFormat = new StringOption("The CI format of error output in build logs, options are None, Auto, AzureDevops and GithubActions.", "Auto");
         }
 
         public StringOption Verbosity
@@ -78,6 +81,22 @@ public StringOption StackTraceVerbosity
             }
         }
 
+        public StringOption CIFormat
+        {
+            get { return _ciFormat; }
+            set
+            {
+                if (_ciFormat == null)
+                {
+                    _ciFormat = value;
+                }
+                else
+                {
+                    _ciFormat = new StringOption(_ciFormat, value?.Value);
+                }
+            }
+        }
+
         private string FixMinimal(string value)
         {
             return string.Equals(value, "Minimal", StringComparison.OrdinalIgnoreCase) ? "Normal" : value;

diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1
@@ -513,8 +513,21 @@ function Get-WriteScreenPlugin ($Verbosity) {
                 throw "Container type '$($container.Type)' is not supported."
             }
 
-            & $SafeCommands["Write-Host"] -ForegroundColor $ReportTheme.Fail "[-] Discovery in $($path) failed with:"
-            Write-ErrorToScreen $Context.Block.ErrorRecord -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
+            $errorHeader = "[-] Discovery in $($path) failed with:"
+
+            $formatErrorParams = @{
+                Err                 = $Context.Block.ErrorRecord
+                StackTraceVerbosity = $PesterPreference.Output.StackTraceVerbosity.Value
+            }
+
+            if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
+                $errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
+                Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorHeader -Message $errorMessage
+            }
+            else {
+                & $SafeCommands["Write-Host"] -ForegroundColor $ReportTheme.Fail $errorHeader
+                Write-ErrorToScreen @formatErrorParams
+            }
         }
     }
 
@@ -573,8 +586,21 @@ function Get-WriteScreenPlugin ($Verbosity) {
         param ($Context)
 
         if ($Context.Result.ErrorRecord.Count -gt 0) {
-            & $SafeCommands["Write-Host"] -ForegroundColor $ReportTheme.Fail "[-] $($Context.Result.Item) failed with:"
-            Write-ErrorToScreen $Context.Result.ErrorRecord -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
+            $errorHeader = "[-] $($Context.Result.Item) failed with:"
+
+            $formatErrorParams = @{
+                Err                 = $Context.Result.ErrorRecord
+                StackTraceVerbosity = $PesterPreference.Output.StackTraceVerbosity.Value
+            }
+
+            if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
+                $errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
+                Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorHeader -Message $errorMessage
+            }
+            else {
+                & $SafeCommands["Write-Host"] -ForegroundColor $ReportTheme.Fail $errorHeader
+                Write-ErrorToScreen $formatErrorParams
+            }
         }
 
         if ('Normal' -eq $PesterPreference.Output.Verbosity.Value) {
@@ -640,6 +666,10 @@ function Get-WriteScreenPlugin ($Verbosity) {
             throw "Unsupported level of stacktrace output '$($PesterPreference.Output.StackTraceVerbosity.Value)'"
         }
 
+        if ($PesterPreference.Output.CIFormat.Value -notin 'None', 'Auto', 'AzureDevops', 'GithubActions') {
+            throw "Unsupported CI format '$($PesterPreference.Output.CIFormat.Value)'"
+        }
+
         $humanTime = "$(Get-HumanTime ($_test.Duration)) ($(Get-HumanTime $_test.UserDuration)|$(Get-HumanTime $_test.FrameworkDuration))"
 
         if ($PesterPreference.Debug.ShowNavigationMarkers.Value) {
@@ -671,10 +701,21 @@ function Get-WriteScreenPlugin ($Verbosity) {
 
                 }
                 else {
-                    & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
-                    & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
+                    $formatErrorParams = @{
+                        Err                 = $_test.ErrorRecord
+                        ErrorMargin         = $error_margin
+                        StackTraceVerbosity = $PesterPreference.Output.StackTraceVerbosity.Value
+                    }
 
-                    Write-ErrorToScreen $_test.ErrorRecord -ErrorMargin $error_margin -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
+                    if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
+                        $errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
+                        Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header "$margin[-] $out $humanTime" -Message $errorMessage
+                    }
+                    else {
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
+                        Write-ErrorToScreen @formatErrorParams
+                    }
                 }
                 break
             }
@@ -760,8 +801,23 @@ function Get-WriteScreenPlugin ($Verbosity) {
         }
 
         foreach ($e in $Context.Block.ErrorRecord) { ConvertTo-FailureLines $e }
-        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.BlockFail "[-] $($Context.Block.FrameworkData.CommandUsed) $($Context.Block.Path -join ".") failed"
-        Write-ErrorToScreen $Context.Block.ErrorRecord $error_margin -StackTraceVerbosity:$PesterPreference.Output.StackTraceVerbosity.Value
+
+        $errorHeader = "[-] $($Context.Block.FrameworkData.CommandUsed) $($Context.Block.Path -join ".") failed"
+
+        $formatErrorParams = @{
+            Err                 = $Context.Block.ErrorRecord
+            ErrorMargin         = $error_margin
+            StackTraceVerbosity = $PesterPreference.Output.StackTraceVerbosity.Value
+        }
+
+        if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
+            $errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
+            Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorHeader -Message $errorMessage
+        }
+        else {
+            & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.BlockFail $errorHeader
+            Write-ErrorToScreen @formatErrorParams
+        }
     }
 
     $p.End = {
@@ -773,6 +829,77 @@ function Get-WriteScreenPlugin ($Verbosity) {
     New-PluginObject @p
 }
 
+function Format-CIErrorMessage {
+    [OutputType([System.Collections.Generic.List[string]])]
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [ValidateSet('AzureDevops', 'GithubActions', IgnoreCase)]
+        [string] $CIFormat,
+
+        [Parameter(Mandatory)]
+        [string] $Header,
+
+        [Parameter(Mandatory)]
+        [string[]] $Message
+    )
+
+    $lines = [System.Collections.Generic.List[string]]@()
+
+    if ($CIFormat -eq 'AzureDevops') {
+
+        # header task issue error, so it gets reported to build log
+        # https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=powershell#example-log-an-error
+        $headerTaskIssueError = "##vso[task.logissue type=error] $Header"
+        $lines.Add($headerTaskIssueError)
+
+        # Add subsequent messages as errors, but do not get reported to build log
+        foreach ($line in $Message) {
+            $lines.Add("##[error] $line")
+        }
+    }
+    elseif ($CIFormat -eq 'GithubActions') {
+
+        # header error, so it gets reported to build log
+        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message
+        $headerError = "::error::$($Header.TrimStart())"
+        $lines.Add($headerError)
+
+        # Add rest of messages inside expandable group
+        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#grouping-log-lines
+        $lines.Add("::group::Message")
+
+        foreach ($line in $Message) {
+            $lines.Add($line.TrimStart())
+        }
+
+        $lines.Add("::endgroup::")
+    }
+
+    return $lines
+}
+
+function Write-CIErrorToScreen {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [ValidateSet('AzureDevops', 'GithubActions', IgnoreCase)]
+        [string] $CIFormat,
+
+        [Parameter(Mandatory)]
+        [string] $Header,
+
+        [Parameter(Mandatory)]
+        [string[]] $Message
+    )
+
+    $errorMessage = Format-CIErrorMessage @PSBoundParameters
+
+    foreach ($line in $errorMessage) {
+        & $SafeCommands['Write-Host'] $line
+    }
+}
+
 function Format-ErrorMessage {
     [CmdletBinding()]
     param (

diff --git a/tst/Pester.Mock.RSpec.ts.ps1 b/tst/Pester.Mock.RSpec.ts.ps1
@@ -342,8 +342,16 @@ i -PassThru:$PassThru {
             }
 
             $r = Invoke-Pester -Configuration ([PesterConfiguration]@{
-                    Run    = @{ ScriptBlock = $sb; PassThru = $true }
-                    Should = @{ ErrorAction = 'Continue' }
+                    Run    = @{
+                        ScriptBlock = $sb
+                        PassThru    = $true
+                    }
+                    Should = @{
+                        ErrorAction = 'Continue'
+                    }
+                    Output = @{
+                        CIFormat = 'None'
+                    }
                 })
 
             $t = $r.Containers[0].Blocks[0].Tests[0]
@@ -1011,7 +1019,13 @@ i -PassThru:$PassThru {
             }
 
             $r = Invoke-Pester -Configuration ([PesterConfiguration]@{
-                    Run = @{ ScriptBlock = $sb; PassThru = $true }
+                    Run    = @{
+                        ScriptBlock = $sb
+                        PassThru    = $true
+                    }
+                    Output = @{
+                        CIFormat = 'None'
+                    }
                 })
 
             $t = $r.Containers[0].Blocks[0].Tests[0]