Merge c71ce03 into b1dacab

docs/Tutorials/Logging/Overview.md

@@ -5,7 +5,7 @@ There are two aspects to logging in Pode: Methods and Types.
 * Methods define how log items should be recorded, such as to a file or terminal.
 * Types define how items to log are transformed, and what should be supplied to the Method.
 
-For example when you supply an Exception to  [`Write-PodeErrorLog`](../../../Functions/Logging/Write-PodeErrorLog), this Exception is first supplied to Pode's inbuilt Error type. This type transforms any Exception (or Error Record) into a string which can then be supplied to the File logging method.
+For example when you supply an Exception to [`Write-PodeErrorLog`](../../../Functions/Logging/Write-PodeErrorLog), this Exception is first supplied to Pode's inbuilt Error logging type. This type transforms any Exception (or Error Record) into a string which can then be supplied to the File logging method.
 
 In Pode you can use File, Terminal or a Custom method. As well as Request, Error, or a Custom type.
 
@@ -53,7 +53,7 @@ Instead of masking the whole value that matches, there is support for two RegEx
 
 Specifying either of these groups in your pattern will keep the original value in place rather than masking it.
 
-For example, expanding on the above example, to keep the `Password=` text you could use the following:
+For example, expanding on the above, to keep the `Password=` text you could use the following:
 
 ```powershell
 @{
@@ -93,3 +93,19 @@ To specify a custom mask, you can do this in the configuration file:
     }
 }
 ```
+
+## Batches
+
+By default all log items are recorded one-by-one, but this can obviously become very slow if a lot of log items are being processed.
+
+To help speed this up, you can specify a batch size on your logging method:
+
+```powershell
+New-PodeLoggingMethod -Terminal -Batch 10 | Enable-PodeRequestLogging
+```
+
+Instead of writing logs one-by-one, the above will keep transformed log items in an array. Once the array matches the batch size of 10, all items will be sent to the method at once.
+
+This means that the method's scriptblock will receive an array of items, rather than a single item.
+
+You can also sent a `-BatchTimeout` value, in seconds, so that if your batch size it 10 but only 5 log items are added, then after the timeout value the logs items will be sent to your method.

examples/web-pages.ps1

@@ -26,7 +26,7 @@ Start-PodeServer -Threads 2 {
     Add-PodeAccessRule -Access Deny -Type IP -Values all
 
     # log requests to the terminal
-    New-PodeLoggingMethod -Terminal | Enable-PodeRequestLogging
+    New-PodeLoggingMethod -Terminal -Batch 10 -BatchTimeout 10 | Enable-PodeRequestLogging
     New-PodeLoggingMethod -Terminal | Enable-PodeErrorLogging
 
     # set view engine to pode renderer

src/Private/Logging.ps1

@@ -7,6 +7,12 @@ function Get-PodeLoggingTerminalMethod
             return
         }
 
+        # check if it's an array from batching
+        if ($item -is [array]) {
+            $item = ($item -join [System.Environment]::NewLine)
+        }
+
+        # protect then write
         $item = ($item | Protect-PodeLogItem)
         $item.ToString() | Out-PodeHost
     }
@@ -17,6 +23,11 @@ function Get-PodeLoggingFileMethod
     return {
         param($item, $options)
 
+        # check if it's an array from batching
+        if ($item -is [array]) {
+            $item = ($item -join [System.Environment]::NewLine)
+        }
+
         # mask values
         $item = ($item | Protect-PodeLogItem)
 
@@ -252,25 +263,60 @@ function Start-PodeLoggingRunspace
     $script = {
         while ($true)
         {
-            # if there are no logs to process, just sleep few a few seconds
+            # if there are no logs to process, just sleep for a few seconds - but after checking the batch
             if ($PodeContext.LogsToProcess.Count -eq 0) {
+                Test-PodeLoggerBatches
                 Start-Sleep -Seconds 5
                 continue
             }
 
-            # safetly pop off the first log from the array
+            # safely pop off the first log from the array
             $log = (Lock-PodeObject -Return -Object $PodeContext.LogsToProcess -ScriptBlock {
                 $log = $PodeContext.LogsToProcess[0]
                 $PodeContext.LogsToProcess.RemoveAt(0) | Out-Null
                 return $log
             })
 
-            if ($null -ne $log) {
-                # run the log item through the appropriate method, then through the storage script
-                $logger = Get-PodeLogger -Name $log.Name
+            # run the log item through the appropriate method
+            $logger = Get-PodeLogger -Name $log.Name
+            $now = [datetime]::Now
+
+            # if the log is null, check batch then sleep and skip
+            if ($null -eq $log) {
+                Start-Sleep -Milliseconds 100
+                continue
+            }
+
+            # convert to log item into a writable format
+            $result = @(Invoke-PodeScriptBlock -ScriptBlock $logger.ScriptBlock -Arguments (@($log.Item) + @($logger.Arguments)) -Return -Splat)
+
+            # check batching
+            $batch = $logger.Method.Batch
+            if ($batch.Size -gt 1) {
+                # add current item to batch
+                $batch.Items += $result
+                $batch.LastUpdate = $now
+                $result = $null
+
+                # if the current amount of items matches the batch, write
+                if ($batch.Items.Length -ge $batch.Size) {
+                    $result = $batch.Items
+                }
+
+                # check if we have timed-out, if so, write
+                if (($batch.Timeout -gt 0) -and ($batch.LastUpdate.AddSeconds($batch.Timeout) -le $now)) {
+                    $result = $batch.Items
+                }
+
+                # if we're writing, reset the items
+                if ($null -ne $result) {
+                    $batch.Items = @()
+                }
+            }
 
-                $result = @(Invoke-PodeScriptBlock -ScriptBlock $logger.ScriptBlock -Arguments (@($log.Item) + @($logger.Arguments)) -Return -Splat)
-                Invoke-PodeScriptBlock -ScriptBlock $logger.Method.ScriptBlock -Arguments (@($result) + @($logger.Method.Arguments)) -Splat
+            # send the writable log item off to the log writer
+            if ($null -ne $result) {
+                Invoke-PodeScriptBlock -ScriptBlock $logger.Method.ScriptBlock -Arguments (@(,$result) + @($logger.Method.Arguments)) -Splat
             }
 
             # small sleep to lower cpu usage
@@ -279,4 +325,22 @@ function Start-PodeLoggingRunspace
     }
 
     Add-PodeRunspace -Type 'Main' -ScriptBlock $script
+}
+
+function Test-PodeLoggerBatches
+{
+    $now = [datetime]::Now
+
+    # check each logger, and see if its batch needs to be written
+    foreach ($logger in $PodeContext.Server.Logging.Types.Values)
+    {
+        $batch = $logger.Method.Batch
+        if (($batch.Size -gt 1) -and ($batch.Items.Length -gt 0) -and
+            ($batch.Timeout -gt 0) -and ($null -ne $batch.LastUpdate) -and ($batch.LastUpdate.AddSeconds($batch.Timeout) -le $now))
+        {
+            $result = $batch.Items
+            $batch.Items = @()
+            Invoke-PodeScriptBlock -ScriptBlock $logger.Method.ScriptBlock -Arguments (@(,$result) + @($logger.Method.Arguments)) -Splat
+        }
+    }
 }

src/Public/Logging.ps1

@@ -17,6 +17,12 @@ The File Path of where to store the logs.
 .PARAMETER Name
 The File Name to prepend new log files using.
 
+.PARAMETER Batch
+An optional batch size to write log items in bulk (Default: 1)
+
+.PARAMETER BatchTimeout
+An optional batch timeout, in seconds, to send items off for writing if a log item isn't received (Default: 0)
+
 .PARAMETER MaxDays
 The maximum number of days to keep logs, before Pode automatically removes them.
 
@@ -62,6 +68,14 @@ function New-PodeLoggingMethod
         [string]
         $Name,
 
+        [Parameter()]
+        [int]
+        $Batch = 1,
+
+        [Parameter()]
+        [int]
+        $BatchTimeout = 0,
+
         [Parameter(ParameterSetName='File')]
         [ValidateScript({
             if ($_ -lt 0) {
@@ -104,10 +118,20 @@ function New-PodeLoggingMethod
         $ArgumentList
     )
 
+    # batch details
+    $batchInfo = @{
+        Size = $Batch
+        Timeout = $BatchTimeout
+        LastUpdate = $null
+        Items = @()
+    }
+
+    # return info on appropriate logging type
     switch ($PSCmdlet.ParameterSetName.ToLowerInvariant()) {
         'terminal' {
             return @{
                 ScriptBlock = (Get-PodeLoggingTerminalMethod)
+                Batch = $batchInfo
                 Arguments = @{}
             }
         }
@@ -119,6 +143,7 @@ function New-PodeLoggingMethod
 
             return @{
                 ScriptBlock = (Get-PodeLoggingFileMethod)
+                Batch = $batchInfo
                 Arguments = @{
                     Name = $Name
                     Path = $Path
@@ -134,6 +159,7 @@ function New-PodeLoggingMethod
         'custom' {
             return @{
                 ScriptBlock = $ScriptBlock
+                Batch = $batchInfo
                 Arguments = $ArgumentList
             }
         }