diff --git a/docs/Tutorials/Compression/Requests.md b/docs/Tutorials/Compression/Requests.md new file mode 100644 index 000000000..e2fbda86c --- /dev/null +++ b/docs/Tutorials/Compression/Requests.md @@ -0,0 +1,118 @@ +# Requests + +You can send Requests to your Pode server that use compression on the payload, such as a JSON payload compressed via GZip. + +Pode supports the following compression methods: + +* gzip +* deflate + +There are a number of ways you can specify the compression type, and these are defined below. When your request uses compression, Pode will first decompress the payload, and then attempt to parse it if needed. + +## Request + +The most common way is to define the a request's compression type in the request's headers. The header to use depends on which web-server type you're using: + +| Server Type | Header | +| ----------- | ------ | +| HttpListener | X-Transfer-Encoding | +| Pode | Transfer-Encoding | + +HttpListener is the default web server, unless you specify `-Type Pode` on `Start-PodeServer`. The reason for HttpListener using a slightly different header to the normal, is because HttpListener doesn't properly support compression; it will error with a 501 if you set the `Transfer-Encoding` header to either `gzip` or `deflate`. + +!!! note + The Pode server does also support `X-Transfer-Encoding`, but it will check `Transfer-Encoding` first. + +## Route + +Like content types, you can force a Route to use a specific transfer encoding by using the `-TransferEncoding` parameter on [`Add-PodeRoute`](../../../Functions/Routes/Add-PodeRoute). If specified, Pode will use this compression type to decompress the payload regardless if the header is present or not. + +```powershell +Add-PodeRoute -Method Get -Path '/' -TransferEncoding gzip -ScriptBlock { + # logic +} +``` + +## Configuration + +Using the `server.psd1` configuration file, you can define a default transfer encoding to use for every route, or you can define patterns to match multiple route paths to set transfer encodings on mass. + +### Default + +To define a default transfer encoding for everything, you can use the following configuration: + +```powershell +@{ + Web = @{ + TransferEncoding = @{ + Default = "gzip" + } + } +} +``` + +### Route Patterns + +You can define patterns to match multiple route paths, and any route that matches (when created) will have the appropriate transfer encoding set. + +For example, the following configuration in your `server.psd1` would bind all `/api` routes to `gzip`, and then all `/status` routes to `deflate`: + +```powershell +@{ + Web = @{ + TransferEncoding = @{ + Routes = @{ + "/api/*" = "gzip" + "/status/*" = "deflate" + } + } + } +} +``` + +## Precedence + +The transfer encoding that will be used is determined by the following order: + +1. Being defined on the Route. +2. The Route matches a pattern defined in the configuration file. +3. A default transfer encoding is defined in the configuration file. +4. The transfer encoding is supplied on the web request. + +## Example + +The following is an example of sending a `gzip` encoded payload to some `/ping` route: + +```powershell +# get the JSON message in bytes +$data = @{ + Name = "Deepthought" + Age = 42 +} + +$message = ($data | ConvertTo-Json) +$bytes = [System.Text.Encoding]::UTF8.GetBytes($message) + +# compress the message using gzip +$ms = New-Object -TypeName System.IO.MemoryStream +$gzip = New-Object System.IO.Compression.GZipStream($ms, [IO.Compression.CompressionMode]::Compress, $true) +$gzip.Write($bytes, 0, $bytes.Length) +$gzip.Close() +$ms.Position = 0 + +# Pode Server +Invoke-RestMethod ` + -Method Post ` + -Uri 'http://localhost:8080/ping' ` + -Body $ms.ToArray() ` + -TransferEncoding gzip ` + -ContentType application/json + +# HttpListener Server +Invoke-RestMethod ` + -Method Post ` + -Uri 'http://localhost:8080/ping' ` + -Body $ms.ToArray() ` + -ContentType application/json ` + -Headers @{'X-Transfer-Encoding' = 'gzip'} +``` diff --git a/examples/web-gzip-request.ps1 b/examples/web-gzip-request.ps1 new file mode 100644 index 000000000..074900bef --- /dev/null +++ b/examples/web-gzip-request.ps1 @@ -0,0 +1,21 @@ +$path = Split-Path -Parent -Path (Split-Path -Parent -Path $MyInvocation.MyCommand.Path) +Import-Module "$($path)/src/Pode.psm1" -Force -ErrorAction Stop + +# or just: +# Import-Module Pode + +# create a server, and start listening on port 8085 +Start-PodeServer -Threads 2 -Type Pode { + + # listen on localhost:8085 + Add-PodeEndpoint -Address * -Port 8085 -Protocol Http + + New-PodeLoggingMethod -Terminal | Enable-PodeErrorLogging + + # GET request that recieves gzip'd json + Add-PodeRoute -Method Post -Path '/users' -ScriptBlock { + param($e) + Write-PodeJsonResponse -Value $e.Data + } + +} \ No newline at end of file diff --git a/src/Private/Context.ps1 b/src/Private/Context.ps1 index ec2d4c13e..9750732ac 100644 --- a/src/Private/Context.ps1 +++ b/src/Private/Context.ps1 @@ -242,6 +242,11 @@ function New-PodeContext $ctx.Server.Middleware = @() $ctx.Server.BodyParsers = @{} + # common support values + $ctx.Server.Supported = @{ + TransferEncodings = @('gzip', 'deflate', 'x-gzip') + } + # endware that needs to run $ctx.Server.Endware = @() @@ -453,6 +458,10 @@ function Set-PodeWebConfiguration Default = $Configuration.ContentType.Default Routes = @{} } + TransferEncoding = @{ + Default = $Configuration.TransferEncoding.Default + Routes = @{} + } } # setup content type route patterns for forced content types @@ -462,6 +471,13 @@ function Set-PodeWebConfiguration $Context.Server.Web.ContentType.Routes[$_pattern] = $_type } + # setup transfer encoding route patterns for forced transfer encodings + $Configuration.TransferEncoding.Routes.Keys | Where-Object { ![string]::IsNullOrWhiteSpace($_) } | ForEach-Object { + $_type = $Configuration.TransferEncoding.Routes[$_] + $_pattern = (Convert-PodePathPatternToRegex -Path $_ -NotSlashes) + $Context.Server.Web.TransferEncoding.Routes[$_pattern] = $_type + } + # setup content type route patterns for error pages $Configuration.ErrorPages.Routes.Keys | Where-Object { ![string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_type = $Configuration.ErrorPages.Routes[$_] diff --git a/src/Private/Helpers.ps1 b/src/Private/Helpers.ps1 index 139cbb7f7..a3049cce3 100644 --- a/src/Private/Helpers.ps1 +++ b/src/Private/Helpers.ps1 @@ -1040,6 +1040,59 @@ function Test-PodeValidNetworkFailure return ($null -ne $match) } +function Get-PodeTransferEncoding +{ + param( + [Parameter()] + [string] + $TransferEncoding, + + [switch] + $ThrowError + ) + + # return if no encoding + if ([string]::IsNullOrWhiteSpace($TransferEncoding)) { + return [string]::Empty + } + + # get the encoding, and store invalid ones for later + $parts = @($TransferEncoding -isplit ',').Trim() + $normal = @('chunked', 'identity') + $invalid = @() + + # if we see a supported one, return immediately. else build up invalid one + foreach ($part in $parts) { + if ($part -iin $PodeContext.Server.Supported.TransferEncodings) { + if ($part -ieq 'x-gzip') { + return 'gzip' + } + + return $part + } + + if ($part -iin $normal) { + continue + } + + $invalid += $part + } + + # if we have any invalid, throw a 415 error + if ($invalid.Length -gt 0) { + if ($ThrowError) { + $err = [System.Net.Http.HttpRequestException]::new() + $err.Data.Add('PodeStatusCode', 415) + throw $err + } + + return $invalid[0] + } + + # else, we're safe + return [string]::Empty +} + function Get-PodeEncodingFromContentType { param( @@ -1052,10 +1105,10 @@ function Get-PodeEncodingFromContentType return [System.Text.Encoding]::UTF8 } - $parts = $ContentType -isplit ';' + $parts = @($ContentType -isplit ';').Trim() foreach ($part in $parts) { - if ($part.Trim().StartsWith('charset')) { + if ($part.StartsWith('charset')) { return [System.Text.Encoding]::GetEncoding(($part -isplit '=')[1].Trim()) } } @@ -1071,7 +1124,11 @@ function ConvertFrom-PodeRequestContent [Parameter()] [string] - $ContentType + $ContentType, + + [Parameter()] + [string] + $TransferEncoding ) # get the requests content type and boundary @@ -1102,11 +1159,31 @@ function ConvertFrom-PodeRequestContent } 'pode' { - $Content = $Request.Body.Value + # if the request is compressed, attempt to uncompress it + if (![string]::IsNullOrWhiteSpace($TransferEncoding)) { + # create a compressed stream to decompress the req bytes + $ms = New-Object -TypeName System.IO.MemoryStream + $ms.Write($Request.Body.Bytes, 0, $Request.Body.Bytes.Length) + $ms.Seek(0, 0) | Out-Null + $stream = New-Object "System.IO.Compression.$($TransferEncoding)Stream"($ms, [System.IO.Compression.CompressionMode]::Decompress) + + # read the decompressed bytes + $Content = Read-PodeStreamToEnd -Stream $stream -Encoding $Encoding + } + else { + $Content = $Request.Body.Value + } } default { - $Content = Read-PodeStreamToEnd -Stream $Request.InputStream -Encoding $Encoding + # if the request is compressed, attempt to uncompress it + if (![string]::IsNullOrWhiteSpace($TransferEncoding)) { + $stream = New-Object "System.IO.Compression.$($TransferEncoding)Stream"($Request.InputStream, [System.IO.Compression.CompressionMode]::Decompress) + $Content = Read-PodeStreamToEnd -Stream $stream -Encoding $Encoding + } + else { + $Content = Read-PodeStreamToEnd -Stream $Request.InputStream -Encoding $Encoding + } } } @@ -1195,8 +1272,8 @@ function ConvertFrom-PodeRequestContent $type = ConvertFrom-PodeBytesToString -Bytes $Lines[$bIndex+2] -Encoding $Encoding -RemoveNewLine $Result.Files.Add($fields.filename, @{ - 'ContentType' = (@($type -isplit ':')[1].Trim()); - 'Bytes' = $null; + ContentType = @($type -isplit ':')[1].Trim() + Bytes = $null }) $bytes = @() diff --git a/src/Private/Mappers.ps1 b/src/Private/Mappers.ps1 index c24d764d4..be07c126d 100644 --- a/src/Private/Mappers.ps1 +++ b/src/Private/Mappers.ps1 @@ -172,6 +172,7 @@ function Get-PodeContentType '.gsm' { return 'audio/x-gsm' } '.gtar' { return 'application/x-gtar' } '.gz' { return 'application/x-gzip' } + '.gzip' { return 'application/x-gzip' } '.h' { return 'text/plain' } '.hdf' { return 'application/x-hdf' } '.hdml' { return 'text/x-hdml' } diff --git a/src/Private/Middleware.ps1 b/src/Private/Middleware.ps1 index 3cde7d87a..7760661d3 100644 --- a/src/Private/Middleware.ps1 +++ b/src/Private/Middleware.ps1 @@ -198,6 +198,11 @@ function Get-PodeRouteValidateMiddleware $e.ContentType = $route.ContentType } + # override the transfer encoding from the route if it's not empty + if (![string]::IsNullOrWhiteSpace($route.TransferEncoding)) { + $e.TransferEncoding = $route.TransferEncoding + } + # set the content type for any pages for the route if it's not empty $e.ErrorType = $route.ErrorType @@ -214,7 +219,7 @@ function Get-PodeBodyMiddleware try { # attempt to parse that data - $result = ConvertFrom-PodeRequestContent -Request $e.Request -ContentType $e.ContentType + $result = ConvertFrom-PodeRequestContent -Request $e.Request -ContentType $e.ContentType -TransferEncoding $e.TransferEncoding # set session data $e.Data = $result.Data @@ -277,7 +282,9 @@ function Get-PodeCookieMiddleware $value = [string]::Empty if ($atoms.Length -gt 1) { - $value = ($atoms[1..($atoms.Length - 1)] -join ([string]::Empty)) + foreach ($atom in $atoms[1..($atoms.Length - 1)]) { + $value += $atom + } } $e.Cookies[$atoms[0]] = [System.Net.Cookie]::new($atoms[0], $value) diff --git a/src/Private/PodeServer.ps1 b/src/Private/PodeServer.ps1 index 98293b7af..d29f5f3ac 100644 --- a/src/Private/PodeServer.ps1 +++ b/src/Private/PodeServer.ps1 @@ -156,6 +156,7 @@ function Invoke-PodeSocketHandler Streamed = $true Route = $null Timestamp = [datetime]::UtcNow + TransferEncoding = $null } # set pode in server response header @@ -206,12 +207,19 @@ function Invoke-PodeSocketHandler Protocol = $req_info.Protocol ProtocolVersion = ($req_info.Protocol -isplit '/')[1] ContentEncoding = (Get-PodeEncodingFromContentType -ContentType $req_info.Headers['Content-Type']) + TransferEncoding = (Get-PodeTransferEncoding -TransferEncoding $req_info.Headers['Transfer-Encoding'] -ThrowError) + } + + # if the transfer encoding is empty, attempt X-Transfer-Encoding for support from HttpListener + if ([string]::IsNullOrWhiteSpace($WebEvent.Request.TransferEncoding)) { + $WebEvent.Request.TransferEncoding = (Get-PodeTransferEncoding -TransferEncoding $req_info.Headers['X-Transfer-Encoding'] -ThrowError) } $WebEvent.Path = $req_info.Uri.AbsolutePath $WebEvent.Method = $req_info.Method.ToLowerInvariant() $WebEvent.Endpoint = $req_info.Headers['Host'] $WebEvent.ContentType = $req_info.Headers['Content-Type'] + $WebEvent.TransferEncoding = $WebEvent.Request.TransferEncoding # parse the query string and convert it to a hashtable $WebEvent.Query = (Convert-PodeQueryStringToHashTable -Uri $req_info.Query) diff --git a/src/Private/Sockets.ps1 b/src/Private/Sockets.ps1 index 696559cb9..ffe140971 100644 --- a/src/Private/Sockets.ps1 +++ b/src/Private/Sockets.ps1 @@ -276,7 +276,7 @@ function Get-PodeServerRequestDetails ) # convert array to string - $Content = $PodeContext.Server.Encoding.GetString($bytes, 0, $bytes.Length) + $Content = $PodeContext.Server.Encoding.GetString($Bytes, 0, $Bytes.Length) # parse the request headers $newLine = "`r`n" @@ -319,9 +319,75 @@ function Get-PodeServerRequestDetails $req_headers[$name] = $value } + # attempt to get content length, and see if content is chunked + $contentLength = $req_headers['Content-Length'] + if (![string]::IsNullOrWhiteSpace($contentLength)) { + $contentLength = 0 + } + + $transferEncoding = $req_headers['Transfer-Encoding'] + if (![string]::IsNullOrWhiteSpace($transferEncoding)) { + $isChunked = $transferEncoding.Contains('chunked') + } + + # if chunked, and we have a content-length, fail + if ($isChunked -and ($contentLength -gt 0)) { + throw [System.Net.Http.HttpRequestException]::new("Cannot supply a Content-Length and a chunked Transfer-Encoding") + } + # then set the request body $req_body = ($req_lines[($req_body_index)..($req_lines.Length - 1)] -join $newLine) - $req_body_bytes = $bytes[($bytes.Length - $req_body.Length)..($bytes.Length - 1)] + + # then set the raw bytes of the request body + $start = 0 + + $lines = $req_lines[0..($req_body_index - 1)] + foreach ($line in $lines) { + $start += $line.Length + } + + $start += ($lines.Length * $newLine.Length) + + # if chunked + if ($isChunked) { + $length = -1 + $req_body_bytes = [byte[]]@() + + while ($length -ne 0) { + # get index of newline char, read start>index bytes as HEX for length + $index = [array]::IndexOf($Bytes, [byte]$newLine[0], $start) + $hexBytes = $Bytes[$start..($index - 1)] + + $hex = [string]::Empty + foreach ($b in $hexBytes) { + $hex += ([char]$b) + } + + # if length is 0, end + $length = [System.Convert]::ToInt32($hex, 16) + if ($length -eq 0) { + continue + } + + # read those X hex bytes from (newline index + newline length) + $start = $index + $newLine.Length + $end = $start + $length - 1 + $req_body_bytes += $Bytes[$start..$end] + + # skip bytes for ending newline, and set new start + $start = ($end + $newLine.Length + 1) + } + } + + # else if content-length + elseif ($contentLength -gt 0) { + $req_body_bytes = $Bytes[$start..($start + $contentLength)] + } + + # else read all + else { + $req_body_bytes = $Bytes[$start..($Bytes.Length - 1)] + } # build required URI details $req_uri = [uri]::new("$($Protocol)://$($req_headers['Host'])$($req_query)") diff --git a/src/Private/WebServer.ps1 b/src/Private/WebServer.ps1 index cebc025f0..0a922cde1 100644 --- a/src/Private/WebServer.ps1 +++ b/src/Private/WebServer.ps1 @@ -108,8 +108,11 @@ function Start-PodeWebServer Streamed = $true Route = $null Timestamp = [datetime]::UtcNow + TransferEncoding = $null } + $WebEvent.TransferEncoding = (Get-PodeTransferEncoding -TransferEncoding (Get-PodeHeader -Name 'x-transfer-encoding') -ThrowError) + # set pode in server response header Set-PodeServerHeader -AllowEmptyType @@ -126,9 +129,18 @@ function Start-PodeWebServer } } } + catch [System.Net.Http.HttpRequestException] { + $code = [int]($_.Exception.Data['PodeStatusCode']) + if ($code -le 0) { + $code = 400 + } + + Set-PodeResponseStatus -Code $code -Exception $_ + } catch { - Set-PodeResponseStatus -Code 500 -Exception $_ $_ | Write-PodeErrorLog + $_.Exception | Write-PodeErrorLog -CheckInnerException + Set-PodeResponseStatus -Code 500 -Exception $_ } finally { Update-PodeServerRequestMetrics -WebEvent $WebEvent diff --git a/src/Public/Routes.ps1 b/src/Public/Routes.ps1 index b4c783ab8..07f918d0c 100644 --- a/src/Public/Routes.ps1 +++ b/src/Public/Routes.ps1 @@ -29,6 +29,9 @@ The EndpointName of an Endpoint(s) this Route should be bound against. .PARAMETER ContentType The content type the Route should use when parsing any payloads. +.PARAMETER TransferEncoding +The transfer encoding the Route should use when parsing any payloads. + .PARAMETER ErrorContentType The content type of any error pages that may get returned. @@ -50,6 +53,9 @@ Add-PodeRoute -Method Post -Path '/users/:userId/message' -Middleware (Get-PodeC .EXAMPLE Add-PodeRoute -Method Post -Path '/user' -ContentType 'application/json' -ScriptBlock { /* logic */ } +.EXAMPLE +Add-PodeRoute -Method Post -Path '/user' -ContentType 'application/json' -TransferEncoding gzip -ScriptBlock { /* logic */ } + .EXAMPLE Add-PodeRoute -Method Get -Path '/api/cpu' -ErrorContentType 'application/json' -ScriptBlock { /* logic */ } @@ -94,6 +100,11 @@ function Add-PodeRoute [string] $ContentType, + [Parameter()] + [ValidateSet('', 'gzip', 'deflate')] + [string] + $TransferEncoding, + [Parameter()] [string] $ErrorContentType, @@ -189,6 +200,21 @@ function Add-PodeRoute } } + # workout a default transfer encoding for the route + if ([string]::IsNullOrWhiteSpace($TransferEncoding)) { + $TransferEncoding = $PodeContext.Server.Web.TransferEncoding.Default + + # find type by pattern from settings + $matched = ($PodeContext.Server.Web.TransferEncoding.Routes.Keys | Where-Object { + $Path -imatch $_ + } | Select-Object -First 1) + + # if we get a match, set it + if (!(Test-IsEmpty $matched)) { + $TransferEncoding = $PodeContext.Server.Web.TransferEncoding.Routes[$matched] + } + } + # add the route(s) Write-Verbose "Adding Route: [$($Method)] $($Path)" $newRoutes = @(foreach ($_endpoint in $endpoints) { @@ -199,6 +225,7 @@ function Add-PodeRoute Endpoint = $_endpoint.Address.Trim() EndpointName = $_endpoint.Name ContentType = $ContentType + TransferEncoding = $TransferEncoding ErrorType = $ErrorContentType Arguments = $ArgumentList Method = $Method