Skip to content

Commit

Permalink
💡feat: allow for simpler override of Response header encoding of Forw…
Browse files Browse the repository at this point in the history 
…arded Requests (#2254)

* add support for ResponseHeaderEncoding in HttpClientConfig (progress commit)

* fix ConfigValidatorTests

* fix ConfigurationConfigProviderTests by updating setup config to set ResponseHeaderEncoding appropriately

* fix bug in suggested interface (probably an old name for the instance?)

* update documentation (also closes issue Document Kestrel response header encoding #1346)

* ..and finally, validate if ForwarderHttpClientFactory leverages the HttpClientConfig.ResponseHeaderEncoding appropriately

* oopsie daisy

* simplify documentation by leveraging XML comments for code

* re-add removed section that educates consumers about ensuring to set server (with e.g. Kestrel) options for header encoding to match SocketsHttpHandler's header encoding options

* Apply suggestions from code review

* Replace links with code references

---------

Co-authored-by: Miha Zupan <mihazupan.zupan1@gmail.com>
  • Loading branch information
@ChintanRaval @MihaZupan
ChintanRaval and MihaZupan committed Sep 22, 2023
1 parent c5899b3 commit 0f491bd
Show file tree
Hide file tree
Showing 13 changed files with 199 additions and 35 deletions.
27 changes: 14 additions & 13 deletions docs/docfx/articles/config-files.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ public Startup(IConfiguration configuration)
Configuration = configuration;
}

public void ConfigureServices(IServiceCollection services)
{
services.AddReverseProxy()
.LoadFromConfig(Configuration.GetSection("ReverseProxy"));
public void ConfigureServices(IServiceCollection services)
{
services.AddReverseProxy()
.LoadFromConfig(Configuration.GetSection("ReverseProxy"));
}

public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
Expand All @@ -27,11 +27,11 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
}

app.UseRouting();
app.UseEndpoints(endpoints =>
app.UseEndpoints(endpoints =>
{
endpoints.MapReverseProxy();
});
}
endpoints.MapReverseProxy();
});
}
```
**Note**: For details about middleware ordering see [here](https://docs.microsoft.com/aspnet/core/fundamentals/middleware/#middleware-order).

Expand Down Expand Up @@ -118,7 +118,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
},
"ReverseProxy": {
// Routes tell the proxy which requests to forward
"Routes": {
"Routes": {
"minimumroute" : {
// Matches anything and routes it to www.example.com
"ClusterId": "minimumcluster",
Expand All @@ -134,7 +134,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
"Authorization Policy" : "Anonymous", // Name of the policy or "Default", "Anonymous"
"CorsPolicy" : "Default", // Name of the CorsPolicy to apply to this route or "Default", "Disable"
"Match": {
"Path": "/something/{**remainder}", // The path to match using ASP.NET syntax.
"Path": "/something/{**remainder}", // The path to match using ASP.NET syntax.
"Hosts" : [ "www.aaaaa.com", "www.bbbbb.com"], // The host names to match, unspecified is any
"Methods" : [ "GET", "PUT" ], // The HTTP methods that match, uspecified is all
"Headers": [ // The headers to match, unspecified is any
Expand All @@ -161,7 +161,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
{
"RequestHeader": "MyHeader",
"Set": "MyValue",
}
}
]
}
},
Expand Down Expand Up @@ -194,7 +194,7 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
}
},
"HealthCheck": {
"Active": { // Makes API calls to validate the health.
"Active": { // Makes API calls to validate the health.
"Enabled": "true",
"Interval": "00:00:10",
"Timeout": "00:00:10",
Expand All @@ -212,7 +212,8 @@ For additional fields see [ClusterConfig](xref:Yarp.ReverseProxy.Configuration.C
"DangerousAcceptAnyServerCertificate" : false,
"MaxConnectionsPerServer" : 1024,
"EnableMultipleHttp2Connections" : true,
"RequestHeaderEncoding" : "Latin1" // How to interpret non ASCII characters in header values
"RequestHeaderEncoding" : "Latin1", // How to interpret non ASCII characters in request header values
"ResponseHeaderEncoding" : "Latin1" // How to interpret non ASCII characters in response header values
},
"HttpRequest" : { // Options for sending request to destination
"ActivityTimeout" : "00:02:00",
Expand Down
16 changes: 10 additions & 6 deletions docs/docfx/articles/http-client-config.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,14 @@ The configuration is represented differently if you're using the [IConfiguration
These types are focused on defining serializable configuration. The code based configuration model is described below in the "Code Configuration" section.

### HttpClient
HTTP client configuration is based on [HttpClientConfig](xref:Yarp.ReverseProxy.Configuration.HttpClientConfig) and represented by the following configuration schema.
HTTP client configuration is based on [HttpClientConfig](xref:Yarp.ReverseProxy.Configuration.HttpClientConfig) and represented by the following configuration schema. If you need a more granular approach, please use a [custom implementation](https://microsoft.github.io/reverse-proxy/articles/http-client-config.html#custom-iforwarderhttpclientfactory) of `IForwarderHttpClientFactory`.
```JSON
"HttpClient": {
"SslProtocols": [ "<protocol-names>" ],
"MaxConnectionsPerServer": "<int>",
"DangerousAcceptAnyServerCertificate": "<bool>",
"RequestHeaderEncoding": "<encoding-name>",
"ResponseHeaderEncoding": "<encoding-name>",
"EnableMultipleHttp2Connections": "<bool>"
"WebProxy": {
"Address": "<url>",
Expand All @@ -44,11 +45,15 @@ Configuration settings:
```JSON
"DangerousAcceptAnyServerCertificate": "true"
```
- RequestHeaderEncoding - enables other than ASCII encoding for outgoing request headers. Setting this value will leverage [`SocketsHttpHandler.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/system.net.http.socketshttphandler.requestheaderencodingselector) and use the selected encoding for all headers. If you need more granular approach, please use custom `IProxyHttpClientFactory`. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/dotnet/api/system.text.encoding.getencoding#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc.
- RequestHeaderEncoding - enables other than ASCII encoding for outgoing request headers. Setting this value will leverage [`SocketsHttpHandler.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/system.net.http.socketshttphandler.requestheaderencodingselector) and use the selected encoding for all headers. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/dotnet/api/system.text.encoding.getencoding#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc.
```JSON
"RequestHeaderEncoding": "utf-8"
```
If you're using an encoding other than ASCII (or UTF-8 for Kestrel) you also need to set your server to accept requests with such headers. For example, use [`KestrelServerOptions.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.RequestHeaderEncodingSelector) to set up Kestrel to accept Latin1 ("iso-8859-1") headers:
- ResponseHeaderEncoding - enables other than ASCII encoding for incoming response headers (from requests that the proxy would forward out). Setting this value will leverage [`SocketsHttpHandler.ResponseHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/system.net.http.socketshttphandler.responseheaderencodingselector) and use the selected encoding for all headers. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/dotnet/api/system.text.encoding.getencoding#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc.
```JSON
"ResponseHeaderEncoding": "utf-8"
```
Note that if you're using an encoding other than ASCII, you also need to set your server to accept requests and/or send responses with such headers. For example, when using Kestrel as the server, use [`KestrelServerOptions.RequestHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.RequestHeaderEncodingSelector) / [`.ResponseHeaderEncodingSelector`](https://docs.microsoft.com/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.ResponseHeaderEncodingSelector) to configure Kestrel to allow `Latin1` ("`iso-8859-1`") headers:
```C#
private static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
Expand All @@ -58,6 +63,8 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
.ConfigureKestrel(kestrel =>
{
kestrel.RequestHeaderEncodingSelector = _ => Encoding.Latin1;
// and/or
kestrel.ResponseHeaderEncodingSelector = _ => Encoding.Latin1;
});
});
```
Expand All @@ -77,9 +84,6 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
}
```

At the moment, there is no solution for changing encoding for response headers in Kestrel (see [aspnetcore#26334](https://github.com/dotnet/aspnetcore/issues/26334)), only ASCII is accepted.


### HttpRequest
HTTP request configuration is based on [ForwarderRequestConfig](xref:Yarp.ReverseProxy.Forwarder.ForwarderRequestConfig) and represented by the following configuration schema.
```JSON
Expand Down
7 changes: 4 additions & 3 deletions samples/ReverseProxy.Config.Sample/appsettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"Authorization Policy": "Anonymous", // Name of the policy or "Default", "Anonymous"
"CorsPolicy": "disable", // Name of the CorsPolicy to apply to this route or "default", "disable"
"Match": { // Rules that have to be met for the route to match the request
"Path": "/download/{**remainder}", // The path to match using ASP.NET syntax.
"Path": "/download/{**remainder}", // The path to match using ASP.NET syntax.
"Hosts": [ "localhost", "www.aaaaa.com", "www.bbbbb.com" ], // The host names to match, unspecified is any
"Methods": [ "GET", "PUT" ], // The HTTP methods that match, unspecified is all
"Headers": [ // The headers to match, unspecified is any
Expand Down Expand Up @@ -91,7 +91,7 @@
"AffinityKeyName": "MySessionCookieName" // Required, no default
},
"HealthCheck": { // Ways to determine which destinations should be filtered out due to unhealthy state
"Active": { // Makes API calls to validate the health of each destination
"Active": { // Makes API calls to validate the health of each destination
"Enabled": "true",
"Interval": "00:00:10", // How often to query for health data
"Timeout": "00:00:10", // Timeout for the health check request/response
Expand All @@ -110,7 +110,8 @@
"DangerousAcceptAnyServerCertificate": true, // Disables destination cert validation
"MaxConnectionsPerServer": 1024, // Destination server can further limit this number
"EnableMultipleHttp2Connections": true,
"RequestHeaderEncoding": "Latin1" // How to interpret non ASCII characters in header values
"RequestHeaderEncoding": "Latin1", // How to interpret non ASCII characters in proxied request's header values
"ResponseHeaderEncoding": "Latin1" // How to interpret non ASCII characters in proxied request's response header values
},
"HttpRequest": { // Options for sending request to destination
"Timeout": "00:02:00", // Timeout for the HttpRequest
Expand Down
3 changes: 2 additions & 1 deletion src/ReverseProxy/Configuration/ConfigProvider/ConfigurationConfigProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ private static RouteQueryParameter CreateRouteQueryParameter(IConfigurationSecti
MaxConnectionsPerServer = section.ReadInt32(nameof(HttpClientConfig.MaxConnectionsPerServer)),
EnableMultipleHttp2Connections = section.ReadBool(nameof(HttpClientConfig.EnableMultipleHttp2Connections)),
RequestHeaderEncoding = section[nameof(HttpClientConfig.RequestHeaderEncoding)],
ResponseHeaderEncoding = section[nameof(HttpClientConfig.ResponseHeaderEncoding)],
WebProxy = webProxy
};
}
Expand Down Expand Up @@ -377,7 +378,7 @@ private static DestinationConfig CreateDestination(IConfigurationSection section
Address = section[nameof(DestinationConfig.Address)]!,
Health = section[nameof(DestinationConfig.Health)],
Metadata = section.GetSection(nameof(DestinationConfig.Metadata)).ReadStringDictionary(),
Host = section[nameof(DestinationConfig.Host)]
Host = section[nameof(DestinationConfig.Host)]
};
}

Expand Down
21 changes: 17 additions & 4 deletions src/ReverseProxy/Configuration/ConfigValidator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -455,16 +455,29 @@ private static void ValidateProxyHttpClient(IList<Exception> errors, ClusterConf
errors.Add(new ArgumentException($"Max connections per server limit set on the cluster '{cluster.ClusterId}' must be positive."));
}

var encoding = cluster.HttpClient.RequestHeaderEncoding;
if (encoding is not null)
var requestHeaderEncoding = cluster.HttpClient.RequestHeaderEncoding;
if (requestHeaderEncoding is not null)
{
try
{
Encoding.GetEncoding(encoding);
Encoding.GetEncoding(requestHeaderEncoding);
}
catch (ArgumentException aex)
{
errors.Add(new ArgumentException($"Invalid header encoding '{encoding}'.", aex));
errors.Add(new ArgumentException($"Invalid request header encoding '{requestHeaderEncoding}'.", aex));
}
}

var responseHeaderEncoding = cluster.HttpClient.ResponseHeaderEncoding;
if (responseHeaderEncoding is not null)
{
try
{
Encoding.GetEncoding(responseHeaderEncoding);
}
catch (ArgumentException aex)
{
errors.Add(new ArgumentException($"Invalid response header encoding '{responseHeaderEncoding}'.", aex));
}
}
}
Expand Down
40 changes: 38 additions & 2 deletions src/ReverseProxy/Configuration/HttpClientConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,20 @@
// Licensed under the MIT License.

using System;
using System.Net.Http;
using System.Security.Authentication;
using System.Text;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Yarp.ReverseProxy.Forwarder;

namespace Yarp.ReverseProxy.Configuration;

/// <summary>
/// Options used for communicating with the destination servers.
/// </summary>
/// <remarks>
/// If you need a more granular approach, please use a <see href="https://microsoft.github.io/reverse-proxy/articles/http-client-config.html#custom-iforwarderhttpclientfactory">custom implementation of <see cref="IForwarderHttpClientFactory"/></see>.
/// </remarks>
public sealed record HttpClientConfig
{
/// <summary>
Expand All @@ -33,7 +40,7 @@ public sealed record HttpClientConfig
public int? MaxConnectionsPerServer { get; init; }

/// <summary>
/// Optional web proxy used when communicating with the destination server.
/// Optional web proxy used when communicating with the destination server.
/// </summary>
public WebProxyConfig? WebProxy { get; init; }

Expand All @@ -45,10 +52,37 @@ public sealed record HttpClientConfig
public bool? EnableMultipleHttp2Connections { get; init; }

/// <summary>
/// Enables non-ASCII header encoding for outgoing requests.
/// Allows overriding the default (ASCII) encoding for outgoing request headers.
/// <para>
/// Setting this value will in turn set <see cref="SocketsHttpHandler.RequestHeaderEncodingSelector"/> and use the selected encoding for all request headers.
/// The value is then parsed by <see cref="Encoding.GetEncoding(string)"/>, so use values like: "utf-8", "iso-8859-1", etc.
/// </para>
/// </summary>
/// <remarks>
/// Note: If you're using an encoding other than UTF-8 here, then you may also need to configure your server to accept request headers with such an encoding via the corresponding options for the server.
/// <para>
/// For example, when using Kestrel as the server, use <see cref="KestrelServerOptions.RequestHeaderEncodingSelector"/> to
/// <see href="https://learn.microsoft.com/en-us/aspnet/core/fundamentals/servers/kestrel/options">configure Kestrel</see> to use the same encoding.
/// </para>
/// </remarks>
public string? RequestHeaderEncoding { get; init; }

/// <summary>
/// Allows overriding the default (Latin1) encoding for incoming request headers.
/// <para>
/// Setting this value will in turn set <see cref="SocketsHttpHandler.ResponseHeaderEncodingSelector"/> and use the selected encoding for all response headers.
/// The value is then parsed by <see cref="Encoding.GetEncoding(string)"/>, so use values like: "utf-8", "iso-8859-1", etc.
/// </para>
/// </summary>
/// <remarks>
/// Note: If you're using an encoding other than ASCII here, then you may also need to configure your server to send response headers with such an encoding via the corresponding options for the server.
/// <para>
/// For example, when using Kestrel as the server, use <see cref="KestrelServerOptions.ResponseHeaderEncodingSelector"/> to
/// <see href="https://learn.microsoft.com/en-us/aspnet/core/fundamentals/servers/kestrel/options">configure Kestrel</see> to use the same encoding.
/// </para>
/// </remarks>
public string? ResponseHeaderEncoding { get; init; }

public bool Equals(HttpClientConfig? other)
{
if (other is null)
Expand All @@ -62,6 +96,7 @@ public bool Equals(HttpClientConfig? other)
&& EnableMultipleHttp2Connections == other.EnableMultipleHttp2Connections
// Comparing by reference is fine here since Encoding.GetEncoding returns the same instance for each encoding.
&& RequestHeaderEncoding == other.RequestHeaderEncoding
&& ResponseHeaderEncoding == other.ResponseHeaderEncoding
&& WebProxy == other.WebProxy;
}

Expand All @@ -72,6 +107,7 @@ public override int GetHashCode()
MaxConnectionsPerServer,
EnableMultipleHttp2Connections,
RequestHeaderEncoding,
ResponseHeaderEncoding,
WebProxy);
}
}
6 changes: 6 additions & 0 deletions src/ReverseProxy/Forwarder/ForwarderHttpClientFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ protected virtual void ConfigureHandler(ForwarderHttpClientContext context, Sock
handler.RequestHeaderEncodingSelector = (_, _) => encoding;
}

if (newConfig.ResponseHeaderEncoding is not null)
{
var encoding = Encoding.GetEncoding(newConfig.ResponseHeaderEncoding);
handler.ResponseHeaderEncodingSelector = (_, _) => encoding;
}

var webProxy = TryCreateWebProxy(newConfig.WebProxy);
if (webProxy is not null)
{
Expand Down
6 changes: 4 additions & 2 deletions test/ReverseProxy.Tests/Configuration/ClusterConfigTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ public void Equals_Same_Value_Returns_True()
SslProtocols = SslProtocols.Tls11 | SslProtocols.Tls12,
MaxConnectionsPerServer = 10,
DangerousAcceptAnyServerCertificate = true,
RequestHeaderEncoding = Encoding.UTF8.WebName
RequestHeaderEncoding = Encoding.UTF8.WebName,
ResponseHeaderEncoding = Encoding.UTF8.WebName
},
HttpRequest = new ForwarderRequestConfig
{
Expand Down Expand Up @@ -161,7 +162,8 @@ public void Equals_Same_Value_Returns_True()
SslProtocols = SslProtocols.Tls11 | SslProtocols.Tls12,
MaxConnectionsPerServer = 10,
DangerousAcceptAnyServerCertificate = true,
RequestHeaderEncoding = Encoding.UTF8.WebName
RequestHeaderEncoding = Encoding.UTF8.WebName,
ResponseHeaderEncoding = Encoding.UTF8.WebName
},
HttpRequest = new ForwarderRequestConfig
{
Expand Down

0 comments on commit 0f491bd

Please sign in to comment.