Add folded response header support to SocketsHttpHandler (#27727)

* Add folded response header support to SocketsHttpHandler

The feature is deprecated by RFC 7230, but some servers still utilize it, and if they do, SocketsHttpHandler currently fails (WinHttpHandler and the netfx handler both allow it, CurlHandler currently fails as well).  This commit updates the header parsing logic to allow for them.

* Address PR feedback

Author: stephentoub

src/Common/tests/System/Net/Http/LoopbackServer.cs

@@ -74,15 +74,15 @@ public static Task CreateServerAsync(Func<LoopbackServer, Uri, Task> funcAsync,
             return CreateServerAsync(server => funcAsync(server, server.Uri), options);
         }
 
-        public static Task CreateClientAndServerAsync(Func<Uri, Task> clientFunc, Func<LoopbackServer, Task> serverFunc)
+        public static Task CreateClientAndServerAsync(Func<Uri, Task> clientFunc, Func<LoopbackServer, Task> serverFunc, Options options = null)
         {
             return CreateServerAsync(async server =>
             {
                 Task clientTask = clientFunc(server.Uri);
                 Task serverTask = serverFunc(server);
 
                 await new Task[] { clientTask, serverTask }.WhenAllOrAnyFailed();
-            });
+            }, options);
         }
 
         public async Task AcceptConnectionAsync(Func<Connection, Task> funcAsync)

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs

@@ -66,7 +66,7 @@ internal partial class HttpConnection : IDisposable
 
         public HttpConnection(
             HttpConnectionPool pool,
-            Stream stream, 
+            Stream stream,
             TransportContext transportContext)
         {
             Debug.Assert(pool != null);
@@ -413,7 +413,7 @@ public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request,
 
                 // Parse the response status line.
                 var response = new HttpResponseMessage() { RequestMessage = request, Content = new HttpConnectionResponseContent() };
-                ParseStatusLine(await ReadNextLineAsync().ConfigureAwait(false), response);
+                ParseStatusLine(await ReadNextResponseHeaderLineAsync().ConfigureAwait(false), response);
 
                 // If we sent an Expect: 100-continue header, handle the response accordingly.
                 if (allowExpect100ToContinue != null)
@@ -441,12 +441,12 @@ public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request,
                         if (response.StatusCode == HttpStatusCode.Continue)
                         {
                             // We got our continue header.  Read the subsequent empty line and parse the additional status line.
-                            if (!LineIsEmpty(await ReadNextLineAsync().ConfigureAwait(false)))
+                            if (!LineIsEmpty(await ReadNextResponseHeaderLineAsync().ConfigureAwait(false)))
                             {
                                 ThrowInvalidHttpResponse();
                             }
 
-                            ParseStatusLine(await ReadNextLineAsync().ConfigureAwait(false), response);
+                            ParseStatusLine(await ReadNextResponseHeaderLineAsync().ConfigureAwait(false), response);
                         }
                     }
                 }
@@ -463,7 +463,7 @@ public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request,
                 // Parse the response headers.
                 while (true)
                 {
-                    ArraySegment<byte> line = await ReadNextLineAsync().ConfigureAwait(false);
+                    ArraySegment<byte> line = await ReadNextResponseHeaderLineAsync(foldedHeadersAllowed: true).ConfigureAwait(false);
                     if (LineIsEmpty(line))
                     {
                         break;
@@ -600,7 +600,7 @@ public Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool doRe
         }
 
         private HttpContentWriteStream CreateRequestContentStream(HttpRequestMessage request)
-    {
+        {
             bool requestTransferEncodingChunked = request.HasHeaders && request.Headers.TransferEncodingChunked == true;
             HttpContentWriteStream requestContentStream = requestTransferEncodingChunked ? (HttpContentWriteStream)
                 new ChunkedEncodingWriteStream(this) :
@@ -1060,16 +1060,13 @@ private bool TryReadNextLine(out ReadOnlySpan<byte> line)
             int bytesConsumed = length + 1;
             _readOffset += bytesConsumed;
             _allowedReadLineBytes -= bytesConsumed;
-            if (_allowedReadLineBytes < 0)
-            {
-                ThrowInvalidHttpResponse();
-            }
+            ThrowIfExceededAllowedReadLineBytes();
 
             line = buffer.Slice(0, length > 0 && buffer[length - 1] == '\r' ? length - 1 : length);
             return true;
         }
 
-        private async ValueTask<ArraySegment<byte>> ReadNextLineAsync()
+        private async ValueTask<ArraySegment<byte>> ReadNextResponseHeaderLineAsync(bool foldedHeadersAllowed = false)
         {
             int previouslyScannedBytes = 0;
             while (true)
@@ -1080,34 +1077,94 @@ private async ValueTask<ArraySegment<byte>> ReadNextLineAsync()
                 {
                     int startIndex = _readOffset;
                     int length = lfIndex - startIndex;
-                    if (length > 0 && _readBuffer[startIndex + length - 1] == '\r')
+                    if (lfIndex > 0 && _readBuffer[lfIndex - 1] == '\r')
                     {
                         length--;
                     }
 
-                    // Advance read position past the LF
-                    _allowedReadLineBytes -= lfIndex + 1 - scanOffset;
-                    if (_allowedReadLineBytes < 0)
+                    // If this isn't the ending header, we need to account for the possibility
+                    // of folded headers, which per RFC2616 are headers split across multiple
+                    // lines, where the continuation line begins with a space or horizontal tab.
+                    // The feature was deprecated in RFC 7230 3.2.4, but some servers still use it.
+                    if (foldedHeadersAllowed && length > 0)
                     {
-                        ThrowInvalidHttpResponse();
+                        // If the newline is the last character we've buffered, we need at least
+                        // one more character in order to see whether it's space/tab, in which
+                        // case it's a folded header.
+                        if (lfIndex + 1 == _readLength)
+                        {
+                            // The LF is at the end of the buffer, so we need to read more
+                            // to determine whether there's a continuation.  We'll read
+                            // and then loop back around again, but to avoid needing to
+                            // rescan the whole header, reposition to one character before
+                            // the newline so that we'll find it quickly.
+                            int backPos = _readBuffer[lfIndex - 1] == '\r' ? lfIndex - 2 : lfIndex - 1;
+                            Debug.Assert(backPos >= 0);
+                            previouslyScannedBytes = backPos - _readOffset;
+                            _allowedReadLineBytes -= backPos - scanOffset;
+                            ThrowIfExceededAllowedReadLineBytes();
+                            await FillAsync().ConfigureAwait(false);
+                            continue;
+                        }
+
+                        // We have at least one more character we can look at.
+                        Debug.Assert(lfIndex + 1 < _readLength);
+                        char nextChar = (char)_readBuffer[lfIndex + 1];
+                        if (nextChar == ' ' || nextChar == '\t')
+                        {
+                            // The next header is a continuation.
+
+                            // Folded headers are only allowed within header field values, not within header field names,
+                            // so if we haven't seen a colon, this is invalid.
+                            if (Array.IndexOf(_readBuffer, (byte)':', _readOffset, lfIndex - _readOffset) == -1)
+                            {
+                                ThrowInvalidHttpResponse();
+                            }
+
+                            // When we return the line, we need the interim newlines filtered out. According
+                            // to RFC 7230 3.2.4, a valid approach to dealing with them is to "replace each
+                            // received obs-fold with one or more SP octets prior to interpreting the field
+                            // value or forwarding the message downstream", so that's what we do.
+                            _readBuffer[lfIndex] = (byte)' ';
+                            if (_readBuffer[lfIndex - 1] == '\r')
+                            {
+                                _readBuffer[lfIndex - 1] = (byte)' ';
+                            }
+
+                            // Update how much we've read, and simply go back to search for the next newline.
+                            previouslyScannedBytes = (lfIndex + 1 - _readOffset);
+                            _allowedReadLineBytes -= (lfIndex + 1 - scanOffset);
+                            ThrowIfExceededAllowedReadLineBytes();
+                            continue;
+                        }
+
+                        // Not at the end of a header with a continuation.
                     }
+
+                    // Advance read position past the LF
+                    _allowedReadLineBytes -= lfIndex + 1 - scanOffset;
+                    ThrowIfExceededAllowedReadLineBytes();
                     _readOffset = lfIndex + 1;
 
                     return new ArraySegment<byte>(_readBuffer, startIndex, length);
                 }
 
-                // Couldn't find LF.  Read more.
-                // Note this may cause _readOffset to change.
+                // Couldn't find LF.  Read more. Note this may cause _readOffset to change.
                 previouslyScannedBytes = _readLength - _readOffset;
                 _allowedReadLineBytes -= _readLength - scanOffset;
-                if (_allowedReadLineBytes < 0)
-                {
-                    ThrowInvalidHttpResponse();
-                }
+                ThrowIfExceededAllowedReadLineBytes();
                 await FillAsync().ConfigureAwait(false);
             }
         }
 
+        private void ThrowIfExceededAllowedReadLineBytes()
+        {
+            if (_allowedReadLineBytes < 0)
+            {
+                ThrowInvalidHttpResponse();
+            }
+        }
+
         // Throws IOException on EOF.  This is only called when we expect more data.
         private async Task FillAsync()
         {

src/System.Net.Http/tests/FunctionalTests/DribbleStream.cs

@@ -26,6 +26,7 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc
                 for (int i = 0; i < count; i++)
                 {
                     await _wrapped.WriteAsync(buffer, offset + i, 1);
+                    await _wrapped.FlushAsync();
                     await Task.Yield(); // introduce short delays, enough to send packets individually but not so long as to extend test duration significantly
                 }
             }
@@ -41,6 +42,7 @@ public override void Write(byte[] buffer, int offset, int count)
                 for (int i = 0; i < count; i++)
                 {
                     _wrapped.Write(buffer, offset + i, 1);
+                    _wrapped.Flush();
                 }
             }
             catch (IOException) when (_clientDisconnectAllowed)