http trailer for http/1.1

src/System.Net.Http/ref/System.Net.Http.cs

@@ -191,6 +191,7 @@ public HttpResponseMessage() { }
         public HttpResponseMessage(System.Net.HttpStatusCode statusCode) { }
         public System.Net.Http.HttpContent Content { get { throw null; } set { } }
         public System.Net.Http.Headers.HttpResponseHeaders Headers { get { throw null; } }
+        public System.Net.Http.Headers.HttpResponseHeaders TrailingHeaders { get { throw null; } set { } }
         public bool IsSuccessStatusCode { get { throw null; } }
         public string ReasonPhrase { get { throw null; } set { } }
         public System.Net.Http.HttpRequestMessage RequestMessage { get { throw null; } set { } }

src/System.Net.Http/src/System.Net.Http.csproj

@@ -621,6 +621,7 @@
     <Reference Include="System.Diagnostics.Tools" />
     <Reference Include="System.Diagnostics.Tracing" />
     <Reference Include="System.IO.Compression" />
+    <Reference Include="System.Linq" />
     <Reference Include="System.Memory" />
     <Reference Include="System.Net.NetworkInformation" />
     <Reference Include="System.Net.Primitives" />

src/System.Net.Http/src/System/Net/Http/Headers/HttpResponseHeaders.cs

@@ -20,6 +20,7 @@ public sealed class HttpResponseHeaders : HttpHeaders
 
         private object[] _specialCollectionsSlots;
         private HttpGeneralHeaders _generalHeaders;
+        private static HttpResponseHeaders _emptyHeaders;
 
         #region Response Headers
 
@@ -160,5 +161,8 @@ internal override void AddHeaders(HttpHeaders sourceHeaders)
         }
 
         private HttpGeneralHeaders GeneralHeaders => _generalHeaders ?? (_generalHeaders = new HttpGeneralHeaders(this));
+
+        // Lazy initialization.
+        internal static HttpResponseHeaders EmptyResponseHeader => _emptyHeaders ?? (_emptyHeaders = new HttpResponseHeaders());
     }
 }

src/System.Net.Http/src/System/Net/Http/HttpResponseMessage.cs

@@ -13,6 +13,7 @@ public class HttpResponseMessage : IDisposable
 
         private HttpStatusCode _statusCode;
         private HttpResponseHeaders _headers;
+        private HttpResponseHeaders _trailingHeaders;
         private string _reasonPhrase;
         private HttpRequestMessage _requestMessage;
         private Version _version;
@@ -115,6 +116,20 @@ public HttpResponseHeaders Headers
             }
         }
 
+        public HttpResponseHeaders TrailingHeaders
+        {
+            get
+            {
+                // If trailing headers are not ready yet, return null.
+                return _trailingHeaders;
+            }
+            set
+            {
+                CheckDisposed();
+                _trailingHeaders = value;
+            }
+        }
+
         public HttpRequestMessage RequestMessage
         {
             get { return _requestMessage; }

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ChunkedEncodingReadStream.cs

@@ -3,8 +3,11 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Buffers.Text;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
+using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -27,8 +30,15 @@ private sealed class ChunkedEncodingReadStream : HttpContentReadStream
             private ulong _chunkBytesRemaining;
             /// <summary>The current state of the parsing state machine for the chunked response.</summary>
             private ParsingState _state = ParsingState.ExpectChunkHeader;
+            /// <summary>The metadata at the end of chunked messages.</summary>
+            private HttpResponseHeaders _trailers;
+            private HttpResponseMessage _response;
 
-            public ChunkedEncodingReadStream(HttpConnection connection) : base(connection) { }
+            public ChunkedEncodingReadStream(HttpConnection connection, HttpResponseMessage response) : base(connection)
+            {
+                Debug.Assert(response != null, "The HttpResponseMessage cannot be null.");
+                _response = response;
+            }
 
             public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken)
             {
@@ -279,6 +289,7 @@ private ReadOnlyMemory<byte> ReadChunkFromConnectionBuffer(int maxBytesToRead, C
 
                             while (true)
                             {
+                                // Folded header was deprecated in RFC 7230 3.2.4. Thus we don't support it.
                                 _connection._allowedReadLineBytes = MaxTrailingHeaderLength;
                                 if (!_connection.TryReadNextLine(out currentLine))
                                 {
@@ -300,8 +311,19 @@ private ReadOnlyMemory<byte> ReadChunkFromConnectionBuffer(int maxBytesToRead, C
                                     _state = ParsingState.Done;
                                     _connection.CompleteResponse();
                                     _connection = null;
+
+                                    // Reached EOF.
+                                    _response.TrailingHeaders = _trailers ??
+                                        HttpResponseHeaders.EmptyResponseHeader;
+
                                     break;
                                 }
+                                // Parse the trailers.
+                                else
+                                {
+                                    if (_trailers == null) _trailers = new HttpResponseHeaders();
+                                    ParseHeaderNameValue(currentLine, _trailers);
+                                }
                             }
 
                             return default;
@@ -407,6 +429,95 @@ public override async Task<bool> DrainAsync(int maxDrainBytes)
                     cts?.Dispose();
                 }
             }
+
+            private void ParseHeaderNameValue(ReadOnlySpan<byte> line, HttpResponseHeaders trailers)
+            {
+                Debug.Assert(line.Length > 0);
+
+                int pos = 0;
+                while (line[pos] != (byte)':' && line[pos] != (byte)' ')
+                {
+                    pos++;
+                    if (pos == line.Length)
+                    {
+                        // Invalid header line that doesn't contain ':'.
+                        return;
+                    }
+                }
+
+                if (pos == 0)
+                {
+                    // Invalid empty header name.
+                    return;
+                }
+
+                if (!HeaderDescriptor.TryGet(line.Slice(0, pos), out HeaderDescriptor descriptor))
+                {
+                    // Invalid header name
+                    ThrowInvalidHttpResponse();
+                }
+
+                if (descriptor.KnownHeader != null && disallowedHeaders.Contains(descriptor.KnownHeader))
+                {
+                    // Disallowed trailer fields.
+                    // A recipient MUST ignore fields that are forbidden to be sent in a trailer.
+                    return;
+                }
+
+                // Eat any trailing whitespace
+                while (line[pos] == (byte)' ')
+                {
+                    pos++;
+                    if (pos == line.Length)
+                    {
+                        // Invalid header line that doesn't contain ':'.
+                        return;
+                    }
+                }
+
+                if (line[pos++] != ':')
+                {
+                    // Invalid header line that doesn't contain ':'.
+                    return;
+                }
+
+                // Skip whitespace after colon
+                while (pos < line.Length && (line[pos] == (byte)' ' || line[pos] == (byte)'\t'))
+                {
+                    pos++;
+                }
+
+                string headerValue = descriptor.GetHeaderValue(line.Slice(pos));
+
+                // Ignore the return value from TryAddWithoutValidation,
+                // if the header can't be added, we silently drop it.
+                trailers.TryAddWithoutValidation(descriptor, headerValue);
+            }
+
+            private static readonly List<KnownHeader> disallowedHeaders = new List<KnownHeader>
+            {
+                // Message framing headers.
+                KnownHeaders.TransferEncoding, KnownHeaders.ContentLength,
+
+                // Routing headers.
+                KnownHeaders.Host,
+
+                // Request modifiers: controls and conditionals.
+                // rfc7231#section-5.1: Controls.
+                KnownHeaders.CacheControl, KnownHeaders.Expect, KnownHeaders.MaxForwards, KnownHeaders.Pragma, KnownHeaders.Range, KnownHeaders.TE,
+                // rfc7231#section-5.2: Conditionals.
+                KnownHeaders.IfMatch, KnownHeaders.IfNoneMatch, KnownHeaders.IfModifiedSince, KnownHeaders.IfUnmodifiedSince, KnownHeaders.IfRange,
+
+                // Authentication headers.
+                KnownHeaders.Authorization, KnownHeaders.SetCookie,
+
+                // Response control data.
+                // rfc7231#section-7.1: Control Data.
+                KnownHeaders.Age, KnownHeaders.Expires, KnownHeaders.Date, KnownHeaders.Location, KnownHeaders.RetryAfter, KnownHeaders.Vary, KnownHeaders.Warning,
+
+                // Content-Encoding, Content-Type, Content-Range, and Trailer itself.
+                KnownHeaders.ContentEncoding, KnownHeaders.ContentType, KnownHeaders.ContentRange, KnownHeaders.Trailer
+            };
         }
     }
 }

src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs

@@ -645,7 +645,7 @@ public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request,
                 }
                 else if (response.Headers.TransferEncodingChunked == true)
                 {
-                    responseStream = new ChunkedEncodingReadStream(this);
+                    responseStream = new ChunkedEncodingReadStream(this, response);
                 }
                 else
                 {