/
HttpConnection.cs
2091 lines (1811 loc) · 95.4 KB
/
HttpConnection.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Buffers;
using System.Buffers.Text;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Net.Http.Headers;
using System.Net.Security;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace System.Net.Http
{
internal sealed partial class HttpConnection : HttpConnectionBase
{
/// <summary>Default size of the read buffer used for the connection.</summary>
private const int InitialReadBufferSize =
#if DEBUG
10;
#else
4096;
#endif
/// <summary>Default size of the write buffer used for the connection.</summary>
private const int InitialWriteBufferSize = InitialReadBufferSize;
/// <summary>
/// Size after which we'll close the connection rather than send the payload in response
/// to final error status code sent by the server when using Expect: 100-continue.
/// </summary>
private const int Expect100ErrorSendThreshold = 1024;
private static readonly byte[] s_contentLength0NewlineAsciiBytes = Encoding.ASCII.GetBytes("Content-Length: 0\r\n");
private static readonly byte[] s_spaceHttp10NewlineAsciiBytes = Encoding.ASCII.GetBytes(" HTTP/1.0\r\n");
private static readonly byte[] s_spaceHttp11NewlineAsciiBytes = Encoding.ASCII.GetBytes(" HTTP/1.1\r\n");
private static readonly byte[] s_httpSchemeAndDelimiter = Encoding.ASCII.GetBytes(Uri.UriSchemeHttp + Uri.SchemeDelimiter);
private static readonly byte[] s_http1DotBytes = Encoding.ASCII.GetBytes("HTTP/1.");
private static readonly ulong s_http10Bytes = BitConverter.ToUInt64(Encoding.ASCII.GetBytes("HTTP/1.0"));
private static readonly ulong s_http11Bytes = BitConverter.ToUInt64(Encoding.ASCII.GetBytes("HTTP/1.1"));
private readonly HttpConnectionPool _pool;
private readonly Socket? _socket; // used for polling; _stream should be used for all reading/writing. _stream owns disposal.
private readonly Stream _stream;
private readonly TransportContext? _transportContext;
private readonly WeakReference<HttpConnection> _weakThisRef;
private HttpRequestMessage? _currentRequest;
private readonly byte[] _writeBuffer;
private int _writeOffset;
private int _allowedReadLineBytes;
/// <summary>Reusable array used to get the values for each header being written to the wire.</summary>
private string[] _headerValues = Array.Empty<string>();
private ValueTask<int>? _readAheadTask;
private int _readAheadTaskLock; // 0 == free, 1 == held
private byte[] _readBuffer;
private int _readOffset;
private int _readLength;
private long _idleSinceTickCount;
private bool _inUse;
private bool _detachedFromPool;
private bool _canRetry;
private bool _startedSendingRequestBody;
private bool _connectionClose; // Connection: close was seen on last response
private const int Status_Disposed = 1;
private const int Status_NotDisposedAndTrackedByTelemetry = 2;
private int _disposed;
public HttpConnection(
HttpConnectionPool pool,
Socket? socket,
Stream stream,
TransportContext? transportContext)
{
Debug.Assert(pool != null);
Debug.Assert(stream != null);
_pool = pool;
_stream = stream;
_socket = socket;
_transportContext = transportContext;
_writeBuffer = new byte[InitialWriteBufferSize];
_readBuffer = new byte[InitialReadBufferSize];
_weakThisRef = new WeakReference<HttpConnection>(this);
_idleSinceTickCount = Environment.TickCount64;
if (HttpTelemetry.Log.IsEnabled())
{
HttpTelemetry.Log.Http11ConnectionEstablished();
_disposed = Status_NotDisposedAndTrackedByTelemetry;
}
if (NetEventSource.Log.IsEnabled()) TraceConnection(_stream);
}
~HttpConnection() => Dispose(disposing: false);
public override void Dispose() => Dispose(disposing: true);
private void Dispose(bool disposing)
{
// Ensure we're only disposed once. Dispose could be called concurrently, for example,
// if the request and the response were running concurrently and both incurred an exception.
int previousValue = Interlocked.Exchange(ref _disposed, Status_Disposed);
if (previousValue != Status_Disposed)
{
if (NetEventSource.Log.IsEnabled()) Trace("Connection closing.");
// Only decrement the connection count if we counted this connection
if (HttpTelemetry.Log.IsEnabled() && previousValue == Status_NotDisposedAndTrackedByTelemetry)
{
HttpTelemetry.Log.Http11ConnectionClosed();
}
if (!_detachedFromPool)
{
_pool.InvalidateHttp11Connection(this, disposing);
}
if (disposing)
{
GC.SuppressFinalize(this);
_stream.Dispose();
// Eat any exceptions from the read-ahead task. We don't need to log, as we expect
// failures from this task due to closing the connection while a read is in progress.
ValueTask<int>? readAheadTask = ConsumeReadAheadTask();
if (readAheadTask != null)
{
IgnoreExceptions(readAheadTask.GetValueOrDefault());
}
}
}
}
/// <summary>Prepare an idle connection to be used for a new request.</summary>
/// <param name="async">Indicates whether the coming request will be sync or async.</param>
/// <returns>True if connection can be used, false if it is invalid due to receiving EOF or unexpected data.</returns>
public bool PrepareForReuse(bool async)
{
// We may already have a read-ahead task if we did a previous scavenge and haven't used the connection since.
// If the read-ahead task is completed, then we've received either EOF or erroneous data the connection, so it's not usable.
if (_readAheadTask is not null)
{
return !_readAheadTask.Value.IsCompleted;
}
// Check to see if we've received anything on the connection; if we have, that's
// either erroneous data (we shouldn't have received anything yet) or the connection
// has been closed; either way, we can't use it.
if (!async && _socket is not null)
{
// Directly poll the socket rather than doing an async read, so that we can
// issue an appropriate sync read when we actually need it.
try
{
return !_socket.Poll(0, SelectMode.SelectRead);
}
catch (Exception e) when (e is SocketException || e is ObjectDisposedException)
{
// Poll can throw when used on a closed socket.
return false;
}
}
else
{
// Perform an async read on the stream, since we're going to need to read from it
// anyway, and in doing so we can avoid the extra syscall.
try
{
#pragma warning disable CA2012 // we're very careful to ensure the ValueTask is only consumed once, even though it's stored into a field
_readAheadTask = _stream.ReadAsync(new Memory<byte>(_readBuffer));
#pragma warning restore CA2012
return !_readAheadTask.Value.IsCompleted;
}
catch (Exception error)
{
// If reading throws, eat the error and don't reuse the connection.
if (NetEventSource.Log.IsEnabled()) Trace($"Error performing read ahead: {error}");
return false;
}
}
}
/// <summary>Check whether a currently idle connection is still usable, or should be scavenged.</summary>
/// <returns>True if connection can be used, false if it is invalid due to receiving EOF or unexpected data.</returns>
public override bool CheckUsabilityOnScavenge()
{
// We may already have a read-ahead task if we did a previous scavenge and haven't used the connection since.
if (_readAheadTask is null)
{
#pragma warning disable CA2012 // we're very careful to ensure the ValueTask is only consumed once, even though it's stored into a field
_readAheadTask = ReadAheadWithZeroByteReadAsync();
#pragma warning restore CA2012
}
// If the read-ahead task is completed, then we've received either EOF or erroneous data the connection, so it's not usable.
return !_readAheadTask.Value.IsCompleted;
async ValueTask<int> ReadAheadWithZeroByteReadAsync()
{
Debug.Assert(_readAheadTask is null);
Debug.Assert(RemainingBuffer.Length == 0);
// Issue a zero-byte read.
// If the underlying stream supports it, this will not complete until the stream has data available,
// which will avoid pinning the connection's read buffer (and possibly allow us to release it to the buffer pool in the future, if desired).
// If not, it will complete immediately.
await _stream.ReadAsync(Memory<byte>.Empty).ConfigureAwait(false);
// We don't know for sure that the stream actually has data available, so we need to issue a real read now.
return await _stream.ReadAsync(new Memory<byte>(_readBuffer)).ConfigureAwait(false);
}
}
private ValueTask<int>? ConsumeReadAheadTask()
{
if (Interlocked.CompareExchange(ref _readAheadTaskLock, 1, 0) == 0)
{
ValueTask<int>? t = _readAheadTask;
_readAheadTask = null;
Volatile.Write(ref _readAheadTaskLock, 0);
return t;
}
// We couldn't get the lock, which means it must already be held
// by someone else who will consume the task.
return null;
}
public override long GetIdleTicks(long nowTicks) => nowTicks - _idleSinceTickCount;
public TransportContext? TransportContext => _transportContext;
public HttpConnectionKind Kind => _pool.Kind;
private int ReadBufferSize => _readBuffer.Length;
private ReadOnlyMemory<byte> RemainingBuffer => new ReadOnlyMemory<byte>(_readBuffer, _readOffset, _readLength - _readOffset);
private void ConsumeFromRemainingBuffer(int bytesToConsume)
{
Debug.Assert(bytesToConsume <= _readLength - _readOffset, $"{bytesToConsume} > {_readLength} - {_readOffset}");
_readOffset += bytesToConsume;
}
private async ValueTask WriteHeadersAsync(HttpHeaders headers, string? cookiesFromContainer, bool async)
{
Debug.Assert(_currentRequest != null);
if (headers.HeaderStore != null)
{
foreach (KeyValuePair<HeaderDescriptor, object> header in headers.HeaderStore)
{
if (header.Key.KnownHeader != null)
{
await WriteBytesAsync(header.Key.KnownHeader.AsciiBytesWithColonSpace, async).ConfigureAwait(false);
}
else
{
await WriteAsciiStringAsync(header.Key.Name, async).ConfigureAwait(false);
await WriteTwoBytesAsync((byte)':', (byte)' ', async).ConfigureAwait(false);
}
int headerValuesCount = HttpHeaders.GetStoreValuesIntoStringArray(header.Key, header.Value, ref _headerValues);
Debug.Assert(headerValuesCount > 0, "No values for header??");
if (headerValuesCount > 0)
{
Encoding? valueEncoding = _pool.Settings._requestHeaderEncodingSelector?.Invoke(header.Key.Name, _currentRequest);
await WriteStringAsync(_headerValues[0], async, valueEncoding).ConfigureAwait(false);
if (cookiesFromContainer != null && header.Key.KnownHeader == KnownHeaders.Cookie)
{
await WriteTwoBytesAsync((byte)';', (byte)' ', async).ConfigureAwait(false);
await WriteStringAsync(cookiesFromContainer, async, valueEncoding).ConfigureAwait(false);
cookiesFromContainer = null;
}
// Some headers such as User-Agent and Server use space as a separator (see: ProductInfoHeaderParser)
if (headerValuesCount > 1)
{
HttpHeaderParser? parser = header.Key.Parser;
string separator = HttpHeaderParser.DefaultSeparator;
if (parser != null && parser.SupportsMultipleValues)
{
separator = parser.Separator!;
}
for (int i = 1; i < headerValuesCount; i++)
{
await WriteAsciiStringAsync(separator, async).ConfigureAwait(false);
await WriteStringAsync(_headerValues[i], async, valueEncoding).ConfigureAwait(false);
}
}
}
await WriteTwoBytesAsync((byte)'\r', (byte)'\n', async).ConfigureAwait(false);
}
}
if (cookiesFromContainer != null)
{
await WriteAsciiStringAsync(HttpKnownHeaderNames.Cookie, async).ConfigureAwait(false);
await WriteTwoBytesAsync((byte)':', (byte)' ', async).ConfigureAwait(false);
Encoding? valueEncoding = _pool.Settings._requestHeaderEncodingSelector?.Invoke(HttpKnownHeaderNames.Cookie, _currentRequest);
await WriteStringAsync(cookiesFromContainer, async, valueEncoding).ConfigureAwait(false);
await WriteTwoBytesAsync((byte)'\r', (byte)'\n', async).ConfigureAwait(false);
}
}
private async ValueTask WriteHostHeaderAsync(Uri uri, bool async)
{
await WriteBytesAsync(KnownHeaders.Host.AsciiBytesWithColonSpace, async).ConfigureAwait(false);
if (_pool.HostHeaderValueBytes != null)
{
Debug.Assert(Kind != HttpConnectionKind.Proxy);
await WriteBytesAsync(_pool.HostHeaderValueBytes, async).ConfigureAwait(false);
}
else
{
Debug.Assert(Kind == HttpConnectionKind.Proxy);
// TODO https://github.com/dotnet/runtime/issues/25782:
// Uri.IdnHost is missing '[', ']' characters around IPv6 address.
// So, we need to add them manually for now.
if (uri.HostNameType == UriHostNameType.IPv6)
{
await WriteByteAsync((byte)'[', async).ConfigureAwait(false);
await WriteAsciiStringAsync(uri.IdnHost, async).ConfigureAwait(false);
await WriteByteAsync((byte)']', async).ConfigureAwait(false);
}
else
{
await WriteAsciiStringAsync(uri.IdnHost, async).ConfigureAwait(false);
}
if (!uri.IsDefaultPort)
{
await WriteByteAsync((byte)':', async).ConfigureAwait(false);
await WriteDecimalInt32Async(uri.Port, async).ConfigureAwait(false);
}
}
await WriteTwoBytesAsync((byte)'\r', (byte)'\n', async).ConfigureAwait(false);
}
private Task WriteDecimalInt32Async(int value, bool async)
{
// Try to format into our output buffer directly.
if (Utf8Formatter.TryFormat(value, new Span<byte>(_writeBuffer, _writeOffset, _writeBuffer.Length - _writeOffset), out int bytesWritten))
{
_writeOffset += bytesWritten;
return Task.CompletedTask;
}
// If we don't have enough room, do it the slow way.
return WriteAsciiStringAsync(value.ToString(), async);
}
private Task WriteHexInt32Async(int value, bool async)
{
// Try to format into our output buffer directly.
if (Utf8Formatter.TryFormat(value, new Span<byte>(_writeBuffer, _writeOffset, _writeBuffer.Length - _writeOffset), out int bytesWritten, 'X'))
{
_writeOffset += bytesWritten;
return Task.CompletedTask;
}
// If we don't have enough room, do it the slow way.
return WriteAsciiStringAsync(value.ToString("X", CultureInfo.InvariantCulture), async);
}
public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request, bool async, CancellationToken cancellationToken)
{
TaskCompletionSource<bool>? allowExpect100ToContinue = null;
Task? sendRequestContentTask = null;
Debug.Assert(_currentRequest == null, $"Expected null {nameof(_currentRequest)}.");
Debug.Assert(RemainingBuffer.Length == 0, "Unexpected data in read buffer");
_currentRequest = request;
HttpMethod normalizedMethod = HttpMethod.Normalize(request.Method);
_canRetry = false;
_startedSendingRequestBody = false;
// Send the request.
if (NetEventSource.Log.IsEnabled()) Trace($"Sending request: {request}");
CancellationTokenRegistration cancellationRegistration = RegisterCancellation(cancellationToken);
try
{
if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.RequestHeadersStart();
Debug.Assert(request.RequestUri != null);
// Write request line
await WriteStringAsync(normalizedMethod.Method, async).ConfigureAwait(false);
await WriteByteAsync((byte)' ', async).ConfigureAwait(false);
if (ReferenceEquals(normalizedMethod, HttpMethod.Connect))
{
// RFC 7231 #section-4.3.6.
// Write only CONNECT foo.com:345 HTTP/1.1
if (!request.HasHeaders || request.Headers.Host == null)
{
throw new HttpRequestException(SR.net_http_request_no_host);
}
await WriteAsciiStringAsync(request.Headers.Host, async).ConfigureAwait(false);
}
else
{
if (Kind == HttpConnectionKind.Proxy)
{
// Proxied requests contain full URL
Debug.Assert(request.RequestUri.Scheme == Uri.UriSchemeHttp);
await WriteBytesAsync(s_httpSchemeAndDelimiter, async).ConfigureAwait(false);
// TODO https://github.com/dotnet/runtime/issues/25782:
// Uri.IdnHost is missing '[', ']' characters around IPv6 address.
// So, we need to add them manually for now.
if (request.RequestUri.HostNameType == UriHostNameType.IPv6)
{
await WriteByteAsync((byte)'[', async).ConfigureAwait(false);
await WriteAsciiStringAsync(request.RequestUri.IdnHost, async).ConfigureAwait(false);
await WriteByteAsync((byte)']', async).ConfigureAwait(false);
}
else
{
await WriteAsciiStringAsync(request.RequestUri.IdnHost, async).ConfigureAwait(false);
}
if (!request.RequestUri.IsDefaultPort)
{
await WriteByteAsync((byte)':', async).ConfigureAwait(false);
await WriteDecimalInt32Async(request.RequestUri.Port, async).ConfigureAwait(false);
}
}
await WriteStringAsync(request.RequestUri.PathAndQuery, async).ConfigureAwait(false);
}
// Fall back to 1.1 for all versions other than 1.0
Debug.Assert(request.Version.Major >= 0 && request.Version.Minor >= 0); // guaranteed by Version class
bool isHttp10 = request.Version.Minor == 0 && request.Version.Major == 1;
await WriteBytesAsync(isHttp10 ? s_spaceHttp10NewlineAsciiBytes : s_spaceHttp11NewlineAsciiBytes, async).ConfigureAwait(false);
// Determine cookies to send
string? cookiesFromContainer = null;
if (_pool.Settings._useCookies)
{
cookiesFromContainer = _pool.Settings._cookieContainer!.GetCookieHeader(request.RequestUri);
if (cookiesFromContainer == "")
{
cookiesFromContainer = null;
}
}
// Write special additional headers. If a host isn't in the headers list, then a Host header
// wasn't sent, so as it's required by HTTP 1.1 spec, send one based on the Request Uri.
if (!request.HasHeaders || request.Headers.Host == null)
{
await WriteHostHeaderAsync(request.RequestUri, async).ConfigureAwait(false);
}
// Write request headers
if (request.HasHeaders || cookiesFromContainer != null)
{
await WriteHeadersAsync(request.Headers, cookiesFromContainer, async).ConfigureAwait(false);
}
if (request.Content == null)
{
// Write out Content-Length: 0 header to indicate no body,
// unless this is a method that never has a body.
if (normalizedMethod.MustHaveRequestBody)
{
await WriteBytesAsync(s_contentLength0NewlineAsciiBytes, async).ConfigureAwait(false);
}
}
else
{
// Write content headers
await WriteHeadersAsync(request.Content.Headers, cookiesFromContainer: null, async).ConfigureAwait(false);
}
// CRLF for end of headers.
await WriteTwoBytesAsync((byte)'\r', (byte)'\n', async).ConfigureAwait(false);
if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.RequestHeadersStop();
if (request.Content == null)
{
// We have nothing more to send, so flush out any headers we haven't yet sent.
await FlushAsync(async).ConfigureAwait(false);
}
else
{
bool hasExpectContinueHeader = request.HasHeaders && request.Headers.ExpectContinue == true;
if (NetEventSource.Log.IsEnabled()) Trace($"Request content is not null, start processing it. hasExpectContinueHeader = {hasExpectContinueHeader}");
// Send the body if there is one. We prefer to serialize the sending of the content before
// we try to receive any response, but if ExpectContinue has been set, we allow the sending
// to run concurrently until we receive the final status line, at which point we wait for it.
if (!hasExpectContinueHeader)
{
await SendRequestContentAsync(request, CreateRequestContentStream(request), async, cancellationToken).ConfigureAwait(false);
}
else
{
// We're sending an Expect: 100-continue header. We need to flush headers so that the server receives
// all of them, and we need to do so before initiating the send, as once we do that, it effectively
// owns the right to write, and we don't want to concurrently be accessing the write buffer.
await FlushAsync(async).ConfigureAwait(false);
// Create a TCS we'll use to block the request content from being sent, and create a timer that's used
// as a fail-safe to unblock the request content if we don't hear back from the server in a timely manner.
// Then kick off the request. The TCS' result indicates whether content should be sent or not.
allowExpect100ToContinue = new TaskCompletionSource<bool>();
var expect100Timer = new Timer(
static s => ((TaskCompletionSource<bool>)s!).TrySetResult(true),
allowExpect100ToContinue, _pool.Settings._expect100ContinueTimeout, Timeout.InfiniteTimeSpan);
sendRequestContentTask = SendRequestContentWithExpect100ContinueAsync(
request, allowExpect100ToContinue.Task, CreateRequestContentStream(request), expect100Timer, async, cancellationToken);
}
}
// Start to read response.
_allowedReadLineBytes = (int)Math.Min(int.MaxValue, _pool.Settings._maxResponseHeadersLength * 1024L);
// We should not have any buffered data here; if there was, it should have been treated as an error
// by the previous request handling. (Note we do not support HTTP pipelining.)
Debug.Assert(_readOffset == _readLength);
// When the connection was taken out of the pool, a pre-emptive read was performed
// into the read buffer. We need to consume that read prior to issuing another read.
ValueTask<int>? t = ConsumeReadAheadTask();
if (t != null)
{
// Handle the pre-emptive read. For the async==false case, hopefully the read has
// already completed and this will be a nop, but if it hasn't, the caller will be forced to block
// waiting for the async operation to complete. We will only hit this case for proxied HTTPS
// requests that use a pooled connection, as in that case we don't have a Socket we
// can poll and are forced to issue an async read.
ValueTask<int> vt = t.GetValueOrDefault();
int bytesRead;
if (vt.IsCompleted)
{
bytesRead = vt.Result;
}
else
{
if (!async)
{
Trace($"Pre-emptive read completed asynchronously for a synchronous request.");
}
bytesRead = await vt.ConfigureAwait(false);
}
if (NetEventSource.Log.IsEnabled()) Trace($"Received {bytesRead} bytes.");
_readOffset = 0;
_readLength = bytesRead;
}
else
{
// No read-ahead, so issue a read ourselves. We will check below for EOF.
await InitialFillAsync(async).ConfigureAwait(false);
}
if (_readLength == 0)
{
// The server shutdown the connection on their end, likely because of an idle timeout.
// If we haven't started sending the request body yet (or there is no request body),
// then we allow the request to be retried.
if (!_startedSendingRequestBody)
{
_canRetry = true;
}
throw new IOException(SR.net_http_invalid_response_premature_eof);
}
// Parse the response status line.
var response = new HttpResponseMessage() { RequestMessage = request, Content = new HttpConnectionResponseContent() };
ParseStatusLine((await ReadNextResponseHeaderLineAsync(async).ConfigureAwait(false)).Span, response);
if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.ResponseHeadersStart();
// Multiple 1xx responses handling.
// RFC 7231: A client MUST be able to parse one or more 1xx responses received prior to a final response,
// even if the client does not expect one. A user agent MAY ignore unexpected 1xx responses.
// In .NET Core, apart from 100 Continue, and 101 Switching Protocols, we will treat all other 1xx responses
// as unknown, and will discard them.
while ((uint)(response.StatusCode - 100) <= 199 - 100)
{
// If other 1xx responses come before an expected 100 continue, we will wait for the 100 response before
// sending request body (if any).
if (allowExpect100ToContinue != null && response.StatusCode == HttpStatusCode.Continue)
{
allowExpect100ToContinue.TrySetResult(true);
allowExpect100ToContinue = null;
}
else if (response.StatusCode == HttpStatusCode.SwitchingProtocols)
{
// 101 Upgrade is a final response as it's used to switch protocols with WebSockets handshake.
// Will return a response object with status 101 and a raw connection stream later.
// RFC 7230: If a server receives both an Upgrade and an Expect header field with the "100-continue" expectation,
// the server MUST send a 100 (Continue) response before sending a 101 (Switching Protocols) response.
// If server doesn't follow RFC, we treat 101 as a final response and stop waiting for 100 continue - as if server
// never sends a 100-continue. The request body will be sent after expect100Timer expires.
break;
}
// In case read hangs which eventually leads to connection timeout.
if (NetEventSource.Log.IsEnabled()) Trace($"Current {response.StatusCode} response is an interim response or not expected, need to read for a final response.");
// Discard headers that come with the interim 1xx responses.
// RFC7231: 1xx responses are terminated by the first empty line after the status-line.
while (!IsLineEmpty(await ReadNextResponseHeaderLineAsync(async).ConfigureAwait(false)));
// Parse the status line for next response.
ParseStatusLine((await ReadNextResponseHeaderLineAsync(async).ConfigureAwait(false)).Span, response);
}
// Parse the response headers. Logic after this point depends on being able to examine headers in the response object.
while (true)
{
ReadOnlyMemory<byte> line = await ReadNextResponseHeaderLineAsync(async, foldedHeadersAllowed: true).ConfigureAwait(false);
if (IsLineEmpty(line))
{
break;
}
ParseHeaderNameValue(this, line.Span, response, isFromTrailer: false);
}
if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.ResponseHeadersStop();
if (allowExpect100ToContinue != null)
{
// If we sent an Expect: 100-continue header, and didn't receive a 100-continue. Handle the final response accordingly.
// Note that the developer may have added an Expect: 100-continue header even if there is no Content.
if ((int)response.StatusCode >= 300 &&
request.Content != null &&
(request.Content.Headers.ContentLength == null || request.Content.Headers.ContentLength.GetValueOrDefault() > Expect100ErrorSendThreshold) &&
!AuthenticationHelper.IsSessionAuthenticationChallenge(response))
{
// For error final status codes, try to avoid sending the payload if its size is unknown or if it's known to be "big".
// If we already sent a header detailing the size of the payload, if we then don't send that payload, the server may wait
// for it and assume that the next request on the connection is actually this request's payload. Thus we mark the connection
// to be closed. However, we may have also lost a race condition with the Expect: 100-continue timeout, so if it turns out
// we've already started sending the payload (we weren't able to cancel it), then we don't need to force close the connection.
// We also must not clone connection if we do NTLM or Negotiate authentication.
allowExpect100ToContinue.TrySetResult(false);
if (!allowExpect100ToContinue.Task.Result) // if Result is true, the timeout already expired and we started sending content
{
_connectionClose = true;
}
}
else
{
// For any success status codes, for errors when the request content length is known to be small,
// or for session-based authentication challenges, send the payload
// (if there is one... if there isn't, Content is null and thus allowExpect100ToContinue is also null, we won't get here).
allowExpect100ToContinue.TrySetResult(true);
}
}
// Determine whether we need to force close the connection when the request/response has completed.
if (response.Headers.ConnectionClose.GetValueOrDefault())
{
_connectionClose = true;
}
// Now that we've received our final status line, wait for the request content to fully send.
// In most common scenarios, the server won't send back a response until all of the request
// content has been received, so this task should generally already be complete.
if (sendRequestContentTask != null)
{
Task sendTask = sendRequestContentTask;
sendRequestContentTask = null;
await sendTask.ConfigureAwait(false);
}
// Now we are sure that the request was fully sent.
if (NetEventSource.Log.IsEnabled()) Trace("Request is fully sent.");
// We're about to create the response stream, at which point responsibility for canceling
// the remainder of the response lies with the stream. Thus we dispose of our registration
// here (if an exception has occurred or does occur while creating/returning the stream,
// we'll still dispose of it in the catch below as part of Dispose'ing the connection).
cancellationRegistration.Dispose();
CancellationHelper.ThrowIfCancellationRequested(cancellationToken); // in case cancellation may have disposed of the stream
// Create the response stream.
Stream responseStream;
if (ReferenceEquals(normalizedMethod, HttpMethod.Head) || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode == HttpStatusCode.NotModified)
{
responseStream = EmptyReadStream.Instance;
CompleteResponse();
}
else if (ReferenceEquals(normalizedMethod, HttpMethod.Connect) && response.StatusCode == HttpStatusCode.OK)
{
// Successful response to CONNECT does not have body.
// What ever comes next should be opaque.
responseStream = new RawConnectionStream(this);
// Don't put connection back to the pool if we upgraded to tunnel.
// We cannot use it for normal HTTP requests any more.
_connectionClose = true;
_pool.InvalidateHttp11Connection(this);
_detachedFromPool = true;
}
else if (response.StatusCode == HttpStatusCode.SwitchingProtocols)
{
responseStream = new RawConnectionStream(this);
// Don't put connection back to the pool if we switched protocols.
// We cannot use it for normal HTTP requests any more.
_connectionClose = true;
_pool.InvalidateHttp11Connection(this);
_detachedFromPool = true;
}
else if (response.Content.Headers.ContentLength != null)
{
long contentLength = response.Content.Headers.ContentLength.GetValueOrDefault();
if (contentLength <= 0)
{
responseStream = EmptyReadStream.Instance;
CompleteResponse();
}
else
{
responseStream = new ContentLengthReadStream(this, (ulong)contentLength);
}
}
else if (response.Headers.TransferEncodingChunked == true)
{
responseStream = new ChunkedEncodingReadStream(this, response);
}
else
{
responseStream = new ConnectionCloseReadStream(this);
}
((HttpConnectionResponseContent)response.Content).SetStream(responseStream);
if (NetEventSource.Log.IsEnabled()) Trace($"Received response: {response}");
// Process Set-Cookie headers.
if (_pool.Settings._useCookies)
{
CookieHelper.ProcessReceivedCookies(response, _pool.Settings._cookieContainer!);
}
return response;
}
catch (Exception error)
{
// Clean up the cancellation registration in case we're still registered.
cancellationRegistration.Dispose();
// Make sure to complete the allowExpect100ToContinue task if it exists.
allowExpect100ToContinue?.TrySetResult(false);
if (NetEventSource.Log.IsEnabled()) Trace($"Error sending request: {error}");
// In the rare case where Expect: 100-continue was used and then processing
// of the response headers encountered an error such that we weren't able to
// wait for the sending to complete, it's possible the sending also encountered
// an exception or potentially is still going and will encounter an exception
// (we're about to Dispose for the connection). In such cases, we don't want any
// exception in that sending task to become unobserved and raise alarm bells, so we
// hook up a continuation that will log it.
if (sendRequestContentTask != null && !sendRequestContentTask.IsCompletedSuccessfully)
{
// In case the connection is disposed, it's most probable that
// expect100Continue timer expired and request content sending failed.
// We're awaiting the task to propagate the exception in this case.
if (Volatile.Read(ref _disposed) == Status_Disposed)
{
try
{
await sendRequestContentTask.ConfigureAwait(false);
}
// Map the exception the same way as we normally do.
catch (Exception ex) when (MapSendException(ex, cancellationToken, out Exception mappedEx))
{
throw mappedEx;
}
}
LogExceptions(sendRequestContentTask);
}
// Now clean up the connection.
Dispose();
// At this point, we're going to throw an exception; we just need to
// determine which exception to throw.
if (MapSendException(error, cancellationToken, out Exception mappedException))
{
throw mappedException;
}
// Otherwise, just allow the original exception to propagate.
throw;
}
}
public Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool async, CancellationToken cancellationToken) =>
SendAsyncCore(request, async, cancellationToken);
private bool MapSendException(Exception exception, CancellationToken cancellationToken, out Exception mappedException)
{
if (CancellationHelper.ShouldWrapInOperationCanceledException(exception, cancellationToken))
{
// Cancellation was requested, so assume that the failure is due to
// the cancellation request. This is a bit unorthodox, as usually we'd
// prioritize a non-OperationCanceledException over a cancellation
// request to avoid losing potentially pertinent information. But given
// the cancellation design where we tear down the underlying connection upon
// a cancellation request, which can then result in a myriad of different
// exceptions (argument exceptions, object disposed exceptions, socket exceptions,
// etc.), as a middle ground we treat it as cancellation, but still propagate the
// original information as the inner exception, for diagnostic purposes.
mappedException = CancellationHelper.CreateOperationCanceledException(exception, cancellationToken);
return true;
}
if (exception is InvalidOperationException)
{
// For consistency with other handlers we wrap the exception in an HttpRequestException.
mappedException = new HttpRequestException(SR.net_http_client_execution_error, exception);
return true;
}
if (exception is IOException ioe)
{
// For consistency with other handlers we wrap the exception in an HttpRequestException.
// If the request is retryable, indicate that on the exception.
mappedException = new HttpRequestException(SR.net_http_client_execution_error, ioe, _canRetry ? RequestRetryType.RetryOnConnectionFailure : RequestRetryType.NoRetry);
return true;
}
// Otherwise, just allow the original exception to propagate.
mappedException = exception;
return false;
}
private HttpContentWriteStream CreateRequestContentStream(HttpRequestMessage request)
{
bool requestTransferEncodingChunked = request.HasHeaders && request.Headers.TransferEncodingChunked == true;
HttpContentWriteStream requestContentStream = requestTransferEncodingChunked ? (HttpContentWriteStream)
new ChunkedEncodingWriteStream(this) :
new ContentLengthWriteStream(this);
return requestContentStream;
}
private CancellationTokenRegistration RegisterCancellation(CancellationToken cancellationToken)
{
// Cancellation design:
// - We register with the SendAsync CancellationToken for the duration of the SendAsync operation.
// - We register with the Read/Write/CopyToAsync methods on the response stream for each such individual operation.
// - The registration disposes of the connection, tearing it down and causing any pending operations to wake up.
// - Because such a tear down can result in a variety of different exception types, we check for a cancellation
// request and prioritize that over other exceptions, wrapping the actual exception as an inner of an OCE.
// - A weak reference to this HttpConnection is stored in the cancellation token, to prevent the token from
// artificially keeping this connection alive.
return cancellationToken.Register(static s =>
{
var weakThisRef = (WeakReference<HttpConnection>)s!;
if (weakThisRef.TryGetTarget(out HttpConnection? strongThisRef))
{
if (NetEventSource.Log.IsEnabled()) strongThisRef.Trace("Cancellation requested. Disposing of the connection.");
strongThisRef.Dispose();
}
}, _weakThisRef);
}
private static bool IsLineEmpty(ReadOnlyMemory<byte> line) => line.Length == 0;
private async ValueTask SendRequestContentAsync(HttpRequestMessage request, HttpContentWriteStream stream, bool async, CancellationToken cancellationToken)
{
// Now that we're sending content, prohibit retries of this request by setting this flag.
_startedSendingRequestBody = true;
Debug.Assert(stream.BytesWritten == 0);
if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.RequestContentStart();
// Copy all of the data to the server.
if (async)
{
await request.Content!.CopyToAsync(stream, _transportContext, cancellationToken).ConfigureAwait(false);
}
else
{
request.Content!.CopyTo(stream, _transportContext, cancellationToken);
}
// Finish the content; with a chunked upload, this includes writing the terminating chunk.
await stream.FinishAsync(async).ConfigureAwait(false);
// Flush any content that might still be buffered.
await FlushAsync(async).ConfigureAwait(false);
if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.RequestContentStop(stream.BytesWritten);
if (NetEventSource.Log.IsEnabled()) Trace("Finished sending request content.");
}
private async Task SendRequestContentWithExpect100ContinueAsync(
HttpRequestMessage request, Task<bool> allowExpect100ToContinueTask,
HttpContentWriteStream stream, Timer expect100Timer, bool async, CancellationToken cancellationToken)
{
// Wait until we receive a trigger notification that it's ok to continue sending content.
// This will come either when the timer fires or when we receive a response status line from the server.
bool sendRequestContent = await allowExpect100ToContinueTask.ConfigureAwait(false);
// Clean up the timer; it's no longer needed.
expect100Timer.Dispose();
// Send the content if we're supposed to. Otherwise, we're done.
if (sendRequestContent)
{
if (NetEventSource.Log.IsEnabled()) Trace($"Sending request content for Expect: 100-continue.");
try
{
await SendRequestContentAsync(request, stream, async, cancellationToken).ConfigureAwait(false);
}
catch
{
// Tear down the connection if called from the timer thread because caller's thread will wait for server status line indefinitely
// or till HttpClient.Timeout tear the connection itself.
Dispose();
throw;
}
}
else
{
if (NetEventSource.Log.IsEnabled()) Trace($"Canceling request content for Expect: 100-continue.");
}
}
private static void ParseStatusLine(ReadOnlySpan<byte> line, HttpResponseMessage response)
{
// We sent the request version as either 1.0 or 1.1.
// We expect a response version of the form 1.X, where X is a single digit as per RFC.
// Validate the beginning of the status line and set the response version.
const int MinStatusLineLength = 12; // "HTTP/1.x 123"
if (line.Length < MinStatusLineLength || line[8] != ' ')
{
throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_status_line, Encoding.ASCII.GetString(line)));
}
ulong first8Bytes = BitConverter.ToUInt64(line);
if (first8Bytes == s_http11Bytes)
{
response.SetVersionWithoutValidation(HttpVersion.Version11);
}
else if (first8Bytes == s_http10Bytes)
{
response.SetVersionWithoutValidation(HttpVersion.Version10);
}
else
{
byte minorVersion = line[7];
if (IsDigit(minorVersion) &&
line.Slice(0, 7).SequenceEqual(s_http1DotBytes))
{
response.SetVersionWithoutValidation(new Version(1, minorVersion - '0'));
}
else
{
throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_status_line, Encoding.ASCII.GetString(line)));
}
}
// Set the status code
byte status1 = line[9], status2 = line[10], status3 = line[11];
if (!IsDigit(status1) || !IsDigit(status2) || !IsDigit(status3))
{
throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_status_code, Encoding.ASCII.GetString(line.Slice(9, 3))));
}
response.SetStatusCodeWithoutValidation((HttpStatusCode)(100 * (status1 - '0') + 10 * (status2 - '0') + (status3 - '0')));
// Parse (optional) reason phrase
if (line.Length == MinStatusLineLength)
{