Skip to content
This repository has been archived by the owner on Dec 18, 2018. It is now read-only.

Commit

Permalink
[2.2] Fix race conditions in HTTP/2 tests (#3078)
Browse files Browse the repository at this point in the history
  • Loading branch information
@halter73
halter73 committed Nov 8, 2018
1 parent 40a9b18 commit a1c1083
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 83 deletions.
100 changes: 49 additions & 51 deletions test/Kestrel.InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -816,19 +816,15 @@ public async Task DATA_Received_StreamClosed_ConnectionError()

await SendDataAsync(1, _helloWorldBytes, endStream: false);

// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
expectedErrorCode: Http2ErrorCode.STREAM_CLOSED,
expectedErrorMessage: null);

// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
var closedMessage = CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.DATA, streamId: 1);
var halfClosedMessage = CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.DATA, streamId: 1);

var message = Assert.Single(TestApplicationErrorLogger.Messages, m => m.Exception is Http2ConnectionErrorException);
Assert.True(message.Exception.Message.IndexOf(closedMessage) >= 0
|| message.Exception.Message.IndexOf(halfClosedMessage) >= 0);
expectedErrorMessage: new[] {
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.DATA, streamId: 1),
CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.DATA, streamId: 1)
});
}

[Fact]
Expand Down Expand Up @@ -1329,19 +1325,15 @@ public async Task HEADERS_Received_StreamClosed_ConnectionError()
// Try to re-use the stream ID (http://httpwg.org/specs/rfc7540.html#rfc.section.5.1.1)
await StartStreamAsync(1, _browserRequestHeaders, endStream: true);

// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
expectedErrorCode: Http2ErrorCode.STREAM_CLOSED,
expectedErrorMessage: null);

// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
var closedMessage = CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.HEADERS, streamId: 1);
var halfClosedMessage = CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.HEADERS, streamId: 1);

var message = Assert.Single(TestApplicationErrorLogger.Messages, m => m.Exception is Http2ConnectionErrorException);
Assert.True(message.Exception.Message.IndexOf(closedMessage) >= 0
|| message.Exception.Message.IndexOf(halfClosedMessage) >= 0);
expectedErrorMessage: new[] {
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.HEADERS, streamId: 1),
CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.HEADERS, streamId: 1)
});
}

[Fact]
Expand Down Expand Up @@ -3782,12 +3774,6 @@ public async Task AppDoesNotReadRequestBody_ResetsAndDrainsRequest(Http2FrameTyp
// Logged without an exception.
Assert.Contains(TestApplicationErrorLogger.Messages, m => m.Message.Contains("the application completed without reading the entire request body."));

// There's a race when the appfunc is exiting about how soon it unregisters the stream.
for (var i = 0; i < 10; i++)
{
await SendDataAsync(1, new byte[100], endStream: false);
}

// These would be refused if the cool-down period had expired
switch (finalFrameType)
{
Expand Down Expand Up @@ -3830,12 +3816,6 @@ public async Task AbortedStream_ResetsAndDrainsRequest(Http2FrameType finalFrame

await WaitForStreamErrorAsync(1, Http2ErrorCode.INTERNAL_ERROR, "The connection was aborted by the application.");

// There's a race when the appfunc is exiting about how soon it unregisters the stream.
for (var i = 0; i < 10; i++)
{
await SendDataAsync(1, new byte[100], endStream: false);
}

// These would be refused if the cool-down period had expired
switch (finalFrameType)
{
Expand All @@ -3860,7 +3840,7 @@ public async Task AbortedStream_ResetsAndDrainsRequest(Http2FrameType finalFrame
}

[Theory]
[InlineData(Http2FrameType.DATA)]
[InlineData(Http2FrameType.DATA, Skip = "Fixed in master with https://github.com/aspnet/KestrelHttpServer/pull/3024/")]
[InlineData(Http2FrameType.HEADERS)]
[InlineData(Http2FrameType.CONTINUATION)]
public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterEndOfStream(Http2FrameType finalFrameType)
Expand All @@ -3877,39 +3857,55 @@ public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterEndOfSt

await WaitForStreamErrorAsync(1, Http2ErrorCode.INTERNAL_ERROR, "The connection was aborted by the application.");

// There's a race when the appfunc is exiting about how soon it unregisters the stream.
for (var i = 0; i < 10; i++)
{
await SendDataAsync(1, new byte[100], endStream: false);
}

switch (finalFrameType)
{
case Http2FrameType.DATA:
await SendDataAsync(1, new byte[100], endStream: true);
// An extra one to break it
await SendDataAsync(1, new byte[100], endStream: true);

await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(ignoreNonGoAwayFrames: false, expectedLastStreamId: 1, Http2ErrorCode.STREAM_CLOSED,
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.DATA, 1));

// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
expectedErrorCode: Http2ErrorCode.STREAM_CLOSED,
expectedErrorMessage: new[] {
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.DATA, streamId: 1),
CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.DATA, streamId: 1)
});
break;

case Http2FrameType.HEADERS:
await SendHeadersAsync(1, Http2HeadersFrameFlags.END_STREAM | Http2HeadersFrameFlags.END_HEADERS, _requestTrailers);
// An extra one to break it
await SendHeadersAsync(1, Http2HeadersFrameFlags.END_STREAM | Http2HeadersFrameFlags.END_HEADERS, _requestTrailers);

await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(ignoreNonGoAwayFrames: false, expectedLastStreamId: 1, Http2ErrorCode.STREAM_CLOSED,
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.HEADERS, 1));
// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
expectedErrorCode: Http2ErrorCode.STREAM_CLOSED,
expectedErrorMessage: new[] {
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.HEADERS, streamId: 1),
CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.HEADERS, streamId: 1)
});
break;

case Http2FrameType.CONTINUATION:
await SendHeadersAsync(1, Http2HeadersFrameFlags.END_STREAM, _requestTrailers);
await SendContinuationAsync(1, Http2ContinuationFrameFlags.END_HEADERS, _requestTrailers);
// An extra one to break it. It's not a Continuation because that would fail with an error that no headers were in progress.
await SendHeadersAsync(1, Http2HeadersFrameFlags.END_STREAM, _requestTrailers);
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(ignoreNonGoAwayFrames: false, expectedLastStreamId: 1, Http2ErrorCode.STREAM_CLOSED,
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.HEADERS, 1));

// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
expectedErrorCode: Http2ErrorCode.STREAM_CLOSED,
expectedErrorMessage: new[] {
CoreStrings.FormatHttp2ErrorStreamClosed(Http2FrameType.HEADERS, streamId: 1),
CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(Http2FrameType.HEADERS, streamId: 1)
});
break;
default:
throw new NotImplementedException(finalFrameType.ToString());
Expand All @@ -3933,11 +3929,6 @@ public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterClientR

await WaitForStreamErrorAsync(1, Http2ErrorCode.INTERNAL_ERROR, "The connection was aborted by the application.");

// There's a race when the appfunc is exiting about how soon it unregisters the stream.
for (var i = 0; i < 10; i++)
{
await SendDataAsync(1, new byte[100], endStream: false);
}
await SendRstStreamAsync(1);

// Send an extra frame to make it fail
Expand All @@ -3955,8 +3946,15 @@ public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterClientR
throw new NotImplementedException(finalFrameType.ToString());
}

await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(ignoreNonGoAwayFrames: false, expectedLastStreamId: 1, Http2ErrorCode.STREAM_CLOSED,
CoreStrings.FormatHttp2ErrorStreamClosed(finalFrameType, 1));
// There's a race where either of these messages could be logged, depending on if the stream cleanup has finished yet.
await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
expectedErrorCode: Http2ErrorCode.STREAM_CLOSED,
expectedErrorMessage: new[] {
CoreStrings.FormatHttp2ErrorStreamClosed(finalFrameType, streamId: 1),
CoreStrings.FormatHttp2ErrorStreamHalfClosedRemote(finalFrameType, streamId: 1)
});
}

public static TheoryData<byte[]> UpperCaseHeaderNameData
Expand Down
7 changes: 4 additions & 3 deletions test/Kestrel.InMemory.FunctionalTests/Http2/Http2TestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1110,7 +1110,7 @@ protected void VerifyGoAway(Http2Frame frame, int expectedLastStreamId, Http2Err
Assert.Equal(expectedErrorCode, frame.GoAwayErrorCode);
}

protected async Task WaitForConnectionErrorAsync<TException>(bool ignoreNonGoAwayFrames, int expectedLastStreamId, Http2ErrorCode expectedErrorCode, string expectedErrorMessage)
protected async Task WaitForConnectionErrorAsync<TException>(bool ignoreNonGoAwayFrames, int expectedLastStreamId, Http2ErrorCode expectedErrorCode, params string[] expectedErrorMessage)
where TException : Exception
{
var frame = await ReceiveFrameAsync();
Expand All @@ -1125,10 +1125,11 @@ protected async Task WaitForConnectionErrorAsync<TException>(bool ignoreNonGoAwa

VerifyGoAway(frame, expectedLastStreamId, expectedErrorCode);

if (expectedErrorMessage != null)
if (expectedErrorMessage?.Length > 0)
{
var message = Assert.Single(TestApplicationErrorLogger.Messages, m => m.Exception is TException);
Assert.Contains(expectedErrorMessage, message.Exception.Message);

Assert.Contains(expectedErrorMessage, expected => message.Exception.Message.Contains(expected));
}

await _connectionTask;
Expand Down
74 changes: 45 additions & 29 deletions test/Kestrel.InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Connections;
using Microsoft.AspNetCore.Server.Kestrel.Core.Features;
Expand Down Expand Up @@ -191,7 +192,7 @@ public async Task ResponseDrain_SlowerThanMinimumDataRate_AbortsConnection()

[Theory]
[InlineData(Http2FrameType.DATA)]
[InlineData(Http2FrameType.HEADERS)]
[InlineData(Http2FrameType.CONTINUATION, Skip = "https://github.com/aspnet/KestrelHttpServer/issues/3077")]
public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterCooldownExpires(Http2FrameType finalFrameType)
{
var mockSystemClock = _serviceContext.MockSystemClock;
Expand All @@ -208,40 +209,55 @@ public async Task AbortedStream_ResetsAndDrainsRequest_RefusesFramesAfterCooldow

await WaitForStreamErrorAsync(1, Http2ErrorCode.INTERNAL_ERROR, "The connection was aborted by the application.");

// There's a race when the appfunc is exiting about how soon it unregisters the stream.
for (var i = 0; i < 10; i++)
{
await SendDataAsync(1, new byte[100], endStream: false);
}

// Just short of the timeout
mockSystemClock.UtcNow += Constants.RequestBodyDrainTimeout;
(_connection as IRequestProcessor).Tick(mockSystemClock.UtcNow);

// Still fine
await SendDataAsync(1, new byte[100], endStream: false);
var cts = new CancellationTokenSource();

// Just past the timeout
mockSystemClock.UtcNow += TimeSpan.FromTicks(1);
(_connection as IRequestProcessor).Tick(mockSystemClock.UtcNow);

// Send an extra frame to make it fail
switch (finalFrameType)
async Task AdvanceClockAndSendFrames()
{
case Http2FrameType.DATA:
await SendDataAsync(1, new byte[100], endStream: true);
break;

case Http2FrameType.HEADERS:
await SendHeadersAsync(1, Http2HeadersFrameFlags.END_STREAM | Http2HeadersFrameFlags.END_HEADERS, _requestTrailers);
break;
if (finalFrameType == Http2FrameType.CONTINUATION)
{
await SendHeadersAsync(1, Http2HeadersFrameFlags.END_STREAM, new byte[0]);
}

default:
throw new NotImplementedException(finalFrameType.ToString());
// There's a race when the appfunc is exiting about how soon it unregisters the stream, so retry until success.
while (!cts.Token.IsCancellationRequested)
{
// Just past the timeout
mockSystemClock.UtcNow += Constants.RequestBodyDrainTimeout + TimeSpan.FromTicks(1);
(_connection as IRequestProcessor).Tick(mockSystemClock.UtcNow);

// Send an extra frame to make it fail
switch (finalFrameType)
{
case Http2FrameType.DATA:
await SendDataAsync(1, new byte[100], endStream: false);
break;

case Http2FrameType.CONTINUATION:
await SendContinuationAsync(1, Http2ContinuationFrameFlags.NONE, new byte[0]);
break;

default:
throw new NotImplementedException(finalFrameType.ToString());
}

if (!cts.Token.IsCancellationRequested)
{
await Task.Delay(10);
}
}
}

await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(ignoreNonGoAwayFrames: false, expectedLastStreamId: 1, Http2ErrorCode.STREAM_CLOSED,
var sendTask = AdvanceClockAndSendFrames();

await WaitForConnectionErrorAsync<Http2ConnectionErrorException>(
ignoreNonGoAwayFrames: false,
expectedLastStreamId: 1,
Http2ErrorCode.STREAM_CLOSED,
CoreStrings.FormatHttp2ErrorStreamClosed(finalFrameType, 1));

cts.Cancel();

await sendTask.DefaultTimeout();
}

[Fact]
Expand Down

0 comments on commit a1c1083

Please sign in to comment.