Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[NOREVIEW][NOMERGE] HTTP/3 stress data corruption hunt #101624

Closed
wants to merge 19 commits into from
Closed

[NOREVIEW][NOMERGE] HTTP/3 stress data corruption hunt #101624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
9103005
Inline POST Duplex Slow validation
rzikm Apr 26, 2024
2ed5dda
Validate request/response contents inside System.Net.Quic
rzikm Apr 26, 2024
2b21f22
Add runtime collection to Linux docker runs
rzikm Apr 26, 2024
0895ba1
Export artifacts when crashed, run in debug
rzikm Apr 26, 2024
81a68ad
More stress per stress
rzikm Apr 26, 2024
47b0f39
Run in Release
rzikm Apr 27, 2024
2e4f942
Fix validation
rzikm Apr 28, 2024
f0e9c3c
More validation
rzikm Apr 29, 2024
455cdbb
Fix script
rzikm Apr 29, 2024
5025990
Move to FailFast instead of Debug.Fail
rzikm Apr 29, 2024
bbef533
Assert in Pipe
rzikm Apr 30, 2024
b6c8de0
Unique dump names
rzikm Apr 30, 2024
1a95bcc
Merge remote-tracking branch 'upstream/main' into http3-missing-data-…
rzikm Jul 11, 2024
376d5b2
Debugging improvements
rzikm Jul 12, 2024
de45b48
Add -aspnetlog
rzikm Jul 15, 2024
014c67e
Add also for Linux
rzikm Jul 15, 2024
bb6422d
Revert "Add also for Linux"
rzikm Jul 15, 2024
623424a
Try again
rzikm Jul 15, 2024
2c8bae0
Overwrite with custom kestrel dll
rzikm Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 93 additions & 22 deletions eng/pipelines/libraries/stress/http.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,30 +60,65 @@ extends:
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
docker-compose up --abort-on-container-exit --no-color
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/2.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/2.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
docker-compose up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 2.0
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/1.1"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/1.1"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
docker-compose up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 1.1
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
docker-compose up --abort-on-container-exit --no-color
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
docker-compose up --abort-on-container-exit --no-color
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
docker-compose up --abort-on-container-exit --no-color
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- bash: |
Expand Down Expand Up @@ -145,33 +180,69 @@ extends:
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- powershell: |
cd '$(httpStressProject)'
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/2.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/2.0"
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- powershell: |
cd '$(httpStressProject)'
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- powershell: |
cd '$(httpStressProject)'
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- powershell: |
cd '$(httpStressProject)'
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 2.0
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- powershell: |
cd '$(httpStressProject)'
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/1.1"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/1.1"
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0 -aspnetlog"
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 1.1
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))

- publish: $(Build.ArtifactStagingDirectory)/dumps
Expand Down
20 changes: 10 additions & 10 deletions src/libraries/Common/src/System/Net/ArrayBuffer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ internal struct ArrayBuffer : IDisposable

public ArrayBuffer(int initialSize, bool usePool = false)
{
Debug.Assert(initialSize > 0 || usePool);
// Debug.Assert(initialSize > 0 || usePool);

_usePool = usePool;
_bytes = initialSize == 0
Expand All @@ -43,7 +43,7 @@ public ArrayBuffer(int initialSize, bool usePool = false)

public ArrayBuffer(byte[] buffer)
{
Debug.Assert(buffer.Length > 0);
// Debug.Assert(buffer.Length > 0);

_usePool = false;
_bytes = buffer;
Expand All @@ -68,8 +68,8 @@ public void Dispose()
// This is different from Dispose as the instance remains usable afterwards (_bytes will not be null).
public void ClearAndReturnBuffer()
{
Debug.Assert(_usePool);
Debug.Assert(_bytes is not null);
// Debug.Assert(_usePool);
// Debug.Assert(_bytes is not null);

_activeStart = 0;
_availableStart = 0;
Expand All @@ -96,7 +96,7 @@ public void ClearAndReturnBuffer()

public void Discard(int byteCount)
{
Debug.Assert(byteCount <= ActiveLength, $"Expected {byteCount} <= {ActiveLength}");
// Debug.Assert(byteCount <= ActiveLength, $"Expected {byteCount} <= {ActiveLength}");
_activeStart += byteCount;

if (_activeStart == _availableStart)
Expand All @@ -108,7 +108,7 @@ public void Discard(int byteCount)

public void Commit(int byteCount)
{
Debug.Assert(byteCount <= AvailableLength);
// Debug.Assert(byteCount <= AvailableLength);
_availableStart += byteCount;
}

Expand All @@ -124,11 +124,11 @@ public void EnsureAvailableSpace(int byteCount)

private void EnsureAvailableSpaceCore(int byteCount)
{
Debug.Assert(AvailableLength < byteCount);
// Debug.Assert(AvailableLength < byteCount);

if (_bytes.Length == 0)
{
Debug.Assert(_usePool && _activeStart == 0 && _availableStart == 0);
// Debug.Assert(_usePool && _activeStart == 0 && _availableStart == 0);
_bytes = ArrayPool<byte>.Shared.Rent(byteCount);
return;
}
Expand All @@ -140,7 +140,7 @@ private void EnsureAvailableSpaceCore(int byteCount)
Buffer.BlockCopy(_bytes, _activeStart, _bytes, 0, ActiveLength);
_availableStart = ActiveLength;
_activeStart = 0;
Debug.Assert(byteCount <= AvailableLength);
// Debug.Assert(byteCount <= AvailableLength);
return;
}

Expand Down Expand Up @@ -168,7 +168,7 @@ private void EnsureAvailableSpaceCore(int byteCount)
_bytes = newBytes;
ReturnBufferIfPooled(oldBytes);

Debug.Assert(byteCount <= AvailableLength);
// Debug.Assert(byteCount <= AvailableLength);
}

public void Grow()
Expand Down
34 changes: 34 additions & 0 deletions src/libraries/System.IO.Pipelines/src/System/IO/Pipelines/Pipe.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public sealed partial class Pipe
private long PauseWriterThreshold => _options.PauseWriterThreshold;
private long ResumeWriterThreshold => _options.ResumeWriterThreshold;

private bool _bailValidation;
private int _validationOffset;

private PipeScheduler ReaderScheduler => _options.ReaderScheduler;
private PipeScheduler WriterScheduler => _options.WriterScheduler;

Expand Down Expand Up @@ -121,6 +124,9 @@ private void ResetState()
_lastExaminedIndex = -1;
_unflushedBytes = 0;
_unconsumedBytes = 0;

_bailValidation = false;
_validationOffset = 0;
}

internal Memory<byte> GetMemory(int sizeHint)
Expand Down Expand Up @@ -333,6 +339,32 @@ internal bool CommitUnsynchronized()
return resumeReader;
}

private void ValidateWritten(Span<byte> written)
{
if (_bailValidation)
{
return;
}

foreach (var b in written)
{
byte expected = (byte)(_validationOffset % 128);
if (expected != b)
{
if (_validationOffset < 16)
{
_bailValidation = true;
break;
}

// System.Console.WriteLine($"Pipe content validation failed at offset {_validationOffset}. Expected: 0x{expected:x2} Actual: {b:x2}");
Environment.FailFast($"Pipe content validation failed at offset {_validationOffset}. Expected: 0x{expected:x2} Actual: {b:x2}");
}

_validationOffset++;
}
}

internal void Advance(int bytes)
{
lock (SyncObj)
Expand All @@ -342,6 +374,8 @@ internal void Advance(int bytes)
ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.bytes);
}

ValidateWritten(_writingHeadMemory.Span.Slice(0, bytes));

// If the reader is completed we no-op Advance but leave GetMemory and FlushAsync alone
if (_readerCompletion.IsCompleted)
{
Expand Down
40 changes: 36 additions & 4 deletions src/libraries/System.Net.Http/tests/StressTests/HttpStress/ClientOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public sealed class RequestContext
private const string alphaNumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";

private readonly Random _random;
public Random Random => _random;
private readonly HttpClient _client;
private readonly CancellationToken _globalToken;
private readonly Configuration _config;
Expand Down Expand Up @@ -107,6 +108,18 @@ HttpResponseMessage WithVersionValidation(HttpResponseMessage m)
}
}

public byte[] GetByteArray(int minLength, int maxLength)
{
int length = _random.Next(minLength, maxLength);
byte[] byteContent = new byte[length];
for (int i = 0; i < length; i++)
{
byteContent[i] = (byte)(i % 128);
}

return byteContent;
}

/// Gets a random ASCII string within specified length range
public string GetRandomString(int minLength, int maxLength, bool alphaNumericOnly = true)
{
Expand Down Expand Up @@ -361,20 +374,21 @@ e is HttpProtocolException protocolException &&
("POST Duplex Slow",
async ctx =>
{
string content = ctx.GetRandomString(0, ctx.MaxContentLength);
byte[] byteContent = Encoding.ASCII.GetBytes(content);
byte[] byteContent = ctx.GetByteArray(0, ctx.MaxContentLength);
// byte[] byteContent = ctx.GetByteArray(0, 20);
// System.Console.WriteLine($"Sending {byteContent.Length} bytes");
ulong checksum = CRC.CalculateCRC(byteContent);

using var req = new HttpRequestMessage(HttpMethod.Post, "/duplexSlow") { Content = new ByteAtATimeNoLengthContent(byteContent) };
using HttpResponseMessage m = await ctx.SendAsync(req, HttpCompletionOption.ResponseHeadersRead);

ValidateStatusCode(m);
string response = await m.Content.ReadAsStringAsync();
byte[] response = await m.Content.ReadAsByteArrayAsync();

// trailing headers not supported for all servers, so do not require checksums
bool isValidChecksum = ValidateServerChecksum(m.TrailingHeaders, checksum, required: false);

ValidateContent(content, response, details: $"server checksum {(isValidChecksum ? "matches" : "does not match")} client value.");
ValidateByteContent(response, byteContent.Length, details: $"server checksum {(isValidChecksum ? "matches" : "does not match")} client value.");

if (!isValidChecksum)
{
Expand Down Expand Up @@ -495,6 +509,24 @@ private static void ValidateContent(string expectedContent, string actualContent
}
}

private static void ValidateByteContent(byte[] content, int length, string? details = null)
{
for (int i = 0; i < content.Length; i++)
{
byte expected = (byte)(i % 128);
if (content[i] != expected)
{
throw new Exception($"Expected response content diverging at index {i}, expected 0x{expected:x2} got 0x{content[i]:x2}. {details}");
}
}

if (content.Length != length)
{
throw new Exception($"Expected response content length {length}, got {content.Length}. {details}");
}
}


private static void ValidateServerContent(string content, int expectedLength)
{
if (content.Length != expectedLength)
Expand Down
Loading
Loading