Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add exponential backoff to the retry algorithm of WebCmdlets #19637

Closed
wants to merge 8 commits into from
Closed

Add exponential backoff to the retry algorithm of WebCmdlets #19637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e41733f
Implememts exponential backoff retry to WebCmdlets
mkht May 10, 2023
837c462
Update src/Microsoft.PowerShell.Commands.Utility/commands/utility/Web…
mkht May 11, 2023
ba5d55b
Update src/Microsoft.PowerShell.Commands.Utility/commands/utility/Web…
mkht May 11, 2023
a4b1b49
Fix type conversion error
mkht May 11, 2023
e9de538
Replace MathF.Pow() to MathF.ScaleB()
mkht May 11, 2023
783f103
Change RetryMode type from string to WebRequestRetryMode enum
mkht May 14, 2023
87535df
Improve switch expression
mkht May 15, 2023
26e4155
Use Random.Shared
mkht May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
67 changes: 56 additions & 11 deletions ...owerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs
View file
Open in desktop
Expand Up @@ -85,6 +85,28 @@ public enum WebSslProtocol
Tls13 = SslProtocols.Tls13
}

/// <summary>
/// The valid values for the -RetryMode parameter for Invoke-RestMethod and Invoke-WebRequest.
/// </summary>
[Flags]
public enum WebRequestRetryMode
{
/// <summary>
/// Specifies fixed time interval between retries.
/// </summary>
Fixed,

/// <summary>
/// Specifies exponential backoff strategy to determine the interval between retries.
/// </summary>
Exponential,

/// <summary>
/// Specifies exponential backoff with jitter strategy to determine the interval between retries.
/// </summary>
ExponentialJitter
}

/// <summary>
/// Base class for Invoke-RestMethod and Invoke-WebRequest commands.
/// </summary>
Expand Down Expand Up @@ -122,6 +144,11 @@ public abstract class WebRequestPSCmdlet : PSCmdlet, IDisposable
/// </summary>
internal Dictionary<string, string> _relationLink = null;

/// <summary>
/// Maximum retry interval for exponential backoff strategy.
/// </summary>
private const int _maximumRetryIntervalInSeconds = 600;

/// <summary>
/// The current size of the local file being resumed.
/// </summary>
Expand Down Expand Up @@ -305,13 +332,6 @@ public abstract class WebRequestPSCmdlet : PSCmdlet, IDisposable
[ValidateRange(0, int.MaxValue)]
public virtual int MaximumRedirection { get; set; } = -1;

/// <summary>
/// Gets or sets the MaximumRetryCount property, which determines the number of retries of a failed web request.
/// </summary>
[Parameter]
[ValidateRange(0, int.MaxValue)]
public virtual int MaximumRetryCount { get; set; }

/// <summary>
/// Gets or sets the PreserveAuthorizationOnRedirect property.
/// </summary>
Expand All @@ -327,14 +347,31 @@ public abstract class WebRequestPSCmdlet : PSCmdlet, IDisposable
[Parameter]
public virtual SwitchParameter PreserveAuthorizationOnRedirect { get; set; }

#endregion Redirect

#region Retry

/// <summary>
/// Gets or sets the RetryMode property.
/// </summary>
[Parameter]
public virtual WebRequestRetryMode RetryMode { get; set; } = WebRequestRetryMode.Fixed;

/// <summary>
/// Gets or sets the MaximumRetryCount property, which determines the number of retries of a failed web request.
/// </summary>
[Parameter]
[ValidateRange(0, int.MaxValue)]
public virtual int MaximumRetryCount { get; set; }

/// <summary>
/// Gets or sets the RetryIntervalSec property, which determines the number seconds between retries.
/// </summary>
[Parameter]
[ValidateRange(1, int.MaxValue)]
public virtual int RetryIntervalSec { get; set; } = 5;

#endregion Redirect
#endregion Retry

#region Method

Expand Down Expand Up @@ -1335,7 +1372,15 @@ internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestM
// When MaximumRetryCount is not specified, the totalRequests is 1.
if (totalRequests > 1 && ShouldRetry(response.StatusCode))
{
int retryIntervalInSeconds = WebSession.RetryIntervalInSeconds;
float retryIntervalInSeconds = WebSession.RetryIntervalInSeconds;
int exponent = WebSession.MaximumRetryCount - totalRequests + 1;

retryIntervalInSeconds = RetryMode switch
{
WebRequestRetryMode.Exponential => MathF.Min(MathF.ScaleB(retryIntervalInSeconds, exponent), _maximumRetryIntervalInSeconds),
WebRequestRetryMode.ExponentialJitter => MathF.Min(MathF.ScaleB(retryIntervalInSeconds, exponent) * Random.Shared.NextSingle(), _maximumRetryIntervalInSeconds),
WebRequestRetryMode.Fixed or _ => WebSession.RetryIntervalInSeconds
};

// If the status code is 429 get the retry interval from the Headers.
// Ignore broken header and its value.
Expand All @@ -1358,13 +1403,13 @@ internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestM
string retryMessage = string.Format(
CultureInfo.CurrentCulture,
WebCmdletStrings.RetryVerboseMsg,
retryIntervalInSeconds,
retryIntervalInSeconds.ToString("G3"),
response.StatusCode);

WriteVerbose(retryMessage);

_cancelToken = new CancellationTokenSource();
Task.Delay(retryIntervalInSeconds * 1000, _cancelToken.Token).GetAwaiter().GetResult();
Task.Delay((int)(retryIntervalInSeconds * 1000), _cancelToken.Token).GetAwaiter().GetResult();
_cancelToken.Cancel();
_cancelToken = null;

Expand Down
74 changes: 74 additions & 0 deletions test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1
View file
Open in desktop
Expand Up @@ -2189,6 +2189,43 @@ Describe "Invoke-WebRequest tests" -Tags "Feature", "RequireAdminOnWindows" {

$verboseFile | Should -FileContentMatch 'Retrying after interval of 3 seconds. Status code for previous attempt: Conflict'
}

It "Invoke-WebRequest -RetryMode Exponential should use the exponential backoff stratefy for retrying." {

$Query = @{
statusCode = 404
reposnsephrase = 'NotFound'
contenttype = 'application/json'
body = '{"message":"oops"}'
}
$uri = Get-WebListenerUrl -Test 'Response' -Query $Query
$verboseFile = Join-Path $TestDrive -ChildPath verbose.txt
$result = Invoke-WebRequest -Uri $uri -RetryMode Exponential -MaximumRetryCount 3 -RetryIntervalSec 1 -SkipHttpErrorCheck -Verbose 4>$verbosefile

$exepectMessage = @(
'Retrying after interval of 1 seconds. Status code for previous attempt: NotFound'
'Retrying after interval of 2 seconds. Status code for previous attempt: NotFound'
'Retrying after interval of 4 seconds. Status code for previous attempt: NotFound'
) -join [System.Environment]::NewLine
$verboseFile | Should -FileContentMatchMultiline $exepectMessage
}

It "Invoke-WebRequest -RetryMode ExponentialJitter should use the exponential backoff stratefy with jitter for retrying." {

$Query = @{
statusCode = 404
reposnsephrase = 'NotFound'
contenttype = 'application/json'
body = '{"message":"oops"}'
}
$uri = Get-WebListenerUrl -Test 'Response' -Query $Query
$verboseFile = Join-Path $TestDrive -ChildPath verbose.txt
$result = Invoke-WebRequest -Uri $uri -RetryMode ExponentialJitter -MaximumRetryCount 3 -RetryIntervalSec 1 -SkipHttpErrorCheck -Verbose 4>$verbosefile

# In the exponential backoff stratefy with jitter, retry interval could not be estimated.
$exepectMessageRegex = 'Retrying after interval of [\.\d]+ seconds. Status code for previous attempt: NotFound'
Select-String -Path $verboseFile -Pattern $exepectMessageRegex -Raw | Should -HaveCount 3
}
}

Context "Regex Parsing" {
Expand Down Expand Up @@ -4111,6 +4148,43 @@ Describe "Invoke-RestMethod tests" -Tags "Feature", "RequireAdminOnWindows" {

$verboseFile | Should -FileContentMatch 'Retrying after interval of 3 seconds. Status code for previous attempt: Conflict'
}

It "Invoke-RestMethod -RetryMode Exponential should use the exponential backoff stratefy for retrying." {

$Query = @{
statusCode = 404
reposnsephrase = 'NotFound'
contenttype = 'application/json'
body = '{"message":"oops"}'
}
$uri = Get-WebListenerUrl -Test 'Response' -Query $Query
$verboseFile = Join-Path $TestDrive -ChildPath verbose.txt
$result = Invoke-RestMethod -Uri $uri -RetryMode Exponential -MaximumRetryCount 3 -RetryIntervalSec 1 -SkipHttpErrorCheck -Verbose 4>$verbosefile

$exepectMessage = @(
'Retrying after interval of 1 seconds. Status code for previous attempt: NotFound'
'Retrying after interval of 2 seconds. Status code for previous attempt: NotFound'
'Retrying after interval of 4 seconds. Status code for previous attempt: NotFound'
) -join [System.Environment]::NewLine
$verboseFile | Should -FileContentMatchMultiline $exepectMessage
}

It "Invoke-RestMethod -RetryMode ExponentialJitter should use the exponential backoff stratefy with jitter for retrying." {

$Query = @{
statusCode = 404
reposnsephrase = 'NotFound'
contenttype = 'application/json'
body = '{"message":"oops"}'
}
$uri = Get-WebListenerUrl -Test 'Response' -Query $Query
$verboseFile = Join-Path $TestDrive -ChildPath verbose.txt
$result = Invoke-RestMethod -Uri $uri -RetryMode ExponentialJitter -MaximumRetryCount 3 -RetryIntervalSec 1 -SkipHttpErrorCheck -Verbose 4>$verbosefile

# In the exponential backoff stratefy with jitter, retry interval could not be estimated.
$exepectMessageRegex = 'Retrying after interval of [\.\d]+ seconds. Status code for previous attempt: NotFound'
Select-String -Path $verboseFile -Pattern $exepectMessageRegex -Raw | Should -HaveCount 3
}
}
}

Expand Down