From 8e740c1f69e9ae4cbe5a054a316b7c90e68f93c7 Mon Sep 17 00:00:00 2001 From: Antonstockmarr <43210147+Antonstockmarr@users.noreply.github.com> Date: Tue, 23 Nov 2021 04:36:19 -0800 Subject: [PATCH 1/2] Ping AOS before every request Make sure that the AOS is warm whenever a new client is created. Improve readability of errors caused by timeout. #patch --- .../AOSCommunicator.cs | 4 +-- .../Utilities/AOSClient.cs | 25 +++++++++++++++++-- .../Utilities/ReliableRun.cs | 6 ++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/AOSCommunicator.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/AOSCommunicator.cs index a5504c1..6a06085 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/AOSCommunicator.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/AOSCommunicator.cs @@ -20,7 +20,7 @@ protected async Task GetScaleUnitAosClient() if (scaleUnitAosClient is null) { ScaleUnitInstance scaleUnit = Config.FindScaleUnitWithId(ScaleUnitContext.GetScaleUnitId()); - SetScaleUnitAosClient(await AOSClient.Construct(scaleUnit)); + await ReliableRun.Execute(async () => SetScaleUnitAosClient(await AOSClient.Construct(scaleUnit)), "Connecting to AOS"); } return scaleUnitAosClient; } @@ -30,7 +30,7 @@ protected async Task GetHubAosClient() if (hubAosClient is null) { ScaleUnitInstance hub = Config.HubScaleUnit(); - SetHubAosClient(await AOSClient.Construct(hub)); + await ReliableRun.Execute(async () => SetHubAosClient(await AOSClient.Construct(hub)), "Connecting to AOS"); } return hubAosClient; } diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs index fc7023e..f6d1854 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs @@ -37,7 +37,19 @@ public static async Task Construct(ScaleUnitInstance scaleUnitInstanc httpClient.Timeout = TimeSpan.FromMinutes(20); httpClient.DefaultRequestHeaders.Add("Authorization", await OAuthHelper.GetAuthenticationHeader(aadTenant, aadClientAppId, aadClientAppSecret, aadResource)); - return new AOSClient(httpClient, scaleUnitInstance.AOSRequestPathPrefix()); + var aosClient = new AOSClient(httpClient, scaleUnitInstance.AOSRequestPathPrefix()); + + await aosClient.Ping(); + + return aosClient; + } + + private async Task Ping() + { + string path = $"{requestPathPrefix}ping/"; + string getPayload = "{}"; + + _ = await SendRequest(path, getPayload); } public async Task WriteScaleUnitConfiguration(ScaleUnitEnvironmentConfiguration config) @@ -183,7 +195,16 @@ private async Task SendRequest(string path, string payload) Content = new StringContent(payload, Encoding.UTF8, "application/json") }; - HttpResponseMessage response = await httpClient.SendAsync(msg); + HttpResponseMessage response; + try + { + response = await httpClient.SendAsync(msg); + } + catch (TaskCanceledException) + { + throw new TaskCanceledException("The server did not respond before timeout"); + } + string result = await response.Content.ReadAsStringAsync(); if (response.IsSuccessStatusCode) diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs index 70f7f43..ca62613 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs @@ -59,7 +59,11 @@ private static bool TryDiagnoseError(Exception exception, string commandName) private static bool IsRetryableAOSCall(Exception exception) { - if (!(exception is AOSClientError)) + if (exception is TaskCanceledException) + { + return true; + } + else if (!(exception is AOSClientError)) { return false; } From d9973155139306fa4fbb598ce22c0ef14f29cb86 Mon Sep 17 00:00:00 2001 From: Antonstockmarr <43210147+Antonstockmarr@users.noreply.github.com> Date: Tue, 23 Nov 2021 04:46:52 -0800 Subject: [PATCH 2/2] Do not retry in case of timeout --- .../WorkloadSetupOrchestrator/Utilities/AOSClient.cs | 2 +- .../WorkloadSetupOrchestrator/Utilities/ReliableRun.cs | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs index f6d1854..b96fc33 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs @@ -202,7 +202,7 @@ private async Task SendRequest(string path, string payload) } catch (TaskCanceledException) { - throw new TaskCanceledException("The server did not respond before timeout"); + throw new Exception("The server did not respond before timeout"); } string result = await response.Content.ReadAsStringAsync(); diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs index ca62613..70f7f43 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs @@ -59,11 +59,7 @@ private static bool TryDiagnoseError(Exception exception, string commandName) private static bool IsRetryableAOSCall(Exception exception) { - if (exception is TaskCanceledException) - { - return true; - } - else if (!(exception is AOSClientError)) + if (!(exception is AOSClientError)) { return false; }