Ping AOS whenever an AOSClient is constructed

src/ScaleUnitManagement/WorkloadSetupOrchestrator/AOSCommunicator.cs

@@ -20,7 +20,7 @@ protected async Task<IAOSClient> GetScaleUnitAosClient()
             if (scaleUnitAosClient is null)
             {
                 ScaleUnitInstance scaleUnit = Config.FindScaleUnitWithId(ScaleUnitContext.GetScaleUnitId());
-                SetScaleUnitAosClient(await AOSClient.Construct(scaleUnit));
+                await ReliableRun.Execute(async () => SetScaleUnitAosClient(await AOSClient.Construct(scaleUnit)), "Connecting to AOS");
             }
             return scaleUnitAosClient;
         }
@@ -30,7 +30,7 @@ protected async Task<IAOSClient> GetHubAosClient()
             if (hubAosClient is null)
             {
                 ScaleUnitInstance hub = Config.HubScaleUnit();
-                SetHubAosClient(await AOSClient.Construct(hub));
+                await ReliableRun.Execute(async () => SetHubAosClient(await AOSClient.Construct(hub)), "Connecting to AOS");
             }
             return hubAosClient;
         }

src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs

@@ -37,7 +37,19 @@ public static async Task<AOSClient> Construct(ScaleUnitInstance scaleUnitInstanc
             httpClient.Timeout = TimeSpan.FromMinutes(20);
             httpClient.DefaultRequestHeaders.Add("Authorization", await OAuthHelper.GetAuthenticationHeader(aadTenant, aadClientAppId, aadClientAppSecret, aadResource));
 
-            return new AOSClient(httpClient, scaleUnitInstance.AOSRequestPathPrefix());
+            var aosClient = new AOSClient(httpClient, scaleUnitInstance.AOSRequestPathPrefix());
+
+            await aosClient.Ping();
+
+            return aosClient;
+        }
+
+        private async Task Ping()
+        {
+            string path = $"{requestPathPrefix}ping/";
+            string getPayload = "{}";
+
+            _ = await SendRequest(path, getPayload);
         }
 
         public async Task<ScaleUnitEnvironmentConfiguration> WriteScaleUnitConfiguration(ScaleUnitEnvironmentConfiguration config)
@@ -183,7 +195,16 @@ private async Task<string> SendRequest(string path, string payload)
                 Content = new StringContent(payload, Encoding.UTF8, "application/json")
             };
 
-            HttpResponseMessage response = await httpClient.SendAsync(msg);
+            HttpResponseMessage response;
+            try
+            {
+                response = await httpClient.SendAsync(msg);
+            }
+            catch (TaskCanceledException)
+            {
+                throw new Exception("The server did not respond before timeout");
+            }
+
             string result = await response.Content.ReadAsStringAsync();
 
             if (response.IsSuccessStatusCode)