Improve error handling from AOS requests

src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
@@ -191,13 +192,13 @@ private async Task<string> SendRequest(string path, string payload)
             }
             else
             {
-                throw RequestFailure((int)response.StatusCode, result);
+                throw RequestFailure(response.StatusCode, result);
             }
         }
 
-        private static Exception RequestFailure(int statusCode, string response)
+        private static Exception RequestFailure(HttpStatusCode statusCode, string response)
         {
-            return new Exception($"[{statusCode}]: Error - {response}");
+            return new AOSClientError(statusCode, response);
         }
     }
 }

src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClientError.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Net;
+
+namespace ScaleUnitManagement.WorkloadSetupOrchestrator.Utilities
+{
+    public sealed class AOSClientError : Exception
+    {
+        public AOSClientError(HttpStatusCode statusCode, string message)
+            : base(message)
+        {
+            StatusCode = statusCode;
+        }
+
+        public HttpStatusCode StatusCode { get; }
+    }
+}

src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs

@@ -3,10 +3,11 @@
 using System.Reflection;
 using System.Threading.Tasks;
 using ScaleUnitManagement.Utilities;
+using System.Net;
 
 namespace ScaleUnitManagement.WorkloadSetupOrchestrator.Utilities
 {
-    static class ReliableRun
+    public static class ReliableRun
     {
         private static readonly string ErrorLogFilePath = Path.Combine(
             Path.GetDirectoryName(Assembly.GetEntryAssembly().Location),
@@ -23,21 +24,75 @@ public static async Task Execute(Func<Task> command, string commandName)
                     await command();
                     break;
                 }
-                catch (Exception e)
+                catch (Exception exception)
                 {
-                    LogErrorToFile(commandName, e);
+                    if (TryDiagnoseError(exception, commandName) || !IsRetryableAOSCall(exception))
+                    {
+                        throw;
+                    }
 
                     if (tryCount == Config.RetryCount - 1)
                     {
-                        Console.WriteLine(commandName + " failed. Retry count exceeded. Execution will not continue.");
+                        Console.WriteLine("Retry count exceeded. Execution will not continue.");
                         throw;
                     }
 
-                    Console.WriteLine($"{commandName} failed. Retrying... (find error output in {ErrorLogFilePath})");
+                    tryCount++;
+                    Console.WriteLine("\nRetrying...");
                 }
+            }
+        }
+
+        private static bool TryDiagnoseError(Exception exception, string commandName)
+        {
+            LogErrorToFile(commandName, exception);
+            Console.WriteLine($"\n{commandName} failed with exception: {exception.Message} \nFind complete error output in {ErrorLogFilePath}");
+
+            string tsg = SuggestTSG(exception);
+            if (!string.IsNullOrEmpty(tsg))
+            {
+                Console.WriteLine(tsg);
+                return true;
+            }
+            else return false;
+        }
+
+        private static bool IsRetryableAOSCall(Exception exception)
+        {
+            if (!(exception is AOSClientError))
+            {
+                return false;
+            }
+            else
+            {
+                var aosClientError = exception as AOSClientError;
+                HttpStatusCode statusCode = aosClientError.StatusCode;
+                return (int)statusCode >= 300 &&
+                    statusCode != HttpStatusCode.BadRequest &&
+                    statusCode != HttpStatusCode.NotFound &&
+                    statusCode != HttpStatusCode.Unauthorized &&
+                    statusCode != HttpStatusCode.Forbidden;
+            }
+        }
 
-                tryCount++;
+        private static string SuggestTSG(Exception exception)
+        {
+            string invalidConfigurationError = "The provided Dynamics.AX.Application.ScaleUnitEnvironmentConfiguration is invalid";
+            if (exception.Message.Contains(invalidConfigurationError))
+            {
+                return "\nThere was a problem with the configuration. This could be caused by:\n" +
+                    "1. Trying to configure the spoke before configuring the hub.\n" +
+                    "2. Empty or missing fields in the configuration file. Make sure that the configuration file is filled out correctly, see details on the devTool wiki page.\n" +
+                    "3. Other errors that make this scale unit unable to communicate with the hub.\n";
+            }
+
+            string axNotRunningError = "No connection could be made because the target machine actively refused it";
+            if (exception.Message.Contains(axNotRunningError))
+            {
+                return "\nIs the IIS and the W3WP instance for AX running? If not, try running the \"Start services\" initialization step.\n";
             }
+
+            return "";
         }
 
         private static void LogErrorToFile(string commandName, Exception exception)