From 9c25393eb5e8977a5bd6461cd5d728681946e7de Mon Sep 17 00:00:00 2001 From: Antonstockmarr <43210147+Antonstockmarr@users.noreply.github.com> Date: Tue, 26 Oct 2021 06:16:34 -0700 Subject: [PATCH 1/4] Improve error handling from AOS requests ReliableRun class will catch AOS exceptions and try to suggest changes. Other errors pass through. If a suggestion is found, or it does not make sense to retry, the request is not repeated. Every AOS error is shown to the user. resolves #105 #patch --- .../Utilities/AOSClient.cs | 7 ++- .../Utilities/AOSClientError.cs | 16 +++++ .../Utilities/ReliableRun.cs | 59 ++++++++++++++++--- 3 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClientError.cs diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs index 83b1ffd..b012688 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net; using System.Net.Http; using System.Text; using System.Threading.Tasks; @@ -190,13 +191,13 @@ private async Task SendRequest(string path, string payload) } else { - throw RequestFailure((int)response.StatusCode, result); + throw RequestFailure(response.StatusCode, result); } } - private static Exception RequestFailure(int statusCode, string response) + private static Exception RequestFailure(HttpStatusCode statusCode, string response) { - return new Exception($"[{statusCode}]: Error - {response}"); + return new AOSClientError(statusCode, response); } } } diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClientError.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClientError.cs new file mode 100644 index 0000000..23d2100 --- /dev/null +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/AOSClientError.cs @@ -0,0 +1,16 @@ +using System; +using System.Net; + +namespace ScaleUnitManagement.WorkloadSetupOrchestrator.Utilities +{ + public sealed class AOSClientError : Exception + { + public AOSClientError(HttpStatusCode statusCode, string message) + : base(message) + { + StatusCode = statusCode; + } + + public HttpStatusCode StatusCode { get; } + } +} diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs index 8d83d45..40798eb 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs @@ -3,10 +3,11 @@ using System.Reflection; using System.Threading.Tasks; using ScaleUnitManagement.Utilities; +using System.Net; namespace ScaleUnitManagement.WorkloadSetupOrchestrator.Utilities { - static class ReliableRun + public static class ReliableRun { private static readonly string ErrorLogFilePath = Path.Combine( Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), @@ -23,23 +24,65 @@ public static async Task Execute(Func command, string commandName) await command(); break; } - catch (Exception e) + catch (AOSClientError e) { - LogErrorToFile(commandName, e); - - if (tryCount == Config.RetryCount - 1) + HandleError(e, commandName, tryCount); + string tsg = SuggestTSG(e); + if (!string.IsNullOrEmpty(tsg)) + { + Console.WriteLine(tsg); + throw; + } + if (!IsRetryableAOSCall(e.StatusCode)) { - Console.WriteLine(commandName + " failed. Retry count exceeded. Execution will not continue."); throw; } - - Console.WriteLine($"{commandName} failed. Retrying... (find error output in {ErrorLogFilePath})"); } tryCount++; + Console.WriteLine("\nRetrying..."); + } + } + + private static void HandleError(Exception e, string commandName, int tryCount) + { + LogErrorToFile(commandName, e); + Console.WriteLine($"\n{commandName} failed with exception: {e.Message} \nFind complete error output in {ErrorLogFilePath}"); + + if (tryCount == Config.RetryCount - 1) + { + Console.WriteLine(commandName + " failed. Retry count exceeded. Execution will not continue."); + throw e; } } + private static bool IsRetryableAOSCall(HttpStatusCode statusCode) + { + Console.WriteLine((int)statusCode); + return (int)statusCode >= 300 && + statusCode != HttpStatusCode.BadRequest && + statusCode != HttpStatusCode.NotFound && + statusCode != HttpStatusCode.Unauthorized && + statusCode != HttpStatusCode.Forbidden; + } + + private static string SuggestTSG(Exception e) + { + string spokeConfiguredBeforeHubError = "The provided Dynamics.AX.Application.ScaleUnitEnvironmentConfiguration is invalid"; + if (e.Message.Contains(spokeConfiguredBeforeHubError)) + { + return "\nDid you try to configure the spoke before configuring the hub?\n"; + } + + string axNotRunningError = "No connection could be made because the target machine actively refused it"; + if (e.Message.Contains(axNotRunningError)) + { + return "\nIs the AX batch service running?\n"; + } + + return ""; + } + private static void LogErrorToFile(string commandName, Exception exception) { File.AppendAllText(ErrorLogFilePath, $"[{DateTime.Now}] Error during {commandName}:\n{exception}\n\n"); From d8ad0013140e7b996445f9d3b02a5ddf64b0bcae Mon Sep 17 00:00:00 2001 From: Antonstockmarr <43210147+Antonstockmarr@users.noreply.github.com> Date: Fri, 29 Oct 2021 02:08:49 -0700 Subject: [PATCH 2/4] More elaborate TSG and proper error handling --- .../Utilities/ReliableRun.cs | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs index 40798eb..a8e3116 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs @@ -26,33 +26,37 @@ public static async Task Execute(Func command, string commandName) } catch (AOSClientError e) { - HandleError(e, commandName, tryCount); - string tsg = SuggestTSG(e); - if (!string.IsNullOrEmpty(tsg)) + if (TryHandleError(e, commandName)) { - Console.WriteLine(tsg); throw; } - if (!IsRetryableAOSCall(e.StatusCode)) + + if (tryCount == Config.RetryCount - 1) { + Console.WriteLine(commandName + " failed. Retry count exceeded. Execution will not continue."); throw; } - } - tryCount++; - Console.WriteLine("\nRetrying..."); + tryCount++; + Console.WriteLine("\nRetrying..."); + } } } - private static void HandleError(Exception e, string commandName, int tryCount) + private static bool TryHandleError(AOSClientError e, string commandName) { LogErrorToFile(commandName, e); Console.WriteLine($"\n{commandName} failed with exception: {e.Message} \nFind complete error output in {ErrorLogFilePath}"); - if (tryCount == Config.RetryCount - 1) + string tsg = SuggestTSG(e); + if (!string.IsNullOrEmpty(tsg)) + { + Console.WriteLine(tsg); + return true; + } + else { - Console.WriteLine(commandName + " failed. Retry count exceeded. Execution will not continue."); - throw e; + return !IsRetryableAOSCall(e.StatusCode); } } @@ -68,16 +72,19 @@ private static bool IsRetryableAOSCall(HttpStatusCode statusCode) private static string SuggestTSG(Exception e) { - string spokeConfiguredBeforeHubError = "The provided Dynamics.AX.Application.ScaleUnitEnvironmentConfiguration is invalid"; - if (e.Message.Contains(spokeConfiguredBeforeHubError)) + string invalidConfigurationError = "The provided Dynamics.AX.Application.ScaleUnitEnvironmentConfiguration is invalid"; + if (e.Message.Contains(invalidConfigurationError)) { - return "\nDid you try to configure the spoke before configuring the hub?\n"; + return "\nThere was a problem with the configuration. This could be caused by:\n" + + "1. Trying to configure the spoke before configuring the hub.\n" + + "2. Empty or missing fields in the configuration file. Make sure that the configuration file is filled out correctly, see details on the devTool wiki page.\n" + + "3. Other errors that make this scale unit unable to communicate with the hub.\n"; } string axNotRunningError = "No connection could be made because the target machine actively refused it"; if (e.Message.Contains(axNotRunningError)) { - return "\nIs the AX batch service running?\n"; + return "\nIs the IIS and the W3WP instance for AX running? If not, try running the \"Start services\" initialization step.\n"; } return ""; From 89cf9165593b0098e8ec055d6f0dce07b7a9550d Mon Sep 17 00:00:00 2001 From: Antonstockmarr <43210147+Antonstockmarr@users.noreply.github.com> Date: Fri, 29 Oct 2021 07:14:41 -0700 Subject: [PATCH 3/4] Changes method name --- .../Utilities/ReliableRun.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs index a8e3116..8f19aaf 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs @@ -26,14 +26,14 @@ public static async Task Execute(Func command, string commandName) } catch (AOSClientError e) { - if (TryHandleError(e, commandName)) + if (TryDiagnoseError(e, commandName) || !IsRetryableAOSCall(e.StatusCode)) { throw; } if (tryCount == Config.RetryCount - 1) { - Console.WriteLine(commandName + " failed. Retry count exceeded. Execution will not continue."); + Console.WriteLine("Retry count exceeded. Execution will not continue."); throw; } @@ -43,7 +43,7 @@ public static async Task Execute(Func command, string commandName) } } - private static bool TryHandleError(AOSClientError e, string commandName) + private static bool TryDiagnoseError(AOSClientError e, string commandName) { LogErrorToFile(commandName, e); Console.WriteLine($"\n{commandName} failed with exception: {e.Message} \nFind complete error output in {ErrorLogFilePath}"); @@ -54,10 +54,7 @@ private static bool TryHandleError(AOSClientError e, string commandName) Console.WriteLine(tsg); return true; } - else - { - return !IsRetryableAOSCall(e.StatusCode); - } + else return false; } private static bool IsRetryableAOSCall(HttpStatusCode statusCode) From 4c9bfc8f4a09daf200e42492ca41afe0ac0c8ad6 Mon Sep 17 00:00:00 2001 From: Antonstockmarr <43210147+Antonstockmarr@users.noreply.github.com> Date: Tue, 9 Nov 2021 02:16:50 -0800 Subject: [PATCH 4/4] Catches Exceptions instead of AOSClientErrors Timeout errors and similar will be caught by reliableRun --- .../Utilities/ReliableRun.cs | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs index 8f19aaf..70f7f43 100644 --- a/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs +++ b/src/ScaleUnitManagement/WorkloadSetupOrchestrator/Utilities/ReliableRun.cs @@ -24,9 +24,9 @@ public static async Task Execute(Func command, string commandName) await command(); break; } - catch (AOSClientError e) + catch (Exception exception) { - if (TryDiagnoseError(e, commandName) || !IsRetryableAOSCall(e.StatusCode)) + if (TryDiagnoseError(exception, commandName) || !IsRetryableAOSCall(exception)) { throw; } @@ -43,12 +43,12 @@ public static async Task Execute(Func command, string commandName) } } - private static bool TryDiagnoseError(AOSClientError e, string commandName) + private static bool TryDiagnoseError(Exception exception, string commandName) { - LogErrorToFile(commandName, e); - Console.WriteLine($"\n{commandName} failed with exception: {e.Message} \nFind complete error output in {ErrorLogFilePath}"); + LogErrorToFile(commandName, exception); + Console.WriteLine($"\n{commandName} failed with exception: {exception.Message} \nFind complete error output in {ErrorLogFilePath}"); - string tsg = SuggestTSG(e); + string tsg = SuggestTSG(exception); if (!string.IsNullOrEmpty(tsg)) { Console.WriteLine(tsg); @@ -57,20 +57,28 @@ private static bool TryDiagnoseError(AOSClientError e, string commandName) else return false; } - private static bool IsRetryableAOSCall(HttpStatusCode statusCode) + private static bool IsRetryableAOSCall(Exception exception) { - Console.WriteLine((int)statusCode); - return (int)statusCode >= 300 && - statusCode != HttpStatusCode.BadRequest && - statusCode != HttpStatusCode.NotFound && - statusCode != HttpStatusCode.Unauthorized && - statusCode != HttpStatusCode.Forbidden; + if (!(exception is AOSClientError)) + { + return false; + } + else + { + var aosClientError = exception as AOSClientError; + HttpStatusCode statusCode = aosClientError.StatusCode; + return (int)statusCode >= 300 && + statusCode != HttpStatusCode.BadRequest && + statusCode != HttpStatusCode.NotFound && + statusCode != HttpStatusCode.Unauthorized && + statusCode != HttpStatusCode.Forbidden; + } } - private static string SuggestTSG(Exception e) + private static string SuggestTSG(Exception exception) { string invalidConfigurationError = "The provided Dynamics.AX.Application.ScaleUnitEnvironmentConfiguration is invalid"; - if (e.Message.Contains(invalidConfigurationError)) + if (exception.Message.Contains(invalidConfigurationError)) { return "\nThere was a problem with the configuration. This could be caused by:\n" + "1. Trying to configure the spoke before configuring the hub.\n" + @@ -79,7 +87,7 @@ private static string SuggestTSG(Exception e) } string axNotRunningError = "No connection could be made because the target machine actively refused it"; - if (e.Message.Contains(axNotRunningError)) + if (exception.Message.Contains(axNotRunningError)) { return "\nIs the IIS and the W3WP instance for AX running? If not, try running the \"Start services\" initialization step.\n"; }