Exit gracefully if a validation error occurs

I've tweaked the way that the startup process for Promitor works so that it runs the validation in the `Main()` method. This gives us the opportunity to exit gracefully if validation fails instead of throwing an exception.

I've also added a new enum to track the possible exit statuses, and made sure that unhandled exceptions continue to use an exit code of `1`.

Fixes tomkerkhove#1113

src/Promitor.Agents.Scraper/ExitStatus.cs

@@ -0,0 +1,24 @@
+namespace Promitor.Agents.Scraper
+{
+    /// <summary>
+    /// The different statuses that the agent scraper can exit with.
+    /// </summary>
+    public enum ExitStatus
+    {
+        /// <summary>
+        /// The application has run successfully.
+        /// </summary>
+        Success = 0,
+
+        /// <summary>
+        /// An unhandled exception was thrown during host startup. This probably
+        /// indicates a bug in Promitor that should be reported.
+        /// </summary>
+        UnhandledException = 1,
+
+        /// <summary>
+        /// Validation failed, so Promitor can't start.
+        /// </summary>
+        ValidationFailed = 2
+    }
+}

src/Promitor.Agents.Scraper/Program.cs

@@ -1,9 +1,11 @@
-using System;
+using System;
 using System.IO;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Promitor.Agents.Core;
 using Promitor.Agents.Core.Configuration.Server;
+using Promitor.Agents.Scraper.Validation;
 using Promitor.Core;
 using Serilog;
 
@@ -20,16 +22,26 @@ public static int Main(string[] args)
                 // Let's hook in a logger for start-up purposes.
                 ConfigureStartupLogging();
 
-                CreateHostBuilder(args)
-                    .Build()
-                    .Run();
+                var host = CreateHostBuilder(args)
+                    .Build();
+
+                using (var scope = host.Services.CreateScope())
+                {
+                    var validator = scope.ServiceProvider.GetRequiredService<RuntimeValidator>();
+                    if (!validator.Run())
+                    {
+                        return (int)ExitStatus.ValidationFailed;
+                    }
+                }
 
-                return 0;
+                host.Run();
+
+                return (int)ExitStatus.Success;
             }
             catch (Exception exception)
             {
                 Log.Fatal(exception, "Host terminated unexpectedly");
-                return 1;
+                return (int)ExitStatus.UnhandledException;
             }
             finally
             {

src/Promitor.Agents.Scraper/Startup.cs

@@ -12,7 +12,6 @@
 using Promitor.Agents.Scraper.Configuration.Sinks;
 using Promitor.Agents.Scraper.Extensions;
 using Promitor.Agents.Scraper.Health;
-using Promitor.Agents.Scraper.Validation;
 using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping;
 using Promitor.Integrations.AzureMonitor.Logging;
 using Serilog;
@@ -55,8 +54,6 @@ public void ConfigureServices(IServiceCollection services)
                 healthCheckBuilder.AddCheck<ResourceDiscoveryHealthCheck>("Promitor Resource Discovery", HealthStatus.Degraded);
             }
 
-            ValidateRuntimeConfiguration(services);
-
             services.UseMetricSinks(Configuration)
                 .ScheduleMetricScraping();
         }
@@ -80,13 +77,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
             UseSerilog(ComponentName, app.ApplicationServices);
         }
 
-        private void ValidateRuntimeConfiguration(IServiceCollection services)
-        {
-            var serviceProvider = services.BuildServiceProvider();
-            var runtimeValidator = serviceProvider.GetService<RuntimeValidator>();
-            runtimeValidator.Run();
-        }
-
         protected override LoggerConfiguration FilterTelemetry(LoggerConfiguration loggerConfiguration)
         {
             var standardConfiguration = base.FilterTelemetry(loggerConfiguration);

src/Promitor.Agents.Scraper/Validation/RuntimeValidator.cs

@@ -39,26 +39,27 @@ public class RuntimeValidator
             };
         }
 
-        public void Run()
+        public bool Run()
         {
             _validationLogger.LogInformation("Starting validation of Promitor setup");
 
             var validationResults = RunValidationSteps();
-            ProcessValidationResults(validationResults);
+            return ProcessValidationResults(validationResults);
         }
 
-        private void ProcessValidationResults(List<ValidationResult> validationResults)
+        private bool ProcessValidationResults(List<ValidationResult> validationResults)
         {
             var failedValidationResults = validationResults.Where(result => result.IsSuccessful == false).ToList();
 
             var validationFailed = failedValidationResults.Any();
             if (validationFailed)
             {
                 _validationLogger.LogCritical("Promitor is not configured correctly. Please fix validation issues and re-run.");
-                throw new ValidationFailedException(failedValidationResults);
+                return false;
             }
 
             _validationLogger.LogInformation("Promitor configuration is valid, we are good to go.");
+            return true;
         }
 
         private List<ValidationResult> RunValidationSteps()