Add the ability to stop the Host when one of its `BackgroundService…

…` instances errors (#50569)

* Added background service exception behavior enum. Update HostOptions and Host accordingly.

* Added corresponding unit tests for host when ignoring or stopping.

* Call StopAsync rather than FailFast

* Call `IApplicationLifetime.StopApplication()` instead of `StopAsync()`.

* Moved enum out of abstractions. Encapsulate logging logic in extensions. Define new EventId. Simplify check, avoid null eval. Update triple dash comments per peer review.

* Set the default as `StopHost`

* Move ref bits from hosting.abstractions to hosting proper.

* Address feedback from peer review

* Added unit test to verify background service 1 gets started, and runs a bit - and other service can cause stop

* Use explicitly controlled timing for unit test and remove args: ex

* Convert from Fact, to Theory. Test both enum values.

* Separate StartAsync and ExecuteTask, let StartAsync bubble. Adjust unit tests...

* Update the background service async exceptions get logged unit test

src/libraries/Microsoft.Extensions.Hosting/ref/Microsoft.Extensions.Hosting.cs

@@ -13,6 +13,11 @@ public static partial class OptionsBuilderExtensions
 }
 namespace Microsoft.Extensions.Hosting
 {
+    public enum BackgroundServiceExceptionBehavior
+    {
+        StopHost,
+        Ignore
+    }
     public partial class ConsoleLifetimeOptions
     {
         public ConsoleLifetimeOptions() { }
@@ -56,6 +61,7 @@ public partial class HostOptions
     {
         public HostOptions() { }
         public System.TimeSpan ShutdownTimeout { get { throw null; } set { } }
+        public Microsoft.Extensions.Hosting.BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get { throw null; } set { } }
     }
 }
 namespace Microsoft.Extensions.Hosting.Internal

src/libraries/Microsoft.Extensions.Hosting/src/BackgroundServiceExceptionBehavior.cs

@@ -0,0 +1,29 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.Hosting
+{
+    /// <summary>
+    /// Specifies a behavior that the <see cref="IHost"/> will honor if an
+    /// unhandled exception occurs in one of its <see cref="BackgroundService"/> instances.
+    /// </summary>
+    public enum BackgroundServiceExceptionBehavior
+    {
+        /// <summary>
+        /// Stops the <see cref="IHost"/> instance.
+        /// </summary>
+        /// <remarks>
+        /// If a <see cref="BackgroundService"/> throws an exception, the <see cref="IHost"/> instance stops, and the process continues.
+        /// </remarks>
+        StopHost = 0,
+
+        /// <summary>
+        /// Ignore exceptions thrown in <see cref="BackgroundService"/>.
+        /// </summary>
+        /// <remarks>
+        /// If a <see cref="BackgroundService"/> throws an exception, the <see cref="IHost"/> will log the error, but otherwise ignore it.
+        /// The <see cref="BackgroundService"/> is not restarted.
+        /// </remarks>
+        Ignore = 1
+    }
+}

src/libraries/Microsoft.Extensions.Hosting/src/HostOptions.cs

@@ -17,6 +17,16 @@ public class HostOptions
         /// </summary>
         public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5);
 
+        /// <summary>
+        /// The behavior the <see cref="IHost"/> will follow when any of
+        /// its <see cref="BackgroundService"/> instances throw an unhandled exception.
+        /// </summary>
+        /// <remarks>
+        /// Defaults to <see cref="BackgroundServiceExceptionBehavior.StopHost"/>.
+        /// </remarks>
+        public BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get; set; } =
+            BackgroundServiceExceptionBehavior.StopHost;
+
         internal void Initialize(IConfiguration configuration)
         {
             var timeoutSeconds = configuration["shutdownTimeoutSeconds"];

src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs

@@ -66,7 +66,7 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
 
                 if (hostedService is BackgroundService backgroundService)
                 {
-                    _ = HandleBackgroundException(backgroundService);
+                    _ = TryExecuteBackgroundServiceAsync(backgroundService);
                 }
             }
 
@@ -76,7 +76,7 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
             _logger.Started();
         }
 
-        private async Task HandleBackgroundException(BackgroundService backgroundService)
+        private async Task TryExecuteBackgroundServiceAsync(BackgroundService backgroundService)
         {
             try
             {
@@ -85,6 +85,11 @@ private async Task HandleBackgroundException(BackgroundService backgroundService
             catch (Exception ex)
             {
                 _logger.BackgroundServiceFaulted(ex);
+                if (_options.BackgroundServiceExceptionBehavior == BackgroundServiceExceptionBehavior.StopHost)
+                {
+                    _logger.BackgroundServiceStoppingHost(ex);
+                    _applicationLifetime.StopApplication();
+                }
             }
         }
 

src/libraries/Microsoft.Extensions.Hosting/src/Internal/HostingLoggerExtensions.cs

@@ -11,8 +11,7 @@ internal static class HostingLoggerExtensions
     {
         public static void ApplicationError(this ILogger logger, EventId eventId, string message, Exception exception)
         {
-            var reflectionTypeLoadException = exception as ReflectionTypeLoadException;
-            if (reflectionTypeLoadException != null)
+            if (exception is ReflectionTypeLoadException reflectionTypeLoadException)
             {
                 foreach (Exception ex in reflectionTypeLoadException.LoaderExceptions)
                 {
@@ -87,5 +86,16 @@ public static void BackgroundServiceFaulted(this ILogger logger, Exception ex)
                     message: "BackgroundService failed");
             }
         }
+
+        public static void BackgroundServiceStoppingHost(this ILogger logger, Exception ex)
+        {
+            if (logger.IsEnabled(LogLevel.Critical))
+            {
+                logger.LogCritical(
+                    eventId: LoggerEventIds.BackgroundServiceStoppingHost,
+                    exception: ex,
+                    message: SR.BackgroundServiceExceptionStoppedHost);
+            }
+        }
     }
 }

src/libraries/Microsoft.Extensions.Hosting/src/Internal/LoggerEventIds.cs

@@ -7,14 +7,15 @@ namespace Microsoft.Extensions.Hosting.Internal
 {
     internal static class LoggerEventIds
     {
-        public static readonly EventId Starting = new EventId(1, "Starting");
-        public static readonly EventId Started = new EventId(2, "Started");
-        public static readonly EventId Stopping = new EventId(3, "Stopping");
-        public static readonly EventId Stopped = new EventId(4, "Stopped");
-        public static readonly EventId StoppedWithException = new EventId(5, "StoppedWithException");
-        public static readonly EventId ApplicationStartupException = new EventId(6, "ApplicationStartupException");
-        public static readonly EventId ApplicationStoppingException = new EventId(7, "ApplicationStoppingException");
-        public static readonly EventId ApplicationStoppedException = new EventId(8, "ApplicationStoppedException");
-        public static readonly EventId BackgroundServiceFaulted = new EventId(9, "BackgroundServiceFaulted");
+        public static readonly EventId Starting = new EventId(1, nameof(Starting));
+        public static readonly EventId Started = new EventId(2, nameof(Started));
+        public static readonly EventId Stopping = new EventId(3, nameof(Stopping));
+        public static readonly EventId Stopped = new EventId(4, nameof(Stopped));
+        public static readonly EventId StoppedWithException = new EventId(5, nameof(StoppedWithException));
+        public static readonly EventId ApplicationStartupException = new EventId(6, nameof(ApplicationStartupException));
+        public static readonly EventId ApplicationStoppingException = new EventId(7, nameof(ApplicationStoppingException));
+        public static readonly EventId ApplicationStoppedException = new EventId(8, nameof(ApplicationStoppedException));
+        public static readonly EventId BackgroundServiceFaulted = new EventId(9, nameof(BackgroundServiceFaulted));
+        public static readonly EventId BackgroundServiceStoppingHost = new EventId(10, nameof(BackgroundServiceStoppingHost));
     }
 }

src/libraries/Microsoft.Extensions.Hosting/src/Resources/Strings.resx

@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!-- 
-    Microsoft ResX Schema 
+  <!--
+    Microsoft ResX Schema
 
     Version 2.0
 
     The primary goals of this format is to allow a simple XML format
-    that is mostly human readable. The generation and parsing of the 
-    various data types are done through the TypeConverter classes 
+    that is mostly human readable. The generation and parsing of the
+    various data types are done through the TypeConverter classes
     associated with the data types.
 
     Example:
@@ -55,7 +55,7 @@
             : and then encoded with base64 encoding.
 
     mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array 
+    value   : The object must be serialized into a byte array
             : using a System.ComponentModel.TypeConverter
             : and then encoded with base64 encoding.
     -->
@@ -117,6 +117,9 @@
   <resheader name="writer">
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
+   <data name="BackgroundServiceExceptionStoppedHost" xml:space="preserve">
+    <value>The HostOptions.BackgroundServiceExceptionBehavior is configured to StopHost. A BackgroundService has thrown an unhandled exception, and the IHost instance is stopping. To avoid this behavior, configure this to Ignore; however the BackgroundService will not be restarted.</value>
+  </data>
   <data name="BuildCalled" xml:space="preserve">
     <value>Build can only be called once.</value>
   </data>
@@ -129,4 +132,4 @@
   <data name="ResolverReturnedNull" xml:space="preserve">
     <value>The resolver returned a null IServiceProviderFactory</value>
   </data>
-</root>
+</root>

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostBuilderTests.cs

@@ -572,6 +572,27 @@ public void HostBuilderConfigureDefaultsInterleavesMissingConfigValues()
             Assert.Equal(expectedContentRootPath, env.ContentRootPath);
         }
 
+        [Theory]
+        [InlineData(BackgroundServiceExceptionBehavior.Ignore)]
+        [InlineData(BackgroundServiceExceptionBehavior.StopHost)]
+        public void HostBuilderCanConfigureBackgroundServiceExceptionBehavior(
+            BackgroundServiceExceptionBehavior testBehavior)
+        {
+            using IHost host = new HostBuilder()
+                .ConfigureServices(
+                    services =>
+                        services.Configure<HostOptions>(
+                            options =>
+                            options.BackgroundServiceExceptionBehavior = testBehavior))
+                .Build();
+
+            var options = host.Services.GetRequiredService<IOptions<HostOptions>>();
+
+            Assert.Equal(
+                testBehavior,
+                options.Value.BackgroundServiceExceptionBehavior);
+        }
+
         private class FakeFileProvider : IFileProvider, IDisposable
         {
             public bool Disposed { get; private set; }

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs

@@ -146,7 +146,7 @@ public void CreateDefaultBuilder_EnablesValidateOnBuild()
         [ActiveIssue("https://github.com/dotnet/runtime/issues/48696")]
         public async Task CreateDefaultBuilder_ConfigJsonDoesNotReload()
         {
-            var reloadFlagConfig = new Dictionary<string, string>() {{ "hostbuilder:reloadConfigOnChange", "false" }};
+            var reloadFlagConfig = new Dictionary<string, string>() { { "hostbuilder:reloadConfigOnChange", "false" } };
             var appSettingsPath = Path.Combine(Path.GetTempPath(), "appsettings.json");
 
             string SaveRandomConfig()