Add the ability to stop the Host when one of its BackgroundService instances errors

src/libraries/Microsoft.Extensions.Hosting.Abstractions/ref/Microsoft.Extensions.Hosting.Abstractions.cs

@@ -23,6 +23,11 @@ public abstract partial class BackgroundService : Microsoft.Extensions.Hosting.I
         public virtual System.Threading.Tasks.Task StartAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
         public virtual System.Threading.Tasks.Task StopAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
     }
+    public enum BackgroundServiceExceptionBehavior
+    {
+        Ignore,
+        StopHost
+    }
     [System.ObsoleteAttribute("This type is obsolete and will be removed in a future version. The recommended alternative is Microsoft.Extensions.Hosting.Environments.", false)]
     public static partial class EnvironmentName
     {

src/libraries/Microsoft.Extensions.Hosting.Abstractions/src/BackgroundService.cs

@@ -75,7 +75,6 @@ public virtual async Task StopAsync(CancellationToken cancellationToken)
                 // Wait until the task completes or the stop token triggers
                 await Task.WhenAny(_executeTask, Task.Delay(Timeout.Infinite, cancellationToken)).ConfigureAwait(false);
             }
-
         }
 
         public virtual void Dispose()

src/libraries/Microsoft.Extensions.Hosting.Abstractions/src/BackgroundServiceExceptionBehavior.cs

@@ -0,0 +1,29 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.Hosting
+{
+    /// <summary>
+    /// Specifies a behavior that the <see cref="IHost"/> will honor if an
+    /// unhandled exception occurs in one of its <see cref="BackgroundService"/> instances.
+    /// </summary>
+    public enum BackgroundServiceExceptionBehavior
+    {
+        /// <summary>
+        /// Ignore exceptions thrown in <see cref="BackgroundService"/>.
+        /// </summary>
+        /// <remarks>
+        /// If a <see cref="BackgroundService"/> throws an exception, the <see cref="IHost"/> will ignore it.
+        /// The <see cref="BackgroundService"/> is not restarted.
+        /// </remarks>
+        Ignore = 0,
+
+        /// <summary>
+        /// Stops the <see cref="IHost"/> instance, and terminates the process.
+        /// </summary>
+        /// <remarks>
+        /// If a <see cref="BackgroundService"/> throws an exception, the <see cref="IHost"/> will terminate the process.
+        /// </remarks>
+        StopHost = 1
+    }
+}

src/libraries/Microsoft.Extensions.Hosting/ref/Microsoft.Extensions.Hosting.cs

@@ -56,6 +56,7 @@ public partial class HostOptions
     {
         public HostOptions() { }
         public System.TimeSpan ShutdownTimeout { get { throw null; } set { } }
+        public Microsoft.Extensions.Hosting.BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get { throw null; } set { } }
     }
 }
 namespace Microsoft.Extensions.Hosting.Internal

src/libraries/Microsoft.Extensions.Hosting/src/HostOptions.cs

@@ -17,6 +17,12 @@ public class HostOptions
         /// </summary>
         public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5);
 
+        /// <summary>
+        /// The behavior the <see cref="IHost"/> will follow when any of
+        /// its <see cref="BackgroundService"/> instances throw an unhandled exception.
+        /// </summary>
+        public BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get; set; }
+
         internal void Initialize(IConfiguration configuration)
         {
             var timeoutSeconds = configuration["shutdownTimeoutSeconds"];

src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs

@@ -61,12 +61,15 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
 
             foreach (IHostedService hostedService in _hostedServices)
             {
-                // Fire IHostedService.Start
-                await hostedService.StartAsync(combinedCancellationToken).ConfigureAwait(false);
-
                 if (hostedService is BackgroundService backgroundService)
                 {
-                    _ = HandleBackgroundException(backgroundService);
+                    _ = TryStartBackgroundServiceAsync(
+                        backgroundService, combinedCancellationToken);
+                }
+                else
+                {
+                    // Fire IHostedService.Start
+                    await hostedService.StartAsync(combinedCancellationToken).ConfigureAwait(false);
                 }
             }
 
@@ -76,15 +79,24 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
             _logger.Started();
         }
 
-        private async Task HandleBackgroundException(BackgroundService backgroundService)
+        private async Task TryStartBackgroundServiceAsync(
+            BackgroundService backgroundService, CancellationToken cancellationToken)
         {
             try
             {
+                // Fire IHostedService.Start
+                await backgroundService.StartAsync(cancellationToken).ConfigureAwait(false);
                 await backgroundService.ExecuteTask.ConfigureAwait(false);
             }
             catch (Exception ex)
             {
                 _logger.BackgroundServiceFaulted(ex);
+                if (_options is { BackgroundServiceExceptionBehavior: BackgroundServiceExceptionBehavior.StopHost })
+                {
+                    _logger.LogCritical(SR.Format(SR.BackgroundServiceExceptionStoppedHost, ex));
+
+                    await StopAsync(CancellationToken.None).ConfigureAwait(false);
+                }
             }
         }
 

src/libraries/Microsoft.Extensions.Hosting/src/Resources/Strings.resx

@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!-- 
-    Microsoft ResX Schema 
+  <!--
+    Microsoft ResX Schema
 
     Version 2.0
 
     The primary goals of this format is to allow a simple XML format
-    that is mostly human readable. The generation and parsing of the 
-    various data types are done through the TypeConverter classes 
+    that is mostly human readable. The generation and parsing of the
+    various data types are done through the TypeConverter classes
     associated with the data types.
 
     Example:
@@ -55,7 +55,7 @@
             : and then encoded with base64 encoding.
 
     mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array 
+    value   : The object must be serialized into a byte array
             : using a System.ComponentModel.TypeConverter
             : and then encoded with base64 encoding.
     -->
@@ -117,6 +117,11 @@
   <resheader name="writer">
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
+   <data name="BackgroundServiceExceptionStoppedHost" xml:space="preserve">
+    <value>The HostOptions.BackgroundServiceExceptionBehavior is configured to StopHost. A BackgroundService has thrown an unhandled exception, and the IHost instance is stopping. To avoid this behavior, configure this to Ignore; however the BackgroundService will not be restarted.
+
+Error: {0}</value>
+  </data>
   <data name="BuildCalled" xml:space="preserve">
     <value>Build can only be called once.</value>
   </data>
@@ -129,4 +134,4 @@
   <data name="ResolverReturnedNull" xml:space="preserve">
     <value>The resolver returned a null IServiceProviderFactory</value>
   </data>
-</root>
+</root>

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs

@@ -145,7 +145,7 @@ public void CreateDefaultBuilder_EnablesValidateOnBuild()
         [ActiveIssue("https://github.com/dotnet/runtime/issues/34580", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
         public async Task CreateDefaultBuilder_ConfigJsonDoesNotReload()
         {
-            var reloadFlagConfig = new Dictionary<string, string>() {{ "hostbuilder:reloadConfigOnChange", "false" }};
+            var reloadFlagConfig = new Dictionary<string, string>() { { "hostbuilder:reloadConfigOnChange", "false" } };
             var appSettingsPath = Path.Combine(Path.GetTempPath(), "appsettings.json");
 
             string SaveRandomConfig()

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Internal/HostTests.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.DotNet.RemoteExecutor;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting.Fakes;
@@ -657,6 +658,120 @@ public async Task WebHostStopAsyncUsesDefaultTimeoutIfNoTokenProvided()
             }
         }
 
+        [Fact]
+        public void HostPropagatesExceptionsThrownWithBackgroundServiceExceptionBehaviorOfStopHost()
+        {
+            RemoteExecutor.Invoke(async () =>
+            {
+                using IHost host = CreateBuilder()
+                    .ConfigureServices(
+                        services =>
+                        {
+                            services.AddHostedService<AsyncThrowingService>();
+                            services.Configure<HostOptions>(
+                              options =>
+                                options.BackgroundServiceExceptionBehavior =
+                                    BackgroundServiceExceptionBehavior.StopHost);
+                        })
+                    .Build();
+
+                await host.StartAsync();
+
+                var lifetime = host.Services.GetRequiredService<IHostApplicationLifetime>();
+                var wasStoppingCalled = false;
+                lifetime.ApplicationStopping.Register(() => wasStoppingCalled = true);
+                var wasStoppedCalled = false;
+                lifetime.ApplicationStopped.Register(() => wasStoppedCalled = true);
+
+                Assert.True(
+                    wasStoppingCalled,
+                    $"The {nameof(AsyncThrowingService)} should have thrown, calling Host.StopAsync, gracefully stopping the host.");
+                Assert.True(
+                    wasStoppedCalled,
+                    $"The {nameof(AsyncThrowingService)} should have thrown, calling Host.StopAsync, gracefully stopping the host.");
+            }).Dispose();
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public void HostPropagatesExceptionsThrownWithBackgroundServiceExceptionBehaviorOfStopHostAfterSometime()
+        {
+            RemoteExecutor.Invoke(async () =>
+            {
+                var backgroundDelayTaskSource = new TaskCompletionSource<bool>();
+
+                using IHost host = CreateBuilder()
+                    .ConfigureServices(
+                        services =>
+                        {
+                            services.AddHostedService(
+                                _ => new AsyncThrowingService(backgroundDelayTaskSource.Task));
+
+                            services.Configure<HostOptions>(
+                              options =>
+                                options.BackgroundServiceExceptionBehavior =
+                                    BackgroundServiceExceptionBehavior.StopHost);
+                        })
+                    .Build();
+
+                var lifetime = host.Services.GetRequiredService<IHostApplicationLifetime>();
+                var wasStoppingCalled = false;
+                lifetime.ApplicationStopping.Register(() => wasStoppingCalled = true);
+                var wasStoppedCalled = false;
+                lifetime.ApplicationStopped.Register(() => wasStoppedCalled = true);
+
+                await Task.WhenAll(host.StartAsync(), DelayThenSignalContinueAsync());
+
+                async Task DelayThenSignalContinueAsync()
+                {
+                    // Emulate the background service doing some work successfully
+                    // for 5 seconds before throwing exception.
+                    await Task.Delay(5000);
+
+                    backgroundDelayTaskSource.SetResult(true);
+                };
+
+                Assert.True(
+                    wasStoppingCalled,
+                    $"The {nameof(AsyncThrowingService)} should have thrown, calling Host.StopAsync, gracefully stopping the host.");
+                Assert.True(
+                    wasStoppedCalled,
+                    $"The {nameof(AsyncThrowingService)} should have thrown, calling Host.StopAsync, gracefully stopping the host.");
+            }).Dispose();
+        }
+
+        [Fact]
+        public void HostHandlesExceptionsThrownWithBackgroundServiceExceptionBehaviorOfIgnore()
+        {
+            var backgroundDelayTaskSource = new TaskCompletionSource<bool>();
+
+            using IHost host = CreateBuilder()
+                .ConfigureServices(
+                    services =>
+                    {
+                        services.AddHostedService(
+                            _ => new AsyncThrowingService(backgroundDelayTaskSource.Task));
+
+                        services.PostConfigure<HostOptions>(
+                          options =>
+                            options.BackgroundServiceExceptionBehavior =
+                                BackgroundServiceExceptionBehavior.Ignore);
+                    })
+                .Build();
+
+            var lifetime = host.Services.GetRequiredService<IHostApplicationLifetime>();
+            var wasStoppingCalled = false;
+            lifetime.ApplicationStopping.Register(() => wasStoppingCalled = true);
+            var wasStoppedCalled = false;
+            lifetime.ApplicationStopped.Register(() => wasStoppedCalled = true);
+
+            host.Start();
+
+            backgroundDelayTaskSource.SetResult(true);
+
+            Assert.False(wasStoppingCalled);
+            Assert.False(wasStoppedCalled);
+        }
+
         [Fact]
         public void HostApplicationLifetimeEventsOrderedCorrectlyDuringShutdown()
         {
@@ -1373,7 +1488,11 @@ public ValueTask DisposeAsync()
 
         private class AsyncThrowingService : BackgroundService
         {
-            private readonly Task _executeDelayTask;
+            private readonly Task? _executeDelayTask;
+
+            internal bool Disposed { get; private set; }
+
+            public AsyncThrowingService() { }
 
             public AsyncThrowingService(Task executeDelayTask)
             {
@@ -1382,10 +1501,18 @@ public AsyncThrowingService(Task executeDelayTask)
 
             protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             {
-                await _executeDelayTask;
+                if (_executeDelayTask is { })
+                    await _executeDelayTask;
 
                 throw new Exception("Background Exception");
             }
+
+            public override void Dispose()
+            {
+                Disposed = true;
+
+                base.Dispose();
+            }
         }
     }
 }

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Microsoft.Extensions.Hosting.Unit.Tests.csproj

@@ -3,6 +3,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent);net461</TargetFrameworks>
     <EnableDefaultItems>true</EnableDefaultItems>
+    <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
   </PropertyGroup>
 
   <ItemGroup>