Ensure function abort on shutdown (#2456) (#2477)

Azure · May 16, 2023 · 3a8795d · 3a8795d
1 parent 31e3c92
commit 3a8795d
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 3 deletions.
diff --git a/release_notes.md b/release_notes.md
@@ -4,7 +4,9 @@
 
 ### Bug Fixes
 
-- Fix connection exhaustion and memory leak due to incorrect caching of `DurableTaskClient`'s in dotnet isolated extension.
+- Fix handling of in-flight orchestrations and activities during host shutdown.
+    - Previously these were considered "failed", now they will be retried.
+    - This only affected dotnet-isolated and java workers.
 
 ### Breaking Changes
 

diff --git a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs
@@ -135,8 +135,8 @@ public async Task CallOrchestratorAsync(DispatchMiddlewareContext dispatchContex
                     context.SetResult(
                         response.Actions.Select(ProtobufUtils.ToOrchestratorAction),
                         response.CustomStatus);
-#pragma warning restore CS0618 // Type or member is obsolete (not intended for general public use)
                 },
+#pragma warning restore CS0618 // Type or member is obsolete (not intended for general public use)
             };
 
             FunctionResult functionResult;
@@ -145,6 +145,12 @@ public async Task CallOrchestratorAsync(DispatchMiddlewareContext dispatchContex
                 functionResult = await function.Executor.TryExecuteAsync(
                     input,
                     cancellationToken: this.HostLifetimeService.OnStopping);
+                if (!functionResult.Succeeded)
+                {
+                    // Shutdown can surface as a completed invocation in a failed state.
+                    // Re-throw so we can abort this invocation.
+                    this.HostLifetimeService.OnStopping.ThrowIfCancellationRequested();
+                }
             }
             catch (Exception hostRuntimeException)
             {
@@ -295,6 +301,12 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F
                 result = await function.Executor.TryExecuteAsync(
                     triggerInput,
                     cancellationToken: this.HostLifetimeService.OnStopping);
+                if (!result.Succeeded)
+                {
+                    // Shutdown can surface as a completed invocation in a failed state.
+                    // Re-throw so we can abort this invocation.
+                    this.HostLifetimeService.OnStopping.ThrowIfCancellationRequested();
+                }
             }
             catch (Exception hostRuntimeException)
             {

diff --git a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj
@@ -6,7 +6,7 @@
     <RootNamespace>Microsoft.Azure.WebJobs.Extensions.DurableTask</RootNamespace>
     <MajorVersion>2</MajorVersion>
     <MinorVersion>9</MinorVersion>
-    <PatchVersion>4</PatchVersion>
+    <PatchVersion>5</PatchVersion>
     <Version>$(MajorVersion).$(MinorVersion).$(PatchVersion)</Version>
     <FileVersion>$(MajorVersion).$(MinorVersion).$(PatchVersion)</FileVersion>
     <AssemblyVersion>$(MajorVersion).0.0.0</AssemblyVersion>