fix: Temporary patch for PubSub Subscriber deadlock issue.

googleapis · May 16, 2023 · d965b93 · d965b93
1 parent 0ec8cc2
commit d965b93
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/apis/Google.Cloud.PubSub.V1/Google.Cloud.PubSub.V1.Tests/SubscriberClientTest.cs b/apis/Google.Cloud.PubSub.V1/Google.Cloud.PubSub.V1.Tests/SubscriberClientTest.cs
@@ -547,7 +547,7 @@ private static RpcException GetExactlyOnceDeliveryMixedException(Rpc.ErrorInfo e
                     Assert.Empty(fake.Subscribers[0].Nacks);
                     Assert.Empty(fake.Subscribers[0].Extends);
                     Assert.Equal(new[] { fake.Time0 + TimeSpan.FromSeconds(1) }, fake.Subscribers[0].WriteCompletes);
-                    Assert.Equal(new[] { fake.Time0 + TimeSpan.FromSeconds(1) }, fake.ClientShutdowns);
+                    Assert.Equal(new[] { fake.Time0 + TimeSpan.FromSeconds(3) }, fake.ClientShutdowns);
                 });
             }
         }
@@ -581,7 +581,7 @@ public void Dispose()
                     Assert.Empty(fake.Subscribers[0].Nacks);
                     Assert.Empty(fake.Subscribers[0].Extends);
                     Assert.Equal(new[] { fake.Time0 + TimeSpan.FromSeconds(1) }, fake.Subscribers[0].WriteCompletes);
-                    Assert.Equal(new[] { fake.Time0 + TimeSpan.FromSeconds(1) }, fake.ClientShutdowns);
+                    Assert.Equal(new[] { fake.Time0 + TimeSpan.FromSeconds(3) }, fake.ClientShutdowns);
                 });
             }
         }
@@ -903,7 +903,7 @@ public void LeaseMaxExtension()
                     var doneTask = fake.Subscriber.StartAsync(async (msg, ct) =>
                     {
                         // Emulate a hanging message-processing task.
-                        await fake.TaskHelper.ConfigureAwait(fake.Scheduler.Delay(TimeSpan.FromHours(24), ct));
+                        await fake.TaskHelper.ConfigureAwait(fake.Scheduler.Delay(TimeSpan.FromHours(23), ct));
                         return SubscriberClient.Reply.Ack;
                     });
                     await fake.TaskHelper.ConfigureAwait(fake.Scheduler.Delay(TimeSpan.FromHours(12), CancellationToken.None));

diff --git a/apis/Google.Cloud.PubSub.V1/Google.Cloud.PubSub.V1/SubscriberClientImpl.cs b/apis/Google.Cloud.PubSub.V1/Google.Cloud.PubSub.V1/SubscriberClientImpl.cs
@@ -157,6 +157,10 @@ private async Task StopCompletionAsync(Task[] subscriberTasks, HashSet<Task> reg
         // Call shutdown function
         if (_shutdown != null)
         {
+            // TODO: Remove this 2 second delay.
+            // This is a temporary patch to avoid race condition between gRPC call cancellation and channel dispose.
+            // Please see https://github.com/grpc/grpc-dotnet/issues/2119 for the deadlock issue in Grpc.Net.Client.
+            await _scheduler.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
             await _taskHelper.ConfigureAwaitHideErrors(_shutdown);
         }
         // Return final result