backoff if we retried polling for more than 50 times in less than 30m… (

#3232) * backoff if we retried polling for more than 50 times in less than 30minutes * run dotnet format * move delay to after no message trace * run dotnet format
actions · Apr 9, 2024 · 4a1e380 · 4a1e380
1 parent f467e9e
commit 4a1e380
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs
@@ -236,6 +236,7 @@ public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
             ArgUtil.NotNull(_settings, nameof(_settings));
             bool encounteringError = false;
             int continuousError = 0;
+            int continuousEmptyMessage = 0;
             string errorMessage = string.Empty;
             Stopwatch heartbeat = new();
             heartbeat.Restart();
@@ -359,16 +360,27 @@ public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
 
                 if (message == null)
                 {
+                    continuousEmptyMessage++;
                     if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
                     {
                         Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
                         heartbeat.Restart();
+                        continuousEmptyMessage = 0;
                     }
                     else
                     {
                         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
                     }
 
+                    if (continuousEmptyMessage > 50)
+                    {
+                        // retried more than 50 times in less than 30mins and still getting empty message
+                        // something is not right on the service side, backoff for 15-30s before retry
+                        _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+
                     continue;
                 }