backoff if we retried polling for more than 50 times in less than 30m… (actions#3232)

* backoff if we retried polling for more than 50 times in less than 30minutes

* run dotnet format

* move delay to after no message trace

* run dotnet format

Author: aiqiaoy

src/Runner.Listener/MessageListener.cs

@@ -236,6 +236,7 @@ public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
             ArgUtil.NotNull(_settings, nameof(_settings));
             bool encounteringError = false;
             int continuousError = 0;
+            int continuousEmptyMessage = 0;
             string errorMessage = string.Empty;
             Stopwatch heartbeat = new();
             heartbeat.Restart();
@@ -359,16 +360,27 @@ public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
 
                 if (message == null)
                 {
+                    continuousEmptyMessage++;
                     if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
                     {
                         Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
                         heartbeat.Restart();
+                        continuousEmptyMessage = 0;
                     }
                     else
                     {
                         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
                     }
 
+                    if (continuousEmptyMessage > 50)
+                    {
+                        // retried more than 50 times in less than 30mins and still getting empty message
+                        // something is not right on the service side, backoff for 15-30s before retry
+                        _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+
                     continue;
                 }