Azure retry hardening

tools/c7n_azure/c7n_azure/constants.py

@@ -86,6 +86,7 @@
 Custom Retry Code Variables
 """
 DEFAULT_MAX_RETRY_AFTER = 30
+DEFAULT_RETRY_AFTER = 5
 
 """
 KeyVault url templates

tools/c7n_azure/c7n_azure/utils.py

@@ -6,6 +6,7 @@
 import hashlib
 import itertools
 import logging
+import random
 import re
 import time
 import uuid
@@ -140,13 +141,17 @@ def custodian_azure_send_override(self, request, headers=None, content=None, **k
                 send_logger.debug(k + ':' + v)
 
         # Retry codes from urllib3/util/retry.py
-        if response.status_code in [413, 429, 503]:
+        if response.status_code in [429, 503]:
             retry_after = None
             for k in response.headers.keys():
                 if StringUtils.equal('retry-after', k):
                     retry_after = int(response.headers[k])
-
-            if retry_after is not None and retry_after < constants.DEFAULT_MAX_RETRY_AFTER:
+            if retry_after is None:
+                # we want to attempt retries even when azure fails to send a header
+                # this has been a constant source of instability in larger environments
+                retry_after = (constants.DEFAULT_RETRY_AFTER * retries) + \
+                    random.randint(1, constants.DEFAULT_RETRY_AFTER)
+            if retry_after < constants.DEFAULT_MAX_RETRY_AFTER:
                 send_logger.warning('Received retriable error code %i. Retry-After: %i'
                                     % (response.status_code, retry_after))
                 time.sleep(retry_after)

tools/c7n_azure/tests_azure/test_azure_utils.py

@@ -238,6 +238,23 @@ def test_custodian_azure_send_override_429_long_retry(self, logger):
         self.assertEqual(mock.orig_send.call_count, 1)
         self.assertEqual(logger.call_count, 1)
 
+    @patch('c7n_azure.utils.send_logger.debug')
+    @patch('c7n_azure.utils.send_logger.warning')
+    def test_custodian_azure_send_override_429_missingheader(self, logger_debug, logger_warning):
+        mock = Mock()
+        mock.send = types.MethodType(custodian_azure_send_override, mock)
+
+        response_dict = {
+            'headers': {},
+            'status_code': 429
+        }
+        mock.orig_send.return_value = type(str('response'), (), response_dict)
+        mock.send('')
+
+        self.assertEqual(mock.orig_send.call_count, 3)
+        self.assertEqual(logger_debug.call_count, 3)
+        self.assertEqual(logger_warning.call_count, 3)
+
     managed_group_return_value = Bag({
         'properties': {
             'name': 'dev',