Retry requests on a specific list of HTTP status codes

usp/helpers.py

@@ -95,7 +95,7 @@ def get_url_retry_on_client_errors(url: str,
         else:
             log.warning("Request for URL {} failed: {}".format(url, response.status_message()))
 
-            if response.encountered_client_error():
+            if response.is_retryable_error():
                 log.info("Retrying URL {} in {} seconds...".format(url, sleep_between_retries))
                 time.sleep(sleep_between_retries)
 

usp/web_client/abstract_client.py

@@ -1,12 +1,75 @@
 """Abstract web client class."""
 
 import abc
+from http import HTTPStatus
 from typing import Optional
 
 
 class AbstractWebClientResponse(object, metaclass=abc.ABCMeta):
     """Abstract web client response."""
 
+    _RETRYABLE_HTTP_STATUS_CODES = {
+
+        # Some servers return "400 Bad Request" initially but upon retry start working again, no idea why
+        HTTPStatus.BAD_REQUEST.value,
+
+        # If we timed out requesting stuff, we can just try again
+        HTTPStatus.REQUEST_TIMEOUT.value,
+
+        # If we got rate limited, it makes sense to wait a bit
+        HTTPStatus.TOO_MANY_REQUESTS.value,
+
+        # Server might be just fine on a subsequent attempt
+        HTTPStatus.INTERNAL_SERVER_ERROR.value,
+
+        # Upstream might reappear on a retry
+        HTTPStatus.BAD_GATEWAY.value,
+
+        # Service might become available again on a retry
+        HTTPStatus.SERVICE_UNAVAILABLE.value,
+
+        # Upstream might reappear on a retry
+        HTTPStatus.GATEWAY_TIMEOUT.value,
+
+        # (unofficial) 509 Bandwidth Limit Exceeded (Apache Web Server/cPanel)
+        509,
+
+        # (unofficial) 598 Network read timeout error
+        598,
+
+        # (unofficial, nginx) 499 Client Closed Request
+        499,
+
+        # (unofficial, Cloudflare) 520 Unknown Error
+        520,
+
+        # (unofficial, Cloudflare) 521 Web Server Is Down
+        521,
+
+        # (unofficial, Cloudflare) 522 Connection Timed Out
+        522,
+
+        # (unofficial, Cloudflare) 523 Origin Is Unreachable
+        523,
+
+        # (unofficial, Cloudflare) 524 A Timeout Occurred
+        524,
+
+        # (unofficial, Cloudflare) 525 SSL Handshake Failed
+        525,
+
+        # (unofficial, Cloudflare) 526 Invalid SSL Certificate
+        526,
+
+        # (unofficial, Cloudflare) 527 Railgun Error
+        527,
+
+        # (unofficial, Cloudflare) 530 Origin DNS Error
+        530,
+
+    }
+    """HTTP status codes on which a request should be retried."""
+
     @abc.abstractmethod
     def status_code(self) -> int:
         """Return HTTP status code of the response."""
@@ -31,9 +94,9 @@ def is_success(self) -> bool:
         """Return True if the request succeeded."""
         return 200 <= self.status_code() < 300
 
-    def encountered_client_error(self) -> bool:
-        """Return True if encountered HTTP client error."""
-        return 400 <= self.status_code() < 500
+    def is_retryable_error(self) -> bool:
+        """Return True if encountered HTTP error which should be retried."""
+        return self.status_code() in self._RETRYABLE_HTTP_STATUS_CODES
 
 
 class AbstractWebClient(object, metaclass=abc.ABCMeta):