Handle HTTP 429, rate limiting

pydiscourse/client.py

@@ -6,8 +6,11 @@
 
 import requests
 
+import time
+
 from pydiscourse.exceptions import (
-    DiscourseError, DiscourseServerError, DiscourseClientError)
+    DiscourseError, DiscourseServerError, DiscourseClientError,
+    DiscourseRateLimitedError)
 from pydiscourse.sso import sso_payload
 
 
@@ -1201,24 +1204,47 @@ def _request(self, verb, path, params={}, files={}, data={}, json={}):
         url = self.host + path
 
         headers = {'Accept': 'application/json; charset=utf-8'}
-        response = requests.request(
-            verb, url, allow_redirects=False, params=params, files=files, data=data, json=json, headers=headers,
-            timeout=self.timeout)
-
-        log.debug('response %s: %s', response.status_code, repr(response.text))
-        if not response.ok:
-            try:
-                msg = u','.join(response.json()['errors'])
-            except (ValueError, TypeError, KeyError):
-                if response.reason:
-                    msg = response.reason
-                else:
-                    msg = u'{0}: {1}'.format(response.status_code, response.text)
-
-            if 400 <= response.status_code < 500:
-                raise DiscourseClientError(msg, response=response)
-
-            raise DiscourseServerError(msg, response=response)
+
+        # How many times should we retry if rate limited
+        retry_count = 4
+        # Extra time (on top of that required by API) to wait on a retry.
+        retry_backoff = 1
+
+        while retry_count > 0:
+            response = requests.request(
+                verb, url, allow_redirects=False, params=params, files=files, data=data, json=json, headers=headers,
+                timeout=self.timeout)
+
+            log.debug('response %s: %s', response.status_code, repr(response.text))
+            if not response.ok:
+                try:
+                    msg = u','.join(response.json()['errors'])
+                except (ValueError, TypeError, KeyError):
+                    if response.reason:
+                        msg = response.reason
+                    else:
+                        msg = u'{0}: {1}'.format(response.status_code, response.text)
+
+                if 400 <= response.status_code < 500:
+                    if 429 == response.status_code:
+                        # This codepath relies on wait_seconds from Discourse v2.0.0.beta3 / v1.9.3 or higher.
+                        rj = response.json()
+                        wait_delay = retry_backoff + rj['extras']['wait_seconds']        # how long to back off for.
+
+                        if retry_count > 1:
+                            time.sleep(wait_delay)
+                        retry_count -= 1
+                        log.info('We have been rate limited and waited {0} seconds ({1} retries left)'.format(wait_delay, retry_count))
+                        log.debug('API returned {0}'.format(rj))
+                        continue
+                    else:
+                        raise DiscourseClientError(msg, response=response)
+
+                # Any other response.ok resulting in False
+                raise DiscourseServerError(msg, response=response)
+
+        if retry_count == 0:
+            raise DiscourseRateLimitedError("Number of rate limit retries exceeded. Increase retry_backoff or retry_count", response=response)
 
         if response.status_code == 302:
             raise DiscourseError(

pydiscourse/exceptions.py

@@ -11,3 +11,8 @@ class DiscourseServerError(DiscourseError):
 
 class DiscourseClientError(DiscourseError):
     """ An invalid request has been made """
+
+
+class DiscourseRateLimitedError(DiscourseError):
+        """ Request required more than the permissible number of retries """
+