Implement backoff

sumologic/backoff.py

@@ -0,0 +1,30 @@
+import requests
+import logging
+import time
+
+logger = logging.getLogger('sumologic.backoff')
+
+MAX_TRIES = 8
+
+def backoff(func):
+    def limited(*args, **kwargs):
+        delay = 0.1
+        tries = 0
+        lastException = None
+        while tries < MAX_TRIES:
+            tries += 1
+            try:
+                return func(*args, **kwargs)
+            except requests.exceptions.HTTPError as e:
+                if e.response.status_code == 429: # rate limited
+                    logging.debug("Rate limited, sleeping for {0}s".format(delay))
+                    time.sleep(delay)
+                    delay *= 2
+                    lastException = e
+                    continue
+                else:
+                    raise
+        logging.debug("Rate limited function still failed after {0} retries.".format(MAX_TRIES))
+        raise lastException
+
+    return limited

sumologic/sumologic.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import requests
+from .backoff import backoff
 
 try:
     import cookielib
@@ -20,7 +21,7 @@ def __init__(self, accessId, accessKey, endpoint=None, cookieFile='cookies.txt')
             self.endpoint = self._get_endpoint()
         else:
             self.endpoint = endpoint
-        if endpoint[-1:] == "/":
+        if self.endpoint[-1:] == "/":
           raise Exception("Endpoint should not end with a slash character")
 
     def _get_endpoint(self):
@@ -41,32 +42,36 @@ def _get_endpoint(self):
         endpoint = self.response.url.replace('/collectors', '')  # dirty hack to sanitise URI and retain domain
         return endpoint
 
+    @backoff
     def delete(self, method, params=None):
         r = self.session.delete(self.endpoint + method, params=params)
         if 400 <= r.status_code < 600:
             r.reason = r.text
         r.raise_for_status()
         return r
 
+    @backoff
     def get(self, method, params=None):
         r = self.session.get(self.endpoint + method, params=params)
         if 400 <= r.status_code < 600:
             r.reason = r.text
         r.raise_for_status()
         return r
 
+    @backoff
     def post(self, method, params, headers=None):
         r = self.session.post(self.endpoint + method, data=json.dumps(params), headers=headers)
         if 400 <= r.status_code < 600:
             r.reason = r.text
         r.raise_for_status()
         return r
 
+    @backoff
     def put(self, method, params, headers=None):
-        r = self.session.put(self.endpoint + method, data=json.dumps(params), headers=headers) 
+        r = self.session.put(self.endpoint + method, data=json.dumps(params), headers=headers)
         if 400 <= r.status_code < 600:
             r.reason = r.text
-        r.raise_for_status() 
+        r.raise_for_status()
         return r
 
     def search(self, query, fromTime=None, toTime=None, timeZone='UTC'):