Add optional retries to connection attempts

splunklib/binding.py

@@ -31,6 +31,7 @@
 import socket
 import ssl
 import sys
+import time
 from base64 import b64encode
 from contextlib import contextmanager
 from datetime import datetime
@@ -452,6 +453,11 @@ class Context(object):
     :type password: ``string``
     :param headers: List of extra HTTP headers to send (optional).
     :type headers: ``list`` of 2-tuples.
+    :param retires: Number of retries for each HTTP connection (optional, the default is 0).
+                    NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER.
+    :type retries: ``int``
+    :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s).
+    :type retryBackoff: ``int`` (in seconds)
     :param handler: The HTTP request handler (optional).
     :returns: A ``Context`` instance.
 
@@ -469,7 +475,8 @@ class Context(object):
     """
     def __init__(self, handler=None, **kwargs):
         self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"),
-                            cert_file=kwargs.get("cert_file"))  # Default to False for backward compat
+                            cert_file=kwargs.get("cert_file"),  # Default to False for backward compat
+                            retries=kwargs.get("retries", 0), retryBackoff=kwargs.get("retryBackoff", 10))
         self.token = kwargs.get("token", _NoAuthenticationToken)
         if self.token is None: # In case someone explicitly passes token=None
             self.token = _NoAuthenticationToken
@@ -1122,12 +1129,14 @@ class HttpLib(object):
 
     If using the default handler, SSL verification can be disabled by passing verify=False.
     """
-    def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None):
+    def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, retries=0, retryBackoff=10):
         if custom_handler is None:
             self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file)
         else:
             self.handler = custom_handler
         self._cookies = {}
+        self.retries = retries
+        self.retryBackoff = retryBackoff
 
     def delete(self, url, headers=None, **kwargs):
         """Sends a DELETE request to a URL.
@@ -1239,7 +1248,16 @@ def request(self, url, message, **kwargs):
             its structure).
         :rtype: ``dict``
         """
-        response = self.handler(url, message, **kwargs)
+        while True:
+            try:
+                response = self.handler(url, message, **kwargs)
+                break
+            except Exception:
+                if self.retries <= 0:
+                    raise
+                else:
+                    time.sleep(self.retryBackoff)
+                    self.retries -= 1
         response = record(response)
         if 400 <= response.status:
             raise HTTPError(response)

splunklib/client.py

@@ -318,6 +318,11 @@ def connect(**kwargs):
     :type username: ``string``
     :param `password`: The password for the Splunk account.
     :type password: ``string``
+    :param retires: Number of retries for each HTTP connection (optional, the default is 0).
+                    NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER.
+    :type retries: ``int``
+    :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s).
+    :type retryBackoff: ``int`` (in seconds)
     :return: An initialized :class:`Service` connection.
 
     **Example**::
@@ -384,6 +389,11 @@ class Service(_BaseService):
     :param `password`: The password, which is used to authenticate the Splunk
                        instance.
     :type password: ``string``
+    :param retires: Number of retries for each HTTP connection (optional, the default is 0).
+                    NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER.
+    :type retries: ``int``
+    :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s).
+    :type retryBackoff: ``int`` (in seconds)
     :return: A :class:`Service` instance.
 
     **Example**::