Retry http_request and json_request failure.

Temporary network outages or keystone server restarts can lead
to a connection refused in auth_token middleware. Rather than
failing immediately, this patch attempts to retry a few times.

Fixes bug 1150299

(pulled from python-keystoneclient commit
9bb29b9dff4d220eaabf86cd286e651c5ae15b98)

Change-Id: I2ecf0d7745290976efcb3e3cd6511817a53d3e0a

keystone/middleware/auth_token.py

@@ -383,26 +383,36 @@ def _get_http_connection(self):
                                           self.key_file,
                                           self.cert_file)
 
-    def _http_request(self, method, path):
+    def _http_request(self, method, path, **kwargs):
         """HTTP request helper used to make unspecified content type requests.
 
         :param method: http method
         :param path: relative request url
-        :return (http response object)
+        :return (http response object, response body)
         :raise ServerError when unable to communicate with keystone
 
         """
         conn = self._get_http_connection()
 
-        try:
-            conn.request(method, path)
-            response = conn.getresponse()
-            body = response.read()
-        except Exception, e:
-            LOG.error('HTTP connection exception: %s' % e)
-            raise ServiceError('Unable to communicate with keystone')
-        finally:
-            conn.close()
+        RETRIES = 3
+        retry = 0
+
+        while True:
+            try:
+                conn.request(method, path, **kwargs)
+                response = conn.getresponse()
+                body = response.read()
+                break
+            except Exception as e:
+                if retry == RETRIES:
+                    LOG.error('HTTP connection exception: %s' % e)
+                    raise ServiceError('Unable to communicate with keystone')
+                # NOTE(vish): sleep 0.5, 1, 2
+                LOG.warn('Retrying on HTTP connection exception: %s' % e)
+                time.sleep(2.0 ** retry / 2)
+                retry += 1
+            finally:
+                conn.close()
 
         return response, body
 
@@ -418,8 +428,6 @@ def _json_request(self, method, path, body=None, additional_headers=None):
         :raise ServerError when unable to communicate with keystone
 
         """
-        conn = self._get_http_connection()
-
         kwargs = {
             'headers': {
                 'Content-type': 'application/json',
@@ -433,16 +441,9 @@ def _json_request(self, method, path, body=None, additional_headers=None):
         if body:
             kwargs['body'] = jsonutils.dumps(body)
 
-        full_path = self.auth_admin_prefix + path
-        try:
-            conn.request(method, full_path, **kwargs)
-            response = conn.getresponse()
-            body = response.read()
-        except Exception, e:
-            LOG.error('HTTP connection exception: %s' % e)
-            raise ServiceError('Unable to communicate with keystone')
-        finally:
-            conn.close()
+        path = self.auth_admin_prefix + path
+
+        response, body = self._http_request(method, path, **kwargs)
 
         try:
             data = jsonutils.loads(body)