Merge pull request #31 from brocaar/retry_on_4xx_status_code

Retry request 5x when the response is in the 4xx range before raising an exception.
spilgames · Jun 27, 2013 · fca8ec2 · fca8ec2
2 parents f459d18 + 3d387ac
commit fca8ec2
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/README.rst b/README.rst
@@ -126,6 +126,9 @@ v2.1.1
 
 * Make sure a shebang does exist on scripts to be run. Use shlex to make
   Popen safer.
+* Retry request 5x when the response is in the 4xx range before raising an
+  exception.
+
 
 v2.1.0
 ~~~~~~

diff --git a/job_runner_worker/models.py b/job_runner_worker/models.py
@@ -41,14 +41,27 @@ def inner_func(*args, **kwargs):
                 return func(*args, **kwargs)
             except (RequestException, RequestServerError):
                 logger.exception(
-                    'Exception raised while calling {0}'.format(
-                        func.__name__))
+                    'Exception raised while calling {0} '
+                    'with arguments {1} '.format(
+                        func.__name__, kwargs))
                 if attempt <= 10:
                     time.sleep(2)
                 elif attempt <= 50:
                     time.sleep(5)
                 else:
                     time.sleep(10)
+            except RequestClientError:
+                # RequestClientError should theoretically not be recoverable
+                # but we'll try max 5 times anyways.
+
+                if attempt >= 5:
+                    raise
+
+                logger.exception(
+                    'Exception raised while calling {0} '
+                    'with arguments {1} '.format(
+                        func.__name__, kwargs))
+                time.sleep(attempt * 10)
 
     return inner_func
 

diff --git a/job_runner_worker/tests/unit/test_models.py b/job_runner_worker/tests/unit/test_models.py
@@ -94,6 +94,7 @@ def config_get_side_effect(*args):
             verify=False,
         )
 
+    @patch('job_runner_worker.models.time', Mock())
     @patch('job_runner_worker.models.HmacAuth')
     @patch('job_runner_worker.models.config')
     @patch('job_runner_worker.models.requests')
@@ -115,6 +116,7 @@ def config_get_side_effect(*args):
         base_model = BaseRestModel('/path/to/resource')
         self.assertRaises(RequestClientError, base_model.patch, {'foo': 'bar'})
 
+    @patch('job_runner_worker.models.time', Mock())
     @patch('job_runner_worker.models.HmacAuth')
     @patch('job_runner_worker.models.config')
     @patch('job_runner_worker.models.requests')
@@ -167,6 +169,7 @@ def config_get_side_effect(*args):
 
         HmacAuth.assert_called_once_with('public', 'key')
 
+    @patch('job_runner_worker.models.time', Mock())
     @patch('job_runner_worker.models.HmacAuth')
     @patch('job_runner_worker.models.config')
     @patch('job_runner_worker.models.requests')
@@ -221,6 +224,7 @@ def config_get_side_effect(*args):
         self.assertEqual({'id': 1, 'resource_uri': 'foo'}, out[0]._data)
         self.assertEqual({'id': 2, 'resource_uri': 'bar'}, out[1]._data)
 
+    @patch('job_runner_worker.models.time', Mock())
     @patch('job_runner_worker.models.HmacAuth')
     @patch('job_runner_worker.models.config')
     @patch('job_runner_worker.models.requests')