Permalink
Browse files

Add --max-retries and --retry-delay

  • Loading branch information...
1 parent b3debba commit 88869abb686dea8deea56d7f66cd06abde444ba7 @cyno cyno committed Oct 31, 2011
Showing with 17 additions and 14 deletions.
  1. +2 −0 S3/Config.py
  2. +12 −9 S3/S3.py
  3. +3 −1 s3cmd
  4. +0 −1 testsuite/etc/brokenlink.png
  5. +0 −1 testsuite/etc/linked.png
  6. +0 −1 testsuite/etc/linked1.png
  7. +0 −1 testsuite/etc/more/linked-dir
View
@@ -78,6 +78,8 @@ class Config(object):
workers = 10
follow_symlinks=False
select_dir = False
+ max_retries = 5
+ retry_delay = 3
## Creating a singleton
def __new__(self, configfile = None):
View
@@ -47,7 +47,7 @@ def update_timestamp(self):
def format_param_str(self):
"""
Format URL parameters from self.params and returns
- ?parm1=val1&parm2=val2 or an empty string if there
+ ?parm1=val1&parm2=val2 or an empty string if there
are no parameters. Output of this function should
be appended directly to self.resource['uri']
"""
@@ -119,9 +119,6 @@ class S3(object):
## S3 sometimes sends HTTP-307 response
redir_map = {}
- ## Maximum attempts of re-issuing failed requests
- _max_retries = 5
-
def __init__(self, config):
self.config = config
@@ -458,9 +455,11 @@ def create_request(self, operation, uri = None, bucket = None, object = None, he
def _fail_wait(self, retries):
# Wait a few seconds. The more it fails the more we wait.
- return (self._max_retries - retries + 1) * 3
+ return (self.config.max_retries - retries + 1) * self.config.retry_delay
- def send_request(self, request, body = None, retries = _max_retries):
+ def send_request(self, request, body = None, retries = -1):
+ if retries == -1:
+ retries = self.config.max_retries
method_string, resource, headers = request.get_triplet()
debug("Processing request, please wait...")
if not headers.has_key('content-length'):
@@ -509,7 +508,9 @@ def send_request(self, request, body = None, retries = _max_retries):
return response
- def send_file(self, request, file, labels, throttle = 0, retries = _max_retries):
+ def send_file(self, request, file, labels, throttle = 0, retries = -1):
+ if retries == -1:
+ retries = self.config.max_retries
method_string, resource, headers = request.get_triplet()
size_left = size_total = headers.get("content-length")
if self.config.progress_meter:
@@ -562,7 +563,7 @@ def send_file(self, request, file, labels, throttle = 0, retries = _max_retries)
if self.config.progress_meter:
progress.done("failed")
if retries:
- if retries < self._max_retries:
+ if retries < self.config.max_retries:
throttle = throttle and throttle * 5 or 0.01
warning("Upload failed: %s (%s)" % (resource['uri'], e))
warning("Retrying on lower speed (throttle=%0.2f)" % throttle)
@@ -634,7 +635,9 @@ def send_file(self, request, file, labels, throttle = 0, retries = _max_retries)
return response
- def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):
+ def recv_file(self, request, stream, labels, start_position = 0, retries = -1):
+ if retries == -1:
+ retries = self.config.max_retries
method_string, resource, headers = request.get_triplet()
if self.config.progress_meter:
progress = self.config.progress_class(labels, 0)
View
@@ -1970,7 +1970,9 @@ def main():
optparser.add_option( "--workers", dest="workers", default=10, help="Sets the number of workers to run for uploading and downloading files (can only be used in conjunction with the --parallel argument)")
optparser.add_option( "--directory", dest="select_dir", action="store_true", default=False, help="Select directories (only for [ls]).")
-
+ optparser.add_option( "--max-retries", dest="max_retries", type="int", action="store", default=5, help="Number of retry before failing GET or PUT.")
+ optparser.add_option( "--retry-delay", dest="retry_delay", type="int", action="store", default=3, help="Time delay to wait after failing GET or PUT.")
+
optparser.set_usage(optparser.usage + " COMMAND [parameters]")
optparser.set_description('S3cmd is a tool for managing objects in '+
'Amazon S3 storage. It allows for making and removing '+

0 comments on commit 88869ab

Please sign in to comment.