Permalink
Browse files

Enable multipart for [sync] - do not check MD5

Multipart-uploaded files don't have a valid MD5 sum in their ETag.
We can detect it and disable MD5 comparison when deciding whether
to sync these files. In such a case only the size (and later on a
timestamp) is compared.
  • Loading branch information...
1 parent 74e7854 commit 5ad69b307cab0ab06be2c00d466208fd4d156273 @mludvig mludvig committed Jan 12, 2012
Showing with 10 additions and 6 deletions.
  1. +2 −0 NEWS
  2. +7 −2 S3/FileLists.py
  3. +1 −0 S3/S3.py
  4. +0 −4 s3cmd
View
2 NEWS
@@ -1,5 +1,7 @@
s3cmd 1.1.0 - ???
===========
+* MultiPart upload enabled for both [put] and [sync]. Default chunk
+ size is 15MB.
* CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo].
* Increased socket_timeout from 10 secs to 5 mins.
* Added "Static WebSite" support [ws-create / ws-delete / ws-info]
View
@@ -301,8 +301,13 @@ def __direction_str(is_remote):
debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
attribs_match = False
- if attribs_match and 'md5' in cfg.sync_checks:
- ## ... same size, check MD5
+ ## Check MD5
+ compare_md5 = 'md5' in cfg.sync_checks
+ # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN"
+ if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
+ compare_md5 = False
+ info(u"Disabled MD5 check for %s" % file)
+ if attribs_match and compare_md5:
try:
if src_remote == False and dst_remote == True:
src_md5 = hash_file_md5(src_list[file]['full_name'])
View
@@ -756,6 +756,7 @@ def send_file_multipart(self, file, headers, uri, size):
upload.upload_all_parts()
response = upload.complete_multipart_upload()
response["speed"] = 0 # XXX
+ response["size"] = size
return response
def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):
View
4 s3cmd
@@ -837,10 +837,6 @@ def cmd_sync_local2remote(args):
s3 = S3(cfg)
- ## FIXME
- cfg.multipart_enabled = False
- warning(u"MultiPart: disabled for 'sync' command. Don't panic, we'll fix it!")
-
if cfg.encrypt:
error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
error(u"Either use unconditional 's3cmd put --recursive'")

0 comments on commit 5ad69b3

Please sign in to comment.