Enable multipart for [sync] - do not check MD5

Multipart-uploaded files don't have a valid MD5 sum in their ETag.
We can detect it and disable MD5 comparison when deciding whether
to sync these files. In such a case only the size (and later on a
timestamp) is compared.

NEWS

@@ -1,5 +1,7 @@
 s3cmd 1.1.0   -   ???
 ===========
+* MultiPart upload enabled for both [put] and [sync]. Default chunk
+  size is 15MB.
 * CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo].
 * Increased socket_timeout from 10 secs to 5 mins.
 * Added "Static WebSite" support [ws-create / ws-delete / ws-info]

S3/FileLists.py

@@ -301,8 +301,13 @@ def __direction_str(is_remote):
                 debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
                 attribs_match = False
 
-            if attribs_match and 'md5' in cfg.sync_checks:
-                ## ... same size, check MD5
+            ## Check MD5
+            compare_md5 = 'md5' in cfg.sync_checks
+            # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN"
+            if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
+                compare_md5 = False
+                info(u"Disabled MD5 check for %s" % file)
+            if attribs_match and compare_md5:
                 try:
                     if src_remote == False and dst_remote == True:
                         src_md5 = hash_file_md5(src_list[file]['full_name'])

S3/S3.py

@@ -756,6 +756,7 @@ def send_file_multipart(self, file, headers, uri, size):
         upload.upload_all_parts()
         response = upload.complete_multipart_upload()
         response["speed"] = 0 # XXX
+        response["size"] = size
         return response
 
     def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):

s3cmd

@@ -837,10 +837,6 @@ def cmd_sync_local2remote(args):
 
     s3 = S3(cfg)
 
-    ## FIXME
-    cfg.multipart_enabled = False
-    warning(u"MultiPart: disabled for 'sync' command. Don't panic, we'll fix it!")
-
     if cfg.encrypt:
         error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
         error(u"Either use unconditional 's3cmd put --recursive'")