Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge pull request #3 from jblomo/master

multipart upload and fix args
  • Loading branch information...
commit a3547524c3fadf22131a632d49676e624347dc78 2 parents efeade5 + 6589e03
authored October 11, 2011

Showing 1 changed file with 8 additions and 3 deletions. Show diff stats Hide diff stats

  1. 11  s3mysqldump.py
11  s3mysqldump.py
@@ -26,6 +26,7 @@
26 26
 import pipes
27 27
 import re
28 28
 import shlex
  29
+import shutil
29 30
 import subprocess
30 31
 import sys
31 32
 import tempfile
@@ -105,6 +106,7 @@ def mysqldump_to_s3(s3_uri, databases=None, tables=None):
105 106
                 if os.path.getsize(file.name) > S3_MAX_PUT_SIZE:
106 107
                     upload_multipart(s3_key, file.name)
107 108
                 else:
  109
+                    log.debug('Upload to %r' % s3_key)
108 110
                     s3_key.set_contents_from_file(file)
109 111
 
110 112
                 log.debug('  Done in %.1fs' % (time.time() - start))
@@ -272,17 +274,20 @@ def upload_multipart(s3_key, large_file):
272 274
     split_dir = tempfile.mkdtemp(prefix='s3mysqldump-split-')
273 275
     split_prefix = "%s/part-" % split_dir
274 276
 
275  
-    args = ['split', "--bytes=%u" % S3_MAX_PUT_SIZE, large_file, split_prefix]
  277
+    args = ['split', "--line-bytes=%u" % S3_MAX_PUT_SIZE, '--suffix-length=4', large_file, split_prefix]
276 278
     log.debug(' '.join(pipes.quote(arg) for arg in args))
277 279
     subprocess.check_call(args)
278 280
 
279 281
     mp = s3_key.bucket.initiate_multipart_upload(s3_key.name)
280  
-    for part, filename in enumerate(glob.glob(split_prefix + '*')):
  282
+    log.debug('Multipart upload to %r' % s3_key)
  283
+    for part, filename in enumerate(sorted(glob.glob(split_prefix + '*'))):
281 284
         with open(filename, 'rb') as file:
282 285
             mp.upload_part_from_file(file, part+1) # counting starts at 1
283 286
 
284 287
     mp.complete_upload()
285 288
 
  289
+    shutil.rmtree(split_dir, True)
  290
+
286 291
 
287 292
 def make_option_parser():
288 293
     usage = '%prog [options] db_name [tbl_name ...] s3_uri_format'
@@ -472,4 +477,4 @@ def mysqldump_to_file(file, databases=None, tables=None, mysqldump_bin=None, my_
472 477
 
473 478
 
474 479
 if __name__ == '__main__':
475  
-    main(sys.argv[:1])
  480
+    main(sys.argv[1:])

0 notes on commit a354752

Please sign in to comment.
Something went wrong with that request. Please try again.