Merge pull request #75 from jmunhoz/test-multipart-cp

Test multipart copy part feature

Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>

s3tests/functional/test_s3.py

@@ -4579,6 +4579,14 @@ def transfer_part(bucket, mp_id, mp_keyname, i, part):
     part_out = StringIO(part)
     mp.upload_part_from_file(part_out, i+1)
 
+def copy_part(src_bucket, src_keyname, dst_bucket, dst_keyname, mp_id, i, start=None, end=None):
+    """Copy a part of a multipart upload from other bucket.
+    """
+    mp = boto.s3.multipart.MultiPartUpload(dst_bucket)
+    mp.key_name = dst_keyname
+    mp.id = mp_id
+    mp.copy_part_from_key(src_bucket, src_keyname, i+1, start, end)
+
 def generate_random(size, part_size=5*1024*1024):
     """
     Generate the specified number random data.
@@ -4619,6 +4627,29 @@ def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list=
 
     return (upload, s)
 
+def _multipart_copy(src_bucketname, src_keyname, dst_bucket, dst_keyname, size, part_size=5*1024*1024, do_list=None, headers=None, metadata=None, resend_parts=[]):
+    upload = dst_bucket.initiate_multipart_upload(dst_keyname, headers=headers, metadata=metadata)
+    i = 0
+    for start_offset in range(0, size, part_size):
+        end_offset = min(start_offset + part_size - 1, size - 1)
+        copy_part(src_bucketname, src_keyname, dst_bucket, dst_keyname, upload.id, i, start_offset, end_offset)
+        if i in resend_parts:
+            copy_part(src_bucketname, src_keyname, dst_bucket, dst_name, upload.id, i, start_offset, end_offset)
+        i = i + 1
+
+    if do_list is not None:
+        l = dst_bucket.list_multipart_uploads()
+        l = list(l)
+
+    return upload
+
+def _create_key_with_random_content(keyname, size=7*1024*1024):
+    bucket = get_new_bucket()
+    key = bucket.new_key(keyname)
+    data = StringIO(str(generate_random(size, size).next()))
+    key.set_contents_from_file(fp=data)
+    return (bucket, key)
+
 @attr(resource='object')
 @attr(method='put')
 @attr(operation='check multipart upload without parts')
@@ -4642,6 +4673,19 @@ def test_multipart_upload_small():
     key2 = bucket.get_key(key)
     eq(key2.size, size)
 
+@attr(resource='object')
+@attr(method='put')
+@attr(operation='check multipart copies with single small part')
+def test_multipart_copy_small():
+    (src_bucket, src_key) = _create_key_with_random_content('foo')
+    dst_bucket = get_new_bucket()
+    dst_keyname = "mymultipart"
+    size = 1
+    copy = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, size)
+    copy.complete_upload()
+    key2 = dst_bucket.get_key(dst_keyname)
+    eq(key2.size, size)
+
 def _check_content_using_range(k, data, step):
     objlen = k.size
     for ofs in xrange(0, k.size, step):
@@ -4736,6 +4780,33 @@ def test_multipart_upload_multiple_sizes():
     (upload, data) = _multipart_upload(bucket, key, 10 * 1024 * 1024)
     upload.complete_upload()
 
+@attr(assertion='successful')
+def test_multipart_copy_multiple_sizes():
+    (src_bucket, src_key) = _create_key_with_random_content('foo', 12 * 1024 * 1024)
+    dst_bucket = get_new_bucket()
+    dst_keyname="mymultipart"
+
+    k = src_bucket.get_key(src_key.name)
+    s = k.get_contents_as_string()
+
+    upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 5 * 1024 * 1024)
+    upload.complete_upload()
+
+    upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 5 * 1024 * 1024 + 100 * 1024)
+    upload.complete_upload()
+
+    upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 5 * 1024 * 1024 + 600 * 1024)
+    upload.complete_upload()
+
+    upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 10 * 1024 * 1024 + 100 * 1024)
+    upload.complete_upload()
+
+    upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 10 * 1024 * 1024 + 600 * 1024)
+    upload.complete_upload()
+
+    upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 10 * 1024 * 1024)
+    upload.complete_upload()
+
 @attr(resource='object')
 @attr(method='put')
 @attr(operation='check failure on multiple multi-part upload with size too small')