From f557487e06bde361c953ed719fc3a5ea728455b4 Mon Sep 17 00:00:00 2001 From: Lucian Branescu Mihaila Date: Tue, 26 Mar 2013 14:52:33 +0000 Subject: [PATCH 01/18] Beginning of multipart upload support. --- moto/s3/models.py | 53 ++++++++++++++++++++++++++++++++++++++++ moto/s3/responses.py | 27 ++++++++++++++++++++ tests/test_s3/test_s3.py | 21 ++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/moto/s3/models.py b/moto/s3/models.py index d80eec41734..2462d59be17 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -1,5 +1,7 @@ # from boto.s3.bucket import Bucket # from boto.s3.key import Key +import os +import base64 import md5 from moto.core import BaseBackend @@ -21,10 +23,40 @@ def size(self): return len(self.value) +class FakeMultipart(object): + def __init__(self, key_name): + self.key_name = key_name + self.parts = {} + self.id = base64.b64encode(os.urandom(43)).replace('=', '') + + def complete(self): + total = bytearray() + + for part_id, index in enumerate(sorted(self.parts.keys()), start=1): + # Make sure part ids are continuous + if part_id != index: + return + + total.extend(self.parts[part_id]) + + if len(total) < 5242880: + return + + return total + + def set_part(self, part_id, value): + if part_id < 1: + return False + + self.parts[part_id] = value + return True + + class FakeBucket(object): def __init__(self, name): self.name = name self.keys = {} + self.multiparts = {} class S3Backend(BaseBackend): @@ -65,6 +97,27 @@ def get_key(self, bucket_name, key_name): if bucket: return bucket.keys.get(key_name) + def initiate_multipart(self, bucket_name, key_name): + bucket = self.buckets[bucket_name] + new_multipart = FakeMultipart(key_name) + bucket.multiparts[new_multipart.id] = new_multipart + + return new_multipart + + def complete_multipart(self, bucket_name, multipart_id): + bucket = self.buckets[bucket_name] + multipart = bucket.multiparts[multipart_id] + value = multipart.complete() + if value is None: + return False + + self.set_key(bucket_name, multipart.key_name, value) + + def set_part(self, bucket_name, multipart_id, part_id, value): + bucket = self.buckets[bucket_name] + multipart = bucket.multiparts[multipart_id] + return multipart.set_part(part_id, value) + def prefix_query(self, bucket, prefix): key_results = set() folder_results = set() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 80a0a9421cc..370c7cf5bad 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -106,6 +106,20 @@ def key_response(uri_info, method, body, headers): removed_key = s3_backend.delete_key(bucket_name, key_name) template = Template(S3_DELETE_OBJECT_SUCCESS) return template.render(bucket=removed_key), dict(status=204) + elif method == 'POST': + if body == '' and uri_info.query == 'uploads': + multipart = s3_backend.initiate_multipart(bucket_name, key_name) + template = Template(S3_MULTIPART_RESPONSE) + response = template.render( + bucket_name=bucket_name, + key_name=key_name, + multipart_id=multipart.id, + ) + print response + return response, dict() + else: + import pdb; pdb.set_trace() + raise NotImplementedError("POST is only allowed for multipart uploads") else: raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method)) @@ -202,3 +216,16 @@ def key_response(uri_info, method, body, headers): 2008-02-18T13:54:10.183Z """ + +S3_MULTIPART_RESPONSE = """ + + {{ bucket_name }} + {{ key_name }} + {{ upload_id }} +""" + +S3_MULTIPART_COMPLETE_RESPONSE = """ +""" + +S3_MULTIPART_ERROR_RESPONSE = """ +""" diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 31e011bfc82..1f713fabb6e 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1,4 +1,5 @@ import urllib2 +from io import BytesIO import boto from boto.exception import S3ResponseError @@ -36,6 +37,26 @@ def test_my_model_save(): conn.get_bucket('mybucket').get_key('steve').get_contents_as_string().should.equal('is awesome') +@mock_s3 +def test_multipart_upload(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + multipart = bucket.initiate_multipart_upload("the-key") + multipart.upload_part_from_file(BytesIO('hello'), 1) + multipart.upload_part_from_file(BytesIO('world'), 1) + # Multipart with total size under 5MB is refused + multipart.complete_upload().should.throw(S3ResponseError) + + multipart = bucket.initiate_multipart_upload("the-key") + part1 = '0' * 5242880 + multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) + part2 = '1' + multipart.upload_part_from_file(BytesIO('1'), 1) + multipart.complete_upload() + bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) + + @mock_s3 def test_missing_key(): conn = boto.connect_s3('the_key', 'the_secret') From 5854219a4c41cef103af63a1a9baf293f58fb519 Mon Sep 17 00:00:00 2001 From: Lucian Branescu Mihaila Date: Tue, 26 Mar 2013 15:50:18 +0000 Subject: [PATCH 02/18] Upload part and complete upload. Somehow, boto doesn't like output I send it, even though it's copy-pasted from its own logs. --- moto/s3/models.py | 14 ++++++++------ moto/s3/responses.py | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 2462d59be17..a2547ea4789 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -37,7 +37,7 @@ def complete(self): if part_id != index: return - total.extend(self.parts[part_id]) + total.extend(self.parts[part_id].value) if len(total) < 5242880: return @@ -46,10 +46,11 @@ def complete(self): def set_part(self, part_id, value): if part_id < 1: - return False + return - self.parts[part_id] = value - return True + key = FakeKey(part_id, value) + self.parts[part_id] = key + return key class FakeBucket(object): @@ -109,9 +110,10 @@ def complete_multipart(self, bucket_name, multipart_id): multipart = bucket.multiparts[multipart_id] value = multipart.complete() if value is None: - return False + return + del bucket.multiparts[multipart_id] - self.set_key(bucket_name, multipart.key_name, value) + return self.set_key(bucket_name, multipart.key_name, value) def set_part(self, bucket_name, multipart_id, part_id, value): bucket = self.buckets[bucket_name] diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 370c7cf5bad..fc1fdcec2f5 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -64,6 +64,7 @@ def key_response(uri_info, method, body, headers): key_name = uri_info.path.lstrip('/') hostname = uri_info.hostname headers = headers_to_dict(headers) + query = parse_qs(uri_info.query) bucket_name = bucket_name_from_hostname(hostname) @@ -74,12 +75,20 @@ def key_response(uri_info, method, body, headers): else: return "", dict(status=404) if method == 'PUT': + if 'uploadId' in query and 'partNumber' in query and body: + upload_id = query['uploadId'][0] + part_number = int(query['partNumber'][0]) + key = s3_backend.set_part(bucket_name, upload_id, part_number, body) + + return '', dict(etag=key.etag) + if 'x-amz-copy-source' in headers: # Copy key src_bucket, src_key = headers.get("x-amz-copy-source").split("/") s3_backend.copy_key(src_bucket, src_key, bucket_name, key_name) template = Template(S3_OBJECT_COPY_RESPONSE) return template.render(key=src_key) + if body is not None: key = s3_backend.get_key(bucket_name, key_name) if not key or body: @@ -107,19 +116,30 @@ def key_response(uri_info, method, body, headers): template = Template(S3_DELETE_OBJECT_SUCCESS) return template.render(bucket=removed_key), dict(status=204) elif method == 'POST': + import pdb; pdb.set_trace() if body == '' and uri_info.query == 'uploads': multipart = s3_backend.initiate_multipart(bucket_name, key_name) - template = Template(S3_MULTIPART_RESPONSE) + template = Template(S3_MULTIPART_INITIATE_RESPONSE) response = template.render( bucket_name=bucket_name, key_name=key_name, multipart_id=multipart.id, ) - print response return response, dict() + + if body == '' and 'uploadId' in query: + upload_id = query['uploadId'][0] + key = s3_backend.complete_multipart(bucket_name, upload_id) + + if key is not None: + template = Template(S3_MULTIPART_COMPLETE_RESPONSE) + return template.render( + bucket_name=bucket_name, + key_name=key.name, + etag=key.etag, + ) else: - import pdb; pdb.set_trace() - raise NotImplementedError("POST is only allowed for multipart uploads") + raise NotImplementedError("Method POST had only been implemented for multipart uploads so far") else: raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method)) @@ -217,15 +237,18 @@ def key_response(uri_info, method, body, headers): """ -S3_MULTIPART_RESPONSE = """ +S3_MULTIPART_INITIATE_RESPONSE = """ {{ bucket_name }} {{ key_name }} {{ upload_id }} """ -S3_MULTIPART_COMPLETE_RESPONSE = """ -""" - -S3_MULTIPART_ERROR_RESPONSE = """ +S3_MULTIPART_COMPLETE_RESPONSE = """ + + http://{{ bucket_name }}.s3.amazonaws.com/{{ key_name }} + {{ bucket_name }} + {{ key_name }} + {{ etag }} + """ From 0b45622dcb10847a3b9980123bd9cbee81b39dfb Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 11:50:41 +0300 Subject: [PATCH 03/18] render part upload response correctly --- moto/s3/responses.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index bc359d87d21..6b0cdc2d585 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -133,8 +133,8 @@ def _key_response(request, full_url, headers): upload_id = query['uploadId'][0] part_number = int(query['partNumber'][0]) key = s3_backend.set_part(bucket_name, upload_id, part_number, body) - - return '', dict(etag=key.etag) + template = Template(S3_MULTIPART_UPLOAD_RESPONSE) + return 200, headers, template.render(part=key) if 'x-amz-copy-source' in request.headers: # Copy key @@ -310,6 +310,12 @@ def _key_response(request, full_url, headers): {{ upload_id }} """ +S3_MULTIPART_UPLOAD_RESPONSE = """ + + {{ part.last_modified_ISO8601 }} + {{ part.etag }} +""" + S3_MULTIPART_COMPLETE_RESPONSE = """ http://{{ bucket_name }}.s3.amazonaws.com/{{ key_name }} From 24ff30f9fc2893ad5aca2d9153e6e20c7b24a836 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 11:51:18 +0300 Subject: [PATCH 04/18] multipart_id doesn't exist, it's upload_id --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6b0cdc2d585..fe37c6f888d 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -187,7 +187,7 @@ def _key_response(request, full_url, headers): response = template.render( bucket_name=bucket_name, key_name=key_name, - multipart_id=multipart.id, + upload_id=multipart.id, ) return 200, headers, response From 9746e72e1d17056cd178437bf84b01358f52b5c8 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 12:09:35 +0300 Subject: [PATCH 05/18] implement list parts --- moto/s3/models.py | 23 +++++++++++++++++------ moto/s3/responses.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 67c48beb4b6..a5fddda9811 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -63,12 +63,8 @@ def __init__(self, key_name): def complete(self): total = bytearray() - for part_id, index in enumerate(sorted(self.parts.keys()), start=1): - # Make sure part ids are continuous - if part_id != index: - return - - total.extend(self.parts[part_id].value) + for part in self.list_parts(): + total.extend(part.value) if len(total) < 5242880: return @@ -83,6 +79,17 @@ def set_part(self, part_id, value): self.parts[part_id] = key return key + def list_parts(self): + parts = [] + + for part_id, index in enumerate(sorted(self.parts.keys()), start=1): + # Make sure part ids are continuous + if part_id != index: + return + parts.append(self.parts[part_id]) + + return parts + class FakeBucket(object): def __init__(self, name): @@ -156,6 +163,10 @@ def complete_multipart(self, bucket_name, multipart_id): return self.set_key(bucket_name, multipart.key_name, value) + def list_multipart(self, bucket_name, multipart_id): + bucket = self.buckets[bucket_name] + return bucket.multiparts[multipart_id].list_parts() + def set_part(self, bucket_name, multipart_id, part_id, value): bucket = self.buckets[bucket_name] multipart = bucket.multiparts[multipart_id] diff --git a/moto/s3/responses.py b/moto/s3/responses.py index fe37c6f888d..a0a6a698930 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -122,6 +122,17 @@ def _key_response(request, full_url, headers): body = request.data if method == 'GET': + if 'uploadId' in query: + upload_id = query['uploadId'][0] + parts = s3_backend.list_multipart(bucket_name, upload_id) + template = Template(S3_MULTIPART_LIST_RESPONSE) + return 200, headers, template.render( + bucket_name=bucket_name, + key_name=key_name, + upload_id=upload_id, + count=len(parts), + parts=parts + ) key = s3_backend.get_key(bucket_name, key_name) if key: headers.update(key.metadata) @@ -316,6 +327,35 @@ def _key_response(request, full_url, headers): {{ part.etag }} """ +S3_MULTIPART_LIST_RESPONSE = """ + + {{ bucket_name }} + {{ key_name }} + {{ upload_id }} + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + STANDARD + 1 + {{ count }} + {{ count }} + false + {% for part in parts %} + + {{ part.name }} + {{ part.last_modified_ISO8601 }} + {{ part.etag }} + {{ part.size }} + + {% endfor %} +""" + S3_MULTIPART_COMPLETE_RESPONSE = """ http://{{ bucket_name }}.s3.amazonaws.com/{{ key_name }} From 04789a59fa01be7517ab11fca8d9f9a5e01aad1b Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 12:10:05 +0300 Subject: [PATCH 06/18] don't forget to send the etag in the headers when uploading a part --- moto/s3/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index a0a6a698930..1b9cf8cac8b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -145,6 +145,7 @@ def _key_response(request, full_url, headers): part_number = int(query['partNumber'][0]) key = s3_backend.set_part(bucket_name, upload_id, part_number, body) template = Template(S3_MULTIPART_UPLOAD_RESPONSE) + headers.update(key.response_dict) return 200, headers, template.render(part=key) if 'x-amz-copy-source' in request.headers: From 3630b3c21a3631a04b05c51e93881c699691b08b Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 12:10:35 +0300 Subject: [PATCH 07/18] remove plus sign from id because it doesn't get parsed correctly in query string --- moto/s3/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index a5fddda9811..6244cdd5c30 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -58,7 +58,7 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - self.id = base64.b64encode(os.urandom(43)).replace('=', '') + self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', ' ') def complete(self): total = bytearray() From aead9bb0d595b0673ce68a39b3cb0971257373fd Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 14:34:56 +0300 Subject: [PATCH 08/18] Complete MultiPart Operation does specify a body --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 1b9cf8cac8b..11ad6b4b29f 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -203,7 +203,7 @@ def _key_response(request, full_url, headers): ) return 200, headers, response - if body == '' and 'uploadId' in query: + if 'uploadId' in query: upload_id = query['uploadId'][0] key = s3_backend.complete_multipart(bucket_name, upload_id) From 4539012db6a6e0ceb1af958cfa38da0690f2fea6 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 14:35:24 +0300 Subject: [PATCH 09/18] throw proper error if multipart upload is too small --- moto/s3/responses.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 11ad6b4b29f..e85b6a0b4a1 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -214,6 +214,8 @@ def _key_response(request, full_url, headers): key_name=key.name, etag=key.etag, ) + template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR) + return 400, headers, template.render() else: raise NotImplementedError("Method POST had only been implemented for multipart uploads so far") else: @@ -365,3 +367,11 @@ def _key_response(request, full_url, headers): {{ etag }} """ + +S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR = """ + + EntityTooSmall + Your proposed upload is smaller than the minimum allowed object size. + asdfasdfsdafds + sdfgdsfgdsfgdfsdsfgdfs +""" From 8f473554930bc87867532297cdac903cb3ee6594 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 14:50:23 +0300 Subject: [PATCH 10/18] remove plus from id (for real this time) --- moto/s3/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 6244cdd5c30..81a7452738e 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -58,7 +58,9 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', ' ') + import pdb; pdb.set_trace() + self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') + def complete(self): total = bytearray() From e49006c72312308f1e5a2a93d2ed0bd243546d06 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 16:36:51 +0300 Subject: [PATCH 11/18] remove pdb --- moto/s3/models.py | 1 - moto/s3/responses.py | 1 - 2 files changed, 2 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 81a7452738e..0abc3e4c2bb 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -58,7 +58,6 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - import pdb; pdb.set_trace() self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e85b6a0b4a1..ded07e2f0b2 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -192,7 +192,6 @@ def _key_response(request, full_url, headers): template = Template(S3_DELETE_OBJECT_SUCCESS) return 204, headers, template.render(bucket=removed_key) elif method == 'POST': - import pdb; pdb.set_trace() if body == '' and parsed_url.query == 'uploads': multipart = s3_backend.initiate_multipart(bucket_name, key_name) template = Template(S3_MULTIPART_INITIATE_RESPONSE) From 6e65b5f6d48543ddcce9afc33d32cf1f066ddade Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 16:37:50 +0300 Subject: [PATCH 12/18] fix part numbering in multipart tests --- tests/test_s3/test_s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 5b5fbf564e1..f4a5ad18d88 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -45,7 +45,7 @@ def test_multipart_upload(): multipart = bucket.initiate_multipart_upload("the-key") multipart.upload_part_from_file(BytesIO('hello'), 1) - multipart.upload_part_from_file(BytesIO('world'), 1) + multipart.upload_part_from_file(BytesIO('world'), 2) # Multipart with total size under 5MB is refused multipart.complete_upload().should.throw(S3ResponseError) @@ -53,7 +53,7 @@ def test_multipart_upload(): part1 = '0' * 5242880 multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) part2 = '1' - multipart.upload_part_from_file(BytesIO('1'), 1) + multipart.upload_part_from_file(BytesIO('1'), 2) multipart.complete_upload() bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) From b1d59c7e173f4f13969f232b83059feee4c5a00f Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 18:36:25 +0300 Subject: [PATCH 13/18] support cancelling of multipart upload --- moto/s3/models.py | 4 ++++ moto/s3/responses.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/moto/s3/models.py b/moto/s3/models.py index 0abc3e4c2bb..a97e2862700 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -164,6 +164,10 @@ def complete_multipart(self, bucket_name, multipart_id): return self.set_key(bucket_name, multipart.key_name, value) + def cancel_multipart(self, bucket_name, multipart_id): + bucket = self.buckets[bucket_name] + del bucket.multiparts[multipart_id] + def list_multipart(self, bucket_name, multipart_id): bucket = self.buckets[bucket_name] return bucket.multiparts[multipart_id].list_parts() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ded07e2f0b2..b2ff1b60bfe 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -188,6 +188,10 @@ def _key_response(request, full_url, headers): else: return 404, headers, "" elif method == 'DELETE': + if 'uploadId' in query: + upload_id = query['uploadId'][0] + s3_backend.cancel_multipart(bucket_name, upload_id) + return 204, headers, "" removed_key = s3_backend.delete_key(bucket_name, key_name) template = Template(S3_DELETE_OBJECT_SUCCESS) return 204, headers, template.render(bucket=removed_key) From b64dbcaa12aa284d7a0ae39d9337747522641ee0 Mon Sep 17 00:00:00 2001 From: Mike Attili Date: Thu, 7 Nov 2013 17:07:56 -0500 Subject: [PATCH 14/18] Remove ()'s on complete_upload since should.throw requires a 'callable'. --- tests/test_s3/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f4a5ad18d88..8cbd489d0fb 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -47,7 +47,7 @@ def test_multipart_upload(): multipart.upload_part_from_file(BytesIO('hello'), 1) multipart.upload_part_from_file(BytesIO('world'), 2) # Multipart with total size under 5MB is refused - multipart.complete_upload().should.throw(S3ResponseError) + multipart.complete_upload.should.throw(S3ResponseError) multipart = bucket.initiate_multipart_upload("the-key") part1 = '0' * 5242880 From d9862aaa65aac24d3f66c4e41aebbb55b50bd853 Mon Sep 17 00:00:00 2001 From: Mike Attili Date: Thu, 7 Nov 2013 17:09:53 -0500 Subject: [PATCH 15/18] Correct size check on multipart uploads. All parts except last must be > 5MB --- moto/s3/models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index a97e2862700..9ab0a5b8bc3 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -60,16 +60,15 @@ def __init__(self, key_name): self.parts = {} self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') - def complete(self): total = bytearray() + last_part_name = len(self.list_parts()) for part in self.list_parts(): + if part.name != last_part_name and len(part.value) < 5242880: + return total.extend(part.value) - if len(total) < 5242880: - return - return total def set_part(self, part_id, value): From bdf5a9e26b129e9dda01b4c550d53c1907ef0b06 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Thu, 14 Nov 2013 17:47:03 +0200 Subject: [PATCH 16/18] convert to bytes for python 2.6 --- moto/s3/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 43453e4661c..7447d0cd961 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -25,7 +25,7 @@ def append_to_value(self, value): @property def etag(self): value_md5 = hashlib.md5() - value_md5.update(self.value) + value_md5.update(bytes(self.value)) return '"{0}"'.format(value_md5.hexdigest()) @property From 3846c4699018f997b2f18e4af284b118771ca6c4 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Fri, 15 Nov 2013 11:53:39 +0200 Subject: [PATCH 17/18] replace literal ints with constants --- moto/s3/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 7447d0cd961..e5955886469 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -7,6 +7,9 @@ from moto.core.utils import iso_8601_datetime, rfc_1123_datetime from .utils import clean_key_name +UPLOAD_ID_BYTES=43 +UPLOAD_PART_MIN_SIZE=5242880 + class FakeKey(object): def __init__(self, name, value): @@ -58,14 +61,14 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') + self.id = base64.b64encode(os.urandom(UPLOAD_ID_BYTES)).replace('=', '').replace('+', '') def complete(self): total = bytearray() last_part_name = len(self.list_parts()) for part in self.list_parts(): - if part.name != last_part_name and len(part.value) < 5242880: + if part.name != last_part_name and len(part.value) < UPLOAD_PART_MIN_SIZE: return total.extend(part.value) From 85e32102fa75e3ef043e43758f4297bbe148a47c Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Fri, 15 Nov 2013 11:59:30 +0200 Subject: [PATCH 18/18] break multipart test in two --- tests/test_s3/test_s3.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 8cbd489d0fb..3d9e3f1fbf9 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -39,7 +39,7 @@ def test_my_model_save(): @mock_s3 -def test_multipart_upload(): +def test_multipart_upload_too_small(): conn = boto.connect_s3('the_key', 'the_secret') bucket = conn.create_bucket("foobar") @@ -49,12 +49,20 @@ def test_multipart_upload(): # Multipart with total size under 5MB is refused multipart.complete_upload.should.throw(S3ResponseError) + +@mock_s3 +def test_multipart_upload(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + multipart = bucket.initiate_multipart_upload("the-key") part1 = '0' * 5242880 - multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) + multipart.upload_part_from_file(BytesIO(part1), 1) + # last part, can be less than 5 MB part2 = '1' - multipart.upload_part_from_file(BytesIO('1'), 2) + multipart.upload_part_from_file(BytesIO(part2), 2) multipart.complete_upload() + # we should get both parts as the key contents bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2)