-
-
Notifications
You must be signed in to change notification settings - Fork 2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Multipart upload support #64
Changes from 19 commits
f557487
5854219
f25caa8
0b45622
24ff30f
9746e72
04789a5
3630b3c
aead9bb
4539012
8f47355
e49006c
6e65b5f
b1d59c7
b64dbca
d9862aa
92bebbb
3628e40
bdf5a9e
3846c46
85e3210
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,5 @@ | ||
import os | ||
import base64 | ||
import datetime | ||
import hashlib | ||
|
||
|
@@ -23,7 +25,7 @@ def append_to_value(self, value): | |
@property | ||
def etag(self): | ||
value_md5 = hashlib.md5() | ||
value_md5.update(self.value) | ||
value_md5.update(bytes(self.value)) | ||
return '"{0}"'.format(value_md5.hexdigest()) | ||
|
||
@property | ||
|
@@ -52,10 +54,48 @@ def size(self): | |
return len(self.value) | ||
|
||
|
||
class FakeMultipart(object): | ||
def __init__(self, key_name): | ||
self.key_name = key_name | ||
self.parts = {} | ||
self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') | ||
|
||
def complete(self): | ||
total = bytearray() | ||
last_part_name = len(self.list_parts()) | ||
|
||
for part in self.list_parts(): | ||
if part.name != last_part_name and len(part.value) < 5242880: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should make 5242880 a constant too. |
||
return | ||
total.extend(part.value) | ||
|
||
return total | ||
|
||
def set_part(self, part_id, value): | ||
if part_id < 1: | ||
return | ||
|
||
key = FakeKey(part_id, value) | ||
self.parts[part_id] = key | ||
return key | ||
|
||
def list_parts(self): | ||
parts = [] | ||
|
||
for part_id, index in enumerate(sorted(self.parts.keys()), start=1): | ||
# Make sure part ids are continuous | ||
if part_id != index: | ||
return | ||
parts.append(self.parts[part_id]) | ||
|
||
return parts | ||
|
||
|
||
class FakeBucket(object): | ||
def __init__(self, name): | ||
self.name = name | ||
self.keys = {} | ||
self.multiparts = {} | ||
|
||
|
||
class S3Backend(BaseBackend): | ||
|
@@ -106,6 +146,36 @@ def get_key(self, bucket_name, key_name): | |
if bucket: | ||
return bucket.keys.get(key_name) | ||
|
||
def initiate_multipart(self, bucket_name, key_name): | ||
bucket = self.buckets[bucket_name] | ||
new_multipart = FakeMultipart(key_name) | ||
bucket.multiparts[new_multipart.id] = new_multipart | ||
|
||
return new_multipart | ||
|
||
def complete_multipart(self, bucket_name, multipart_id): | ||
bucket = self.buckets[bucket_name] | ||
multipart = bucket.multiparts[multipart_id] | ||
value = multipart.complete() | ||
if value is None: | ||
return | ||
del bucket.multiparts[multipart_id] | ||
|
||
return self.set_key(bucket_name, multipart.key_name, value) | ||
|
||
def cancel_multipart(self, bucket_name, multipart_id): | ||
bucket = self.buckets[bucket_name] | ||
del bucket.multiparts[multipart_id] | ||
|
||
def list_multipart(self, bucket_name, multipart_id): | ||
bucket = self.buckets[bucket_name] | ||
return bucket.multiparts[multipart_id].list_parts() | ||
|
||
def set_part(self, bucket_name, multipart_id, part_id, value): | ||
bucket = self.buckets[bucket_name] | ||
multipart = bucket.multiparts[multipart_id] | ||
return multipart.set_part(part_id, value) | ||
|
||
def prefix_query(self, bucket, prefix, delimiter): | ||
key_results = set() | ||
folder_results = set() | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,5 @@ | ||
import urllib2 | ||
from io import BytesIO | ||
|
||
import boto | ||
from boto.exception import S3ResponseError | ||
|
@@ -37,6 +38,26 @@ def test_my_model_save(): | |
conn.get_bucket('mybucket').get_key('steve').get_contents_as_string().should.equal('is awesome') | ||
|
||
|
||
@mock_s3 | ||
def test_multipart_upload(): | ||
conn = boto.connect_s3('the_key', 'the_secret') | ||
bucket = conn.create_bucket("foobar") | ||
|
||
multipart = bucket.initiate_multipart_upload("the-key") | ||
multipart.upload_part_from_file(BytesIO('hello'), 1) | ||
multipart.upload_part_from_file(BytesIO('world'), 2) | ||
# Multipart with total size under 5MB is refused | ||
multipart.complete_upload.should.throw(S3ResponseError) | ||
|
||
multipart = bucket.initiate_multipart_upload("the-key") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it would make sense to break this second part into a separate test. |
||
part1 = '0' * 5242880 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe add a comment about what we're doing here? |
||
multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) | ||
part2 = '1' | ||
multipart.upload_part_from_file(BytesIO('1'), 2) | ||
multipart.complete_upload() | ||
bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) | ||
|
||
|
||
@mock_s3 | ||
def test_missing_key(): | ||
conn = boto.connect_s3('the_key', 'the_secret') | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's make 43 a constant.