From f73abb13436cc3cf51d4a2f7054c683312f0f51d Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Mon, 1 Dec 2014 17:28:20 +1100 Subject: [PATCH 1/5] [LIBCLOUD-639] Add erroring test for this Python 3 bytes issue so I don't need to explain as much magic in an earlier test. --- libcloud/test/storage/test_cloudfiles.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index 6590270ff3..77bcdc5ad5 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -665,10 +665,6 @@ def intercept_request(request_path, try: self.driver.upload_object_via_stream( - # We never reach the Python 3 only bytes vs int error - # currently at libcloud/utils/py3.py:89 - # raise TypeError("Invalid argument %r for b()" % (s,)) - # because I raise a NotImplementedError. iterator=iter(b'blob data like an image or video'), container=container, object_name="test_object", @@ -682,6 +678,17 @@ def intercept_request(request_path, self.fail('Expected NotImplementedError to be thrown to ' 'verify we actually checked the expected headers') + def test_upload_object_via_stream_python3_bytes_error(self): + container = Container(name='py3', extra={}, driver=self.driver) + bytes_blob = b'blob data like an image or video' + + # This is mostly to check we didn't discover other errors along the way + mocked_response = container.upload_object_via_stream( + iterator=iter(bytes_blob), + object_name="img_or_vid", + ) + self.assertEqual(len(bytes_blob), mocked_response.size) + def test__upload_object_manifest(self): hash_function = self.driver._get_hash_function() hash_function.update(b('')) @@ -1087,6 +1094,11 @@ class CloudFilesMockRawResponse(MockRawResponse): fixtures = StorageFileFixtures('cloudfiles') base_headers = {'content-type': 'application/json; charset=UTF-8'} + def _v1_MossoCloudFS_py3_img_or_vid(self, method, url, body, headers): + headers = {'etag': 'e2378cace8712661ce7beec3d9362ef6'} + headers.update(self.base_headers) + return httplib.CREATED, '', headers, httplib.responses[httplib.CREATED] + def _v1_MossoCloudFS_foo_bar_container_foo_test_upload( self, method, url, body, headers): # test_object_upload_success From 7eaa0a6447c1eb20e31595be6b60c446cef85a01 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Mon, 1 Dec 2014 17:28:58 +1100 Subject: [PATCH 2/5] Fix test under Python 3.4 at least. http://stackoverflow.com/a/21017834 --- libcloud/utils/py3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libcloud/utils/py3.py b/libcloud/utils/py3.py index 4e054197e3..2b695a483e 100644 --- a/libcloud/utils/py3.py +++ b/libcloud/utils/py3.py @@ -85,6 +85,8 @@ def b(s): return s.encode('utf-8') elif isinstance(s, bytes): return s + elif isinstance(s, int): + return bytes([s]) else: raise TypeError("Invalid argument %r for b()" % (s,)) From fe6cca9405a8958db6a5f5b795792cbfef37df41 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Tue, 2 Dec 2014 12:56:11 +1100 Subject: [PATCH 3/5] Don't send one byte at a time, read the iterator in chunks and fill the chunks, otherwise it turns a 1.6MB file into more than 400MB (when I gave up and killed the request). --- libcloud/storage/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 66e5d44885..ead1545ae3 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -742,7 +742,8 @@ def _stream_data(self, response, iterator, chunked=False, if calculate_hash: data_hash = self._get_hash_function() - generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size) + generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size, + fill_size=True) bytes_transferred = 0 try: From 776f6e5b5496f896f8b238d1d6b26d1d04a52e54 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Mon, 1 Dec 2014 18:48:49 +1100 Subject: [PATCH 4/5] Typo missing i. --- libcloud/utils/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/utils/files.py b/libcloud/utils/files.py index a71e1c488f..663677a18c 100644 --- a/libcloud/utils/files.py +++ b/libcloud/utils/files.py @@ -38,7 +38,7 @@ def read_in_chunks(iterator, chunk_size=None, fill_size=False, """ Return a generator which yields data in chunks. - :param terator: An object which implements an iterator interface + :param iterator: An object which implements an iterator interface or a File like object with read method. :type iterator: :class:`object` which implements iterator interface. From 3bf7af1e9747ba97302a77f96b8a345688f37899 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Mon, 8 Dec 2014 11:40:09 +1100 Subject: [PATCH 5/5] Add regression test for `fill_size=True` change in https://github.com/apache/libcloud/commit/fe6cca9405a8958db6a5f5b795792cbfef37df41 --- libcloud/test/storage/test_cloudfiles.py | 49 +++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index 77bcdc5ad5..3e911dafe3 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -30,7 +30,7 @@ from libcloud.utils.py3 import urlquote from libcloud.common.types import LibcloudError, MalformedResponseError -from libcloud.storage.base import Container, Object +from libcloud.storage.base import CHUNK_SIZE, Container, Object from libcloud.storage.types import ContainerAlreadyExistsError from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ContainerIsNotEmptyError @@ -689,6 +689,53 @@ def test_upload_object_via_stream_python3_bytes_error(self): ) self.assertEqual(len(bytes_blob), mocked_response.size) + def test_upload_object_via_stream_chunked_encoding(self): + + # Create enough bytes it should get split into two chunks + bytes_blob = ''.join(['\0' for _ in range(CHUNK_SIZE + 1)]) + hex_chunk_size = ('%X' % CHUNK_SIZE).encode('utf8') + expected = [ + # Chunk 1 + hex_chunk_size + b'\r\n', + bytes(bytes_blob[:CHUNK_SIZE].encode('utf8')), + b'\r\n', + + # Chunk 2 + b'1\r\n', + bytes(bytes_blob[CHUNK_SIZE:].encode('utf8')), + b'\r\n', + + # If chunked, also send a final message + b'0\r\n\r\n', + ] + logged_data = [] + + class InterceptResponse(CloudFilesMockRawResponse): + def __init__(self, connection): + super(InterceptResponse, self).__init__(connection=connection) + old_send = self.connection.connection.send + + def intercept_send(data): + old_send(data) + logged_data.append(data) + self.connection.connection.send = intercept_send + + def _v1_MossoCloudFS_py3_img_or_vid2(self, + method, url, body, headers): + headers = {'etag': 'd79fb00c27b50494a463e680d459c90c'} + headers.update(self.base_headers) + _201 = httplib.CREATED + return _201, '', headers, httplib.responses[_201] + + self.driver_klass.connectionCls.rawResponseCls = InterceptResponse + + container = Container(name='py3', extra={}, driver=self.driver) + container.upload_object_via_stream( + iterator=iter(bytes_blob), + object_name="img_or_vid2", + ) + self.assertListEqual(expected, logged_data) + def test__upload_object_manifest(self): hash_function = self.driver._get_hash_function() hash_function.update(b(''))