From d453ef99bffdeaa7c41f5fb76a5df6985ddce2ae Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Thu, 27 Nov 2014 11:20:38 +1100 Subject: [PATCH 1/5] Remove misleading . --- docs/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 8899df9d49..0b8328b35c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -36,7 +36,7 @@ Code style guide * Use 79 characters in a line * Make sure edited file doesn't contain any trailing whitespace * You can verify that your modifications don't break any rules by running the - ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py.`` or + ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py`` or ``tox -e lint``. Second command fill run flake8 on all the files in the repository. From a1b078b729e0f68bbc83cd56ed51bbf8b70726e1 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Thu, 27 Nov 2014 11:20:57 +1100 Subject: [PATCH 2/5] Typo: fill instead of will. --- docs/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 0b8328b35c..09cfe2b13f 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -38,7 +38,7 @@ Code style guide * You can verify that your modifications don't break any rules by running the ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py`` or ``tox -e lint``. - Second command fill run flake8 on all the files in the repository. + Second command will run flake8 on all the files in the repository. And most importantly, follow the existing style in the file you are editing and **be consistent**. From 5c9ad396f54b90912cefe272f68df5fc1b5926fe Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Thu, 27 Nov 2014 14:01:07 +1100 Subject: [PATCH 3/5] [LIBCLOUD-638] Add, document and test for headers to ``upload_object_via_stream``, and similar methods to enable supporting CORS (or other) headers in the cloudfiles driver: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Access-Control-Allow-Origin http://docs.rackspace.com/files/api/v1/cf-devguide/content/Assigning_CORS_Headers_to_Requests-d1e2120.html --- libcloud/storage/base.py | 15 +++++++-- libcloud/storage/drivers/cloudfiles.py | 14 ++++---- libcloud/test/storage/test_cloudfiles.py | 41 ++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 5e4f09afd4..a829b645ee 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -361,7 +361,7 @@ def download_object_as_stream(self, obj, chunk_size=None): 'download_object_as_stream not implemented for this driver') def upload_object(self, file_path, container, object_name, extra=None, - verify_hash=True): + verify_hash=True, headers=None): """ Upload an object currently located on a disk. @@ -380,6 +380,11 @@ def upload_object(self, file_path, container, object_name, extra=None, :param extra: Extra attributes (driver specific). (optional) :type extra: ``dict`` + :type headers: ``dict`` + :param headers: (optional) Additional request headers, + such as CORS headers. For example: + headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :rtype: :class:`Object` """ raise NotImplementedError( @@ -387,7 +392,8 @@ def upload_object(self, file_path, container, object_name, extra=None, def upload_object_via_stream(self, iterator, container, object_name, - extra=None): + extra=None, + headers=None): """ Upload an object using an iterator. @@ -419,6 +425,11 @@ def upload_object_via_stream(self, iterator, container, This dictionary must contain a 'content_type' key which represents a content type of the stored object. + :type headers: ``dict`` + :param headers: (optional) Additional request headers, + such as CORS headers. For example: + headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :rtype: ``object`` """ raise NotImplementedError( diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index a2189c1c62..314586b229 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -414,7 +414,7 @@ def download_object_as_stream(self, obj, chunk_size=None): success_status_code=httplib.OK) def upload_object(self, file_path, container, object_name, extra=None, - verify_hash=True): + verify_hash=True, headers=None): """ Upload an object. @@ -427,10 +427,11 @@ def upload_object(self, file_path, container, object_name, extra=None, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, extra=extra, file_path=file_path, - verify_hash=verify_hash) + verify_hash=verify_hash, headers=headers) def upload_object_via_stream(self, iterator, - container, object_name, extra=None): + container, object_name, extra=None, + headers=None): if isinstance(iterator, file): iterator = iter(iterator) @@ -440,7 +441,8 @@ def upload_object_via_stream(self, iterator, return self._put_object(container=container, object_name=object_name, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, - extra=extra, iterator=iterator) + extra=extra, iterator=iterator, + headers=headers) def delete_object(self, obj): container_name = self._encode_container_name(obj.container.name) @@ -752,7 +754,7 @@ def iterate_container_objects(self, container, ex_prefix=None): def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, - iterator=None, verify_hash=True): + iterator=None, verify_hash=True, headers=None): extra = extra or {} container_name_encoded = self._encode_container_name(container.name) object_name_encoded = self._encode_object_name(object_name) @@ -760,7 +762,7 @@ def _put_object(self, container, object_name, upload_func, meta_data = extra.get('meta_data', None) content_disposition = extra.get('content_disposition', None) - headers = {} + headers = headers or {} if meta_data: for key, value in list(meta_data.items()): key = 'X-Object-Meta-%s' % (key) diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index 70e52c80bc..4c823d6a71 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -635,6 +635,47 @@ def test__upload_object_part(self): self.assertEqual(func_kwargs['object_name'], expected_name) self.assertEqual(func_kwargs['container'], container) + def test_upload_object_via_stream_with_cors_headers(self): + """ + Test we can add some ``Cross-origin resource sharing`` headers + to the request about to be sent. + """ + cors_headers = { + 'Access-Control-Allow-Origin': 'http://mozilla.com', + 'Origin': 'http://storage.clouddrive.com', + } + expected_headers = { + # Automatically added headers + 'Content-Type': 'application/octet-stream', + 'Transfer-Encoding': 'chunked', + } + expected_headers.update(cors_headers) + + def intercept_request(request_path, + method=None, data=None, + headers=None, raw=True): + + # What we're actually testing + # ... would prefer assertDictEqual but for Python <=2.6 support + self.assertEqual(expected_headers, headers) + + raise NotImplementedError('oops') + self.driver.connection.request = intercept_request + + container = Container(name='CORS', extra={}, driver=self.driver) + + try: + self.driver.upload_object_via_stream( + iterator=iter(b'blob data like an image or video'), + container=container, + object_name="test_object", + headers=cors_headers, + ) + except NotImplementedError: + # Don't care about the response we'd have to mock anyway + # as long as we intercepted the request and checked its headers + pass + def test__upload_object_manifest(self): hash_function = self.driver._get_hash_function() hash_function.update(b('')) From 599f33a5eaff7408a7bc577f3863f302fe98e5e7 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Thu, 27 Nov 2014 14:44:47 +1100 Subject: [PATCH 4/5] Document something that affects a full usage of a bytes iterator but I have no idea on the correct fix for. e.g. bytes(37) returns 37 null bytes... --- libcloud/test/storage/test_cloudfiles.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index 4c823d6a71..2e55aee332 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -666,6 +666,10 @@ def intercept_request(request_path, try: self.driver.upload_object_via_stream( + # We never reach the Python 3 only bytes vs int error + # currently at libcloud/utils/py3.py:89 + # raise TypeError("Invalid argument %r for b()" % (s,)) + # because I raise a NotImplementedError. iterator=iter(b'blob data like an image or video'), container=container, object_name="test_object", From bf0ef50dfc83e1a78c60f080cabe07faddaa9d71 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Thu, 27 Nov 2014 16:05:43 +1100 Subject: [PATCH 5/5] Consistently order as :param arg: then :type arg: (if there's a higher level coding convention I'll happily swap the other half of this file instead). --- libcloud/storage/base.py | 60 ++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index a829b645ee..66e5d44885 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -380,10 +380,10 @@ def upload_object(self, file_path, container, object_name, extra=None, :param extra: Extra attributes (driver specific). (optional) :type extra: ``dict`` - :type headers: ``dict`` :param headers: (optional) Additional request headers, such as CORS headers. For example: headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :type headers: ``dict`` :rtype: :class:`Object` """ @@ -411,24 +411,24 @@ def upload_object_via_stream(self, iterator, container, function which uses fs.stat function to determine the file size and it doesn't need to buffer whole object in the memory. - :type iterator: :class:`object` :param iterator: An object which implements the iterator interface. + :type iterator: :class:`object` - :type container: :class:`Container` :param container: Destination container. + :type container: :class:`Container` - :type object_name: ``str`` :param object_name: Object name. + :type object_name: ``str`` - :type extra: ``dict`` :param extra: (optional) Extra attributes (driver specific). Note: This dictionary must contain a 'content_type' key which represents a content type of the stored object. + :type extra: ``dict`` - :type headers: ``dict`` :param headers: (optional) Additional request headers, such as CORS headers. For example: headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'} + :type headers: ``dict`` :rtype: ``object`` """ @@ -439,8 +439,8 @@ def delete_object(self, obj): """ Delete an object. - :type obj: :class:`Object` :param obj: Object instance. + :type obj: :class:`Object` :return: ``bool`` True on success. :rtype: ``bool`` @@ -452,8 +452,8 @@ def create_container(self, container_name): """ Create a new container. - :type container_name: ``str`` :param container_name: Container name. + :type container_name: ``str`` :return: Container instance on success. :rtype: :class:`Container` @@ -465,8 +465,8 @@ def delete_container(self, container): """ Delete a container. - :type container: :class:`Container` :param container: Container instance + :type container: :class:`Container` :return: ``True`` on success, ``False`` otherwise. :rtype: ``bool`` @@ -479,23 +479,23 @@ def _get_object(self, obj, callback, callback_kwargs, response, """ Call passed callback and start transfer of the object' - :type obj: :class:`Object` :param obj: Object instance. + :type obj: :class:`Object` - :type callback: :class:`function` :param callback: Function which is called with the passed callback_kwargs + :type callback: :class:`function` - :type callback_kwargs: ``dict`` :param callback_kwargs: Keyword arguments which are passed to the callback. + :type callback_kwargs: ``dict`` - :typed response: :class:`Response` :param response: Response instance. + :type response: :class:`Response` - :type success_status_code: ``int`` :param success_status_code: Status code which represents a successful transfer (defaults to httplib.OK) + :type success_status_code: ``int`` :return: ``True`` on success, ``False`` otherwise. :rtype: ``bool`` @@ -518,26 +518,26 @@ def _save_object(self, response, obj, destination_path, """ Save object to the provided path. - :type response: :class:`RawResponse` :param response: RawResponse instance. + :type response: :class:`RawResponse` - :type obj: :class:`Object` :param obj: Object instance. + :type obj: :class:`Object` - :type destination_path: ``str`` :param destination_path: Destination directory. + :type destination_path: ``str`` - :type delete_on_failure: ``bool`` :param delete_on_failure: True to delete partially downloaded object if the download fails. + :type delete_on_failure: ``bool`` - :type overwrite_existing: ``bool`` :param overwrite_existing: True to overwrite a local path if it already exists. + :type overwrite_existing: ``bool`` - :type chunk_size: ``int`` :param chunk_size: Optional chunk size (defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb) + :type chunk_size: ``int`` :return: ``True`` on success, ``False`` otherwise. :rtype: ``bool`` @@ -671,20 +671,20 @@ def _upload_data(self, response, data, calculate_hash=True): """ Upload data stored in a string. - :type response: :class:`RawResponse` :param response: RawResponse object. + :type response: :class:`RawResponse` - :type data: ``str`` :param data: Data to upload. + :type data: ``str`` - :type calculate_hash: ``bool`` :param calculate_hash: True to calculate hash of the transferred data. - (defauls to True). + (defaults to True). + :type calculate_hash: ``bool`` - :rtype: ``tuple`` :return: First item is a boolean indicator of success, second one is the uploaded data MD5 hash and the third one is the number of transferred bytes. + :rtype: ``tuple`` """ bytes_transferred = 0 data_hash = None @@ -712,23 +712,23 @@ def _stream_data(self, response, iterator, chunked=False, """ Stream a data over an http connection. - :type response: :class:`RawResponse` :param response: RawResponse object. + :type response: :class:`RawResponse` - :type iterator: :class:`object` :param response: An object which implements an iterator interface or a File like object with read method. + :type iterator: :class:`object` - :type chunked: ``bool`` :param chunked: True if the chunked transfer encoding should be used (defauls to False). + :type chunked: ``bool`` - :type calculate_hash: ``bool`` :param calculate_hash: True to calculate hash of the transferred data. (defauls to True). + :type calculate_hash: ``bool`` - :type chunk_size: ``int`` :param chunk_size: Optional chunk size (defaults to ``CHUNK_SIZE``) + :type chunk_size: ``int`` :rtype: ``tuple`` :return: First item is a boolean indicator of success, second