From 7953b24e5126a193b1a5925ee7fa52c7cf0a3a3d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 17:27:27 +0200 Subject: [PATCH 1/7] Fix Azure blobs driver, make sure Content-Length header value is a string and not a number. Part of LIBCLOUD-945 --- libcloud/storage/drivers/azure_blobs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py index e1bb7225f3..82ee03a68f 100644 --- a/libcloud/storage/drivers/azure_blobs.py +++ b/libcloud/storage/drivers/azure_blobs.py @@ -634,7 +634,7 @@ def _upload_in_chunks(self, response, data, iterator, object_path, chunk_hash = base64.b64encode(b(chunk_hash.digest())) headers['Content-MD5'] = chunk_hash.decode('utf-8') - headers['Content-Length'] = content_length + headers['Content-Length'] = str(content_length) if blob_type == 'BlockBlob': # Block id can be any unique string that is base64 encoded @@ -861,10 +861,10 @@ def _prepare_upload_headers(self, object_name, object_size, self._update_metadata(headers, meta_data) if object_size is not None: - headers['Content-Length'] = object_size + headers['Content-Length'] = str(object_size) if blob_type == 'PageBlob': - headers['Content-Length'] = 0 + headers['Content-Length'] = str('0') headers['x-ms-blob-content-length'] = object_size return headers From 6e19a63847a6a554aef9078c7efce60c175ad32e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 17:28:08 +0200 Subject: [PATCH 2/7] Add test cases for it. Part of LIBCLOUD-745. --- libcloud/test/storage/test_azure_blobs.py | 25 ++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py index eb12c0d1ba..95ab3f1122 100644 --- a/libcloud/test/storage/test_azure_blobs.py +++ b/libcloud/test/storage/test_azure_blobs.py @@ -17,7 +17,6 @@ import os import sys -import unittest import tempfile from io import BytesIO @@ -25,6 +24,7 @@ from libcloud.utils.py3 import urlparse from libcloud.utils.py3 import parse_qs from libcloud.utils.py3 import b +from libcloud.utils.py3 import basestring from libcloud.common.types import InvalidCredsError from libcloud.common.types import LibcloudError @@ -39,12 +39,13 @@ from libcloud.storage.drivers.azure_blobs import AZURE_BLOCK_MAX_SIZE from libcloud.storage.drivers.azure_blobs import AZURE_PAGE_CHUNK_SIZE +from libcloud.test import unittest from libcloud.test import MockHttp, generate_random_data # pylint: disable-msg=E0611 from libcloud.test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 from libcloud.test.secrets import STORAGE_AZURE_BLOBS_PARAMS -class AzureBlobsMockHttp(MockHttp): +class AzureBlobsMockHttp(MockHttp, unittest.TestCase): fixtures = StorageFileFixtures('azure_blobs') base_headers = {} @@ -247,6 +248,8 @@ def _foo_bar_container_foo_bar_object_DELETE(self, method, url, body, headers): def _foo_bar_container_foo_test_upload(self, method, url, body, headers): # test_upload_object_success + self._assert_content_length_header_is_string(headers=headers) + body = '' headers = {} headers['etag'] = '0x8CFB877BB56A6FB' @@ -259,6 +262,8 @@ def _foo_bar_container_foo_test_upload(self, method, url, body, headers): def _foo_bar_container_foo_test_upload_block(self, method, url, body, headers): # test_upload_object_success + self._assert_content_length_header_is_string(headers=headers) + body = '' headers = {} headers['etag'] = '0x8CFB877BB56A6FB' @@ -281,6 +286,8 @@ def _foo_bar_container_foo_test_upload_page(self, method, url, def _foo_bar_container_foo_test_upload_blocklist(self, method, url, body, headers): # test_upload_object_success + self._assert_content_length_header_is_string(headers=headers) + body = '' headers = {} headers['etag'] = '0x8CFB877BB56A6FB' @@ -294,6 +301,8 @@ def _foo_bar_container_foo_test_upload_blocklist(self, method, url, def _foo_bar_container_foo_test_upload_lease(self, method, url, body, headers): # test_upload_object_success + self._assert_content_length_header_is_string(headers=headers) + action = headers['x-ms-lease-action'] rheaders = {'x-ms-lease-id': 'someleaseid'} body = '' @@ -318,12 +327,14 @@ def _foo_bar_container_foo_test_upload_lease(self, method, url, def _foo_bar_container_foo_test_upload_INVALID_HASH(self, method, url, body, headers): + # test_upload_object_invalid_hash1 + self._assert_content_length_header_is_string(headers=headers) + body = '' headers = {} headers['etag'] = '0x8CFB877BB56A6FB' headers['content-md5'] = 'd4fe4c9829f7ca1cc89db7ad670d2bbd' - # test_upload_object_invalid_hash1 return (httplib.CREATED, body, headers, @@ -331,6 +342,8 @@ def _foo_bar_container_foo_test_upload_INVALID_HASH(self, method, url, def _foo_bar_container_foo_bar_object(self, method, url, body, headers): # test_upload_object_invalid_file_size + self._assert_content_length_header_is_string(headers=headers) + body = generate_random_data(1000) return (httplib.OK, body, @@ -340,12 +353,18 @@ def _foo_bar_container_foo_bar_object(self, method, url, body, headers): def _foo_bar_container_foo_bar_object_INVALID_SIZE(self, method, url, body, headers): # test_upload_object_invalid_file_size + self._assert_content_length_header_is_string(headers=headers) + body = '' return (httplib.OK, body, headers, httplib.responses[httplib.OK]) + def _assert_content_length_header_is_string(self, headers): + if 'Content-Length' in headers: + self.assertTrue(isinstance(headers['Content-Length'], basestring)) + class AzureBlobsTests(unittest.TestCase): driver_type = AzureBlobsStorageDriver From 119cd4a6ebcd9402edc3668006e8c39cc3929f90 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 17:39:01 +0200 Subject: [PATCH 3/7] Move functionality for normalizing header values to a utility method. --- libcloud/http.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/libcloud/http.py b/libcloud/http.py index 8f41124312..567a2a2a21 100644 --- a/libcloud/http.py +++ b/libcloud/http.py @@ -201,10 +201,8 @@ def verification(self): def request(self, method, url, body=None, headers=None, raw=False, stream=False): url = urlparse.urljoin(self.host, url) - # all headers should be strings - for header, value in headers.items(): - if isinstance(headers[header], int): - headers[header] = str(value) + headers = self._normalize_headers(headers=headers) + self.response = self.session.request( method=method.lower(), url=url, @@ -217,10 +215,8 @@ def request(self, method, url, body=None, headers=None, raw=False, def prepared_request(self, method, url, body=None, headers=None, raw=False, stream=False): - # all headers should be strings - for header, value in headers.items(): - if isinstance(headers[header], int): - headers[header] = str(value) + headers = self._normalize_headers(headers=headers) + req = requests.Request(method, ''.join([self.host, url]), data=body, headers=headers) @@ -262,6 +258,16 @@ def close(self): # pragma: no cover # return connection back to pool self.response.close() + def _normalize_headers(self, headers): + headers = headers or {} + + # all headers should be strings + for key, value in headers.items(): + if isinstance(value, (int, float)): + headers[key] = str(value) + + return headers + class HttpLibResponseProxy(object): """ From 61d68d2cfdb1e8625f7bf444bb87528c5cd7da2b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 17:47:52 +0200 Subject: [PATCH 4/7] Also make sure we normalize all the header values in the connection mock classes. --- libcloud/test/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libcloud/test/__init__.py b/libcloud/test/__init__.py index 6691766d35..645105e4f8 100644 --- a/libcloud/test/__init__.py +++ b/libcloud/test/__init__.py @@ -133,6 +133,7 @@ def _get_request(self, method, url, body=None, headers=None): return meth(method, url, body, headers) def request(self, method, url, body=None, headers=None, raw=False, stream=False): + headers = self._normalize_headers(headers=headers) r_status, r_body, r_headers, r_reason = self._get_request(method, url, body, headers) if r_body is None: r_body = '' @@ -153,6 +154,7 @@ def request(self, method, url, body=None, headers=None, raw=False, stream=False) def prepared_request(self, method, url, body=None, headers=None, raw=False, stream=False): + headers = self._normalize_headers(headers=headers) r_status, r_body, r_headers, r_reason = self._get_request(method, url, body, headers) with requests_mock.mock() as m: From f5dab2432c1a5f48504e6bab9b72960fd0b05a51 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 19:10:53 +0200 Subject: [PATCH 5/7] Add changelog entry. --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 1df56dd99c..bbabae4612 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,15 @@ Compute manner to finish faster. [Tomaz Muraus] +Tests +~~~~~ + +- Make sure we normalize header values and cast all the numbers to string in + base connection classes used by tests. (LIBCLOUD-945, GITHUB-1111) + + Reported by Erich Eckner. + [Tomaz Muraus] + Changes in Apache Libcloud 2.2.0 -------------------------------- From 3a73d08a49cd9ae08224b7c4457e0280148b35d6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 19:13:10 +0200 Subject: [PATCH 6/7] Fix typo. --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index bbabae4612..79f9eff1d7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,7 +25,7 @@ Compute Tests ~~~~~ -- Make sure we normalize header values and cast all the numbers to string in +- Make sure we normalize header values and cast all the numbers to strings in base connection classes used by tests. (LIBCLOUD-945, GITHUB-1111) Reported by Erich Eckner. From eafbc1165e2dab41f4968dbf078dd441d2c07c5a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Sep 2017 20:51:52 +0200 Subject: [PATCH 7/7] Only run code codecov on travis. --- .travis.yml | 2 +- tox.ini | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8109850f5f..fe62c3ed40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: before_script: TOX_ENV=pylint - env: ENV=coverage python: 2.7 - before_script: TOX_ENV=coverage + before_script: TOX_ENV=coverage-travis - env: ENV=docs python: 3.5 before_script: TOX_ENV=docs-travis diff --git a/tox.ini b/tox.ini index f33912e6ff..535522371b 100644 --- a/tox.ini +++ b/tox.ini @@ -91,6 +91,13 @@ deps = -r{toxinidir}/integration/requirements.txt commands = python -m integration [testenv:coverage] +deps = + -r{toxinidir}/requirements-tests.txt +set-env = +commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py + coverage run --source=libcloud setup.py test + +[testenv:coverage-travis] passenv = TOXENV CI TRAVIS TRAVIS_* deps = -r{toxinidir}/requirements-tests.txt