From 14a11676854da2a72a43b833cc54cf5cca311f77 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 18 May 2018 01:22:00 -0700 Subject: [PATCH 1/2] Avoid logging expected errors --- .../azure/storage/blob/baseblobservice.py | 25 ++++- .../azure/storage/common/_error.py | 12 ++- .../azure/storage/common/storageclient.py | 18 +++- .../azure/storage/file/fileservice.py | 31 +++++- .../azure/storage/queue/queueservice.py | 18 +++- tests/blob/test_common_blob.py | 32 +++++- tests/queues/test_queue.py | 22 ++++ ...common_blob.test_blob_snapshot_exists.yaml | 100 ++++++++++++++++++ ...on_blob.test_blob_snapshot_not_exists.yaml | 69 ++++++++++++ ...ueue_already_exist_different_metadata.yaml | 48 +++++++++ ...ueue_fail_on_exist_different_metadata.yaml | 48 +++++++++ tests/testcase.py | 4 +- 12 files changed, 404 insertions(+), 23 deletions(-) create mode 100644 tests/recordings/test_common_blob.test_blob_snapshot_exists.yaml create mode 100644 tests/recordings/test_common_blob.test_blob_snapshot_not_exists.yaml create mode 100644 tests/recordings/test_queue.test_create_queue_already_exist_different_metadata.yaml create mode 100644 tests/recordings/test_queue.test_create_queue_fail_on_exist_different_metadata.yaml diff --git a/azure-storage-blob/azure/storage/blob/baseblobservice.py b/azure-storage-blob/azure/storage/blob/baseblobservice.py index b41b04f7..d598b7a4 100644 --- a/azure-storage-blob/azure/storage/blob/baseblobservice.py +++ b/azure-storage-blob/azure/storage/blob/baseblobservice.py @@ -85,6 +85,10 @@ __version__ as package_version, ) +_CONTAINER_ALREADY_EXISTS_ERROR_CODE = 'ContainerAlreadyExists' +_BLOB_NOT_FOUND_ERROR_CODE = 'BlobNotFound' +_CONTAINER_NOT_FOUND_ERROR_CODE = 'ContainerNotFound' + if sys.version_info >= (3,): from io import BytesIO else: @@ -622,7 +626,7 @@ def create_container(self, container_name, metadata=None, if not fail_on_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=[_CONTAINER_ALREADY_EXISTS_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_on_exist(ex) @@ -1575,10 +1579,21 @@ def exists(self, container_name, blob_name=None, snapshot=None, timeout=None): ''' _validate_not_none('container_name', container_name) try: - if blob_name is None: - self.get_container_properties(container_name, timeout=timeout) - else: - self.get_blob_properties(container_name, blob_name, snapshot=snapshot, timeout=timeout) + # make head request to see if container/blob/snapshot exists + request = HTTPRequest() + request.method = 'GET' if blob_name is None else 'HEAD' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(container_name, blob_name) + request.query = { + 'snapshot': _to_str(snapshot), + 'timeout': _int_to_str(timeout), + 'restype': 'container' if blob_name is None else None, + } + + expected_errors = [_CONTAINER_NOT_FOUND_ERROR_CODE] if blob_name is None \ + else [_CONTAINER_NOT_FOUND_ERROR_CODE, _BLOB_NOT_FOUND_ERROR_CODE] + self._perform_request(request, expected_errors=expected_errors) + return True except AzureHttpError as ex: _dont_fail_not_exist(ex) diff --git a/azure-storage-common/azure/storage/common/_error.py b/azure-storage-common/azure/storage/common/_error.py index a8f67864..621f4bf6 100644 --- a/azure-storage-common/azure/storage/common/_error.py +++ b/azure-storage-common/azure/storage/common/_error.py @@ -104,11 +104,19 @@ def _dont_fail_not_exist(error): def _http_error_handler(http_error): ''' Simple error handler for azure.''' message = str(http_error) + error_code = None + if 'x-ms-error-code' in http_error.respheader: - message += 'ErrorCode: ' + http_error.respheader['x-ms-error-code'] + error_code = http_error.respheader['x-ms-error-code'] + message += ' ErrorCode: ' + error_code + if http_error.respbody is not None: message += '\n' + http_error.respbody.decode('utf-8-sig') - raise AzureHttpError(message, http_error.status) + + ex = AzureHttpError(message, http_error.status) + ex.error_code = error_code + + raise ex def _validate_type_bytes(param_name, param): diff --git a/azure-storage-common/azure/storage/common/storageclient.py b/azure-storage-common/azure/storage/common/storageclient.py index 88eae359..859a729d 100644 --- a/azure-storage-common/azure/storage/common/storageclient.py +++ b/azure-storage-common/azure/storage/common/storageclient.py @@ -14,6 +14,7 @@ import requests from azure.common import ( AzureException, + AzureHttpError, ) from ._constants import ( @@ -209,7 +210,7 @@ def extract_date_and_request_id(retry_context): else: return "" - def _perform_request(self, request, parser=None, parser_args=None, operation_context=None): + def _perform_request(self, request, parser=None, parser_args=None, operation_context=None, expected_errors=None): ''' Sends the request and return response. Catches HTTPError and hands it to error handler @@ -281,7 +282,7 @@ def _perform_request(self, request, parser=None, parser_args=None, operation_con self.extract_date_and_request_id(retry_context), response.status, response.message, - str(request.headers).replace('\n', '')) + str(response.headers).replace('\n', '')) # Parse and wrap HTTP errors in AzureHttpError which inherits from AzureException if response.status >= 300: @@ -325,6 +326,16 @@ def _perform_request(self, request, parser=None, parser_args=None, operation_con status_code = retry_context.response.status if retry_context.response is not None else 'Unknown' timestamp_and_request_id = self.extract_date_and_request_id(retry_context) + # if the http error was expected, we should short-circuit + if isinstance(ex, AzureHttpError) and expected_errors is not None and ex.error_code in expected_errors: + logger.info("%s Received expected http error: " + "%s, HTTP status code=%s, Exception=%s.", + client_request_id_prefix, + timestamp_and_request_id, + status_code, + exception_str_in_one_line) + raise ex + logger.info("%s Operation failed: checking if the operation should be retried. " "Current retry count=%s, %s, HTTP status code=%s, Exception=%s.", client_request_id_prefix, @@ -376,4 +387,5 @@ def _perform_request(self, request, parser=None, parser_args=None, operation_con # note: to cover the emulator scenario, the host_location is grabbed # from request.host_locations(which includes the dev account name) # instead of request.host(which at this point no longer includes the dev account name) - operation_context.host_location = {retry_context.location_mode: request.host_locations[retry_context.location_mode]} + operation_context.host_location = { + retry_context.location_mode: request.host_locations[retry_context.location_mode]} diff --git a/azure-storage-file/azure/storage/file/fileservice.py b/azure-storage-file/azure/storage/file/fileservice.py index 2f8cc0a5..1bf42a6f 100644 --- a/azure-storage-file/azure/storage/file/fileservice.py +++ b/azure-storage-file/azure/storage/file/fileservice.py @@ -83,6 +83,12 @@ __version__ as package_version, ) +_SHARE_NOT_FOUND_ERROR_CODE = 'ShareNotFound' +_PARENT_NOT_FOUND_ERROR_CODE = 'ParentNotFound' +_RESOURCE_NOT_FOUND_ERROR_CODE = 'ResourceNotFound' +_RESOURCE_ALREADY_EXISTS_ERROR_CODE = 'ResourceAlreadyExists' +_SHARE_ALREADY_EXISTS_ERROR_CODE = 'ShareAlreadyExists' + if sys.version_info >= (3,): from io import BytesIO else: @@ -648,7 +654,7 @@ def create_share(self, share_name, metadata=None, quota=None, if not fail_on_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=[_SHARE_ALREADY_EXISTS_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_on_exist(ex) @@ -977,7 +983,7 @@ def create_directory(self, share_name, directory_name, metadata=None, if not fail_on_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=_RESOURCE_ALREADY_EXISTS_ERROR_CODE) return True except AzureHttpError as ex: _dont_fail_on_exist(ex) @@ -1273,12 +1279,27 @@ def exists(self, share_name, directory_name=None, file_name=None, timeout=None, ''' _validate_not_none('share_name', share_name) try: + request = HTTPRequest() + request.method = 'HEAD' if file_name is not None else 'GET' + request.host_locations = self._get_host_locations() + request.path = _get_path(share_name, directory_name, file_name) + if file_name is not None: - self.get_file_properties(share_name, directory_name, file_name, timeout=timeout, snapshot=snapshot) + restype = None + expected_errors = [_RESOURCE_NOT_FOUND_ERROR_CODE, _PARENT_NOT_FOUND_ERROR_CODE] elif directory_name is not None: - self.get_directory_properties(share_name, directory_name, timeout=timeout, snapshot=snapshot) + restype = 'directory' + expected_errors = [_RESOURCE_NOT_FOUND_ERROR_CODE, _SHARE_NOT_FOUND_ERROR_CODE] else: - self.get_share_properties(share_name, timeout=timeout, snapshot=snapshot) + restype = 'share' + expected_errors = [_SHARE_NOT_FOUND_ERROR_CODE] + + request.query = { + 'restype': restype, + 'timeout': _int_to_str(timeout), + 'sharesnapshot': _to_str(snapshot) + } + self._perform_request(request, expected_errors=expected_errors) return True except AzureHttpError as ex: _dont_fail_not_exist(ex) diff --git a/azure-storage-queue/azure/storage/queue/queueservice.py b/azure-storage-queue/azure/storage/queue/queueservice.py index 709272bb..27ee5ff7 100644 --- a/azure-storage-queue/azure/storage/queue/queueservice.py +++ b/azure-storage-queue/azure/storage/queue/queueservice.py @@ -74,6 +74,8 @@ __version__ as package_version, ) +_QUEUE_ALREADY_EXISTS_ERROR_CODE = 'QueueAlreadyExists' +_QUEUE_NOT_FOUND_ERROR_CODE = 'QueueNotFound' _HTTP_RESPONSE_NO_CONTENT = 204 @@ -526,7 +528,8 @@ def _return_request(request): if not fail_on_exist: try: - response = self._perform_request(request, parser=_return_request) + response = self._perform_request(request, parser=_return_request, + expected_errors=[_QUEUE_ALREADY_EXISTS_ERROR_CODE]) if response.status == _HTTP_RESPONSE_NO_CONTENT: return False return True @@ -644,8 +647,19 @@ def exists(self, queue_name, timeout=None): :return: A boolean indicating whether the queue exists. :rtype: bool ''' + _validate_not_none('queue_name', queue_name) + try: - self.get_queue_metadata(queue_name, timeout=timeout) + request = HTTPRequest() + request.method = 'GET' + request.host_locations = self._get_host_locations(secondary=True) + request.path = _get_path(queue_name) + request.query = { + 'comp': 'metadata', + 'timeout': _int_to_str(timeout), + } + + self._perform_request(request, expected_errors=[_QUEUE_NOT_FOUND_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_not_exist(ex) diff --git a/tests/blob/test_common_blob.py b/tests/blob/test_common_blob.py index 89ec38b8..0189c26c 100644 --- a/tests/blob/test_common_blob.py +++ b/tests/blob/test_common_blob.py @@ -141,23 +141,47 @@ def test_blob_exists(self): self.assertTrue(exists) @record - def test_blob_container_not_exists(self): + def test_blob_not_exists(self): # Arrange blob_name = self._get_blob_reference() # Act - exists = self.bs.exists(self._get_container_reference(), blob_name) + exists = self.bs.exists(self.container_name, blob_name) # Assert self.assertFalse(exists) @record - def test_blob_not_exists(self): + def test_blob_snapshot_exists(self): + # Arrange + blob_name = self._create_block_blob() + snapshot = self.bs.snapshot_blob(self.container_name, blob_name) + + # Act + exists = self.bs.exists(self.container_name, blob_name, snapshot.snapshot) + + # Assert + self.assertTrue(exists) + + @record + def test_blob_snapshot_not_exists(self): + # Arrange + blob_name = self._create_block_blob() + + # Act + exists = self.bs.exists(self.container_name, blob_name, "1988-08-18T07:52:31.6690068Z") + + # Assert + self.assertFalse(exists) + + @record + def test_blob_container_not_exists(self): + # In this case both the blob and container do not exist # Arrange blob_name = self._get_blob_reference() # Act - exists = self.bs.exists(self.container_name, blob_name) + exists = self.bs.exists(self._get_container_reference(), blob_name) # Assert self.assertFalse(exists) diff --git a/tests/queues/test_queue.py b/tests/queues/test_queue.py index 8718a5fe..888aecc6 100644 --- a/tests/queues/test_queue.py +++ b/tests/queues/test_queue.py @@ -105,6 +105,28 @@ def test_create_queue_fail_on_exist(self): # Asserts self.assertTrue(created) + @record + def test_create_queue_already_exist_different_metadata(self): + # Action + queue_name = self._get_queue_reference() + created1 = self.qs.create_queue(queue_name) + created2 = self.qs.create_queue(queue_name, {"val": "value"}) + + # Asserts + self.assertTrue(created1) + self.assertFalse(created2) + + @record + def test_create_queue_fail_on_exist_different_metadata(self): + # Action + queue_name = self._get_queue_reference() + created = self.qs.create_queue(queue_name, None, True) + with self.assertRaises(AzureConflictHttpError): + self.qs.create_queue(queue_name, {"val": "value"}, True) + + # Asserts + self.assertTrue(created) + @record def test_create_queue_with_options(self): # Action diff --git a/tests/recordings/test_common_blob.test_blob_snapshot_exists.yaml b/tests/recordings/test_common_blob.test_blob_snapshot_exists.yaml new file mode 100644 index 00000000..68b29b51 --- /dev/null +++ b/tests/recordings/test_common_blob.test_blob_snapshot_exists.yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: !!binary | + GouZRVe/gTjfwn0LpBMqDU3mQpiqiyKYNYGThiXlHXRms8hhQLJK5AbKg3zeej42Mdh5uXsipP3j + yW5zEywx3ljRMi/HOb68CKa46640OrGpRfsArakeZJsfy5Y3pc/dhZnak8kHzuEYNXMcnMn9iopH + YApQe5ZgcPkJ2mlaiL1bJe942Ht652WaW2dbBNq7XHw7B2grvmocsVYLuBKKF4buIZTr9Sf40/Z0 + 8EdSX5CmEf41wMTuvFWc2i8jeBx7URtZz19ozVNrsQHsmPzM4ml0ekD+MjxGEM7Z5+odMEkaVlHx + YOnnuo+Yt7szbjJEOUrSRqc0jJXLuLlwQu5fL38/ed0edc4f2Fx9Gv10e+bK5+n5u3xLwGmkmpee + 1kVMB45b8HLDFxZJJvhPDfQrE8O3pVKXEeRG3TYVZ33hsPTKdS5+QjkIbacaxBvWzQFBnCSGGaUf + Z01Clw4cNZNNIOuNDkwFKfx5GTzqou+ggVwkM3JCp0WoiaGNbhfDb6uQU4/0rkNcw4fVlqhSY1Hd + c6VzEoknNpWpdyzQFVjYyMfPfU7QUGcCzOEua71aguwt25SydCbu7phwqoQsTa6VSC0RcFFKfx9i + 8DDnyBPfMjPUh2OE1rvOrVaUq7zy7uB5wU4R+rpO121sWrYSJAMGqPre8S7jSKh3ILwCISIQNes9 + GxhPui3Tk3jlVTnMl+Wi4HvXWzFgzeXm55yi88k336uDjocN2lVEDBWf3Yxlz/qQcyFMfWKP/Nl7 + OWRHKknUFiXUPnp/d4jxj+vdELKZLBoY55VrFJFakR2T521BgFiKgdzbLdd6U/75hZzVxBLWt8lH + oOfFDEjOIFdgn7m8N4a/jIT7INXfEPvSew28pL/2bdRuir9pncoPHpnsqu87FvoJWRxI2qffh2fj + U6mzPkAbNzn2VmAuX1Wuv4oNiuZj4cIBHe70zycHFu5oCl4/dVvcyNI7GkZMQp8G2+XpEPxgQP04 + uODVZOjkUr/6HhKTfOVFfM/4JjGtMryO0Z94ARYHOYRBsFqOCSKe9QmRCQ/9RyRothgBx3raJzu/ + o2x43hGJUIz76CQDFq70uGbz1HYTu7DnYB4e6ZXMDWWcAODNMxCm7gf+dTfNsOFZA+L2O2Hvkk0V + d7cU1TvPLrQoHVqacZDgEtLaumisQrNYnX/BcXYPhcjAGTCJ6w1ge59ICB98pUTAvxEfJyWQBKef + 9eeiHObjpnMCNz/oEYQZDRGTbfJYsdfYIg2sLY+FhW4tKk0MHi7idMCdaGSf+bavCnCMpMe/7Q4h + hLrrA1xdT2H+RWd77HJcnhwj5rk1V6PqJG98Unk8Afz7CP5mmJyiKZ+TYClkY9rpjDKq3GgVmg== + headers: + Connection: [keep-alive] + Content-Length: ['1024'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-blob-type: [BlockBlob] + x-ms-client-request-id: [1a751bba-5f27-11e8-ab94-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:18 GMT'] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.blob.core.windows.net/utcontainer72d31161/blob72d31161 + response: + body: {string: !!python/unicode ''} + headers: + content-md5: [LV9fUZsBRmjgoDiPQ4fVQQ==] + date: ['Thu, 24 May 2018 07:50:18 GMT'] + etag: ['"0x8D5C14AFEC203A0"'] + last-modified: ['Thu, 24 May 2018 07:50:18 GMT'] + server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0] + transfer-encoding: [chunked] + x-ms-request-id: [5c3c08d5-a01e-013e-4b33-f3a4a0000000] + x-ms-request-server-encrypted: ['true'] + x-ms-version: ['2017-11-09'] + status: {code: 201, message: Created} +- request: + body: null + headers: + Connection: [keep-alive] + Content-Length: ['0'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [1a989087-5f27-11e8-bcc7-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:18 GMT'] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.blob.core.windows.net/utcontainer72d31161/blob72d31161?comp=snapshot + response: + body: {string: !!python/unicode ''} + headers: + date: ['Thu, 24 May 2018 07:50:18 GMT'] + etag: ['"0x8D5C14AFEC203A0"'] + last-modified: ['Thu, 24 May 2018 07:50:18 GMT'] + server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0] + transfer-encoding: [chunked] + x-ms-request-id: [5c3c090c-a01e-013e-7f33-f3a4a0000000] + x-ms-snapshot: ['2018-05-24T07:50:18.9642369Z'] + x-ms-version: ['2017-11-09'] + status: {code: 201, message: Created} +- request: + body: null + headers: + Connection: [keep-alive] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [1aa02e4a-5f27-11e8-935a-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:18 GMT'] + x-ms-version: ['2017-11-09'] + method: HEAD + uri: https://storagename.blob.core.windows.net/utcontainer72d31161/blob72d31161?snapshot=2018-05-24T07%3A50%3A18.9642369Z + response: + body: {string: !!python/unicode ''} + headers: + accept-ranges: [bytes] + access-control-allow-origin: ['*'] + access-control-expose-headers: [x-ms-client-request-id] + content-length: ['1024'] + content-md5: [LV9fUZsBRmjgoDiPQ4fVQQ==] + content-type: [application/octet-stream] + date: ['Thu, 24 May 2018 07:50:18 GMT'] + etag: ['"0x8D5C14AFEC203A0"'] + last-modified: ['Thu, 24 May 2018 07:50:18 GMT'] + server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0] + x-ms-blob-type: [BlockBlob] + x-ms-creation-time: ['Thu, 24 May 2018 07:50:18 GMT'] + x-ms-request-id: [5c3c092a-a01e-013e-1833-f3a4a0000000] + x-ms-server-encrypted: ['true'] + x-ms-version: ['2017-11-09'] + status: {code: 200, message: OK} +version: 1 diff --git a/tests/recordings/test_common_blob.test_blob_snapshot_not_exists.yaml b/tests/recordings/test_common_blob.test_blob_snapshot_not_exists.yaml new file mode 100644 index 00000000..ade5093a --- /dev/null +++ b/tests/recordings/test_common_blob.test_blob_snapshot_not_exists.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: !!binary | + NXUGO56Thf0hqDDEqLhe4SHaEuk2z1oE7ZU9KF7XvwGWooXFPI35ZNPSvRu0PU2jUTPttOJS1N4e + omyMiz0JiSw0+h2LynVfQIEhe0vdb497Fqcfdo4SZqjp+FhwIuA633nW9bntjAl6y86cRzeBC54W + FySCtHegLjG0TtzfMFvLvppbf6tFFq+WVQm/QHf1REIKSG8ntO2RybnWxwTy1q+DZS0xD5Yx2DVb + kNMs4VTRXw/aCn8PHmnjaSJlJkCPK+kJz62sUGxL46IDSPccrn0N1R5QSQ7Zm4axR+4y8pagD2xT + zAviRDYr1eXUqE5iACzNw27s/lZZTiOncx2kowRTu6PU8oSCmhuFIhyE5n8IjdrqTxo7R194Nwxx + +glvFuGHiN/gwUW8JtlDF17spZmNzk+HSAyNQahLcU+sWnFq2Y5XHTPMxbXcpzoint56Bcx+q2sr + 4EGQ99x1eL40jTuuqC3sWguWY7ehKtJPhyl7aZkx9ZyS4PyzJDKA10QmnwtGF4NSWwl6e9tv7LAM + 6owz53sqM8VkNVvQo+PyvwzVWe2TUs2Q45VhJOlfDO22L9PZQezu/bVvPvXgK6BrIAHnIMUkJG8x + Xqfo7bhH11Y/3bapQS48HJ4JRadtMZ99+ofoywjkYQV/NDxsGpDdNtrR0LaWLvWmsPromEvWIEb2 + nEuLzm2TaB26UedbxiTplSoAq30RjpnVcew0wAQfw+Sr0eEXl/fmz0xsLx0abGBQJda3Pe6xWMKz + sTugw4xwckpIo1ZDFQhMCQBSBX9tx3wSkuB24FvH8ESdO9II92w6pwWok64LABZMG7u2Sqi+vfcd + rjjXaj/xUFWmfFApD911RfFcSPruRM3SLq9U+iHAtNaTwauuSteRcfurUiuT+oiVeYQMRmKX9xo9 + abY31IEHpP0DMcp4nnXsqugWqBrVock/Txz8Mf4XSEptTXfTp4fXB7AUzNmWp8nfr/78ZXiSqWPO + wjiopldUoDdQSFMDLhgRgTFJHOpyNIjKj5E6NCNAPcGh0BBHMlRW67kypwsHm/qVhTiyzD8UEHRH + qgxLYXSOXWAlan8IYgnM6LQKZaxyDUNuqmEJC8ifKwHvTkfNQrMLhYjGDqdS8Xw40KZ74dvqR3az + TKujmdbM7df+FvtO4cRgr0EsCef4QGbrQGeSPysaRy918gcU20I5ymLVrYqp1pg5+n3OhneIGm2U + gRwowGTNUhmhZZh0Q9MsYUV129rgNOm0qwHobNUmAmRZoOpfOww0TClRwGyTd4ZCvvPlbloIVY9F + lAnnT39EU4raG/w/cQEJJOhEUM2Sn9sTggxVsoIUAVCgjawhPxicGBJG+W/z28awftxys3wNSw== + headers: + Connection: [keep-alive] + Content-Length: ['1024'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-blob-type: [BlockBlob] + x-ms-client-request-id: [1c739054-5f27-11e8-84f0-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:22 GMT'] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.blob.core.windows.net/utcontainerbc431311/blobbc431311 + response: + body: {string: !!python/unicode ''} + headers: + content-md5: [zBAlRPUsu32smIvbseKKPw==] + date: ['Thu, 24 May 2018 07:50:21 GMT'] + etag: ['"0x8D5C14B00BE583D"'] + last-modified: ['Thu, 24 May 2018 07:50:22 GMT'] + server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0] + transfer-encoding: [chunked] + x-ms-request-id: [8ee71281-f01e-0126-7b33-f38935000000] + x-ms-request-server-encrypted: ['true'] + x-ms-version: ['2017-11-09'] + status: {code: 201, message: Created} +- request: + body: null + headers: + Connection: [keep-alive] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [1c8cc9f5-5f27-11e8-a174-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:22 GMT'] + x-ms-version: ['2017-11-09'] + method: HEAD + uri: https://storagename.blob.core.windows.net/utcontainerbc431311/blobbc431311?snapshot=1988-08-18T07%3A52%3A31.6690068Z + response: + body: {string: !!python/unicode ''} + headers: + access-control-allow-origin: ['*'] + access-control-expose-headers: [x-ms-client-request-id] + date: ['Thu, 24 May 2018 07:50:21 GMT'] + server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0] + transfer-encoding: [chunked] + x-ms-error-code: [BlobNotFound] + x-ms-request-id: [8ee71297-f01e-0126-0e33-f38935000000] + x-ms-version: ['2017-11-09'] + status: {code: 404, message: The specified blob does not exist.} +version: 1 diff --git a/tests/recordings/test_queue.test_create_queue_already_exist_different_metadata.yaml b/tests/recordings/test_queue.test_create_queue_already_exist_different_metadata.yaml new file mode 100644 index 00000000..754e04f6 --- /dev/null +++ b/tests/recordings/test_queue.test_create_queue_already_exist_different_metadata.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: null + headers: + Connection: [keep-alive] + Content-Length: ['0'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [121d104f-5f27-11e8-871b-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:04 GMT'] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.queue.core.windows.net/queuea35190d + response: + body: {string: !!python/unicode ''} + headers: + date: ['Thu, 24 May 2018 07:50:04 GMT'] + server: [Windows-Azure-Queue/1.0 Microsoft-HTTPAPI/2.0] + transfer-encoding: [chunked] + x-ms-request-id: [2b0911f5-0003-0037-4433-f3f87b000000] + x-ms-version: ['2017-11-09'] + status: {code: 201, message: Created} +- request: + body: null + headers: + Connection: [keep-alive] + Content-Length: ['0'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [1254ffb0-5f27-11e8-909d-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:05 GMT'] + x-ms-meta-val: [value] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.queue.core.windows.net/queuea35190d + response: + body: {string: "\uFEFFQueueAlreadyExistsThe + specified queue already exists.\nRequestId:2b09121e-0003-0037-6a33-f3f87b000000\nTime:2018-05-24T07:50:05.0982733Z"} + headers: + content-length: ['222'] + content-type: [application/xml] + date: ['Thu, 24 May 2018 07:50:04 GMT'] + server: [Windows-Azure-Queue/1.0 Microsoft-HTTPAPI/2.0] + x-ms-error-code: [QueueAlreadyExists] + x-ms-request-id: [2b09121e-0003-0037-6a33-f3f87b000000] + x-ms-version: ['2017-11-09'] + status: {code: 409, message: The specified queue already exists.} +version: 1 diff --git a/tests/recordings/test_queue.test_create_queue_fail_on_exist_different_metadata.yaml b/tests/recordings/test_queue.test_create_queue_fail_on_exist_different_metadata.yaml new file mode 100644 index 00000000..463b1991 --- /dev/null +++ b/tests/recordings/test_queue.test_create_queue_fail_on_exist_different_metadata.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: null + headers: + Connection: [keep-alive] + Content-Length: ['0'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [13b79ad4-5f27-11e8-be77-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:07 GMT'] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.queue.core.windows.net/queue9101903 + response: + body: {string: !!python/unicode ''} + headers: + date: ['Thu, 24 May 2018 07:50:09 GMT'] + server: [Windows-Azure-Queue/1.0 Microsoft-HTTPAPI/2.0] + transfer-encoding: [chunked] + x-ms-request-id: [8734b47d-8003-00c3-7533-f3dd97000000] + x-ms-version: ['2017-11-09'] + status: {code: 201, message: Created} +- request: + body: null + headers: + Connection: [keep-alive] + Content-Length: ['0'] + User-Agent: [Azure-Storage/1.2.0rc1-1.2.0rc1 (Python CPython 2.7.10; Darwin + 17.5.0)] + x-ms-client-request-id: [14ef75ca-5f27-11e8-aa66-b8e8564491f6] + x-ms-date: ['Thu, 24 May 2018 07:50:09 GMT'] + x-ms-meta-val: [value] + x-ms-version: ['2017-11-09'] + method: PUT + uri: https://storagename.queue.core.windows.net/queue9101903 + response: + body: {string: "\uFEFFQueueAlreadyExistsThe + specified queue already exists.\nRequestId:8734b51a-8003-00c3-7833-f3dd97000000\nTime:2018-05-24T07:50:09.4788588Z"} + headers: + content-length: ['222'] + content-type: [application/xml] + date: ['Thu, 24 May 2018 07:50:09 GMT'] + server: [Windows-Azure-Queue/1.0 Microsoft-HTTPAPI/2.0] + x-ms-error-code: [QueueAlreadyExists] + x-ms-request-id: [8734b51a-8003-00c3-7833-f3dd97000000] + x-ms-version: ['2017-11-09'] + status: {code: 409, message: The specified queue already exists.} +version: 1 diff --git a/tests/testcase.py b/tests/testcase.py index 32acad28..efd9c6ae 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -25,8 +25,8 @@ # logging is not enabled by default because it pollutes the CI logs # uncommenting the following two lines make debugging much easier -# import logging -# logging.basicConfig(format='%(asctime)s %(name)-20s %(levelname)-5s %(message)s', level=logging.INFO) +import logging +logging.basicConfig(format='%(asctime)s %(name)-20s %(levelname)-5s %(message)s', level=logging.INFO) try: import tests.settings_real as settings From 5c9b2dba1b7c263e727ee302dc5a5e13097fc501 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 5 Jun 2018 22:58:11 -0700 Subject: [PATCH 2/2] Added unit test to validate that expected errors are not printed --- .../azure/storage/blob/baseblobservice.py | 2 +- .../azure/storage/file/fileservice.py | 4 +- .../azure/storage/queue/queueservice.py | 2 +- tests/blob/test_container.py | 22 ++++++-- tests/file/test_directory.py | 21 +++++-- tests/file/test_share.py | 27 +++++++-- tests/queues/test_queue.py | 23 ++++++-- tests/settings_fake.py | 4 ++ tests/testcase.py | 56 +++++++++++++++++-- tool_clean_build_files.sh | 4 ++ 10 files changed, 136 insertions(+), 29 deletions(-) create mode 100755 tool_clean_build_files.sh diff --git a/azure-storage-blob/azure/storage/blob/baseblobservice.py b/azure-storage-blob/azure/storage/blob/baseblobservice.py index d598b7a4..bb0e4645 100644 --- a/azure-storage-blob/azure/storage/blob/baseblobservice.py +++ b/azure-storage-blob/azure/storage/blob/baseblobservice.py @@ -877,7 +877,7 @@ def delete_container(self, container_name, fail_not_exist=False, if not fail_not_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=[_CONTAINER_NOT_FOUND_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_not_exist(ex) diff --git a/azure-storage-file/azure/storage/file/fileservice.py b/azure-storage-file/azure/storage/file/fileservice.py index 1bf42a6f..2870fefc 100644 --- a/azure-storage-file/azure/storage/file/fileservice.py +++ b/azure-storage-file/azure/storage/file/fileservice.py @@ -935,7 +935,7 @@ def delete_share(self, share_name, fail_not_exist=False, timeout=None, snapshot= if not fail_not_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=[_SHARE_NOT_FOUND_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_not_exist(ex) @@ -1029,7 +1029,7 @@ def delete_directory(self, share_name, directory_name, if not fail_not_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=[_RESOURCE_NOT_FOUND_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_not_exist(ex) diff --git a/azure-storage-queue/azure/storage/queue/queueservice.py b/azure-storage-queue/azure/storage/queue/queueservice.py index 27ee5ff7..156af56d 100644 --- a/azure-storage-queue/azure/storage/queue/queueservice.py +++ b/azure-storage-queue/azure/storage/queue/queueservice.py @@ -574,7 +574,7 @@ def delete_queue(self, queue_name, fail_not_exist=False, timeout=None): request.query = {'timeout': _int_to_str(timeout)} if not fail_not_exist: try: - self._perform_request(request) + self._perform_request(request, expected_errors=[_QUEUE_NOT_FOUND_ERROR_CODE]) return True except AzureHttpError as ex: _dont_fail_not_exist(ex) diff --git a/tests/blob/test_container.py b/tests/blob/test_container.py index 778c7270..d4123bb0 100644 --- a/tests/blob/test_container.py +++ b/tests/blob/test_container.py @@ -13,7 +13,7 @@ Include, PublicAccess) from azure.storage.common import AccessPolicy -from tests.testcase import StorageTestCase, TestMode, record +from tests.testcase import StorageTestCase, TestMode, record, LogCaptured #------------------------------------------------------------------------------ TEST_CONTAINER_PREFIX = 'container' @@ -160,7 +160,11 @@ def test_container_not_exists(self): container_name = self._get_container_reference() # Act - exists = self.bs.exists(container_name) + with LogCaptured(self) as log_captured: + exists = self.bs.exists(container_name) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(exists) @@ -664,7 +668,11 @@ def test_delete_container_with_non_existing_container(self): container_name = self._get_container_reference() # Act - deleted = self.bs.delete_container(container_name) + with LogCaptured(self) as log_captured: + deleted = self.bs.delete_container(container_name) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(deleted) @@ -675,8 +683,12 @@ def test_delete_container_with_non_existing_container_fail_not_exist(self): container_name = self._get_container_reference() # Act - with self.assertRaises(AzureMissingResourceHttpError): - self.bs.delete_container(container_name, True) + with LogCaptured(self) as log_captured: + with self.assertRaises(AzureMissingResourceHttpError): + self.bs.delete_container(container_name, True) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' in log_as_str) # Assert diff --git a/tests/file/test_directory.py b/tests/file/test_directory.py index 66cea173..8e40ccf7 100644 --- a/tests/file/test_directory.py +++ b/tests/file/test_directory.py @@ -19,6 +19,7 @@ from tests.testcase import ( StorageTestCase, record, + LogCaptured, ) @@ -166,7 +167,11 @@ def test_directory_not_exists(self): # Arrange # Act - exists = self.fs.exists(self.share_name, 'missing') + with LogCaptured(self) as log_captured: + exists = self.fs.exists(self.share_name, 'missing') + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(exists) @@ -240,7 +245,11 @@ def test_delete_directory_with_non_existing_directory(self): # Arrange # Act - deleted = self.fs.delete_directory(self.share_name, 'dir1', False) + with LogCaptured(self) as log_captured: + deleted = self.fs.delete_directory(self.share_name, 'dir1', False) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(deleted) @@ -250,8 +259,12 @@ def test_delete_directory_with_non_existing_directory_fail_not_exist(self): # Arrange # Act - with self.assertRaises(AzureMissingResourceHttpError): - self.fs.delete_directory(self.share_name, 'dir1', True) + with LogCaptured(self) as log_captured: + with self.assertRaises(AzureMissingResourceHttpError): + self.fs.delete_directory(self.share_name, 'dir1', True) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' in log_as_str) # Assert diff --git a/tests/file/test_share.py b/tests/file/test_share.py index 1a80abdf..dfaa632e 100644 --- a/tests/file/test_share.py +++ b/tests/file/test_share.py @@ -28,6 +28,7 @@ StorageTestCase, TestMode, record, + LogCaptured, ) # ------------------------------------------------------------------------------ @@ -224,7 +225,11 @@ def test_share_not_exists(self): share_name = self._get_share_reference() # Act - exists = self.fs.exists(share_name) + with LogCaptured(self) as log_captured: + exists = self.fs.exists(share_name) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(exists) @@ -248,7 +253,11 @@ def test_share_snapshot_not_exists(self): made_up_snapshot = '2017-07-19T06:53:46.0000000Z' # Act - exists = self.fs.exists(share_name, snapshot=made_up_snapshot) + with LogCaptured(self) as log_captured: + exists = self.fs.exists(share_name, snapshot=made_up_snapshot) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(exists) @@ -475,7 +484,11 @@ def test_delete_share_with_non_existing_share(self): share_name = self._get_share_reference() # Act - deleted = self.fs.delete_share(share_name) + with LogCaptured(self) as log_captured: + deleted = self.fs.delete_share(share_name) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(deleted) @@ -486,10 +499,12 @@ def test_delete_share_with_non_existing_share_fail_not_exist(self): share_name = self._get_share_reference() # Act - with self.assertRaises(AzureMissingResourceHttpError): - self.fs.delete_share(share_name, True) + with LogCaptured(self) as log_captured: + with self.assertRaises(AzureMissingResourceHttpError): + self.fs.delete_share(share_name, True) - # Assert + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' in log_as_str) @record def test_get_share_stats(self): diff --git a/tests/queues/test_queue.py b/tests/queues/test_queue.py index 888aecc6..21a3fb4e 100644 --- a/tests/queues/test_queue.py +++ b/tests/queues/test_queue.py @@ -34,6 +34,7 @@ StorageTestCase, TestMode, record, + LogCaptured, ) # ------------------------------------------------------------------------------ @@ -146,7 +147,12 @@ def test_create_queue_with_options(self): def test_delete_queue_not_exist(self): # Action queue_name = self._get_queue_reference() - deleted = self.qs.delete_queue(queue_name) + + with LogCaptured(self) as log_captured: + deleted = self.qs.delete_queue(queue_name) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Asserts self.assertFalse(deleted) @@ -155,10 +161,13 @@ def test_delete_queue_not_exist(self): def test_delete_queue_fail_not_exist_not_exist(self): # Action queue_name = self._get_queue_reference() - with self.assertRaises(AzureMissingResourceHttpError): - self.qs.delete_queue(queue_name, True) - # Asserts + with LogCaptured(self) as log_captured: + with self.assertRaises(AzureMissingResourceHttpError): + self.qs.delete_queue(queue_name, True) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' in log_as_str) @record def test_delete_queue_fail_not_exist_already_exist(self): @@ -269,7 +278,11 @@ def test_queue_not_exists(self): # Arrange # Act - exists = self.qs.exists(self.get_resource_name('missing')) + with LogCaptured(self) as log_captured: + exists = self.qs.exists(self.get_resource_name('missing')) + + log_as_str = log_captured.getvalue() + self.assertTrue('ERROR' not in log_as_str) # Assert self.assertFalse(exists) diff --git a/tests/settings_fake.py b/tests/settings_fake.py index 0ba12938..9bba6947 100644 --- a/tests/settings_fake.py +++ b/tests/settings_fake.py @@ -41,6 +41,10 @@ # - RunLiveNoRecord: run tests against live storage without altering recordings TEST_MODE = 'RunLiveNoRecord' +# Set to true to enable logging for the tests +# logging is not enabled by default because it pollutes the CI logs +ENABLE_LOGGING = False + # Set up proxy support USE_PROXY = False PROXY_HOST = "192.168.15.116" diff --git a/tests/testcase.py b/tests/testcase.py index efd9c6ae..bd2c30ab 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -22,17 +22,20 @@ import sys import random import tests.settings_fake as fake_settings - -# logging is not enabled by default because it pollutes the CI logs -# uncommenting the following two lines make debugging much easier import logging -logging.basicConfig(format='%(asctime)s %(name)-20s %(levelname)-5s %(message)s', level=logging.INFO) + +try: + from cStringIO import StringIO # Python 2 +except ImportError: + from io import StringIO try: import tests.settings_real as settings except ImportError: settings = None +LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' + class TestMode(object): none = 'None'.lower() # this will be for unit test, no need for any recordings @@ -78,6 +81,18 @@ def setUp(self): self._testMethodName, ) + # enable logging if desired + self.configure_logging() + + def configure_logging(self): + if self.settings.ENABLE_LOGGING: + logging.basicConfig(format=LOGGING_FORMAT, level=logging.INFO) + else: + # disable logging if the user does not want to see them + logger = logging.getLogger('azure.storage') + logger.propagate = False + logger.level = logging.CRITICAL + def sleep(self, seconds): if not self.is_playback(): time.sleep(seconds) @@ -375,4 +390,35 @@ def override_first_status(self, response): def override_status(self, response): if response.status == self.status: response.status = self.new_status - self.count += 1 \ No newline at end of file + self.count += 1 + + +class LogCaptured(object): + def __init__(self, test_case=None): + # accept the test case so that we may reset logging after capturing logs + self.test_case = test_case + + def __enter__(self): + # create a string stream to send the logs to + self.log_stream = StringIO() + + # the handler needs to be stored so that we can remove it later + self.handler = logging.StreamHandler(self.log_stream) + self.handler.setFormatter(logging.Formatter(LOGGING_FORMAT)) + + # get and enable the logger to send the outputs to the string stream + self.logger = logging.getLogger('azure.storage') + self.logger.level = logging.INFO + self.logger.addHandler(self.handler) + + # the stream is returned to the user so that the capture logs can be retrieved + return self.log_stream + + def __exit__(self, exc_type, exc_val, exc_tb): + # stop the handler, and close the stream to exit + self.logger.removeHandler(self.handler) + self.log_stream.close() + + # reset logging since we messed with the setting + if self.test_case is not None: + self.test_case.configure_logging() diff --git a/tool_clean_build_files.sh b/tool_clean_build_files.sh new file mode 100755 index 00000000..8c97a3a6 --- /dev/null +++ b/tool_clean_build_files.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# clean up all build files that were generated by setup.py +rm -vrf ./azure-storage-*/build ./azure-storage-*/*.egg-info ./dist