From 9e1dfe12413b8104564ff76c8283ce93276a0bc7 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Tue, 26 Feb 2019 14:34:58 -0500 Subject: [PATCH] [LIBCLOUD-1037] Add Azurite support for Azure Blob Storage driver This requires the following main changes: - Protect against a missing content md5 since the Azurite emulator doesn't set this value. - Protect against a missing metadata tag since the Azurite emulator doesn't set this value. - Implement routing to a specific account via URL prefix (e.g. /someaccount) instead of hostname (e.g. someaccount.blob.core.windows.net). --- .../storage/azure/instantiate_azurite.py | 11 +++++ docs/storage/drivers/azure_blobs.rst | 16 +++++++ libcloud/storage/drivers/azure_blobs.py | 43 +++++++++++++++--- libcloud/test/secrets.py-dist | 1 + .../azurite_blobs/list_containers_1.xml | 25 +++++++++++ .../azurite_blobs/list_containers_2.xml | 26 +++++++++++ .../azurite_blobs/list_containers_empty.xml | 8 ++++ .../fixtures/azurite_blobs/list_objects_1.xml | 45 +++++++++++++++++++ .../fixtures/azurite_blobs/list_objects_2.xml | 37 +++++++++++++++ .../azurite_blobs/list_objects_empty.xml | 6 +++ libcloud/test/storage/test_azure_blobs.py | 19 ++++++++ 11 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 docs/examples/storage/azure/instantiate_azurite.py create mode 100644 libcloud/test/storage/fixtures/azurite_blobs/list_containers_1.xml create mode 100644 libcloud/test/storage/fixtures/azurite_blobs/list_containers_2.xml create mode 100644 libcloud/test/storage/fixtures/azurite_blobs/list_containers_empty.xml create mode 100644 libcloud/test/storage/fixtures/azurite_blobs/list_objects_1.xml create mode 100644 libcloud/test/storage/fixtures/azurite_blobs/list_objects_2.xml create mode 100644 libcloud/test/storage/fixtures/azurite_blobs/list_objects_empty.xml diff --git a/docs/examples/storage/azure/instantiate_azurite.py b/docs/examples/storage/azure/instantiate_azurite.py new file mode 100644 index 0000000000..e3753007ee --- /dev/null +++ b/docs/examples/storage/azure/instantiate_azurite.py @@ -0,0 +1,11 @@ +from libcloud.storage.types import Provider +from libcloud.storage.providers import get_driver + +cls = get_driver(Provider.AZURE_BLOBS) + +driver = cls(key='devstoreaccount1', + secret='Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6' + 'IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', + host='localhost', + port=10000, + secure=False) diff --git a/docs/storage/drivers/azure_blobs.rst b/docs/storage/drivers/azure_blobs.rst index ac9e881ed1..a453e79302 100644 --- a/docs/storage/drivers/azure_blobs.rst +++ b/docs/storage/drivers/azure_blobs.rst @@ -34,3 +34,19 @@ below. .. literalinclude:: /examples/storage/azure/instantiate.py :language: python + +Connecting to self-hosted Azure Storage implementations +------------------------------------------------------- + +To facilitate integration testing, libcloud supports connecting to the +`Azurite storage emulator`_. After starting the emulator, you can +instantiate the driver as shown below. + +.. literalinclude:: /examples/storage/azure/instantiate_azurite.py + :language: python + +This instantiation strategy can also be adapted to connect to other self-hosted +Azure Storage implementations such as `Azure Blob Storage on IoT Edge`_. + +.. _`Azurite storage emulator`: https://github.com/Azure/Azurite +.. _`Azure Blob Storage on IoT Edge`: https://docs.microsoft.com/en-us/azure/iot-edge/how-to-store-data-blob diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py index 96578cf627..b7db039316 100644 --- a/libcloud/storage/drivers/azure_blobs.py +++ b/libcloud/storage/drivers/azure_blobs.py @@ -154,8 +154,35 @@ def __exit__(self, type, value, traceback): class AzureBlobsConnection(AzureConnection): """ - Represents a single connection to Azure Blobs + Represents a single connection to Azure Blobs. + + The main Azure Blob Storage service uses a prefix in the hostname to + distinguish between accounts, e.g. ``theaccount.blob.core.windows.net``. + However, some custom deployments of the service, such as the Azurite + emulator, instead use a URL prefix such as ``/theaccount``. To support + these deployments, the parameter ``account_prefix`` must be set on the + connection. This is done by instantiating the driver with arguments such + as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom + host without an account prefix, e.g. for use-cases where the custom host + implements an auditing proxy or similar, the driver can be instantiated + with ``host='theaccount.somewhere.tld'`` and ``key=''``. + + :param account_prefix: Optional prefix identifying the sotrage account. + Used when connecting to a custom deployment of the + storage service like Azurite or IoT Edge Storage. + :type account_prefix: ``str`` """ + def __init__(self, *args, **kwargs): + self.account_prefix = kwargs.pop('account_prefix', None) + super(AzureBlobsConnection, self).__init__(*args, **kwargs) + + def morph_action_hook(self, action): + action = super(AzureBlobsConnection, self).morph_action_hook(action) + + if self.account_prefix is not None: + action = '/%s%s' % (self.account_prefix, action) + + return action # this is the minimum api version supported by storage accounts of kinds # StorageV2, Storage and BlobStorage @@ -188,6 +215,8 @@ def _ex_connection_class_kwargs(self): # host argument has precedence if not self._host_argument_set: result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX) + else: + result['account_prefix'] = self.key return result @@ -218,8 +247,9 @@ def _xml_to_container(self, node): 'meta_data': {} } - for meta in list(metadata): - extra['meta_data'][meta.tag] = meta.text + if metadata is not None: + for meta in list(metadata): + extra['meta_data'][meta.tag] = meta.text return Container(name=name, extra=extra, driver=self) @@ -303,8 +333,9 @@ def _xml_to_object(self, container, blob): extra['md5_hash'] = value meta_data = {} - for meta in list(metadata): - meta_data[meta.tag] = meta.text + if metadata is not None: + for meta in list(metadata): + meta_data[meta.tag] = meta.text return Object(name=name, size=size, hash=etag, meta_data=meta_data, extra=extra, container=container, driver=self) @@ -955,7 +986,7 @@ def _put_object(self, container, object_name, object_size, 'Unexpected status code, status_code=%s' % (response.status), driver=self) - server_hash = headers['content-md5'] + server_hash = headers.get('content-md5') if server_hash: server_hash = binascii.hexlify(base64.b64decode(b(server_hash))) diff --git a/libcloud/test/secrets.py-dist b/libcloud/test/secrets.py-dist index 638e17d131..1c84359dae 100644 --- a/libcloud/test/secrets.py-dist +++ b/libcloud/test/secrets.py-dist @@ -70,6 +70,7 @@ STORAGE_GOOGLE_STORAGE_PARAMS = ('GOOG0123456789ABCXYZ', 'secret') # Azure key is b64 encoded and must be decoded before signing requests STORAGE_AZURE_BLOBS_PARAMS = ('account', 'cGFzc3dvcmQ=') +STORAGE_AZURITE_BLOBS_PARAMS = ('account', 'cGFzc3dvcmQ=', False, 'localhost', 10000) # Loadbalancer LB_BRIGHTBOX_PARAMS = ('user', 'key') diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_containers_1.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_1.xml new file mode 100644 index 0000000000..e536efdb9a --- /dev/null +++ b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_1.xml @@ -0,0 +1,25 @@ + + + 2 + + + container1 + + Mon, 07 Jan 2013 06:31:06 GMT + "0x8CFBAB7B4F23346" + unlocked + available + + + + container2 + + Mon, 07 Jan 2013 06:31:07 GMT + "0x8CFBAB7B5B82D8E" + unlocked + available + + + + /account/container3 + diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_containers_2.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_2.xml new file mode 100644 index 0000000000..9b16f30e56 --- /dev/null +++ b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_2.xml @@ -0,0 +1,26 @@ + + + /account/container3 + 2 + + + container3 + + Mon, 07 Jan 2013 06:31:08 GMT + "0x8CFBAB7B6452A71" + unlocked + available + + + + container4 + + Fri, 04 Jan 2013 08:32:41 GMT + "0x8CFB86D32305484" + unlocked + available + + + + + diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_containers_empty.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_empty.xml new file mode 100644 index 0000000000..55fd83c6da --- /dev/null +++ b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_empty.xml @@ -0,0 +1,8 @@ + + + + + 100 + + + diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_objects_1.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_1.xml new file mode 100644 index 0000000000..10c9290c0d --- /dev/null +++ b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_1.xml @@ -0,0 +1,45 @@ + + + 2 + + + object1.txt + + Fri, 04 Jan 2013 09:48:06 GMT + 0x8CFB877BB56A6FB + 0 + application/octet-stream + + + + BlockBlob + unlocked + available + + + value1 + value2 + + + + object2.txt + + Sat, 05 Jan 2013 03:51:42 GMT + 0x8CFB90F1BA8CD8F + 1048576 + application/octet-stream + + + + BlockBlob + unlocked + available + + + value1 + value2 + + + + 2!76!MDAwMDExIXNvbWUxMTcudHh0ITAwMDAyOCE5OTk5LTEyLTMxVDIzOjU5OjU5Ljk5OTk5OTlaIQ-- + diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_objects_2.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_2.xml new file mode 100644 index 0000000000..b793711ffe --- /dev/null +++ b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_2.xml @@ -0,0 +1,37 @@ + + + object3.txt + 2 + + + object3.txt + + Sat, 05 Jan 2013 03:52:08 GMT + 0x8CFB90F2B6FC022 + 1048576 + application/octet-stream + + + + BlockBlob + unlocked + available + + + + object4.txt + + Fri, 04 Jan 2013 10:20:14 GMT + 0x8CFB87C38717450 + 0 + application/octet-stream + + + BlockBlob + unlocked + available + + + + + diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_objects_empty.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_empty.xml new file mode 100644 index 0000000000..235801cef7 --- /dev/null +++ b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_empty.xml @@ -0,0 +1,6 @@ + + + 2 + + + diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py index 68e8564497..a2d24fc0c5 100644 --- a/libcloud/test/storage/test_azure_blobs.py +++ b/libcloud/test/storage/test_azure_blobs.py @@ -43,6 +43,7 @@ from libcloud.test import MockHttp, generate_random_data # pylint: disable-msg=E0611 from libcloud.test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 from libcloud.test.secrets import STORAGE_AZURE_BLOBS_PARAMS +from libcloud.test.secrets import STORAGE_AZURITE_BLOBS_PARAMS class AzureBlobsMockHttp(MockHttp, unittest.TestCase): @@ -366,6 +367,19 @@ def _assert_content_length_header_is_string(self, headers): self.assertTrue(isinstance(headers['Content-Length'], basestring)) +class AzuriteBlobsMockHttp(AzureBlobsMockHttp): + fixtures = StorageFileFixtures('azurite_blobs') + + def _get_method_name(self, *args, **kwargs): + method_name = super(AzuriteBlobsMockHttp, self).\ + _get_method_name(*args, **kwargs) + + if method_name.startswith('_account'): + method_name = method_name[8:] + + return method_name + + class AzureBlobsTests(unittest.TestCase): driver_type = AzureBlobsStorageDriver driver_args = STORAGE_AZURE_BLOBS_PARAMS @@ -988,5 +1002,10 @@ def test_storage_driver_host(self): self.assertEqual(host3, 'test.foo.bar.com') +class AzuriteBlobsTests(AzureBlobsTests): + driver_args = STORAGE_AZURITE_BLOBS_PARAMS + mock_response_klass = AzuriteBlobsMockHttp + + if __name__ == '__main__': sys.exit(unittest.main())