[LIBCLOUD-1037] Add Azurite support for Azure Blob Storage driver

This requires the following main changes: - Protect against a missing content md5 since the Azurite emulator doesn't set this value. - Protect against a missing metadata tag since the Azurite emulator doesn't set this value. - Implement routing to a specific account via URL prefix (e.g. /someaccount) instead of hostname (e.g. someaccount.blob.core.windows.net).
apache · Jul 16, 2019 · 9e1dfe1 · 9e1dfe1
1 parent 15e3c3c
commit 9e1dfe1
Show file tree

Hide file tree

Showing 11 changed files with 231 additions and 6 deletions.
diff --git a/docs/examples/storage/azure/instantiate_azurite.py b/docs/examples/storage/azure/instantiate_azurite.py
@@ -0,0 +1,11 @@
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+cls = get_driver(Provider.AZURE_BLOBS)
+
+driver = cls(key='devstoreaccount1',
+             secret='Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6'
+                    'IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
+             host='localhost',
+             port=10000,
+             secure=False)
diff --git a/docs/storage/drivers/azure_blobs.rst b/docs/storage/drivers/azure_blobs.rst
@@ -34,3 +34,19 @@ below.
 
 .. literalinclude:: /examples/storage/azure/instantiate.py
    :language: python
+
+Connecting to self-hosted Azure Storage implementations
+-------------------------------------------------------
+
+To facilitate integration testing, libcloud supports connecting to the
+`Azurite storage emulator`_. After starting the emulator, you can
+instantiate the driver as shown below.
+
+.. literalinclude:: /examples/storage/azure/instantiate_azurite.py
+   :language: python
+
+This instantiation strategy can also be adapted to connect to other self-hosted
+Azure Storage implementations such as `Azure Blob Storage on IoT Edge`_.
+
+.. _`Azurite storage emulator`: https://github.com/Azure/Azurite
+.. _`Azure Blob Storage on IoT Edge`: https://docs.microsoft.com/en-us/azure/iot-edge/how-to-store-data-blob
diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py
@@ -154,8 +154,35 @@ def __exit__(self, type, value, traceback):
 
 class AzureBlobsConnection(AzureConnection):
     """
-    Represents a single connection to Azure Blobs
+    Represents a single connection to Azure Blobs.
+
+    The main Azure Blob Storage service uses a prefix in the hostname to
+    distinguish between accounts, e.g. ``theaccount.blob.core.windows.net``.
+    However, some custom deployments of the service, such as the Azurite
+    emulator, instead use a URL prefix such as ``/theaccount``. To support
+    these deployments, the parameter ``account_prefix`` must be set on the
+    connection. This is done by instantiating the driver with arguments such
+    as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
+    host without an account prefix, e.g. for use-cases where the custom host
+    implements an auditing proxy or similar, the driver can be instantiated
+    with ``host='theaccount.somewhere.tld'`` and ``key=''``.
+
+    :param account_prefix: Optional prefix identifying the sotrage account.
+                           Used when connecting to a custom deployment of the
+                           storage service like Azurite or IoT Edge Storage.
+    :type account_prefix: ``str``
     """
+    def __init__(self, *args, **kwargs):
+        self.account_prefix = kwargs.pop('account_prefix', None)
+        super(AzureBlobsConnection, self).__init__(*args, **kwargs)
+
+    def morph_action_hook(self, action):
+        action = super(AzureBlobsConnection, self).morph_action_hook(action)
+
+        if self.account_prefix is not None:
+            action = '/%s%s' % (self.account_prefix, action)
+
+        return action
 
     # this is the minimum api version supported by storage accounts of kinds
     # StorageV2, Storage and BlobStorage
@@ -188,6 +215,8 @@ def _ex_connection_class_kwargs(self):
         # host argument has precedence
         if not self._host_argument_set:
             result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)
+        else:
+            result['account_prefix'] = self.key
 
         return result
 
@@ -218,8 +247,9 @@ def _xml_to_container(self, node):
             'meta_data': {}
         }
 
-        for meta in list(metadata):
-            extra['meta_data'][meta.tag] = meta.text
+        if metadata is not None:
+            for meta in list(metadata):
+                extra['meta_data'][meta.tag] = meta.text
 
         return Container(name=name, extra=extra, driver=self)
 
@@ -303,8 +333,9 @@ def _xml_to_object(self, container, blob):
             extra['md5_hash'] = value
 
         meta_data = {}
-        for meta in list(metadata):
-            meta_data[meta.tag] = meta.text
+        if metadata is not None:
+            for meta in list(metadata):
+                meta_data[meta.tag] = meta.text
 
         return Object(name=name, size=size, hash=etag, meta_data=meta_data,
                       extra=extra, container=container, driver=self)
@@ -955,7 +986,7 @@ def _put_object(self, container, object_name, object_size,
                 'Unexpected status code, status_code=%s' % (response.status),
                 driver=self)
 
-        server_hash = headers['content-md5']
+        server_hash = headers.get('content-md5')
 
         if server_hash:
             server_hash = binascii.hexlify(base64.b64decode(b(server_hash)))

diff --git a/libcloud/test/secrets.py-dist b/libcloud/test/secrets.py-dist
@@ -70,6 +70,7 @@ STORAGE_GOOGLE_STORAGE_PARAMS = ('GOOG0123456789ABCXYZ', 'secret')
 
 # Azure key is b64 encoded and must be decoded before signing requests
 STORAGE_AZURE_BLOBS_PARAMS = ('account', 'cGFzc3dvcmQ=')
+STORAGE_AZURITE_BLOBS_PARAMS = ('account', 'cGFzc3dvcmQ=', False, 'localhost', 10000)
 
 # Loadbalancer
 LB_BRIGHTBOX_PARAMS = ('user', 'key')

diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_containers_1.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_1.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults ServiceEndpoint="http://localhost:10000/account">
+    <MaxResults>2</MaxResults>
+    <Containers>
+        <Container>
+            <Name>container1</Name>
+            <Properties>
+                <Last-Modified>Mon, 07 Jan 2013 06:31:06 GMT</Last-Modified>
+                <Etag>"0x8CFBAB7B4F23346"</Etag>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+        </Container>
+        <Container>
+            <Name>container2</Name>
+            <Properties>
+                <Last-Modified>Mon, 07 Jan 2013 06:31:07 GMT</Last-Modified>
+                <Etag>"0x8CFBAB7B5B82D8E"</Etag>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+        </Container>
+    </Containers>
+    <NextMarker>/account/container3</NextMarker>
+</EnumerationResults>
diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_containers_2.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_2.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults ServiceEndpoint="http://localhost:10000/account">
+    <Marker>/account/container3</Marker>
+    <MaxResults>2</MaxResults>
+    <Containers>
+        <Container>
+            <Name>container3</Name>
+            <Properties>
+                <Last-Modified>Mon, 07 Jan 2013 06:31:08 GMT</Last-Modified>
+                <Etag>"0x8CFBAB7B6452A71"</Etag>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+        </Container>
+        <Container>
+            <Name>container4</Name>
+            <Properties>
+                <Last-Modified>Fri, 04 Jan 2013 08:32:41 GMT</Last-Modified>
+                <Etag>"0x8CFB86D32305484"</Etag>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+        </Container>
+    </Containers>
+    <NextMarker />
+</EnumerationResults>
diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_containers_empty.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_containers_empty.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults ServiceEndpoint="http://localhost:10000/account">
+  <Prefix></Prefix>
+  <Marker></Marker>
+  <MaxResults>100</MaxResults>
+  <Containers />
+  <NextMarker />
+</EnumerationResults>
diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_objects_1.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_1.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults ContainerName="test_container" ServiceEndpoint="http://localhost:10000/account">
+    <MaxResults>2</MaxResults>
+    <Blobs>
+        <Blob>
+            <Name>object1.txt</Name>
+            <Properties>
+                <Last-Modified>Fri, 04 Jan 2013 09:48:06 GMT</Last-Modified>
+                <Etag>0x8CFB877BB56A6FB</Etag>
+                <Content-Length>0</Content-Length>
+                <Content-Type>application/octet-stream</Content-Type>
+                <Content-Encoding />
+                <Content-Language />
+                <Cache-Control />
+                <BlobType>BlockBlob</BlobType>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+            <Metadata>
+                <meta1>value1</meta1>
+                <meta2>value2</meta2>
+            </Metadata>
+        </Blob>
+        <Blob>
+            <Name>object2.txt</Name>
+            <Properties>
+                <Last-Modified>Sat, 05 Jan 2013 03:51:42 GMT</Last-Modified>
+                <Etag>0x8CFB90F1BA8CD8F</Etag>
+                <Content-Length>1048576</Content-Length>
+                <Content-Type>application/octet-stream</Content-Type>
+                <Content-Encoding />
+                <Content-Language />
+                <Cache-Control />
+                <BlobType>BlockBlob</BlobType>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+            <Metadata>
+                <meta1>value1</meta1>
+                <meta2>value2</meta2>
+            </Metadata>
+        </Blob>
+    </Blobs>
+    <NextMarker>2!76!MDAwMDExIXNvbWUxMTcudHh0ITAwMDAyOCE5OTk5LTEyLTMxVDIzOjU5OjU5Ljk5OTk5OTlaIQ--</NextMarker>
+</EnumerationResults>
diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_objects_2.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_2.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults ContainerName="test_container" ServiceEndpoint="http://localhost:10000/account">
+    <Marker>object3.txt</Marker>
+    <MaxResults>2</MaxResults>
+    <Blobs>
+        <Blob>
+            <Name>object3.txt</Name>
+            <Properties>
+                <Last-Modified>Sat, 05 Jan 2013 03:52:08 GMT</Last-Modified>
+                <Etag>0x8CFB90F2B6FC022</Etag>
+                <Content-Length>1048576</Content-Length>
+                <Content-Type>application/octet-stream</Content-Type>
+                <Content-Encoding />
+                <Content-Language />
+                <Cache-Control />
+                <BlobType>BlockBlob</BlobType>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+        </Blob>
+        <Blob>
+            <Name>object4.txt</Name>
+            <Properties>
+                <Last-Modified>Fri, 04 Jan 2013 10:20:14 GMT</Last-Modified>
+                <Etag>0x8CFB87C38717450</Etag>
+                <Content-Length>0</Content-Length>
+                <Content-Type>application/octet-stream</Content-Type>
+                <Content-Encoding /><Content-Language />
+                <Cache-Control />
+                <BlobType>BlockBlob</BlobType>
+                <LeaseStatus>unlocked</LeaseStatus>
+                <LeaseState>available</LeaseState>
+            </Properties>
+        </Blob>
+    </Blobs>
+    <NextMarker />
+</EnumerationResults>
diff --git a/libcloud/test/storage/fixtures/azurite_blobs/list_objects_empty.xml b/libcloud/test/storage/fixtures/azurite_blobs/list_objects_empty.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults ContainerName="test_container" ServiceEndpoint="http://localhost:10000/account">
+    <MaxResults>2</MaxResults>
+    <Blobs />
+    <NextMarker />
+</EnumerationResults>
diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py
@@ -43,6 +43,7 @@
 from libcloud.test import MockHttp, generate_random_data  # pylint: disable-msg=E0611
 from libcloud.test.file_fixtures import StorageFileFixtures  # pylint: disable-msg=E0611
 from libcloud.test.secrets import STORAGE_AZURE_BLOBS_PARAMS
+from libcloud.test.secrets import STORAGE_AZURITE_BLOBS_PARAMS
 
 
 class AzureBlobsMockHttp(MockHttp, unittest.TestCase):
@@ -366,6 +367,19 @@ def _assert_content_length_header_is_string(self, headers):
             self.assertTrue(isinstance(headers['Content-Length'], basestring))
 
 
+class AzuriteBlobsMockHttp(AzureBlobsMockHttp):
+    fixtures = StorageFileFixtures('azurite_blobs')
+
+    def _get_method_name(self, *args, **kwargs):
+        method_name = super(AzuriteBlobsMockHttp, self).\
+            _get_method_name(*args, **kwargs)
+
+        if method_name.startswith('_account'):
+            method_name = method_name[8:]
+
+        return method_name
+
+
 class AzureBlobsTests(unittest.TestCase):
     driver_type = AzureBlobsStorageDriver
     driver_args = STORAGE_AZURE_BLOBS_PARAMS
@@ -988,5 +1002,10 @@ def test_storage_driver_host(self):
         self.assertEqual(host3, 'test.foo.bar.com')
 
 
+class AzuriteBlobsTests(AzureBlobsTests):
+    driver_args = STORAGE_AZURITE_BLOBS_PARAMS
+    mock_response_klass = AzuriteBlobsMockHttp
+
+
 if __name__ == '__main__':
     sys.exit(unittest.main())