Add support for Azure Government to Azure Blobs storage driver

CHANGES.rst

@@ -1,6 +1,19 @@
 Changelog
 =========
 
+Changes in Apache Libcloud 3.3.2
+--------------------------------
+
+Storage
+~~~~~~~
+
+- [Azure Blobs] Enable the Azure storage driver to be used with
+  Azure Government, Azure China, and Azure Private Link by setting
+  the driver host argument to the endpoint suffix for the environment.
+
+  Reported by Melissa Kersh - @mkcello96
+  (GITHUB-1551)
+
 Changes in Apache Libcloud 3.3.1
 --------------------------------
 

docs/examples/storage/azure/instantiate_gov.py

@@ -0,0 +1,8 @@
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+cls = get_driver(Provider.AZURE_BLOBS)
+
+driver = cls(key='your storage account name',
+             secret='your access key',
+             host='blob.core.usgovcloudapi.net')

docs/storage/drivers/azure_blobs.rst

@@ -33,6 +33,19 @@ below.
 .. literalinclude:: /examples/storage/azure/instantiate.py
    :language: python
 
+Connecting to Azure Government
+------------------------------
+
+To target an `Azure Government`_ storage account, you can instantiate the driver
+by setting a custom storage host argument as shown below.
+
+.. literalinclude:: /examples/storage/azure/instantiate_gov.py
+   :language: python
+
+Setting a custom host argument can also be leveraged to customize the blob
+endpoint and connect to a storage account in `Azure China`_ or
+`Azure Private Link`_.
+
 Connecting to self-hosted Azure Storage implementations
 -------------------------------------------------------
 
@@ -51,3 +64,6 @@ Azure Storage implementations such as `Azure Blob Storage on IoT Edge`_.
 .. _`BlockBlobStorage accounts`: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#blockblobstorage-accounts
 .. _`Azurite storage emulator`: https://github.com/Azure/Azurite
 .. _`Azure Blob Storage on IoT Edge`: https://docs.microsoft.com/en-us/azure/iot-edge/how-to-store-data-blob
+.. _`Azure Government`: https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide
+.. _`Azure China`: https://docs.microsoft.com/en-us/azure/china/resources-developer-guide
+.. _`Azure Private Link`: https://docs.microsoft.com/en-us/azure/private-link/private-link-overview

libcloud/common/base.py

@@ -984,29 +984,31 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None,
         self.key = key
         self.secret = secret
         self.secure = secure
+        self.api_version = api_version
+        self.region = region
+
+        conn_kwargs = self._ex_connection_class_kwargs()
+        conn_kwargs.update({'timeout': kwargs.pop('timeout', None),
+                            'retry_delay': kwargs.pop('retry_delay', None),
+                            'backoff': kwargs.pop('backoff', None),
+                            'proxy_url': kwargs.pop('proxy_url', None)})
+
         args = [self.key]
 
         if self.secret is not None:
             args.append(self.secret)
 
         args.append(secure)
 
+        host = conn_kwargs.pop('host', None) or host
+
         if host is not None:
             args.append(host)
 
         if port is not None:
             args.append(port)
 
-        self.api_version = api_version
-        self.region = region
-
-        conn_kwargs = self._ex_connection_class_kwargs()
-        conn_kwargs.update({'timeout': kwargs.pop('timeout', None),
-                            'retry_delay': kwargs.pop('retry_delay', None),
-                            'backoff': kwargs.pop('backoff', None),
-                            'proxy_url': kwargs.pop('proxy_url', None)})
         self.connection = self.connectionCls(*args, **conn_kwargs)
-
         self.connection.driver = self
         self.connection.connect()
 

libcloud/storage/drivers/azure_blobs.py

@@ -68,6 +68,9 @@
 )
 
 AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
+AZURE_STORAGE_HOST_SUFFIX_CHINA = 'blob.core.chinacloudapi.cn'
+AZURE_STORAGE_HOST_SUFFIX_GOVERNMENT = 'blob.core.usgovcloudapi.net'
+AZURE_STORAGE_HOST_SUFFIX_PRIVATELINK = 'privatelink.blob.core.windows.net'
 
 AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
 
@@ -173,11 +176,12 @@ class AzureBlobsConnection(AzureConnection):
     these deployments, the parameter ``account_prefix`` must be set on the
     connection. This is done by instantiating the driver with arguments such
     as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
-    host without an account prefix, e.g. for use-cases where the custom host
-    implements an auditing proxy or similar, the driver can be instantiated
-    with ``host='theaccount.somewhere.tld'`` and ``key=''``.
+    host without an account prefix, e.g. to connect to Azure Government or
+    Azure China, the driver can be instantiated with the appropriate storage
+    endpoint suffix, e.g. ``host='blob.core.usgovcloudapi.net'`` and
+    ``key='theaccount'``.
 
-    :param account_prefix: Optional prefix identifying the sotrage account.
+    :param account_prefix: Optional prefix identifying the storage account.
                            Used when connecting to a custom deployment of the
                            storage service like Azurite or IoT Edge Storage.
     :type account_prefix: ``str``
@@ -206,7 +210,7 @@ class AzureBlobsStorageDriver(StorageDriver):
 
     def __init__(self, key, secret=None, secure=True, host=None, port=None,
                  **kwargs):
-        self._host_argument_set = bool(host)
+        self._host = host
 
         # B64decode() this key and keep it, so that we don't have to do
         # so for every request. Minor performance improvement
@@ -217,15 +221,34 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None,
                                                       port=port, **kwargs)
 
     def _ex_connection_class_kwargs(self):
-        result = {}
-
-        # host argument has precedence
-        if not self._host_argument_set:
-            result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)
+        # if the user didn't provide a custom host value, assume we're
+        # targeting the default Azure Storage endpoints
+        if self._host is None:
+            return {'host': '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)}
+
+        # connecting to a special storage region like Azure Government or
+        # Azure China requires setting a custom storage endpoint but we
+        # still use the same scheme to identify a specific account as for
+        # the standard storage endpoint
+        try:
+            host_suffix = next(
+                host_suffix
+                for host_suffix in (
+                    AZURE_STORAGE_HOST_SUFFIX_CHINA,
+                    AZURE_STORAGE_HOST_SUFFIX_GOVERNMENT,
+                    AZURE_STORAGE_HOST_SUFFIX_PRIVATELINK,
+                )
+                if self._host.endswith(host_suffix)
+            )
+        except StopIteration:
+            pass
         else:
-            result['account_prefix'] = self.key
+            return {'host': '%s.%s' % (self.key, host_suffix)}
 
-        return result
+        # if the host isn't targeting one of the special storage regions, it
+        # must be pointing to Azurite or IoT Edge Storage so switch to prefix
+        # identification
+        return {'account_prefix': self.key}
 
     def _xml_to_container(self, node):
         """

libcloud/test/storage/test_azure_blobs.py

@@ -935,6 +935,35 @@ def test_storage_driver_host(self):
         self.assertEqual(host2, 'fakeaccount2.blob.core.windows.net')
         self.assertEqual(host3, 'test.foo.bar.com')
 
+    def test_storage_driver_host_govcloud(self):
+        driver1 = self.driver_type(
+            'fakeaccount1', 'deadbeafcafebabe==',
+            host='blob.core.usgovcloudapi.net')
+        driver2 = self.driver_type(
+            'fakeaccount2', 'deadbeafcafebabe==',
+            host='fakeaccount2.blob.core.usgovcloudapi.net')
+
+        host1 = driver1.connection.host
+        host2 = driver2.connection.host
+        account_prefix_1 = driver1.connection.account_prefix
+        account_prefix_2 = driver2.connection.account_prefix
+
+        self.assertEqual(host1, 'fakeaccount1.blob.core.usgovcloudapi.net')
+        self.assertEqual(host2, 'fakeaccount2.blob.core.usgovcloudapi.net')
+        self.assertIsNone(account_prefix_1)
+        self.assertIsNone(account_prefix_2)
+
+    def test_storage_driver_host_azurite(self):
+        driver = self.driver_type(
+            'fakeaccount1', 'deadbeafcafebabe==',
+            host='localhost', port=10000, secure=False)
+
+        host = driver.connection.host
+        account_prefix = driver.connection.account_prefix
+
+        self.assertEqual(host, 'localhost')
+        self.assertEqual(account_prefix, 'fakeaccount1')
+
 
 class AzuriteBlobsTests(AzureBlobsTests):
     driver_args = STORAGE_AZURITE_BLOBS_PARAMS