Implemented chunked upload for Azure Blobs storage driver

CHANGES.rst

@@ -43,6 +43,38 @@ Compute
   (GITHUB-1346)
   [Tomaz Muraus]
 
+Storage
+~~~~~~~
+
+- [Azure Blobs] Implement chunked upload in the Azure Storage driver.
+
+  Previously, the maximum object size that could be uploaded with the
+  Azure Storage driver was capped at 100 MB: the maximum size that could
+  be uploaded in a single request to Azure. Chunked upload removes this
+  limitation and now enables uploading objects up to Azure's maximum block
+  blob size (~5 TB). The size of the chunks uploaded by the driver can be
+  configured via the ``LIBCLOUD_AZURE_UPLOAD_CHUNK_SIZE_MB`` environment
+  variable and defaults to 4 MB per chunk. Increasing this number trades-off
+  higher memory usage for a lower number of http requests executed by the
+  driver.
+
+  Reported by @rvolykh.
+  (GITHUB-1399, GITHUB-1400)
+  [Clemens Wolff - @c-w]
+
+- [Azure Blobs] Drop support for uploading PageBlob objects via the Azure
+  Storage driver.
+
+  Previously, both PageBlob and BlockBlob objects could be uploaded via the
+  ``upload_object`` and ``upload_object_via_stream`` methods by specifying the
+  ``ex_blob_type`` and ``ex_page_blob_size`` arguments. To simplify the API,
+  these options were removed and all uploaded objects are now of BlockBlob
+  type. Passing ``ex_blob_type`` or ``ex_page_blob_size`` will now raise a
+  ``ValueError``.
+
+  (GITHUB-1400)
+  [Clemens Wolff - @c-w]
+
 Changes in Apache Libcloud v2.8.0
 ---------------------------------
 

docs/upgrade_notes.rst

@@ -17,6 +17,11 @@ Libcloud 3.0.0
 * This release removes VMware vSphere driver which relied on old and
   unmaintained ``pysphere`` library which doesn't support Python 3.
 
+* This release removes support for PageBlob objects from the Azure Blobs
+  storage driver. The ``ex_blob_type`` and ``ex_page_blob_size`` arguments
+  have been removed from the ``upload_object`` and ``upload_object_via_stream``
+  methods.
+
 Libcloud 2.8.0
 --------------
 

libcloud/storage/base.py

@@ -584,7 +584,6 @@ def _save_object(self, response, obj, destination_path,
     def _upload_object(self, object_name, content_type, request_path,
                        request_method='PUT',
                        headers=None, file_path=None, stream=None,
-                       upload_func=None, upload_func_kwargs=None,
                        chunked=False, multipart=False):
         """
         Helper function for setting common request headers and calling the
@@ -600,23 +599,9 @@ def _upload_object(self, object_name, content_type, request_path,
             raise AttributeError('iterator object must implement next() ' +
                                  'method.')
 
-        if not content_type:
-            if file_path:
-                name = file_path
-            else:
-                name = object_name
-            content_type, _ = libcloud.utils.files.guess_file_mime_type(name)
-
-            if not content_type:
-                if self.strict_mode:
-                    raise AttributeError('File content-type could not be '
-                                         'guessed and no content_type value '
-                                         'is provided')
-                else:
-                    # Fallback to a content-type
-                    content_type = DEFAULT_CONTENT_TYPE
-
-        headers['Content-Type'] = content_type
+        headers['Content-Type'] = self._determine_content_type(
+            content_type, object_name, file_path=file_path)
+
         if stream:
             response = self.connection.request(
                 request_path,
@@ -639,13 +624,25 @@ def _upload_object(self, object_name, content_type, request_path,
         if not response.success():
             response.parse_error()
 
-        if upload_func:
-            upload_func(**upload_func_kwargs)
-
         return {'response': response,
                 'bytes_transferred': stream_length,
                 'data_hash': stream_hash}
 
+    def _determine_content_type(self, content_type, object_name,
+                                file_path=None):
+        if content_type:
+            return content_type
+
+        name = file_path or object_name
+        content_type, _ = libcloud.utils.files.guess_file_mime_type(name)
+
+        if self.strict_mode and not content_type:
+            raise AttributeError('File content-type could not be guessed for '
+                                 '"%s" and no content_type value is provided'
+                                 % name)
+
+        return content_type or DEFAULT_CONTENT_TYPE
+
     def _hash_buffered_stream(self, stream, hasher, blocksize=65536):
         total_len = 0
 

libcloud/storage/drivers/atmos.py

@@ -30,7 +30,7 @@
 if PY3:
     from io import FileIO as file
 
-from libcloud.utils.files import read_in_chunks, guess_file_mime_type
+from libcloud.utils.files import read_in_chunks
 from libcloud.common.base import ConnectionUserAndKey, XmlResponse
 from libcloud.common.types import LibcloudError
 
@@ -271,13 +271,8 @@ def upload_object_via_stream(self, iterator, container, object_name,
             content_type = extra.get('content_type', None)
         else:
             content_type = None
-        if not content_type:
-            content_type, _ = guess_file_mime_type(object_name)
 
-            if not content_type:
-                raise AttributeError(
-                    'File content-type could not be guessed and' +
-                    ' no content_type value provided')
+        content_type = self._determine_content_type(content_type, object_name)
 
         try:
             self.connection.request(path + '?metadata/system')