Fixes #7: Add support for chunk size when uploading

scottwernervt · Jun 20, 2017 · 62ef0f6 · 62ef0f6
1 parent 51ae832
commit 62ef0f6
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 9 deletions.
diff --git a/cloudstorage/base.py b/cloudstorage/base.py
@@ -569,7 +569,7 @@ def delete(self) -> None:
     def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None,
                     acl: str = None, meta_data: MetaData = None,
                     content_type: str = None, content_disposition: str = None,
-                    extra: ExtraOptions = None) -> Blob:
+                    chunk_size: int = 1024, extra: ExtraOptions = None) -> Blob:
         """Upload a filename or file like object to a container.
         
         If `content_type` is `None`, Cloud Storage will attempt to guess the 
@@ -631,6 +631,8 @@ def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None,
           storage-api-reference/object-services-operations/
           #create-or-update-object>`_
 
+        :param chunk_size:
+        :type chunk_size:
         :param filename: A file handle open for reading or the path to the file.
         :type filename: file or str
 
@@ -663,6 +665,10 @@ def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None,
                                     information for the blob.
         :type content_disposition: str or None
 
+        :param chunk_size: (optional) Optional chunk size for streaming a
+                   transfer.
+        :type chunk_size: int
+
         :param extra: (optional) Extra parameters for the request.
         :type extra: Dict[str, str] or None
 
@@ -674,6 +680,7 @@ def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None,
                                        meta_data=meta_data,
                                        content_type=content_type,
                                        content_disposition=content_disposition,
+                                       chunk_size=chunk_size,
                                        extra=extra)
 
     def get_blob(self, blob_name: str) -> Blob:
@@ -1165,11 +1172,13 @@ def disable_container_cdn(self, container: Container) -> bool:
     def upload_blob(self, container: Container, filename: Union[str, FileLike],
                     blob_name: str = None, acl: str = None,
                     meta_data: MetaData = None, content_type: str = None,
-                    content_disposition: str = None,
+                    content_disposition: str = None, chunk_size=1024,
                     extra: ExtraOptions = None) -> Blob:
         """Upload a filename or file like object to a container.
         
         .. important:: This class method is called by 
+                       :param chunk_size:
+                       :type chunk_size:
                        :meth:`.Container.upload_blob`.
         
         :param container: The container to upload the blob to.
@@ -1196,6 +1205,10 @@ def upload_blob(self, container: Container, filename: Union[str, FileLike],
         :param content_disposition: (optional) Specifies presentational 
                                     information for the blob.
         :type content_disposition: str or None
+
+        :param chunk_size: (optional) Optional chunk size for streaming a
+                           transfer.
+        :type chunk_size: int
         
         :param extra: (optional) Extra parameters for the request.
         :type extra: Dict[str, str] or None

diff --git a/cloudstorage/drivers/amazon.py b/cloudstorage/drivers/amazon.py
@@ -280,13 +280,15 @@ def disable_container_cdn(self, container: Container) -> bool:
     def upload_blob(self, container: Container, filename: Union[str, FileLike],
                     blob_name: str = None, acl: str = None,
                     meta_data: MetaData = None, content_type: str = None,
-                    content_disposition: str = None,
+                    content_disposition: str = None, chunk_size: int = 1024,
                     extra: ExtraOptions = None) -> Blob:
         meta_data = {} if meta_data is None else meta_data
         extra = {} if extra is None else extra
 
         extra_args = self._normalize_parameters(extra, self._PUT_OBJECT_KEYS)
 
+        config = boto3.s3.transfer.TransferConfig(io_chunksize=chunk_size)
+
         # Default arguments
         if acl:
             extra_args.setdefault('ACL', acl.lower())
@@ -313,11 +315,13 @@ def upload_blob(self, container: Container, filename: Union[str, FileLike],
         if isinstance(filename, str):
             self.s3.Bucket(container.name).upload_file(Filename=filename,
                                                        Key=blob_name,
-                                                       ExtraArgs=extra_args)
+                                                       ExtraArgs=extra_args,
+                                                       Config=config)
         else:
             self.s3.Bucket(container.name).upload_fileobj(Fileobj=filename,
                                                           Key=blob_name,
-                                                          ExtraArgs=extra_args)
+                                                          ExtraArgs=extra_args,
+                                                          Config=config)
 
         return self.get_blob(container, blob_name)
 

diff --git a/cloudstorage/drivers/google.py b/cloudstorage/drivers/google.py
@@ -274,7 +274,7 @@ def disable_container_cdn(self, container: Container) -> bool:
     def upload_blob(self, container: Container, filename: Union[str, FileLike],
                     blob_name: str = None, acl: str = None,
                     meta_data: MetaData = None, content_type: str = None,
-                    content_disposition: str = None,
+                    content_disposition: str = None, chunk_size: int = 1024,
                     extra: ExtraOptions = None) -> Blob:
         extra = extra if extra is not None else {}
         extra_args = self._normalize_parameters(extra, self._PUT_OBJECT_KEYS)
@@ -296,7 +296,8 @@ def upload_blob(self, container: Container, filename: Union[str, FileLike],
             blob.upload_from_filename(filename=filename,
                                       content_type=content_type)
         else:
-            blob.upload_from_file(file_obj=filename, content_type=content_type)
+            blob.upload_from_file(file_obj=filename, content_type=content_type,
+                                  size=chunk_size)
 
         if acl:
             blob.acl.save_predefined(acl)

diff --git a/cloudstorage/drivers/local.py b/cloudstorage/drivers/local.py
@@ -393,7 +393,7 @@ def disable_container_cdn(self, container: Container) -> bool:
     def upload_blob(self, container: Container, filename: Union[str, FileLike],
                     blob_name: str = None, acl: str = None,
                     meta_data: MetaData = None, content_type: str = None,
-                    content_disposition: str = None,
+                    content_disposition: str = None, chunk_size: int = 1024,
                     extra: ExtraOptions = None) -> Blob:
         if acl:
             logger.info(option_not_supported % 'acl')

diff --git a/cloudstorage/drivers/rackspace.py b/cloudstorage/drivers/rackspace.py
@@ -477,7 +477,7 @@ def disable_container_cdn(self, container: Container) -> bool:
     def upload_blob(self, container: Container, filename: Union[str, FileLike],
                     blob_name: str = None, acl: str = None,
                     meta_data: MetaData = None, content_type: str = None,
-                    content_disposition: str = None,
+                    content_disposition: str = None, chunk_size: int = 1024,
                     extra: ExtraOptions = None) -> Blob:
         if acl:
             logger.warning(option_not_supported % 'acl')