diff --git a/cloudstorage/base.py b/cloudstorage/base.py index def7ffd..6032f33 100644 --- a/cloudstorage/base.py +++ b/cloudstorage/base.py @@ -569,7 +569,7 @@ def delete(self) -> None: def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None, acl: str = None, meta_data: MetaData = None, content_type: str = None, content_disposition: str = None, - extra: ExtraOptions = None) -> Blob: + chunk_size: int = 1024, extra: ExtraOptions = None) -> Blob: """Upload a filename or file like object to a container. If `content_type` is `None`, Cloud Storage will attempt to guess the @@ -631,6 +631,8 @@ def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None, storage-api-reference/object-services-operations/ #create-or-update-object>`_ + :param chunk_size: + :type chunk_size: :param filename: A file handle open for reading or the path to the file. :type filename: file or str @@ -663,6 +665,10 @@ def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None, information for the blob. :type content_disposition: str or None + :param chunk_size: (optional) Optional chunk size for streaming a + transfer. + :type chunk_size: int + :param extra: (optional) Extra parameters for the request. :type extra: Dict[str, str] or None @@ -674,6 +680,7 @@ def upload_blob(self, filename: Union[str, FileLike], blob_name: str = None, meta_data=meta_data, content_type=content_type, content_disposition=content_disposition, + chunk_size=chunk_size, extra=extra) def get_blob(self, blob_name: str) -> Blob: @@ -1165,11 +1172,13 @@ def disable_container_cdn(self, container: Container) -> bool: def upload_blob(self, container: Container, filename: Union[str, FileLike], blob_name: str = None, acl: str = None, meta_data: MetaData = None, content_type: str = None, - content_disposition: str = None, + content_disposition: str = None, chunk_size=1024, extra: ExtraOptions = None) -> Blob: """Upload a filename or file like object to a container. .. important:: This class method is called by + :param chunk_size: + :type chunk_size: :meth:`.Container.upload_blob`. :param container: The container to upload the blob to. @@ -1196,6 +1205,10 @@ def upload_blob(self, container: Container, filename: Union[str, FileLike], :param content_disposition: (optional) Specifies presentational information for the blob. :type content_disposition: str or None + + :param chunk_size: (optional) Optional chunk size for streaming a + transfer. + :type chunk_size: int :param extra: (optional) Extra parameters for the request. :type extra: Dict[str, str] or None diff --git a/cloudstorage/drivers/amazon.py b/cloudstorage/drivers/amazon.py index 8bd01c7..0161139 100644 --- a/cloudstorage/drivers/amazon.py +++ b/cloudstorage/drivers/amazon.py @@ -280,13 +280,15 @@ def disable_container_cdn(self, container: Container) -> bool: def upload_blob(self, container: Container, filename: Union[str, FileLike], blob_name: str = None, acl: str = None, meta_data: MetaData = None, content_type: str = None, - content_disposition: str = None, + content_disposition: str = None, chunk_size: int = 1024, extra: ExtraOptions = None) -> Blob: meta_data = {} if meta_data is None else meta_data extra = {} if extra is None else extra extra_args = self._normalize_parameters(extra, self._PUT_OBJECT_KEYS) + config = boto3.s3.transfer.TransferConfig(io_chunksize=chunk_size) + # Default arguments if acl: extra_args.setdefault('ACL', acl.lower()) @@ -313,11 +315,13 @@ def upload_blob(self, container: Container, filename: Union[str, FileLike], if isinstance(filename, str): self.s3.Bucket(container.name).upload_file(Filename=filename, Key=blob_name, - ExtraArgs=extra_args) + ExtraArgs=extra_args, + Config=config) else: self.s3.Bucket(container.name).upload_fileobj(Fileobj=filename, Key=blob_name, - ExtraArgs=extra_args) + ExtraArgs=extra_args, + Config=config) return self.get_blob(container, blob_name) diff --git a/cloudstorage/drivers/google.py b/cloudstorage/drivers/google.py index a9127e7..f50501f 100644 --- a/cloudstorage/drivers/google.py +++ b/cloudstorage/drivers/google.py @@ -274,7 +274,7 @@ def disable_container_cdn(self, container: Container) -> bool: def upload_blob(self, container: Container, filename: Union[str, FileLike], blob_name: str = None, acl: str = None, meta_data: MetaData = None, content_type: str = None, - content_disposition: str = None, + content_disposition: str = None, chunk_size: int = 1024, extra: ExtraOptions = None) -> Blob: extra = extra if extra is not None else {} extra_args = self._normalize_parameters(extra, self._PUT_OBJECT_KEYS) @@ -296,7 +296,8 @@ def upload_blob(self, container: Container, filename: Union[str, FileLike], blob.upload_from_filename(filename=filename, content_type=content_type) else: - blob.upload_from_file(file_obj=filename, content_type=content_type) + blob.upload_from_file(file_obj=filename, content_type=content_type, + size=chunk_size) if acl: blob.acl.save_predefined(acl) diff --git a/cloudstorage/drivers/local.py b/cloudstorage/drivers/local.py index 8276dfd..e843fa1 100644 --- a/cloudstorage/drivers/local.py +++ b/cloudstorage/drivers/local.py @@ -393,7 +393,7 @@ def disable_container_cdn(self, container: Container) -> bool: def upload_blob(self, container: Container, filename: Union[str, FileLike], blob_name: str = None, acl: str = None, meta_data: MetaData = None, content_type: str = None, - content_disposition: str = None, + content_disposition: str = None, chunk_size: int = 1024, extra: ExtraOptions = None) -> Blob: if acl: logger.info(option_not_supported % 'acl') diff --git a/cloudstorage/drivers/rackspace.py b/cloudstorage/drivers/rackspace.py index c65dc97..c817087 100644 --- a/cloudstorage/drivers/rackspace.py +++ b/cloudstorage/drivers/rackspace.py @@ -477,7 +477,7 @@ def disable_container_cdn(self, container: Container) -> bool: def upload_blob(self, container: Container, filename: Union[str, FileLike], blob_name: str = None, acl: str = None, meta_data: MetaData = None, content_type: str = None, - content_disposition: str = None, + content_disposition: str = None, chunk_size: int = 1024, extra: ExtraOptions = None) -> Blob: if acl: logger.warning(option_not_supported % 'acl')