Skip to content

高级下载支持进度回调 #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhy1985555 merged 1 commit into from
Dec 17, 2021
Merged

高级下载支持进度回调 #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions qcloud_cos/cos_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3194,7 +3194,7 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num,
already_exist_parts[part_num] = part['ETag']
return True

def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, EnableCRC=False, **Kwargs):
def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, EnableCRC=False, progress_callback=None, **Kwargs):
"""小于等于20MB的文件简单下载，大于20MB的文件使用续传下载

:param Bucket(string): 存储桶名称.
Expand Down Expand Up @@ -3224,8 +3224,13 @@ def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, Ena
response['Body'].get_stream_to_file(DestFilePath)
return

# 支持回调查看进度
callback = None
if progress_callback:
callback = ProgressCallback(file_size, progress_callback)

downloader = ResumableDownLoader(self, Bucket, Key, DestFilePath, object_info, PartSize, MAXThread, EnableCRC,
**Kwargs)
callback, **Kwargs)
downloader.start()

def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, EnableMD5=False, progress_callback=None,
Expand Down
11 changes: 10 additions & 1 deletion qcloud_cos/resumable_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@

class ResumableDownLoader(object):
def __init__(self, cos_client, bucket, key, dest_filename, object_info, part_size=20, max_thread=5,
enable_crc=False, **kwargs):
enable_crc=False, progress_callback=None, **kwargs):
self.__cos_client = cos_client
self.__bucket = bucket
self.__key = key
self.__dest_file_path = os.path.abspath(dest_filename)
self.__object_info = object_info
self.__max_thread = max_thread
self.__enable_crc = enable_crc
self.__progress_callback = progress_callback
self.__headers = kwargs

self.__max_part_count = 100 # 取决于服务端是否对并发有限制
Expand All @@ -50,6 +51,11 @@ def start(self):
assert self.__tmp_file
open(self.__tmp_file, 'a').close()

# 已完成分块先设置下载进度
if self.__progress_callback:
for finished_part in self.__finished_parts:
self.__progress_callback.report(finished_part.length)

parts_need_to_download = self.__get_parts_need_to_download()
logger.debug('parts_need_to_download: {0}'.format(parts_need_to_download))
pool = SimpleThreadPool(self.__max_thread)
Expand Down Expand Up @@ -126,6 +132,9 @@ def __download_part(self, part, headers):

self.__finish_part(part)

if self.__progress_callback:
self.__progress_callback.report(part.length)

def __finish_part(self, part):
logger.debug('download part finished,bucket: {0}, key: {1}, part_id: {2}'.
format(self.__bucket, self.__key, part.part_id))
Expand Down