From d5ed4d9d0e9ffda8ccde7cb552acf0ca20f7b5cb Mon Sep 17 00:00:00 2001 From: John Casey Date: Sat, 4 Dec 2021 17:53:52 -0600 Subject: [PATCH 1/2] Add file progress counters to upload / delete logs --- charon/storage.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/charon/storage.py b/charon/storage.py index 51d02dc9..20b7346a 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -108,12 +108,17 @@ def upload_files( uploaded_files = [] - def path_upload_handler(full_file_path: str, path: str) -> bool: + def path_upload_handler(full_file_path: str, path: str, index: int, total: int) -> bool: if not os.path.isfile(full_file_path): logger.warning('Warning: file %s does not exist during uploading. Product: %s', full_file_path, product) return False - logger.info('Uploading %s to bucket %s', full_file_path, bucket_name) + + logger.info( + '(%d/%d) Uploading %s to bucket %s', + index, total, full_file_path, bucket_name + ) + path_key = os.path.join(key_prefix, path) if key_prefix else path fileObject = bucket.Object(path_key) existed = self.file_exists(fileObject) @@ -201,12 +206,17 @@ def upload_metadatas( uploaded_files = [] - def path_upload_handler(full_file_path: str, path: str): + def path_upload_handler(full_file_path: str, path: str, index: int, total: int): if not os.path.isfile(full_file_path): logger.warning('Warning: file %s does not exist during uploading. Product: %s', full_file_path, product) return False - logger.info('Updating metadata %s to bucket %s', path, bucket_name) + + logger.info( + '(%d/%d) Updating metadata %s to bucket %s', + index, total, path, bucket_name + ) + path_key = os.path.join(key_prefix, path) if key_prefix else path file_object = bucket.Object(path_key) existed = self.file_exists(file_object) @@ -282,8 +292,8 @@ def delete_files( deleted_files = [] - def path_delete_handler(full_file_path: str, path: str): - logger.info('Deleting %s from bucket %s', path, bucket_name) + def path_delete_handler(full_file_path: str, path: str, index: int, total: int): + logger.info('(%d/%d) Deleting %s from bucket %s', index, total, path, bucket_name) path_key = os.path.join(key_prefix, path) if key_prefix else path fileObject = bucket.Object(path_key) existed = self.file_exists(fileObject) @@ -449,10 +459,12 @@ def __do_path_cut_and( if not root.endswith("/"): slash_root = slash_root + "/" failed_paths = [] + index = 1 for full_path in file_paths: path = full_path if path.startswith(slash_root): path = path[len(slash_root):] - if not fn(full_path, path): + if not fn(full_path, path, index, len(file_paths)): failed_paths.append(path) + index += 1 return failed_paths From 3568e982b523aa3524227778d64a01a78ae59b17 Mon Sep 17 00:00:00 2001 From: John Casey Date: Tue, 7 Dec 2021 11:02:34 -0600 Subject: [PATCH 2/2] minor corrections to account for file_paths length and typing --- charon/storage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charon/storage.py b/charon/storage.py index 20b7346a..853bbca1 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -453,18 +453,19 @@ def __update_file_metadata( def __do_path_cut_and( self, file_paths: List[str], - fn: Callable[[str, str], bool], root="/" + fn: Callable[[str, str, int, int], bool], root="/" ) -> List[str]: slash_root = root if not root.endswith("/"): slash_root = slash_root + "/" failed_paths = [] index = 1 + file_paths_count = len(file_paths) for full_path in file_paths: path = full_path if path.startswith(slash_root): path = path[len(slash_root):] - if not fn(full_path, path, index, len(file_paths)): + if not fn(full_path, path, index, file_paths_count): failed_paths.append(path) index += 1 return failed_paths