diff --git a/charon/storage.py b/charon/storage.py index 51d02dc9..853bbca1 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -108,12 +108,17 @@ def upload_files( uploaded_files = [] - def path_upload_handler(full_file_path: str, path: str) -> bool: + def path_upload_handler(full_file_path: str, path: str, index: int, total: int) -> bool: if not os.path.isfile(full_file_path): logger.warning('Warning: file %s does not exist during uploading. Product: %s', full_file_path, product) return False - logger.info('Uploading %s to bucket %s', full_file_path, bucket_name) + + logger.info( + '(%d/%d) Uploading %s to bucket %s', + index, total, full_file_path, bucket_name + ) + path_key = os.path.join(key_prefix, path) if key_prefix else path fileObject = bucket.Object(path_key) existed = self.file_exists(fileObject) @@ -201,12 +206,17 @@ def upload_metadatas( uploaded_files = [] - def path_upload_handler(full_file_path: str, path: str): + def path_upload_handler(full_file_path: str, path: str, index: int, total: int): if not os.path.isfile(full_file_path): logger.warning('Warning: file %s does not exist during uploading. Product: %s', full_file_path, product) return False - logger.info('Updating metadata %s to bucket %s', path, bucket_name) + + logger.info( + '(%d/%d) Updating metadata %s to bucket %s', + index, total, path, bucket_name + ) + path_key = os.path.join(key_prefix, path) if key_prefix else path file_object = bucket.Object(path_key) existed = self.file_exists(file_object) @@ -282,8 +292,8 @@ def delete_files( deleted_files = [] - def path_delete_handler(full_file_path: str, path: str): - logger.info('Deleting %s from bucket %s', path, bucket_name) + def path_delete_handler(full_file_path: str, path: str, index: int, total: int): + logger.info('(%d/%d) Deleting %s from bucket %s', index, total, path, bucket_name) path_key = os.path.join(key_prefix, path) if key_prefix else path fileObject = bucket.Object(path_key) existed = self.file_exists(fileObject) @@ -443,16 +453,19 @@ def __update_file_metadata( def __do_path_cut_and( self, file_paths: List[str], - fn: Callable[[str, str], bool], root="/" + fn: Callable[[str, str, int, int], bool], root="/" ) -> List[str]: slash_root = root if not root.endswith("/"): slash_root = slash_root + "/" failed_paths = [] + index = 1 + file_paths_count = len(file_paths) for full_path in file_paths: path = full_path if path.startswith(slash_root): path = path[len(slash_root):] - if not fn(full_path, path): + if not fn(full_path, path, index, file_paths_count): failed_paths.append(path) + index += 1 return failed_paths