From a39bdf15a41715f36357858ef5665d504fd2a7b1 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Sun, 6 Nov 2022 18:34:45 +0400 Subject: [PATCH 1/2] fixing chunk size issue with listing names --- src/superannotate/lib/core/usecases/images.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index d7d978e6e..26184a443 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -849,6 +849,7 @@ def execute(self) -> Response: class UploadImagesToProject(BaseInteractiveUseCase): MAX_WORKERS = 10 + LIST_NAME_CHUNK_SIZE = 500 def __init__( self, @@ -1023,6 +1024,7 @@ def filter_paths(self, paths: List[str]): for path in paths: name_path_map[Path(path).name].append(path) + CHUNK_SIZE = UploadImagesFromFolderToProject.LIST_NAME_CHUNK_SIZE filtered_paths = [] duplicated_paths = [] for file_name in name_path_map: @@ -1030,17 +1032,22 @@ def filter_paths(self, paths: List[str]): duplicated_paths.append(name_path_map[file_name][1:]) filtered_paths.append(name_path_map[file_name][0]) - response = self._service_provider.items.list_by_names( - project=self._project, - folder=self._folder, - names=[image.split("/")[-1] for image in filtered_paths], - ) - if not response.ok: - self._response.errors = AppException(response.error) - return self._response + image_list = [] + for i in range(0, len(filtered_paths), CHUNK_SIZE): + response = self._service_provider.items.list_by_names( + project=self._project, + folder=self._folder, + names=[ + image.split("/")[-1] for image in filtered_paths[i : i + CHUNK_SIZE] + ], + ) + + if not response.ok: + raise AppException(response.error) + image_list.extend([image.name for image in response.data]) + image_list=set(image_list) images_to_upload = [] - image_list = [image.name for image in response.data] for path in filtered_paths: if Path(path).name not in image_list: From db74f92e9ca64ad09780b478787684a9820133b8 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Mon, 7 Nov 2022 10:17:09 +0400 Subject: [PATCH 2/2] using correct baseclasses --- src/superannotate/lib/core/usecases/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 26184a443..7954b0bcd 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -1024,7 +1024,7 @@ def filter_paths(self, paths: List[str]): for path in paths: name_path_map[Path(path).name].append(path) - CHUNK_SIZE = UploadImagesFromFolderToProject.LIST_NAME_CHUNK_SIZE + CHUNK_SIZE = UploadImagesToProject.LIST_NAME_CHUNK_SIZE filtered_paths = [] duplicated_paths = [] for file_name in name_path_map: