From ddd126946092b18b41dbd14200aa7827e238da45 Mon Sep 17 00:00:00 2001 From: nareksa Date: Tue, 2 Apr 2024 20:26:36 +0400 Subject: [PATCH 1/2] fix upload_images_to_project in case of duplicate paths --- src/superannotate/lib/core/usecases/images.py | 2 +- ...e_image_upload.py => test_image_upload.py} | 56 ++++++++++++++++++- tests/integration/test_single_image_upload.py | 43 -------------- 3 files changed, 55 insertions(+), 46 deletions(-) rename tests/integration/{test_duplicate_image_upload.py => test_image_upload.py} (52%) delete mode 100644 tests/integration/test_single_image_upload.py diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index f66df1d48..3e60e4e79 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -1024,7 +1024,7 @@ def filter_paths(self, paths: List[str]): duplicated_paths = [] for file_name in name_path_map: if len(name_path_map[file_name]) > 1: - duplicated_paths.append(name_path_map[file_name][1:]) + duplicated_paths.extend(name_path_map[file_name][1:]) filtered_paths.append(name_path_map[file_name][0]) image_list = [] diff --git a/tests/integration/test_duplicate_image_upload.py b/tests/integration/test_image_upload.py similarity index 52% rename from tests/integration/test_duplicate_image_upload.py rename to tests/integration/test_image_upload.py index f1a4385b3..c4013dd54 100644 --- a/tests/integration/test_duplicate_image_upload.py +++ b/tests/integration/test_image_upload.py @@ -1,3 +1,4 @@ +import io import os from os.path import dirname @@ -7,8 +8,8 @@ sa = SAClient() -class TestDuplicateImage(BaseTestCase): - PROJECT_NAME = "duplicate_image" +class TestSingleImageUpload(BaseTestCase): + PROJECT_NAME = "single_image_upload" PROJECT_DESCRIPTION = "Desc" PROJECT_TYPE = "Vector" TEST_FOLDER_PTH = "data_set" @@ -25,6 +26,41 @@ def classes_path(self): ) def test_single_image_upload(self): + sa.upload_image_to_project( + self.PROJECT_NAME, + self.folder_path + "/example_image_1.jpg", + annotation_status="InProgress", + ) + assert len(sa.search_items(self.PROJECT_NAME)) == 1 + + with open(self.folder_path + "/example_image_1.jpg", "rb") as f: + img = io.BytesIO(f.read()) + + sa.upload_image_to_project( + self.PROJECT_NAME, img, image_name="rr.jpg", annotation_status="InProgress" + ) + + assert len(sa.search_items(self.PROJECT_NAME)) == 2 + + +class TestMultipleImageUpload(BaseTestCase): + PROJECT_NAME = "test_multiple_image_upload" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PTH = "data_set" + TEST_FOLDER_PATH = "data_set/sample_project_vector" + + @property + def folder_path(self): + return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) + + @property + def classes_path(self): + return os.path.join( + dirname(dirname(__file__)), self.TEST_FOLDER_PATH, "classes/classes.json" + ) + + def test_multiple_image_upload(self): (uploaded, could_not_upload, existing_images,) = sa.upload_images_to_project( self.PROJECT_NAME, [ @@ -59,3 +95,19 @@ def test_single_image_upload(self): self.assertEqual(len(uploaded), 0) self.assertEqual(len(could_not_upload), 1) self.assertEqual(len(existing_images), 0) + + def test_multiple_image_upload_with_duplicates(self): + (uploaded, could_not_upload, existing_images,) = sa.upload_images_to_project( + self.PROJECT_NAME, + [ + f"{self.folder_path}/example_image_1.jpg", + f"{self.folder_path}/example_image_1.jpg", + f"{self.folder_path}/example_image_1.jpg", + f"{self.folder_path}/example_image_3.jpg", + f"{self.folder_path}/example_image_4.jpg", + ], + annotation_status="InProgress", + ) + self.assertEqual(len(uploaded), 3) + self.assertEqual(len(could_not_upload), 0) + self.assertEqual(len(existing_images), 2) diff --git a/tests/integration/test_single_image_upload.py b/tests/integration/test_single_image_upload.py deleted file mode 100644 index 7bec9ac51..000000000 --- a/tests/integration/test_single_image_upload.py +++ /dev/null @@ -1,43 +0,0 @@ -import io -import os -from os.path import dirname - -from src.superannotate import SAClient -from tests.integration.base import BaseTestCase - -sa = SAClient() - - -class TestSingleImageUpload(BaseTestCase): - PROJECT_NAME = "single_image_upload" - PROJECT_DESCRIPTION = "Desc" - PROJECT_TYPE = "Vector" - TEST_FOLDER_PTH = "data_set" - TEST_FOLDER_PATH = "data_set/sample_project_vector" - - @property - def folder_path(self): - return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) - - @property - def classes_path(self): - return os.path.join( - dirname(dirname(__file__)), self.TEST_FOLDER_PATH, "classes/classes.json" - ) - - def test_single_image_upload(self): - sa.upload_image_to_project( - self.PROJECT_NAME, - self.folder_path + "/example_image_1.jpg", - annotation_status="InProgress", - ) - assert len(sa.search_items(self.PROJECT_NAME)) == 1 - - with open(self.folder_path + "/example_image_1.jpg", "rb") as f: - img = io.BytesIO(f.read()) - - sa.upload_image_to_project( - self.PROJECT_NAME, img, image_name="rr.jpg", annotation_status="InProgress" - ) - - assert len(sa.search_items(self.PROJECT_NAME)) == 2 From ae5ac902bac985da9bcc0bd8d771dcbe8f9dfe80 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 3 Apr 2024 11:15:53 +0400 Subject: [PATCH 2/2] Updated duplicatinos detection behevior --- .../lib/app/interface/sdk_interface.py | 3 +- src/superannotate/lib/core/usecases/images.py | 4 +- src/superannotate/logger.py | 35 ---------------- tests/integration/test_image_upload.py | 41 ++++++++++++------- 4 files changed, 31 insertions(+), 52 deletions(-) delete mode 100644 src/superannotate/logger.py diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 710cd48a9..20aade691 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2051,8 +2051,7 @@ def upload_images_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", - len(duplicates), + f"{len(duplicates)} duplicated images found that won't be uploaded." ) logger.info(f"Uploading {len(images_to_upload)} images to project {project}.") uploaded, failed_images, duplications = [], [], duplicates diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 3e60e4e79..71e99c101 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -1020,6 +1020,7 @@ def filter_paths(self, paths: List[str]): name_path_map[Path(path).name].append(path) CHUNK_SIZE = UploadImagesToProject.LIST_NAME_CHUNK_SIZE + filtered_paths = [] duplicated_paths = [] for file_name in name_path_map: @@ -1059,7 +1060,7 @@ def images_to_upload(self): def execute(self): if self.is_valid(): - images_to_upload, duplications = self.images_to_upload + images_to_upload, _ = self.images_to_upload images_to_upload = images_to_upload[: self.auth_data["availableImageCount"]] if not images_to_upload: return self._response @@ -1082,6 +1083,7 @@ def execute(self): yield uploaded = [] + duplications = [] # existing items for i in range(0, len(uploaded_images), 100): response = AttachFileUrlsUseCase( project=self._project, diff --git a/src/superannotate/logger.py b/src/superannotate/logger.py deleted file mode 100644 index 12101e28b..000000000 --- a/src/superannotate/logger.py +++ /dev/null @@ -1,35 +0,0 @@ -import logging.config -import os -from logging.handlers import RotatingFileHandler - -import superannotate.lib.core as constances - - -loggers = {} - - -def get_server_logger(): - global loggers - if loggers.get("sa_server"): - return loggers.get("sa_server") - else: - logger = logging.getLogger("sa_server") - logger.propagate = False - logger.setLevel(logging.INFO) - stream_handler = logging.StreamHandler() - logger.addHandler(stream_handler) - try: - log_file_path = os.path.join(constances.LOG_FILE_LOCATION, "sa_server.log") - if os.access(log_file_path, os.W_OK): - file_handler = RotatingFileHandler( - log_file_path, - maxBytes=5 * 1024 * 1024, - backupCount=2, - mode="a", - ) - logger.addHandler(file_handler) - except OSError: - pass - finally: - loggers["sa_server"] = logger - return logger diff --git a/tests/integration/test_image_upload.py b/tests/integration/test_image_upload.py index c4013dd54..e9b60e1e0 100644 --- a/tests/integration/test_image_upload.py +++ b/tests/integration/test_image_upload.py @@ -97,17 +97,30 @@ def test_multiple_image_upload(self): self.assertEqual(len(existing_images), 0) def test_multiple_image_upload_with_duplicates(self): - (uploaded, could_not_upload, existing_images,) = sa.upload_images_to_project( - self.PROJECT_NAME, - [ - f"{self.folder_path}/example_image_1.jpg", - f"{self.folder_path}/example_image_1.jpg", - f"{self.folder_path}/example_image_1.jpg", - f"{self.folder_path}/example_image_3.jpg", - f"{self.folder_path}/example_image_4.jpg", - ], - annotation_status="InProgress", - ) - self.assertEqual(len(uploaded), 3) - self.assertEqual(len(could_not_upload), 0) - self.assertEqual(len(existing_images), 2) + with self.assertLogs("sa", level="INFO") as mock_log: + ( + uploaded, + could_not_upload, + existing_images, + ) = sa.upload_images_to_project( + self.PROJECT_NAME, + [ + f"{self.folder_path}/example_image_1.jpg", + f"{self.folder_path}/example_image_1.jpg", + f"{self.folder_path}/example_image_1.jpg", + f"{self.folder_path}/example_image_3.jpg", + f"{self.folder_path}/example_image_4.jpg", + ], + annotation_status="InProgress", + ) + assert ( + mock_log.records[0].msg + == "2 duplicated images found that won't be uploaded." + ) + assert ( + mock_log.records[1].msg + == "Uploading 3 images to project test_multiple_image_upload." + ) + self.assertEqual(len(uploaded), 3) + self.assertEqual(len(could_not_upload), 0) + self.assertEqual(len(existing_images), 2)