From c68a04ad3eb5d43471d5742393203c78a4144da8 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 29 Sep 2021 13:00:19 +0400 Subject: [PATCH 1/3] Fix pixel recursive --- .../lib/app/interface/sdk_interface.py | 13 ++-- src/superannotate/lib/core/usecases/images.py | 60 +++++++++---------- .../test_recursive_folder_pixel.py | 32 ++++++++++ 3 files changed, 70 insertions(+), 35 deletions(-) create mode 100644 tests/integration/test_recursive_folder_pixel.py diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 462732d08..2ecd32560 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2392,11 +2392,14 @@ def upload_annotations_from_folder_to_project( annotation_paths=annotation_paths, # noqa: E203 client_s3_bucket=from_s3_bucket, ) - with tqdm( - total=len(annotation_paths), desc="Uploading annotations" - ) as progress_bar: - for _ in use_case.execute(): - progress_bar.update(1) + if use_case.is_valid(): + with tqdm( + total=len(use_case.annotations_to_upload), desc="Uploading annotations" + ) as progress_bar: + for _ in use_case.execute(): + progress_bar.update(1) + else: + raise AppException(use_case.response.errors) return use_case.data diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 1b8dcb947..75bd6c14d 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -2662,6 +2662,8 @@ def execute(self): else: from_s3 = None + for _ in range(len(annotations_to_upload) - len(response.data.images)): + yield with concurrent.futures.ThreadPoolExecutor( max_workers=self.MAX_WORKERS ) as executor: @@ -2693,7 +2695,6 @@ def execute(self): failed_annotations = [ annotation.path for annotation in failed_annotations ] - self._response.data = ( uploaded_annotations, failed_annotations, @@ -2705,42 +2706,41 @@ def execute(self): def upload_to_s3( self, image_id: int, image_info, bucket, from_s3, image_id_name_map ): - if from_s3: - file = io.BytesIO() - s3_object = from_s3.Object( - self._client_s3_bucket, image_id_name_map[image_id].path - ) - s3_object.download_fileobj(file) - file.seek(0) - annotation_json = json.load(file) - else: - annotation_json = json.load(open(image_id_name_map[image_id].path)) - - self.fill_classes_data(annotation_json) - - if not self._is_valid_json(annotation_json): - logger.warning(f"Invalid json {image_id_name_map[image_id].path}") - return image_id_name_map[image_id], False - bucket.put_object( - Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json), - ) - if self._project.project_type == constances.ProjectType.PIXEL.value: - mask_filename = ( - image_id_name_map[image_id].name + constances.ANNOTATION_MASK_POSTFIX - ) + try: if from_s3: file = io.BytesIO() s3_object = from_s3.Object( - self._client_s3_bucket, f"{self._folder_path}/{mask_filename}" + self._client_s3_bucket, image_id_name_map[image_id].path ) s3_object.download_fileobj(file) file.seek(0) + annotation_json = json.load(file) else: - with open(f"{self._folder_path}/{mask_filename}", "rb") as mask_file: - file = io.BytesIO(mask_file.read()) - - bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file) - return image_id_name_map[image_id], True + annotation_json = json.load(open(image_id_name_map[image_id].path)) + self.fill_classes_data(annotation_json) + if not self._is_valid_json(annotation_json): + logger.warning(f"Invalid json {image_id_name_map[image_id].path}") + return image_id_name_map[image_id], False + bucket.put_object( + Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json), + ) + if self._project.project_type == constances.ProjectType.PIXEL.value: + mask_path = image_id_name_map[image_id].path.replace("___pixel.json", constances.ANNOTATION_MASK_POSTFIX) + if from_s3: + file = io.BytesIO() + s3_object = from_s3.Object( + self._client_s3_bucket, + mask_path + ) + s3_object.download_fileobj(file) + file.seek(0) + else: + with open(mask_path, "rb") as mask_file: + file = io.BytesIO(mask_file.read()) + bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file) + return image_id_name_map[image_id], True + except Exception as _: + return image_id_name_map[image_id], False def report_missing_data(self): if self.missing_classes: diff --git a/tests/integration/test_recursive_folder_pixel.py b/tests/integration/test_recursive_folder_pixel.py new file mode 100644 index 000000000..069979968 --- /dev/null +++ b/tests/integration/test_recursive_folder_pixel.py @@ -0,0 +1,32 @@ +import os +from os.path import dirname + +import src.superannotate as sa +from tests.integration.base import BaseTestCase + + +class TestRecursiveFolderPixel(BaseTestCase): + PROJECT_NAME = "pixel_recursive_test" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Pixel" + S3_FOLDER_PATH = "pixel_all_fuse" + JSON_POSTFIX = "*.json" + + + def test_recursive_upload_pixel(self): + uploaded, _, duplicated = sa.upload_images_from_folder_to_project(self.PROJECT_NAME, + self.S3_FOLDER_PATH, + from_s3_bucket="test-openseadragon-1212", + recursive_subfolders=True + ) + + uploaded, failed, missing = sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME, + self.S3_FOLDER_PATH, + from_s3_bucket="test-openseadragon-1212", + recursive_subfolders=True + ) + self.assertEqual(115, len(uploaded)) + self.assertEqual(0, len(failed)) + self.assertEqual(0, len(missing)) + + From e8244cc10e5547c7be9654fe4a545987be22a532 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 29 Sep 2021 14:30:58 +0400 Subject: [PATCH 2/3] validate --- src/superannotate/lib/core/__init__.py | 12 +++-- src/superannotate/lib/core/usecases/images.py | 44 +++++++++++++------ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index e7e560cc8..612ba4a27 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -83,12 +83,18 @@ ATTACH_USER_LIMIT_ERROR_MESSAGE = "The number of items you want to attach exceeds the limit of your subscription plan." -COPY_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of 50 000 items per folder." +COPY_FOLDER_LIMIT_ERROR_MESSAGE = ( + "The number of items you want to copy exceeds the limit of 50 000 items per folder." +) COPY_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of 500 000 items per project." -COPY_SUPER_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of your subscription plan." +COPY_SUPER_LIMIT_ERROR_MESSAGE = ( + "The number of items you want to copy exceeds the limit of your subscription plan." +) -MOVE_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to move exceeds the limit of 50 000 items per folder." +MOVE_FOLDER_LIMIT_ERROR_MESSAGE = ( + "The number of items you want to move exceeds the limit of 50 000 items per folder." +) MOVE_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to move exceeds the limit of 500 000 items per project." __alL__ = ( diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 75bd6c14d..ab36caa6b 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -833,14 +833,17 @@ def execute(self): outline_color = 4 * (255,) for instance in self.annotations["instances"]: - if (not instance.get("className")) or (not class_color_map.get(instance["className"])): + if (not instance.get("className")) or ( + not class_color_map.get(instance["className"]) + ): continue color = class_color_map.get(instance["className"]) if not color: class_color_map[instance["className"]] = self.generate_color() for image in images: fill_color = ( - *class_color_map[instance["className"]], 255 if image.type == "fuse" else self.TRANSPARENCY + *class_color_map[instance["className"]], + 255 if image.type == "fuse" else self.TRANSPARENCY, ) if instance["type"] == "bbox": image.content.draw_bbox( @@ -909,7 +912,9 @@ def execute(self): weight, height = image.get_size() empty_image_arr = np.full((height, weight, 4), [0, 0, 0, 255], np.uint8) for annotation in self.annotations["instances"]: - if (not annotation.get("className")) or (not class_color_map.get(annotation["className"])): + if (not annotation.get("className")) or ( + not class_color_map.get(annotation["className"]) + ): continue fill_color = *class_color_map[annotation["className"]], 255 for part in annotation["parts"]: @@ -1989,17 +1994,28 @@ def validate_limitations(self): if self._move: if self._from_project.uuid == self._to_project.uuid: if self._from_folder.uuid == self._to_folder.uuid: - raise AppValidationException("Cannot move image if source_project == destination_project.") + raise AppValidationException( + "Cannot move image if source_project == destination_project." + ) elif response.data.folder_limit.remaining_image_count < 1: - raise AppValidationException(constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE) + raise AppValidationException( + constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE + ) elif response.data.project_limit.remaining_image_count < 1: - raise AppValidationException(constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE) + raise AppValidationException( + constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE + ) else: if response.data.folder_limit.remaining_image_count < 1: raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE) if response.data.project_limit.remaining_image_count < 1: - raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE) - if response.data.super_user_limit and response.data.super_user_limit.remaining_image_count < 1: + raise AppValidationException( + constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE + ) + if ( + response.data.super_user_limit + and response.data.super_user_limit.remaining_image_count < 1 + ): raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE) def execute(self) -> Response: @@ -2722,16 +2738,16 @@ def upload_to_s3( logger.warning(f"Invalid json {image_id_name_map[image_id].path}") return image_id_name_map[image_id], False bucket.put_object( - Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json), + Key=image_info["annotation_json_path"], + Body=json.dumps(annotation_json), ) if self._project.project_type == constances.ProjectType.PIXEL.value: - mask_path = image_id_name_map[image_id].path.replace("___pixel.json", constances.ANNOTATION_MASK_POSTFIX) + mask_path = image_id_name_map[image_id].path.replace( + "___pixel.json", constances.ANNOTATION_MASK_POSTFIX + ) if from_s3: file = io.BytesIO() - s3_object = from_s3.Object( - self._client_s3_bucket, - mask_path - ) + s3_object = from_s3.Object(self._client_s3_bucket, mask_path) s3_object.download_fileobj(file) file.seek(0) else: From dbd973208593d04a9ea7ba640b9a233a940f0a61 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 29 Sep 2021 15:18:27 +0400 Subject: [PATCH 3/3] Add log --- src/superannotate/lib/app/interface/sdk_interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 2ecd32560..7b7d75f21 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2249,6 +2249,8 @@ def attach_image_urls_to_project( logger.warning( constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) ) + logger.info(constances.ATTACHING_FILES_MESSAGE.format(len(images_to_upload))) + use_case = controller.interactive_attach_urls( project_name=project_name, folder_name=folder_name, @@ -3516,6 +3518,7 @@ def attach_document_urls_to_project( logger.warning( constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) ) + logger.info(constances.ATTACHING_FILES_MESSAGE.format(len(images_to_upload))) use_case = controller.interactive_attach_urls( project_name=project_name, folder_name=folder_name,