diff --git a/pytest.ini b/pytest.ini index af955261e..bcc45fa11 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.0 log_cli=true python_files = test_*.py -addopts = -n32 --dist=loadscope +;addopts = -n32 --dist=loadscope diff --git a/requirements.txt b/requirements.txt index 85b2f5f5c..6ee0714a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ pydicom>=2.0.0 boto3>=1.14.53 requests==2.26.0 requests-toolbelt>=0.9.1 -tqdm=4.48.2 +tqdm==4.48.2 pillow>=7.2.0 numpy>=1.19.0 matplotlib>=3.3.1 diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 89a14f174..e7d3dd435 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -59,8 +59,8 @@ def init(path_to_config_json: str): :param path_to_config_json: Location to config JSON file :type path_to_config_json: str or Path """ - global controller - controller = Controller(logger, path_to_config_json) + # global controller + controller.init(path_to_config_json) @validate_input @@ -2599,9 +2599,25 @@ def upload_preannotations_from_folder_to_project( "The function does not support projects containing videos attached with URLs" ) + if recursive_subfolders: + logger.info( + "When using recursive subfolder parsing same name annotations in different subfolders will overwrite each other.", + ) + + logger.info( + "The JSON files should follow specific naming convention. For Vector projects they should be named '___objects.json', for Pixel projects JSON file should be names '___pixel.json' and also second mask image file should be present with the name '___save.png'. In both cases image with should be already present on the platform." + ) + logger.info("Existing annotations will be overwritten.",) + logger.info( + "Uploading all annotations from %s to project %s.", folder_path, project_name + ) + annotation_paths = get_annotation_paths( folder_path, from_s3_bucket, recursive_subfolders ) + logger.info( + "Uploading %s annotations to project %s.", len(annotation_paths), project_name + ) uploaded_annotations = [] failed_annotations = [] missing_annotations = [] diff --git a/src/superannotate/lib/app/mixp/decorators.py b/src/superannotate/lib/app/mixp/decorators.py index 20cb32683..4ddec7a2b 100644 --- a/src/superannotate/lib/app/mixp/decorators.py +++ b/src/superannotate/lib/app/mixp/decorators.py @@ -1,5 +1,4 @@ import functools -import logging import sys from mixpanel import Mixpanel @@ -11,7 +10,7 @@ mp = Mixpanel(TOKEN) -controller = Controller(logger=logging.getLogger()) +controller = Controller.get_instance() res = controller.get_team() user_id, team_name = res.data.creator_id, res.data.name diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index d12ac70d3..4a02c22eb 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -303,7 +303,7 @@ def delete_image_annotations( project_id: int, folder_id: int = None, image_names: List[str] = None, - ) -> int: + ) -> dict: raise NotImplementedError def get_annotations_delete_progress( diff --git a/src/superannotate/lib/core/usecases.py b/src/superannotate/lib/core/usecases.py index 087748836..a5be1b1be 100644 --- a/src/superannotate/lib/core/usecases.py +++ b/src/superannotate/lib/core/usecases.py @@ -4077,12 +4077,14 @@ def execute(self): image_ids = [image["id"] for image in images] image_names = [image["name"] for image in images] - self._service.run_segmentation( + res = self._service.run_segmentation( self._project.team_id, self._project.uuid, model_name=self._ml_model_name, image_ids=image_ids, ) + if not res.ok: + return self._response succeded_imgs = [] failed_imgs = [] @@ -4158,12 +4160,14 @@ def execute(self): if model.name == self._ml_model_name: ml_model = model - self._service.run_prediction( + res = self._service.run_prediction( team_id=self._project.team_id, project_id=self._project.uuid, ml_model_id=ml_model.uuid, image_ids=image_ids, ) + if not res.ok: + return self._response success_images = [] failed_images = [] @@ -4266,7 +4270,7 @@ def __init__( extensions=constances.DEFAULT_IMAGE_EXTENSIONS, annotation_status="NotStarted", from_s3_bucket=None, - exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + exclude_file_patterns: List[str] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, recursive_sub_folders: bool = False, image_quality_in_editor=None, ): @@ -4286,6 +4290,10 @@ def __init__( self._from_s3_bucket = from_s3_bucket self._extensions = extensions self._recursive_sub_folders = recursive_sub_folders + if exclude_file_patterns: + list(exclude_file_patterns).extend( + list(constances.DEFAULT_FILE_EXCLUDE_PATTERNS) + ) self._exclude_file_patterns = exclude_file_patterns self._annotation_status = annotation_status @@ -4302,12 +4310,19 @@ def exclude_file_patterns(self): return self._exclude_file_patterns def validate_annotation_status(self): - if self._annotation_status and self._annotation_status not in constances.AnnotationStatus.values(): + if ( + self._annotation_status + and self._annotation_status.lower() + not in constances.AnnotationStatus.values() + ): raise AppValidationException("Invalid annotations status") def validate_extensions(self): if self._extensions and not all( - [extension in constances.DEFAULT_IMAGE_EXTENSIONS for extension in self._extensions] + [ + extension in constances.DEFAULT_IMAGE_EXTENSIONS + for extension in self._extensions + ] ): raise AppValidationException("") @@ -4420,14 +4435,16 @@ def paths(self): paths.append(key) break - paths = [str(path) for path in paths] - return [ - path - for path in paths - if "___objects" not in path - and "___fuse" not in path - and "___pixel" not in path - ] + data = [] + for path in paths: + if all( + [ + True if exclude_pattern not in str(path) else False + for exclude_pattern in self.exclude_file_patterns + ] + ): + data.append(str(path)) + return data @property def images_to_upload(self): @@ -4490,8 +4507,9 @@ def execute(self): annotation_status=self._annotation_status, ).execute() - attachments, duplications = response.data + attachments, attach_duplications = response.data uploaded.extend(attachments) + duplications.extend(attach_duplications) uploaded = [image["name"] for image in uploaded] failed_images = [image.split("/")[-1] for image in failed_images] @@ -4517,17 +4535,12 @@ def __init__( def execute(self) -> Response: - if self._folder.name == "root" and not self._image_names: - response = self._backend_service.delete_image_annotations( - project_id=self._project.uuid, team_id=self._project.team_id, image_names=self._image_names - ) - else: - response = self._backend_service.delete_image_annotations( - project_id=self._project.uuid, - team_id=self._project.team_id, - folder_id=self._folder.uuid, - image_names=self._image_names, - ) + response = self._backend_service.delete_image_annotations( + project_id=self._project.uuid, + team_id=self._project.team_id, + folder_id=self._folder.uuid, + image_names=self._image_names, + ) if response: timeout_start = time.time() diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index c88270572..06844900e 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -38,45 +38,58 @@ def __call__(cls, *args, **kwargs): SingleInstanceMetaClass._instances[cls] = super().__call__(*args, **kwargs) return SingleInstanceMetaClass._instances[cls] + def get_instance(cls): + if cls._instances: + return cls._instances[cls] + class BaseController(metaclass=SingleInstanceMetaClass): def __init__(self, logger, config_path=constances.CONFIG_FILE_LOCATION): - self._config_path = config_path + self._config_path = None + self._backend_client = None self._logger = logger + self._s3_upload_auth_data = None + self._projects = None + self._folders = None + self._teams = None + self._images = None + self._ml_models = None + self._team_id = None + self.init(config_path) + + def init(self, config_path): + self._config_path = config_path token, main_endpoint = ( - self.configs.get_one("token"), - self.configs.get_one("main_endpoint"), + self.configs.get_one("token").value, + self.configs.get_one("main_endpoint").value, ) if not main_endpoint: self.configs.insert(ConfigEntity("main_endpoint", constances.BACKEND_URL)) if not token: self.configs.insert(ConfigEntity("token", "")) - logger.warning("Fill config.json") + self._logger.warning("Fill config.json") return verify_ssl_entity = self.configs.get_one("ssl_verify") if not verify_ssl_entity: verify_ssl = True else: verify_ssl = verify_ssl_entity.value - self._backend_client = SuperannotateBackendService( - api_url=self.configs.get_one("main_endpoint").value, - auth_token=ConfigRepository().get_one("token").value, - logger=logger, - verify_ssl=verify_ssl - ) - self._s3_upload_auth_data = None - self._projects = None - self._folders = None - self._teams = None - self._images = None - self._ml_models = None - self._team_id = None + if not self._backend_client: + self._backend_client = SuperannotateBackendService( + api_url=self.configs.get_one("main_endpoint").value, + auth_token=self.configs.get_one("token").value, + logger=self._logger, + verify_ssl=verify_ssl, + ) + else: + self._backend_client.api_url = self.configs.get_one("main_endpoint").value + self._backend_client._auth_token = self.configs.get_one("token").value def set_token(self, token): self.configs.insert(ConfigEntity("token", token)) self._backend_client = SuperannotateBackendService( api_url=self.configs.get_one("main_endpoint").value, - auth_token=ConfigRepository().get_one("token").value, + auth_token=self.configs.get_one("token").value, logger=self._logger, ) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index eeedf1597..8b25e4862 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -932,7 +932,7 @@ def run_segmentation( params={"team_id": team_id, "project_id": project_id}, data={"model_name": model_name, "image_ids": image_ids}, ) - return res.json() + return res def run_prediction( self, team_id: int, project_id: int, ml_model_id: int, image_ids: list @@ -948,7 +948,7 @@ def run_prediction( "image_ids": image_ids, }, ) - return res.json() + return res def delete_image_annotations( self, @@ -956,7 +956,7 @@ def delete_image_annotations( project_id: int, folder_id: int = None, image_names: List[str] = None, - ) -> int: + ) -> dict: delete_annotations_url = urljoin(self.api_url, self.URL_DELETE_ANNOTATIONS) params = {"team_id": team_id, "project_id": project_id} data = {} diff --git a/tests/integration/test_annotation_delete.py b/tests/integration/test_annotation_delete.py index 6c294d227..f3b4d19dd 100644 --- a/tests/integration/test_annotation_delete.py +++ b/tests/integration/test_annotation_delete.py @@ -46,6 +46,25 @@ def test_delete_annotations(self): self.assertIsNotNone(data["annotation_json_filename"]) self.assertIsNone(data["annotation_mask"]) + @pytest.mark.skip( + "waiting for deployment to dev", + ) + def test_delete_annotations_by_name(self): + sa.upload_images_from_folder_to_project( + self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" + ) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, self.folder_path + "/classes/classes.json" + ) + sa.upload_annotations_from_folder_to_project( + self.PROJECT_NAME, f"{self.folder_path}" + ) + sa.delete_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1]) + data = sa.get_image_annotations(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1) + self.assertIsNone(data["annotation_json"]) + self.assertIsNotNone(data["annotation_json_filename"]) + self.assertIsNone(data["annotation_mask"]) + @pytest.mark.skip( "waiting for deployment to dev", ) @@ -61,3 +80,39 @@ def test_delete_annotations_by_not_existing_name(self): ) self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2]) + @pytest.mark.skip( + "waiting for deployment to dev", + ) + def test_delete_annotations_wrong_path(self): + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) + sa.upload_images_from_folder_to_project( + f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.folder_path, annotation_status="InProgress" + ) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, self.folder_path + "/classes/classes.json" + ) + sa.upload_annotations_from_folder_to_project( + f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", f"{self.folder_path}" + ) + self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1]) + + @pytest.mark.skip( + "waiting for deployment to dev", + ) + def test_delete_annotations_from_folder(self): + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) + + sa.upload_images_from_folder_to_project( + f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.folder_path, annotation_status="InProgress" + ) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, self.folder_path + "/classes/classes.json" + ) + sa.upload_annotations_from_folder_to_project( + f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", f"{self.folder_path}" + ) + sa.delete_annotations(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", [self.EXAMPLE_IMAGE_1]) + data = sa.get_image_annotations(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.EXAMPLE_IMAGE_1) + self.assertIsNone(data["annotation_json"]) + self.assertIsNotNone(data["annotation_json_filename"]) + self.assertIsNone(data["annotation_mask"]) diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 60d07c4cb..e6aaed147 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -58,7 +58,7 @@ def test_upload_annotations_from_folder_to_project(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, - annotation_status="InProgress", + annotation_status="Completed", ) uploaded_annotations, _, _ = sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, self.folder_path