Re design sdk

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-addopts = -n32 --dist=loadscope
+;addopts = -n32 --dist=loadscope

requirements.txt

@@ -2,7 +2,7 @@ pydicom>=2.0.0
 boto3>=1.14.53
 requests==2.26.0
 requests-toolbelt>=0.9.1
-tqdm=4.48.2
+tqdm==4.48.2
 pillow>=7.2.0
 numpy>=1.19.0
 matplotlib>=3.3.1

src/superannotate/lib/app/interface/sdk_interface.py

@@ -59,8 +59,8 @@ def init(path_to_config_json: str):
     :param path_to_config_json: Location to config JSON file
     :type path_to_config_json: str or Path
     """
-    global controller
-    controller = Controller(logger, path_to_config_json)
+    # global controller
+    controller.init(path_to_config_json)
 
 
 @validate_input
@@ -2599,9 +2599,25 @@ def upload_preannotations_from_folder_to_project(
             "The function does not support projects containing videos attached with URLs"
         )
 
+    if recursive_subfolders:
+        logger.info(
+            "When using recursive subfolder parsing same name annotations in different subfolders will overwrite each other.",
+        )
+
+    logger.info(
+        "The JSON files should follow specific naming convention. For Vector projects they should be named '<image_name>___objects.json', for Pixel projects JSON file should be names '<image_name>___pixel.json' and also second mask image file should be present with the name '<image_name>___save.png'. In both cases image with <image_name> should be already present on the platform."
+    )
+    logger.info("Existing annotations will be overwritten.",)
+    logger.info(
+        "Uploading all annotations from %s to project %s.", folder_path, project_name
+    )
+
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
+    logger.info(
+        "Uploading %s annotations to project %s.", len(annotation_paths), project_name
+    )
     uploaded_annotations = []
     failed_annotations = []
     missing_annotations = []

src/superannotate/lib/app/mixp/decorators.py

@@ -1,5 +1,4 @@
 import functools
-import logging
 import sys
 
 from mixpanel import Mixpanel
@@ -11,7 +10,7 @@
 
 mp = Mixpanel(TOKEN)
 
-controller = Controller(logger=logging.getLogger())
+controller = Controller.get_instance()
 res = controller.get_team()
 user_id, team_name = res.data.creator_id, res.data.name
 

src/superannotate/lib/core/serviceproviders.py

@@ -303,7 +303,7 @@ def delete_image_annotations(
         project_id: int,
         folder_id: int = None,
         image_names: List[str] = None,
-    ) -> int:
+    ) -> dict:
         raise NotImplementedError
 
     def get_annotations_delete_progress(

src/superannotate/lib/core/usecases.py

@@ -4077,12 +4077,14 @@ def execute(self):
         image_ids = [image["id"] for image in images]
         image_names = [image["name"] for image in images]
 
-        self._service.run_segmentation(
+        res = self._service.run_segmentation(
             self._project.team_id,
             self._project.uuid,
             model_name=self._ml_model_name,
             image_ids=image_ids,
         )
+        if not res.ok:
+            return self._response
 
         succeded_imgs = []
         failed_imgs = []
@@ -4158,12 +4160,14 @@ def execute(self):
             if model.name == self._ml_model_name:
                 ml_model = model
 
-        self._service.run_prediction(
+        res = self._service.run_prediction(
             team_id=self._project.team_id,
             project_id=self._project.uuid,
             ml_model_id=ml_model.uuid,
             image_ids=image_ids,
         )
+        if not res.ok:
+            return self._response
 
         success_images = []
         failed_images = []
@@ -4266,7 +4270,7 @@ def __init__(
         extensions=constances.DEFAULT_IMAGE_EXTENSIONS,
         annotation_status="NotStarted",
         from_s3_bucket=None,
-        exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS,
+        exclude_file_patterns: List[str] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS,
         recursive_sub_folders: bool = False,
         image_quality_in_editor=None,
     ):
@@ -4286,6 +4290,10 @@ def __init__(
         self._from_s3_bucket = from_s3_bucket
         self._extensions = extensions
         self._recursive_sub_folders = recursive_sub_folders
+        if exclude_file_patterns:
+            list(exclude_file_patterns).extend(
+                list(constances.DEFAULT_FILE_EXCLUDE_PATTERNS)
+            )
         self._exclude_file_patterns = exclude_file_patterns
         self._annotation_status = annotation_status
 
@@ -4302,12 +4310,19 @@ def exclude_file_patterns(self):
         return self._exclude_file_patterns
 
     def validate_annotation_status(self):
-        if self._annotation_status and self._annotation_status not in constances.AnnotationStatus.values():
+        if (
+            self._annotation_status
+            and self._annotation_status.lower()
+            not in constances.AnnotationStatus.values()
+        ):
             raise AppValidationException("Invalid annotations status")
 
     def validate_extensions(self):
         if self._extensions and not all(
-                [extension in constances.DEFAULT_IMAGE_EXTENSIONS  for extension in self._extensions]
+            [
+                extension in constances.DEFAULT_IMAGE_EXTENSIONS
+                for extension in self._extensions
+            ]
         ):
             raise AppValidationException("")
 
@@ -4420,14 +4435,16 @@ def paths(self):
                             paths.append(key)
                             break
 
-        paths = [str(path) for path in paths]
-        return [
-            path
-            for path in paths
-            if "___objects" not in path
-            and "___fuse" not in path
-            and "___pixel" not in path
-        ]
+        data = []
+        for path in paths:
+            if all(
+                [
+                    True if exclude_pattern not in str(path) else False
+                    for exclude_pattern in self.exclude_file_patterns
+                ]
+            ):
+                data.append(str(path))
+        return data
 
     @property
     def images_to_upload(self):
@@ -4490,8 +4507,9 @@ def execute(self):
                     annotation_status=self._annotation_status,
                 ).execute()
 
-                attachments, duplications = response.data
+                attachments, attach_duplications = response.data
                 uploaded.extend(attachments)
+                duplications.extend(attach_duplications)
             uploaded = [image["name"] for image in uploaded]
             failed_images = [image.split("/")[-1] for image in failed_images]
 
@@ -4517,17 +4535,12 @@ def __init__(
 
     def execute(self) -> Response:
 
-        if self._folder.name == "root" and not self._image_names:
-            response = self._backend_service.delete_image_annotations(
-                project_id=self._project.uuid, team_id=self._project.team_id, image_names=self._image_names
-            )
-        else:
-            response = self._backend_service.delete_image_annotations(
-                project_id=self._project.uuid,
-                team_id=self._project.team_id,
-                folder_id=self._folder.uuid,
-                image_names=self._image_names,
-            )
+        response = self._backend_service.delete_image_annotations(
+            project_id=self._project.uuid,
+            team_id=self._project.team_id,
+            folder_id=self._folder.uuid,
+            image_names=self._image_names,
+        )
 
         if response:
             timeout_start = time.time()

src/superannotate/lib/infrastructure/controller.py

@@ -38,45 +38,58 @@ def __call__(cls, *args, **kwargs):
             SingleInstanceMetaClass._instances[cls] = super().__call__(*args, **kwargs)
         return SingleInstanceMetaClass._instances[cls]
 
+    def get_instance(cls):
+        if cls._instances:
+            return cls._instances[cls]
+
 
 class BaseController(metaclass=SingleInstanceMetaClass):
     def __init__(self, logger, config_path=constances.CONFIG_FILE_LOCATION):
-        self._config_path = config_path
+        self._config_path = None
+        self._backend_client = None
         self._logger = logger
+        self._s3_upload_auth_data = None
+        self._projects = None
+        self._folders = None
+        self._teams = None
+        self._images = None
+        self._ml_models = None
+        self._team_id = None
+        self.init(config_path)
+
+    def init(self, config_path):
+        self._config_path = config_path
         token, main_endpoint = (
-            self.configs.get_one("token"),
-            self.configs.get_one("main_endpoint"),
+            self.configs.get_one("token").value,
+            self.configs.get_one("main_endpoint").value,
         )
         if not main_endpoint:
             self.configs.insert(ConfigEntity("main_endpoint", constances.BACKEND_URL))
         if not token:
             self.configs.insert(ConfigEntity("token", ""))
-            logger.warning("Fill config.json")
+            self._logger.warning("Fill config.json")
             return
         verify_ssl_entity = self.configs.get_one("ssl_verify")
         if not verify_ssl_entity:
             verify_ssl = True
         else:
             verify_ssl = verify_ssl_entity.value
-        self._backend_client = SuperannotateBackendService(
-            api_url=self.configs.get_one("main_endpoint").value,
-            auth_token=ConfigRepository().get_one("token").value,
-            logger=logger,
-            verify_ssl=verify_ssl
-        )
-        self._s3_upload_auth_data = None
-        self._projects = None
-        self._folders = None
-        self._teams = None
-        self._images = None
-        self._ml_models = None
-        self._team_id = None
+        if not self._backend_client:
+            self._backend_client = SuperannotateBackendService(
+                api_url=self.configs.get_one("main_endpoint").value,
+                auth_token=self.configs.get_one("token").value,
+                logger=self._logger,
+                verify_ssl=verify_ssl,
+            )
+        else:
+            self._backend_client.api_url = self.configs.get_one("main_endpoint").value
+            self._backend_client._auth_token = self.configs.get_one("token").value
 
     def set_token(self, token):
         self.configs.insert(ConfigEntity("token", token))
         self._backend_client = SuperannotateBackendService(
             api_url=self.configs.get_one("main_endpoint").value,
-            auth_token=ConfigRepository().get_one("token").value,
+            auth_token=self.configs.get_one("token").value,
             logger=self._logger,
         )
 

src/superannotate/lib/infrastructure/services.py

@@ -932,7 +932,7 @@ def run_segmentation(
             params={"team_id": team_id, "project_id": project_id},
             data={"model_name": model_name, "image_ids": image_ids},
         )
-        return res.json()
+        return res
 
     def run_prediction(
         self, team_id: int, project_id: int, ml_model_id: int, image_ids: list
@@ -948,15 +948,15 @@ def run_prediction(
                 "image_ids": image_ids,
             },
         )
-        return res.json()
+        return res
 
     def delete_image_annotations(
         self,
         team_id: int,
         project_id: int,
         folder_id: int = None,
         image_names: List[str] = None,
-    ) -> int:
+    ) -> dict:
         delete_annotations_url = urljoin(self.api_url, self.URL_DELETE_ANNOTATIONS)
         params = {"team_id": team_id, "project_id": project_id}
         data = {}

tests/integration/test_annotation_delete.py

@@ -46,6 +46,25 @@ def test_delete_annotations(self):
         self.assertIsNotNone(data["annotation_json_filename"])
         self.assertIsNone(data["annotation_mask"])
 
+    @pytest.mark.skip(
+        "waiting for deployment to dev",
+    )
+    def test_delete_annotations_by_name(self):
+        sa.upload_images_from_folder_to_project(
+            self.PROJECT_NAME, self.folder_path, annotation_status="InProgress"
+        )
+        sa.create_annotation_classes_from_classes_json(
+            self.PROJECT_NAME, self.folder_path + "/classes/classes.json"
+        )
+        sa.upload_annotations_from_folder_to_project(
+            self.PROJECT_NAME, f"{self.folder_path}"
+        )
+        sa.delete_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1])
+        data = sa.get_image_annotations(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1)
+        self.assertIsNone(data["annotation_json"])
+        self.assertIsNotNone(data["annotation_json_filename"])
+        self.assertIsNone(data["annotation_mask"])
+
     @pytest.mark.skip(
         "waiting for deployment to dev",
     )
@@ -61,3 +80,39 @@ def test_delete_annotations_by_not_existing_name(self):
         )
         self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2])
 
+    @pytest.mark.skip(
+        "waiting for deployment to dev",
+    )
+    def test_delete_annotations_wrong_path(self):
+        sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME)
+        sa.upload_images_from_folder_to_project(
+            f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.folder_path, annotation_status="InProgress"
+        )
+        sa.create_annotation_classes_from_classes_json(
+            self.PROJECT_NAME, self.folder_path + "/classes/classes.json"
+        )
+        sa.upload_annotations_from_folder_to_project(
+            f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", f"{self.folder_path}"
+        )
+        self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1])
+
+    @pytest.mark.skip(
+        "waiting for deployment to dev",
+    )
+    def test_delete_annotations_from_folder(self):
+        sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME)
+
+        sa.upload_images_from_folder_to_project(
+            f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.folder_path, annotation_status="InProgress"
+        )
+        sa.create_annotation_classes_from_classes_json(
+            self.PROJECT_NAME, self.folder_path + "/classes/classes.json"
+        )
+        sa.upload_annotations_from_folder_to_project(
+            f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", f"{self.folder_path}"
+        )
+        sa.delete_annotations(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", [self.EXAMPLE_IMAGE_1])
+        data = sa.get_image_annotations(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.EXAMPLE_IMAGE_1)
+        self.assertIsNone(data["annotation_json"])
+        self.assertIsNotNone(data["annotation_json_filename"])
+        self.assertIsNone(data["annotation_mask"])

tests/integration/test_interface.py

@@ -58,7 +58,7 @@ def test_upload_annotations_from_folder_to_project(self):
         sa.upload_images_from_folder_to_project(
             self.PROJECT_NAME,
             self.folder_path,
-            annotation_status="InProgress",
+            annotation_status="Completed",
         )
         uploaded_annotations, _, _ = sa.upload_annotations_from_folder_to_project(
             self.PROJECT_NAME, self.folder_path