From 72038bc441018468e29c53523294cdf0be4b95b6 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 7 Nov 2022 10:40:57 +0400 Subject: [PATCH 1/2] Deprecation Tasks --- docs/source/superannotate.sdk.rst | 2 - .../lib/app/interface/cli_interface.py | 38 +--- .../lib/app/interface/sdk_interface.py | 197 ++---------------- src/superannotate/lib/core/usecases/images.py | 143 ------------- .../lib/infrastructure/controller.py | 26 --- 5 files changed, 23 insertions(+), 383 deletions(-) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index fd4e7fac6..53b626f4d 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -42,7 +42,6 @@ ________ .. automethod:: superannotate.SAClient.upload_videos_from_folder_to_project .. _ref_upload_annotations_from_folder_to_project: .. automethod:: superannotate.SAClient.upload_annotations_from_folder_to_project -.. automethod:: superannotate.SAClient.upload_preannotations_from_folder_to_project .. automethod:: superannotate.SAClient.add_contributors_to_project .. automethod:: superannotate.SAClient.get_project_settings .. automethod:: superannotate.SAClient.set_project_default_image_quality_in_editor @@ -107,7 +106,6 @@ ______ .. automethod:: superannotate.SAClient.download_image .. automethod:: superannotate.SAClient.download_image_annotations .. automethod:: superannotate.SAClient.upload_image_annotations -.. automethod:: superannotate.SAClient.copy_image .. automethod:: superannotate.SAClient.pin_image .. automethod:: superannotate.SAClient.add_annotation_bbox_to_image .. automethod:: superannotate.SAClient.add_annotation_point_to_image diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index 7fe0ffec0..410e3c506 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -129,27 +129,6 @@ def export_project( ) sys.exit(0) - def upload_preannotations( - self, project, folder, dataset_name=None, task=None, format=None - ): - """ - To upload preannotations from folder to project use - Optional argument format accepts input annotation format. It can have COCO or SuperAnnotate values. - If the argument is not given then SuperAnnotate (the native annotation format) is assumed. - Only when COCO format is specified dataset-name and task arguments are required. - dataset-name specifies JSON filename (without extension) in . - task specifies the COCO task for conversion. Please see import_annotation_format for more details. - """ - self._upload_annotations( - project=project, - folder=folder, - format=format, - dataset_name=dataset_name, - task=task, - pre=True, - ) - sys.exit(0) - def upload_annotations( self, project, folder, dataset_name=None, task=None, format=None ): @@ -167,13 +146,10 @@ def upload_annotations( format=format, dataset_name=dataset_name, task=task, - pre=False, ) sys.exit(0) - def _upload_annotations( - self, project, folder, format, dataset_name, task, pre=True - ): + def _upload_annotations(self, project, folder, format, dataset_name, task): project_folder_name = project project_name, folder_name = split_project_path(project) project = SAClient().controller.get_project(project_name) @@ -197,14 +173,10 @@ def _upload_annotations( task=task, ) annotations_path = temp_dir - if pre: - SAClient().upload_preannotations_from_folder_to_project( - project_folder_name, annotations_path - ) - else: - SAClient().upload_annotations_from_folder_to_project( - project_folder_name, annotations_path - ) + + SAClient().upload_annotations_from_folder_to_project( + project_folder_name, annotations_path + ) sys.exit(0) def attach_image_urls( diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index e8d0a620f..5c969c030 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -437,112 +437,6 @@ def search_folders( ] return [folder.name for folder in data if not folder.is_root] - def copy_image( - self, - source_project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = False, - copy_annotation_status: Optional[StrictBool] = False, - copy_pin: Optional[StrictBool] = False, - ): - """Copy image to a project. The image's project is the same as destination - project then the name will be changed to _()., - where is the next available number deducted from project image list. - - :param source_project: project name plus optional subfolder in the project (e.g., "project1/folder1") or - metadata of the project of source project - :type source_project: str or dict - :param image_name: image name - :type image_name: str - :param destination_project: project name or metadata of the project of destination project - :type destination_project: str or dict - :param include_annotations: enables annotations copy - :type include_annotations: bool - :param copy_annotation_status: enables annotations status copy - :type copy_annotation_status: bool - :param copy_pin: enables image pin status copy - :type copy_pin: bool - """ - warning_msg = "The SAClient.copy_image method will be deprecated with the Superannotate Python SDK 4.4.6 release" - warnings.warn(warning_msg, DeprecationWarning) - logger.warning(warning_msg) - source_project_name, source_folder_name = extract_project_folder(source_project) - destination_project_name, destination_folder_name = extract_project_folder( - destination_project - ) - source_project_metadata = self.controller.projects.get_by_name( - source_project_name - ).data - destination_project_metadata = self.controller.projects.get_by_name( - destination_project_name - ).data - - if destination_project_metadata.type.value in [ - constants.ProjectType.VIDEO.value, - constants.ProjectType.DOCUMENT.value, - ] or source_project_metadata.type.value in [ - constants.ProjectType.VIDEO.value, - constants.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[source_project_metadata.type]) - - response = self.controller.copy_image( - from_project_name=source_project_name, - from_folder_name=source_folder_name, - to_project_name=destination_project_name, - to_folder_name=destination_folder_name, - image_name=image_name, - copy_annotation_status=copy_annotation_status, - ) - if response.errors: - raise AppException(response.errors) - if copy_pin: - destination_project = self.controller.get_project( - destination_project_metadata - ) - _folder = self.controller.get_folder( - destination_project, destination_folder_name - ) - item = self.controller.items.get_by_name( - destination_project_metadata, _folder, image_name - ).data - item.is_pinned = 1 - self.controller.items.update( - project=destination_project_metadata, - folder=_folder, - image_name=image_name, - is_pinned=1, - ) - if include_annotations: - source_project = self.controller.get_project(source_project_name) - source_folder = self.controller.get_folder( - source_project, source_folder_name - ) - source_image = self.controller.items.get_by_name( - source_project, source_folder, image_name - ).data - destination_project = self.controller.get_project(destination_project) - destination_folder = self.controller.get_folder( - destination_project, destination_folder_name - ) - destination_image = self.controller.items.get_by_name( - destination_project, destination_folder, image_name - ).data - self.controller.annotation_classes.copy_multiple( - source_project=source_project, - source_folder=source_folder, - source_item=source_image, - destination_project=destination_project, - destination_folder=destination_folder, - destination_item=destination_image, - ) - - logger.info( - f"Copied image {source_project}/{image_name}" - f" to {destination_project_name}/{destination_folder_name}." - ) - def get_project_metadata( self, project: Union[NotEmptyStr, dict], @@ -1616,79 +1510,6 @@ def upload_annotations_from_folder_to_project( raise AppException(response.errors) return response.data - def upload_preannotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, - ): - """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. - - The JSON files should follow specific naming convention. For Vector - projects they should be named "___objects.json" (e.g., if - image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects - JSON file should be named "___pixel.json" and also second mask - image file should be present with the name "___save.png". In both cases - image with should be already present on the platform. - - Existing pre-annotations will be overwritten. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param folder_path: from which folder to upload the pre-annotations - :type folder_path: Path-like (str or Path) - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - :param recursive_subfolders: enable recursive subfolder parsing - :type recursive_subfolders: bool - - :return: paths to pre-annotations uploaded and could-not-upload - :rtype: tuple of list of strs - """ - warning_msg = ( - "The SAClient.upload_preannotations_from_folder_to_project" - " method will be deprecated with the Superannotate Python SDK 4.4.6 release" - ) - warnings.warn(warning_msg, DeprecationWarning) - logger.warning(warning_msg) - project_name, folder_name = extract_project_folder(project) - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - project = self.controller.get_project(project_name) - if project.type in [ - constants.ProjectType.VIDEO, - constants.ProjectType.DOCUMENT, - ]: - raise AppException(LIMITED_FUNCTIONS[project.type]) - if recursive_subfolders: - logger.info( - "When using recursive subfolder parsing same name annotations in different " - "subfolders will overwrite each other.", - ) - logger.info( - "The JSON files should follow a specific naming convention, matching file names already present " - "on the platform. Existing annotations will be overwritten" - ) - annotation_paths = get_annotation_paths( - folder_path, from_s3_bucket, recursive_subfolders - ) - logger.info( - f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}." - ) - project, folder = self.controller.get_project_folder(project_name, folder_name) - response = self.controller.annotations.upload_from_folder( - project=project, - folder=folder, - team=self.controller.team, - annotation_paths=annotation_paths, # noqa: E203 - client_s3_bucket=from_s3_bucket, - folder_path=folder_path, - is_pre_annotations=True, - ) - if response.errors: - raise AppException(response.errors) - return response.data - def upload_image_annotations( self, project: Union[NotEmptyStr, dict], @@ -1971,6 +1792,12 @@ def add_annotation_bbox_to_image( :param error: if not None, marks annotation as error (True) or no-error (False) :type error: bool """ + warning_msg = ( + "The SAClient.add_annotation_bbox_to_image method will " + "be deprecated with the Superannotate Python SDK 4.4.7 release" + ) + warnings.warn(warning_msg, DeprecationWarning) + logger.warning(warning_msg) project_name, folder_name = extract_project_folder(project) project = self.controller.get_project(project_name) @@ -2035,6 +1862,12 @@ def add_annotation_point_to_image( :param error: if not None, marks annotation as error (True) or no-error (False) :type error: bool """ + warning_msg = ( + "The SAClient.add_annotation_point_to_image method will " + "be deprecated with the Superannotate Python SDK 4.4.7 release" + ) + warnings.warn(warning_msg, DeprecationWarning) + logger.warning(warning_msg) project, folder = self.controller.get_project_folder_by_path(project) if project.type in [ constants.ProjectType.VIDEO, @@ -2091,6 +1924,12 @@ def add_annotation_comment_to_image( :param resolved: comment resolve status :type resolved: bool """ + warning_msg = ( + "The SAClient.add_annotation_comment_to_image method will " + "be deprecated with the Superannotate Python SDK 4.4.7 release" + ) + warnings.warn(warning_msg, DeprecationWarning) + logger.warning(warning_msg) project_name, folder_name = extract_project_folder(project) project = self.controller.projects.get_by_name(project_name).data if project.type in [ diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index d7d978e6e..46c1f47a8 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -1287,149 +1287,6 @@ def execute(self): return self._response -class CopyImageUseCase(BaseUseCase): - def __init__( - self, - from_project: ProjectEntity, - from_folder: FolderEntity, - image_name: str, - to_project: ProjectEntity, - to_folder: FolderEntity, - service_provider: BaseServiceProvider, - s3_repo, - include_annotations: Optional[bool] = True, - copy_annotation_status: Optional[bool] = True, - copy_pin: Optional[bool] = True, - move=False, - ): - super().__init__() - self._from_project = from_project - self._from_folder = from_folder - self._image_name = image_name - self._to_project = to_project - self._to_folder = to_folder - self._s3_repo = s3_repo - self._include_annotations = include_annotations - self._copy_annotation_status = copy_annotation_status - self._copy_pin = copy_pin - self._service_provider = service_provider - self._move = move - - def validate_copy_path(self): - if ( - self._from_project.name == self._to_project.name - and self._from_folder.name == self._to_folder.name - ): - raise AppValidationException( - "Cannot move image if source_project == destination_project." - ) - - def validate_project_type(self): - if self._from_project.type in ( - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ): - raise AppValidationException( - constances.LIMITED_FUNCTIONS[self._from_project.type] - ) - - def validate_limitations(self): - response = self._service_provider.get_limitations( - project=self._to_project, - folder=self._to_folder, - ) - if not response.ok: - raise AppValidationException(response.error) - - if self._move and self._from_project.id == self._to_project.id: - if self._from_folder.id == self._to_folder.id: - raise AppValidationException( - "Cannot move image if source_project == destination_project." - ) - - if response.data.folder_limit.remaining_image_count < 1: - raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE) - if response.data.project_limit.remaining_image_count < 1: - raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE) - if ( - response.data.user_limit - and response.data.user_limit.remaining_image_count < 1 - ): - raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE) - - @property - def s3_repo(self): - self._auth_data = self._service_provider.get_s3_upload_auth_token( - project=self._to_project, folder=self._to_folder - ).data - if "error" in self._auth_data: - raise AppException(self._auth_data.get("error")) - return self._s3_repo( - self._auth_data["accessKeyId"], - self._auth_data["secretAccessKey"], - self._auth_data["sessionToken"], - self._auth_data["bucket"], - ) - - def execute(self) -> Response: - if self.is_valid(): - image = ( - GetImageUseCase( - project=self._from_project, - folder=self._from_folder, - image_name=self._image_name, - service_provider=self._service_provider, - ) - .execute() - .data - ) - - image_bytes = ( - GetImageBytesUseCase( - project=self._from_project, - folder=self._from_folder, - image=image, - service_provider=self._service_provider, - ) - .execute() - .data - ) - image_path = f"{self._to_folder}/{self._image_name}" - - auth_data = self._service_provider.get_s3_upload_auth_token( - project=self._to_project, folder=self._to_folder - ) - if not auth_data.ok: - raise AppException(auth_data.error) - s3_response = UploadImageS3UseCase( - project=self._to_project, - image_path=image_path, - image=image_bytes, - service_provider=self._service_provider, - upload_path=auth_data.data["filePath"], - s3_repo=self.s3_repo, - ).execute() - if s3_response.errors: - raise AppException(s3_response.errors) - image_entity = s3_response.data - del image_bytes - - attach_response = AttachFileUrlsUseCase( - project=self._to_project, - folder=self._to_folder, - attachments=[image_entity], - service_provider=self._service_provider, - annotation_status=image.annotation_status_code - if self._copy_annotation_status - else None, - upload_state_code=constances.UploadState.BASIC.value, - ).execute() - if attach_response.errors: - raise AppException(attach_response.errors) - self._response.data = image_entity - return self._response - - class DeleteAnnotations(BaseUseCase): POLL_AWAIT_TIME = 2 CHUNK_SIZE = 2000 diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 2a89894db..604ff60c9 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -997,32 +997,6 @@ def update(self, project: ProjectEntity, folder: FolderEntity): ) return use_case.execute() - def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, - ): - from_project = self.get_project(from_project_name) - to_project = self.get_project(to_project_name) - to_folder = self.get_folder(to_project, to_folder_name) - use_case = usecases.CopyImageUseCase( - from_project=from_project, - from_folder=self.get_folder(from_project, from_folder_name), - to_project=to_project, - to_folder=to_folder, - service_provider=self.service_provider, - image_name=image_name, - s3_repo=self.s3_repo, - copy_annotation_status=copy_annotation_status, - move=move, - ) - return use_case.execute() - def un_assign_folder(self, project_name: str, folder_name: str): project_entity = self.get_project(project_name) folder = self.get_folder(project_entity, folder_name) From 3a5403f70411f256d38f8cd207501edd06821001 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 7 Nov 2022 18:11:17 +0400 Subject: [PATCH 2/2] Fix encoding issue --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/usecases/annotations.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 5c969c030..04df7a0cd 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -1277,7 +1277,7 @@ def create_annotation_classes_from_classes_json( file.seek(0) data = file else: - data = open(classes_json) + data = open(classes_json, encoding="utf-8") classes_json = json.load(data) try: annotation_classes = parse_obj_as(List[AnnotationClassEntity], classes_json) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index d0290e6dc..fcc2c04c4 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -532,7 +532,7 @@ async def get_annotation( if self._project.type == constants.ProjectType.PIXEL.value: mask = self.get_annotation_from_s3(self._client_s3_bucket, mask_path) else: - async with aiofiles.open(path) as file: + async with aiofiles.open(path, encoding="utf-8") as file: content = await file.read() if self._project.type == constants.ProjectType.PIXEL.value: async with aiofiles.open(mask_path, "rb") as mask: @@ -835,7 +835,9 @@ def _get_annotation_json(self) -> tuple: ), ) else: - annotation_json = json.load(open(self._annotation_path)) + annotation_json = json.load( + open(self._annotation_path, encoding="utf-8") + ) if self._project.type == constants.ProjectType.PIXEL.value: mask = open( self._annotation_path.replace( @@ -1313,7 +1315,7 @@ def download_annotation_classes(self, path: str): if response.ok: classes_path = Path(path) / "classes" classes_path.mkdir(parents=True, exist_ok=True) - with open(classes_path / "classes.json", "w+") as file: + with open(classes_path / "classes.json", "w+", encoding="utf-8") as file: json.dump( [ i.dict(