From 0eb04dccfabd0c5bf1ce380e470d23c154b46be1 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 5 Apr 2023 10:29:07 +0400 Subject: [PATCH 01/17] Updated add_contributors_to_project --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index b2a135187..653a6f2ca 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2229,7 +2229,7 @@ def add_contributors_to_project( for email in emails ] response = self.controller.projects.add_contributors( - team=self.controller.team, project=project, contributors=contributors + team=self.controller.get_team().data, project=project, contributors=contributors ) if response.errors: raise AppException(response.errors) From 77b7e4c6245b64582997461cdc5e577b4b9e829c Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 5 Apr 2023 10:42:39 +0400 Subject: [PATCH 02/17] Fixed run_prediction --- src/superannotate/lib/core/usecases/models.py | 60 +++++++++---------- .../lib/infrastructure/repositories.py | 13 ---- .../lib/infrastructure/serviceprovider.py | 1 + tests/integration/test_ml_funcs.py | 11 ++-- 4 files changed, 36 insertions(+), 49 deletions(-) diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 4bc6d1f52..83c8048ee 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -479,7 +479,7 @@ def execute(self): images = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=self._images_list ).data - image_ids = [image.uuid for image in images] + image_ids = [image.id for image in images] image_names = [image.name for image in images] if not len(image_names): @@ -502,36 +502,36 @@ def execute(self): ml_model_id=ml_model.id, image_ids=image_ids, ) - if not res.ok: - return self._response.data - - success_images = [] - failed_images = [] - while len(success_images) + len(failed_images) != len(image_ids): - images_metadata = self._service_provider.items.list_by_names( - project=self._project, folder=self._folder, names=self._images_list - ).data - - success_images = [ - img.name - for img in images_metadata - if img.prediction_status - == constances.SegmentationStatus.COMPLETED.value - ] - failed_images = [ - img.name - for img in images_metadata - if img.prediction_status - == constances.SegmentationStatus.FAILED.value - ] - - complete_images = success_images + failed_images - logger.info( - f"prediction complete on {len(complete_images)} / {len(image_ids)} images" - ) - time.sleep(5) + if res.ok: + success_images = [] + failed_images = [] + while len(success_images) + len(failed_images) != len(image_ids): + images_metadata = self._service_provider.items.list_by_names( + project=self._project, folder=self._folder, names=self._images_list + ).data + + success_images = [ + img.name + for img in images_metadata + if img.prediction_status + == constances.SegmentationStatus.COMPLETED.value + ] + failed_images = [ + img.name + for img in images_metadata + if img.prediction_status + == constances.SegmentationStatus.FAILED.value + ] + + complete_images = success_images + failed_images + logger.info( + f"prediction complete on {len(complete_images)} / {len(image_ids)} images" + ) + time.sleep(5) - self._response.data = (success_images, failed_images) + self._response.data = (success_images, failed_images) + else: + self._response.errors = res.error return self._response diff --git a/src/superannotate/lib/infrastructure/repositories.py b/src/superannotate/lib/infrastructure/repositories.py index 3357bcf26..83c827f5c 100644 --- a/src/superannotate/lib/infrastructure/repositories.py +++ b/src/superannotate/lib/infrastructure/repositories.py @@ -1,8 +1,4 @@ import io -from typing import List - -from lib.core.conditions import Condition -from lib.core.entities import ProjectEntity from lib.core.entities import S3FileEntity from lib.core.repositories import BaseS3Repository @@ -22,12 +18,3 @@ def insert(self, entity: S3FileEntity) -> S3FileEntity: data["Metadata"] = temp self.bucket.put_object(**data) return entity - - def update(self, entity: ProjectEntity): - self._service.update_project(entity.to_dict()) - - def delete(self, uuid: int): - self._service.delete_project(uuid) - - def get_all(self, condition: Condition = None) -> List[ProjectEntity]: - pass diff --git a/src/superannotate/lib/infrastructure/serviceprovider.py b/src/superannotate/lib/infrastructure/serviceprovider.py index 7e000488b..71965a507 100644 --- a/src/superannotate/lib/infrastructure/serviceprovider.py +++ b/src/superannotate/lib/infrastructure/serviceprovider.py @@ -195,6 +195,7 @@ def run_prediction( self.URL_PREDICTION, "post", data={ + "team_id": project.team_id, "project_id": project.id, "ml_model_id": ml_model_id, "image_ids": image_ids, diff --git a/tests/integration/test_ml_funcs.py b/tests/integration/test_ml_funcs.py index 02167694f..3a39a8180 100644 --- a/tests/integration/test_ml_funcs.py +++ b/tests/integration/test_ml_funcs.py @@ -1,10 +1,10 @@ import os -import time from os.path import dirname -import pytest from src.superannotate import SAClient +from src.superannotate import AppException from tests.integration.base import BaseTestCase +import pytest sa = SAClient() @@ -23,17 +23,16 @@ def folder_path(self): return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) def test_run_prediction_with_non_exist_images(self): - with pytest.raises(Exception) as e: + with self.assertRaisesRegexp(AppException, 'No valid image names were provided.'): sa.run_prediction( - self.PROJECT_NAME, ["NonExistantImage.jpg"], self.MODEL_NAME + self.PROJECT_NAME, ["NotExistingImage.jpg"], self.MODEL_NAME ) - @pytest.mark.skip(reason="Need to adjust") + @pytest.mark.skip(reason="Test skipped due to long execution") def test_run_prediction_for_all_images(self): sa.upload_images_from_folder_to_project( project=self.PROJECT_NAME, folder_path=self.folder_path ) - time.sleep(2) image_names_vector = [i["name"] for i in sa.search_items(self.PROJECT_NAME)] succeeded_images, failed_images = sa.run_prediction( self.PROJECT_NAME, image_names_vector, self.MODEL_NAME From dbcbf35f3f4d53efd9ca9381d62ddd54080e3d63 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 5 Apr 2023 10:50:05 +0400 Subject: [PATCH 03/17] structure changes --- .../lib/core/usecases/annotations.py | 262 +++++++++--------- 1 file changed, 137 insertions(+), 125 deletions(-) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 648694c90..1c24060e3 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -10,12 +10,12 @@ import time import traceback import typing -from threading import Thread from dataclasses import dataclass from datetime import datetime from itertools import islice from operator import itemgetter from pathlib import Path +from threading import Thread from typing import Any from typing import Callable from typing import Dict @@ -60,11 +60,20 @@ class AsyncThread(Thread): - def __init__(self, group=None, target=None, name=None, - args=(), kwargs=None, *, daemon=None): - super().__init__(group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon) + def __init__( + self, group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None + ): + super().__init__( + group=group, + target=target, + name=name, + args=args, + kwargs=kwargs, + daemon=daemon, + ) self._exc = None self._response = None + @property def response(self): return self._response @@ -81,11 +90,14 @@ def join(self, timeout=None) -> typing.Any: raise self._exc return self._response + def run_async(f): response = [None] + def wrapper(func: typing.Callable): response[0] = asyncio.run(func) # noqa return response[0] + thread = AsyncThread(target=wrapper, args=(f,)) thread.start() thread.join() @@ -113,7 +125,7 @@ def divide_to_chunks(it, size): def log_report( - report: Report, + report: Report, ): if report.missing_classes: logger.warning( @@ -145,18 +157,18 @@ class Config: def set_annotation_statuses_in_progress( - service_provider: BaseServiceProvider, - project: ProjectEntity, - folder: FolderEntity, - item_names: List[str], - chunk_size=500, + service_provider: BaseServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + item_names: List[str], + chunk_size=500, ) -> bool: failed_on_chunk = False for i in range(0, len(item_names), chunk_size): status_changed = service_provider.items.set_statuses( project=project, folder=folder, - item_names=item_names[i: i + chunk_size], # noqa: E203 + item_names=item_names[i : i + chunk_size], # noqa: E203 annotation_status=constants.AnnotationStatus.IN_PROGRESS.value, ) if not status_changed.ok: @@ -166,13 +178,13 @@ def set_annotation_statuses_in_progress( async def upload_small_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def upload(_chunk): failed_annotations, missing_classes, missing_attr_groups, missing_attrs = ( @@ -217,9 +229,9 @@ async def upload(_chunk): queue.put_nowait(None) break if ( - _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB - or sum([len(i.item.name) for i in chunk]) - >= URI_THRESHOLD - (len(chunk) + 1) * 14 + _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB + or sum([len(i.item.name) for i in chunk]) + >= URI_THRESHOLD - (len(chunk) + 1) * 14 ): await upload(chunk) chunk = [] @@ -231,13 +243,13 @@ async def upload(_chunk): async def upload_big_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]: try: @@ -273,13 +285,13 @@ class UploadAnnotationsUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - annotations: List[dict], - service_provider: BaseServiceProvider, - keep_status: bool = False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + annotations: List[dict], + service_provider: BaseServiceProvider, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -306,7 +318,7 @@ def _validate_json(self, json_data: dict) -> list: def list_existing_items(self, item_names: List[str]) -> List[BaseItemEntity]: existing_items = [] for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -466,17 +478,17 @@ class UploadAnnotationsFromFolderUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - team: TeamEntity, - annotation_paths: List[str], - service_provider: BaseServiceProvider, - pre_annotation: bool = False, - client_s3_bucket=None, - folder_path: str = None, - keep_status=False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + team: TeamEntity, + annotation_paths: List[str], + service_provider: BaseServiceProvider, + pre_annotation: bool = False, + client_s3_bucket=None, + folder_path: str = None, + keep_status=False, ): super().__init__(reporter) self._project = project @@ -517,7 +529,7 @@ def get_name_path_mappings(annotation_paths): return name_path_mappings def _log_report( - self, + self, ): if self._report.missing_classes: logger.warning( @@ -574,7 +586,7 @@ def prepare_annotation(self, annotation: dict, size) -> dict: return annotation async def get_annotation( - self, path: str + self, path: str ) -> (Optional[Tuple[io.StringIO]], Optional[io.BytesIO]): mask = None mask_path = path.replace( @@ -619,12 +631,12 @@ def extract_name(value: Path): return path def get_existing_name_item_mapping( - self, name_path_mappings: Dict[str, str] + self, name_path_mappings: Dict[str, str] ) -> dict: item_names = list(name_path_mappings.keys()) existing_name_item_mapping = {} for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -645,7 +657,7 @@ def annotation_upload_data(self) -> UploadAnnotationAuthData: tmp = self._service_provider.get_annotation_upload_data( project=self._project, folder=self._folder, - item_ids=self._item_ids[i: i + CHUNK_SIZE], + item_ids=self._item_ids[i : i + CHUNK_SIZE], ) if not tmp.ok: raise AppException(tmp.error) @@ -811,22 +823,22 @@ def execute(self): class UploadAnnotationUseCase(BaseReportableUseCase): def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - image: ImageEntity, - team: TeamEntity, - service_provider: BaseServiceProvider, - reporter: Reporter, - annotation_upload_data: UploadAnnotationAuthData = None, - annotations: dict = None, - s3_bucket=None, - client_s3_bucket=None, - mask=None, - verbose: bool = True, - annotation_path: str = None, - pass_validation: bool = False, - keep_status: bool = False, + self, + project: ProjectEntity, + folder: FolderEntity, + image: ImageEntity, + team: TeamEntity, + service_provider: BaseServiceProvider, + reporter: Reporter, + annotation_upload_data: UploadAnnotationAuthData = None, + annotations: dict = None, + s3_bucket=None, + client_s3_bucket=None, + mask=None, + verbose: bool = True, + annotation_path: str = None, + pass_validation: bool = False, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -1008,8 +1020,8 @@ def execute(self): ) if ( - self._project.type == constants.ProjectType.PIXEL.value - and mask + self._project.type == constants.ProjectType.PIXEL.value + and mask ): self.s3_bucket.put_object( Key=self.annotation_upload_data.images[self._image.id][ @@ -1043,14 +1055,14 @@ def execute(self): class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( - self, - config: ConfigEntity, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - video_name: str, - fps: int, - service_provider: BaseServiceProvider, + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + video_name: str, + fps: int, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._config = config @@ -1103,13 +1115,13 @@ class UploadPriorityScoresUseCase(BaseReportableUseCase): CHUNK_SIZE = 100 def __init__( - self, - reporter, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScoreEntity], - project_folder_name: str, - service_provider: BaseServiceProvider, + self, + reporter, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScoreEntity], + project_folder_name: str, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._project = project @@ -1165,8 +1177,8 @@ def execute(self): if iterations: for i in iterations: priorities_to_upload = priorities[ - i: i + self.CHUNK_SIZE - ] # noqa: E203 + i : i + self.CHUNK_SIZE + ] # noqa: E203 res = self._service_provider.projects.upload_priority_scores( project=self._project, folder=self._folder, @@ -1196,12 +1208,12 @@ class ValidateAnnotationUseCase(BaseReportableUseCase): } def __init__( - self, - reporter: Reporter, - team_id: int, - project_type: int, - annotation: dict, - service_provider: BaseServiceProvider, + self, + reporter: Reporter, + team_id: int, + project_type: int, + annotation: dict, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._team_id = team_id @@ -1369,9 +1381,9 @@ def extract_messages(self, path, error, report): for sub_error in sorted(error.context, key=lambda e: e.schema_path): tmp_path = sub_error.path # if sub_error.path else real_path _path = ( - f"{''.join(path)}" - + ("." if tmp_path else "") - + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) + f"{''.join(path)}" + + ("." if tmp_path else "") + + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) ) if sub_error.context: self.extract_messages(_path, sub_error, report) @@ -1407,13 +1419,13 @@ def execute(self) -> Response: class GetAnnotations(BaseReportableUseCase): def __init__( - self, - config: ConfigEntity, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - item_names: Optional[List[str]], - service_provider: BaseServiceProvider, + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + item_names: Optional[List[str]], + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._config = config @@ -1480,9 +1492,9 @@ async def get_small_annotations(self, item_ids: List[int]): ) async def run_workers( - self, - big_annotations: List[BaseItemEntity], - small_annotations: List[List[Dict]], + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[List[Dict]], ): annotations = [] if big_annotations: @@ -1506,7 +1518,7 @@ async def run_workers( ) if small_annotations: for chunks in divide_to_chunks( - small_annotations, self._config.MAX_COROUTINE_COUNT + small_annotations, self._config.MAX_COROUTINE_COUNT ): tasks = [] for chunk in chunks: @@ -1573,16 +1585,16 @@ def execute(self): class DownloadAnnotations(BaseReportableUseCase): def __init__( - self, - config: ConfigEntity, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - destination: str, - recursive: bool, - item_names: List[str], - service_provider: BaseServiceProvider, - callback: Callable = None, + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + destination: str, + recursive: bool, + item_names: List[str], + service_provider: BaseServiceProvider, + callback: Callable = None, ): super().__init__(reporter) self._config = config @@ -1609,7 +1621,7 @@ def validate_destination(self): if self._destination: destination = str(self._destination) if not os.path.exists(destination) or not os.access( - destination, os.X_OK | os.W_OK + destination, os.X_OK | os.W_OK ): raise AppException( f"Local path {destination} is not an existing directory or access denied." @@ -1673,7 +1685,7 @@ async def download_big_annotations(self, export_path): break async def download_small_annotations( - self, item_ids: List[int], export_path, folder: FolderEntity + self, item_ids: List[int], export_path, folder: FolderEntity ): postfix = self.get_postfix() await self._service_provider.annotations.download_small_annotations( @@ -1687,11 +1699,11 @@ async def download_small_annotations( ) async def run_workers( - self, - big_annotations: List[BaseItemEntity], - small_annotations: List[List[dict]], - folder: FolderEntity, - export_path, + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[List[dict]], + folder: FolderEntity, + export_path, ): if big_annotations: self._big_file_queue = asyncio.Queue() @@ -1707,7 +1719,7 @@ async def run_workers( if small_annotations: for chunks in divide_to_chunks( - small_annotations, self._config.MAX_COROUTINE_COUNT + small_annotations, self._config.MAX_COROUTINE_COUNT ): tasks = [] for chunk in chunks: From 302cef5373178cf07ff15c683efb8ab25771b448 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 5 Apr 2023 11:22:56 +0400 Subject: [PATCH 04/17] removed create_project_from_metadata get_project_image_count --- docs/source/api_reference/api_project.rst | 2 - .../lib/app/interface/sdk_interface.py | 74 -------- .../lib/core/usecases/projects.py | 43 ----- .../lib/infrastructure/controller.py | 16 -- tests/integration/folders/test_folders.py | 37 ++-- .../test_create_project_from_metadata.py | 158 ------------------ tests/integration/settings/test_settings.py | 20 --- .../test_depricated_functions_document.py | 4 - .../test_depricated_functions_video.py | 4 - 9 files changed, 11 insertions(+), 347 deletions(-) delete mode 100644 tests/integration/projects/test_create_project_from_metadata.py diff --git a/docs/source/api_reference/api_project.rst b/docs/source/api_reference/api_project.rst index ca6e23344..96c86fe39 100644 --- a/docs/source/api_reference/api_project.rst +++ b/docs/source/api_reference/api_project.rst @@ -5,7 +5,6 @@ Projects .. _ref_search_projects: .. automethod:: superannotate.SAClient.create_project .. automethod:: superannotate.SAClient.search_projects -.. automethod:: superannotate.SAClient.create_project_from_metadata .. automethod:: superannotate.SAClient.clone_project .. automethod:: superannotate.SAClient.rename_project .. automethod:: superannotate.SAClient.delete_project @@ -13,7 +12,6 @@ Projects .. automethod:: superannotate.SAClient.get_project_by_id .. automethod:: superannotate.SAClient.set_project_status .. automethod:: superannotate.SAClient.get_project_metadata -.. automethod:: superannotate.SAClient.get_project_image_count .. automethod:: superannotate.SAClient.upload_images_to_project .. automethod:: superannotate.SAClient.attach_items_from_integrated_storage .. automethod:: superannotate.SAClient.upload_image_to_project diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index b2a135187..98d4826a4 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -5,7 +5,6 @@ import logging import os import sys -import warnings from pathlib import Path from typing import Callable from typing import Dict @@ -47,7 +46,6 @@ from lib.app.serializers import TeamSerializer from lib.core import LIMITED_FUNCTIONS from lib.core import entities -from lib.core import enums from lib.core.conditions import CONDITION_EQ as EQ from lib.core.conditions import Condition from lib.core.conditions import EmptyCondition @@ -385,45 +383,6 @@ def create_project( project.workflows = self.controller.projects.list_workflow(project).data return ProjectSerializer(project).serialize() - def create_project_from_metadata(self, project_metadata: Project): - """Create a new project in the team using project metadata object dict. - - | Mandatory keys: “name”, “description” and “type” (Vector or Pixel) - | Non-mandatory keys: “workflow”, “settings” and “annotation_classes” - - :param project_metadata: project metadata - :type project_metadata: dict - - :return: dict object metadata the new project - :rtype: dict - """ - deprecation_msg = '"create_project_from_metadata" is deprecated and replaced by "create_project"' - warnings.warn(deprecation_msg, DeprecationWarning) - logger.warning(deprecation_msg) - - project_metadata = project_metadata.dict() - if project_metadata["type"] not in enums.ProjectType.titles(): - raise AppException( - "Please provide a valid project type: Vector, Pixel, Document, or Video." - ) - - response = self.controller.projects.create( - entities.ProjectEntity( - name=project_metadata["name"], - description=project_metadata.get("description"), - type=constants.ProjectType.get_value(project_metadata["type"]), - settings=parse_obj_as( - List[SettingEntity], project_metadata.get("settings", []) - ), - classes=project_metadata.get("classes", []), - workflows=project_metadata.get("workflows", []), - instructions_link=project_metadata.get("instructions_link"), - ), - ) - if response.errors: - raise AppException(response.errors) - return ProjectSerializer(response.data).serialize() - def clone_project( self, project_name: Union[NotEmptyStr, dict], @@ -1112,39 +1071,6 @@ def upload_images_from_folder_to_project( return use_case.data raise AppException(use_case.response.errors) - def get_project_image_count( - self, - project: Union[NotEmptyStr, dict], - with_all_subfolders: Optional[StrictBool] = False, - ): - """Returns number of images in the project. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param with_all_subfolders: enables recursive folder counting - :type with_all_subfolders: bool - - :return: number of images in the project - :rtype: int - """ - deprecation_msg = ( - "“get_project_image_count” is deprecated and replaced" - " by “item_count” value will be included in project metadata." - ) - - warnings.warn(deprecation_msg, DeprecationWarning) - logger.warning(deprecation_msg) - project_name, folder_name = extract_project_folder(project) - - response = self.controller.get_project_image_count( - project_name=project_name, - folder_name=folder_name, - with_all_subfolders=with_all_subfolders, - ) - if response.errors: - raise AppException(response.errors) - return response.data - def download_image_annotations( self, project: Union[NotEmptyStr, dict], diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index e671e32cd..69d55ccdf 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -7,7 +7,6 @@ from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import ContributorEntity -from lib.core.entities import FolderEntity from lib.core.entities import ProjectEntity from lib.core.entities import SettingEntity from lib.core.entities import TeamEntity @@ -508,48 +507,6 @@ def execute(self): return self._response -class GetProjectImageCountUseCase(BaseUseCase): - def __init__( - self, - service_provider: BaseServiceProvider, - project: ProjectEntity, - folder: FolderEntity, - with_all_sub_folders: bool = False, - ): - super().__init__() - self._service_provider = service_provider - self._project = project - self._folder = folder - self._with_all_sub_folders = with_all_sub_folders - - def validate_user_input(self): - if not self._folder.name == "root" and self._with_all_sub_folders: - raise AppValidationException("The folder does not contain any sub-folders.") - - def validate_project_type(self): - if self._project.type in constances.LIMITED_FUNCTIONS: - raise AppValidationException( - constances.LIMITED_FUNCTIONS[self._project.type] - ) - - def execute(self): - if self.is_valid(): - data = self._service_provider.get_project_images_count(self._project).data - count = 0 - if self._folder.name == "root": - count += data["images"]["count"] - if self._with_all_sub_folders: - for i in data["folders"]["data"]: - count += i["imagesCount"] - else: - for i in data["folders"]["data"]: - if i["id"] == self._folder.id: - count = i["imagesCount"] - - self._response.data = count - return self._response - - class SetWorkflowUseCase(BaseUseCase): def __init__( self, diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index e2e2d2d41..dcc9ff12f 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1073,22 +1073,6 @@ def get_exports(self, project_name: str, return_metadata: bool): ) return use_case.execute() - def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool - ): - - project = self.get_project(project_name) - folder = self.get_folder(project=project, name=folder_name) - - use_case = usecases.GetProjectImageCountUseCase( - service_provider=self.service_provider, - project=project, - folder=folder, - with_all_sub_folders=with_all_subfolders, - ) - - return use_case.execute() - def download_image( self, project_name: str, diff --git a/tests/integration/folders/test_folders.py b/tests/integration/folders/test_folders.py index 609fbd58c..345397a23 100644 --- a/tests/integration/folders/test_folders.py +++ b/tests/integration/folders/test_folders.py @@ -174,8 +174,6 @@ def test_project_folder_image_count(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) - num_images = sa.get_project_image_count(self.PROJECT_NAME) - self.assertEqual(num_images, 4) sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) sa.upload_images_from_folder_to_project( @@ -183,18 +181,11 @@ def test_project_folder_image_count(self): self.folder_path, annotation_status="InProgress", ) - num_images = sa.get_project_image_count(self.PROJECT_NAME) - self.assertEqual(num_images, 4) + num_images = sa.search_items(self.PROJECT_NAME) + assert len(num_images) == 4 - num_images = sa.get_project_image_count( - self.PROJECT_NAME + f"/{self.TEST_FOLDER_NAME_1}" - ) - self.assertEqual(num_images, 4) - - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 8) + num_images = sa.search_items(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}") + assert len(num_images) == 4 def test_delete_items(self): sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) @@ -204,26 +195,20 @@ def test_delete_items(self): self.folder_path, annotation_status="InProgress", ) - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 4) + num_images = sa.search_items(self.PROJECT_NAME) + assert len(num_images) == 4 sa.delete_items( f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", [self.EXAMPLE_IMAGE_2, self.EXAMPLE_IMAGE_3], ) - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 2) + num_images = sa.search_items(self.PROJECT_NAME) + assert len(num_images) == 2 sa.delete_items(self.PROJECT_NAME, None) - time.sleep(2) - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=False - ) - self.assertEqual(num_images, 0) + + num_images = sa.search_items(self.PROJECT_NAME) + assert len(num_images) == 0 @pytest.mark.flaky(reruns=2) def test_project_completed_count(self): diff --git a/tests/integration/projects/test_create_project_from_metadata.py b/tests/integration/projects/test_create_project_from_metadata.py deleted file mode 100644 index fa6892d6e..000000000 --- a/tests/integration/projects/test_create_project_from_metadata.py +++ /dev/null @@ -1,158 +0,0 @@ -from unittest import TestCase - -from src.superannotate import SAClient - -sa = SAClient() - - -class CreateProjectFromMetadata(TestCase): - PROJECT_1 = "pr_1" - PROJECT_2 = "pr_2" - - def setUp(self) -> None: - self.tearDown() - - def tearDown(self) -> None: - for project_name in self.PROJECT_1, self.PROJECT_2: - try: - sa.delete_project(project_name) - except Exception: - pass - - def test_create_project_with_default_attribute(self): - sa.create_project( - self.PROJECT_1, project_type="Vector", project_description="Desc" - ) - sa.create_annotation_classes_from_classes_json( - self.PROJECT_1, - classes_json=[ - { - "name": "Personal vehicle", - "color": "#ecb65f", - "count": 25, - "createdAt": "2020-10-12T11:35:20.000Z", - "updatedAt": "2020-10-12T11:48:19.000Z", - "attribute_groups": [ - { - "name": "test", - "attributes": [ - {"name": "Car"}, - {"name": "Track"}, - {"name": "Bus"}, - ], - "default_value": "Bus", - "is_multiselect": 0, - } - ], - } - ], - ) - pr_1_metadata = sa.get_project_metadata( - self.PROJECT_1, include_annotation_classes=True - ) - pr_1_metadata["name"] = self.PROJECT_2 - sa.create_project_from_metadata(pr_1_metadata) - pr_2_metadata = sa.get_project_metadata( - self.PROJECT_2, include_annotation_classes=True - ) - assert ( - pr_2_metadata["classes"][0]["attribute_groups"][0]["default_value"] == "Bus" - ) - assert ( - "is_multiselect" not in pr_2_metadata["classes"][0]["attribute_groups"][0] - ) - - def test_metadata_create_workflow(self): - sa.create_project( - self.PROJECT_1, project_type="Vector", project_description="Desc" - ) - sa.create_annotation_class( - self.PROJECT_1, - "class1", - "#FFAAFF", - [ - { - "name": "tall", - "is_multiselect": 0, - "attributes": [{"name": "yes"}, {"name": "no"}], - }, - { - "name": "age", - "is_multiselect": 0, - "attributes": [{"name": "young"}, {"name": "old"}], - }, - ], - ) - sa.create_annotation_class( - self.PROJECT_1, - "class2", - "#FFAAFF", - [ - { - "name": "tall", - "is_multiselect": 0, - "attributes": [{"name": "yes"}, {"name": "no"}], - }, - { - "name": "age", - "is_multiselect": 0, - "attributes": [{"name": "young"}, {"name": "old"}], - }, - ], - ) - sa.set_project_workflow( - self.PROJECT_1, - [ - { - "step": 1, - "className": "class1", - "tool": 3, - "attribute": [ - { - "attribute": { - "name": "young", - "attribute_group": {"name": "age"}, - } - }, - { - "attribute": { - "name": "yes", - "attribute_group": {"name": "tall"}, - } - }, - ], - }, - { - "step": 2, - "className": "class2", - "tool": 3, - "attribute": [ - { - "attribute": { - "name": "young", - "attribute_group": {"name": "age"}, - } - }, - { - "attribute": { - "name": "yes", - "attribute_group": {"name": "tall"}, - } - }, - ], - }, - ], - ) - - pr_1_metadata = sa.get_project_metadata( - self.PROJECT_1, include_annotation_classes=True, include_workflow=True - ) - pr_1_metadata["name"] = self.PROJECT_2 - sa.create_project_from_metadata(pr_1_metadata) - pr_2_metadata = sa.get_project_metadata( - self.PROJECT_2, include_workflow=True, include_annotation_classes=True - ) - assert pr_2_metadata["workflows"][0]["className"] == "class1" - assert pr_2_metadata["workflows"][1]["className"] == "class2" - self.assertEqual(pr_2_metadata["classes"][0]["name"], "class1") - self.assertEqual(pr_2_metadata["classes"][1]["name"], "class2") diff --git a/tests/integration/settings/test_settings.py b/tests/integration/settings/test_settings.py index 65c640a58..cf2fb85d7 100644 --- a/tests/integration/settings/test_settings.py +++ b/tests/integration/settings/test_settings.py @@ -62,26 +62,6 @@ def test_create_project_with_settings(self): else: raise Exception("Test failed") - def test_create_from_metadata(self): - sa.create_project( - self.PROJECT_NAME, - self.PROJECT_DESCRIPTION, - self.PROJECT_TYPE, - [{"attribute": "ImageQuality", "value": "original"}], - ) - project_metadata = sa.get_project_metadata( - self.PROJECT_NAME, include_settings=True - ) - project_metadata["name"] = self.SECOND_PROJECT_NAME - sa.create_project_from_metadata(project_metadata) - settings = sa.get_project_settings(self.SECOND_PROJECT_NAME) - for setting in settings: - if setting["attribute"] == "ImageQuality": - assert setting["value"] == "original" - break - else: - raise Exception("Test failed") - def test_frame_rate_invalid_range_value(self): with self.assertRaisesRegexp( AppException, "FrameRate is available only for Video projects" diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py index 1bf247a17..481bb2950 100644 --- a/tests/integration/test_depricated_functions_document.py +++ b/tests/integration/test_depricated_functions_document.py @@ -100,10 +100,6 @@ def test_deprecated_functions(self): sa.upload_video_to_project(self.PROJECT_NAME, "some path") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.get_project_image_count(self.PROJECT_NAME) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.set_project_workflow(self.PROJECT_NAME, [{}]) except AppException as e: diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py index 8e16622d9..52b771cfb 100644 --- a/tests/integration/test_depricated_functions_video.py +++ b/tests/integration/test_depricated_functions_video.py @@ -94,10 +94,6 @@ def test_deprecated_functions(self): sa.upload_video_to_project(self.PROJECT_NAME, "some path") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.get_project_image_count(self.PROJECT_NAME) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.set_project_workflow(self.PROJECT_NAME, [{}]) except AppException as e: From ebf053cfba2ed036d61cad3718f065ae7effe4d3 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 6 Apr 2023 17:27:17 +0400 Subject: [PATCH 05/17] Updated PerFrame convertor --- src/superannotate/lib/core/video_convertor.py | 24 +- .../unit/one_frame_video_annotation.json | 90 +++ tests/data_set/unit/video_annotation.json | 674 ++++++++++++++++++ tests/unit/test_per_frame_convertor.py | 97 +++ 4 files changed, 877 insertions(+), 8 deletions(-) create mode 100644 tests/data_set/unit/one_frame_video_annotation.json create mode 100644 tests/data_set/unit/video_annotation.json create mode 100644 tests/unit/test_per_frame_convertor.py diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 758478f7e..6bce376e1 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -74,8 +74,12 @@ def _interpolate( "x": round(data["x"] + steps["x"] * idx, 2), "y": round(data["y"] + steps["y"] * idx, 2), } - elif annotation_type != AnnotationTypes.EVENT: - tmp_data["points"] = data["points"] + elif ( + annotation_type != AnnotationTypes.EVENT + ): # AnnotationTypes.POLYGON, AnnotationTypes.POLYLINE + tmp_data["points"] = [] + for i in range(len(data["points"])): + tmp_data["points"].append(data["points"][i] + idx * steps[i]) annotations[frame_idx] = Annotation( instanceId=instance_id, @@ -157,12 +161,16 @@ def _interpolate_frames( "y": (to_frame["y"] - from_frame["y"]) / frames_diff, } elif annotation_type in (AnnotationTypes.POLYGON, AnnotationTypes.POLYLINE): - steps = [ - (to_point - from_point) / frames_diff - for from_point, to_point in zip( - from_frame["points"], to_frame["points"] - ) - ] + if len(from_frame["points"]) == len(to_frame["points"]): + steps = [ + (to_point - from_point) / frames_diff + for from_point, to_point in zip( + from_frame["points"], to_frame["points"] + ) + ] + else: + steps = [0] * len(from_frame["points"]) + return self._interpolate( class_name=class_name, class_id=class_id, diff --git a/tests/data_set/unit/one_frame_video_annotation.json b/tests/data_set/unit/one_frame_video_annotation.json new file mode 100644 index 000000000..710804031 --- /dev/null +++ b/tests/data_set/unit/one_frame_video_annotation.json @@ -0,0 +1,90 @@ +{ + "metadata": { + "width": 992, + "height": 568, + "duration": 6.5, + "name": null, + "lastAction": { + "timestamp": 1680778235176, + "email": "karen@superannotate.com" + } + }, + "instances": [ + { + "id": "MzE4NzcuODYwMzQ2Nzc5NTg=", + "type": "bbox", + "classId": -1, + "createdBy": { + "email": "karen@superannotate.com", + "role": "Admin" + }, + "createdAt": "2023-04-06T10:49:58.410Z", + "updatedBy": { + "email": "karen@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2023-04-06T10:49:59.799Z", + "locked": false, + "timeline": { + "0": { + "active": true, + "points": { + "x1": 161.52, + "y1": 82, + "x2": 372.8, + "y2": 318.92 + } + }, + "0.100000": { + "active": false, + "points": { + "x1": 161.52, + "y1": 82, + "x2": 372.8, + "y2": 318.92 + } + } + }, + "pointLabels": {} + }, + { + "id": "NDEyMDguNzUwNzYwMDc4MjU2", + "type": "bbox", + "classId": -1, + "createdBy": { + "email": "karen@superannotate.com", + "role": "Admin" + }, + "createdAt": "2023-04-06T10:50:07.741Z", + "updatedBy": { + "email": "karen@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2023-04-06T10:50:09.264Z", + "locked": false, + "timeline": { + "0.900001": { + "active": true, + "points": { + "x1": 186.35, + "y1": 164.85, + "x2": 375.17, + "y2": 366.01 + } + }, + "1.000000": { + "active": false, + "points": { + "x1": 186.35, + "y1": 164.85, + "x2": 375.17, + "y2": 366.01 + } + } + }, + "pointLabels": {} + } + ], + "tags": [], + "name": "ernest mtom 10 fps" +} \ No newline at end of file diff --git a/tests/data_set/unit/video_annotation.json b/tests/data_set/unit/video_annotation.json new file mode 100644 index 000000000..ce16d8ee2 --- /dev/null +++ b/tests/data_set/unit/video_annotation.json @@ -0,0 +1,674 @@ +{ + "metadata": { + "name": "1.mp4", + "duration": 30526667, + "width": 1920, + "height": 1080, + "lastAction": { + "timestamp": 1680762192661, + "email": "arturn@superannotate.com" + }, + "projectId": 460663, + "url": "https://sa-public-files.s3.us-west-2.amazonaws.com/Video+project/video_file_example_1.mp4", + "status": "InProgress", + "error": null, + "annotatorEmail": null, + "qaEmail": null + }, + "instances": [ + { + "meta": { + "type": "polygon", + "classId": 4797926, + "className": "class_obj_1", + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2023-04-05T08:24:12.256Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2023-04-05T08:27:34.436Z", + "start": 900001, + "end": 5400001 + }, + "parameters": [ + { + "start": 900001, + "end": 5400001, + "timestamps": [ + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 226.36, + 240.15, + 191.47, + 456.31, + 283.59, + 604.41, + 585.2, + 808.18, + 949.66, + 592.18, + 957.41, + 235.89, + 596.02, + 186.13 + ], + "timestamp": 900001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 226.36, + 240.15, + 191.47, + 456.31, + 240.71, + 648.53, + 585.2, + 808.18, + 949.66, + 592.18, + 957.41, + 235.89, + 596.02, + 186.13 + ], + "timestamp": 2300001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 226.36, + 240.15, + 191.47, + 456.31, + 240.71, + 648.53, + 585.2, + 808.18, + 1342.69, + 779.37, + 957.41, + 235.89, + 596.02, + 186.13 + ], + "timestamp": 4000001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 226.36, + 240.15, + 191.47, + 456.31, + 240.71, + 648.53, + 585.2, + 808.18, + 906.83, + 933.12, + 1342.69, + 779.37, + 957.41, + 235.89, + 596.02, + 186.13 + ], + "timestamp": 4500001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 226.36, + 240.15, + 191.47, + 456.31, + 437.49, + 569.12, + 585.2, + 808.18, + 906.83, + 933.12, + 801.66, + 625.14, + 957.41, + 235.89, + 606.13, + 563.07 + ], + "timestamp": 4900001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 226.36, + 240.15, + 191.47, + 456.31, + 437.49, + 569.12, + 585.2, + 808.18, + 906.83, + 933.12, + 801.66, + 625.14, + 957.41, + 235.89, + 606.13, + 563.07 + ], + "timestamp": 5400001 + } + ] + } + ] + }, + { + "meta": { + "type": "polyline", + "classId": 4797926, + "className": "class_obj_1", + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2023-04-06T06:22:05.555Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2023-04-06T06:23:20.303Z", + "start": 6200001, + "end": 11000001 + }, + "parameters": [ + { + "start": 6200001, + "end": 11000001, + "timestamps": [ + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 958.95, + 86.53, + 953.95, + 509.98, + 1085.52, + 221.98 + ], + "timestamp": 6200001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1008.46, + 86.38, + 953.95, + 509.98, + 1085.52, + 221.98 + ], + "timestamp": 6400001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1065.41, + 102.04, + 953.95, + 509.98, + 1085.52, + 221.98 + ], + "timestamp": 6800001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1164.59, + 153.47, + 953.95, + 509.98, + 1085.52, + 221.98 + ], + "timestamp": 7300001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 953.95, + 509.98, + 1110.52, + 232.45 + ], + "timestamp": 7700001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 1197.22, + 435.16, + 953.95, + 509.98, + 1110.52, + 232.45 + ], + "timestamp": 8200001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 1262.25, + 529.69, + 953.95, + 509.98, + 1110.52, + 232.45 + ], + "timestamp": 8800001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 1270.38, + 587.25, + 953.95, + 509.98, + 1110.52, + 232.45 + ], + "timestamp": 9300001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 1270.38, + 587.25, + 995.77, + 913.51, + 953.95, + 509.98, + 1110.52, + 232.45 + ], + "timestamp": 9800001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 1270.38, + 587.25, + 995.77, + 913.51, + 953.95, + 509.98, + 1234.05, + 191.96 + ], + "timestamp": 10300001 + }, + { + "attributes": [ + { + "id": 9251166, + "groupId": 4402994, + "name": "atr_multi1", + "groupName": "group_MULTI" + }, + { + "id": 9251168, + "groupId": 4402994, + "name": "atr_multi3", + "groupName": "group_MULTI" + }, + { + "id": 9251170, + "groupId": 4402995, + "name": "atr_single2", + "groupName": "group_SINGLE" + } + ], + "points": [ + 1237.61, + 194.59, + 1270.38, + 587.25, + 995.77, + 913.51, + 953.95, + 509.98, + 1234.05, + 191.96 + ], + "timestamp": 11000001 + } + ] + } + ] + } + ], + "tags": [] +} \ No newline at end of file diff --git a/tests/unit/test_per_frame_convertor.py b/tests/unit/test_per_frame_convertor.py new file mode 100644 index 000000000..0926bdf86 --- /dev/null +++ b/tests/unit/test_per_frame_convertor.py @@ -0,0 +1,97 @@ +import json +import os.path +from unittest import TestCase + +from src.superannotate.lib.core.video_convertor import VideoFrameGenerator +from tests import DATA_SET_PATH + + +class TestConvertor(TestCase): + ANNOTATION_PATH = os.path.join(DATA_SET_PATH, "unit", "video_annotation.json") + ONE_FRAME_ANNOTATION_PATH = os.path.join( + DATA_SET_PATH, "unit", "one_frame_video_annotation.json" + ) + + def test_polygon_polyline_convertor(self): + payload = json.load(open(self.ANNOTATION_PATH)) + generator = VideoFrameGenerator(payload, fps=10) + data = [i for i in generator] + + from collections import defaultdict + + point_frame_map = defaultdict(list) + frame_point_map = { + i["frame"]: j["points"] for i in data for j in i["annotations"] + } + for frame, points in frame_point_map.items(): + point_frame_map[tuple(points)].append(frame) + + point_frame_count_pairs = [ + ( + ( + 226.36, + 240.15, + 191.47, + 456.31, + 240.71, + 648.53, + 585.2, + 808.18, + 1342.69, + 779.37, + 957.41, + 235.89, + 596.02, + 186.13, + ), + 5, + ), + ( + ( + 226.36, + 240.15, + 191.47, + 456.31, + 437.49, + 569.12, + 585.2, + 808.18, + 906.83, + 933.12, + 801.66, + 625.14, + 957.41, + 235.89, + 606.13, + 563.07, + ), + 6, + ), + ((1237.61, 194.59, 953.95, 509.98, 1110.52, 232.45), 5), + ((1237.61, 194.59, 1270.38, 587.25, 953.95, 509.98, 1110.52, 232.45), 5), + ( + ( + 1237.61, + 194.59, + 1270.38, + 587.25, + 995.77, + 913.51, + 953.95, + 509.98, + 1234.05, + 191.96, + ), + 8, + ), + ] + for points, frame_count in point_frame_count_pairs: + assert len(point_frame_map.pop(points)) == frame_count + assert all([len(frames) == 1 for frames in point_frame_map.values()]) + + # TODO write tests for one frame annotation + # def test_one_frame_annotation(self): + # payload = json.load(open(self.ONE_FRAME_ANNOTATION_PATH)) + # generator = VideoFrameGenerator(payload, fps=10) + # data = [i for i in generator] + # assert data From ba2729195a7320c74e227e93f838e51cf4a93704 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 10 Apr 2023 17:52:26 +0400 Subject: [PATCH 06/17] Fix S3Repo interface --- src/superannotate/lib/core/repositories.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/superannotate/lib/core/repositories.py b/src/superannotate/lib/core/repositories.py index a345573ac..409de413a 100644 --- a/src/superannotate/lib/core/repositories.py +++ b/src/superannotate/lib/core/repositories.py @@ -12,24 +12,24 @@ class BaseReadOnlyRepository(ABC): - @abstractmethod + def get_one(self, uuid: Union[Condition, int]) -> Optional[Union[BaseModel]]: raise NotImplementedError - @abstractmethod + def get_all(self, condition: Optional[Condition] = None) -> List[Union[BaseModel]]: raise NotImplementedError class BaseManageableRepository(BaseReadOnlyRepository): - @abstractmethod + def insert(self, entity: BaseEntity) -> BaseEntity: raise NotImplementedError def update(self, entity: BaseEntity) -> BaseEntity: raise NotImplementedError - @abstractmethod + def delete(self, uuid: Any): raise NotImplementedError From 7d9336b0c2269fef12e1ba2e398d00194a9cf060 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 12 Apr 2023 09:52:40 +0400 Subject: [PATCH 07/17] Update __init__.py --- src/superannotate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 5c1b72fe8..27e2f0f30 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys import typing -__version__ = "4.4.11" +__version__ = "4.4.12dev1" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) From 3ddbd3b5ca32cb3cb4fdb0e35c56df4e764aaf8f Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Thu, 13 Apr 2023 16:46:56 +0400 Subject: [PATCH 08/17] changed SDK auth flow --- .../lib/app/interface/base_interface.py | 2 +- .../lib/app/interface/sdk_interface.py | 8 ++- .../lib/core/entities/__init__.py | 2 + .../lib/core/entities/project.py | 3 + src/superannotate/lib/core/repositories.py | 5 -- src/superannotate/lib/core/service_types.py | 4 ++ .../lib/core/serviceproviders.py | 5 ++ .../lib/core/usecases/annotations.py | 14 ++-- src/superannotate/lib/core/usecases/models.py | 4 +- .../lib/core/usecases/projects.py | 19 +++++ .../lib/infrastructure/controller.py | 19 +++-- .../lib/infrastructure/repositories.py | 1 + .../lib/infrastructure/serviceprovider.py | 7 ++ .../test_annotations_pre_processing.py | 2 +- tests/integration/client/test_client_init.py | 71 ------------------- .../mixpanel/test_mixpanel_decorator.py | 23 +++--- tests/integration/test_ml_funcs.py | 8 ++- 17 files changed, 92 insertions(+), 105 deletions(-) delete mode 100644 tests/integration/client/test_client_init.py diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 930d84e03..8a5cb4ec1 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -207,7 +207,7 @@ def _track_method(self, args, kwargs, success: bool): function_name = self.function.__name__ if self.function else "" arguments = self.extract_arguments(self.function, *args, **kwargs) event_name, properties = self.default_parser(function_name, arguments) - user_id = client.controller.team_data.creator_id + user_id = client.controller.current_user.email team_name = client.controller.team_data.name properties["Success"] = success diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index eac126415..f4980b4b3 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -1717,7 +1717,7 @@ def upload_annotations_from_folder_to_project( response = self.controller.annotations.upload_from_folder( project=project, folder=folder, - team=self.controller.team, + user=self.controller.current_user, annotation_paths=annotation_paths, # noqa: E203 client_s3_bucket=from_s3_bucket, folder_path=folder_path, @@ -1794,7 +1794,7 @@ def upload_image_annotations( folder=folder, image=image, annotations=annotation_json, - team=self.controller.team, + user=self.controller.current_user, mask=mask, verbose=verbose, keep_status=keep_status, @@ -2155,7 +2155,9 @@ def add_contributors_to_project( for email in emails ] response = self.controller.projects.add_contributors( - team=self.controller.get_team().data, project=project, contributors=contributors + team=self.controller.get_team().data, + project=project, + contributors=contributors, ) if response.errors: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index e2a9ab89a..cc81ba8a7 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -16,6 +16,7 @@ from lib.core.entities.project import ProjectEntity from lib.core.entities.project import SettingEntity from lib.core.entities.project import TeamEntity +from lib.core.entities.project import UserEntity from lib.core.entities.project import WorkflowEntity from lib.core.entities.project_entities import BaseEntity from lib.core.entities.project_entities import ImageInfoEntity @@ -47,6 +48,7 @@ "S3FileEntity", "AnnotationClassEntity", "TeamEntity", + "UserEntity", "MLModelEntity", "IntegrationEntity", ] diff --git a/src/superannotate/lib/core/entities/project.py b/src/superannotate/lib/core/entities/project.py index 1ae6693b1..65c725285 100644 --- a/src/superannotate/lib/core/entities/project.py +++ b/src/superannotate/lib/core/entities/project.py @@ -172,6 +172,9 @@ class UserEntity(BaseModel): picture: Optional[str] user_role: Optional[int] + class Config: + extra = Extra.ignore + class TeamEntity(BaseModel): id: Optional[int] diff --git a/src/superannotate/lib/core/repositories.py b/src/superannotate/lib/core/repositories.py index 409de413a..411b40e84 100644 --- a/src/superannotate/lib/core/repositories.py +++ b/src/superannotate/lib/core/repositories.py @@ -1,5 +1,4 @@ from abc import ABC -from abc import abstractmethod from typing import Any from typing import List from typing import Optional @@ -12,24 +11,20 @@ class BaseReadOnlyRepository(ABC): - def get_one(self, uuid: Union[Condition, int]) -> Optional[Union[BaseModel]]: raise NotImplementedError - def get_all(self, condition: Optional[Condition] = None) -> List[Union[BaseModel]]: raise NotImplementedError class BaseManageableRepository(BaseReadOnlyRepository): - def insert(self, entity: BaseEntity) -> BaseEntity: raise NotImplementedError def update(self, entity: BaseEntity) -> BaseEntity: raise NotImplementedError - def delete(self, uuid: Any): raise NotImplementedError diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index 11f981776..4374aec26 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -150,6 +150,10 @@ class TeamResponse(ServiceResponse): data: entities.TeamEntity = None +class UserResponse(ServiceResponse): + data: entities.UserEntity = None + + class ModelListResponse(ServiceResponse): data: List[entities.AnnotationClassEntity] = None diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 53ac6f798..0d0c1054e 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -26,6 +26,7 @@ from lib.core.service_types import UploadAnnotationsResponse from lib.core.service_types import UploadCustomFieldValuesResponse from lib.core.service_types import UserLimitsResponse +from lib.core.service_types import UserResponse from lib.core.types import Attachment from lib.core.types import AttachmentMeta @@ -507,6 +508,10 @@ class BaseServiceProvider: def get_team(self, team_id: int) -> TeamResponse: raise NotImplementedError + @abstractmethod + def get_user(self, team_id: int) -> UserResponse: + raise NotImplementedError + @abstractmethod def list_templates(self) -> ServiceResponse: raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 1c24060e3..e73da9998 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -37,7 +37,7 @@ from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity from lib.core.entities import ProjectEntity -from lib.core.entities import TeamEntity +from lib.core.entities import UserEntity from lib.core.exceptions import AppException from lib.core.reporter import Reporter from lib.core.response import Response @@ -482,7 +482,7 @@ def __init__( reporter: Reporter, project: ProjectEntity, folder: FolderEntity, - team: TeamEntity, + user: UserEntity, annotation_paths: List[str], service_provider: BaseServiceProvider, pre_annotation: bool = False, @@ -493,7 +493,7 @@ def __init__( super().__init__(reporter) self._project = project self._folder = folder - self._team = team + self._user = user self._service_provider = service_provider self._annotation_classes = service_provider.annotation_classes.list( Condition("project_id", project.id, EQ) @@ -581,7 +581,7 @@ def prepare_annotation(self, annotation: dict, size) -> dict: raise AppException(errors) annotation = UploadAnnotationUseCase.set_defaults( - self._team.creator_id, annotation, self._project.type + self._user.email, annotation, self._project.type ) return annotation @@ -827,7 +827,7 @@ def __init__( project: ProjectEntity, folder: FolderEntity, image: ImageEntity, - team: TeamEntity, + user: UserEntity, service_provider: BaseServiceProvider, reporter: Reporter, annotation_upload_data: UploadAnnotationAuthData = None, @@ -844,7 +844,7 @@ def __init__( self._project = project self._folder = folder self._image = image - self._team = team + self._user = user self._service_provider = service_provider self._annotation_classes = service_provider.annotation_classes.list( Condition("project_id", project.id, EQ) @@ -973,7 +973,7 @@ def execute(self): annotation_json, mask = self._get_annotation_json() errors = self._validate_json(annotation_json) annotation_json = UploadAnnotationUseCase.set_defaults( - self._team.creator_id, annotation_json, self._project.type + self._user.email, annotation_json, self._project.type ) if not errors: annotation_file = io.StringIO() diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 83c8048ee..d9e718ad1 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -507,7 +507,9 @@ def execute(self): failed_images = [] while len(success_images) + len(failed_images) != len(image_ids): images_metadata = self._service_provider.items.list_by_names( - project=self._project, folder=self._folder, names=self._images_list + project=self._project, + folder=self._folder, + names=self._images_list, ).data success_images = [ diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 69d55ccdf..92f75e4ed 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -609,6 +609,25 @@ def execute(self): return self._response +class GetCurrentUserUseCase(BaseUseCase): + def __init__(self, service_provider: BaseServiceProvider, team_id: int): + super().__init__() + self._service_provider = service_provider + self._team_id = team_id + + def execute(self): + try: + response = self._service_provider.get_user(self._team_id) + if not response.ok: + raise AppException(response.error) + self._response.data = response.data + except Exception: + raise AppException( + "Unable to retrieve user data. Please verify your credentials." + ) from None + return self._response + + class SearchContributorsUseCase(BaseUseCase): def __init__( self, diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index dcc9ff12f..cd5da2f14 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -23,6 +23,7 @@ from lib.core.entities import ProjectEntity from lib.core.entities import SettingEntity from lib.core.entities import TeamEntity +from lib.core.entities import UserEntity from lib.core.entities.classes import AnnotationClassEntity from lib.core.entities.integrations import IntegrationEntity from lib.core.exceptions import AppException @@ -583,7 +584,7 @@ def upload_from_folder( project: ProjectEntity, folder: FolderEntity, annotation_paths: List[str], - team: TeamEntity, + user: UserEntity, keep_status: bool = False, client_s3_bucket=None, is_pre_annotations: bool = False, @@ -592,7 +593,7 @@ def upload_from_folder( use_case = usecases.UploadAnnotationsFromFolderUseCase( project=project, folder=folder, - team=team, + user=user, annotation_paths=annotation_paths, service_provider=self.service_provider, pre_annotation=is_pre_annotations, @@ -608,7 +609,7 @@ def upload_image_annotations( project: ProjectEntity, folder: FolderEntity, image: ImageEntity, - team: TeamEntity, + user: UserEntity, annotations: dict, mask: io.BytesIO = None, verbose: bool = True, @@ -617,7 +618,7 @@ def upload_image_annotations( use_case = usecases.UploadAnnotationUseCase( project=project, folder=folder, - team=team, + user=user, service_provider=self.service_provider, image=image, annotations=annotations, @@ -795,6 +796,7 @@ def __init__(self, config: ConfigEntity): self.service_provider = ServiceProvider(http_client) self._team = self.get_team().data + self._user = self.get_current_user().data self.annotation_classes = AnnotationClassManager(self.service_provider) self.projects = ProjectManager(self.service_provider) self.folders = FolderManager(self.service_provider) @@ -805,6 +807,10 @@ def __init__(self, config: ConfigEntity): self.models = ModelManager(self.service_provider) self.integrations = IntegrationManager(self.service_provider) + @property + def current_user(self): + return self._user + @staticmethod def validate_token(token: str): try: @@ -828,6 +834,11 @@ def get_team(self): service_provider=self.service_provider, team_id=self.team_id ).execute() + def get_current_user(self): + return usecases.GetCurrentUserUseCase( + service_provider=self.service_provider, team_id=self.team_id + ).execute() + @property def team_data(self): if not self._team_data: diff --git a/src/superannotate/lib/infrastructure/repositories.py b/src/superannotate/lib/infrastructure/repositories.py index 83c827f5c..e94ec427b 100644 --- a/src/superannotate/lib/infrastructure/repositories.py +++ b/src/superannotate/lib/infrastructure/repositories.py @@ -1,4 +1,5 @@ import io + from lib.core.entities import S3FileEntity from lib.core.repositories import BaseS3Repository diff --git a/src/superannotate/lib/infrastructure/serviceprovider.py b/src/superannotate/lib/infrastructure/serviceprovider.py index 71965a507..840fc2e6f 100644 --- a/src/superannotate/lib/infrastructure/serviceprovider.py +++ b/src/superannotate/lib/infrastructure/serviceprovider.py @@ -9,6 +9,7 @@ from lib.core.service_types import TeamResponse from lib.core.service_types import UploadAnnotationAuthDataResponse from lib.core.service_types import UserLimitsResponse +from lib.core.service_types import UserResponse from lib.core.serviceproviders import BaseServiceProvider from lib.infrastructure.services.annotation import AnnotationService from lib.infrastructure.services.annotation_class import AnnotationClassService @@ -31,6 +32,7 @@ class ServiceProvider(BaseServiceProvider): URL_GET_TEMPLATES = "templates" URL_PREPARE_EXPORT = "export" URL_GET_EXPORTS = "exports" + URL_USER = "user/ME" URL_USERS = "users" URL_GET_EXPORT = "export/{}" URL_GET_MODEL_METRICS = "ml_models/{}/getCurrentMetrics" @@ -59,6 +61,11 @@ def get_team(self, team_id: int) -> TeamResponse: f"{self.URL_TEAM}/{team_id}", "get", content_type=TeamResponse ) + def get_user(self, team_id: int) -> UserResponse: + return self.client.request( + self.URL_USER, "get", params={"team_id": team_id}, content_type=UserResponse + ) + def list_templates(self): return self.client.request(self.URL_GET_TEMPLATES, "get") diff --git a/tests/integration/annotations/test_annotations_pre_processing.py b/tests/integration/annotations/test_annotations_pre_processing.py index abf7a838b..60ec4fafc 100644 --- a/tests/integration/annotations/test_annotations_pre_processing.py +++ b/tests/integration/annotations/test_annotations_pre_processing.py @@ -39,7 +39,7 @@ def test_annotation_last_action_and_creation_type(self): self.assertEqual(instance["creationType"], "Preannotation") assert ( annotation["metadata"]["lastAction"]["email"] - == sa.controller.team_data.creator_id + == sa.controller.current_user.email ) self.assertEqual( type(annotation["metadata"]["lastAction"]["timestamp"]), int diff --git a/tests/integration/client/test_client_init.py b/tests/integration/client/test_client_init.py deleted file mode 100644 index 8526ec180..000000000 --- a/tests/integration/client/test_client_init.py +++ /dev/null @@ -1,71 +0,0 @@ -# import json -# import os -# import tempfile -# from unittest import TestCase -# -# import superannotate.lib.core as constants -# from superannotate import SAClient -# -# -# class TestClientInit(TestCase): -# TEST_TOKEN = "test=6085" -# TEST_URL = "https://test.com" -# -# def setUp(self) -> None: -# os.environ.pop('SA_TOKEN', None) -# os.environ.pop('SA_URL', None) -# os.environ.pop('SA_SSL', None) -# -# def test_via_token(self): -# sa = SAClient(token=self.TEST_TOKEN) -# assert sa.controller._token == self.TEST_TOKEN -# assert sa.controller._backend_client.api_url == constants.BACKEND_URL -# -# def test_via_env_token(self): -# os.environ.update( -# {"SA_TOKEN": self.TEST_TOKEN} -# ) -# sa = SAClient() -# assert sa.controller._token == self.TEST_TOKEN -# assert sa.controller._backend_client.api_url == constants.BACKEND_URL -# -# def test_via_env_vars(self): -# os.environ.update( -# { -# "SA_TOKEN": self.TEST_TOKEN, -# "SA_URL": self.TEST_URL, -# "SA_SSL": "False" -# } -# ) -# sa = SAClient() -# assert sa.controller._token == self.TEST_TOKEN -# assert sa.controller._backend_client.api_url == self.TEST_URL -# assert sa.controller._backend_client._verify_ssl == False -# -# def test_via_config_path_with_url_token(self): -# data = { -# "token": self.TEST_TOKEN, -# "main_endpoint": self.TEST_URL, -# "ssl_verify": True -# } -# with tempfile.TemporaryDirectory() as temp_dir: -# file_path = f"{temp_dir}/config.json" -# with open(file_path, "w") as file: -# json.dump(data, file) -# sa = SAClient(config_path=file_path) -# assert sa.controller._token == self.TEST_TOKEN -# assert sa.controller._backend_client.api_url == self.TEST_URL -# assert sa.controller._backend_client._verify_ssl == True -# -# def test_via_config_path_with_token(self): -# data = { -# "token": self.TEST_TOKEN, -# } -# with tempfile.TemporaryDirectory() as temp_dir: -# file_path = f"{temp_dir}/config.json" -# with open(file_path, "w") as file: -# json.dump(data, file) -# sa = SAClient(config_path=file_path) -# assert sa.controller._token == self.TEST_TOKEN -# assert sa.controller._backend_client.api_url == constants.BACKEND_URL -# assert sa.controller._backend_client._verify_ssl == True diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py index a1c5e1abd..ada792fc3 100644 --- a/tests/integration/mixpanel/test_mixpanel_decorator.py +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -14,11 +14,10 @@ class TestMixpanel(TestCase): CLIENT = SAClient() - TEAM_DATA = CLIENT.get_team_metadata() BLANK_PAYLOAD = { "SDK": True, - "Team": TEAM_DATA["name"], - "Team Owner": TEAM_DATA["creator_id"], + "Team": CLIENT.get_team_metadata()["name"], + "Team Owner": CLIENT.controller.current_user.email, "Version": __version__, "Success": True, "Python version": platform.python_version(), @@ -65,7 +64,10 @@ def test_init(self, track_method): @patch("lib.app.interface.base_interface.Tracker._track") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_token(self, get_team_use_case, track_method): + @patch("lib.core.usecases.GetCurrentUserUseCase") + def test_init_via_token( + self, get_current_user_use_case, get_team_use_case, track_method + ): SAClient(token="test=3232") result = list(track_method.call_args)[0] payload = self.default_payload @@ -74,7 +76,7 @@ def test_init_via_token(self, get_team_use_case, track_method): "sa_token": "True", "config_path": "False", "Team": get_team_use_case().execute().data.name, - "Team Owner": get_team_use_case().execute().data.creator_id, + "Team Owner": get_current_user_use_case().execute().data.email, } ) assert result[1] == "__init__" @@ -82,7 +84,10 @@ def test_init_via_token(self, get_team_use_case, track_method): @patch("lib.app.interface.base_interface.Tracker._track") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_config_file(self, get_team_use_case, track_method): + @patch("lib.core.usecases.GetCurrentUserUseCase") + def test_init_via_config_file( + self, get_current_user_use_case, get_team_use_case, track_method + ): with tempfile.TemporaryDirectory() as config_dir: config_ini_path = f"{config_dir}/config.ini" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path): @@ -99,7 +104,7 @@ def test_init_via_config_file(self, get_team_use_case, track_method): "sa_token": "False", "config_path": "True", "Team": get_team_use_case().execute().data.name, - "Team Owner": get_team_use_case().execute().data.creator_id, + "Team Owner": get_current_user_use_case().execute().data.email, } ) assert result[1] == "__init__" @@ -107,8 +112,8 @@ def test_init_via_config_file(self, get_team_use_case, track_method): @patch("lib.app.interface.base_interface.Tracker._track") def test_get_team_metadata(self, track_method): - team = self.CLIENT.get_team_metadata() - team_owner = team["creator_id"] + self.CLIENT.get_team_metadata() + team_owner = self.CLIENT.controller.current_user.email result = list(track_method.call_args)[0] payload = self.default_payload assert result[0] == team_owner diff --git a/tests/integration/test_ml_funcs.py b/tests/integration/test_ml_funcs.py index 3a39a8180..70aed5d03 100644 --- a/tests/integration/test_ml_funcs.py +++ b/tests/integration/test_ml_funcs.py @@ -1,10 +1,10 @@ import os from os.path import dirname -from src.superannotate import SAClient +import pytest from src.superannotate import AppException +from src.superannotate import SAClient from tests.integration.base import BaseTestCase -import pytest sa = SAClient() @@ -23,7 +23,9 @@ def folder_path(self): return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) def test_run_prediction_with_non_exist_images(self): - with self.assertRaisesRegexp(AppException, 'No valid image names were provided.'): + with self.assertRaisesRegexp( + AppException, "No valid image names were provided." + ): sa.run_prediction( self.PROJECT_NAME, ["NotExistingImage.jpg"], self.MODEL_NAME ) From 16eb5280d5c3f60cddaac6e4b45d9e92d1906a1f Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Thu, 13 Apr 2023 18:30:19 +0400 Subject: [PATCH 09/17] changes in SDK auth flow --- .../lib/app/interface/base_interface.py | 10 ++--- .../lib/core/entities/project.py | 1 - .../lib/core/usecases/projects.py | 14 +++---- .../lib/infrastructure/controller.py | 17 +++----- .../mixpanel/test_mixpanel_decorator.py | 40 ++++++++++--------- 5 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 8a5cb4ec1..50b1ecdf5 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -132,11 +132,11 @@ def get_mp_instance(self) -> Mixpanel: return Mixpanel(mp_token) @staticmethod - def get_default_payload(team_name, user_id): + def get_default_payload(team_name, user_email): return { "SDK": True, "Team": team_name, - "Team Owner": user_id, + "User Email": user_email, "Version": __version__, "Python version": platform.python_version(), "Python interpreter type": platform.python_implementation(), @@ -207,13 +207,13 @@ def _track_method(self, args, kwargs, success: bool): function_name = self.function.__name__ if self.function else "" arguments = self.extract_arguments(self.function, *args, **kwargs) event_name, properties = self.default_parser(function_name, arguments) - user_id = client.controller.current_user.email + user_email = client.controller.current_user.email team_name = client.controller.team_data.name properties["Success"] = success - default = self.get_default_payload(team_name=team_name, user_id=user_id) + default = self.get_default_payload(team_name=team_name, user_email=user_email) self._track( - user_id, + user_email, event_name, {**default, **properties, **CONFIG.get_current_session().data}, ) diff --git a/src/superannotate/lib/core/entities/project.py b/src/superannotate/lib/core/entities/project.py index 65c725285..eb4556c87 100644 --- a/src/superannotate/lib/core/entities/project.py +++ b/src/superannotate/lib/core/entities/project.py @@ -169,7 +169,6 @@ class UserEntity(BaseModel): first_name: Optional[str] last_name: Optional[str] email: Optional[str] - picture: Optional[str] user_role: Optional[int] class Config: diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 92f75e4ed..b863b62d5 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -616,15 +616,13 @@ def __init__(self, service_provider: BaseServiceProvider, team_id: int): self._team_id = team_id def execute(self): - try: - response = self._service_provider.get_user(self._team_id) - if not response.ok: - raise AppException(response.error) - self._response.data = response.data - except Exception: - raise AppException( + response = self._service_provider.get_user(self._team_id) + if not response.ok: + self._response.errors = AppException( "Unable to retrieve user data. Please verify your credentials." - ) from None + ) + else: + self._response.data = response.data return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index cd5da2f14..7b3cb3574 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -796,7 +796,7 @@ def __init__(self, config: ConfigEntity): self.service_provider = ServiceProvider(http_client) self._team = self.get_team().data - self._user = self.get_current_user().data + self._user = self.get_current_user() self.annotation_classes = AnnotationClassManager(self.service_provider) self.projects = ProjectManager(self.service_provider) self.folders = FolderManager(self.service_provider) @@ -811,14 +811,6 @@ def __init__(self, config: ConfigEntity): def current_user(self): return self._user - @staticmethod - def validate_token(token: str): - try: - int(token.split("=")[-1]) - except ValueError: - raise AppException("Invalid token.") - return token - @property def user_id(self): if not self._user_id: @@ -834,10 +826,13 @@ def get_team(self): service_provider=self.service_provider, team_id=self.team_id ).execute() - def get_current_user(self): - return usecases.GetCurrentUserUseCase( + def get_current_user(self) -> UserEntity: + response = usecases.GetCurrentUserUseCase( service_provider=self.service_provider, team_id=self.team_id ).execute() + if response.errors: + raise AppException(response.errors) + return response.data @property def team_data(self): diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py index ada792fc3..87263d991 100644 --- a/tests/integration/mixpanel/test_mixpanel_decorator.py +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -12,12 +12,14 @@ from src.superannotate import SAClient +sa = SAClient() + + class TestMixpanel(TestCase): - CLIENT = SAClient() BLANK_PAYLOAD = { "SDK": True, - "Team": CLIENT.get_team_metadata()["name"], - "Team Owner": CLIENT.controller.current_user.email, + "Team": sa.get_team_metadata()["name"], + "User Email": sa.controller.current_user.email, "Version": __version__, "Success": True, "Python version": platform.python_version(), @@ -32,7 +34,7 @@ class TestMixpanel(TestCase): def setUpClass(cls) -> None: cls.tearDownClass() print(cls.PROJECT_NAME) - cls._project = cls.CLIENT.create_project( + cls._project = sa.create_project( cls.PROJECT_NAME, cls.PROJECT_DESCRIPTION, cls.PROJECT_TYPE ) @@ -42,10 +44,10 @@ def tearDownClass(cls) -> None: @classmethod def _safe_delete_project(cls, project_name): - projects = cls.CLIENT.search_projects(project_name, return_metadata=True) + projects = sa.search_projects(project_name, return_metadata=True) for project in projects: try: - cls.CLIENT.delete_project(project) + sa.delete_project(project) except Exception: raise @@ -64,9 +66,9 @@ def test_init(self, track_method): @patch("lib.app.interface.base_interface.Tracker._track") @patch("lib.core.usecases.GetTeamUseCase") - @patch("lib.core.usecases.GetCurrentUserUseCase") + @patch("lib.infrastructure.serviceprovider.ServiceProvider.get_user") def test_init_via_token( - self, get_current_user_use_case, get_team_use_case, track_method + self, get_user, get_team_use_case, track_method ): SAClient(token="test=3232") result = list(track_method.call_args)[0] @@ -76,7 +78,7 @@ def test_init_via_token( "sa_token": "True", "config_path": "False", "Team": get_team_use_case().execute().data.name, - "Team Owner": get_current_user_use_case().execute().data.email, + "User Email": get_user().data.email, } ) assert result[1] == "__init__" @@ -84,9 +86,9 @@ def test_init_via_token( @patch("lib.app.interface.base_interface.Tracker._track") @patch("lib.core.usecases.GetTeamUseCase") - @patch("lib.core.usecases.GetCurrentUserUseCase") + @patch("lib.infrastructure.serviceprovider.ServiceProvider.get_user") def test_init_via_config_file( - self, get_current_user_use_case, get_team_use_case, track_method + self, get_user, get_team_use_case, track_method ): with tempfile.TemporaryDirectory() as config_dir: config_ini_path = f"{config_dir}/config.ini" @@ -104,7 +106,7 @@ def test_init_via_config_file( "sa_token": "False", "config_path": "True", "Team": get_team_use_case().execute().data.name, - "Team Owner": get_current_user_use_case().execute().data.email, + "User Email": get_user().data.email, } ) assert result[1] == "__init__" @@ -112,8 +114,8 @@ def test_init_via_config_file( @patch("lib.app.interface.base_interface.Tracker._track") def test_get_team_metadata(self, track_method): - self.CLIENT.get_team_metadata() - team_owner = self.CLIENT.controller.current_user.email + sa.get_team_metadata() + team_owner = sa.controller.current_user.email result = list(track_method.call_args)[0] payload = self.default_payload assert result[0] == team_owner @@ -128,7 +130,7 @@ def test_search_team_contributors(self, track_method): "last_name": "last_name", "return_metadata": False, } - self.CLIENT.search_team_contributors(**kwargs) + sa.search_team_contributors(**kwargs) result = list(track_method.call_args)[0] payload = self.default_payload payload.update(kwargs) @@ -143,7 +145,7 @@ def test_search_projects(self, track_method): "status": "NotStarted", "return_metadata": False, } - self.CLIENT.search_projects(**kwargs) + sa.search_projects(**kwargs) result = list(track_method.call_args)[0] payload = self.default_payload payload.update(kwargs) @@ -162,7 +164,7 @@ def test_create_project(self, track_method): "instructions_link": None, } try: - self.CLIENT.create_project(**kwargs) + sa.create_project(**kwargs) except AppException: pass result = list(track_method.call_args)[0] @@ -190,10 +192,10 @@ def test_create_project_multi_thread(self, track_method): "project_type": self.PROJECT_TYPE, } thread_1 = threading.Thread( - target=self.CLIENT.create_project, kwargs=kwargs_1 + target=sa.create_project, kwargs=kwargs_1 ) thread_2 = threading.Thread( - target=self.CLIENT.create_project, kwargs=kwargs_2 + target=sa.create_project, kwargs=kwargs_2 ) thread_1.start() thread_2.start() From ce7f1fb7c2483b4182fd3fec67735e86bda6ddc3 Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Mon, 17 Apr 2023 16:32:03 +0400 Subject: [PATCH 10/17] added History page in docs --- CHANGELOG.md | 379 ----------- CHANGELOG.rst | 596 ++++++++++++++++++ README.rst | 2 +- docs/source/changelog_link.rst | 1 + docs/source/index.rst | 1 + .../lib/app/interface/base_interface.py | 4 +- .../mixpanel/test_mixpanel_decorator.py | 16 +- 7 files changed, 606 insertions(+), 393 deletions(-) delete mode 100644 CHANGELOG.md create mode 100644 CHANGELOG.rst create mode 100644 docs/source/changelog_link.rst diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index f29c383d0..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,379 +0,0 @@ -# Changelog -All release highlights of this project will be documented in this file. -## 4.4.11 - April 2, 2023 -### Added -- `SAClient.set_project_status()` method. -- `SAClient.set_folder_status()` method. -### Updated -- `SAClient.create_annotation_class()` added OCR type attribute group support in the vector projects. -- `SAClient.create_annotation_classes_from_classes_json()` added OCR type attribute group support in the vector projects. -## 4.4.10 - March 12, 2023 -### Updated -- Configuration file creation flow -- `SAClient.search_projects()` method, removed `include_complete_image_count` argument, use `include_complete_item_count` instead. -- `SAClient.get_project_metadata()` method, removed `include_complete_image_count` argument, use `include_complete_item_count` instead. -- `SAClient.create_project()` method to support classes, workflows and instructions_link. -### Fixed -- `SAClient.clone_project()` method to address the issue of FPS mode is not being copied. -### Deprecated -- `SAClient.create_project_from_metadata()` method, use `SAClient.create_project()` instead. -- `SAClient.get_project_image_count()` method, use `SAClient.get_project_metadata()` instead. -### Removed -- `SAClient.class_distribution()` method -- `SAClient.benchmark()` method -## 4.4.9 - January 29, 2023 -### Added -- `SAClient.set_approval_statuses` _method_ function to change the approval status of items (images, audio / videos) in bulk. -### Updated -- `SAClient.convert_project_type` _method_ updated from Pixel to Vector converter, added polygon holes handling. -## 4.4.8 - December 25, 2022 -### Added -- New project types `Tiled`, `PointCloud`, `Other`. -- `SAClient.get_project_by_id` _method_ to get project metadata by id. -- `SAClient.get_folder_by_id` _method_ to get folder metadata by id. -- `SAClient.get_item_by_id` _method_ to get item metadata by id. -### Updated -- `SAClient.consensus` _method_ to compute agreement scores between tag type annotations. -### -## 4.4.7 - December 04, 2022 -### Updated -- `SAClient.search_folders` _method_ to add a new `status` argument for searching folders via status. -- Schemas for `Annotation Classes` and `Video Annotation` to support **text** and **numeric input** attribute group types. -### Fixed -- `SAClient.query` _method_ to address invalid exceptions. -- `SAClient.download_export` _method_ to address the issue with downloading for Windows OS. -- `SAClient.attach_items_from_integrated_storage` _method_ to address "integration not found" error. -- `SAClient.aggregate_annotations_as_df` _method_ to support files without "___objects" in their naming. -### Removed -- `SAClient.add_annotation_bbox_to_image` _method_, use `SAClient.upload_annotations` instead. -- `SAClient.add_annotation_point_to_image` _method_, use `SAClient.upload_annotations` instead. -- `SAClient.add_annotation_comment_to_image` _method_, use `SAClient.upload_annotations` instead. -### -## 4.4.6 - November 23, 2022 -### Updated -- `SAClient.aggregate_annotations_as_df` method to aggregate "comment" type instances. -- `SAClient.add_annotation_bbox_to_image`, `SAClient.add_annotation_point_to_image`, `SAClient.add_annotation_comment_to_image` _methods_ to add deprecation warnings. -### Fixed -- Special characters are being encoded after annotation upload (Windows) -- `SAClient.assign_folder` _method_ to address the invalid argument name. -- `SAClient.upload_images_from_folder_to_project` _method_ to address uploading of more than 500 items. -- `SAClient.upload_annotations_from_folder_to_project` _method_ to address the issue of a folder size being more than 25,5 MB. -- `SAClient.download_image` _method_ to address the KeyError 'id' when `include_annotations` is set to `True`. -### Removed -`SAClient.upload_preannotations_from_folder_to_project` _method_ -`SAClient.copy_image` _method_ -### -## 4.4.5 - October 23, 2022 -### Added -- `SAClient.add_items_to_subset` _method_ to associate given items with a Subset. -- `SAClient.upload_annotations` _method_ to upload annotations in SA format from the system memory. -### Updated -- `SAClient.upload_annotations_from_folder_to_project` & `SAClient.upload_image_annotations` _methods_ to add `keep_status` argument to prevent the annotation status from changing to **In Progress** after the annotation upload. -- Item metadata to add a new key for holding the id of an item. --`SAClient.upload_preannotations_from_folder_to_project` to add a deprecation warning message. --`SAClient.copy_image` to add a deprecation warning message. -### Fixed -- `SAClient.validate_annotations` _method_. -- `SAClient.search_items`, `SAClient.get_item_metadata` _methods_ to address defects related to pydantic 1.8.2. -- A defect related to editor to address the issue of uploading a tag instance without attributes. -### -## 4.4.4 - September 11, 2022 -### Updated -- Improvements on working with large files. -### Fixed -- `SAClient.upload_annotations_from_folder_to_project()` _method_ to address the issue of the dissaperaing progress bar. -- `SAClient.run_prediction()` _method_ to address the issue of the OCR model. -- `SAClient.validate_annotations()` _method_ to address the issue of missing log messages. -- `SAClient.create_project_from_metadata()` _method_ to address the issue of returning deprecated `is_multiselect` key. -- `SAClient.get_annotations()` _method_ to address the issue of returning error messages as annotation dicts. -### -## 4.4.2, 4.4.3 - August 21, 2022 -### Updated -- the **schema** of `classes JSON` to support new values for the `"group_type"` key for a given attribute group. `"group_type": "radio" | "checklist" | "text" | "numeric"`. -- the **schema** of `video annotation JSON` to support instances that have a `"tag"` type. -### Fixed -- `SAClient.get_annotations()` _method_ to address the issue of working with the large projects. -- `SAClient.get_annotations_per_frame()` _method_ to address the issue of throwing an error on small videos when the fps is set to 1. -- `SAClient.upload_annotations_from_folder_to_project()` to address the issue of timestamp values represented in seconds for the `"lastAction"`. -- `SAClient.download_export()` _method_ to address the issue of empty logs. -- `SAClient.clone_project()` _method_ to address the issue of having a corrupted project clone, when the source project has a keypoint workflow. -### -## 4.4.1 - July 24, 2022 -### Added -- `SAClient.create_custom_fields()` _method_ to create/add new custom fields to a project’s custom field schema. -- `SAClient.get_custom_fields()` _method_ to get a project’s custom field schema. -- `SAClient.delete_custom_fields()` _method_ to remove existing custom fields from a project’s custom field schema. -- `SAClient.upload_custom_values()` _method_ to attach custom field-value pairs to items. -- `SAClient.delete_custom_values()` _method_ to remove custom field-value pairs from items. -### Updated -- the **schema** of `classes JSON` to support the new `"default_value"` key to set a default attribute(s) for a given attribute group. -- `SAClient.get_item_metadata()` _method_ to add a new input argument `include_custom_metadata` to return custom metadata in the result items. -- `SAClient.search_items()` _method_ to add a new input argument `include_custom_metadata` to return custom metadata in the result items. -- `SAClient.query()` _method_ to return custom metadata in the result items. -### Fixed -- `SAClient` _class_ to address the system crash that occurs on instantiation via `config.json` file. -- `SAClient.query()` _method_ to address the issue of not returning more than 50 items. -- `SAClient.upload_annotations_from_folder_to_project()` to address the issue of some fields not being auto populated after the upload is finished. -- `SAClient.get_folder_metadata()`, `SAClient.search_folders()` to address the issue of transforming the ‘+’ sign in a folder to a whitespace. -### Removed -- `superannotate.assign_images()` _function_. Please use the `SAClient.assign_items()` _method_ instead. -- `superannotate.unassign_images()` _function_. Please use the `SAClient.unassign_items()` _method_ instead. -- `superannotate.delete_images()` _function_. Please use the `SAClient.delete_items()` _method_ instead. -### -## 4.4.0 - July 03, 2022 -### Added -- `superannotate.SAClient()` _class_ to instantiate team-level authentication and inheriting methods to access the back-end. -- `SAClient.download_annotations()` _method_ to download annotations without preparing an Export object. -- `SAClient.get_subsets()` _method_ to get the existing subsets for a given project. -- `SAClient.assign_items()` _method_ to assign items in a given project to annotators or quality specialists. -- `SAClient.unassign_items()` _method_ to remove assignments from items. -- `SAClient.delete_items()` _method_ to delete items in a given project. -### Updated -- `JSON Schema` for video annotations to version `1.0.45` to show **polygon** and **polyline** annotations. -- `SAClient.get_annotations_per_frame()` _method_ to show **polygon** and **polyline** annotations. -- `SAClient.get_annotations_per_frame()` _method_ to pick instances closer to a given **frame start** instead of the **median**. -- `SAClient.query()` _method_ to add the `subset` argument to support querying in a subset. -### Fixed -- `SAClient.set_annotation_statuses()` _method_ to address the issue occurring with more than 500 items. -- `SAClient.get_annotations()` _method_ to address the `PayloadError` occurring with more than 20000 items. -- `SAClient.get_annotations()` _method_ to address the missing `'duration'` and `'tags'` keys for newly uploaded and unannotated videos. -- `SAClient.get_annotations_per_frame()` _method_ to address missing `'duration'` and `'tags'` keys for newly uploaded and unannotated videos. -- `SAClient.get_annotations_per_frame()` _method_ to address the wrong `classId` value for unclassified instances. -### Removed -- `superannotate.init()` _function_. Please instantiate `superannotate.SAClient()` _class_ to authenticate. -- `superannotate.set_image_annotation_status()` _function_. Please use the `SAClient.set_annotation_statuses()` _method_ instead. -- `superannotate.set_images_annotations_statuses()` _function_. Please use the `SAClient.set_annotation_statuses()` _method_ instead. -## 4.3.4 - May 22, 2022 -### Updated -- `JSON Schema` for video annotations to version `x` to reflect point annotations. -- `superannotate.download_export()` function to preserve SA folder structure while downloading to S3 bucket. -- `superannotate.get_item_metadata()` function to have string type values instead of int type for the `approval_status` key. -- `superannotate.get_item_metadata()` function to change the value for the `path` key in the item metadata from `project/folder/item` format to `project/folder`. -- `superannotate.get_item_metadata()` function to add the `is_pinned` key in the returned metadata. -- `superannotate.clone_project()` function to have `NotStarted` project status for the newly created project. -### Fixed -- `superannotate.query()` function to address the missing value for the `path` key. -- `superannotate.import_annotation()` function to address the extension issue with JPEG files while converting from `VOC` to SA. -- `superannotate.import_annotation()` function to address int type pointlabels in the converted `JSON` from `COCO` to SA. -- `superannotate_get_annotations()` & `superannotate.add_annotation_comment_to_image()` to address the issue with `asyncio` occuring on Windows. -- `superannotate.set_image_annotation_status()` function add a deprecation warning. -- `superannotate.set_images_annotation_statuses()` function add a deprecation warning. -### Removed -- `share_projects()` function. -- `superannotate.attach_image_urls_to_project()` function. Please use the `superannotate.attach_items()` function instead. -- `superannotate.attach_document_urls_to_project()` function. Please use the `superannotate.attach_items()` function instead. -- `superannotate.attach_video_urls_to_project()` function. Please use the `superannotate.attach_items()` function instead. -- `superannotate.copy_images()` function. Please use the `superannotate.copy_items()` function instead. -- `superannotate.move_images()` function. Please use the `superannotate.move_items()` function instead. -### -## 4.3.3 - May 01 2022 -### Added -- `attach_items()` function to link items (images, videos, and documents) from external storages to SuperAnnotate using URLs. -- `copy_items()` function to copy items (images, videos, and documents) in bulk between folders in a project. -- `move_items()` function to move items (images, videos, and documents) in bulk between folders in a project. -- `set_annotation_statuses()` function to change the annotation status of items (images, videos, and documents) in bulk. -### Updated -- `aggregate_annotations_as_df()` function now supports Text Projects. -### Fixed -- `validate_annotations()` function to accept only numeric type values for the `points` field. -- `prepare_export()` function to address the issue when the entire project is prepared when a wrong folder name is provided. -- `search_team_contributors()` function to address the error message when `email` parameter is used. -- `get_item_metadata()` to address the issue with approved/disapproved items. -### Removed -- `get_project_and_folder_metadata()` function. -- `get_image_metadata()` function. Please use `get_item_metadata()` instead. -- `search_images()` function. Please use `search_items()` instead. -- `search images_all_folders()` function. Please use `search_items()` instead. -### -## 4.3.2 - April 10 2022 -### Added -- `query()` function to run SAQuL queries via SDK. -- `search_items()` function to search items by various filtering criteria for all supported project types. `search_images()` and `search_images_all_folders()` functions will be deprecated. -- `get_item_metadata()` function to get item metadata for all supported project types. `get_image_metadata()` will be deprecated. -### Updated -- `search_projects()` function to add new parameter that gives an option to filter projects by project `status`. -- `get_annotation_per_frame()` function to add a unique identifier for each annotation instance. -### Fixed -- pixel annotations to address the issue with the hex code. -- `sa.validate_annotations()` function to address the incorrect error message. -- `create_project_from_metadata()` function to address the issue with instructions. -### Removed -- `get_image_annotations()` function. Please use `get_annotations()` -- `upload_images_from_public_urls()` function. -### -## 4.3.1 - March 20 2022 -### Added -- `get_integrations()` to list all existing integrations with cloud storages. -- `attach_items_from_integrated_storage()` to attach items from an integrated cloud storage. -- `upload_priority_scores()` to set priority scores for a given list of items. -### Updated -- `JSON Schema` to version `1.0.40` to add instance type differentiation for text annotations and `"exclude"` key for subtracted polygon instances for image annotations. -- `validate_annotations()` to validate text and image annotations based on JSON schema version `1.0.40`. -- `get_annotations()` to get annotation instances based on JSON schema version `1.0.40`. -- `prepare_export()` to prepare for the download annotations with based on JSON schema version `1.0.40`. -- `upload_annotations_from_folder_to_project()` & `upload_preannotations_from_folder_to_project()` to handle upload based on JSON schema version `1.0.40`. -- `create_project()` to add `"status"` key in returned metadata. -- `get_project_metadata()` to add `"status"` key. -- `create_project_from_project_metadata()` to make `"description"` key not required. -- `clone_project()` to add generic `"description"`. -### Fixed -- `sa.get_annotations_per_frame()` to take correct attributes. -- `sa.get_annotations_per_frame()` & `get_annotations()` to eliminate duplicate instances. -### -## 4.3.0 - Feb 27 2022 -### Added -- `get_annotations` to load annotations for the list of items. -- `get_annotations_per_frame` to generate frame by frame annotations for the given video. -### Updated -- `get_image_annotations()` to reference `get_annotations()`. -- `create_annotation_class()` to add `class_type` in parameters to specify class type on creation. -- `create_annotation_classes_from_classes_json()` to handle class type in classes JSON. -- `search_annotation_classes()` to return class type in metadata. -- `upload_annotations_from_folder_to_project()` to handle tag annotations. -- `upload_preannotations_from_folder_to_project()` to handle tag annotations. -- `upload_image_annotations()` to handle tag annotations. -- `validate_annotations()` to validate vector annotation schema with tag instances. -- `aggregate_annotations_as_df()` to handle tag annotations in annotations df. -- `class_distribution()` to handle class distribution of tag instances. -- `upload_images_from_public_urls()` for deprecation log. -### Fixed -- `upload_images_from_folder_to_project()` to upload images without invalid rotation. -- `upload-annotations` CLI to upload annotations to specified folder. -- `create_project_from_metadata()` to setup image quality and workflow from given metadata. -- `get_project_metadata()` to return information on project contributors. -- `get_project_metadata()` to return number of completed images in project root. -- `get_project_workflow()` to return `className` in project workflow. -- file handler permissions in GColab at `import` stage of the package. -### -## 4.2.9 - Jan 30 2022 -### Added -- `superannotate_schemas` as a stand alone package on annotation schemas. -### Updated -- `upload_annotations_from_folder_to_project()` to reference the `validate_annotations()`. -- `upload_videos_from_folder_to_project()` to remove code duplications. -- `clone_project()` to set upload state of clone project to inital. -### Fixed -- `validate_annotations()` to fix rotated bounding box schema. -### Removed -- Third party logs from logging mechanism. -### -## 4.2.8 - Jan 9 2022 -### Added -- `invite_contributers_to_team()` for bulk team invite. -- `add_contributors_to_project()` for bulk project sharing. -### Updated -- `upload_images_from_folder_to_project()` for non existing S3 path handling. -- `upload_annotations_from_folder_to_project()` for template name and class processing on template annotation upload. -- `add_annotation_comment_to_image()` for unrecognized author processing. -- `add_annotation_point_to_image()` for valid point addition on empty state. -- `add_annotation_bbox_to_image()` for valid bbox addition on empty state. -- `add_annotation_comment_to_image()` for valid comment addition on empty state. -### Fixed -- `superannotatecli upload_images` to accept default list of image extensions. -### Removed -- `invite_contributor_to_team()` use `invite_contributors_to_team()` instead. -### -## 4.2.7 - Dec 12 2021 -### Added -- Logging mechanism. -### Updated -- Cloning projects with attached URLs. -- Improve relation between image status and annotations. -- Deprecate functions of zero usage. -### Fixed -- Small bug fix & enhancements. -### -## 4.2.6 - Nov 21 2021 -### Added -- Validation schemas for annotations. -- Dataframe aggregation for video projects. -### Fixed -- Minor bug fixes and enhancements. -### -## 4.2.4 - Nov 2 2021 -### Fixed -- Minor bug fixes and enhancements. -### -## 4.2.3 - Oct 31 2021 -### Fixed -- Minor bug fixes and enhancements. -### -## 4.2.2 - Oct 22 2021 -### Fixed -- Minor bug fixes and enhancements. -### -## 4.2.1 - Oct 13 2021 -### Fixed -- `init` functionality. -- `upload_annotation` functionality. -### -## 4.2.0 - Oct 10 2021 -### Added -- `delete_annotations()` for bulk annotation delete. -### Updated -- Project/folder limitations. -### Fixed -- Refactor and major bug fix. -## 4.1.9 - Sep 22 2021 -### Added -- Text project support. -## 4.1.8 - Aug 15 2021 -### Added -- Video project release. -### -## 4.1.7 - Aug 1 2021 -### Fixed -- Video upload refinements. -### -## 4.1.6 - Jul 19 2021 -### Added -- Training/Test data with folder structure. -- Token validation. -### Updated -- Add success property on mixpanel events. -### Fixed -- Upload template enhancements. -### -## 4.1.5 - Jun 16 2021 -### Added -- Folder assignment. -### Updated -- COCO keypoint enhancements. -### -## 4.1.4 - May 26 2021 -### Added -- Mixpanel Integration. -### Updated -- Image upload enhancements. -- Video upload enhancements. -- Annotation upload enhancements. -- Consensus enhancements. -- Image copy/move enhancements. -- COCO import/export enhancements. -- AWS region enhancements. -### -## 4.1.3 - Apr 19 2021 -### Added -- Folder limitations. -### -## 4.1.2 - Apr 1 2021 -### Fixed -- Video upload to folder. -### -## 4.1.1 - Mar 31 2021 -### Added -- Attach image URLs. -### -## 4.1.0 - Mar 22 2021 -### Added -- Folder structure on platform. -### -## 4.0.1 - Mar 15 2021 -### Updated -- The FPS change during video upload has more stable frame choosing algorithm now. -### -## 4.0.0 - Feb 28 2021 -### Updated -- Improved image storage structure on platform, which requires this upgrade in SDK. This change in platform is backward incompatible with previous versions of SDK. -### -Changelog not maintained before version 4.0.0. diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 000000000..d20da23b3 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,596 @@ +.. _ref_history: + +======= +History +======= + +All release highlights of this project will be documented in this file. + +4.4.11 - April 2, 2023 +______________________ + +**Added** + + - ``SAClient.set_project_status()`` method. + - ``SAClient.set_folder_status()`` method. + +**Updated** + + - ``SAClient.create_annotation_class()`` added OCR type attribute group support in the vector projects. + - ``SAClient.create_annotation_classes_from_classes_json()`` added OCR type attribute group support in the vector projects. + +4.4.10 - March 12, 2023 +_______________________ + +**Updated** + + - Configuration file creation flow + - ``SAClient.search_projects()`` method, removed ``include_complete_image_count`` argument, use ``include_complete_item_count`` instead. + - ``SAClient.get_project_metadata()`` method, removed ``include_complete_image_count`` argument, use ``include_complete_item_count`` instead. + - ``SAClient.create_project()`` method to support classes, workflows and instructions_link. + +**Fixed** + + - ``SAClient.clone_project()`` method to address the issue of FPS mode is not being copied. + +**Deprecated** + + - ``SAClient.create_project_from_metadata()`` method, use ``SAClient.create_project()`` instead. + - ``SAClient.get_project_image_count()`` method, use ``SAClient.get_project_metadata()`` instead. + +**Removed** + + - ``SAClient.class_distribution()`` method + - ``SAClient.benchmark()`` method + +4.4.9 - January 29, 2023 +________________________ + +**Added** + + - ``SAClient.set_approval_statuses`` method function to change the approval status of items (images, audio / videos) in bulk. + +**Updated** + + - ``SAClient.convert_project_type`` method updated from Pixel to Vector converter, added polygon holes handling. + +4.4.8 - December 25, 2022 +____________________________ + +**Added** + + - New project types ``Tiled``, ``PointCloud``, ``Other``. + - ``SAClient.get_project_by_id`` method to get project metadata by id. + - ``SAClient.get_folder_by_id`` method to get folder metadata by id. + - ``SAClient.get_item_by_id`` method to get item metadata by id. + +**Updated** + + - ``SAClient.consensus`` method to compute agreement scores between tag type annotations. + +4.4.7 - December 04, 2022 +_________________________ + +**Updated** + + - ``SAClient.search_folders`` method to add a new ``status`` argument for searching folders via status. + - Schemas for ``Annotation Classes`` and ``Video Annotation`` to support **text** and **numeric input** attribute group types. + +**Fixed** + + - ``SAClient.query`` method to address invalid exceptions. + - ``SAClient.download_export`` method to address the issue with downloading for Windows OS. + - ``SAClient.attach_items_from_integrated_storage`` method to address "integration not found" error. + - ``SAClient.aggregate_annotations_as_df`` method to support files without "___objects" in their naming. + +**Removed** + + - ``SAClient.add_annotation_bbox_to_image`` method, use ``SAClient.upload_annotations`` instead. + - ``SAClient.add_annotation_point_to_image`` method, use ``SAClient.upload_annotations`` instead. + - ``SAClient.add_annotation_comment_to_image`` method, use ``SAClient.upload_annotations`` instead. + +4.4.6 - November 23, 2022 +_________________________ + +**Updated** + + - ``SAClient.aggregate_annotations_as_df`` method to aggregate "comment" type instances. + - ``SAClient.add_annotation_bbox_to_image``, ``SAClient.add_annotation_point_to_image``, ``SAClient.add_annotation_comment_to_image`` methods to add deprecation warnings. + +**Fixed** + + - Special characters are being encoded after annotation upload (Windows) + - ``SAClient.assign_folder`` method to address the invalid argument name. + - ``SAClient.upload_images_from_folder_to_project`` method to address uploading of more than 500 items. + - ``SAClient.upload_annotations_from_folder_to_project`` method to address the issue of a folder size being more than 25,5 MB. + - ``SAClient.download_image`` method to address the KeyError 'id' when ``include_annotations`` is set to ``True``. + +**Removed** + + - ``SAClient.upload_preannotations_from_folder_to_project`` method + - ``SAClient.copy_image`` method + +4.4.5 - October 23, 2022 +________________________ + +**Added** + + - ``SAClient.add_items_to_subset`` method to associate given items with a Subset. + - ``SAClient.upload_annotations`` method to upload annotations in SA format from the system memory. + +**Updated** + + - ``SAClient.upload_annotations_from_folder_to_project`` & ``SAClient.upload_image_annotations`` methods to add ``keep_status`` argument to prevent the annotation status from changing to **In Progress** after the annotation upload. + - Item metadata to add a new key for holding the id of an item. + - ``SAClient.upload_preannotations_from_folder_to_project`` to add a deprecation warning message. + - ``SAClient.copy_image`` to add a deprecation warning message. + +**Fixed** + + - ``SAClient.validate_annotations`` method. + - ``SAClient.search_items``, ``SAClient.get_item_metadata`` methods to address defects related to pydantic 1.8.2. + - A defect related to editor to address the issue of uploading a tag instance without attributes. + +4.4.4 - September 11, 2022 +__________________________ + +**Updated** + + - Improvements on working with large files. + +**Fixed** + + - ``SAClient.upload_annotations_from_folder_to_project()`` method to address the issue of the disappearing progress bar. + - ``SAClient.run_prediction()`` method to address the issue of the OCR model. + - ``SAClient.validate_annotations()`` method to address the issue of missing log messages. + - ``SAClient.create_project_from_metadata()`` method to address the issue of returning deprecated ``is_multiselect`` key. + - ``SAClient.get_annotations()`` method to address the issue of returning error messages as annotation dicts. + +4.4.2, 4.4.3 - August 21, 2022 +______________________________ + +**Updated** + + - the **schema** of ``classes JSON`` to support new values for the ``"group_type"`` key for a given attribute group. ``"group_type": "radio" | "checklist" | "text" | "numeric"``. + - the **schema** of ``video annotation JSON`` to support instances that have a ``"tag"`` type. + +**Fixed** + + - ``SAClient.get_annotations()`` method to address the issue of working with the large projects. + - ``SAClient.get_annotations_per_frame()`` method to address the issue of throwing an error on small videos when the fps is set to 1. + - ``SAClient.upload_annotations_from_folder_to_project()`` to address the issue of timestamp values represented in seconds for the ``"lastAction"``. + - ``SAClient.download_export()`` method to address the issue of empty logs. + - ``SAClient.clone_project()`` method to address the issue of having a corrupted project clone, when the source project has a keypoint workflow. + +4.4.1 - July 24, 2022 +_____________________ + +**Added** + + - ``SAClient.create_custom_fields()`` method to create/add new custom fields to a project’s custom field schema. + - ``SAClient.get_custom_fields()`` method to get a project’s custom field schema. + - ``SAClient.delete_custom_fields()`` method to remove existing custom fields from a project’s custom field schema. + - ``SAClient.upload_custom_values()`` method to attach custom field-value pairs to items. + - ``SAClient.delete_custom_values()`` method to remove custom field-value pairs from items. + +**Updated** + + - The **schema** of ``classes JSON`` to support the new ``"default_value"`` key to set a default attribute(s) for a given attribute group. + - ``SAClient.get_item_metadata()`` method to add a new input argument ``include_custom_metadata`` to return custom metadata in the result items. + - ``SAClient.search_items()`` method to add a new input argument ``include_custom_metadata`` to return custom metadata in the result items. + - ``SAClient.query()`` method to return custom metadata in the result items. + +**Fixed** + + - ``SAClient`` class to address the system crash that occurs on instantiation via ``config.json`` file. + - ``SAClient.query()`` method to address the issue of not returning more than 50 items. + - ``SAClient.upload_annotations_from_folder_to_project()`` to address the issue of some fields not being auto populated after the upload is finished. + - ``SAClient.get_folder_metadata()``, ``SAClient.search_folders()`` to address the issue of transforming the ‘+’ sign in a folder to a whitespace. + +**Removed** + + - ``superannotate.assign_images()`` function. Please use the ``SAClient.assign_items()`` method instead. + - ``superannotate.unassign_images()`` function. Please use the ``SAClient.unassign_items()`` method instead. + - ``superannotate.delete_images()`` function. Please use the ``SAClient.delete_items()`` method instead. + +4.4.0 - July 03, 2022 +_____________________ + +**Added** + + - ``superannotate.SAClient()`` class to instantiate team-level authentication and inheriting methods to access the back-end. + - ``SAClient.download_annotations()`` method to download annotations without preparing an Export object. + - ``SAClient.get_subsets()`` method to get the existing subsets for a given project. + - ``SAClient.assign_items()`` method to assign items in a given project to annotators or quality specialists. + - ``SAClient.unassign_items()`` method to remove assignments from items. + - ``SAClient.delete_items()`` method to delete items in a given project. + +**Updated** + + - ``JSON Schema`` for video annotations to version ``1.0.45`` to show **polygon** and **polyline** annotations. + - ``SAClient.get_annotations_per_frame()`` method to show **polygon** and **polyline** annotations. + - ``SAClient.get_annotations_per_frame()`` method to pick instances closer to a given **frame start** instead of the **median**. + - ``SAClient.query()`` method to add the ``subset`` argument to support querying in a subset. + +**Fixed** + + - ``SAClient.set_annotation_statuses()`` method to address the issue occurring with more than 500 items. + - ``SAClient.get_annotations()`` method to address the ``PayloadError`` occurring with more than 20000 items. + - ``SAClient.get_annotations()`` method to address the missing ``'duration'`` and ``'tags'`` keys for newly uploaded and unannotated videos. + - ``SAClient.get_annotations_per_frame()`` method to address missing ``'duration'`` and ``'tags'`` keys for newly uploaded and unannotated videos. + - ``SAClient.get_annotations_per_frame()`` method to address the wrong ``classId`` value for unclassified instances. + +**Removed** + + - ``superannotate.init()`` function. Please instantiate ``superannotate.SAClient()`` class to authenticate. + - ``superannotate.set_image_annotation_status()`` function. Please use the ``SAClient.set_annotation_statuses()`` method instead. + - ``superannotate.set_images_annotations_statuses()`` function. Please use the ``SAClient.set_annotation_statuses()`` method instead. + +4.3.4 - May 22, 2022 +____________________ + +**Updated** + + - ``JSON Schema`` for video annotations to version ``x`` to reflect point annotations. + - ``superannotate.download_export()`` function to preserve SA folder structure while downloading to S3 bucket. + - ``superannotate.get_item_metadata()`` function to have string type values instead of int type for the ``approval_status`` key. + - ``superannotate.get_item_metadata()`` function to change the value for the ``path`` key in the item metadata from ``project/folder/item`` format to ``project/folder``. + - ``superannotate.get_item_metadata()`` function to add the ``is_pinned`` key in the returned metadata. + - ``superannotate.clone_project()`` function to have ``NotStarted`` project status for the newly created project. + +**Fixed** + + - ``superannotate.query()`` function to address the missing value for the ``path`` key. + - ``superannotate.import_annotation()`` function to address the extension issue with JPEG files while converting from ``VOC`` to SA. + - ``superannotate.import_annotation()`` function to address int type pointlabels in the converted ``JSON`` from ``COCO`` to SA. + - ``superannotate_get_annotations()`` & ``superannotate.add_annotation_comment_to_image()`` to address the issue with ``asyncio`` occurring on Windows. + - ``superannotate.set_image_annotation_status()`` function add a deprecation warning. + - ``superannotate.set_images_annotation_statuses()`` function add a deprecation warning. + +**Removed** + + - ``share_projects()`` function. + - ``superannotate.attach_image_urls_to_project()`` function. Please use the ``superannotate.attach_items()`` function instead. + - ``superannotate.attach_document_urls_to_project()`` function. Please use the ``superannotate.attach_items()`` function instead. + - ``superannotate.attach_video_urls_to_project()`` function. Please use the ``superannotate.attach_items()`` function instead. + - ``superannotate.copy_images()`` function. Please use the ``superannotate.copy_items()`` function instead. + - ``superannotate.move_images()`` function. Please use the ``superannotate.move_items()`` function instead. + +4.3.3 - May 01 2022 +___________________ + +**Added** + + - ``attach_items()`` function to link items (images, videos, and documents) from external storages to SuperAnnotate using URLs. + - ``copy_items()`` function to copy items (images, videos, and documents) in bulk between folders in a project. + - ``move_items()`` function to move items (images, videos, and documents) in bulk between folders in a project. + - ``set_annotation_statuses()`` function to change the annotation status of items (images, videos, and documents) in bulk. + +**Updated** + + - ``aggregate_annotations_as_df()`` function now supports Text Projects. + +**Fixed** + + - ``validate_annotations()`` function to accept only numeric type values for the ``points`` field. + - ``prepare_export()`` function to address the issue when the entire project is prepared when a wrong folder name is provided. + - ``search_team_contributors()`` function to address the error message when ``email`` parameter is used. + - ``get_item_metadata()`` to address the issue with approved/disapproved items. + +**Removed** + + - ``get_project_and_folder_metadata()`` function. + - ``get_image_metadata()`` function. Please use ``get_item_metadata()`` instead. + - ``search_images()`` function. Please use ``search_items()`` instead. + - ``search images_all_folders()`` function. Please use ``search_items()`` instead. + +4.3.2 - April 10 2022 +_____________________ + +**Added** + + - ``query()`` function to run SAQuL queries via SDK. + - ``search_items()`` function to search items by various filtering criteria for all supported project types. ``search_images()`` and ``search_images_all_folders()`` functions will be deprecated. + - ``get_item_metadata()`` function to get item metadata for all supported project types. ``get_image_metadata()`` will be deprecated. + +**Updated** + + - ``search_projects()`` function to add new parameter that gives an option to filter projects by project ``status``. + - ``get_annotation_per_frame()`` function to add a unique identifier for each annotation instance. + +**Fixed** + + - pixel annotations to address the issue with the hex code. + - ``sa.validate_annotations()`` function to address the incorrect error message. + - ``create_project_from_metadata()`` function to address the issue with instructions. + +**Removed** + + - ``get_image_annotations()`` function. Please use ``get_annotations()`` + - ``upload_images_from_public_urls()`` function. + +4.3.1 - March 20 2022 +_____________________ + +**Added** + + - ``get_integrations()`` to list all existing integrations with cloud storages. + - ``attach_items_from_integrated_storage()`` to attach items from an integrated cloud storage. + - ``upload_priority_scores()`` to set priority scores for a given list of items. + +**Updated** + + - ``JSON Schema`` to version ``1.0.40`` to add instance type differentiation for text annotations and ``"exclude"`` key for subtracted polygon instances for image annotations. + - ``validate_annotations()`` to validate text and image annotations based on JSON schema version ``1.0.40``. + - ``get_annotations()`` to get annotation instances based on JSON schema version ``1.0.40``. + - ``prepare_export()`` to prepare for the download annotations with based on JSON schema version ``1.0.40``. + - ``upload_annotations_from_folder_to_project()`` & ``upload_preannotations_from_folder_to_project()`` to handle upload based on JSON schema version ``1.0.40``. + - ``create_project()`` to add ``"status"`` key in returned metadata. + - ``get_project_metadata()`` to add ``"status"`` key. + - ``create_project_from_project_metadata()`` to make ``"description"`` key not required. + - ``clone_project()`` to add generic ``"description"``. + +**Fixed** + + - ``sa.get_annotations_per_frame()`` to take correct attributes. + - ``sa.get_annotations_per_frame()`` & ``get_annotations()`` to eliminate duplicate instances. + +4.3.0 - Feb 27 2022 +___________________ + +**Added** + + - ``get_annotations`` to load annotations for the list of items. + - ``get_annotations_per_frame`` to generate frame by frame annotations for the given video. + +**Updated** + + - ``get_image_annotations()`` to reference ``get_annotations()``. + - ``create_annotation_class()`` to add ``class_type`` in parameters to specify class type on creation. + - ``create_annotation_classes_from_classes_json()`` to handle class type in classes JSON. + - ``search_annotation_classes()`` to return class type in metadata. + - ``upload_annotations_from_folder_to_project()`` to handle tag annotations. + - ``upload_preannotations_from_folder_to_project()`` to handle tag annotations. + - ``upload_image_annotations()`` to handle tag annotations. + - ``validate_annotations()`` to validate vector annotation schema with tag instances. + - ``aggregate_annotations_as_df()`` to handle tag annotations in annotations df. + - ``class_distribution()`` to handle class distribution of tag instances. + - ``upload_images_from_public_urls()`` for deprecation log. + +**Fixed** + + - ``upload_images_from_folder_to_project()`` to upload images without invalid rotation. + - ``upload-annotations`` CLI to upload annotations to specified folder. + - ``create_project_from_metadata()`` to setup image quality and workflow from given metadata. + - ``get_project_metadata()`` to return information on project contributors. + - ``get_project_metadata()`` to return number of completed images in project root. + - ``get_project_workflow()`` to return ``className`` in project workflow. + - file handler permissions in GColab at ``import`` stage of the package. + +4.2.9 - Jan 30 2022 +___________________ + +**Added** + + - ``superannotate_schemas`` as a stand alone package on annotation schemas. + +**Updated** + + - ``upload_annotations_from_folder_to_project()`` to reference the ``validate_annotations()``. + - ``upload_videos_from_folder_to_project()`` to remove code duplications. + - ``clone_project()`` to set upload state of clone project to initial. + +**Fixed** + + - ``validate_annotations()`` to fix rotated bounding box schema. + +**Removed** + + - Third party logs from logging mechanism. + +4.2.8 - Jan 9 2022 +__________________ + +**Added** + + - ``invite_contributers_to_team()`` for bulk team invite. + - ``add_contributors_to_project()`` for bulk project sharing. + +**Updated** + + - ``upload_images_from_folder_to_project()`` for non existing S3 path handling. + - ``upload_annotations_from_folder_to_project()`` for template name and class processing on template annotation upload. + - ``add_annotation_comment_to_image()`` for unrecognized author processing. + - ``add_annotation_point_to_image()`` for valid point addition on empty state. + - ``add_annotation_bbox_to_image()`` for valid bbox addition on empty state. + - ``add_annotation_comment_to_image()`` for valid comment addition on empty state. + +**Fixed** + + - ``superannotatecli upload_images`` to accept default list of image extensions. + +**Removed** + + - ``invite_contributor_to_team()`` use ``invite_contributors_to_team()`` instead. + +4.2.7 - Dec 12 2021 +___________________ + +**Added** + + - Logging mechanism. + +**Updated** + + - Cloning projects with attached URLs. + - Improve relation between image status and annotations. + - Deprecate functions of zero usage. + +**Fixed** + + - Small bug fix & enhancements. + +4.2.6 - Nov 21 2021 +___________________ + +**Added** + + - Validation schemas for annotations. + - Dataframe aggregation for video projects. + +**Fixed** + + - Minor bug fixes and enhancements. + +4.2.4 - Nov 2 2021 +__________________ + +**Fixed** + + - Minor bug fixes and enhancements. + +4.2.3 - Oct 31 2021 +___________________ + +**Fixed** + + - Minor bug fixes and enhancements. + +4.2.2 - Oct 22 2021 +___________________ + +**Fixed** + + - Minor bug fixes and enhancements. + +4.2.1 - Oct 13 2021 +___________________ + +**Fixed** + + - ``init`` functionality. + - ``upload_annotation`` functionality. + +4.2.0 - Oct 10 2021 +___________________ + +**Added** + + - ``delete_annotations()`` for bulk annotation delete. + +**Updated** + + - Project/folder limitations. + +**Fixed** + + - Refactor and major bug fix. + +4.1.9 - Sep 22 2021 +___________________ + +**Added** + + - Text project support. + +4.1.8 - Aug 15 2021 +___________________ + +**Added** + + - Video project release. + +4.1.7 - Aug 1 2021 +__________________ + +**Fixed** + + - Video upload refinements. + +4.1.6 - Jul 19 2021 +___________________ + +**Added** + + - Training/Test data with folder structure. + - Token validation. + +**Updated** + + - Add success property on mixpanel events. + +**Fixed** + + - Upload template enhancements. + +4.1.5 - Jun 16 2021 +___________________ + +**Added** + + - Folder assignment. + +**Updated** + + - COCO keypoint enhancements. + +4.1.4 - May 26 2021 +___________________ + +**Added** + + - Mixpanel Integration. + +**Updated** + + - Image upload enhancements. + - Video upload enhancements. + - Annotation upload enhancements. + - Consensus enhancements. + - Image copy/move enhancements. + - COCO import/export enhancements. + - AWS region enhancements. + +4.1.3 - Apr 19 2021 +___________________ + +**Added** + + - Folder limitations. + +4.1.2 - Apr 1 2021 +__________________ + +**Fixed** + + - Video upload to folder. + +4.1.1 - Mar 31 2021 +___________________ + +**Added** + + - Attach image URLs. + +4.1.0 - Mar 22 2021 +___________________ + +**Added** + + - Folder structure on platform. + +4.0.1 - Mar 15 2021 +___________________ + +**Updated** + + - The FPS change during video upload has more stable frame choosing algorithm now. + +4.0.0 - Feb 28 2021 +___________________ + +**Updated** + + - Improved image storage structure on platform, which requires this upgrade in SDK. This change in platform is backward incompatible with previous versions of SDK. + +Changelog not maintained before version 4.0.0. diff --git a/README.rst b/README.rst index 4d7c71847..675982ff3 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ Welcome to the SuperAnnotate Python Software Development Kit (SDK), which enable :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/LICENSE/ :alt: License .. |Changelog| image:: https://img.shields.io/static/v1?label=change&message=log&color=yellow&style=flat-square - :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/CHANGELOG.md + :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/CHANGELOG.rst :alt: Changelog Resources diff --git a/docs/source/changelog_link.rst b/docs/source/changelog_link.rst new file mode 100644 index 000000000..09929fe43 --- /dev/null +++ b/docs/source/changelog_link.rst @@ -0,0 +1 @@ +.. include:: ../../CHANGELOG.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index fe53b1efb..b4d9afc86 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,6 +16,7 @@ API Reference CLI Reference SA Server + History LICENSE.rst ---------- diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 50b1ecdf5..5725a58a0 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -211,7 +211,9 @@ def _track_method(self, args, kwargs, success: bool): team_name = client.controller.team_data.name properties["Success"] = success - default = self.get_default_payload(team_name=team_name, user_email=user_email) + default = self.get_default_payload( + team_name=team_name, user_email=user_email + ) self._track( user_email, event_name, diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py index 87263d991..9a34011b0 100644 --- a/tests/integration/mixpanel/test_mixpanel_decorator.py +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -67,9 +67,7 @@ def test_init(self, track_method): @patch("lib.app.interface.base_interface.Tracker._track") @patch("lib.core.usecases.GetTeamUseCase") @patch("lib.infrastructure.serviceprovider.ServiceProvider.get_user") - def test_init_via_token( - self, get_user, get_team_use_case, track_method - ): + def test_init_via_token(self, get_user, get_team_use_case, track_method): SAClient(token="test=3232") result = list(track_method.call_args)[0] payload = self.default_payload @@ -87,9 +85,7 @@ def test_init_via_token( @patch("lib.app.interface.base_interface.Tracker._track") @patch("lib.core.usecases.GetTeamUseCase") @patch("lib.infrastructure.serviceprovider.ServiceProvider.get_user") - def test_init_via_config_file( - self, get_user, get_team_use_case, track_method - ): + def test_init_via_config_file(self, get_user, get_team_use_case, track_method): with tempfile.TemporaryDirectory() as config_dir: config_ini_path = f"{config_dir}/config.ini" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path): @@ -191,12 +187,8 @@ def test_create_project_multi_thread(self, track_method): "project_description": self.PROJECT_DESCRIPTION, "project_type": self.PROJECT_TYPE, } - thread_1 = threading.Thread( - target=sa.create_project, kwargs=kwargs_1 - ) - thread_2 = threading.Thread( - target=sa.create_project, kwargs=kwargs_2 - ) + thread_1 = threading.Thread(target=sa.create_project, kwargs=kwargs_1) + thread_2 = threading.Thread(target=sa.create_project, kwargs=kwargs_2) thread_1.start() thread_2.start() thread_1.join() From 607a7bf45736ac12c7dd325d3876c580590d7007 Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan <124172833+nareksa@users.noreply.github.com> Date: Mon, 17 Apr 2023 16:57:13 +0400 Subject: [PATCH 11/17] Update __init__.py --- src/superannotate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 27e2f0f30..abe409bd3 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys import typing -__version__ = "4.4.12dev1" +__version__ = "4.4.12dev2" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) From 804c48c752dd6e88dd28c6a3638be5f690af60de Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 19 Apr 2023 11:24:30 +0400 Subject: [PATCH 12/17] Docs upadates --- docs/source/api_reference/api_folder.rst | 1 + docs/source/index.rst | 111 +----------------- requirements.txt | 6 +- .../lib/app/interface/sdk_interface.py | 2 +- 4 files changed, 6 insertions(+), 114 deletions(-) diff --git a/docs/source/api_reference/api_folder.rst b/docs/source/api_reference/api_folder.rst index d4890066c..916a2121d 100644 --- a/docs/source/api_reference/api_folder.rst +++ b/docs/source/api_reference/api_folder.rst @@ -3,6 +3,7 @@ Folders ======= .. automethod:: superannotate.SAClient.search_folders +.. automethod:: superannotate.SAClient.set_folder_status .. automethod:: superannotate.SAClient.assign_folder .. automethod:: superannotate.SAClient.unassign_folder .. automethod:: superannotate.SAClient.get_folder_by_id diff --git a/docs/source/index.rst b/docs/source/index.rst index b4d9afc86..c89ca3a6a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,113 +19,4 @@ History LICENSE.rst ----------- - -SuperAnnotate Python SDK documentation -================================================================== - -SuperAnnotate Python SDK allows access to the platform without web browser: - -.. code-block:: python - - from superannotate import SAClient - - - sa_client = SAClient() - - project = 'Dogs' - - sa_client.create_project( - project_name=project, - project_description='Test project generated via SDK', - project_type='Vector' - ) - - sa_client.create_annotation_class( - project=project, - name='dog', - color='#F9E0FA', - class_type='tag' - ) - - sa_client.attach_items( - project=project, - attachments=[ - { - 'url': 'https://drive.google.com/uc?export=download&id=1ipOrZNSTlPUkI_hnrW9aUD5yULqqq5Vl', - 'name': 'dog.jpeg' - } - ] - ) - - sa_client.upload_annotations( - project=project, - annotations=[ - { - 'metadata': {'name': 'dog.jpeg'}, - 'instances': [ - {'type': 'tag', 'className': 'dog'} - ] - } - ] - ) - - sa_client.get_annotations(project=project, items=['dog.jpeg']) - ----------- - -Installation -____________ - - -SDK is available on PyPI: - -.. code-block:: bash - - pip install superannotate - - -The package officially supports Python 3.7+ and was tested under Linux and -Windows (`Anaconda `_) platforms. - ----------- - -Supported Features -__________________ - -- Search projects -- Create/delete a project -- Upload images to a project from a local or AWS S3 folder -- Upload videos to a project from a local folder -- Upload annotations/pre-annotations to a project from local or AWS S3 folder -- Set the annotation status of the images being uploaded -- Export annotations from a project to a local or AWS S3 folder -- Share and unshare a project with a team contributor -- Invite a team contributor -- Search images in a project -- Download a single image -- Copy/move image between projects -- Get image bytes (e.g., for numpy array creation) -- Set image annotation status -- Download image annotations/pre-annotations -- Create/download project annotation classes -- Add annotations to images on platform -- Convert annotation format from/to COCO -- Add annotations to local SuperAnnotate format JSONs -- CLI commands for simple tasks -- Aggregate class/attribute distribution as histogram - ----------- - -License -_______ - -This SDK is distributed under the :ref:`MIT License `. - ----------- - -Questions and Issues -____________________ - -For questions and issues please use issue tracker on -`GitHub `_. +.. include:: ../../README.rst diff --git a/requirements.txt b/requirements.txt index e7777454d..6f594e2d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,17 +8,17 @@ pillow==9.4.0 matplotlib>=3.3.1 xmltodict==0.12.0 opencv-python>=4.4.0.42 -wheel>=0.38.4 +wheel>=0.40.0 packaging>=20.4 plotly>=4.1.0 ffmpeg-python>=0.2.0 fire==0.4.0 mixpanel==4.8.3 -pydantic>=1.10.4 +pydantic>=1.10.7 setuptools>=57.4.0 email-validator>=1.0.3 jsonschema==3.2.0 pandas>=1.1.4 aiofiles==0.8.0 -Werkzeug==2.2.2 +Werkzeug==2.2.3 Jinja2==3.1.2 diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index f4980b4b3..ad4b4324e 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2192,7 +2192,7 @@ def get_annotations( :param project: project name or folder path (e.g., “project1/folder1”). :type project: str - :param items: item names. If None all items in the project will be exported + :param items: item names. If None all items in specified directory :type items: list of strs :return: list of annotations From 0415f81701079aa8fd683a97765872320bac5cb4 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:39:24 +0400 Subject: [PATCH 13/17] Update __init__.py --- src/superannotate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index abe409bd3..5ca137a9e 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys import typing -__version__ = "4.4.12dev2" +__version__ = "4.4.12dev3" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) From 5b170b8aa3fa0a6a0b5fdcefb9bc62b91d323b3b Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 19 Apr 2023 15:14:23 +0400 Subject: [PATCH 14/17] doc string updates --- .../lib/app/interface/sdk_interface.py | 8 +++--- .../lib/infrastructure/controller.py | 2 +- tests/unit/__init__.py | 1 + tests/unit/test_init.py | 25 +++++++++++++------ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index f4980b4b3..364ac128e 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2087,7 +2087,7 @@ def delete_annotations( :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param item_names: image names. If None, all image annotations from a given project/folder will be deleted. + :param item_names: item names. If None, all the items in the specified directory will be deleted. :type item_names: list of strs """ @@ -2192,7 +2192,7 @@ def get_annotations( :param project: project name or folder path (e.g., “project1/folder1”). :type project: str - :param items: item names. If None all items in the project will be exported + :param items: item names. If None, all the items in the specified directory will be used. :type items: list of strs :return: list of annotations @@ -2232,7 +2232,7 @@ def get_annotations_per_frame( return response.data def upload_priority_scores(self, project: NotEmptyStr, scores: List[PriorityScore]): - """Returns per frame annotations for the given video. + """Upload priority scores for the given list of items. :param project: project name or folder path (e.g., “project1/folder1”) :type project: str @@ -2673,7 +2673,7 @@ def set_annotation_statuses( ♦ “Skipped” \n :type annotation_status: str - :param items: item names to set the mentioned status for. If None, all the items in the project will be used. + :param items: item names. If None, all the items in the specified directory will be used. :type items: list of strs """ diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 7b3cb3574..881df1234 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -795,8 +795,8 @@ def __init__(self, config: ConfigEntity): ) self.service_provider = ServiceProvider(http_client) - self._team = self.get_team().data self._user = self.get_current_user() + self._team = self.get_team().data self.annotation_classes = AnnotationClassManager(self.service_provider) self.projects = ProjectManager(self.service_provider) self.folders = FolderManager(self.service_provider) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index e69de29bb..e6b6f186e 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +from superannotate.lib.infrastructure.validators import validators \ No newline at end of file diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index d882ba293..5e51081b6 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -19,17 +19,20 @@ def test_init_via_invalid_token(self): with self.assertRaisesRegexp(AppException, r"(\s+)token(\s+)Invalid token."): SAClient(token=_token) + @patch("lib.infrastructure.controller.Controller.get_current_user") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_token(self, get_team_use_case): + def test_init_via_token(self, get_team_use_case, get_current_user): sa = SAClient(token=self._token) assert get_team_use_case.call_args_list[0].kwargs["team_id"] == int( self._token.split("=")[-1] ) + assert get_current_user.call_count == 1 assert sa.controller._config.API_TOKEN == self._token assert sa.controller._config.API_URL == constants.BACKEND_URL + @patch("lib.infrastructure.controller.Controller.get_current_user") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_config_json(self, get_team_use_case): + def test_init_via_config_json(self, get_team_use_case, get_current_user): with tempfile.TemporaryDirectory() as config_dir: config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" @@ -40,11 +43,13 @@ def test_init_via_config_json(self, get_team_use_case): json.dump({"token": self._token}, config_json) for kwargs in ({}, {"config_path": f"{config_dir}/config.json"}): sa = SAClient(**kwargs) + assert sa.controller._config.API_TOKEN == self._token assert sa.controller._config.API_URL == constants.BACKEND_URL assert get_team_use_case.call_args_list[0].kwargs["team_id"] == int( self._token.split("=")[-1] ) + assert get_current_user.call_count == 2 def test_init_via_config_json_invalid_json(self): with tempfile.TemporaryDirectory() as config_dir: @@ -61,8 +66,9 @@ def test_init_via_config_json_invalid_json(self): ): SAClient(**kwargs) + @patch("lib.infrastructure.controller.Controller.get_current_user") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_config_ini(self, get_team_use_case): + def test_init_via_config_ini(self, get_team_use_case, get_current_user): with tempfile.TemporaryDirectory() as config_dir: config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" @@ -85,9 +91,11 @@ def test_init_via_config_ini(self, get_team_use_case): assert get_team_use_case.call_args_list[0].kwargs["team_id"] == int( self._token.split("=")[-1] ) + assert get_current_user.call_count == 2 + @patch("lib.infrastructure.controller.Controller.get_current_user") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_config_relative_filepath(self, get_team_use_case): + def test_init_via_config_relative_filepath(self, get_team_use_case, get_current_user): with tempfile.TemporaryDirectory(dir=Path("~").expanduser()) as config_dir: config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" @@ -113,14 +121,17 @@ def test_init_via_config_relative_filepath(self, get_team_use_case): assert get_team_use_case.call_args_list[0].kwargs["team_id"] == int( self._token.split("=")[-1] ) + assert get_current_user.call_count == 2 - @patch("lib.core.usecases.GetTeamUseCase") + @patch("lib.infrastructure.controller.Controller.get_current_user") + @patch("lib.infrastructure.controller.Controller.get_team") @patch.dict(os.environ, {"SA_URL": "SOME_URL", "SA_TOKEN": "SOME_TOKEN=123"}) - def test_init_env(self, get_team_use_case): + def test_init_env(self, get_team, get_current_user): sa = SAClient() assert sa.controller._config.API_TOKEN == "SOME_TOKEN=123" assert sa.controller._config.API_URL == "SOME_URL" - assert get_team_use_case.call_args_list[0].kwargs["team_id"] == 123 + assert get_team.call_count == 1 + assert get_current_user.call_count == 1 @patch.dict(os.environ, {"SA_URL": "SOME_URL", "SA_TOKEN": "SOME_TOKEN"}) def test_init_env_invalid_token(self): From 7c520ba8311ac122d802313e6abe5e71ce452193 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 19 Apr 2023 16:06:42 +0400 Subject: [PATCH 15/17] Upgraded requests package --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f594e2d3..5723ff2a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pydicom>=2.0.0 boto3>=1.14.53 -requests==2.26.0 +requests==2.28.2 requests-toolbelt>=0.9.1 aiohttp>=3.8.1 tqdm==4.64.0 From cccef36263a7ea3eaa68568b35ea73cb96ac5724 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 19 Apr 2023 17:16:27 +0400 Subject: [PATCH 16/17] Folder tests update --- .../lib/app/interface/sdk_interface.py | 49 ++- src/superannotate/lib/core/entities/folder.py | 3 +- .../lib/core/entities/project.py | 3 + tests/integration/folders/__init__.py | 10 + .../integration/folders/test_create_folder.py | 31 ++ .../folders/test_delete_folders.py | 44 +++ tests/integration/folders/test_folders.py | 285 ------------------ .../folders/test_get_folder_metadata.py | 27 ++ .../folders/test_search_folders.py | 38 +++ tests/unit/__init__.py | 1 - tests/unit/test_init.py | 4 +- 11 files changed, 181 insertions(+), 314 deletions(-) create mode 100644 tests/integration/folders/test_create_folder.py create mode 100644 tests/integration/folders/test_delete_folders.py delete mode 100644 tests/integration/folders/test_folders.py create mode 100644 tests/integration/folders/test_get_folder_metadata.py create mode 100644 tests/integration/folders/test_search_folders.py diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 364ac128e..e0632848a 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -24,7 +24,6 @@ from typing import TypedDict, NotRequired, Required # noqa import boto3 -from pydantic import StrictBool from pydantic import conlist from pydantic import constr from pydantic import parse_obj_as @@ -388,10 +387,10 @@ def clone_project( project_name: Union[NotEmptyStr, dict], from_project: Union[NotEmptyStr, dict], project_description: Optional[NotEmptyStr] = None, - copy_annotation_classes: Optional[StrictBool] = True, - copy_settings: Optional[StrictBool] = True, - copy_workflow: Optional[StrictBool] = False, - copy_contributors: Optional[StrictBool] = False, + copy_annotation_classes: Optional[bool] = True, + copy_settings: Optional[bool] = True, + copy_workflow: Optional[bool] = False, + copy_contributors: Optional[bool] = False, ): """Create a new project in the team using annotation classes and settings from from_project. @@ -578,7 +577,7 @@ def search_folders( project: NotEmptyStr, folder_name: Optional[NotEmptyStr] = None, status: Optional[Union[FOLDER_STATUS, List[FOLDER_STATUS]]] = None, - return_metadata: Optional[StrictBool] = False, + return_metadata: Optional[bool] = False, ): """Folder name based case-insensitive search for folders in project. @@ -625,11 +624,11 @@ def search_folders( def get_project_metadata( self, project: Union[NotEmptyStr, dict], - include_annotation_classes: Optional[StrictBool] = False, - include_settings: Optional[StrictBool] = False, - include_workflow: Optional[StrictBool] = False, - include_contributors: Optional[StrictBool] = False, - include_complete_item_count: Optional[StrictBool] = False, + include_annotation_classes: Optional[bool] = False, + include_settings: Optional[bool] = False, + include_workflow: Optional[bool] = False, + include_contributors: Optional[bool] = False, + include_complete_item_count: Optional[bool] = False, ): """Returns project metadata @@ -809,7 +808,7 @@ def pin_image( self, project: Union[NotEmptyStr, dict], image_name: str, - pin: Optional[StrictBool] = True, + pin: Optional[bool] = True, ): """Pins (or unpins) image @@ -966,7 +965,7 @@ def upload_images_from_folder_to_project( exclude_file_patterns: Optional[ Iterable[NotEmptyStr] ] = constants.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders: Optional[StrictBool] = False, + recursive_subfolders: Optional[bool] = False, image_quality_in_editor: Optional[str] = None, ): """Uploads all images with given extensions from folder_path to the project. @@ -1103,7 +1102,7 @@ def download_image_annotations( return res.data def get_exports( - self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False + self, project: NotEmptyStr, return_metadata: Optional[bool] = False ): """Get all prepared exports of the project. @@ -1125,7 +1124,7 @@ def prepare_export( project: Union[NotEmptyStr, dict], folder_names: Optional[List[NotEmptyStr]] = None, annotation_statuses: Optional[List[ANNOTATION_STATUS]] = None, - include_fuse: Optional[StrictBool] = False, + include_fuse: Optional[bool] = False, only_pinned=False, ): """Prepare annotations and classes.json for export. Original and fused images for images with @@ -1180,7 +1179,7 @@ def upload_videos_from_folder_to_project( Union[Tuple[NotEmptyStr], List[NotEmptyStr]] ] = constants.DEFAULT_VIDEO_EXTENSIONS, exclude_file_patterns: Optional[List[NotEmptyStr]] = (), - recursive_subfolders: Optional[StrictBool] = False, + recursive_subfolders: Optional[bool] = False, target_fps: Optional[int] = None, start_time: Optional[float] = 0.0, end_time: Optional[float] = None, @@ -1513,7 +1512,7 @@ def download_export( project: Union[NotEmptyStr, dict], export: Union[NotEmptyStr, dict], folder_path: Union[str, Path], - extract_zip_contents: Optional[StrictBool] = True, + extract_zip_contents: Optional[bool] = True, to_s3_bucket=None, ): """Download prepared export. @@ -1572,9 +1571,9 @@ def download_image( project: Union[NotEmptyStr, dict], image_name: NotEmptyStr, local_dir_path: Optional[Union[str, Path]] = "./", - include_annotations: Optional[StrictBool] = False, - include_fuse: Optional[StrictBool] = False, - include_overlay: Optional[StrictBool] = False, + include_annotations: Optional[bool] = False, + include_fuse: Optional[bool] = False, + include_overlay: Optional[bool] = False, variant: Optional[str] = "original", ): """Downloads the image (and annotation if not None) to local_dir_path @@ -1659,7 +1658,7 @@ def upload_annotations_from_folder_to_project( project: Union[NotEmptyStr, dict], folder_path: Union[str, Path], from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + recursive_subfolders: Optional[bool] = False, keep_status=False, ): """Finds and uploads all JSON files in the folder_path as annotations to the project. @@ -1733,7 +1732,7 @@ def upload_image_annotations( image_name: str, annotation_json: Union[str, Path, dict], mask: Optional[Union[str, Path, bytes]] = None, - verbose: Optional[StrictBool] = True, + verbose: Optional[bool] = True, keep_status: bool = False, ): """Upload annotations from JSON (also mask for pixel annotations) @@ -1948,7 +1947,7 @@ def search_models( type_: Optional[NotEmptyStr] = None, # noqa project_id: Optional[int] = None, task: Optional[NotEmptyStr] = None, - include_global: Optional[StrictBool] = True, + include_global: Optional[bool] = True, ): r"""Search for ML models. @@ -2164,7 +2163,7 @@ def add_contributors_to_project( return response.data def invite_contributors_to_team( - self, emails: conlist(EmailStr, min_items=1), admin: StrictBool = False + self, emails: conlist(EmailStr, min_items=1), admin: bool = False ) -> Tuple[List[str], List[str]]: """Invites contributors to the team. @@ -2575,7 +2574,7 @@ def copy_items( source: Union[NotEmptyStr, dict], destination: Union[NotEmptyStr, dict], items: Optional[List[NotEmptyStr]] = None, - include_annotations: Optional[StrictBool] = True, + include_annotations: Optional[bool] = True, ): """Copy images in bulk between folders in a project diff --git a/src/superannotate/lib/core/entities/folder.py b/src/superannotate/lib/core/entities/folder.py index 7acde8065..a1c0ebe3c 100644 --- a/src/superannotate/lib/core/entities/folder.py +++ b/src/superannotate/lib/core/entities/folder.py @@ -11,11 +11,10 @@ class FolderEntity(TimedBaseModel): name: Optional[str] status: Optional[FolderStatus] project_id: Optional[int] - parent_id: Optional[int] team_id: Optional[int] is_root: Optional[bool] = (False,) folder_users: Optional[List[dict]] completedCount: Optional[int] class Config: - extra = Extra.allow + extra = Extra.ignore diff --git a/src/superannotate/lib/core/entities/project.py b/src/superannotate/lib/core/entities/project.py index eb4556c87..60a646b8b 100644 --- a/src/superannotate/lib/core/entities/project.py +++ b/src/superannotate/lib/core/entities/project.py @@ -185,3 +185,6 @@ class TeamEntity(BaseModel): users: Optional[List[UserEntity]] pending_invitations: Optional[List[Any]] creator_id: Optional[str] + + class Config: + extra = Extra.ignore diff --git a/tests/integration/folders/__init__.py b/tests/integration/folders/__init__.py index e69de29bb..a70688ea7 100644 --- a/tests/integration/folders/__init__.py +++ b/tests/integration/folders/__init__.py @@ -0,0 +1,10 @@ +FOLDER_KEYS = [ + "createdAt", + "updatedAt", + "id", + "name", + "status", + "project_id", + "team_id", + "folder_users", +] diff --git a/tests/integration/folders/test_create_folder.py b/tests/integration/folders/test_create_folder.py new file mode 100644 index 000000000..bc7cd1b7f --- /dev/null +++ b/tests/integration/folders/test_create_folder.py @@ -0,0 +1,31 @@ +from src.superannotate import AppException +from src.superannotate import SAClient +from tests.integration.base import BaseTestCase + +sa = SAClient() + + +class TestCreateFolder(BaseTestCase): + PROJECT_NAME = "test TestCreateFolder" + PROJECT_DESCRIPTION = "desc" + PROJECT_TYPE = "Vector" + SPECIAL_CHARS = r"/\:*?“<>|" + TEST_FOLDER_NAME = "folder_" + + def test_create_long_name(self): + err_msg = "The folder name is too long. The maximum length for this field is 80 characters." + with self.assertRaisesRegexp(AppException, err_msg): + sa.create_folder( + self.PROJECT_NAME, + "A while back I needed to count the amount of letters that " + "a piece of text in an email template had (to avoid passing any)", + ) + + def test_create_folder_with_special_chars(self): + sa.create_folder(self.PROJECT_NAME, self.SPECIAL_CHARS) + folder = sa.get_folder_metadata( + self.PROJECT_NAME, "_" * len(self.SPECIAL_CHARS) + ) + self.assertIsNotNone(folder) + assert "completedCount" not in folder.keys() + assert "is_root" not in folder.keys() diff --git a/tests/integration/folders/test_delete_folders.py b/tests/integration/folders/test_delete_folders.py new file mode 100644 index 000000000..ab76b01cd --- /dev/null +++ b/tests/integration/folders/test_delete_folders.py @@ -0,0 +1,44 @@ +from src.superannotate import AppException +from src.superannotate import SAClient +from tests.integration.base import BaseTestCase + + +sa = SAClient() + + +class TestDeleteFolders(BaseTestCase): + PROJECT_NAME = "test TestDeleteFolders" + PROJECT_DESCRIPTION = "desc" + PROJECT_TYPE = "Vector" + SPECIAL_CHARS = r"/\:*?“<>|" + TEST_FOLDER_NAME_1 = "folder_1" + TEST_FOLDER_NAME_2 = "folder_2" + + def test_search_folders(self): + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_2) + sa.delete_folders(self.PROJECT_NAME, folder_names=[self.TEST_FOLDER_NAME_1]) + folders = sa.search_folders(self.PROJECT_NAME) + assert len(folders) == 1 + + sa.delete_folders(self.PROJECT_NAME, folder_names=[self.TEST_FOLDER_NAME_2]) + folders = sa.search_folders(self.PROJECT_NAME) + assert len(folders) == 0 + + # test delete multiple + folder_names = [f"folder_{i}" for i in range(5)] + [ + sa.create_folder(self.PROJECT_NAME, folder_name) + for folder_name in folder_names + ] + + with self.assertRaisesRegexp(AppException, "There is no folder to delete."): + sa.delete_folders(self.PROJECT_NAME, []) + pattern = r"(\s+)folder_names(\s+)none is not an allowed value" + + with self.assertRaisesRegexp(AppException, pattern): + sa.delete_folders(self.PROJECT_NAME, None) # noqa + + sa.delete_folders(self.PROJECT_NAME, folder_names) + folders = sa.search_folders(self.PROJECT_NAME) + assert len(folders) == 0 diff --git a/tests/integration/folders/test_folders.py b/tests/integration/folders/test_folders.py deleted file mode 100644 index 345397a23..000000000 --- a/tests/integration/folders/test_folders.py +++ /dev/null @@ -1,285 +0,0 @@ -import os -import time - -import pytest -from src.superannotate import AppException -from src.superannotate import SAClient -from tests import DATA_SET_PATH -from tests.integration.base import BaseTestCase - -sa = SAClient() - - -class TestFolders(BaseTestCase): - PROJECT_NAME = "test folders" - TEST_FOLDER_PATH = "sample_project_vector" - PROJECT_DESCRIPTION = "desc" - PROJECT_TYPE = "Vector" - SPECIAL_CHARS = r"/\:*?“<>|" - TEST_FOLDER_NAME_1 = "folder_1" - TEST_FOLDER_NAME_2 = "folder_2" - EXAMPLE_IMAGE_1_NAME = "example_image_1" - EXAMPLE_IMAGE_2_NAME = "example_image_2" - EXAMPLE_IMAGE_1 = "example_image_1.jpg" - EXAMPLE_IMAGE_2 = "example_image_2.jpg" - EXAMPLE_IMAGE_3 = "example_image_3.jpg" - EXAMPLE_IMAGE_4 = "example_image_4.jpg" - - @property - def folder_path(self): - return os.path.join(DATA_SET_PATH, self.TEST_FOLDER_PATH) - - @property - def classes_json(self): - return f"{self.folder_path}/classes/classes.json" - - def test_get_folder_metadata(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - folder_metadata = sa.get_folder_metadata( - self.PROJECT_NAME, self.TEST_FOLDER_NAME_1 - ) - assert "is_root" not in folder_metadata - - def test_search_folders(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_2) - folders = sa.search_folders(self.PROJECT_NAME, return_metadata=True) - folders = sa.search_folders(self.PROJECT_NAME) - assert all(["is_root" not in folder for folder in folders]) - - def test_basic_folders(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) - images = sa.search_items(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1) - self.assertEqual(len(images), 1) - - folders = sa.search_folders(self.PROJECT_NAME) - self.assertEqual(len(folders), 0) - - folder_metadata = sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - self.assertEqual(folder_metadata["name"], self.TEST_FOLDER_NAME_1) - - folders = sa.search_folders(self.PROJECT_NAME, return_metadata=True) - self.assertEqual(len(folders), 1) - - self.assertEqual(folders[0]["name"], self.TEST_FOLDER_NAME_1) - - folders = sa.search_folders(self.PROJECT_NAME) - self.assertEqual(len(folders), 1) - - self.assertEqual(folders[0], self.TEST_FOLDER_NAME_1) - - images = sa.search_items( - self.PROJECT_NAME + f"/{self.TEST_FOLDER_NAME_1}", self.EXAMPLE_IMAGE_1 - ) - self.assertEqual(len(images), 0) - - images = sa.search_items(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1) - self.assertEqual(len(images), 1) - - folder = sa.get_folder_metadata(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - self.assertIsInstance(folder, dict) - self.assertEqual(folder["name"], self.TEST_FOLDER_NAME_1) - with self.assertRaisesRegexp(Exception, "Folder not found"): - sa.get_folder_metadata(self.PROJECT_NAME, self.TEST_FOLDER_NAME_2) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", - self.folder_path, - annotation_status="InProgress", - ) - images = sa.search_items( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", self.EXAMPLE_IMAGE_1 - ) - self.assertEqual(len(images), 1) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", - self.folder_path, - annotation_status="InProgress", - ) - images = sa.search_items(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}") - self.assertEqual(len(images), 4) - - folder_metadata = sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_2) - self.assertEqual(folder_metadata["name"], self.TEST_FOLDER_NAME_2) - - folders = sa.search_folders(self.PROJECT_NAME) - self.assertEqual(len(folders), 2) - - folders = sa.search_folders(self.PROJECT_NAME, folder_name="folder") - self.assertEqual(len(folders), 2) - - folders = sa.search_folders( - self.PROJECT_NAME, folder_name=self.TEST_FOLDER_NAME_2 - ) - self.assertEqual(len(folders), 1) - self.assertEqual(folders[0], self.TEST_FOLDER_NAME_2) - - folders = sa.search_folders( - self.PROJECT_NAME, folder_name=self.TEST_FOLDER_NAME_1 - ) - self.assertEqual(len(folders), 1) - self.assertEqual(folders[0], self.TEST_FOLDER_NAME_1) - - folders = sa.search_folders(self.PROJECT_NAME, folder_name="old") - self.assertEqual(len(folders), 2) - - def test_folder_annotations(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) - sa.create_annotation_classes_from_classes_json( - self.PROJECT_NAME, self.classes_json - ) - folder_metadata = sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - self.assertEqual(folder_metadata["name"], self.TEST_FOLDER_NAME_1) - folders = sa.search_folders(self.PROJECT_NAME, return_metadata=True) - self.assertEqual(len(folders), 1) - - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME + "/" + folders[0]["name"], - self.folder_path, - annotation_status="InProgress", - ) - sa.upload_annotations_from_folder_to_project( - self.PROJECT_NAME + "/" + folders[0]["name"], self.folder_path - ) - annotations = sa.get_annotations( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", - [self.EXAMPLE_IMAGE_1], - ) - self.assertGreater(len(annotations[0]["instances"]), 0) - - def test_delete_folders(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_2) - - self.assertEqual(len(sa.search_folders(self.PROJECT_NAME)), 2) - - sa.delete_folders(self.PROJECT_NAME, [self.TEST_FOLDER_NAME_1]) - self.assertEqual(len(sa.search_folders(self.PROJECT_NAME)), 1) - sa.delete_folders(self.PROJECT_NAME, [self.TEST_FOLDER_NAME_2]) - self.assertEqual(len(sa.search_folders(self.PROJECT_NAME)), 0) - sa.create_folder(self.PROJECT_NAME, "folder5") - sa.create_folder(self.PROJECT_NAME, "folder6") - self.assertEqual(len(sa.search_folders(self.PROJECT_NAME)), 2) - - sa.delete_folders(self.PROJECT_NAME, ["folder2", "folder5"]) - self.assertEqual(len(sa.search_folders(self.PROJECT_NAME)), 1) - self.assertEqual(sa.search_folders(self.PROJECT_NAME)[0], "folder6") - - def test_project_folder_image_count(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) - - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME + f"/{self.TEST_FOLDER_NAME_1}", - self.folder_path, - annotation_status="InProgress", - ) - num_images = sa.search_items(self.PROJECT_NAME) - assert len(num_images) == 4 - - num_images = sa.search_items(f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}") - assert len(num_images) == 4 - - def test_delete_items(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", - self.folder_path, - annotation_status="InProgress", - ) - num_images = sa.search_items(self.PROJECT_NAME) - assert len(num_images) == 4 - - sa.delete_items( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}", - [self.EXAMPLE_IMAGE_2, self.EXAMPLE_IMAGE_3], - ) - num_images = sa.search_items(self.PROJECT_NAME) - assert len(num_images) == 2 - - sa.delete_items(self.PROJECT_NAME, None) - - num_images = sa.search_items(self.PROJECT_NAME) - assert len(num_images) == 0 - - @pytest.mark.flaky(reruns=2) - def test_project_completed_count(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="Completed" - ) - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - project = f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME_1}" - sa.upload_images_from_folder_to_project( - project, self.folder_path, annotation_status="Completed" - ) - project_metadata = sa.get_project_metadata( - self.PROJECT_NAME, include_complete_item_count=True - ) - self.assertEqual(project_metadata["completed_items_count"], 8) - self.assertEqual(project_metadata["root_folder_completed_items_count"], 4) - - def test_folder_misnamed(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - self.assertTrue(self.TEST_FOLDER_NAME_1 in sa.search_folders(self.PROJECT_NAME)) - - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - self.assertTrue( - f"{self.TEST_FOLDER_NAME_1} (1)" in sa.search_folders(self.PROJECT_NAME) - ) - - sa.create_folder(self.PROJECT_NAME, f"{self.TEST_FOLDER_NAME_2}\\") - self.assertTrue( - f"{self.TEST_FOLDER_NAME_2}_" in sa.search_folders(self.PROJECT_NAME) - ) - - def test_create_folder_with_special_chars(self): - sa.create_folder(self.PROJECT_NAME, self.SPECIAL_CHARS) - folder = sa.get_folder_metadata( - self.PROJECT_NAME, "_" * len(self.SPECIAL_CHARS) - ) - self.assertIsNotNone(folder) - assert "completedCount" not in folder.keys() - assert "is_root" not in folder.keys() - - def test_create_long_name(self): - err_msg = "The folder name is too long. The maximum length for this field is 80 characters." - with self.assertRaisesRegexp(AppException, err_msg): - sa.create_folder( - self.PROJECT_NAME, - "A while back I needed to count the amount of letters that " - "a piece of text in an email template had (to avoid passing any)", - ) - - def test_search_folder(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) - time.sleep(1) - folders = sa.search_folders( - self.PROJECT_NAME, self.TEST_FOLDER_NAME_1, return_metadata=True - ) - assert len(folders) == 1 - assert folders[0]["name"] == self.TEST_FOLDER_NAME_1 - assert folders[0]["status"] == "NotStarted" - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1 + "1") - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1 + "2") - folders = sa.search_folders(self.PROJECT_NAME, return_metadata=True) - assert len(folders) == 3 - folders = sa.search_folders( - self.PROJECT_NAME, status="Completed", return_metadata=True - ) - assert len(folders) == 0 - folders = sa.search_folders( - self.PROJECT_NAME, status="OnHold", return_metadata=True - ) - assert len(folders) == 0 - folders = sa.search_folders( - self.PROJECT_NAME, status="NotStarted", return_metadata=True - ) - assert len(folders) == 3 diff --git a/tests/integration/folders/test_get_folder_metadata.py b/tests/integration/folders/test_get_folder_metadata.py new file mode 100644 index 000000000..08c683029 --- /dev/null +++ b/tests/integration/folders/test_get_folder_metadata.py @@ -0,0 +1,27 @@ +from src.superannotate import AppException +from src.superannotate import SAClient +from tests.integration.base import BaseTestCase +from tests.integration.folders import FOLDER_KEYS + +sa = SAClient() + + +class TestGetFolderMetadata(BaseTestCase): + PROJECT_NAME = "test TestGetFolderMetadata" + PROJECT_DESCRIPTION = "desc" + PROJECT_TYPE = "Vector" + SPECIAL_CHARS = r"/\:*?“<>|" + TEST_FOLDER_NAME = "folder_" + + def test_get_folder_metadata(self): + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) + folder_metadata = sa.get_folder_metadata( + self.PROJECT_NAME, self.TEST_FOLDER_NAME + ) + assert "is_root" not in folder_metadata + self.assertListEqual(list(folder_metadata.keys()), FOLDER_KEYS) + + # get not exiting folder + with self.assertRaises(AppException) as cm: + sa.get_folder_metadata(self.PROJECT_NAME, "dummy folder") + assert str(cm.exception) == "Folder not found." diff --git a/tests/integration/folders/test_search_folders.py b/tests/integration/folders/test_search_folders.py new file mode 100644 index 000000000..9701a5cee --- /dev/null +++ b/tests/integration/folders/test_search_folders.py @@ -0,0 +1,38 @@ +from src.superannotate import AppException +from src.superannotate import SAClient +from tests.integration.base import BaseTestCase +from tests.integration.folders import FOLDER_KEYS + +sa = SAClient() + + +class TestSearchFolders(BaseTestCase): + PROJECT_NAME = "test TestSearchFolders" + PROJECT_DESCRIPTION = "desc" + PROJECT_TYPE = "Vector" + SPECIAL_CHARS = r"/\:*?“<>|" + TEST_FOLDER_NAME_1 = "folder_1" + TEST_FOLDER_NAME_2 = "folder_2" + + def test_search_folders(self): + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_2) + folders = sa.search_folders(self.PROJECT_NAME) + assert all(["is_root" not in folder for folder in folders]) + assert len(folders) == 2 + # with metadata + folders = sa.search_folders(self.PROJECT_NAME, return_metadata=True) + for folder in folders: + self.assertListEqual(list(folder.keys()), FOLDER_KEYS) + + # with status + folders = sa.search_folders(self.PROJECT_NAME, status="NotStarted") + assert len(folders) == 2 + + # with invalid status + pattern = ( + r"(\s+)status(\s+)Available values are 'NotStarted', " + r"'InProgress', 'Completed', 'OnHold'.(\s+)value is not a valid list" + ) + with self.assertRaisesRegexp(AppException, pattern): + folders = sa.search_folders(self.PROJECT_NAME, status="dummy") # noqa diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index e6b6f186e..e69de29bb 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1 +0,0 @@ -from superannotate.lib.infrastructure.validators import validators \ No newline at end of file diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 5e51081b6..b53fd1ed1 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -95,7 +95,9 @@ def test_init_via_config_ini(self, get_team_use_case, get_current_user): @patch("lib.infrastructure.controller.Controller.get_current_user") @patch("lib.core.usecases.GetTeamUseCase") - def test_init_via_config_relative_filepath(self, get_team_use_case, get_current_user): + def test_init_via_config_relative_filepath( + self, get_team_use_case, get_current_user + ): with tempfile.TemporaryDirectory(dir=Path("~").expanduser()) as config_dir: config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" From 9a8ff00b065812f8dc7168e74075a3a280cc8a2c Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:53:12 +0400 Subject: [PATCH 17/17] Update __init__.py --- src/superannotate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 5ca137a9e..ab2169765 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys import typing -__version__ = "4.4.12dev3" +__version__ = "4.4.12dev4" sys.path.append(os.path.split(os.path.realpath(__file__))[0])