From 1a241c8eb5e558315ad7e174c7a05dbea7f3a7cf Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 16:44:54 +0400 Subject: [PATCH 01/50] initial priority --- src/superannotate/__init__.py | 4 ++ .../lib/app/interface/sdk_interface.py | 27 ++++++++++ .../lib/app/mixp/utils/parsers.py | 8 +++ .../lib/core/serviceproviders.py | 6 +++ src/superannotate/lib/core/types.py | 5 ++ .../lib/core/usecases/annotations.py | 50 +++++++++++++++++++ .../lib/infrastructure/controller.py | 12 +++++ .../lib/infrastructure/services.py | 15 +++++- .../test_upload_priority_scores.py | 34 +++++++++++++ 9 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_upload_priority_scores.py diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index afb4e8cc5..a8517b7e5 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -99,6 +99,9 @@ from superannotate.lib.app.interface.sdk_interface import ( upload_images_from_public_urls_to_project, ) +from superannotate.lib.app.interface.sdk_interface import ( + upload_priority_scores, +) from superannotate.lib.app.interface.sdk_interface import upload_images_to_project from superannotate.lib.app.interface.sdk_interface import ( upload_preannotations_from_folder_to_project, @@ -151,6 +154,7 @@ "clone_project", "share_project", "delete_project", + "upload_priority_scores", # Images Section "search_images", "copy_image", diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 472a3b47a..e0673a6f9 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -40,6 +40,7 @@ from lib.core.types import AttributeGroup from lib.core.types import MLModel from lib.core.types import Project +from lib.core.types import PriorityScore from lib.infrastructure.controller import Controller from pydantic import conlist from pydantic import parse_obj_as @@ -2893,3 +2894,29 @@ def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int if response.errors: raise AppException(response.errors) return response.data + +@Trackable +@validate_arguments +def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): + """Returns per frame annotations for the given video. + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str + + :param scores: list of score objects + :type scores: list of dicts + + :return: lists of uploaded, skipped items + :rtype: tuple (2 members) of lists of strs + """ + project_name, folder_name = extract_project_folder(project) + project_folder_name = project + use_case = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) + with tqdm( + total=len(scores), desc="Uploading priority scores" + ) as progress_bar: + for _ in use_case.execute(): + progress_bar.update() + progress_bar.close() + if use_case.response.errors: + raise AppException(use_case.errors) + return use_case.response.data \ No newline at end of file diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index e93011cc2..d954379c8 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1227,3 +1227,11 @@ def get_annotations_per_frame(*args, **kwargs): "event_name": "get_annotations_per_frame", "properties": {"Project": project, "fps": fps}, } + + +def upload_priority_scores(*args, **kwargs): + scores = kwargs.get("scores", args[1]) + return { + "event_name": "get_annotations_per_frame", + "properties": {"Score Count": len(scores)}, + } \ No newline at end of file diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index f85cb3979..69ab0e751 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -314,3 +314,9 @@ def get_annotations( reporter: Reporter ) -> List[dict]: raise NotImplementedError + + def upload_priority_scores( + self, team_id: int, project_id: int, folder_id: int, priorities: list = [] + ) -> ServiceResponse: + raise NotImplementedError + diff --git a/src/superannotate/lib/core/types.py b/src/superannotate/lib/core/types.py index a5236d01b..31aa7d746 100644 --- a/src/superannotate/lib/core/types.py +++ b/src/superannotate/lib/core/types.py @@ -40,3 +40,8 @@ class MLModel(BaseModel): class Config: extra = Extra.allow + + +class PriorityScore(BaseModel): + name: NotEmptyStr + priority: float diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 3bcf1eb39..c11c80796 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -24,6 +24,7 @@ from lib.core.exceptions import AppException from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository +from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import SuerannotateServiceProvider from lib.core.usecases.base import BaseReportableUseCae @@ -32,6 +33,7 @@ from lib.core.video_convertor import VideoFrameGenerator from superannotate.logger import get_default_logger from superannotate_schemas.validators import AnnotationValidators +from lib.core.types import PriorityScore logger = get_default_logger() @@ -593,3 +595,51 @@ def execute(self): else: self._response.data = [] return self._response + + +class UploadPriorityScoresUseCase(BaseInteractiveUseCase): + + CHUNK_SIZE = 100 + + def __init__( + self, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScore], + project_folder_name: str, + backend_service_provider: SuerannotateServiceProvider + ): + super().__init__() + self._project = project + self._folder = folder + self._scores = scores + self._client = backend_service_provider + self._project_folder_name = project_folder_name + + def execute(self): + logger.info(f"Uploading priority scores for {len(self._scores)} item(s) from {self._project_folder_name}.") + priorities = [] + to_send = [] + for i in self._scores: + priorities.append({ + "name": i.name, + "entropy_value": i.priority + }) + to_send.append(i.name) + + uploaded = [] + for i in range(0, len(priorities), self.CHUNK_SIZE): + res = self._client.upload_priority_scores( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._folder.uuid, + priorities=priorities[i : i + self.CHUNK_SIZE], # noqa: E203 + ) + uploaded += res["data"] + yield + + uploaded = [i["name"] for i in uploaded] + skipped = list(set(to_send) - set(uploaded)) + self._response.data = (uploaded, skipped) + return self._response + diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index d637e839f..e73b89cad 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1647,3 +1647,15 @@ def get_annotations_per_frame(self, project_name: str, folder_name: str, video_n backend_service_provider=self.backend_client ) return use_case.execute() + + def upload_priority_scores(self, project_name, folder_name, scores, project_folder_name): + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + use_case = usecases.UploadPriorityScoresUseCase( + project=project, + folder=folder, + scores=scores, + backend_service_provider=self.backend_client, + project_folder_name=project_folder_name + ) + return use_case diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index b1eecc2b7..c6e880c91 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -50,7 +50,7 @@ def __init__( self._auth_token = auth_token self.logger = logger self._paginate_by = paginate_by - self._verify_ssl = verify_ssl + self._verify_ssl = False self.team_id = auth_token.split("=")[-1] self._testing = testing self.get_session() @@ -224,6 +224,19 @@ class SuperannotateBackendService(BaseBackendService): URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus" URL_GET_LIMITS = "project/{}/limitationDetails" URL_GET_ANNOTATIONS = "images/annotations/stream" + URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" + + def upload_priority_scores( + self, team_id: int, project_id: int, folder_id: int, priorities: list = [] + ) -> ServiceResponse: + upload_priority_score_url = urljoin(self.api_url, self.URL_UPLOAD_PRIORITY_SCORES) + res = self._request( + upload_priority_score_url, + "post", + params={"team_id": team_id, "project_id": project_id, "folder_id": folder_id}, + data={"image_entropies": priorities} + ) + return res.json() def get_project(self, uuid: int, team_id: int): get_project_url = urljoin(self.api_url, self.URL_GET_PROJECT.format(uuid)) diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py new file mode 100644 index 000000000..e19992ec8 --- /dev/null +++ b/tests/integration/test_upload_priority_scores.py @@ -0,0 +1,34 @@ +import os +import src.superannotate as sa +from tests.integration.base import BaseTestCase +from pathlib import Path + +class TestUploadPriorityScores(BaseTestCase): + PROJECT_NAME = "TestUploadPriorityScores" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PATH = "data_set/sample_project_vector" + + @property + def folder_path(self): + return os.path.join(Path(__file__).parent.parent, self.TEST_FOLDER_PATH) + + def test_upload_priority_scores(self): + sa.upload_images_from_folder_to_project( + self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" + ) + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + "name": "example_image_1.jpg", + "priority": 1 + }]) + self.assertEqual(len(uploaded), 1) + self.assertEqual(len(skipped), 0) + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + "name": "non-exist.jpg", + "priority": 1 + }, { + "name": "non-exist-2.jpg", + "priority": 1 + }]) + self.assertEqual(len(uploaded), 0) + self.assertEqual(len(skipped), 2) From b6b08b3e3318259602a45d4147e39f71adca9fde Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 16:58:05 +0400 Subject: [PATCH 02/50] fix parser --- src/superannotate/lib/app/mixp/utils/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index d954379c8..b742f3c1e 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1232,6 +1232,6 @@ def get_annotations_per_frame(*args, **kwargs): def upload_priority_scores(*args, **kwargs): scores = kwargs.get("scores", args[1]) return { - "event_name": "get_annotations_per_frame", + "event_name": "upload_priority_scores", "properties": {"Score Count": len(scores)}, } \ No newline at end of file From 10df0dde8127c5e43272b8509b129ea136434529 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 17:22:25 +0400 Subject: [PATCH 03/50] add tqdm --- .../lib/app/interface/sdk_interface.py | 7 ++++--- .../lib/core/usecases/annotations.py | 15 +++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index e0673a6f9..70dd4c0fd 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2911,12 +2911,13 @@ def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): project_name, folder_name = extract_project_folder(project) project_folder_name = project use_case = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) + logger.info(f"Uploading priority scores for {len(scores)} item(s) from {project_folder_name}.") with tqdm( total=len(scores), desc="Uploading priority scores" ) as progress_bar: - for _ in use_case.execute(): - progress_bar.update() + for uploaded_count in use_case.execute(): + progress_bar.update(uploaded_count) progress_bar.close() if use_case.response.errors: raise AppException(use_case.errors) - return use_case.response.data \ No newline at end of file + return use_case.response.data diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index c11c80796..57f7c1ce2 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -602,12 +602,12 @@ class UploadPriorityScoresUseCase(BaseInteractiveUseCase): CHUNK_SIZE = 100 def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScore], - project_folder_name: str, - backend_service_provider: SuerannotateServiceProvider + self, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScore], + project_folder_name: str, + backend_service_provider: SuerannotateServiceProvider ): super().__init__() self._project = project @@ -617,7 +617,6 @@ def __init__( self._project_folder_name = project_folder_name def execute(self): - logger.info(f"Uploading priority scores for {len(self._scores)} item(s) from {self._project_folder_name}.") priorities = [] to_send = [] for i in self._scores: @@ -636,7 +635,7 @@ def execute(self): priorities=priorities[i : i + self.CHUNK_SIZE], # noqa: E203 ) uploaded += res["data"] - yield + yield len(to_send[:i + self.CHUNK_SIZE]) uploaded = [i["name"] for i in uploaded] skipped = list(set(to_send) - set(uploaded)) From ce3d1da2f84e1068a6c95b6f10ae8e1c38f6ad23 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 17:31:04 +0400 Subject: [PATCH 04/50] Update docs --- docs/source/superannotate.sdk.rst | 1 + src/superannotate/lib/app/interface/sdk_interface.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 099fb3a8e..103be461b 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -92,6 +92,7 @@ ______ .. autofunction:: superannotate.add_annotation_bbox_to_image .. autofunction:: superannotate.add_annotation_point_to_image .. autofunction:: superannotate.add_annotation_comment_to_image +.. autofunction:: superannotate.upload_priority_scores ---------- diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 70dd4c0fd..a9ed40a02 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2898,7 +2898,9 @@ def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int @Trackable @validate_arguments def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): - """Returns per frame annotations for the given video. + """ + Returns per frame annotations for the given video. + :param project: project name or folder path (e.g., “project1/folder1”) :type project: str From 88cb53c3589c552caa1ee4610a425748bc8e62d8 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 17:34:07 +0400 Subject: [PATCH 05/50] Update ssl --- src/superannotate/lib/infrastructure/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index c6e880c91..026bacb3b 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -50,7 +50,7 @@ def __init__( self._auth_token = auth_token self.logger = logger self._paginate_by = paginate_by - self._verify_ssl = False + self._verify_ssl = verify_ssl self.team_id = auth_token.split("=")[-1] self._testing = testing self.get_session() From 52de4684018b23066ba42ac4e99ea8f551301d95 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 18:16:09 +0400 Subject: [PATCH 06/50] Add clean priority --- .../lib/core/usecases/annotations.py | 15 ++++++++++++++- tests/integration/test_upload_priority_scores.py | 9 +++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 57f7c1ce2..ec251453c 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -616,13 +616,26 @@ def __init__( self._client = backend_service_provider self._project_folder_name = project_folder_name + @staticmethod + def get_clean_priority(priority): + if len(str(priority)) > 8: + priority = float(str(priority)[:8]) + if priority > 1000000: + priority = 1000000 + if priority < 0: + priority = 0 + if str(float(priority)).split('.')[1:2]: + if len(str(float(priority)).split('.')[1]) > 5: + priority = float(str(float(priority)).split('.')[0] + '.' + str(float(priority)).split('.')[1][:5]) + return priority + def execute(self): priorities = [] to_send = [] for i in self._scores: priorities.append({ "name": i.name, - "entropy_value": i.priority + "entropy_value": self.get_clean_priority(i.priority) }) to_send.append(i.name) diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index e19992ec8..d98e2a265 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -32,3 +32,12 @@ def test_upload_priority_scores(self): }]) self.assertEqual(len(uploaded), 0) self.assertEqual(len(skipped), 2) + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + "name": "example_image_3.jpg", + "priority": 1.1234567890 + }, { + "name": "example_image_4.jpg", + "priority": 100000000 + }]) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_4.jpg")['entropy_value'], 1000000) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_3.jpg")['entropy_value'], 1.12345) From 8d2e2362659e30a132403cd6bae5b6d454f57f8e Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Sun, 27 Feb 2022 11:55:51 +0400 Subject: [PATCH 07/50] round frames count in VideoFrameGenerator --- src/superannotate/lib/core/video_convertor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index ffb92702d..0f77f38dc 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -42,7 +42,7 @@ def __init__(self, annotation_data: dict, fps: int): self.fps = fps self.ratio = 1000 * 1000 / fps self._frame_id = 1 - self.frames_count = self.duration * fps + self.frames_count = int(self.duration * fps) self.annotations: dict = {} self._mapping = {} self._process() From 7dd287324314e11d2d1561319082176584778389 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 2 Mar 2022 12:38:00 +0400 Subject: [PATCH 08/50] refactor upload_priority_scores --- src/superannotate/__init__.py | 6 +- .../lib/app/interface/sdk_interface.py | 21 ++---- .../lib/app/mixp/utils/parsers.py | 2 +- .../lib/core/serviceproviders.py | 5 +- .../lib/core/usecases/annotations.py | 65 +++++++++++-------- .../lib/infrastructure/controller.py | 4 +- .../lib/infrastructure/services.py | 4 +- .../test_upload_priority_scores.py | 10 +-- 8 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index a8517b7e5..3cdb13ccc 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -99,13 +99,13 @@ from superannotate.lib.app.interface.sdk_interface import ( upload_images_from_public_urls_to_project, ) -from superannotate.lib.app.interface.sdk_interface import ( - upload_priority_scores, -) from superannotate.lib.app.interface.sdk_interface import upload_images_to_project from superannotate.lib.app.interface.sdk_interface import ( upload_preannotations_from_folder_to_project, ) +from superannotate.lib.app.interface.sdk_interface import ( + upload_priority_scores, +) from superannotate.lib.app.interface.sdk_interface import upload_video_to_project from superannotate.lib.app.interface.sdk_interface import ( upload_videos_from_folder_to_project, diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index a9ed40a02..71de74d87 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -39,8 +39,8 @@ from lib.core.exceptions import AppException from lib.core.types import AttributeGroup from lib.core.types import MLModel -from lib.core.types import Project from lib.core.types import PriorityScore +from lib.core.types import Project from lib.infrastructure.controller import Controller from pydantic import conlist from pydantic import parse_obj_as @@ -2895,11 +2895,11 @@ def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int raise AppException(response.errors) return response.data + @Trackable @validate_arguments def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): - """ - Returns per frame annotations for the given video. + """Returns per frame annotations for the given video. :param project: project name or folder path (e.g., “project1/folder1”) :type project: str @@ -2912,14 +2912,7 @@ def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): """ project_name, folder_name = extract_project_folder(project) project_folder_name = project - use_case = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) - logger.info(f"Uploading priority scores for {len(scores)} item(s) from {project_folder_name}.") - with tqdm( - total=len(scores), desc="Uploading priority scores" - ) as progress_bar: - for uploaded_count in use_case.execute(): - progress_bar.update(uploaded_count) - progress_bar.close() - if use_case.response.errors: - raise AppException(use_case.errors) - return use_case.response.data + response = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) + if response.errors: + raise AppException(response.errors) + return response.data diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index b742f3c1e..da4f789b0 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1234,4 +1234,4 @@ def upload_priority_scores(*args, **kwargs): return { "event_name": "upload_priority_scores", "properties": {"Score Count": len(scores)}, - } \ No newline at end of file + } diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 69ab0e751..0a6f294c0 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -316,7 +316,6 @@ def get_annotations( raise NotImplementedError def upload_priority_scores( - self, team_id: int, project_id: int, folder_id: int, priorities: list = [] - ) -> ServiceResponse: + self, team_id: int, project_id: int, folder_id: int, priorities: list + ) -> dict: raise NotImplementedError - diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index ec251453c..0f44cb587 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -24,16 +24,16 @@ from lib.core.exceptions import AppException from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository -from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.types import PriorityScore from lib.core.usecases.base import BaseReportableUseCae from lib.core.usecases.images import GetBulkImages from lib.core.usecases.images import ValidateAnnotationUseCase from lib.core.video_convertor import VideoFrameGenerator from superannotate.logger import get_default_logger from superannotate_schemas.validators import AnnotationValidators -from lib.core.types import PriorityScore + logger = get_default_logger() @@ -597,19 +597,20 @@ def execute(self): return self._response -class UploadPriorityScoresUseCase(BaseInteractiveUseCase): +class UploadPriorityScoresUseCase(BaseReportableUseCae): CHUNK_SIZE = 100 def __init__( self, + reporter, project: ProjectEntity, folder: FolderEntity, scores: List[PriorityScore], project_folder_name: str, backend_service_provider: SuerannotateServiceProvider ): - super().__init__() + super().__init__(reporter) self._project = project self._folder = folder self._scores = scores @@ -629,29 +630,37 @@ def get_clean_priority(priority): priority = float(str(float(priority)).split('.')[0] + '.' + str(float(priority)).split('.')[1][:5]) return priority - def execute(self): - priorities = [] - to_send = [] - for i in self._scores: - priorities.append({ - "name": i.name, - "entropy_value": self.get_clean_priority(i.priority) - }) - to_send.append(i.name) - - uploaded = [] - for i in range(0, len(priorities), self.CHUNK_SIZE): - res = self._client.upload_priority_scores( - team_id=self._project.team_id, - project_id=self._project.uuid, - folder_id=self._folder.uuid, - priorities=priorities[i : i + self.CHUNK_SIZE], # noqa: E203 - ) - uploaded += res["data"] - yield len(to_send[:i + self.CHUNK_SIZE]) + @property + def folder_path(self): + return f"{self._project.name}{f'/{self._folder.name}'if self._folder.name != 'root' else ''}" - uploaded = [i["name"] for i in uploaded] - skipped = list(set(to_send) - set(uploaded)) - self._response.data = (uploaded, skipped) + def execute(self): + if self.is_valid(): + priorities = [] + initial_scores = [] + for i in self._scores: + priorities.append({ + "name": i.name, + "entropy_value": self.get_clean_priority(i.priority) + }) + initial_scores.append(i.name) + uploaded_score_names = [] + self.reporter.log_info(f"Uploading priority scores for {len(priorities)} item(s) from {self.folder_path}.") + iterations = range(0, len(priorities), self.CHUNK_SIZE) + self.reporter.start_progress(iterations, "Uploading priority scores") + if iterations: + for i in iterations: + priorities_to_upload = priorities[i : i + self.CHUNK_SIZE] # noqa: E203 + res = self._client.upload_priority_scores( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._folder.uuid, + priorities=priorities_to_upload + ) + self.reporter.update_progress(len(priorities_to_upload)) + uploaded_score_names.extend(list(map(lambda x: x["name"], res.get("data", [])))) + skipped_score_names = list(set(initial_scores) - set(uploaded_score_names)) + self._response.data = (uploaded_score_names, skipped_score_names) + else: + self.reporter.warning_messages("Empty scores.") return self._response - diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index e73b89cad..00962a14f 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -38,6 +38,7 @@ class BaseController(metaclass=ABCMeta): + def __init__(self, config_path: str = None, token: str = None): self._team_data = None self._token = None @@ -1652,10 +1653,11 @@ def upload_priority_scores(self, project_name, folder_name, scores, project_fold project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.UploadPriorityScoresUseCase( + reporter=self.default_reporter, project=project, folder=folder, scores=scores, backend_service_provider=self.backend_client, project_folder_name=project_folder_name ) - return use_case + return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 026bacb3b..c62b9749d 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -227,8 +227,8 @@ class SuperannotateBackendService(BaseBackendService): URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" def upload_priority_scores( - self, team_id: int, project_id: int, folder_id: int, priorities: list = [] - ) -> ServiceResponse: + self, team_id: int, project_id: int, folder_id: int, priorities: list + ) -> dict: upload_priority_score_url = urljoin(self.api_url, self.URL_UPLOAD_PRIORITY_SCORES) res = self._request( upload_priority_score_url, diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index d98e2a265..4ebb9b20a 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -1,7 +1,9 @@ import os +from pathlib import Path + import src.superannotate as sa from tests.integration.base import BaseTestCase -from pathlib import Path + class TestUploadPriorityScores(BaseTestCase): PROJECT_NAME = "TestUploadPriorityScores" @@ -17,7 +19,7 @@ def test_upload_priority_scores(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) - uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + uploaded, skipped = sa.upload_priority_2scores(self.PROJECT_NAME, scores=[{ "name": "example_image_1.jpg", "priority": 1 }]) @@ -39,5 +41,5 @@ def test_upload_priority_scores(self): "name": "example_image_4.jpg", "priority": 100000000 }]) - self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_4.jpg")['entropy_value'], 1000000) - self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_3.jpg")['entropy_value'], 1.12345) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME, "example_image_4.jpg")['entropy_value'], 1000000) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME, "example_image_3.jpg")['entropy_value'], 1.12345) From b444a6845fff45421d71f59a6729a5af3e40c2f3 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 16:44:54 +0400 Subject: [PATCH 09/50] initial priority --- src/superannotate/__init__.py | 4 ++ .../lib/app/interface/sdk_interface.py | 27 ++++++++++ .../lib/app/mixp/utils/parsers.py | 8 +++ .../lib/core/serviceproviders.py | 6 +++ src/superannotate/lib/core/types.py | 5 ++ .../lib/core/usecases/annotations.py | 50 +++++++++++++++++++ .../lib/infrastructure/controller.py | 12 +++++ .../lib/infrastructure/services.py | 15 +++++- .../test_upload_priority_scores.py | 34 +++++++++++++ 9 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_upload_priority_scores.py diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index afb4e8cc5..a8517b7e5 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -99,6 +99,9 @@ from superannotate.lib.app.interface.sdk_interface import ( upload_images_from_public_urls_to_project, ) +from superannotate.lib.app.interface.sdk_interface import ( + upload_priority_scores, +) from superannotate.lib.app.interface.sdk_interface import upload_images_to_project from superannotate.lib.app.interface.sdk_interface import ( upload_preannotations_from_folder_to_project, @@ -151,6 +154,7 @@ "clone_project", "share_project", "delete_project", + "upload_priority_scores", # Images Section "search_images", "copy_image", diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index c6fc9db58..3a0ffe6f0 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -40,6 +40,7 @@ from lib.core.types import AttributeGroup from lib.core.types import MLModel from lib.core.types import Project +from lib.core.types import PriorityScore from lib.infrastructure.controller import Controller from pydantic import conlist from pydantic import parse_obj_as @@ -2900,3 +2901,29 @@ def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int if response.errors: raise AppException(response.errors) return response.data + +@Trackable +@validate_arguments +def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): + """Returns per frame annotations for the given video. + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str + + :param scores: list of score objects + :type scores: list of dicts + + :return: lists of uploaded, skipped items + :rtype: tuple (2 members) of lists of strs + """ + project_name, folder_name = extract_project_folder(project) + project_folder_name = project + use_case = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) + with tqdm( + total=len(scores), desc="Uploading priority scores" + ) as progress_bar: + for _ in use_case.execute(): + progress_bar.update() + progress_bar.close() + if use_case.response.errors: + raise AppException(use_case.errors) + return use_case.response.data \ No newline at end of file diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index b71e61cb0..fa63ac745 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1230,3 +1230,11 @@ def get_annotations_per_frame(*args, **kwargs): "event_name": "get_annotations_per_frame", "properties": {"Project": project, "fps": fps}, } + + +def upload_priority_scores(*args, **kwargs): + scores = kwargs.get("scores", args[1]) + return { + "event_name": "get_annotations_per_frame", + "properties": {"Score Count": len(scores)}, + } \ No newline at end of file diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index f85cb3979..69ab0e751 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -314,3 +314,9 @@ def get_annotations( reporter: Reporter ) -> List[dict]: raise NotImplementedError + + def upload_priority_scores( + self, team_id: int, project_id: int, folder_id: int, priorities: list = [] + ) -> ServiceResponse: + raise NotImplementedError + diff --git a/src/superannotate/lib/core/types.py b/src/superannotate/lib/core/types.py index a5236d01b..31aa7d746 100644 --- a/src/superannotate/lib/core/types.py +++ b/src/superannotate/lib/core/types.py @@ -40,3 +40,8 @@ class MLModel(BaseModel): class Config: extra = Extra.allow + + +class PriorityScore(BaseModel): + name: NotEmptyStr + priority: float diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index da061e623..55864680f 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -24,6 +24,7 @@ from lib.core.exceptions import AppException from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository +from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import SuerannotateServiceProvider from lib.core.usecases.base import BaseReportableUseCae @@ -32,6 +33,7 @@ from lib.core.video_convertor import VideoFrameGenerator from superannotate.logger import get_default_logger from superannotate_schemas.validators import AnnotationValidators +from lib.core.types import PriorityScore logger = get_default_logger() @@ -622,3 +624,51 @@ def execute(self): else: self._response.errors = "Couldn't get annotations." return self._response + + +class UploadPriorityScoresUseCase(BaseInteractiveUseCase): + + CHUNK_SIZE = 100 + + def __init__( + self, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScore], + project_folder_name: str, + backend_service_provider: SuerannotateServiceProvider + ): + super().__init__() + self._project = project + self._folder = folder + self._scores = scores + self._client = backend_service_provider + self._project_folder_name = project_folder_name + + def execute(self): + logger.info(f"Uploading priority scores for {len(self._scores)} item(s) from {self._project_folder_name}.") + priorities = [] + to_send = [] + for i in self._scores: + priorities.append({ + "name": i.name, + "entropy_value": i.priority + }) + to_send.append(i.name) + + uploaded = [] + for i in range(0, len(priorities), self.CHUNK_SIZE): + res = self._client.upload_priority_scores( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._folder.uuid, + priorities=priorities[i : i + self.CHUNK_SIZE], # noqa: E203 + ) + uploaded += res["data"] + yield + + uploaded = [i["name"] for i in uploaded] + skipped = list(set(to_send) - set(uploaded)) + self._response.data = (uploaded, skipped) + return self._response + diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index d637e839f..e73b89cad 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1647,3 +1647,15 @@ def get_annotations_per_frame(self, project_name: str, folder_name: str, video_n backend_service_provider=self.backend_client ) return use_case.execute() + + def upload_priority_scores(self, project_name, folder_name, scores, project_folder_name): + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + use_case = usecases.UploadPriorityScoresUseCase( + project=project, + folder=folder, + scores=scores, + backend_service_provider=self.backend_client, + project_folder_name=project_folder_name + ) + return use_case diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index e1189d70a..78e661197 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -50,7 +50,7 @@ def __init__( self._auth_token = auth_token self.logger = logger self._paginate_by = paginate_by - self._verify_ssl = verify_ssl + self._verify_ssl = False self.team_id = auth_token.split("=")[-1] self._testing = testing self.get_session() @@ -224,6 +224,19 @@ class SuperannotateBackendService(BaseBackendService): URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus" URL_GET_LIMITS = "project/{}/limitationDetails" URL_GET_ANNOTATIONS = "images/annotations/stream" + URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" + + def upload_priority_scores( + self, team_id: int, project_id: int, folder_id: int, priorities: list = [] + ) -> ServiceResponse: + upload_priority_score_url = urljoin(self.api_url, self.URL_UPLOAD_PRIORITY_SCORES) + res = self._request( + upload_priority_score_url, + "post", + params={"team_id": team_id, "project_id": project_id, "folder_id": folder_id}, + data={"image_entropies": priorities} + ) + return res.json() def get_project(self, uuid: int, team_id: int): get_project_url = urljoin(self.api_url, self.URL_GET_PROJECT.format(uuid)) diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py new file mode 100644 index 000000000..e19992ec8 --- /dev/null +++ b/tests/integration/test_upload_priority_scores.py @@ -0,0 +1,34 @@ +import os +import src.superannotate as sa +from tests.integration.base import BaseTestCase +from pathlib import Path + +class TestUploadPriorityScores(BaseTestCase): + PROJECT_NAME = "TestUploadPriorityScores" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PATH = "data_set/sample_project_vector" + + @property + def folder_path(self): + return os.path.join(Path(__file__).parent.parent, self.TEST_FOLDER_PATH) + + def test_upload_priority_scores(self): + sa.upload_images_from_folder_to_project( + self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" + ) + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + "name": "example_image_1.jpg", + "priority": 1 + }]) + self.assertEqual(len(uploaded), 1) + self.assertEqual(len(skipped), 0) + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + "name": "non-exist.jpg", + "priority": 1 + }, { + "name": "non-exist-2.jpg", + "priority": 1 + }]) + self.assertEqual(len(uploaded), 0) + self.assertEqual(len(skipped), 2) From 1d4582c520071d57d3331adb75447eceb649db89 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 16:58:05 +0400 Subject: [PATCH 10/50] fix parser --- src/superannotate/lib/app/mixp/utils/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index fa63ac745..581f7747c 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1235,6 +1235,6 @@ def get_annotations_per_frame(*args, **kwargs): def upload_priority_scores(*args, **kwargs): scores = kwargs.get("scores", args[1]) return { - "event_name": "get_annotations_per_frame", + "event_name": "upload_priority_scores", "properties": {"Score Count": len(scores)}, } \ No newline at end of file From 64e068bf353c043fc807acbfdb0210dd43d79bfa Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 17:22:25 +0400 Subject: [PATCH 11/50] add tqdm --- .../lib/app/interface/sdk_interface.py | 7 ++++--- .../lib/core/usecases/annotations.py | 15 +++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 3a0ffe6f0..70fe18f16 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2918,12 +2918,13 @@ def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): project_name, folder_name = extract_project_folder(project) project_folder_name = project use_case = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) + logger.info(f"Uploading priority scores for {len(scores)} item(s) from {project_folder_name}.") with tqdm( total=len(scores), desc="Uploading priority scores" ) as progress_bar: - for _ in use_case.execute(): - progress_bar.update() + for uploaded_count in use_case.execute(): + progress_bar.update(uploaded_count) progress_bar.close() if use_case.response.errors: raise AppException(use_case.errors) - return use_case.response.data \ No newline at end of file + return use_case.response.data diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 55864680f..3900a33aa 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -631,12 +631,12 @@ class UploadPriorityScoresUseCase(BaseInteractiveUseCase): CHUNK_SIZE = 100 def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScore], - project_folder_name: str, - backend_service_provider: SuerannotateServiceProvider + self, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScore], + project_folder_name: str, + backend_service_provider: SuerannotateServiceProvider ): super().__init__() self._project = project @@ -646,7 +646,6 @@ def __init__( self._project_folder_name = project_folder_name def execute(self): - logger.info(f"Uploading priority scores for {len(self._scores)} item(s) from {self._project_folder_name}.") priorities = [] to_send = [] for i in self._scores: @@ -665,7 +664,7 @@ def execute(self): priorities=priorities[i : i + self.CHUNK_SIZE], # noqa: E203 ) uploaded += res["data"] - yield + yield len(to_send[:i + self.CHUNK_SIZE]) uploaded = [i["name"] for i in uploaded] skipped = list(set(to_send) - set(uploaded)) From d36ba895d8991049a73ffd2682785c8d3a1f9508 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 17:31:04 +0400 Subject: [PATCH 12/50] Update docs --- docs/source/superannotate.sdk.rst | 1 + src/superannotate/lib/app/interface/sdk_interface.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 099fb3a8e..103be461b 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -92,6 +92,7 @@ ______ .. autofunction:: superannotate.add_annotation_bbox_to_image .. autofunction:: superannotate.add_annotation_point_to_image .. autofunction:: superannotate.add_annotation_comment_to_image +.. autofunction:: superannotate.upload_priority_scores ---------- diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 70fe18f16..5739b02f6 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2905,7 +2905,9 @@ def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int @Trackable @validate_arguments def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): - """Returns per frame annotations for the given video. + """ + Returns per frame annotations for the given video. + :param project: project name or folder path (e.g., “project1/folder1”) :type project: str From e0cad32af604bfb09ffa27932a07058e528f59b0 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 17:34:07 +0400 Subject: [PATCH 13/50] Update ssl --- src/superannotate/lib/infrastructure/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 78e661197..12ac6ae34 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -50,7 +50,7 @@ def __init__( self._auth_token = auth_token self.logger = logger self._paginate_by = paginate_by - self._verify_ssl = False + self._verify_ssl = verify_ssl self.team_id = auth_token.split("=")[-1] self._testing = testing self.get_session() From 2e82afe44f8680397a72955c251625792c925500 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Wed, 23 Feb 2022 18:16:09 +0400 Subject: [PATCH 14/50] Add clean priority --- .../lib/core/usecases/annotations.py | 15 ++++++++++++++- tests/integration/test_upload_priority_scores.py | 9 +++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 3900a33aa..61c65da84 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -645,13 +645,26 @@ def __init__( self._client = backend_service_provider self._project_folder_name = project_folder_name + @staticmethod + def get_clean_priority(priority): + if len(str(priority)) > 8: + priority = float(str(priority)[:8]) + if priority > 1000000: + priority = 1000000 + if priority < 0: + priority = 0 + if str(float(priority)).split('.')[1:2]: + if len(str(float(priority)).split('.')[1]) > 5: + priority = float(str(float(priority)).split('.')[0] + '.' + str(float(priority)).split('.')[1][:5]) + return priority + def execute(self): priorities = [] to_send = [] for i in self._scores: priorities.append({ "name": i.name, - "entropy_value": i.priority + "entropy_value": self.get_clean_priority(i.priority) }) to_send.append(i.name) diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index e19992ec8..d98e2a265 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -32,3 +32,12 @@ def test_upload_priority_scores(self): }]) self.assertEqual(len(uploaded), 0) self.assertEqual(len(skipped), 2) + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + "name": "example_image_3.jpg", + "priority": 1.1234567890 + }, { + "name": "example_image_4.jpg", + "priority": 100000000 + }]) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_4.jpg")['entropy_value'], 1000000) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_3.jpg")['entropy_value'], 1.12345) From 7272af8f1da14e9d2965d7d2d87dcdd5eea53be7 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 2 Mar 2022 12:38:00 +0400 Subject: [PATCH 15/50] refactor upload_priority_scores --- src/superannotate/__init__.py | 6 +- .../lib/app/interface/sdk_interface.py | 21 ++---- .../lib/app/mixp/utils/parsers.py | 2 +- .../lib/core/serviceproviders.py | 5 +- .../lib/core/usecases/annotations.py | 65 +++++++++++-------- .../lib/infrastructure/controller.py | 4 +- .../lib/infrastructure/services.py | 4 +- .../test_upload_priority_scores.py | 10 +-- 8 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index a8517b7e5..3cdb13ccc 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -99,13 +99,13 @@ from superannotate.lib.app.interface.sdk_interface import ( upload_images_from_public_urls_to_project, ) -from superannotate.lib.app.interface.sdk_interface import ( - upload_priority_scores, -) from superannotate.lib.app.interface.sdk_interface import upload_images_to_project from superannotate.lib.app.interface.sdk_interface import ( upload_preannotations_from_folder_to_project, ) +from superannotate.lib.app.interface.sdk_interface import ( + upload_priority_scores, +) from superannotate.lib.app.interface.sdk_interface import upload_video_to_project from superannotate.lib.app.interface.sdk_interface import ( upload_videos_from_folder_to_project, diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 5739b02f6..3325ec7c6 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -39,8 +39,8 @@ from lib.core.exceptions import AppException from lib.core.types import AttributeGroup from lib.core.types import MLModel -from lib.core.types import Project from lib.core.types import PriorityScore +from lib.core.types import Project from lib.infrastructure.controller import Controller from pydantic import conlist from pydantic import parse_obj_as @@ -2902,11 +2902,11 @@ def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int raise AppException(response.errors) return response.data + @Trackable @validate_arguments def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): - """ - Returns per frame annotations for the given video. + """Returns per frame annotations for the given video. :param project: project name or folder path (e.g., “project1/folder1”) :type project: str @@ -2919,14 +2919,7 @@ def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): """ project_name, folder_name = extract_project_folder(project) project_folder_name = project - use_case = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) - logger.info(f"Uploading priority scores for {len(scores)} item(s) from {project_folder_name}.") - with tqdm( - total=len(scores), desc="Uploading priority scores" - ) as progress_bar: - for uploaded_count in use_case.execute(): - progress_bar.update(uploaded_count) - progress_bar.close() - if use_case.response.errors: - raise AppException(use_case.errors) - return use_case.response.data + response = Controller.get_default().upload_priority_scores(project_name, folder_name, scores, project_folder_name) + if response.errors: + raise AppException(response.errors) + return response.data diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index 581f7747c..ca14cf5c0 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1237,4 +1237,4 @@ def upload_priority_scores(*args, **kwargs): return { "event_name": "upload_priority_scores", "properties": {"Score Count": len(scores)}, - } \ No newline at end of file + } diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 69ab0e751..0a6f294c0 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -316,7 +316,6 @@ def get_annotations( raise NotImplementedError def upload_priority_scores( - self, team_id: int, project_id: int, folder_id: int, priorities: list = [] - ) -> ServiceResponse: + self, team_id: int, project_id: int, folder_id: int, priorities: list + ) -> dict: raise NotImplementedError - diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 61c65da84..f9464b96d 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -24,16 +24,16 @@ from lib.core.exceptions import AppException from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository -from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.types import PriorityScore from lib.core.usecases.base import BaseReportableUseCae from lib.core.usecases.images import GetBulkImages from lib.core.usecases.images import ValidateAnnotationUseCase from lib.core.video_convertor import VideoFrameGenerator from superannotate.logger import get_default_logger from superannotate_schemas.validators import AnnotationValidators -from lib.core.types import PriorityScore + logger = get_default_logger() @@ -626,19 +626,20 @@ def execute(self): return self._response -class UploadPriorityScoresUseCase(BaseInteractiveUseCase): +class UploadPriorityScoresUseCase(BaseReportableUseCae): CHUNK_SIZE = 100 def __init__( self, + reporter, project: ProjectEntity, folder: FolderEntity, scores: List[PriorityScore], project_folder_name: str, backend_service_provider: SuerannotateServiceProvider ): - super().__init__() + super().__init__(reporter) self._project = project self._folder = folder self._scores = scores @@ -658,29 +659,37 @@ def get_clean_priority(priority): priority = float(str(float(priority)).split('.')[0] + '.' + str(float(priority)).split('.')[1][:5]) return priority - def execute(self): - priorities = [] - to_send = [] - for i in self._scores: - priorities.append({ - "name": i.name, - "entropy_value": self.get_clean_priority(i.priority) - }) - to_send.append(i.name) - - uploaded = [] - for i in range(0, len(priorities), self.CHUNK_SIZE): - res = self._client.upload_priority_scores( - team_id=self._project.team_id, - project_id=self._project.uuid, - folder_id=self._folder.uuid, - priorities=priorities[i : i + self.CHUNK_SIZE], # noqa: E203 - ) - uploaded += res["data"] - yield len(to_send[:i + self.CHUNK_SIZE]) + @property + def folder_path(self): + return f"{self._project.name}{f'/{self._folder.name}'if self._folder.name != 'root' else ''}" - uploaded = [i["name"] for i in uploaded] - skipped = list(set(to_send) - set(uploaded)) - self._response.data = (uploaded, skipped) + def execute(self): + if self.is_valid(): + priorities = [] + initial_scores = [] + for i in self._scores: + priorities.append({ + "name": i.name, + "entropy_value": self.get_clean_priority(i.priority) + }) + initial_scores.append(i.name) + uploaded_score_names = [] + self.reporter.log_info(f"Uploading priority scores for {len(priorities)} item(s) from {self.folder_path}.") + iterations = range(0, len(priorities), self.CHUNK_SIZE) + self.reporter.start_progress(iterations, "Uploading priority scores") + if iterations: + for i in iterations: + priorities_to_upload = priorities[i : i + self.CHUNK_SIZE] # noqa: E203 + res = self._client.upload_priority_scores( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._folder.uuid, + priorities=priorities_to_upload + ) + self.reporter.update_progress(len(priorities_to_upload)) + uploaded_score_names.extend(list(map(lambda x: x["name"], res.get("data", [])))) + skipped_score_names = list(set(initial_scores) - set(uploaded_score_names)) + self._response.data = (uploaded_score_names, skipped_score_names) + else: + self.reporter.warning_messages("Empty scores.") return self._response - diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index e73b89cad..00962a14f 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -38,6 +38,7 @@ class BaseController(metaclass=ABCMeta): + def __init__(self, config_path: str = None, token: str = None): self._team_data = None self._token = None @@ -1652,10 +1653,11 @@ def upload_priority_scores(self, project_name, folder_name, scores, project_fold project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.UploadPriorityScoresUseCase( + reporter=self.default_reporter, project=project, folder=folder, scores=scores, backend_service_provider=self.backend_client, project_folder_name=project_folder_name ) - return use_case + return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 12ac6ae34..f1369eaff 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -227,8 +227,8 @@ class SuperannotateBackendService(BaseBackendService): URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" def upload_priority_scores( - self, team_id: int, project_id: int, folder_id: int, priorities: list = [] - ) -> ServiceResponse: + self, team_id: int, project_id: int, folder_id: int, priorities: list + ) -> dict: upload_priority_score_url = urljoin(self.api_url, self.URL_UPLOAD_PRIORITY_SCORES) res = self._request( upload_priority_score_url, diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index d98e2a265..4ebb9b20a 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -1,7 +1,9 @@ import os +from pathlib import Path + import src.superannotate as sa from tests.integration.base import BaseTestCase -from pathlib import Path + class TestUploadPriorityScores(BaseTestCase): PROJECT_NAME = "TestUploadPriorityScores" @@ -17,7 +19,7 @@ def test_upload_priority_scores(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) - uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ + uploaded, skipped = sa.upload_priority_2scores(self.PROJECT_NAME, scores=[{ "name": "example_image_1.jpg", "priority": 1 }]) @@ -39,5 +41,5 @@ def test_upload_priority_scores(self): "name": "example_image_4.jpg", "priority": 100000000 }]) - self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_4.jpg")['entropy_value'], 1000000) - self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME,"example_image_3.jpg")['entropy_value'], 1.12345) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME, "example_image_4.jpg")['entropy_value'], 1000000) + self.assertEqual(sa.get_image_metadata(self.PROJECT_NAME, "example_image_3.jpg")['entropy_value'], 1.12345) From cf1c78db0d33f376709aeb85aeb039cf918ffc0a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 2 Mar 2022 12:41:15 +0400 Subject: [PATCH 16/50] tod --- src/superannotate/lib/app/serializers.py | 1 - .../lib/core/usecases/annotations.py | 1 - src/superannotate/lib/core/video_convertor.py | 185 ++++++++++-------- src/superannotate/version.py | 1 - 4 files changed, 99 insertions(+), 89 deletions(-) diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index 6af747131..163e0d383 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -114,4 +114,3 @@ def serialize(self): if data["attribute"] == "ImageQuality": data["value"] = constance.ImageQuality.get_name(data["value"]) return data - diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index f9464b96d..b4ecdbcfb 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -536,7 +536,6 @@ def validate_item_names(self): self._item_names = [item.name for item in self._images.get_all(condition)] def _prettify_annotations(self, annotations: List[dict]): - if self._item_names_provided: try: data = [] diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 0f77f38dc..fe10938ef 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -1,3 +1,4 @@ +import itertools import math from collections import defaultdict from typing import Any @@ -62,117 +63,129 @@ def interpolate_annotations( data: dict, steps: dict = None, annotation_type: str = "bbox" - ): - for idx, frame_idx in enumerate(range(from_frame, to_frame), 1): - keyframe = False - if idx == 1: - keyframe = True + ) -> dict: + annotations = {} + for idx, frame_idx in enumerate(range(from_frame + 1, to_frame), 1): points = None - if annotation_type == "bbox": + if annotation_type == "bbox" and data.get("points") and steps: points = { "x1": round(data["points"]["x1"] + steps["x1"] * idx, 2), "y1": round(data["points"]["y1"] + steps["y1"] * idx, 2), "x2": round(data["points"]["x2"] + steps["x2"] * idx, 2), "y2": round(data["points"]["y2"] + steps["y2"] * idx, 2), } - self._add_annotation( - frame_idx, - annotation_type=annotation_type, - class_name=class_name, + annotations[frame_idx] = Annotation( + type=annotation_type, + className=class_name, points=points, attributes=data["attributes"], - keyframe=keyframe + keyframe=False ) + return annotations def _add_annotation( self, frame_no: int, - annotation_type: str, - class_name: str, - points: list = None, - attributes: list = None, - keyframe: bool = False + annotation: Annotation ): + frame = self.get_frame(frame_no) - frame.annotations.append(Annotation( - type=annotation_type, - className=class_name, - points=points, - attributes=attributes, - keyframe=keyframe - )) + frame.annotations.append(annotation) + + @staticmethod + def pairwise(data: list): + a, b = itertools.tee(data) + next(b, None) + return zip(a, b) + + def get_median(self, annotations: List[dict]) -> dict: + if len(annotations) == 1: + return annotations[0] + first_annotations = annotations[:1][0] + median = (first_annotations["timestamp"] // self.ratio) + (self.ratio / 2) + median_annotation = first_annotations + distance = abs(median - first_annotations["timestamp"]) + for annotation in annotations[1:]: + annotation_distance = abs(median - annotation["timestamp"]) + if annotation_distance < distance: + distance = annotation_distance + median_annotation = annotation + return median_annotation + + def merge_first_frame(self, frames_mapping): + try: + if 0 in frames_mapping: + frames_mapping[1].extend(frames_mapping[0]) + del frames_mapping[0] + finally: + return frames_mapping def _process(self): for instance in self._annotation_data["instances"]: + annotation_type = instance["meta"]["type"] + class_name = instance["meta"]["className"] for parameter in instance["parameters"]: - time_stamp_frame_map = [] + frames_mapping = defaultdict(list) + last_frame_no = None + last_annotation = None + interpolated_frames = {} for timestamp in parameter["timestamps"]: - time_stamp_frame_map.append((int(math.ceil(timestamp["timestamp"] / self.ratio)), timestamp)) - skip_current = False - for idx, (frame_no, timestamp_data) in enumerate(time_stamp_frame_map): - annotation_type = instance["meta"]["type"] - try: - next_frame_no, next_timestamp = time_stamp_frame_map[idx + 1] - if frame_no == next_frame_no: - median = (timestamp_data["timestamp"] // self.ratio) + (self.ratio / 2) - if abs(median - timestamp_data["timestamp"]) < abs( - median - next_timestamp["timestamp"]) and not skip_current: - time_stamp_frame_map[idx + 1] = (frame_no, timestamp_data) - self._add_annotation( - frame_no=frame_no, - annotation_type=annotation_type, - class_name=instance["meta"]["className"], - points=timestamp_data.get("points"), - attributes=timestamp_data.get("attributes"), - keyframe=True - ) - skip_current = True - elif skip_current: - frame_no += 1 - skip_current = False - - frames_diff = next_frame_no - frame_no + frames_mapping[int(math.ceil(timestamp["timestamp"] / self.ratio))].append(timestamp) + frames_mapping = self.merge_first_frame(frames_mapping) + for from_frame_no, to_frame_no in self.pairwise(sorted(frames_mapping)): + last_frame_no = to_frame_no + from_frame, to_frame = frames_mapping[from_frame_no][-1], frames_mapping[to_frame_no][0] + frames_diff = to_frame_no - from_frame_no + if frames_diff > 1: steps = None - if annotation_type == "bbox": - if not frames_diff: - steps = { - "y1": 0, - "x2": 0, - "x1": 0, - "y2": 0 - } - else: - steps = { - "y1": round( - (next_timestamp["points"]["y1"] - timestamp_data["points"]["y1"]) / frames_diff, - 2), - "x2": round( - (next_timestamp["points"]["x2"] - timestamp_data["points"]["x2"]) / frames_diff, - 2), - "x1": round( - (next_timestamp["points"]["x1"] - timestamp_data["points"]["x1"]) / frames_diff, - 2), - "y2": round( - (next_timestamp["points"]["y2"] - timestamp_data["points"]["y2"]) / frames_diff, - 2), - } - self.interpolate_annotations( - class_name=instance["meta"]["className"], - from_frame=frame_no, - to_frame=next_frame_no, - data=timestamp_data, + if annotation_type == "bbox" and from_frame.get("points") and to_frame.get("points"): + steps = { + "y1": round( + (to_frame["points"]["y1"] - from_frame["points"]["y1"]) / frames_diff, + 2), + "x2": round( + (to_frame["points"]["x2"] - from_frame["points"]["x2"]) / frames_diff, + 2), + "x1": round( + (to_frame["points"]["x1"] - from_frame["points"]["x1"]) / frames_diff, + 2), + "y2": round( + (to_frame["points"]["y2"] - from_frame["points"]["y2"]) / frames_diff, + 2), + } + interpolated_frames.update(self.interpolate_annotations( + class_name=class_name, + from_frame=from_frame_no, + to_frame=to_frame_no, + data=from_frame, steps=steps, annotation_type=annotation_type - ) - except IndexError: - frame = self.get_frame(frame_no) - frame.annotations.append(Annotation( - type=annotation_type, - className=instance["meta"]["className"], - points=timestamp_data.get("points"), - attributes=timestamp_data.get("attributes"), - keyframe=True )) + start_median_frame = self.get_median(frames_mapping[from_frame_no]) + end_median_frame = self.get_median(frames_mapping[to_frame_no]) + interpolated_frames[from_frame_no] = Annotation( + type=annotation_type, + className=class_name, + points=start_median_frame.get("points"), + attributes=start_median_frame["attributes"], + keyframe=True + ) + last_annotation = Annotation( + type=annotation_type, + className=class_name, + points=end_median_frame.get("points"), + attributes=end_median_frame["attributes"], + keyframe=True + ) + # interpolated_frames[to_frame_no] = Annotation( + # type=annotation_type, + # className=class_name, + # points=end_median_frame.get("points"), + # attributes=end_median_frame["attributes"], + # keyframe=True + # ) + self._add_annotation(last_frame_no, last_annotation) + [self._add_annotation(frame_no, annotation) for frame_no, annotation in interpolated_frames.items()] def __iter__(self): for frame_no in range(1, int(self.frames_count) + 1): diff --git a/src/superannotate/version.py b/src/superannotate/version.py index a17f5643c..d386f167f 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1,2 +1 @@ __version__ = "4.2.9" - From ae71d0aa999324adb3baef38f252ed1cc3105944 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 2 Mar 2022 12:45:14 +0400 Subject: [PATCH 17/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index d386f167f..91535874d 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.2.9" +__version__ = "4.3.0.dev1" From fe70a1dd58b3a1bf6b37edd3555e1c7c218a24a9 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 2 Mar 2022 15:05:08 +0400 Subject: [PATCH 18/50] fix convertor --- src/superannotate/lib/core/usecases/annotations.py | 5 ++++- src/superannotate/lib/core/video_convertor.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index b4ecdbcfb..25d019de2 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -592,7 +592,10 @@ def __init__( def validate_project_type(self): if self._project.project_type != constances.ProjectType.VIDEO.value: - raise AppException("The function only supports video projects.") + raise AppException( + "The function is not supported for" + f" {constances.ProjectType.get_name(self._project.project_type)} projects." + ) def execute(self): self.reporter.disable_info() diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index fe10938ef..bdc44492a 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -112,10 +112,11 @@ def get_median(self, annotations: List[dict]) -> dict: median_annotation = annotation return median_annotation - def merge_first_frame(self, frames_mapping): + @staticmethod + def merge_first_frame(frames_mapping): try: if 0 in frames_mapping: - frames_mapping[1].extend(frames_mapping[0]) + frames_mapping[1].extendleft(frames_mapping[0]) del frames_mapping[0] finally: return frames_mapping From dfd634fa4f6f647f5d1be6d62e3986744d711909 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 2 Mar 2022 15:05:41 +0400 Subject: [PATCH 19/50] Change version --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 91535874d..b1820ff87 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev1" +__version__ = "4.3.0.dev2" From 2750f6dfad480eb7a67ff8ca593eb488ccd811e4 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 2 Mar 2022 17:02:14 +0400 Subject: [PATCH 20/50] Minor changes --- .../lib/core/usecases/annotations.py | 212 +++++++++--------- src/superannotate/lib/core/video_convertor.py | 3 +- .../lib/infrastructure/controller.py | 2 +- src/superannotate/version.py | 2 +- .../test_upload_priority_scores.py | 3 +- 5 files changed, 115 insertions(+), 107 deletions(-) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 25d019de2..3814620a5 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -8,9 +8,11 @@ from typing import Tuple import boto3 +from superannotate_schemas.validators import AnnotationValidators + import lib.core as constances -from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ +from lib.core.conditions import Condition from lib.core.data_handlers import ChainedAnnotationHandlers from lib.core.data_handlers import DocumentTagHandler from lib.core.data_handlers import LastActionHandler @@ -32,8 +34,6 @@ from lib.core.usecases.images import ValidateAnnotationUseCase from lib.core.video_convertor import VideoFrameGenerator from superannotate.logger import get_default_logger -from superannotate_schemas.validators import AnnotationValidators - logger = get_default_logger() @@ -45,20 +45,20 @@ class UploadAnnotationsUseCase(BaseReportableUseCae): ImageInfo = namedtuple("ImageInfo", ["path", "name", "id"]) def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - team: TeamEntity, - images: BaseManageableRepository, - annotation_classes: List[AnnotationClassEntity], - annotation_paths: List[str], - backend_service_provider: SuerannotateServiceProvider, - templates: List[dict], - validators: AnnotationValidators, - pre_annotation: bool = False, - client_s3_bucket=None, - folder_path: str = None, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + team: TeamEntity, + images: BaseManageableRepository, + annotation_classes: List[AnnotationClassEntity], + annotation_paths: List[str], + backend_service_provider: SuerannotateServiceProvider, + templates: List[dict], + validators: AnnotationValidators, + pre_annotation: bool = False, + client_s3_bucket=None, + folder_path: str = None, ): super().__init__(reporter) self._project = project @@ -82,8 +82,8 @@ def __init__( @property def annotation_postfix(self): if self._project.project_type in ( - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, ): return constances.ATTACHED_VIDEO_ANNOTATION_POSTFIX elif self._project.project_type == constances.ProjectType.VECTOR.value: @@ -95,8 +95,8 @@ def annotation_postfix(self): def extract_name(value: str): return os.path.basename( value.replace(constances.PIXEL_ANNOTATION_POSTFIX, "") - .replace(constances.VECTOR_ANNOTATION_POSTFIX, "") - .replace(constances.ATTACHED_VIDEO_ANNOTATION_POSTFIX, ""), + .replace(constances.VECTOR_ANNOTATION_POSTFIX, "") + .replace(constances.ATTACHED_VIDEO_ANNOTATION_POSTFIX, ""), ) @property @@ -120,8 +120,8 @@ def annotations_to_upload(self): folder_id=self._folder.uuid, images=[image.name for image in images_detail], ) - .execute() - .data + .execute() + .data ) for image_data in images_data: for idx, detail in enumerate(images_detail): @@ -151,7 +151,7 @@ def missing_annotations(self): return self._missing_annotations def get_annotation_upload_data( - self, image_ids: List[int] + self, image_ids: List[int] ) -> UploadAnnotationAuthData: if self._pre_annotation: function = self._backend_service.get_pre_annotation_upload_data @@ -167,12 +167,12 @@ def get_annotation_upload_data( return response.data def _upload_annotation( - self, - image_id: int, - image_name: str, - upload_data: UploadAnnotationAuthData, - path: str, - bucket, + self, + image_id: int, + image_name: str, + upload_data: UploadAnnotationAuthData, + path: str, + bucket, ): try: self.reporter.disable_warnings() @@ -256,8 +256,8 @@ def execute(self): ) for step in iterations_range: annotations_to_upload = self.annotations_to_upload[ - step : step + self.AUTH_DATA_CHUNK_SIZE # noqa: E203 - ] + step: step + self.AUTH_DATA_CHUNK_SIZE # noqa: E203 + ] upload_data = self.get_annotation_upload_data( [int(image.id) for image in annotations_to_upload] ) @@ -270,11 +270,11 @@ def execute(self): } # dummy progress for _ in range( - len(annotations_to_upload) - len(upload_data.images) + len(annotations_to_upload) - len(upload_data.images) ): self.reporter.update_progress() with concurrent.futures.ThreadPoolExecutor( - max_workers=self.MAX_WORKERS + max_workers=self.MAX_WORKERS ) as executor: results = [ executor.submit( @@ -306,25 +306,25 @@ def execute(self): class UploadAnnotationUseCase(BaseReportableUseCae): def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - image: ImageEntity, - images: BaseManageableRepository, - team: TeamEntity, - annotation_classes: List[AnnotationClassEntity], - backend_service_provider: SuerannotateServiceProvider, - reporter: Reporter, - templates: List[dict], - validators: AnnotationValidators, - annotation_upload_data: UploadAnnotationAuthData = None, - annotations: dict = None, - s3_bucket=None, - client_s3_bucket=None, - mask=None, - verbose: bool = True, - annotation_path: str = None, - pass_validation: bool = False, + self, + project: ProjectEntity, + folder: FolderEntity, + image: ImageEntity, + images: BaseManageableRepository, + team: TeamEntity, + annotation_classes: List[AnnotationClassEntity], + backend_service_provider: SuerannotateServiceProvider, + reporter: Reporter, + templates: List[dict], + validators: AnnotationValidators, + annotation_upload_data: UploadAnnotationAuthData = None, + annotations: dict = None, + s3_bucket=None, + client_s3_bucket=None, + mask=None, + verbose: bool = True, + annotation_path: str = None, + pass_validation: bool = False, ): super().__init__(reporter) self._project = project @@ -413,18 +413,18 @@ def set_annotation_json(self): @staticmethod def prepare_annotations( - project_type: int, - annotations: dict, - annotation_classes: List[AnnotationClassEntity], - templates: List[dict], - reporter: Reporter, - team: TeamEntity, + project_type: int, + annotations: dict, + annotation_classes: List[AnnotationClassEntity], + templates: List[dict], + reporter: Reporter, + team: TeamEntity, ) -> dict: handlers_chain = ChainedAnnotationHandlers() if project_type in ( - constances.ProjectType.VECTOR.value, - constances.ProjectType.PIXEL.value, - constances.ProjectType.DOCUMENT.value, + constances.ProjectType.VECTOR.value, + constances.ProjectType.PIXEL.value, + constances.ProjectType.DOCUMENT.value, ): handlers_chain.attach( MissingIDsHandler(annotation_classes, templates, reporter) @@ -436,7 +436,7 @@ def prepare_annotations( handlers_chain.attach(LastActionHandler(team.creator_id)) return handlers_chain.handle(annotations) - def clean_json(self, json_data: dict,) -> Tuple[bool, dict]: + def clean_json(self, json_data: dict, ) -> Tuple[bool, dict]: use_case = ValidateAnnotationUseCase( constances.ProjectType.get_name(self._project.project_type), annotation=json_data, @@ -466,8 +466,8 @@ def execute(self): Body=json.dumps(annotation_json), ) if ( - self._project.project_type == constances.ProjectType.PIXEL.value - and self._mask + self._project.project_type == constances.ProjectType.PIXEL.value + and self._mask ): bucket.put_object( Key=self.annotation_upload_data.images[self._image.uuid][ @@ -540,7 +540,7 @@ def _prettify_annotations(self, annotations: List[dict]): try: data = [] for annotation in annotations: - data.append((self._item_names.index(annotation["metadata"]["name"]), annotation)) + data.append((self._item_names.index(annotation["metadata"]["name"]), annotation)) return [i[1] for i in sorted(data, key=lambda x: x[0])] except KeyError: raise AppException("Broken data.") @@ -598,48 +598,48 @@ def validate_project_type(self): ) def execute(self): - self.reporter.disable_info() - response = GetAnnotations( - reporter=self.reporter, - project=self._project, - folder=self._folder, - images=self._images, - item_names=[self._video_name], - backend_service_provider=self._client, - show_process=False - ).execute() - self.reporter.enable_info() - if response.data: - generator = VideoFrameGenerator(response.data[0], fps=self._fps) - - self.reporter.log_info(f"Getting annotations for {generator.frames_count} frames from {self._video_name}.") - if response.errors: - self._response.errors = response.errors - return self._response - if not response.data: - self._response.errors = AppException(f"Video {self._video_name} not found.") - annotations = response.data - if annotations: - self._response.data = list(generator) + if self.is_valid(): + self.reporter.disable_info() + response = GetAnnotations( + reporter=self.reporter, + project=self._project, + folder=self._folder, + images=self._images, + item_names=[self._video_name], + backend_service_provider=self._client, + show_process=False + ).execute() + self.reporter.enable_info() + if response.data: + generator = VideoFrameGenerator(response.data[0], fps=self._fps) + + self.reporter.log_info(f"Getting annotations for {generator.frames_count} frames from {self._video_name}.") + if response.errors: + self._response.errors = response.errors + return self._response + if not response.data: + self._response.errors = AppException(f"Video {self._video_name} not found.") + annotations = response.data + if annotations: + self._response.data = list(generator) + else: + self._response.data = [] else: - self._response.data = [] - else: - self._response.errors = "Couldn't get annotations." + self._response.errors = "Couldn't get annotations." return self._response class UploadPriorityScoresUseCase(BaseReportableUseCae): - CHUNK_SIZE = 100 def __init__( - self, - reporter, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScore], - project_folder_name: str, - backend_service_provider: SuerannotateServiceProvider + self, + reporter, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScore], + project_folder_name: str, + backend_service_provider: SuerannotateServiceProvider ): super().__init__(reporter) self._project = project @@ -663,7 +663,12 @@ def get_clean_priority(priority): @property def folder_path(self): - return f"{self._project.name}{f'/{self._folder.name}'if self._folder.name != 'root' else ''}" + return f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}" + + @property + def uploading_info(self): + data_len: int = len(self._scores) + return f"Uploading priority scores for {data_len} item{'(s)' if data_len < 1 else ''} to {self.folder_path}." def execute(self): if self.is_valid(): @@ -676,12 +681,12 @@ def execute(self): }) initial_scores.append(i.name) uploaded_score_names = [] - self.reporter.log_info(f"Uploading priority scores for {len(priorities)} item(s) from {self.folder_path}.") + self.reporter.log_info(self.uploading_info) iterations = range(0, len(priorities), self.CHUNK_SIZE) self.reporter.start_progress(iterations, "Uploading priority scores") if iterations: for i in iterations: - priorities_to_upload = priorities[i : i + self.CHUNK_SIZE] # noqa: E203 + priorities_to_upload = priorities[i: i + self.CHUNK_SIZE] # noqa: E203 res = self._client.upload_priority_scores( team_id=self._project.team_id, project_id=self._project.uuid, @@ -690,6 +695,7 @@ def execute(self): ) self.reporter.update_progress(len(priorities_to_upload)) uploaded_score_names.extend(list(map(lambda x: x["name"], res.get("data", [])))) + self.reporter.finish_progress() skipped_score_names = list(set(initial_scores) - set(uploaded_score_names)) self._response.data = (uploaded_score_names, skipped_score_names) else: diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index bdc44492a..1ee52d977 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -116,7 +116,8 @@ def get_median(self, annotations: List[dict]) -> dict: def merge_first_frame(frames_mapping): try: if 0 in frames_mapping: - frames_mapping[1].extendleft(frames_mapping[0]) + frames_mapping[0].extend(frames_mapping[1]) + frames_mapping[1] = frames_mapping[0] del frames_mapping[0] finally: return frames_mapping diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 00962a14f..67fb78b2e 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -98,7 +98,7 @@ def retrieve_configs(self, path: Path, raise_exception=True): ) self._token = token self._backend_url = backend_url or self._backend_url - self._ssl_verify = ssl_verify or self._ssl_verify + self._ssl_verify = ssl_verify if ssl_verify is not None else True @staticmethod def _validate_token(token: str): diff --git a/src/superannotate/version.py b/src/superannotate/version.py index b1820ff87..312635073 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev2" +__version__ = "4.3.0.dev5" diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index 4ebb9b20a..371f742e3 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -16,10 +16,11 @@ def folder_path(self): return os.path.join(Path(__file__).parent.parent, self.TEST_FOLDER_PATH) def test_upload_priority_scores(self): + sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) - uploaded, skipped = sa.upload_priority_2scores(self.PROJECT_NAME, scores=[{ + uploaded, skipped = sa.upload_priority_scores(self.PROJECT_NAME, scores=[{ "name": "example_image_1.jpg", "priority": 1 }]) From 3f4368b23fd515d107a5d3ad56746e849f4980da Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 3 Mar 2022 17:24:52 +0400 Subject: [PATCH 21/50] Added integrations --- src/superannotate/__init__.py | 5 + .../lib/app/interface/sdk_interface.py | 47 ++ .../lib/app/mixp/utils/parsers.py | 34 ++ src/superannotate/lib/app/serializers.py | 41 +- .../lib/core/entities/__init__.py | 2 + src/superannotate/lib/core/entities/base.py | 10 + .../lib/core/entities/integrations.py | 14 + .../lib/core/serviceproviders.py | 165 +++--- .../lib/core/usecases/__init__.py | 1 + .../lib/core/usecases/annotations.py | 2 +- .../lib/core/usecases/integrations.py | 81 +++ .../lib/infrastructure/controller.py | 471 ++++++++++-------- .../lib/infrastructure/repositories.py | 17 + .../lib/infrastructure/services.py | 31 ++ tests/convertors/test_labelbox.py | 41 +- .../integrations/test_get_integrations.py | 24 + 16 files changed, 662 insertions(+), 324 deletions(-) create mode 100644 src/superannotate/lib/core/entities/base.py create mode 100644 src/superannotate/lib/core/entities/integrations.py create mode 100644 src/superannotate/lib/core/usecases/integrations.py create mode 100644 tests/integration/integrations/test_get_integrations.py diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 3cdb13ccc..8af10fa64 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -21,6 +21,8 @@ from superannotate.lib.app.interface.sdk_interface import aggregate_annotations_as_df from superannotate.lib.app.interface.sdk_interface import assign_folder from superannotate.lib.app.interface.sdk_interface import assign_images +from superannotate.lib.app.interface.sdk_interface import get_integrations +from superannotate.lib.app.interface.sdk_interface import attach_items_from_integrated_storage from superannotate.lib.app.interface.sdk_interface import ( attach_document_urls_to_project, ) @@ -135,6 +137,9 @@ # annotations "get_annotations", "get_annotations_per_frame", + # integrations + "get_integrations", + "attach_items_from_integrated_storage", # converters "convert_json_version", "import_annotation", diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 3325ec7c6..b5b7b0064 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -34,6 +34,7 @@ from lib.app.serializers import SettingsSerializer from lib.app.serializers import TeamSerializer from lib.core import LIMITED_FUNCTIONS +from lib.core.entities.integrations import IntegrationEntity from lib.core.entities.project_entities import AnnotationClassEntity from lib.core.enums import ImageQuality from lib.core.exceptions import AppException @@ -2923,3 +2924,49 @@ def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): if response.errors: raise AppException(response.errors) return response.data + + +@Trackable +@validate_arguments +def get_integrations(): + """Get all integrations per team + + :return: metadata objects of all integrations of the team. + :rtype: list of dicts + """ + response = Controller.get_default().get_integrations() + if response.errors: + raise AppException(response.errors) + integrations = response.data + return BaseSerializers.serialize_iterable(integrations, ("name", "type", "root")) + + +@Trackable +@validate_arguments +def attach_items_from_integrated_storage( + project: NotEmptyStr, + integration: Union[NotEmptyStr, IntegrationEntity], + folder_path: Optional[NotEmptyStr] = None +): + """Link images from integrated external storage to SuperAnnotate. + + :param project: project name or folder path where items should be attached (e.g., “project1/folder1”). + :type project: str + + :param project: project name or folder path where items should be attached (e.g., “project1/folder1”). + :type project: str + + :param integration: existing integration name or metadata dict to pull items from. + Mandatory keys in integration metadata’s dict is “name”. + :type integration: str + + :param folder_path: Points to an exact folder/directory within given storage. + If None, items are fetched from the root directory. + :type folder_path: str + """ + project_name, folder_name = extract_project_folder(project) + if isinstance(integration, str): + integration = IntegrationEntity(name=integration) + response = Controller.get_default().attach_integrations(project_name, folder_name, integration, folder_path) + if response.errors: + raise AppException(response.errors) diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index ca14cf5c0..9b9c0ba5f 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1,6 +1,7 @@ import lib.core as constances from lib.app.helpers import extract_project_folder from lib.core.enums import ProjectType +from lib.core.entities import IntegrationEntity from lib.infrastructure.controller import Controller @@ -1238,3 +1239,36 @@ def upload_priority_scores(*args, **kwargs): "event_name": "upload_priority_scores", "properties": {"Score Count": len(scores)}, } + + +def get_integrations(*args, **kwargs): + return { + "event_name": "get_integrations", + "properties": {}, + } + + +def attach_items_from_integrated_storage(*args, **kwargs): + project = kwargs.get("project") + integration = kwargs.get("integration") + folder_path = kwargs.get("folder_path") + if not project: + project = args[0] + if not integration: + integration = args[1] + if not folder_path: + folder_path = args[2] + + project_name, _ = extract_project_folder(project) + if isinstance(integration, str): + integration = IntegrationEntity(name=integration) + project = Controller.get_default().get_project_metadata(project_name) + return { + "event_name": "attach_items_from_integrated_storage", + "properties": { + "project_type": ProjectType.get_name(project.project_type), + "integration_name": integration.name, + "folder_path": bool(folder_path) + }, + } + diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index 163e0d383..d3d3103ee 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -1,4 +1,8 @@ from abc import ABC +from typing import List +from typing import Set +from typing import Union +from typing import Any import superannotate.lib.core as constance from pydantic import BaseModel @@ -11,12 +15,37 @@ class BaseSerializers(ABC): def __init__(self, entity: BaseEntity): self._entity = entity - def serialize(self): - if isinstance(self._entity, dict): - return self._entity - if isinstance(self._entity, BaseModel): - return self._entity.dict(by_alias=True) - return self._entity.to_dict() + def serialize(self, fields: List[str] = None, by_alias: bool = True, flat: bool = False): + return self._serialize(self._entity, fields, by_alias, flat) + + @staticmethod + def _serialize(entity: Any, fields: List[str] = None, by_alias: bool = False, flat: bool = False): + if isinstance(entity, dict): + return entity + if isinstance(entity, BaseModel): + if fields: + fields = set(fields) + if len(fields) == 1: + if flat: + return entity.dict(include=fields, by_alias=by_alias)[next(iter(fields))] + else: + return entity.dict(include=fields, by_alias=by_alias) + return entity.dict(include=fields, by_alias=by_alias) + return entity.dict(by_alias=by_alias) + return entity.to_dict() + + @classmethod + def serialize_iterable( + cls, + data: List[Any], + fields: Union[List[str], Set[str]] = None, + by_alias: bool = False, + flat: bool = False + ) -> List[Any]: + serialized_data = [] + for i in data: + serialized_data.append(cls._serialize(i, fields, by_alias, flat)) + return serialized_data class UserSerializer(BaseSerializers): diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index 596e28932..753dddf6b 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -11,6 +11,7 @@ from lib.core.entities.project_entities import TeamEntity from lib.core.entities.project_entities import UserEntity from lib.core.entities.project_entities import WorkflowEntity +from lib.core.entities.integrations import IntegrationEntity from superannotate_schemas.schemas.internal.document import DocumentAnnotation from superannotate_schemas.schemas.internal.pixel import PixelAnnotation from superannotate_schemas.schemas.internal.vector import VectorAnnotation @@ -33,6 +34,7 @@ "UserEntity", "TeamEntity", "MLModelEntity", + "IntegrationEntity", # annotations "DocumentAnnotation", "VideoAnnotation", diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py new file mode 100644 index 000000000..740a52042 --- /dev/null +++ b/src/superannotate/lib/core/entities/base.py @@ -0,0 +1,10 @@ +from datetime import datetime + +from pydantic import BaseModel +from pydantic import Field + + +class TimedBaseModel(BaseModel): + created_at: datetime = Field(None, alias="createdAt") + updated_at: datetime = Field(None, alias="updatedAt") + diff --git a/src/superannotate/lib/core/entities/integrations.py b/src/superannotate/lib/core/entities/integrations.py new file mode 100644 index 000000000..b4016daf7 --- /dev/null +++ b/src/superannotate/lib/core/entities/integrations.py @@ -0,0 +1,14 @@ +from pydantic import Field +from lib.core.entities.base import TimedBaseModel + + +class IntegrationEntity(TimedBaseModel): + id: int = None + user_id: str = None + name: str + type: str = "aws" + root: str = Field(None, alias="bucket_name") + source: int = None + + class Config: + arbitrary_types_allowed = True diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 0a6f294c0..2503d1426 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -12,20 +12,20 @@ class SuerannotateServiceProvider: @abstractmethod def attach_files( - self, - project_id: int, - folder_id: int, - team_id: int, - files: List[Dict], - annotation_status_code: int, - upload_state_code: int, - meta: Dict, + self, + project_id: int, + folder_id: int, + team_id: int, + files: List[Dict], + annotation_status_code: int, + upload_state_code: int, + meta: Dict, ): raise NotImplementedError @abstractmethod def get_annotation_classes( - self, project_id: int, team_id: int, name_prefix: str = None + self, project_id: int, team_id: int, name_prefix: str = None ): raise NotImplementedError @@ -35,19 +35,19 @@ def share_project_bulk(self, project_id: int, team_id: int, users: Iterable): @abstractmethod def invite_contributors( - self, team_id: int, team_role: int, emails: Iterable + self, team_id: int, team_role: int, emails: Iterable ) -> Tuple[List[str], List[str]]: raise NotImplementedError @abstractmethod def prepare_export( - self, - project_id: int, - team_id: int, - folders: List[str], - annotation_statuses: Iterable[Any], - include_fuse: bool, - only_pinned: bool, + self, + project_id: int, + team_id: int, + folders: List[str], + annotation_statuses: Iterable[Any], + include_fuse: bool, + only_pinned: bool, ): raise NotImplementedError @@ -99,17 +99,17 @@ def update_folder(self, project_id: int, team_id: int, folder_data: dict): raise NotImplementedError def get_download_token( - self, - project_id: int, - team_id: int, - folder_id: int, - image_id: int, - include_original: int = 1, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, + include_original: int = 1, ) -> dict: raise NotImplementedError def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, project_id: int, team_id: int, folder_id: int, image_id: int, ) -> dict: raise NotImplementedError @@ -117,24 +117,24 @@ def update_image(self, image_id: int, team_id: int, project_id: int, data: dict) raise NotImplementedError def copy_images_between_folders_transaction( - self, - team_id: int, - project_id: int, - from_folder_id: int, - to_folder_id: int, - images: List[str], - include_annotations: bool = False, - include_pin: bool = False, + self, + team_id: int, + project_id: int, + from_folder_id: int, + to_folder_id: int, + images: List[str], + include_annotations: bool = False, + include_pin: bool = False, ) -> int: raise NotImplementedError def move_images_between_folders( - self, - team_id: int, - project_id: int, - from_folder_id: int, - to_folder_id: int, - images: List[str], + self, + team_id: int, + project_id: int, + from_folder_id: int, + to_folder_id: int, + images: List[str], ) -> List[str]: """ Returns list of moved images. @@ -142,22 +142,22 @@ def move_images_between_folders( raise NotImplementedError def get_duplicated_images( - self, project_id: int, team_id: int, folder_id: int, images: List[str] + self, project_id: int, team_id: int, folder_id: int, images: List[str] ): raise NotImplementedError def get_progress( - self, project_id: int, team_id: int, poll_id: int + self, project_id: int, team_id: int, poll_id: int ) -> Tuple[int, int]: raise NotImplementedError def set_images_statuses_bulk( - self, - image_names: List[str], - team_id: int, - project_id: int, - folder_id: int, - annotation_status: int, + self, + image_names: List[str], + team_id: int, + project_id: int, + folder_id: int, + annotation_status: int, ): raise NotImplementedError @@ -165,49 +165,49 @@ def delete_images(self, project_id: int, team_id: int, image_ids: List[int]): raise NotImplementedError def assign_images( - self, - team_id: int, - project_id: int, - folder_name: str, - user: str, - image_names: list, + self, + team_id: int, + project_id: int, + folder_name: str, + user: str, + image_names: list, ): raise NotImplementedError def get_bulk_images( - self, project_id: int, team_id: int, folder_id: int, images: List[str] + self, project_id: int, team_id: int, folder_id: int, images: List[str] ) -> List[dict]: raise NotImplementedError def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, team_id: int, project_id: int, folder_name: str, ): raise NotImplementedError def assign_folder( - self, team_id: int, project_id: int, folder_name: str, users: list + self, team_id: int, project_id: int, folder_name: str, users: list ): raise NotImplementedError def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: list, + self, team_id: int, project_id: int, folder_name: str, image_names: list, ): raise NotImplementedError def un_share_project( - self, team_id: int, project_id: int, user_id: str, + self, team_id: int, project_id: int, user_id: str, ): raise NotImplementedError def upload_form_s3( - self, - project_id: int, - team_id: int, - access_key: str, - secret_key: str, - bucket_name: str, - from_folder_name: str, - to_folder_id: int, + self, + project_id: int, + team_id: int, + access_key: str, + secret_key: str, + bucket_name: str, + from_folder_name: str, + to_folder_id: int, ): raise NotImplementedError @@ -227,7 +227,7 @@ def get_s3_upload_auth_token(self, team_id: int, folder_id: int, project_id: int raise NotImplementedError def delete_annotation_class( - self, team_id: int, project_id: int, annotation_class_id: int + self, team_id: int, project_id: int, annotation_class_id: int ): raise NotImplementedError @@ -238,17 +238,17 @@ def set_project_workflow_bulk(self, project_id: int, team_id: int, steps: list): raise NotImplementedError def set_project_workflow_attributes_bulk( - self, project_id: int, team_id: int, attributes: list + self, project_id: int, team_id: int, attributes: list ): raise NotImplementedError def get_pre_annotation_upload_data( - self, project_id: int, team_id: int, image_ids: List[int], folder_id: int + self, project_id: int, team_id: int, image_ids: List[int], folder_id: int ): raise NotImplementedError def get_annotation_upload_data( - self, project_id: int, team_id: int, image_ids: List[int], folder_id: int + self, project_id: int, team_id: int, image_ids: List[int], folder_id: int ) -> ServiceResponse: raise NotImplementedError @@ -262,7 +262,7 @@ def get_model_metrics(self, team_id: int, model_id: int) -> dict: raise NotImplementedError def get_models( - self, name: str, team_id: int, project_id: int, model_type: str + self, name: str, team_id: int, project_id: int, model_type: str ) -> List: raise NotImplementedError @@ -276,31 +276,31 @@ def delete_model(self, team_id: int, model_id: int): raise NotImplementedError def get_ml_model_download_tokens( - self, team_id: int, model_id: int + self, team_id: int, model_id: int ) -> ServiceResponse: raise NotImplementedError def run_prediction( - self, team_id: int, project_id: int, ml_model_id: int, image_ids: list + self, team_id: int, project_id: int, ml_model_id: int, image_ids: list ): raise NotImplementedError def delete_image_annotations( - self, - team_id: int, - project_id: int, - folder_id: int = None, - image_names: List[str] = None, + self, + team_id: int, + project_id: int, + folder_id: int = None, + image_names: List[str] = None, ) -> dict: raise NotImplementedError def get_annotations_delete_progress( - self, team_id: int, project_id: int, poll_id: int + self, team_id: int, project_id: int, poll_id: int ): raise NotImplementedError def get_limitations( - self, team_id: int, project_id: int, folder_id: int = None + self, team_id: int, project_id: int, folder_id: int = None ) -> ServiceResponse: raise NotImplementedError @@ -319,3 +319,10 @@ def upload_priority_scores( self, team_id: int, project_id: int, folder_id: int, priorities: list ) -> dict: raise NotImplementedError + + def get_integrations(self, team_id: int) -> List[dict]: + raise NotImplementedError + + def attach_integrations(self, team_id: int, project_id: int, integration_id: int, folder_id: int, + folder_name: str) -> bool: + raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/__init__.py b/src/superannotate/lib/core/usecases/__init__.py index 07c42e079..0dfc971cd 100644 --- a/src/superannotate/lib/core/usecases/__init__.py +++ b/src/superannotate/lib/core/usecases/__init__.py @@ -3,3 +3,4 @@ from lib.core.usecases.images import * # noqa: F403 F401 from lib.core.usecases.models import * # noqa: F403 F401 from lib.core.usecases.projects import * # noqa: F403 F401 +from lib.core.usecases.integrations import * # noqa: F403 F401 diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 3814620a5..079fd6d8a 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -668,7 +668,7 @@ def folder_path(self): @property def uploading_info(self): data_len: int = len(self._scores) - return f"Uploading priority scores for {data_len} item{'(s)' if data_len < 1 else ''} to {self.folder_path}." + return f"Uploading priority scores for {data_len} item(s) to {self.folder_path}." def execute(self): if self.is_valid(): diff --git a/src/superannotate/lib/core/usecases/integrations.py b/src/superannotate/lib/core/usecases/integrations.py new file mode 100644 index 000000000..9d1d48719 --- /dev/null +++ b/src/superannotate/lib/core/usecases/integrations.py @@ -0,0 +1,81 @@ +from lib.core.usecases import BaseReportableUseCae +from lib.core.reporter import Reporter +from lib.core.entities import IntegrationEntity +from lib.core.entities import TeamEntity +from lib.core.entities import FolderEntity +from lib.core.entities import ProjectEntity +from lib.core.response import Response +from lib.core.exceptions import AppException +from lib.core.repositories import BaseReadOnlyRepository +from lib.core.serviceproviders import SuerannotateServiceProvider + + +class GetIntegrations(BaseReportableUseCae): + def __init__( + self, + reporter: Reporter, + team: TeamEntity, + integrations: BaseReadOnlyRepository, + ): + + super().__init__(reporter) + self._team = team + self._integrations = integrations + + def execute(self) -> Response: + integrations = self._integrations.get_all() + integrations = list(sorted(integrations, key=lambda x: x.created_at)) + integrations.reverse() + self._response.data = integrations + return self._response + + +class AttachIntegrations(BaseReportableUseCae): + def __init__( + self, + reporter: Reporter, + team: TeamEntity, + project: ProjectEntity, + folder: FolderEntity, + integrations: BaseReadOnlyRepository, + backend_service: SuerannotateServiceProvider, + integration: IntegrationEntity, + folder_path: str = None + ): + + super().__init__(reporter) + self._project = project + self._folder = folder + self._team = team + self._integration = integration + self._client = backend_service + self._folder_path = folder_path + self._integrations = integrations + + def execute(self) -> Response: + integrations = self._integrations.get_all() + integration_name_lower = self._integration.name.lower() + integration = next((i for i in integrations if i.name.lower() == integration_name_lower), None) + if integration: + self.reporter.log_info( + "Attaching file(s) from " + f"{integration.root}{f'/{self._folder.name}' if self._folder.name != 'root' else ''} " + f"to {self._project.name}. This may take some time." + ) + attached = self._client.attach_integrations( + self._team.uuid, + self._project.uuid, + integration.id, + self._folder.uuid, + self._folder_path + ) + if not attached: + self._response.errors = AppException( + f"An error occurred for {self._integration.name}. Please make sure: " + "\n - The bucket exists." + "\n - The connection is valid." + "\n - The path to a specified directory is correct." + ) + else: + self._response.errors = AppException("Integration not found.") + return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 67fb78b2e..f38e53193 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -9,15 +9,18 @@ from typing import Optional from typing import Union +from superannotate_schemas.validators import AnnotationValidators + import lib.core as constances from lib.core import usecases -from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ +from lib.core.conditions import Condition from lib.core.entities import AnnotationClassEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity from lib.core.entities import MLModelEntity from lib.core.entities import ProjectEntity +from lib.core.entities.integrations import IntegrationEntity from lib.core.exceptions import AppException from lib.core.reporter import Reporter from lib.core.response import Response @@ -26,6 +29,7 @@ from lib.infrastructure.repositories import ConfigRepository from lib.infrastructure.repositories import FolderRepository from lib.infrastructure.repositories import ImageRepository +from lib.infrastructure.repositories import IntegrationRepository from lib.infrastructure.repositories import MLModelRepository from lib.infrastructure.repositories import ProjectRepository from lib.infrastructure.repositories import ProjectSettingsRepository @@ -34,7 +38,6 @@ from lib.infrastructure.repositories import WorkflowRepository from lib.infrastructure.services import SuperannotateBackendService from superannotate.logger import get_default_logger -from superannotate_schemas.validators import AnnotationValidators class BaseController(metaclass=ABCMeta): @@ -52,6 +55,7 @@ def __init__(self, config_path: str = None, token: str = None): self._folders = None self._teams = None self._images = None + self._integrations = None self._ml_models = None self._user_id = None self._team_name = None @@ -182,6 +186,11 @@ def images(self): self._images = ImageRepository(self._backend_client) return self._images + def get_integrations_repo(self, team_id: int): + if not self._integrations: + self._integrations = IntegrationRepository(self._backend_client, team_id) + return self._integrations + @property def configs(self): return ConfigRepository(self._config_path) @@ -277,7 +286,7 @@ def get_folder_name(name: str = None): return "root" def search_project( - self, name: str = None, include_complete_image_count=False + self, name: str = None, include_complete_image_count=False ) -> Response: condition = Condition.get_empty_condition() if name: @@ -290,13 +299,13 @@ def search_project( return use_case.execute() def create_project( - self, - name: str, - description: str, - project_type: str, - settings: Iterable = tuple(), - annotation_classes: Iterable = tuple(), - workflows: Iterable = tuple(), + self, + name: str, + description: str, + project_type: str, + settings: Iterable = tuple(), + annotation_classes: Iterable = tuple(), + workflows: Iterable = tuple(), ) -> Response: try: @@ -341,14 +350,14 @@ def update_project(self, name: str, project_data: dict) -> Response: return use_case.execute() def upload_image_to_project( - self, - project_name: str, - folder_name: str, - image_name: str, - image: Union[str, io.BytesIO] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + image_name: str, + image: Union[str, io.BytesIO] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -376,13 +385,13 @@ def upload_image_to_project( ).execute() def upload_images_to_project( - self, - project_name: str, - folder_name: str, - paths: List[str], - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + paths: List[str], + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -402,16 +411,16 @@ def upload_images_to_project( ) def upload_images_from_folder_to_project( - self, - project_name: str, - folder_name: str, - folder_path: str, - extensions: Optional[List[str]] = None, - annotation_status: str = None, - exclude_file_patterns: Optional[List[str]] = None, - recursive_sub_folders: Optional[bool] = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + folder_path: str, + extensions: Optional[List[str]] = None, + annotation_status: str = None, + exclude_file_patterns: Optional[List[str]] = None, + recursive_sub_folders: Optional[bool] = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -434,13 +443,13 @@ def upload_images_from_folder_to_project( ) def upload_images_from_public_urls_to_project( - self, - project_name: str, - folder_name: str, - image_urls: List[str], - image_names: List[str] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, + self, + project_name: str, + folder_name: str, + image_urls: List[str], + image_names: List[str] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -459,14 +468,14 @@ def upload_images_from_public_urls_to_project( ) def clone_project( - self, - name: str, - from_name: str, - project_description: str, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, + self, + name: str, + from_name: str, + project_description: str, + copy_annotation_classes=True, + copy_settings=True, + copy_workflow=True, + copy_contributors=False, ): project = self._get_project(from_name) @@ -492,12 +501,12 @@ def clone_project( return use_case.execute() def interactive_attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, + self, + project_name: str, + files: List[ImageEntity], + folder_name: str = None, + annotation_status: str = None, + upload_state_code: int = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -532,7 +541,7 @@ def get_folder(self, project_name: str, folder_name: str): return use_case.execute() def search_folders( - self, project_name: str, folder_name: str = None, include_users=False, **kwargs + self, project_name: str, folder_name: str = None, include_users=False, **kwargs ): condition = Condition.get_empty_condition() if kwargs: @@ -562,12 +571,12 @@ def delete_folders(self, project_name: str, folder_names: List[str]): return use_case.execute() def prepare_export( - self, - project_name: str, - folder_names: List[str], - include_fuse: bool, - only_pinned: bool, - annotation_statuses: List[str] = None, + self, + project_name: str, + folder_names: List[str], + include_fuse: bool, + only_pinned: bool, + annotation_statuses: List[str] = None, ): project = self._get_project(project_name) @@ -599,11 +608,11 @@ def search_team_contributors(self, **kwargs): return use_case.execute() def search_images( - self, - project_name: str, - folder_path: str = None, - annotation_status: str = None, - image_name_prefix: str = None, + self, + project_name: str, + folder_path: str = None, + annotation_status: str = None, + image_name_prefix: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -618,7 +627,7 @@ def search_images( return use_case.execute() def _get_image( - self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, + self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, ) -> ImageEntity: response = usecases.GetImageUseCase( service=self._backend_client, @@ -632,7 +641,7 @@ def _get_image( return response.data def get_image( - self, project_name: str, image_name: str, folder_path: str = None + self, project_name: str, image_name: str, folder_path: str = None ) -> ImageEntity: project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -643,18 +652,18 @@ def update_folder(self, project_name: str, folder_name: str, folder_data: dict): folder = self._get_folder(project, folder_name) for field, value in folder_data.items(): setattr(folder, field, value) - use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder,) + use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder, ) return use_case.execute() def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, + copy_annotation_status: bool = False, + move: bool = False, ): from_project = self._get_project(from_project_name) to_project = self._get_project(to_project_name) @@ -677,12 +686,12 @@ def copy_image( return use_case.execute() def copy_image_annotation_classes( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, ): from_project = self._get_project(from_project_name) from_folder = self._get_folder(from_project, from_folder_name) @@ -717,7 +726,7 @@ def copy_image_annotation_classes( return use_case.execute() def update_image( - self, project_name: str, image_name: str, folder_name: str = None, **kwargs + self, project_name: str, image_name: str, folder_name: str = None, **kwargs ): image = self.get_image( project_name=project_name, image_name=image_name, folder_path=folder_name @@ -728,7 +737,7 @@ def update_image( return use_case.execute() def download_image_from_public_url( - self, project_name: str, image_url: str, image_name: str = None + self, project_name: str, image_url: str, image_name: str = None ): use_case = usecases.DownloadImageFromPublicUrlUseCase( project=self._get_project(project_name), @@ -738,13 +747,13 @@ def download_image_from_public_url( return use_case.execute() def bulk_copy_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], - include_annotations: bool, - include_pin: bool, + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], + include_annotations: bool, + include_pin: bool, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -761,11 +770,11 @@ def bulk_copy_images( return use_case.execute() def bulk_move_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -780,13 +789,13 @@ def bulk_move_images( return use_case.execute() def get_project_metadata( - self, - project_name: str, - include_annotation_classes: bool = False, - include_settings: bool = False, - include_workflow: bool = False, - include_contributors: bool = False, - include_complete_image_count: bool = False, + self, + project_name: str, + include_annotation_classes: bool = False, + include_settings: bool = False, + include_workflow: bool = False, + include_contributors: bool = False, + include_complete_image_count: bool = False, ): project = self._get_project(project_name) @@ -868,11 +877,11 @@ def get_image_metadata(self, project_name: str, folder_name: str, image_name: st return use_case.execute() def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, + self, + project_name: str, + folder_name: str, + image_names: list, + annotation_status: str, ): project_entity = self._get_project(project_name) folder_entity = self._get_folder(project_entity, folder_name) @@ -890,7 +899,7 @@ def set_images_annotation_statuses( return use_case.execute() def delete_images( - self, project_name: str, folder_name: str, image_names: List[str] = None, + self, project_name: str, folder_name: str, image_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -905,7 +914,7 @@ def delete_images( return use_case.execute() def assign_images( - self, project_name: str, folder_name: str, image_names: list, user: str + self, project_name: str, folder_name: str, image_names: list, user: str ): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) @@ -968,7 +977,7 @@ def un_share_project(self, project_name: str, user_id: str): return use_case.execute() def get_image_annotations( - self, project_name: str, folder_name: str, image_name: str + self, project_name: str, folder_name: str, image_name: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -984,7 +993,7 @@ def get_image_annotations( return use_case.execute() def download_image_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -1002,7 +1011,7 @@ def download_image_annotations( return use_case.execute() def download_image_pre_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -1036,14 +1045,14 @@ def get_exports(self, project_name: str, return_metadata: bool): return use_case.execute() def backend_upload_from_s3( - self, - project_name: str, - folder_name: str, - access_key: str, - secret_key: str, - bucket_name: str, - folder_path: str, - image_quality: str, + self, + project_name: str, + folder_name: str, + access_key: str, + secret_key: str, + bucket_name: str, + folder_path: str, + image_quality: str, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1061,7 +1070,7 @@ def backend_upload_from_s3( return use_case.execute() def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool + self, project_name: str, folder_name: str, with_all_subfolders: bool ): project = self._get_project(project_name) @@ -1077,17 +1086,17 @@ def get_project_image_count( return use_case.execute() def extract_video_frames( - self, - project_name: str, - folder_name: str, - video_path: str, - extract_path: str, - start_time: float, - end_time: float = None, - target_fps: float = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - limit: int = None, + self, + project_name: str, + folder_name: str, + video_path: str, + extract_path: str, + start_time: float, + end_time: float = None, + target_fps: float = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + limit: int = None, ): annotation_status_code = ( constances.AnnotationStatus.get_value(annotation_status) @@ -1126,12 +1135,12 @@ def get_duplicate_images(self, project_name: str, folder_name: str, images: list ) def create_annotation_class( - self, - project_name: str, - name: str, - color: str, - attribute_groups: List[dict], - class_type: str, + self, + project_name: str, + name: str, + color: str, + attribute_groups: List[dict], + class_type: str, ): project = self._get_project(project_name) annotation_classes = AnnotationClassRepository( @@ -1194,15 +1203,15 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) return use_case.execute() def download_image( - self, - project_name: str, - image_name: str, - download_path: str, - folder_name: str = None, - image_variant: str = None, - include_annotations: bool = None, - include_fuse: bool = None, - include_overlay: bool = None, + self, + project_name: str, + image_name: str, + download_path: str, + folder_name: str = None, + image_variant: str = None, + include_annotations: bool = None, + include_fuse: bool = None, + include_overlay: bool = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1242,13 +1251,13 @@ def set_project_workflow(self, project_name: str, steps: list): return use_case.execute() def upload_annotations_from_folder( - self, - project_name: str, - folder_name: str, - annotation_paths: List[str], - client_s3_bucket=None, - is_pre_annotations: bool = False, - folder_path: str = None, + self, + project_name: str, + folder_name: str, + annotation_paths: List[str], + client_s3_bucket=None, + is_pre_annotations: bool = False, + folder_path: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1274,13 +1283,13 @@ def upload_annotations_from_folder( return use_case.execute() def upload_image_annotations( - self, - project_name: str, - folder_name: str, - image_name: str, - annotations: dict, - mask: io.BytesIO = None, - verbose: bool = True, + self, + project_name: str, + folder_name: str, + image_name: str, + annotations: dict, + mask: io.BytesIO = None, + verbose: bool = True, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1310,14 +1319,14 @@ def upload_image_annotations( return use_case.execute() def create_model( - self, - model_name: str, - model_description: str, - task: str, - base_model_name: str, - train_data_paths: Iterable[str], - test_data_paths: Iterable[str], - hyper_parameters: dict, + self, + model_name: str, + model_description: str, + task: str, + base_model_name: str, + train_data_paths: Iterable[str], + test_data_paths: Iterable[str], + hyper_parameters: dict, ): use_case = usecases.CreateModelUseCase( base_model_name=base_model_name, @@ -1353,12 +1362,12 @@ def delete_model(self, model_id: int): return use_case.execute() def download_export( - self, - project_name: str, - export_name: str, - folder_path: str, - extract_zip_contents: bool, - to_s3_bucket: bool, + self, + project_name: str, + export_name: str, + folder_path: str, + extract_zip_contents: bool, + to_s3_bucket: bool, ): project = self._get_project(project_name) return usecases.DownloadExportUseCase( @@ -1389,14 +1398,14 @@ def download_ml_model(self, model_data: dict, download_path: str): return use_case.execute() def benchmark( - self, - project_name: str, - ground_truth_folder_name: str, - folder_names: List[str], - export_root: str, - image_list: List[str], - annot_type: str, - show_plots: bool, + self, + project_name: str, + ground_truth_folder_name: str, + folder_names: List[str], + export_root: str, + image_list: List[str], + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1432,13 +1441,13 @@ def benchmark( return use_case.execute() def consensus( - self, - project_name: str, - folder_names: list, - export_path: str, - image_list: list, - annot_type: str, - show_plots: bool, + self, + project_name: str, + folder_names: list, + export_path: str, + image_list: list, + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1472,7 +1481,7 @@ def consensus( return use_case.execute() def run_prediction( - self, project_name: str, images_list: list, model_name: str, folder_name: str + self, project_name: str, images_list: list, model_name: str, folder_name: str ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1490,7 +1499,7 @@ def run_prediction( return use_case.execute() def list_images( - self, project_name: str, annotation_status: str = None, name_prefix: str = None, + self, project_name: str, annotation_status: str = None, name_prefix: str = None, ): project = self._get_project(project_name) @@ -1503,12 +1512,12 @@ def list_images( return use_case.execute() def search_models( - self, - name: str, - model_type: str = None, - project_id: int = None, - task: str = None, - include_global: bool = True, + self, + name: str, + model_type: str = None, + project_id: int = None, + task: str = None, + include_global: bool = True, ): ml_models_repo = MLModelRepository( service=self._backend_client, team_id=self.team_id @@ -1531,10 +1540,10 @@ def search_models( return use_case.execute() def delete_annotations( - self, - project_name: str, - folder_name: str, - image_names: Optional[List[str]] = None, + self, + project_name: str, + folder_name: str, + image_names: Optional[List[str]] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1548,7 +1557,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = False ): use_case = usecases.ValidateAnnotationUseCase( project_type, @@ -1585,17 +1594,17 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): return use_case.execute() def upload_videos( - self, - project_name: str, - folder_name: str, - paths: List[str], - start_time: float, - extensions: List[str] = None, - exclude_file_patterns: List[str] = None, - end_time: Optional[float] = None, - target_fps: Optional[int] = None, - annotation_status: Optional[str] = None, - image_quality_in_editor: Optional[str] = None, + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1661,3 +1670,29 @@ def upload_priority_scores(self, project_name, folder_name, scores, project_fold project_folder_name=project_folder_name ) return use_case.execute() + + def get_integrations(self): + team = self.team_data.data + use_cae = usecases.GetIntegrations( + reporter=self.default_reporter, + team=self.team_data.data, + integrations=self.get_integrations_repo(team_id=team.uuid) + ) + return use_cae.execute() + + def attach_integrations(self, project_name: str, folder_name: str, integration: IntegrationEntity, + folder_path: str): + team = self.team_data.data + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + use_case = usecases.AttachIntegrations( + reporter=self.default_reporter, + team=self.team_data.data, + backend_service=self.backend_client, + project=project, + folder=folder, + integrations=self.get_integrations_repo(team_id=team.uuid), + integration=integration, + folder_path=folder_path + ) + return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/repositories.py b/src/superannotate/lib/infrastructure/repositories.py index 0fd0aa177..567278d0a 100644 --- a/src/superannotate/lib/infrastructure/repositories.py +++ b/src/superannotate/lib/infrastructure/repositories.py @@ -18,6 +18,7 @@ from lib.core.entities import S3FileEntity from lib.core.entities import TeamEntity from lib.core.entities import UserEntity +from lib.core.entities import IntegrationEntity from lib.core.entities import WorkflowEntity from lib.core.enums import ClassTypeEnum from lib.core.enums import ImageQuality @@ -493,3 +494,19 @@ def dict2entity(data: dict): is_global=data["is_global"], training_status=data["training_status"], ) + + +class IntegrationRepository(BaseReadOnlyRepository): + def __init__(self, service: SuperannotateBackendService, team_id: int): + self._service = service + self._team_id = team_id + + def get_one(self, uuid: int) -> Optional[TeamEntity]: + raise NotImplementedError + + def get_all(self, condition: Optional[Condition] = None) -> List[IntegrationEntity]: + return [self.dict2entity(integration) for integration in self._service.get_integrations(self._team_id)] + + @staticmethod + def dict2entity(data: dict) -> IntegrationEntity: + return IntegrationEntity(**data) \ No newline at end of file diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index f1369eaff..d722e1670 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -225,6 +225,8 @@ class SuperannotateBackendService(BaseBackendService): URL_GET_LIMITS = "project/{}/limitationDetails" URL_GET_ANNOTATIONS = "images/annotations/stream" URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" + URL_GET_INTEGRATIONS = "integrations" + URL_ATTACH_INTEGRATIONS = "image/integration/create" def upload_priority_scores( self, team_id: int, project_id: int, folder_id: int, priorities: list @@ -1059,3 +1061,32 @@ def get_annotations( chunk_size=self.DEFAULT_CHUNK_SIZE, map_function=lambda x: {"image_names": x} )) + + def get_integrations(self, team_id: int) -> List[dict]: + get_integrations_url = urljoin(self.api_url, self.URL_GET_INTEGRATIONS.format(team_id)) + + response = self._request( + get_integrations_url, + "get", + params={"team_id": team_id} + ) + if response.ok: + return response.json().get("integrations", []) + return [] + + def attach_integrations(self, team_id: int, project_id: int, integration_id: int, folder_id: int, folder_name: str = None) -> bool: + attach_integrations_url = urljoin(self.api_url, self.URL_ATTACH_INTEGRATIONS.format(team_id)) + data = { + "team_id": team_id, + "project_id": project_id, + "folder_id": folder_id, + "integration_id": integration_id + } + if folder_name: + data["customer_folder_name"] = folder_name + response = self._request( + attach_integrations_url, + "post", + data=data + ) + return response.ok diff --git a/tests/convertors/test_labelbox.py b/tests/convertors/test_labelbox.py index a2e4b16fc..a3ca4eab7 100644 --- a/tests/convertors/test_labelbox.py +++ b/tests/convertors/test_labelbox.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest + import superannotate as sa @@ -9,11 +10,11 @@ def test_labelbox_convert_vector(tmpdir): project_name = "labelbox_vector_annotation" input_dir = ( - Path("tests") - / "converter_test" - / "LabelBox" - / "vector_annotations" - / "toSuperAnnotate" + Path("tests") + / "converter_test" + / "LabelBox" + / "vector_annotations" + / "toSuperAnnotate" ) out_dir = Path(tmpdir) / project_name dataset_name = "labelbox_example" @@ -31,11 +32,11 @@ def test_labelbox_convert_object(tmpdir): project_name = "labelbox_object_vector" input_dir = ( - Path("tests") - / "converter_test" - / "LabelBox" - / "vector_annotations" - / "toSuperAnnotate" + Path("tests") + / "converter_test" + / "LabelBox" + / "vector_annotations" + / "toSuperAnnotate" ) out_dir = Path(tmpdir) / project_name dataset_name = "labelbox_example" @@ -53,11 +54,11 @@ def test_labelbox_convert_instance(tmpdir): project_name = "labelbox_vector_instance" input_dir = ( - Path("tests") - / "converter_test" - / "LabelBox" - / "vector_annotations" - / "toSuperAnnotate" + Path("tests") + / "converter_test" + / "LabelBox" + / "vector_annotations" + / "toSuperAnnotate" ) out_dir = Path(tmpdir) / project_name dataset_name = "labelbox_example" @@ -75,11 +76,11 @@ def test_labelbox_convert_instance_pixel(tmpdir): project_name = "labelbox_pixel_instance" input_dir = ( - Path("tests") - / "converter_test" - / "LabelBox" - / "instance_segmentation" - / "toSuperAnnotate" + Path("tests") + / "converter_test" + / "LabelBox" + / "instance_segmentation" + / "toSuperAnnotate" ) out_dir = Path(tmpdir) / project_name dataset_name = "labelbox_example" diff --git a/tests/integration/integrations/test_get_integrations.py b/tests/integration/integrations/test_get_integrations.py new file mode 100644 index 000000000..f3a023fc7 --- /dev/null +++ b/tests/integration/integrations/test_get_integrations.py @@ -0,0 +1,24 @@ +import os +from os.path import dirname + +import pytest +import src.superannotate as sa +from tests.integration.base import BaseTestCase + + +class TestGetIntegrations(BaseTestCase): + PROJECT_NAME = "TestGetIntegrations" + TEST_FOLDER_PATH = "data_set/sample_project_vector" + TEST_FOLDER_NAME = "test_folder" + PROJECT_DESCRIPTION = "desc" + PROJECT_TYPE = "Vector" + EXAMPLE_IMAGE = "example_image_1.jpg" + + @property + def folder_path(self): + return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) + + @pytest.mark.skip(reason="Need to adjust") + def test_get(self): + integrations = sa.get_integrations() + integrations = sa.attach_items_from_integrated_storage(self.PROJECT_NAME, integrations[0]["name"]) From 79aa3cf4a46493f2206d933a96e2895c74dd2072 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Fri, 4 Mar 2022 14:02:02 +0400 Subject: [PATCH 22/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 312635073..b54da73d4 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev5" +__version__ = "4.3.0.dev6" From 29747d0439364bdb6804f1861721134c5bf38d6b Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Fri, 4 Mar 2022 15:15:50 +0400 Subject: [PATCH 23/50] Adjusted integrations --- docs/source/superannotate.sdk.rst | 4 +- src/superannotate/__init__.py | 4 +- .../lib/app/interface/sdk_interface.py | 2 +- .../lib/app/mixp/utils/parsers.py | 15 ++--- src/superannotate/lib/app/serializers.py | 7 +-- .../lib/core/entities/__init__.py | 2 +- src/superannotate/lib/core/entities/base.py | 1 - .../lib/core/entities/integrations.py | 2 +- src/superannotate/lib/core/repositories.py | 4 +- .../lib/core/serviceproviders.py | 2 +- .../lib/core/usecases/__init__.py | 2 +- .../lib/core/usecases/annotations.py | 17 +++--- .../lib/core/usecases/folders.py | 4 +- src/superannotate/lib/core/usecases/images.py | 60 +++++++++---------- .../lib/core/usecases/integrations.py | 26 ++++---- src/superannotate/lib/core/usecases/models.py | 16 ++--- .../lib/core/usecases/projects.py | 24 ++++---- .../lib/infrastructure/controller.py | 5 +- .../lib/infrastructure/repositories.py | 4 +- .../lib/infrastructure/services.py | 12 +++- 20 files changed, 108 insertions(+), 105 deletions(-) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 103be461b..7216c8bae 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -38,6 +38,7 @@ ________ .. autofunction:: superannotate.attach_image_urls_to_project .. autofunction:: superannotate.upload_images_from_public_urls_to_project .. autofunction:: superannotate.attach_document_urls_to_project +.. autofunction:: superannotate.attach_items_from_integrated_storage .. autofunction:: superannotate.upload_image_to_project .. autofunction:: superannotate.delete_annotations .. _ref_upload_images_from_folder_to_project: @@ -108,10 +109,11 @@ __________________ ---------- -Team contributors +Team _________________ .. autofunction:: superannotate.get_team_metadata +.. autofunction:: superannotate.get_integrations .. autofunction:: superannotate.invite_contributors_to_team .. autofunction:: superannotate.search_team_contributors diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 8af10fa64..0ee1b246d 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -21,12 +21,11 @@ from superannotate.lib.app.interface.sdk_interface import aggregate_annotations_as_df from superannotate.lib.app.interface.sdk_interface import assign_folder from superannotate.lib.app.interface.sdk_interface import assign_images -from superannotate.lib.app.interface.sdk_interface import get_integrations -from superannotate.lib.app.interface.sdk_interface import attach_items_from_integrated_storage from superannotate.lib.app.interface.sdk_interface import ( attach_document_urls_to_project, ) from superannotate.lib.app.interface.sdk_interface import attach_image_urls_to_project +from superannotate.lib.app.interface.sdk_interface import attach_items_from_integrated_storage from superannotate.lib.app.interface.sdk_interface import attach_video_urls_to_project from superannotate.lib.app.interface.sdk_interface import benchmark from superannotate.lib.app.interface.sdk_interface import clone_project @@ -58,6 +57,7 @@ from superannotate.lib.app.interface.sdk_interface import get_folder_metadata from superannotate.lib.app.interface.sdk_interface import get_image_annotations from superannotate.lib.app.interface.sdk_interface import get_image_metadata +from superannotate.lib.app.interface.sdk_interface import get_integrations from superannotate.lib.app.interface.sdk_interface import ( get_project_and_folder_metadata, ) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index b5b7b0064..860a862e2 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2958,7 +2958,7 @@ def attach_items_from_integrated_storage( :param integration: existing integration name or metadata dict to pull items from. Mandatory keys in integration metadata’s dict is “name”. - :type integration: str + :type integration: str or dict :param folder_path: Points to an exact folder/directory within given storage. If None, items are fetched from the root directory. diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index 9b9c0ba5f..8fafb8448 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1,7 +1,7 @@ import lib.core as constances from lib.app.helpers import extract_project_folder -from lib.core.enums import ProjectType from lib.core.entities import IntegrationEntity +from lib.core.enums import ProjectType from lib.infrastructure.controller import Controller @@ -1249,15 +1249,9 @@ def get_integrations(*args, **kwargs): def attach_items_from_integrated_storage(*args, **kwargs): - project = kwargs.get("project") - integration = kwargs.get("integration") - folder_path = kwargs.get("folder_path") - if not project: - project = args[0] - if not integration: - integration = args[1] - if not folder_path: - folder_path = args[2] + project = kwargs.get("project", args[0]) + integration = kwargs.get("integration", args[1]) + folder_path = kwargs.get("folder_path", args[2]) project_name, _ = extract_project_folder(project) if isinstance(integration, str): @@ -1271,4 +1265,3 @@ def attach_items_from_integrated_storage(*args, **kwargs): "folder_path": bool(folder_path) }, } - diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index d3d3103ee..983d5623b 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -1,8 +1,8 @@ from abc import ABC +from typing import Any from typing import List from typing import Set from typing import Union -from typing import Any import superannotate.lib.core as constance from pydantic import BaseModel @@ -16,7 +16,7 @@ def __init__(self, entity: BaseEntity): self._entity = entity def serialize(self, fields: List[str] = None, by_alias: bool = True, flat: bool = False): - return self._serialize(self._entity, fields, by_alias, flat) + return self._serialize(self._entity, fields, by_alias, flat) @staticmethod def _serialize(entity: Any, fields: List[str] = None, by_alias: bool = False, flat: bool = False): @@ -28,8 +28,7 @@ def _serialize(entity: Any, fields: List[str] = None, by_alias: bool = False, fl if len(fields) == 1: if flat: return entity.dict(include=fields, by_alias=by_alias)[next(iter(fields))] - else: - return entity.dict(include=fields, by_alias=by_alias) + return entity.dict(include=fields, by_alias=by_alias) return entity.dict(include=fields, by_alias=by_alias) return entity.dict(by_alias=by_alias) return entity.to_dict() diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index 753dddf6b..1fe08e016 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -1,3 +1,4 @@ +from lib.core.entities.integrations import IntegrationEntity from lib.core.entities.project_entities import AnnotationClassEntity from lib.core.entities.project_entities import BaseEntity from lib.core.entities.project_entities import ConfigEntity @@ -11,7 +12,6 @@ from lib.core.entities.project_entities import TeamEntity from lib.core.entities.project_entities import UserEntity from lib.core.entities.project_entities import WorkflowEntity -from lib.core.entities.integrations import IntegrationEntity from superannotate_schemas.schemas.internal.document import DocumentAnnotation from superannotate_schemas.schemas.internal.pixel import PixelAnnotation from superannotate_schemas.schemas.internal.vector import VectorAnnotation diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 740a52042..43c132e37 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -7,4 +7,3 @@ class TimedBaseModel(BaseModel): created_at: datetime = Field(None, alias="createdAt") updated_at: datetime = Field(None, alias="updatedAt") - diff --git a/src/superannotate/lib/core/entities/integrations.py b/src/superannotate/lib/core/entities/integrations.py index b4016daf7..3b48253c1 100644 --- a/src/superannotate/lib/core/entities/integrations.py +++ b/src/superannotate/lib/core/entities/integrations.py @@ -1,5 +1,5 @@ -from pydantic import Field from lib.core.entities.base import TimedBaseModel +from pydantic import Field class IntegrationEntity(TimedBaseModel): diff --git a/src/superannotate/lib/core/repositories.py b/src/superannotate/lib/core/repositories.py index 9d08b5438..58f2a5dac 100644 --- a/src/superannotate/lib/core/repositories.py +++ b/src/superannotate/lib/core/repositories.py @@ -8,7 +8,7 @@ from lib.core.conditions import Condition from lib.core.entities import BaseEntity from lib.core.entities import ProjectEntity -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider class BaseReadOnlyRepository(ABC): @@ -50,7 +50,7 @@ def _drop_nones(data: dict): class BaseProjectRelatedManageableRepository(BaseManageableRepository): - def __init__(self, service: SuerannotateServiceProvider, project: ProjectEntity): + def __init__(self, service: SuperannotateServiceProvider, project: ProjectEntity): self._service = service self._project = project diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 2503d1426..43d69a109 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -9,7 +9,7 @@ from lib.core.service_types import ServiceResponse -class SuerannotateServiceProvider: +class SuperannotateServiceProvider: @abstractmethod def attach_files( self, diff --git a/src/superannotate/lib/core/usecases/__init__.py b/src/superannotate/lib/core/usecases/__init__.py index 0dfc971cd..6dbf5bbc2 100644 --- a/src/superannotate/lib/core/usecases/__init__.py +++ b/src/superannotate/lib/core/usecases/__init__.py @@ -1,6 +1,6 @@ from lib.core.usecases.annotations import * # noqa: F403 F401 from lib.core.usecases.folders import * # noqa: F403 F401 from lib.core.usecases.images import * # noqa: F403 F401 +from lib.core.usecases.integrations import * # noqa: F403 F401 from lib.core.usecases.models import * # noqa: F403 F401 from lib.core.usecases.projects import * # noqa: F403 F401 -from lib.core.usecases.integrations import * # noqa: F403 F401 diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 079fd6d8a..ab8583cfe 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -8,11 +8,9 @@ from typing import Tuple import boto3 -from superannotate_schemas.validators import AnnotationValidators - import lib.core as constances -from lib.core.conditions import CONDITION_EQ as EQ from lib.core.conditions import Condition +from lib.core.conditions import CONDITION_EQ as EQ from lib.core.data_handlers import ChainedAnnotationHandlers from lib.core.data_handlers import DocumentTagHandler from lib.core.data_handlers import LastActionHandler @@ -27,13 +25,14 @@ from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository from lib.core.service_types import UploadAnnotationAuthData -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.types import PriorityScore from lib.core.usecases.base import BaseReportableUseCae from lib.core.usecases.images import GetBulkImages from lib.core.usecases.images import ValidateAnnotationUseCase from lib.core.video_convertor import VideoFrameGenerator from superannotate.logger import get_default_logger +from superannotate_schemas.validators import AnnotationValidators logger = get_default_logger() @@ -53,7 +52,7 @@ def __init__( images: BaseManageableRepository, annotation_classes: List[AnnotationClassEntity], annotation_paths: List[str], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, templates: List[dict], validators: AnnotationValidators, pre_annotation: bool = False, @@ -313,7 +312,7 @@ def __init__( images: BaseManageableRepository, team: TeamEntity, annotation_classes: List[AnnotationClassEntity], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, reporter: Reporter, templates: List[dict], validators: AnnotationValidators, @@ -500,7 +499,7 @@ def __init__( folder: FolderEntity, images: BaseManageableRepository, item_names: Optional[List[str]], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, show_process: bool = True ): super().__init__(reporter) @@ -580,7 +579,7 @@ def __init__( images: BaseManageableRepository, video_name: str, fps: int, - backend_service_provider: SuerannotateServiceProvider + backend_service_provider: SuperannotateServiceProvider ): super().__init__(reporter) self._project = project @@ -639,7 +638,7 @@ def __init__( folder: FolderEntity, scores: List[PriorityScore], project_folder_name: str, - backend_service_provider: SuerannotateServiceProvider + backend_service_provider: SuperannotateServiceProvider ): super().__init__(reporter) self._project = project diff --git a/src/superannotate/lib/core/usecases/folders.py b/src/superannotate/lib/core/usecases/folders.py index 70d99d10e..b64f26cd0 100644 --- a/src/superannotate/lib/core/usecases/folders.py +++ b/src/superannotate/lib/core/usecases/folders.py @@ -9,7 +9,7 @@ from lib.core.exceptions import AppValidationException from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseUseCase from superannotate.logger import get_default_logger @@ -181,7 +181,7 @@ def execute(self): class AssignFolderUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project_entity: ProjectEntity, folder: FolderEntity, users: List[str], diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 0e583e6c2..c2fefe430 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -40,7 +40,7 @@ from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.usecases.base import BaseReportableUseCae from lib.core.usecases.base import BaseUseCase @@ -108,7 +108,7 @@ def __init__( folder: FolderEntity, image_name: str, images: BaseReadOnlyRepository, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, ): super().__init__() self._project = project @@ -140,7 +140,7 @@ class GetAllImagesUseCase(BaseUseCase): def __init__( self, project: ProjectEntity, - service_provider: SuerannotateServiceProvider, + service_provider: SuperannotateServiceProvider, annotation_status: str = None, name_prefix: str = None, ): @@ -174,7 +174,7 @@ def execute(self): class GetBulkImages(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project_id: int, team_id: int, folder_id: int, @@ -210,7 +210,7 @@ def __init__( project: ProjectEntity, folder: FolderEntity, attachments: List[ImageEntity], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, annotation_status: str = None, upload_state_code: int = constances.UploadState.EXTERNAL.value, ): @@ -303,7 +303,7 @@ class GetImageBytesUseCase(BaseUseCase): def __init__( self, image: ImageEntity, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, image_variant: str = "original", ): super().__init__() @@ -337,7 +337,7 @@ def __init__( to_project_s3_repo: BaseManageableRepository, to_project_annotation_classes: BaseReadOnlyRepository, from_project_annotation_classes: BaseReadOnlyRepository, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, from_folder: FolderEntity = None, to_folder: FolderEntity = None, annotation_type: str = "MAIN", @@ -574,7 +574,7 @@ def __init__( from_folder: FolderEntity, to_folder: FolderEntity, image_names: List[str], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, include_annotations: bool, include_pin: bool, ): @@ -657,7 +657,7 @@ def __init__( image_name: str, project: ProjectEntity, folder: FolderEntity, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, ): super().__init__() self._image_name = image_name @@ -701,7 +701,7 @@ def __init__( from_folder: FolderEntity, to_folder: FolderEntity, image_names: List[str], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, ): super().__init__() self._project = project @@ -968,7 +968,7 @@ def __init__( image: ImageEntity, images: BaseManageableRepository, classes: BaseManageableRepository, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, annotation_classes: BaseReadOnlyRepository, download_path: str, image_variant: str = "original", @@ -1079,7 +1079,7 @@ def __init__( folder: FolderEntity, s3_repo, settings: BaseManageableRepository, - backend_client: SuerannotateServiceProvider, + backend_client: SuperannotateServiceProvider, annotation_status: str, image_bytes: io.BytesIO = None, image_path: str = None, @@ -1218,7 +1218,7 @@ def __init__( folder: FolderEntity, settings: BaseManageableRepository, s3_repo, - backend_client: SuerannotateServiceProvider, + backend_client: SuperannotateServiceProvider, paths: List[str], extensions=constances.DEFAULT_IMAGE_EXTENSIONS, annotation_status="NotStarted", @@ -1485,7 +1485,7 @@ def __init__( folder: FolderEntity, settings: BaseManageableRepository, s3_repo, - backend_client: SuerannotateServiceProvider, + backend_client: SuperannotateServiceProvider, folder_path: str, extensions=constances.DEFAULT_IMAGE_EXTENSIONS, annotation_status="NotStarted", @@ -1563,7 +1563,7 @@ def __init__( self, project: ProjectEntity, folder: FolderEntity, - backend_service: SuerannotateServiceProvider, + backend_service: SuperannotateServiceProvider, settings: List[ProjectSettingEntity], s3_repo, image_urls: List[str], @@ -1862,7 +1862,7 @@ def __init__( project: ProjectEntity, folder: FolderEntity, attachments: List[ImageEntity], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, annotation_status: str = None, upload_state_code: int = constances.UploadState.EXTERNAL.value, ): @@ -1953,7 +1953,7 @@ def __init__( image_name: str, to_project: ProjectEntity, to_folder: FolderEntity, - backend_service: SuerannotateServiceProvider, + backend_service: SuperannotateServiceProvider, images: BaseManageableRepository, s3_repo, project_settings: List[ProjectSettingEntity], @@ -2101,7 +2101,7 @@ def __init__( self, project: ProjectEntity, folder: FolderEntity, - backend_service: SuerannotateServiceProvider, + backend_service: SuperannotateServiceProvider, image_names: Optional[List[str]] = None, ): super().__init__() @@ -2179,7 +2179,7 @@ def __init__( self, project: ProjectEntity, folder: FolderEntity, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, images: BaseReadOnlyRepository, image_names: List[str] = None, ): @@ -2231,7 +2231,7 @@ def execute(self): class DownloadImageAnnotationsUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, image_name: str, @@ -2393,7 +2393,7 @@ def execute(self): class DownloadImagePreAnnotationsUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, image_name: str, @@ -2472,7 +2472,7 @@ class GetImageAnnotationsUseCase(BaseReportableUseCae): def __init__( self, reporter: Reporter, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, image_name: str, @@ -2553,7 +2553,7 @@ class AssignImagesUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, image_names: list, @@ -2597,7 +2597,7 @@ class UnAssignImagesUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project_entity: ProjectEntity, folder: FolderEntity, image_names: list, @@ -2628,7 +2628,7 @@ def execute(self): class UnAssignFolderUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project_entity: ProjectEntity, folder: FolderEntity, ): @@ -2653,7 +2653,7 @@ class SetImageAnnotationStatuses(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, projects: BaseReadOnlyRepository, image_names: list, team_id: int, @@ -2834,7 +2834,7 @@ class CreateAnnotationClassesUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, annotation_classes_repo: BaseManageableRepository, annotation_classes: List[AnnotationClassEntity], project: ProjectEntity, @@ -2893,7 +2893,7 @@ def execute(self): class ExtractFramesUseCase(BaseInteractiveUseCase): def __init__( self, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, video_path: str, @@ -2997,7 +2997,7 @@ def execute(self): class UploadS3ImagesBackendUseCase(BaseUseCase): def __init__( self, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, settings: BaseReadOnlyRepository, project: ProjectEntity, folder: FolderEntity, @@ -3115,7 +3115,7 @@ class UploadVideosAsImages(BaseReportableUseCae): def __init__( self, reporter: Reporter, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, settings: BaseManageableRepository, diff --git a/src/superannotate/lib/core/usecases/integrations.py b/src/superannotate/lib/core/usecases/integrations.py index 9d1d48719..b92994297 100644 --- a/src/superannotate/lib/core/usecases/integrations.py +++ b/src/superannotate/lib/core/usecases/integrations.py @@ -1,13 +1,15 @@ -from lib.core.usecases import BaseReportableUseCae -from lib.core.reporter import Reporter -from lib.core.entities import IntegrationEntity -from lib.core.entities import TeamEntity +from typing import List + from lib.core.entities import FolderEntity +from lib.core.entities import IntegrationEntity from lib.core.entities import ProjectEntity -from lib.core.response import Response +from lib.core.entities import TeamEntity from lib.core.exceptions import AppException +from lib.core.reporter import Reporter from lib.core.repositories import BaseReadOnlyRepository -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.response import Response +from lib.core.serviceproviders import SuperannotateServiceProvider +from lib.core.usecases import BaseReportableUseCae class GetIntegrations(BaseReportableUseCae): @@ -38,7 +40,7 @@ def __init__( project: ProjectEntity, folder: FolderEntity, integrations: BaseReadOnlyRepository, - backend_service: SuerannotateServiceProvider, + backend_service: SuperannotateServiceProvider, integration: IntegrationEntity, folder_path: str = None ): @@ -52,15 +54,19 @@ def __init__( self._folder_path = folder_path self._integrations = integrations + @property + def _upload_path(self): + return f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}" + def execute(self) -> Response: - integrations = self._integrations.get_all() + integrations: List[IntegrationEntity] = self._integrations.get_all() integration_name_lower = self._integration.name.lower() integration = next((i for i in integrations if i.name.lower() == integration_name_lower), None) if integration: self.reporter.log_info( "Attaching file(s) from " - f"{integration.root}{f'/{self._folder.name}' if self._folder.name != 'root' else ''} " - f"to {self._project.name}. This may take some time." + f"{integration.root}{f'/{self._folder.name}' if self._folder_path else ''} " + f"to {self._upload_path}. This may take some time." ) attached = self._client.attach_integrations( self._team.uuid, diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 5b165eb8a..7952cfdd2 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -25,7 +25,7 @@ from lib.core.exceptions import AppValidationException from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.images import GetBulkImages @@ -40,7 +40,7 @@ def __init__( self, project: ProjectEntity, folder_names: List[str], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, include_fuse: bool, only_pinned: bool, annotation_statuses: List[str] = None, @@ -112,7 +112,7 @@ def execute(self): class GetExportsUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, return_metadata: bool = False, ): @@ -142,7 +142,7 @@ def __init__( team_id: int, train_data_paths: Iterable[str], test_data_paths: Iterable[str], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, projects: BaseReadOnlyRepository, folders: BaseReadOnlyRepository, ml_models: BaseManageableRepository, @@ -286,7 +286,7 @@ def __init__( self, model_id: int, team_id: int, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, ): super().__init__() self._model_id = model_id @@ -329,7 +329,7 @@ def execute(self): class DownloadExportUseCase(BaseInteractiveUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, export_name: str, folder_path: str, @@ -437,7 +437,7 @@ def __init__( self, model: MLModelEntity, download_path: str, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, team_id: int, ): super().__init__() @@ -682,7 +682,7 @@ def __init__( ml_model_repo: BaseManageableRepository, ml_model_name: str, images_list: list, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, folder: FolderEntity, ): super().__init__() diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 40663a291..7708c446e 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -17,7 +17,7 @@ from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseReportableUseCae from lib.core.usecases.base import BaseUseCase from lib.core.usecases.base import BaseUserBasedUseCase @@ -75,7 +75,7 @@ class GetProjectMetaDataUseCase(BaseUseCase): def __init__( self, project: ProjectEntity, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, annotation_classes: BaseManageableRepository, settings: BaseManageableRepository, workflows: BaseManageableRepository, @@ -163,7 +163,7 @@ def __init__( self, project: ProjectEntity, projects: BaseManageableRepository, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, settings_repo: Type[BaseManageableRepository], annotation_classes_repo: BaseManageableRepository, workflows_repo: BaseManageableRepository, @@ -352,7 +352,7 @@ def __init__( settings_repo: Type[BaseManageableRepository], workflows_repo: Type[BaseManageableRepository], annotation_classes_repo: Type[BaseManageableRepository], - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, include_annotation_classes: bool = True, include_settings: bool = True, include_workflow: bool = True, @@ -594,7 +594,7 @@ def execute(self): class ShareProjectUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project_entity: ProjectEntity, user_id: str, user_role: str, @@ -625,7 +625,7 @@ def execute(self): class UnShareProjectUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project_entity: ProjectEntity, user_id: str, ): @@ -712,7 +712,7 @@ def __init__( projects: BaseReadOnlyRepository, settings: BaseManageableRepository, to_update: List, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, project_id: int, team_id: int, ): @@ -773,7 +773,7 @@ def execute(self): class GetProjectImageCountUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, project: ProjectEntity, folder: FolderEntity, with_all_sub_folders: bool = False, @@ -817,7 +817,7 @@ def execute(self): class SetWorkflowUseCase(BaseUseCase): def __init__( self, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, annotation_classes_repo: BaseManageableRepository, workflow_repo: BaseManageableRepository, steps: list, @@ -921,7 +921,7 @@ def execute(self): class SearchContributorsUseCase(BaseUseCase): def __init__( self, - backend_service_provider: SuerannotateServiceProvider, + backend_service_provider: SuperannotateServiceProvider, team_id: int, condition: Condition = None, ): @@ -955,7 +955,7 @@ def __init__( project: ProjectEntity, emails: list, role: str, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, ): super().__init__(reporter, emails) self._team = team @@ -1020,7 +1020,7 @@ def __init__( team: TeamEntity, emails: list, set_admin: bool, - service: SuerannotateServiceProvider, + service: SuperannotateServiceProvider, ): super().__init__(reporter, emails) self._team = team diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f38e53193..f73f665c6 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -9,12 +9,10 @@ from typing import Optional from typing import Union -from superannotate_schemas.validators import AnnotationValidators - import lib.core as constances from lib.core import usecases -from lib.core.conditions import CONDITION_EQ as EQ from lib.core.conditions import Condition +from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import AnnotationClassEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity @@ -38,6 +36,7 @@ from lib.infrastructure.repositories import WorkflowRepository from lib.infrastructure.services import SuperannotateBackendService from superannotate.logger import get_default_logger +from superannotate_schemas.validators import AnnotationValidators class BaseController(metaclass=ABCMeta): diff --git a/src/superannotate/lib/infrastructure/repositories.py b/src/superannotate/lib/infrastructure/repositories.py index 567278d0a..bf49f95fb 100644 --- a/src/superannotate/lib/infrastructure/repositories.py +++ b/src/superannotate/lib/infrastructure/repositories.py @@ -12,13 +12,13 @@ from lib.core.entities import ConfigEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity +from lib.core.entities import IntegrationEntity from lib.core.entities import MLModelEntity from lib.core.entities import ProjectEntity from lib.core.entities import ProjectSettingEntity from lib.core.entities import S3FileEntity from lib.core.entities import TeamEntity from lib.core.entities import UserEntity -from lib.core.entities import IntegrationEntity from lib.core.entities import WorkflowEntity from lib.core.enums import ClassTypeEnum from lib.core.enums import ImageQuality @@ -509,4 +509,4 @@ def get_all(self, condition: Optional[Condition] = None) -> List[IntegrationEnti @staticmethod def dict2entity(data: dict) -> IntegrationEntity: - return IntegrationEntity(**data) \ No newline at end of file + return IntegrationEntity(**data) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index d722e1670..40eed8468 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -18,7 +18,7 @@ from lib.core.service_types import ServiceResponse from lib.core.service_types import UploadAnnotationAuthData from lib.core.service_types import UserLimits -from lib.core.serviceproviders import SuerannotateServiceProvider +from lib.core.serviceproviders import SuperannotateServiceProvider from lib.infrastructure.helpers import timed_lru_cache from lib.infrastructure.stream_data_handler import StreamedAnnotations from requests.exceptions import HTTPError @@ -34,7 +34,7 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) -class BaseBackendService(SuerannotateServiceProvider): +class BaseBackendService(SuperannotateServiceProvider): AUTH_TYPE = "sdk" PAGINATE_BY = 100 LIMIT = 100 @@ -1074,7 +1074,13 @@ def get_integrations(self, team_id: int) -> List[dict]: return response.json().get("integrations", []) return [] - def attach_integrations(self, team_id: int, project_id: int, integration_id: int, folder_id: int, folder_name: str = None) -> bool: + def attach_integrations( + self, + team_id: int, + project_id: int, + integration_id: int, + folder_id: int, + folder_name: str = None) -> bool: attach_integrations_url = urljoin(self.api_url, self.URL_ATTACH_INTEGRATIONS.format(team_id)) data = { "team_id": team_id, From 5f6a7b10d67f2b71c4e31292be9f95bbd0e2a0a3 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Fri, 4 Mar 2022 15:18:14 +0400 Subject: [PATCH 24/50] Update version.py --- src/superannotate/lib/core/usecases/integrations.py | 2 +- src/superannotate/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/core/usecases/integrations.py b/src/superannotate/lib/core/usecases/integrations.py index b92994297..345b734fa 100644 --- a/src/superannotate/lib/core/usecases/integrations.py +++ b/src/superannotate/lib/core/usecases/integrations.py @@ -65,7 +65,7 @@ def execute(self) -> Response: if integration: self.reporter.log_info( "Attaching file(s) from " - f"{integration.root}{f'/{self._folder.name}' if self._folder_path else ''} " + f"{integration.root}{f'/{self._folder_path}' if self._folder_path else ''} " f"to {self._upload_path}. This may take some time." ) attached = self._client.attach_integrations( diff --git a/src/superannotate/version.py b/src/superannotate/version.py index b54da73d4..de4576569 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev6" +__version__ = "4.3.0.dev7" From 0e723fc49f4befbadd3ded4374f15f32535b0e5f Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 7 Mar 2022 10:42:42 +0400 Subject: [PATCH 25/50] Added ProjectStatus Enum --- src/superannotate/lib/app/serializers.py | 4 ++++ src/superannotate/lib/core/__init__.py | 2 ++ src/superannotate/lib/core/enums.py | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index 983d5623b..ce0cb5850 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -71,6 +71,10 @@ class ProjectSerializer(BaseSerializers): def serialize(self): data = super().serialize() data["type"] = constance.ProjectType.get_name(data["type"]) + if data.get("status"): + data["status"] = constance.ProjectStatus.get_name(data["status"]) + else: + data["status"] = "Undefined" if data.get("upload_state"): data["upload_state"] = constance.UploadState(data["upload_state"]).name if data.get("users"): diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index bd77c0536..e72a5caf8 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -1,5 +1,6 @@ from pathlib import Path +from superannotate.lib.core.enums import ProjectStatus from superannotate.lib.core.enums import AnnotationStatus from superannotate.lib.core.enums import ImageQuality from superannotate.lib.core.enums import ProjectType @@ -127,6 +128,7 @@ INVALID_JSON_MESSAGE = "Invalid json" __alL__ = ( + ProjectStatus, ProjectType, UserRole, UploadState, diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index b9314536e..7056ee0d9 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -62,6 +62,13 @@ class ImageQuality(BaseTitledEnum): COMPRESSED = "compressed", 60 +class ProjectStatus(BaseTitledEnum): + NotStarted = "NotStarted", 1 + InProgress = "InProgress", 2 + Completed = "Completed", 3 + OnHold = "OnHold", 4 + + class ExportStatus(BaseTitledEnum): IN_PROGRESS = "inProgress", 1 COMPLETE = "complete", 2 From a67247a19529290882c6fc40ea70722be03a6766 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:43:20 +0400 Subject: [PATCH 26/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index de4576569..99b11aef7 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev7" +__version__ = "4.3.0.dev8" From b1bdbefc632ffe0282236303fa34093da23cc00d Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:45:22 +0400 Subject: [PATCH 27/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 99b11aef7..ff215547f 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev8" +__version__ = "4.3.0.dev9" From 1089d95226266c1a0c6686a185b6c965deea3d6f Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 7 Mar 2022 15:08:38 +0400 Subject: [PATCH 28/50] Friday 840 --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/usecases/projects.py | 7 +------ src/superannotate/lib/infrastructure/controller.py | 4 +++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 860a862e2..1378ec9b8 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -191,7 +191,7 @@ def create_project_from_metadata(project_metadata: Project): project_metadata = project_metadata.dict() response = Controller.get_default().create_project( name=project_metadata["name"], - description=project_metadata["description"], + description=project_metadata.get("description", "..."), project_type=project_metadata["type"], settings=project_metadata.get("settings", []), annotation_classes=project_metadata.get("classes", []), diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 7708c446e..cb80f487b 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -212,14 +212,9 @@ def validate_project_name(self): f"To use SDK please make project names unique." ) - def validate_description(self): - if not self._project.description: - raise AppValidationException("Please provide a project description.") - def execute(self): if self.is_valid(): - # TODO add status in the constants - self._project.status = 0 + self._project.status = constances.ProjectStatus.NotStarted.value entity = self._projects.insert(self._project) self._response.data = entity data = {} diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f73f665c6..13ce9bf3d 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -254,7 +254,7 @@ def set_default(cls, obj): cls.DEFAULT = obj return cls.DEFAULT - def _get_project(self, name: str): + def _get_project(self, name: str) -> ProjectEntity: use_case = usecases.GetProjectByNameUseCase( name=name, team_id=self.team_id, @@ -482,6 +482,8 @@ def clone_project( project_to_create.name = name if project_description: project_to_create.description = project_description + elif not project.description: + project.description = f"Copy of {from_name}." use_case = usecases.CloneProjectUseCase( reporter=self.default_reporter, From 3766bc733076c1e6576b9d60689045115176a368 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:09:15 +0400 Subject: [PATCH 29/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index ff215547f..03ab66921 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev9" +__version__ = "4.3.0.dev10" From d2b24a063eeb515849c7d2295f573835bb4be503 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 7 Mar 2022 15:11:14 +0400 Subject: [PATCH 30/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 03ab66921..4fb5a5636 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev10" +__version__ = "4.3.0.dev11" From e2220d30a4356806edd74e5da6650202d5615daa Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 10 Mar 2022 12:48:37 +0400 Subject: [PATCH 31/50] Added tags handeling, tests adjuted --- requirements_dev.txt | 2 +- .../lib/core/entities/project_entities.py | 2 +- .../lib/core/usecases/projects.py | 4 +-- .../lib/infrastructure/controller.py | 3 --- .../lib/infrastructure/repositories.py | 25 +++++++++++-------- .../lib/infrastructure/services.py | 4 +-- .../text_file_example_1.json | 1 + .../text_file_example_1.json | 1 + .../classes/test_create_annotation_class.py | 6 ++--- .../projects/test_basic_project.py | 2 +- .../test_depricated_functions_document.py | 3 +-- .../test_depricated_functions_video.py | 4 +-- tests/integration/test_df_processing.py | 7 ++++-- 13 files changed, 33 insertions(+), 31 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index d4966b9d7..5a296cca2 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1 +1 @@ -superannotate_schemas>=1.0.38.b1 +superannotate_schemas==1.0.40b2 diff --git a/src/superannotate/lib/core/entities/project_entities.py b/src/superannotate/lib/core/entities/project_entities.py index b56ce0a7e..5ef574dca 100644 --- a/src/superannotate/lib/core/entities/project_entities.py +++ b/src/superannotate/lib/core/entities/project_entities.py @@ -124,7 +124,7 @@ def __copy__(self): team_id=self.team_id, name=self.name, project_type=self.project_type, - description=self.description, + description=self.description if self.description else f"Copy of {self.name}.", status=self.status, folder_id=self.folder_id, users=self.users, diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index cb80f487b..954ee9edf 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -165,8 +165,8 @@ def __init__( projects: BaseManageableRepository, backend_service_provider: SuperannotateServiceProvider, settings_repo: Type[BaseManageableRepository], - annotation_classes_repo: BaseManageableRepository, - workflows_repo: BaseManageableRepository, + annotation_classes_repo: Type[BaseManageableRepository], + workflows_repo: Type[BaseManageableRepository], settings: List[ProjectSettingEntity] = None, workflows: List[WorkflowEntity] = None, annotation_classes: List[AnnotationClassEntity] = None, diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 13ce9bf3d..e4cca9841 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -482,9 +482,6 @@ def clone_project( project_to_create.name = name if project_description: project_to_create.description = project_description - elif not project.description: - project.description = f"Copy of {from_name}." - use_case = usecases.CloneProjectUseCase( reporter=self.default_reporter, project=project, diff --git a/src/superannotate/lib/infrastructure/repositories.py b/src/superannotate/lib/infrastructure/repositories.py index bf49f95fb..c3c8caf9c 100644 --- a/src/superannotate/lib/infrastructure/repositories.py +++ b/src/superannotate/lib/infrastructure/repositories.py @@ -99,6 +99,8 @@ def get_all(self, condition: Condition = None) -> List[ProjectEntity]: def insert(self, entity: ProjectEntity) -> ProjectEntity: project_data = self._drop_nones(entity.to_dict()) + if not project_data.get("status"): + project_data["status"] = constance.ProjectStatus.NotStarted.value result = self._service.create_project(project_data) return self.dict2entity(result) @@ -118,21 +120,21 @@ def delete(self, entity: ProjectEntity): ) @staticmethod - def dict2entity(data: dict): + def dict2entity(data: dict) -> ProjectEntity: try: return ProjectEntity( uuid=data["id"], team_id=data["team_id"], name=data["name"], project_type=data["type"], - status=data["status"], - attachment_name=data["attachment_name"], - attachment_path=data["attachment_path"], - entropy_status=data["entropy_status"], + status=data.get("status"), + attachment_name=data.get("attachment_name"), + attachment_path=data.get("attachment_path"), + entropy_status=data.get("entropy_status"), sharing_status=data.get("sharing_status"), creator_id=data["creator_id"], upload_state=data["upload_state"], - description=data["description"], + description=data.get("description"), folder_id=data.get("folder_id"), users=data.get("users", ()), unverified_users=data.get("unverified_users", ()), @@ -148,6 +150,7 @@ def dict2entity(data: dict): class S3Repository(BaseS3Repository): + def get_one(self, uuid: str) -> S3FileEntity: file = io.BytesIO() self._resource.Object(self._bucket, uuid).download_fileobj(file) @@ -242,7 +245,7 @@ def update(self, entity: WorkflowEntity): raise NotImplementedError @staticmethod - def dict2entity(data: dict): + def dict2entity(data: dict) -> WorkflowEntity: return WorkflowEntity( uuid=data["id"], project_id=data["project_id"], @@ -294,7 +297,7 @@ def bulk_delete(self, entities: List[FolderEntity]): return self._service.delete_folders(entity.project_id, entity.team_id, ids) @staticmethod - def dict2entity(data: dict): + def dict2entity(data: dict) -> FolderEntity: try: return FolderEntity( uuid=data["id"], @@ -397,7 +400,7 @@ def update(self, entity: ImageEntity): return entity @staticmethod - def dict2entity(data: dict): + def dict2entity(data: dict) -> ImageEntity: return ImageEntity( uuid=data["id"], name=data["name"], @@ -416,7 +419,7 @@ def dict2entity(data: dict): class UserRepository(BaseReadOnlyRepository): @staticmethod - def dict2entity(data: dict): + def dict2entity(data: dict) -> UserEntity: return UserEntity( uuid=data["id"], first_name=data["first_name"], @@ -480,7 +483,7 @@ def update(self, entity: MLModelEntity): return self.dict2entity(data) @staticmethod - def dict2entity(data: dict): + def dict2entity(data: dict) -> MLModelEntity: return MLModelEntity( uuid=data["id"], name=data["name"], diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 40eed8468..65c26ce0c 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -318,7 +318,7 @@ def delete_project(self, uuid: int, query_string: str = None) -> bool: res = self._request(url, "delete") return res.ok - def update_project(self, data: dict, query_string: str = None) -> bool: + def update_project(self, data: dict, query_string: str = None) -> dict: url = urljoin(self.api_url, self.URL_GET_PROJECT.format(data["id"])) if query_string: url = f"{url}?{query_string}" @@ -689,8 +689,6 @@ def get_bulk_images( self, project_id: int, team_id: int, folder_id: int, images: List[str] ) -> List[dict]: bulk_get_images_url = urljoin(self.api_url, self.URL_BULK_GET_IMAGES) - time.sleep(1) - res = self._request( bulk_get_images_url, "post", diff --git a/tests/data_set/document_annotation/text_file_example_1.json b/tests/data_set/document_annotation/text_file_example_1.json index 7f058766c..1974ec727 100644 --- a/tests/data_set/document_annotation/text_file_example_1.json +++ b/tests/data_set/document_annotation/text_file_example_1.json @@ -13,6 +13,7 @@ }, "instances": [ { + "type": "entity", "start": 253, "end": 593, "classId": 873208, diff --git a/tests/data_set/document_annotation_without_class_data/text_file_example_1.json b/tests/data_set/document_annotation_without_class_data/text_file_example_1.json index 712c5a7a6..5ef92f8a2 100644 --- a/tests/data_set/document_annotation_without_class_data/text_file_example_1.json +++ b/tests/data_set/document_annotation_without_class_data/text_file_example_1.json @@ -13,6 +13,7 @@ }, "instances": [ { + "type": "entity", "start": 253, "end": 593, "classId": 873208, diff --git a/tests/integration/classes/test_create_annotation_class.py b/tests/integration/classes/test_create_annotation_class.py index ab1ba7e16..dd756b54c 100644 --- a/tests/integration/classes/test_create_annotation_class.py +++ b/tests/integration/classes/test_create_annotation_class.py @@ -8,7 +8,7 @@ class TestCreateAnnotationClass(BaseTestCase): - PROJECT_NAME = "test_create_annotation_class" + PROJECT_NAME = "TestCreateAnnotationClass" PROJECT_TYPE = "Vector" PROJECT_DESCRIPTION = "Example " TEST_LARGE_CLASSES_JSON = "large_classes_json.json" @@ -29,7 +29,7 @@ def test_create_annotations_classes_from_class_json(self): class TestCreateAnnotationClassNonVectorWithError(BaseTestCase): - PROJECT_NAME = "test_create_annotation_class" + PROJECT_NAME = "TestCreateAnnotationClassNonVectorWithError" PROJECT_TYPE = "Video" PROJECT_DESCRIPTION = "Example Project test pixel basic images" @@ -43,7 +43,7 @@ def test_create_annotation_class(self): class TestCreateAnnotationClassesNonVectorWithError(BaseTestCase): - PROJECT_NAME = "test_create_annotation_class" + PROJECT_NAME = "TestCreateAnnotationClassesNonVectorWithError" PROJECT_TYPE = "Video" PROJECT_DESCRIPTION = "Example Project test pixel basic images" diff --git a/tests/integration/projects/test_basic_project.py b/tests/integration/projects/test_basic_project.py index 95c724906..14231e28b 100644 --- a/tests/integration/projects/test_basic_project.py +++ b/tests/integration/projects/test_basic_project.py @@ -98,7 +98,7 @@ def test_workflow_get(self): class TestProjectCreateMetadata(BaseTestCase): - PROJECT_NAME = "sample_basic_project" + PROJECT_NAME = "TestProjectCreateMetadata" PROJECT_TYPE = "Vector" OTHER_PROJECT_NAME = "other_project" PROJECT_DESCRIPTION = "DESCRIPTION" diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py index a1eddcc64..70b2b876e 100644 --- a/tests/integration/test_depricated_functions_document.py +++ b/tests/integration/test_depricated_functions_document.py @@ -64,7 +64,6 @@ def image_path(self): @pytest.mark.flaky(reruns=2) def test_deprecated_functions(self): - try: sa.upload_images_from_folder_to_project(self.PROJECT_NAME, "some") except AppException as e: @@ -143,7 +142,7 @@ def test_deprecated_functions(self): self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.add_annotation_comment_to_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "some comment", [1, 2], - "some user") + "some@user.com") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py index 7f2f229b3..ddb5785d6 100644 --- a/tests/integration/test_depricated_functions_video.py +++ b/tests/integration/test_depricated_functions_video.py @@ -137,7 +137,7 @@ def test_deprecated_functions(self): self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.add_annotation_comment_to_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "some comment", [1, 2], - "some user") + "some@user.com") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: @@ -196,7 +196,7 @@ def test_deprecated_functions(self): except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: - sa.assign_images(self.PROJECT_NAME, [self.UPLOAD_IMAGE_NAME], "some user") + sa.assign_images(self.PROJECT_NAME, [self.UPLOAD_IMAGE_NAME], "some@user.com") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: diff --git a/tests/integration/test_df_processing.py b/tests/integration/test_df_processing.py index 6b1707e72..f96d72013 100644 --- a/tests/integration/test_df_processing.py +++ b/tests/integration/test_df_processing.py @@ -2,6 +2,8 @@ from os.path import dirname from pathlib import Path +import pytest + import src.superannotate as sa from tests.integration.base import BaseTestCase @@ -29,7 +31,7 @@ def test_filter_instances(self): ) -class TestDFWithTagInstace(BaseTestCase): +class TestDFWithTagInstance(BaseTestCase): PROJECT_TYPE = "Vector" TEST_FOLDER_PATH = "data_set/sample_project_vector_with_tag" @@ -44,7 +46,7 @@ def test_filter_instances(self): self.assertEqual(df.iloc[0]["type"], "tag") -class TestClassDistibutionWithTagInstance(BaseTestCase): +class TestClassDistributionWithTagInstance(BaseTestCase): PROJECT_TYPE = "Vector" EXPORT_ROOT_PATH = "data_set" PROJECT_NAME = "sample_project_vector_with_tag" @@ -55,6 +57,7 @@ def root_path(self): Path(os.path.join(dirname(dirname(__file__)), self.EXPORT_ROOT_PATH)) ) + @pytest.mark.skip(reason="Need to adjust") def test_filter_instances(self): df = sa.class_distribution(export_root=self.root_path, project_names=[self.PROJECT_NAME]) self.assertEqual(df.iloc[0]['count'], 1) From 6e442bb4d82c7b8c3986119d38785aac0dd68781 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 10 Mar 2022 12:49:32 +0400 Subject: [PATCH 32/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 4fb5a5636..01ab2fb2a 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev11" +__version__ = "4.3.0.dev12" From c7e87b73cecf80c33cfb4978476002f133a7ef8f Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 10 Mar 2022 17:16:06 +0400 Subject: [PATCH 33/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 01ab2fb2a..dfc687c3d 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev12" +__version__ = "4.3.0.dev13" From 3afbe3c1c9fe36cd106badbb49656e82d0f79986 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Fri, 11 Mar 2022 11:38:02 +0400 Subject: [PATCH 34/50] Deleted default description valie ... --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/entities/project_entities.py | 3 ++- src/superannotate/lib/infrastructure/controller.py | 2 +- tests/integration/projects/test_clone_project.py | 3 --- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 1378ec9b8..0e274637d 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -191,7 +191,7 @@ def create_project_from_metadata(project_metadata: Project): project_metadata = project_metadata.dict() response = Controller.get_default().create_project( name=project_metadata["name"], - description=project_metadata.get("description", "..."), + description=project_metadata.get("description"), project_type=project_metadata["type"], settings=project_metadata.get("settings", []), annotation_classes=project_metadata.get("classes", []), diff --git a/src/superannotate/lib/core/entities/project_entities.py b/src/superannotate/lib/core/entities/project_entities.py index 5ef574dca..f6e70185e 100644 --- a/src/superannotate/lib/core/entities/project_entities.py +++ b/src/superannotate/lib/core/entities/project_entities.py @@ -3,6 +3,7 @@ from typing import Any from typing import Iterable from typing import List +from typing import Union from lib.core.enums import ClassTypeEnum from lib.core.enums import SegmentationStatus @@ -79,7 +80,7 @@ def __init__( team_id: int = None, name: str = None, project_type: int = None, - description: str = None, + description: Union[str, None] = None, attachment_name: str = None, attachment_path: str = None, creator_id: str = None, diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index e4cca9841..37bd92ccd 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -480,7 +480,7 @@ def clone_project( project = self._get_project(from_name) project_to_create = copy.copy(project) project_to_create.name = name - if project_description: + if project_description is not None: project_to_create.description = project_description use_case = usecases.CloneProjectUseCase( reporter=self.default_reporter, diff --git a/tests/integration/projects/test_clone_project.py b/tests/integration/projects/test_clone_project.py index f278cd20c..8c33c9e92 100644 --- a/tests/integration/projects/test_clone_project.py +++ b/tests/integration/projects/test_clone_project.py @@ -50,7 +50,6 @@ def test_create_like_project(self): ) sa.set_project_default_image_quality_in_editor(self.PROJECT_NAME_1, self.IMAGE_QUALITY) - sa.set_project_workflow( self.PROJECT_NAME_1, [ @@ -78,7 +77,6 @@ def test_create_like_project(self): new_project = sa.clone_project( self.PROJECT_NAME_2, self.PROJECT_NAME_1, copy_contributors=True ) - source_project = sa.get_project_metadata(self.PROJECT_NAME_1) self.assertEqual(new_project['upload_state'], constances.UploadState.INITIAL.name) new_settings = sa.get_project_settings(self.PROJECT_NAME_2) @@ -112,7 +110,6 @@ def test_create_like_project(self): self.assertEqual(ann_classes[0]["color"], "#faf") - class TestCloneProjectAttachedUrls(TestCase): PROJECT_NAME_1 = "TestCloneProjectAttachedUrls_1" PROJECT_NAME_2 = "TestCloneProjectAttachedUrls_2" From 63c6b8ab9b1fbc674e5e0931d657c0dc120a62e3 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Fri, 11 Mar 2022 11:38:52 +0400 Subject: [PATCH 35/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index dfc687c3d..4a37f1ca0 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev13" +__version__ = "4.3.0.dev14" From 2ad01b55f0a5593716099d5b0dd242c2f08d2569 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 15 Mar 2022 11:17:46 +0400 Subject: [PATCH 36/50] Udpate scemas version, set project status not started on create --- requirements_dev.txt | 2 +- src/superannotate/lib/infrastructure/repositories.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 5a296cca2..378318914 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1 +1 @@ -superannotate_schemas==1.0.40b2 +superannotate_schemas>=1.0.40b3 diff --git a/src/superannotate/lib/infrastructure/repositories.py b/src/superannotate/lib/infrastructure/repositories.py index c3c8caf9c..a15a4957d 100644 --- a/src/superannotate/lib/infrastructure/repositories.py +++ b/src/superannotate/lib/infrastructure/repositories.py @@ -99,8 +99,8 @@ def get_all(self, condition: Condition = None) -> List[ProjectEntity]: def insert(self, entity: ProjectEntity) -> ProjectEntity: project_data = self._drop_nones(entity.to_dict()) - if not project_data.get("status"): - project_data["status"] = constance.ProjectStatus.NotStarted.value + # new projects can only have the status of NotStarted + project_data["status"] = constance.ProjectStatus.NotStarted.value result = self._service.create_project(project_data) return self.dict2entity(result) From b5fbf3da86324b4f26da5ecbf0041e68374274c7 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 15 Mar 2022 11:18:47 +0400 Subject: [PATCH 37/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 4a37f1ca0..294e17add 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev14" +__version__ = "4.3.0.dev15" From 732460bedf620215fc9583602be1f2958fb47e7d Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 15 Mar 2022 13:51:42 +0400 Subject: [PATCH 38/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 294e17add..479b6da91 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev15" +__version__ = "4.3.0.dev16" From bd26bf5a680b372bed3f474a47329429838fc5b4 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 16 Mar 2022 18:55:33 +0400 Subject: [PATCH 39/50] fix project validations on create annotation classes --- src/superannotate/lib/core/__init__.py | 2 +- src/superannotate/lib/core/usecases/images.py | 8 ++++---- src/superannotate/lib/infrastructure/services.py | 1 - .../projects/test_create_project_from_metadata.py | 0 4 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 tests/integration/projects/test_create_project_from_metadata.py diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index e72a5caf8..7d88e3527 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -1,8 +1,8 @@ from pathlib import Path -from superannotate.lib.core.enums import ProjectStatus from superannotate.lib.core.enums import AnnotationStatus from superannotate.lib.core.enums import ImageQuality +from superannotate.lib.core.enums import ProjectStatus from superannotate.lib.core.enums import ProjectType from superannotate.lib.core.enums import SegmentationStatus from superannotate.lib.core.enums import TrainingStatus diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index c2fefe430..57aa0d30b 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -2732,7 +2732,7 @@ def validate_uniqueness(self): def validate_project_type(self): if ( - self._project.project_type != ProjectType.VECTOR.value + self._project.project_type in (ProjectType.PIXEL.value and ProjectType.VIDEO.value) and self._annotation_class.type == "tag" ): raise AppException( @@ -2846,9 +2846,9 @@ def __init__( self._project = project def validate_project_type(self): - if self._project.project_type != ProjectType.VECTOR.value and "tag" in [ - i.type for i in self._annotation_classes - ]: + if self._project.project_type in (ProjectType.PIXEL.value, ProjectType.VIDEO.value) and "tag" in any([ + True for i in self._annotation_classes if i.type == "tag" + ]): raise AppException( f"Predefined tagging functionality is not supported for projects of type {ProjectType.get_name(self._project.project_type)}." ) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 65c26ce0c..736654d46 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -1,6 +1,5 @@ import asyncio import json -import time from contextlib import contextmanager from datetime import datetime from typing import Dict diff --git a/tests/integration/projects/test_create_project_from_metadata.py b/tests/integration/projects/test_create_project_from_metadata.py new file mode 100644 index 000000000..e69de29bb From 34df048be6eb04c2d984a57ae33fae8079c0163b Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 16 Mar 2022 18:56:00 +0400 Subject: [PATCH 40/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 479b6da91..483b0fd66 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev16" +__version__ = "4.3.0.dev17" From 92876c45f7d3185d6b0ff25a7664810bf9711890 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 16 Mar 2022 18:56:00 +0400 Subject: [PATCH 41/50] Update version.py --- src/superannotate/lib/core/usecases/images.py | 2 +- src/superannotate/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 57aa0d30b..9275c432d 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -2732,7 +2732,7 @@ def validate_uniqueness(self): def validate_project_type(self): if ( - self._project.project_type in (ProjectType.PIXEL.value and ProjectType.VIDEO.value) + self._project.project_type in (ProjectType.PIXEL.value, ProjectType.VIDEO.value) and self._annotation_class.type == "tag" ): raise AppException( diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 479b6da91..483b0fd66 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev16" +__version__ = "4.3.0.dev17" From b16c20727851d67b3044f09a86873f01796d026e Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 16 Mar 2022 19:38:35 +0400 Subject: [PATCH 42/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 483b0fd66..facf01197 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev17" +__version__ = "4.3.0.dev18" From e726512cf0353098165705a948e90bd26abe10d4 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 17 Mar 2022 11:00:56 +0400 Subject: [PATCH 43/50] added test --- .../lib/app/interface/sdk_interface.py | 2 + src/superannotate/lib/core/usecases/images.py | 2 +- .../classes/classes.json | 33 ++ .../video_annotation_with_tags/video.mp4.json | 299 ++++++++++++++++++ .../test_create_annotation_class.py | 23 ++ 5 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 tests/data_set/video_annotation_with_tags/classes/classes.json create mode 100644 tests/data_set/video_annotation_with_tags/video.mp4.json create mode 100644 tests/integration/annotation_classes/test_create_annotation_class.py diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 0e274637d..fcceaad08 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -1574,6 +1574,8 @@ def create_annotation_class( attribute_groups=attribute_groups, class_type=class_type, ) + if response.errors: + raise AppException(response.errors) return BaseSerializers(response.data).serialize() diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 9275c432d..4c1bc74dd 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -2846,7 +2846,7 @@ def __init__( self._project = project def validate_project_type(self): - if self._project.project_type in (ProjectType.PIXEL.value, ProjectType.VIDEO.value) and "tag" in any([ + if self._project.project_type in (ProjectType.PIXEL.value, ProjectType.VIDEO.value) and any([ True for i in self._annotation_classes if i.type == "tag" ]): raise AppException( diff --git a/tests/data_set/video_annotation_with_tags/classes/classes.json b/tests/data_set/video_annotation_with_tags/classes/classes.json new file mode 100644 index 000000000..5284d5a49 --- /dev/null +++ b/tests/data_set/video_annotation_with_tags/classes/classes.json @@ -0,0 +1,33 @@ +[ + { + "id": 857627, + "project_id": 150845, + "name": "vid", + "type": "tag", + "color": "#0fc1c9", + "count": 0, + "createdAt": "2021-10-01T13:03:51.000Z", + "updatedAt": "2021-10-01T13:03:51.000Z", + "attribute_groups": [ + { + "id": 337487, + "class_id": 857627, + "name": "attr g", + "is_multiselect": 0, + "createdAt": "2021-10-04T07:01:29.000Z", + "updatedAt": "2021-10-04T07:01:29.000Z", + "attributes": [ + { + "id": 1174520, + "group_id": 337487, + "project_id": 150845, + "name": "attr", + "count": 0, + "createdAt": "2021-10-04T07:01:31.000Z", + "updatedAt": "2021-10-04T07:01:31.000Z" + } + ] + } + ] + } +] \ No newline at end of file diff --git a/tests/data_set/video_annotation_with_tags/video.mp4.json b/tests/data_set/video_annotation_with_tags/video.mp4.json new file mode 100644 index 000000000..4687b418f --- /dev/null +++ b/tests/data_set/video_annotation_with_tags/video.mp4.json @@ -0,0 +1,299 @@ +{ + "metadata": { + "name": "video.mp4", + "width": 480, + "height": 270, + "status": "NotStarted", + "url": "https://file-examples-com.github.io/uploads/2017/04/file_example_MP4_480_1_5MG.mp4", + "duration": 30526667, + "projectId": 152038, + "error": null, + "annotatorEmail": null, + "qaEmail": null + }, + "instances": [ + { + "meta": { + "type": "bbox", + "classId": 859496, + "className": "vid", + "pointLabels": { + "3": "point label bro" + }, + "start": 0, + "end": 30526667 + }, + "parameters": [ + { + "start": 0, + "end": 30526667, + "timestamps": [ + { + "points": { + "x1": 223.32, + "y1": 78.45, + "x2": 312.31, + "y2": 176.66 + }, + "timestamp": 0, + "attributes": [] + }, + { + "points": { + "x1": 182.08, + "y1": 33.18, + "x2": 283.45, + "y2": 131.39 + }, + "timestamp": 17271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 182.32, + "y1": 36.33, + "x2": 284.01, + "y2": 134.54 + }, + "timestamp": 18271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 181.49, + "y1": 45.09, + "x2": 283.18, + "y2": 143.3 + }, + "timestamp": 19271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 181.9, + "y1": 48.35, + "x2": 283.59, + "y2": 146.56 + }, + "timestamp": 19725864, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 181.49, + "y1": 52.46, + "x2": 283.18, + "y2": 150.67 + }, + "timestamp": 20271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 181.49, + "y1": 63.7, + "x2": 283.18, + "y2": 161.91 + }, + "timestamp": 21271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 182.07, + "y1": 72.76, + "x2": 283.76, + "y2": 170.97 + }, + "timestamp": 22271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 182.07, + "y1": 81.51, + "x2": 283.76, + "y2": 179.72 + }, + "timestamp": 23271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 182.42, + "y1": 97.19, + "x2": 284.11, + "y2": 195.4 + }, + "timestamp": 24271058, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "points": { + "x1": 182.42, + "y1": 97.19, + "x2": 284.11, + "y2": 195.4 + }, + "timestamp": 30526667, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "bbox", + "classId": 859496, + "className": "vid", + "start": 29713736, + "end": 30526667 + }, + "parameters": [ + { + "start": 29713736, + "end": 30526667, + "timestamps": [ + { + "points": { + "x1": 132.82, + "y1": 129.12, + "x2": 175.16, + "y2": 188 + }, + "timestamp": 29713736, + "attributes": [] + }, + { + "points": { + "x1": 132.82, + "y1": 129.12, + "x2": 175.16, + "y2": 188 + }, + "timestamp": 30526667, + "attributes": [] + } + ] + } + ] + }, + { + "meta": { + "type": "event", + "classId": 859496, + "className": "vid", + "start": 5528212, + "end": 7083022 + }, + "parameters": [ + { + "start": 5528212, + "end": 7083022, + "timestamps": [ + { + "timestamp": 5528212, + "attributes": [] + }, + { + "timestamp": 6702957, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + }, + { + "timestamp": 7083022, + "attributes": [ + { + "id": 1175876, + "groupId": 338357, + "name": "attr", + "groupName": "attr g" + } + ] + } + ] + } + ] + } + ], + "tags": [ + "some tag" + ] +} \ No newline at end of file diff --git a/tests/integration/annotation_classes/test_create_annotation_class.py b/tests/integration/annotation_classes/test_create_annotation_class.py new file mode 100644 index 000000000..d42187ebd --- /dev/null +++ b/tests/integration/annotation_classes/test_create_annotation_class.py @@ -0,0 +1,23 @@ +import os +from pathlib import Path + +import src.superannotate as sa +from tests.integration.base import BaseTestCase + + +class TestDocumentCreateAnnotationClass(BaseTestCase): + PROJECT_NAME = "TestDocumentCreateAnnotationClass" + TEST_FOLDER_PATH = "data_set/sample_project_vector" + PROJECT_DESCRIPTION = "desc" + PROJECT_TYPE = "Document" + + def test_create_annotation_class(self): + sa.create_annotation_class( + self.PROJECT_NAME, + "test_add", + "#FF0000", + [{"name": "height", "attributes": [{"name": "tall"}, {"name": "short"}]}], + class_type="tag" + ) + + self.assertEqual(len(sa.search_annotation_classes(self.PROJECT_NAME)), 1) From 2820c91a501a51dd289f5f80b90ae89798c7e5cd Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:01:36 +0400 Subject: [PATCH 44/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 475abb24a..a179e5364 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev19" +__version__ = "4.3.0.dev20" From 47d838f737d88c1d1dc415364ee24d8c50f8001a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 17 Mar 2022 12:19:36 +0400 Subject: [PATCH 45/50] new version --- src/superannotate/lib/infrastructure/controller.py | 1 - src/superannotate/version.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 37bd92ccd..9af4491a6 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1152,7 +1152,6 @@ def create_annotation_class( annotation_class=annotation_class, project=project, ) - use_case.execute() return use_case.execute() def delete_annotation_class(self, project_name: str, annotation_class_name: str): diff --git a/src/superannotate/version.py b/src/superannotate/version.py index a179e5364..66e896faf 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev20" +__version__ = "4.3.0.dev21" From 968c31e013952e97428bbe70dbc84b382baa0418 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 17 Mar 2022 17:43:04 +0400 Subject: [PATCH 46/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 66e896faf..caef5fbf8 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.dev21" +__version__ = "4.3.0.b1" From 6b97c0869da4000774163bc7ceb386f8a6772fce Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 17 Mar 2022 18:51:20 +0400 Subject: [PATCH 47/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index caef5fbf8..f088bf137 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.0.b1" +__version__ = "4.3.1.b1" From 1c983479d41ae7091f13b6714411b4f54092e24d Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 17 Mar 2022 19:03:11 +0400 Subject: [PATCH 48/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index f088bf137..b2f04708d 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.1.b1" +__version__ = "4.3.1.b2" From 91f87e749683cf674466fd5fd7bfd8fdfb21bf9a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Fri, 18 Mar 2022 12:00:08 +0400 Subject: [PATCH 49/50] fix cli --- src/superannotate/lib/app/interface/cli_interface.py | 9 +++------ .../integration/annotations/test_annotation_class_new.py | 2 -- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index 6a3a0f3d9..515a9da25 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -23,9 +23,6 @@ from lib.infrastructure.repositories import ConfigRepository -controller = Controller() -controller.retrieve_configs(constances.CONFIG_FILE_LOCATION) - class CLIFacade(BaseInterfaceFacade): """ @@ -124,12 +121,12 @@ def export_project( folders = None if folder_name: folders = [folder_name] - export_res = controller.prepare_export( + export_res = Controller.get_default().prepare_export( project_name, folders, include_fuse, False, annotation_statuses ) export_name = export_res.data["name"] - use_case = controller.download_export( + use_case = Controller.get_default().download_export( project_name=project_name, export_name=export_name, folder_path=folder, @@ -189,7 +186,7 @@ def _upload_annotations( ): project_folder_name = project project_name, folder_name = split_project_path(project) - project = controller.get_project_metadata(project_name=project_name).data + project = Controller.get_default().get_project_metadata(project_name=project_name).data if not format: format = "SuperAnnotate" if not dataset_name and format == "COCO": diff --git a/tests/integration/annotations/test_annotation_class_new.py b/tests/integration/annotations/test_annotation_class_new.py index cfefa5076..0fe7e9370 100644 --- a/tests/integration/annotations/test_annotation_class_new.py +++ b/tests/integration/annotations/test_annotation_class_new.py @@ -42,8 +42,6 @@ def test_create_annotation_class(self): classes = sa.search_annotation_classes(self.PROJECT_NAME) self.assertEqual(len(classes), 1) self.assertEqual(classes[0]['type'], 'object') - sa.create_annotation_class(self.PROJECT_NAME, "tt", "#FFFFFF") - self.assertEqual(len(sa.search_annotation_classes(self.PROJECT_NAME)), 1) def test_create_annotation_class_from_json(self): sa.create_annotation_classes_from_classes_json( From df691e057b50286a8c0016689970cac2f219f1b8 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Fri, 18 Mar 2022 12:48:30 +0400 Subject: [PATCH 50/50] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index b2f04708d..7804f75ca 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.1.b2" +__version__ = "4.3.1.b3"