diff --git a/docs/source/api_reference/api_metadata.rst b/docs/source/api_reference/api_metadata.rst index a7aba3382..f6692ed68 100644 --- a/docs/source/api_reference/api_metadata.rst +++ b/docs/source/api_reference/api_metadata.rst @@ -20,6 +20,7 @@ Project metadata example: "attachment_path": None, "entropy_status": 1, "status": "NotStarted", + "item_count": 123, "...": "..." } diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 2ab396e8c..5dde9e57b 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -5,6 +5,7 @@ import os import sys import tempfile +import warnings from pathlib import Path from typing import Callable from typing import Dict @@ -51,6 +52,7 @@ from lib.core.conditions import Condition from lib.core.conditions import EmptyCondition from lib.core.entities import AttachmentEntity +from lib.core.entities import WorkflowEntity from lib.core.entities import SettingEntity from lib.core.entities.classes import AnnotationClassEntity from lib.core.entities.classes import AttributeGroup @@ -233,7 +235,7 @@ def search_projects( self, name: Optional[NotEmptyStr] = None, return_metadata: bool = False, - include_complete_image_count: bool = False, + include_complete_item_count: bool = False, status: Optional[Union[PROJECT_STATUS, List[PROJECT_STATUS]]] = None, ): """ @@ -246,8 +248,9 @@ def search_projects( :param return_metadata: return metadata of projects instead of names :type return_metadata: bool - :param include_complete_image_count: return projects that have completed images and include the number of completed images in response. - :type include_complete_image_count: bool + :param include_complete_item_count: return projects that have completed items and include + the number of completed items in response. + :type include_complete_item_count: bool :param status: search projects via project status :type status: str @@ -265,9 +268,9 @@ def search_projects( condition = Condition.get_empty_condition() if name: condition &= Condition("name", name, EQ) - if include_complete_image_count: + if include_complete_item_count: condition &= Condition( - "completeImagesCount", include_complete_image_count, EQ + "completeImagesCount", include_complete_item_count, EQ ) for status in statuses: condition &= Condition( @@ -280,7 +283,13 @@ def search_projects( if return_metadata: return [ ProjectSerializer(project).serialize( - exclude={"settings", "workflows", "contributors", "classes"} + exclude={ + "settings", + "workflows", + "contributors", + "classes", + "item_count", + } ) for project in response.data ] @@ -293,6 +302,9 @@ def create_project( project_description: NotEmptyStr, project_type: PROJECT_TYPE, settings: List[Setting] = None, + classes: List[AnnotationClassEntity] = None, + workflows: List = None, + instructions_link: str = None, ): """Create a new project in the team. @@ -308,25 +320,71 @@ def create_project( :param settings: list of settings objects :type settings: list of dicts + :param classes: list of class objects + :type classes: list of dicts + + :param workflows: list of information for each step + :type workflows: list of dicts + + :param instructions_link: str of instructions URL + :type instructions_link: str + :return: dict object metadata the new project :rtype: dict """ + if workflows: + if project_type.capitalize() not in ( + constants.ProjectType.VECTOR.name, + constants.ProjectType.PIXEL.name, + ): + raise AppException( + f"Workflow is not supported in {project_type} project." + ) + parse_obj_as(List[WorkflowEntity], workflows) + if workflows and not classes: + raise AppException( + "Project with workflows can not be created without classes." + ) if settings: settings = parse_obj_as(List[SettingEntity], settings) else: settings = [] - response = self.controller.projects.create( + if classes: + classes = parse_obj_as(List[AnnotationClassEntity], classes) + if workflows and classes: + invalid_classes = [] + class_names = [_class.name for _class in classes] + for step in workflows: + if step["className"] not in class_names: + invalid_classes.append(step["className"]) + if invalid_classes: + raise AppException( + f"There are no [{', '.join(invalid_classes)}] classes created in the project." + ) + project_response = self.controller.projects.create( entities.ProjectEntity( name=project_name, description=project_description, type=constants.ProjectType.get_value(project_type), settings=settings, + instructions_link=instructions_link, ) ) - if response.errors: - raise AppException(response.errors) - - return ProjectSerializer(response.data).serialize() + project_response.raise_for_status() + project = project_response.data + if classes: + classes_response = self.controller.annotation_classes.create_multiple( + project, classes + ) + classes_response.raise_for_status() + project.classes = classes_response.data + if workflows: + workflow_response = self.controller.projects.set_workflows( + project, workflows + ) + workflow_response.raise_for_status() + project.workflows = self.controller.projects.list_workflow(project).data + return ProjectSerializer(project).serialize() def create_project_from_metadata(self, project_metadata: Project): """Create a new project in the team using project metadata object dict. @@ -340,6 +398,10 @@ def create_project_from_metadata(self, project_metadata: Project): :return: dict object metadata the new project :rtype: dict """ + deprecation_msg = '"create_project_from_metadata" is deprecated and replaced by "create_project"' + warnings.warn(deprecation_msg, DeprecationWarning) + logger.warning(deprecation_msg) + project_metadata = project_metadata.dict() if project_metadata["type"] not in enums.ProjectType.titles(): raise AppException( @@ -557,7 +619,7 @@ def get_project_metadata( include_settings: Optional[StrictBool] = False, include_workflow: Optional[StrictBool] = False, include_contributors: Optional[StrictBool] = False, - include_complete_image_count: Optional[StrictBool] = False, + include_complete_item_count: Optional[StrictBool] = False, ): """Returns project metadata @@ -576,9 +638,9 @@ def get_project_metadata( the key "contributors" :type include_contributors: bool - :param include_complete_image_count: enables project complete image count output under - the key "completed_images_count" - :type include_complete_image_count: bool + :param include_complete_item_count: enables project complete item count output under + the key "completed_items_count" + :type include_complete_item_count: bool :return: metadata of project :rtype: dict @@ -591,7 +653,7 @@ def get_project_metadata( include_settings, include_workflow, include_contributors, - include_complete_image_count, + include_complete_item_count, ) if response.errors: raise AppException(response.errors) @@ -968,7 +1030,13 @@ def get_project_image_count( :return: number of images in the project :rtype: int """ + deprecation_msg = ( + "“get_project_image_count” is deprecated and replaced" + " by “item_count” value will be included in project metadata." + ) + warnings.warn(deprecation_msg, DeprecationWarning) + logger.warning(deprecation_msg) project_name, folder_name = extract_project_folder(project) response = self.controller.get_project_image_count( diff --git a/src/superannotate/lib/core/entities/project.py b/src/superannotate/lib/core/entities/project.py index 97004ad6e..b036dc152 100644 --- a/src/superannotate/lib/core/entities/project.py +++ b/src/superannotate/lib/core/entities/project.py @@ -97,8 +97,9 @@ class ProjectEntity(TimedBaseModel): settings: List[SettingEntity] = [] classes: List[AnnotationClassEntity] = [] workflows: Optional[List[WorkflowEntity]] = [] - completed_images_count: Optional[int] = Field(None, alias="completedImagesCount") - root_folder_completed_images_count: Optional[int] = Field( + item_count: Optional[int] = Field(None, alias="imageCount") + completed_items_count: Optional[int] = Field(None, alias="completedImagesCount") + root_folder_completed_items_count: Optional[int] = Field( None, alias="rootFolderCompletedImagesCount" ) diff --git a/src/superannotate/lib/core/response.py b/src/superannotate/lib/core/response.py index 49d6b035c..74f3bb180 100644 --- a/src/superannotate/lib/core/response.py +++ b/src/superannotate/lib/core/response.py @@ -1,5 +1,8 @@ +from typing import NoReturn from typing import Union +from lib.core.exceptions import AppException + class Response: def __init__(self, status: str = None, data: Union[dict, list] = None): @@ -11,6 +14,10 @@ def __init__(self, status: str = None, data: Union[dict, list] = None): def __str__(self): return f"Response object with status:{self.status}, data : {self.data}, errors: {self.errors} " + def raise_for_status(self) -> NoReturn: + if self.errors: + raise AppException(self.errors) + @property def data(self): return self._data diff --git a/src/superannotate/lib/core/usecases/classes.py b/src/superannotate/lib/core/usecases/classes.py index a7a3525d8..9ab99bee6 100644 --- a/src/superannotate/lib/core/usecases/classes.py +++ b/src/superannotate/lib/core/usecases/classes.py @@ -8,7 +8,6 @@ from lib.core.entities import ProjectEntity from lib.core.enums import ProjectType from lib.core.exceptions import AppException -from lib.core.reporter import Spinner from lib.core.serviceproviders import BaseServiceProvider from lib.core.usecases.base import BaseUseCase @@ -150,21 +149,17 @@ def execute(self): ) created = [] chunk_failed = False - with Spinner(): - # this is in reverse order because of the front-end - for i in range(len(unique_annotation_classes), 0, -self.CHUNK_SIZE): - response = ( - self._service_provider.annotation_classes.create_multiple( - project=self._project, - classes=unique_annotation_classes[ - i - self.CHUNK_SIZE : i - ], # noqa - ) - ) - if response.ok: - created.extend(response.data) - else: - chunk_failed = True + # this is in reverse order because of the front-end + for i in range(len(unique_annotation_classes), 0, -self.CHUNK_SIZE): + response = self._service_provider.annotation_classes.create_multiple( + project=self._project, + classes=unique_annotation_classes[i - self.CHUNK_SIZE : i], # noqa + ) + if response.ok: + created.extend(response.data) + else: + logger.debug(response.error) + chunk_failed = True if created: logger.info( f"{len(created)} annotation classes were successfully created in {self._project.name}." diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index f8e17752d..a523f1ee4 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -137,8 +137,8 @@ def execute(self): root_completed_count = folder.completedCount # noqa except AttributeError: pass - project.root_folder_completed_images_count = root_completed_count - project.completed_images_count = total_completed_count + project.root_folder_completed_items_count = root_completed_count + project.completed_items_count = total_completed_count if self._include_annotation_classes: project.classes = self._service_provider.annotation_classes.list( Condition("project_id", self._project.id, EQ) @@ -836,10 +836,7 @@ def execute(self): del step["id"] step["class_id"] = annotation_classes_map.get(step["className"], None) if not step["class_id"]: - raise AppException( - "Annotation class not found in set_project_workflow." - ) - + raise AppException("Annotation class not found.") self._service_provider.projects.set_workflows( project=self._project, steps=self._steps, @@ -864,7 +861,7 @@ def execute(self): None, ): raise AppException( - "Attribute group name or attribute name not found in set_project_workflow." + f"Attribute group name or attribute name not found {attribute_group_name}." ) if not existing_workflows_map.get(step["step"], None): diff --git a/tests/integration/base.py b/tests/integration/base.py index f6c396f42..bc88c80a4 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -34,11 +34,14 @@ def tearDown(self) -> None: except Exception as e: print(str(e)) - def _attach_items(self): + def _attach_items(self, count=5, folder=None): + path = self.PROJECT_NAME + if folder: + path = f"{self.PROJECT_NAME}/{folder}" sa.attach_items( - self.PROJECT_NAME, + path, [ {"name": f"example_image_{i}.jpg", "url": f"url_{i}"} - for i in range(1, 5) + for i in range(1, count + 1) ], # noqa ) diff --git a/tests/integration/folders/test_folders.py b/tests/integration/folders/test_folders.py index f34bd08e2..528adbace 100644 --- a/tests/integration/folders/test_folders.py +++ b/tests/integration/folders/test_folders.py @@ -236,10 +236,10 @@ def test_project_completed_count(self): project, self.folder_path, annotation_status="Completed" ) project_metadata = sa.get_project_metadata( - self.PROJECT_NAME, include_complete_image_count=True + self.PROJECT_NAME, include_complete_item_count=True ) - self.assertEqual(project_metadata["completed_images_count"], 8) - self.assertEqual(project_metadata["root_folder_completed_images_count"], 4) + self.assertEqual(project_metadata["completed_items_count"], 8) + self.assertEqual(project_metadata["root_folder_completed_items_count"], 4) def test_folder_misnamed(self): sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME_1) diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py index d93d646f0..3fa71026e 100644 --- a/tests/integration/mixpanel/test_mixpanel_decorator.py +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -78,7 +78,7 @@ def test_search_team_contributors(self, track_method): def test_search_projects(self, track_method): kwargs = { "name": self.PROJECT_NAME, - "include_complete_image_count": True, + "include_complete_item_count": True, "status": "NotStarted", "return_metadata": False, } @@ -96,6 +96,9 @@ def test_create_project(self, track_method): "project_description": self.PROJECT_DESCRIPTION, "project_type": self.PROJECT_TYPE, "settings": {"a": 1, "b": 2}, + "classes": None, + "workflows": None, + 'instructions_link': None } try: self.CLIENT.create_project(**kwargs) diff --git a/tests/integration/projects/test_basic_project.py b/tests/integration/projects/test_basic_project.py index bf89bff81..89e22d9c7 100644 --- a/tests/integration/projects/test_basic_project.py +++ b/tests/integration/projects/test_basic_project.py @@ -102,12 +102,12 @@ def test_workflow_get(self): self.assertEqual(workflows[1]["className"], "class2") def test_include_complete_image_count(self): - self._attach_items() # will attach 4 items + self._attach_items(count=4) sa.set_annotation_statuses(self.PROJECT_NAME, annotation_status="Completed") metadata = sa.get_project_metadata( - self.PROJECT_NAME, include_complete_image_count=True + self.PROJECT_NAME, include_complete_item_count=True ) - assert metadata["completed_images_count"] == 4 + assert metadata["completed_items_count"] == 4 class TestProject(BaseTestCase): diff --git a/tests/integration/projects/test_create_project.py b/tests/integration/projects/test_create_project.py index 2d158b03f..a0be9692b 100644 --- a/tests/integration/projects/test_create_project.py +++ b/tests/integration/projects/test_create_project.py @@ -1,3 +1,4 @@ +import copy from unittest import TestCase from src.superannotate import AppException @@ -30,24 +31,81 @@ class TestSearchProjectVector(BaseTestCase): PROJECT_1 = "project_1TestSearchProject" PROJECT_2 = "project_2TestSearchProject" PROJECT_TYPE = "Vector" + CLASSES = [ + { + "type": 1, + "name": "Personal vehicle", + "color": "#ecb65f", + "attribute_groups": [ + { + "name": "test", + "group_type": "checklist", + "attributes": [ + {"name": "Car"}, + {"name": "Track"}, + {"name": "Bus"}, + ], + "default_value": ["Bus"], + } + ], + } + ] + + WORKFLOWS = [ + { + "step": 1, + "className": "Personal vehicle", + "tool": 3, + "attribute": [ + { + "attribute": { + "name": "Car", + "attribute_group": {"name": "test"}, + } + }, + { + "attribute": { + "name": "Bus", + "attribute_group": {"name": "test"}, + } + }, + ], + }, + { + "step": 2, + "className": "Personal vehicle", + "tool": 3, + "attribute": [ + { + "attribute": { + "name": "Track", + "attribute_group": {"name": "test"}, + } + }, + { + "attribute": { + "name": "Bus", + "attribute_group": {"name": "test"}, + } + }, + ], + }, + ] @property def projects(self): return self.PROJECT_2, self.PROJECT_1 - def test_project_metadata(self): + def test_created_project(self): + # check datetime project = sa.create_project(self.PROJECT_1, "desc", self.PROJECT_TYPE) - pr = sa.get_project_metadata(project["name"]) - assert "Z" not in pr["createdAt"] - - def test_create_project_without_settings(self): - project = sa.create_project(self.PROJECT_1, "desc", self.PROJECT_TYPE) - assert project["name"] == self.PROJECT_1 + metadata = sa.get_project_metadata(project["name"]) + assert "Z" not in metadata["createdAt"] def test_create_project_wrong_type(self): with self.assertRaisesRegexp( - AppException, - "Available values are 'Vector', 'Pixel', 'Video', 'Document', 'Tiled', 'Other', 'PointCloud'.", + AppException, + "Available values are 'Vector', 'Pixel', 'Video', 'Document', 'Tiled', 'Other', 'PointCloud'.", ): sa.create_project(self.PROJECT_1, "desc", "wrong_type") @@ -63,6 +121,40 @@ def test_create_project_with_settings(self): if setting["attribute"] == "ImageQuality": assert setting["value"] == "original" + def test_create_with_classes_and_workflows(self): + project = sa.create_project(self.PROJECT_1, "desc", self.PROJECT_TYPE, classes=self.CLASSES, + workflows=self.WORKFLOWS) + assert len(project['classes']) == 1 + assert len(project['classes'][0]['attribute_groups']) == 1 + assert len(project['classes'][0]['attribute_groups'][0]['attributes']) == 3 + + assert len(project['workflows']) == 2 + assert project['workflows'][0]['className'] == self.CLASSES[0]['name'] + assert project['workflows'][0]['attribute'][0]['attribute']['name'] == 'Car' + assert project['workflows'][0]['attribute'][1]['attribute']['name'] == 'Bus' + assert project['workflows'][1]['attribute'][0]['attribute']['name'] == 'Track' + assert project['workflows'][1]['attribute'][1]['attribute']['name'] == 'Bus' + + def test_create_with_workflow_without_classes(self): + with self.assertRaisesRegexp(AppException, 'Project with workflows can not be created without classes.'): + sa.create_project(self.PROJECT_1, "desc", self.PROJECT_TYPE, workflows=self.WORKFLOWS) + + def test_create_wrong_project_type(self): + with self.assertRaisesRegexp(AppException, 'Workflow is not supported in Video project.'): + sa.create_project(self.PROJECT_1, "desc", "Video", workflows=self.WORKFLOWS) + + def test_create_with_workflow_wrong_classes(self): + # with self.assertRaisesRegexp didnt work + try: + workflows = copy.copy(self.WORKFLOWS) + workflows[0]['className'] = "1" + workflows[1]['className'] = "2" + sa.create_project( + self.PROJECT_1, "desc", self.PROJECT_TYPE, classes=self.CLASSES, workflows=self.WORKFLOWS + ) + except AppException as e: + assert str(e) == 'There are no [1, 2] classes created in the project.' + class TestSearchProjectVideo(BaseTestCase): PROJECT_1 = "project_1TestSearchProjectVideo" @@ -73,10 +165,6 @@ class TestSearchProjectVideo(BaseTestCase): def projects(self): return self.PROJECT_2, self.PROJECT_1 - def test_create_project_without_settings(self): - project = sa.create_project(self.PROJECT_1, "desc", self.PROJECT_TYPE) - assert project["name"] == self.PROJECT_1 - def test_create_project_with_settings(self): sa.create_project( self.PROJECT_1, diff --git a/tests/integration/projects/test_get_project_metadata.py b/tests/integration/projects/test_get_project_metadata.py new file mode 100644 index 000000000..68fa10b47 --- /dev/null +++ b/tests/integration/projects/test_get_project_metadata.py @@ -0,0 +1,25 @@ +from src.superannotate import SAClient +from tests.integration.base import BaseTestCase + +sa = SAClient() + + +class TestGetProjectMetadata(BaseTestCase): + PROJECT_NAME = "TestGetProjectMetadata" + PROJECT_TYPE = "Vector" + PROJECT_DESCRIPTION = "DESCRIPTION" + + def test_metadata_payload(self): + """ + checking item_count for project retrieve + and exclude in list + """ + project = sa.get_project_metadata(self.PROJECT_NAME) + assert project['item_count'] == 0 + self._attach_items(count=5) + sa.create_folder(self.PROJECT_NAME, "tmp") + self._attach_items(count=5, folder="tmp") + project = sa.get_project_metadata(self.PROJECT_NAME) + assert project['item_count'] == 10 + projects = sa.search_projects(name=self.PROJECT_NAME, return_metadata=True) + assert 'item_count' not in projects[0] diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index aa4380cea..2a83e14ba 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -107,9 +107,9 @@ def test_search_project(self): self.PROJECT_NAME, "Completed", [self.EXAMPLE_IMAGE_1] ) data = sa.search_projects( - self.PROJECT_NAME, return_metadata=True, include_complete_image_count=True + self.PROJECT_NAME, return_metadata=True, include_complete_item_count=True ) - self.assertIsNotNone(data[0]["completed_images_count"]) + self.assertIsNotNone(data[0]["completed_items_count"]) def test_overlay_fuse(self): sa.upload_image_to_project(