From 0dee3463d9fc6dadfd0a1369326a14a4dc3fa4ef Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Fri, 24 Feb 2023 16:18:28 +0400 Subject: [PATCH] fix video project clone issue and refactoring --- .../lib/app/interface/base_interface.py | 12 +- .../lib/app/interface/sdk_interface.py | 70 +++- src/superannotate/lib/app/interface/types.py | 1 + src/superannotate/lib/app/serializers.py | 2 + .../lib/core/entities/__init__.py | 4 +- src/superannotate/lib/core/entities/base.py | 4 +- .../lib/core/entities/project.py | 32 +- src/superannotate/lib/core/enums.py | 2 +- .../lib/core/usecases/projects.py | 315 ++---------------- .../lib/infrastructure/controller.py | 31 +- .../infrastructure/services/http_client.py | 2 +- ...load_annotations_from_folder_to_project.py | 5 +- .../projects/test_clone_project.py | 180 +++++++--- .../projects/test_create_project.py | 112 +++---- tests/integration/settings/test_settings.py | 18 - tests/unit/test_init.py | 13 +- 16 files changed, 331 insertions(+), 472 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index f146b3494..5d4e141a5 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -34,7 +34,9 @@ def __init__(self, token: TokenStr = None, config_path: str = None): config = ConfigEntity(SA_TOKEN=token) elif config_path: config_path = Path(config_path) - if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + if not Path(config_path).is_file() or not os.access( + config_path, os.R_OK + ): raise AppException( f"SuperAnnotate config file {str(config_path)} not found." ) @@ -77,8 +79,12 @@ def _retrieve_configs_from_json(path: Path) -> typing.Union[ConfigEntity]: config = ConfigEntity(SA_TOKEN=token) except pydantic.ValidationError: raise pydantic.ValidationError( - [pydantic.error_wrappers.ErrorWrapper(ValueError("Invalid token."), loc='token')], - model=ConfigEntity + [ + pydantic.error_wrappers.ErrorWrapper( + ValueError("Invalid token."), loc="token" + ) + ], + model=ConfigEntity, ) host = json_data.get("main_endpoint") verify_ssl = json_data.get("ssl_verify") diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 2cde74788..0084db457 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -431,7 +431,7 @@ def clone_project( project_description: Optional[NotEmptyStr] = None, copy_annotation_classes: Optional[StrictBool] = True, copy_settings: Optional[StrictBool] = True, - copy_workflow: Optional[StrictBool] = True, + copy_workflow: Optional[StrictBool] = False, copy_contributors: Optional[StrictBool] = False, ): """Create a new project in the team using annotation classes and settings from from_project. @@ -455,22 +455,54 @@ def clone_project( :return: dict object metadata of the new project :rtype: dict """ - project = self.controller.get_project(from_project) - new_project = copy.copy(project) - new_project.name = project_name - if project_description: - new_project.description = project_description - response = self.controller.projects.clone( - project=project, - new_project=new_project, - copy_annotation_classes=copy_annotation_classes, - copy_settings=copy_settings, - copy_workflow=copy_workflow, - copy_contributors=copy_contributors, + response = self.controller.projects.get_metadata( + self.controller.get_project(from_project), + include_annotation_classes=copy_annotation_classes, + include_settings=copy_settings, + include_workflow=copy_workflow, + include_contributors=copy_contributors, ) - if response.errors: - raise AppException(response.errors) - return ProjectSerializer(response.data).serialize() + response.raise_for_status() + project: entities.ProjectEntity = response.data + if copy_workflow and project.type not in ( + constants.ProjectType.VECTOR, + constants.ProjectType.PIXEL, + ): + raise AppException( + f"Workflow is not supported in {project.type.name} project." + ) + logger.info( + f"Created project {project_name} with type {constants.ProjectType.get_name(project.type)}." + ) + project_copy = copy.copy(project) + if project_description: + project_copy.description = project_description + project_copy.name = project_name + create_response = self.controller.projects.create(project_copy) + create_response.raise_for_status() + new_project = create_response.data + if copy_contributors: + logger.info(f"Cloning contributors from {from_project} to {project_name}.") + self.controller.projects.add_contributors( + self.controller.team, new_project, project.contributors + ) + if copy_annotation_classes: + logger.info( + f"Cloning annotation classes from {from_project} to {project_name}." + ) + classes_response = self.controller.annotation_classes.create_multiple( + new_project, project.classes + ) + classes_response.raise_for_status() + project.classes = classes_response.data + if copy_workflow: + logger.info(f"Cloning workflow from {from_project} to {project_name}.") + workflow_response = self.controller.projects.set_workflows( + new_project, project.workflows + ) + workflow_response.raise_for_status() + project.workflows = self.controller.projects.list_workflow(project).data + return ProjectSerializer(new_project).serialize() def create_folder(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Create a new folder in the project. @@ -2123,8 +2155,12 @@ def add_contributors_to_project( :rtype: tuple (2 members) of lists of strs """ project = self.controller.projects.get_by_name(project).data + contributors = [ + entities.ContributorEntity(email=email, user_role=constants.UserRole(role)) + for email in emails + ] response = self.controller.projects.add_contributors( - project=project, team=self.controller.team, emails=emails, role=role + team=self.controller.team, project=project, contributors=contributors ) if response.errors: raise AppException(response.errors) diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index 5bc92b438..0f8faf8ef 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -51,4 +51,5 @@ def wrapped(self, *args, **kwargs): return pydantic_validate_arguments(func)(self, *args, **kwargs) except ValidationError as e: raise AppException(wrap_error(e)) from e + return wrapped diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index 032811fd7..c7d650088 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -128,11 +128,13 @@ def serialize( data["settings"] = [ SettingsSerializer(setting).serialize() for setting in data["settings"] ] + if not data.get("status"): data["status"] = "Undefined" if data.get("upload_state"): data["upload_state"] = constance.UploadState(data["upload_state"]).name + if data.get("users"): for contributor in data["users"]: contributor["user_role"] = constance.UserRole.get_name( diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index 2f00f2a66..e2a9ab89a 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -11,11 +11,11 @@ from lib.core.entities.items import TiledEntity from lib.core.entities.items import VideoEntity from lib.core.entities.project import AttachmentEntity +from lib.core.entities.project import ContributorEntity from lib.core.entities.project import MLModelEntity from lib.core.entities.project import ProjectEntity from lib.core.entities.project import SettingEntity from lib.core.entities.project import TeamEntity -from lib.core.entities.project import UserEntity from lib.core.entities.project import WorkflowEntity from lib.core.entities.project_entities import BaseEntity from lib.core.entities.project_entities import ImageInfoEntity @@ -39,13 +39,13 @@ "AttachmentEntity", # project "ProjectEntity", + "ContributorEntity", "ConfigEntity", "WorkflowEntity", "FolderEntity", "ImageInfoEntity", "S3FileEntity", "AnnotationClassEntity", - "UserEntity", "TeamEntity", "MLModelEntity", "IntegrationEntity", diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 22f29feea..86e87a41e 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -15,8 +15,8 @@ from lib.core.enums import BaseTitledEnum from pydantic import BaseModel as PydanticBaseModel from pydantic import Extra -from pydantic import StrictStr from pydantic import Field +from pydantic import StrictStr from pydantic.datetime_parse import parse_datetime from pydantic.typing import is_namedtuple from pydantic.utils import ROOT_KEY @@ -295,7 +295,7 @@ def map_fields(entity: dict) -> dict: class TokenStr(StrictStr): - regex = r'^[-.@_A-Za-z0-9]+=\d+$' + regex = r"^[-.@_A-Za-z0-9]+=\d+$" @classmethod def validate(cls, value: Union[str]) -> Union[str]: diff --git a/src/superannotate/lib/core/entities/project.py b/src/superannotate/lib/core/entities/project.py index b036dc152..371599f5f 100644 --- a/src/superannotate/lib/core/entities/project.py +++ b/src/superannotate/lib/core/entities/project.py @@ -10,6 +10,7 @@ from lib.core.enums import BaseTitledEnum from lib.core.enums import ProjectStatus from lib.core.enums import ProjectType +from lib.core.enums import UserRole from pydantic import Extra from pydantic import Field from pydantic import StrictBool @@ -77,6 +78,17 @@ def __copy__(self): return SettingEntity(attribute=self.attribute, value=self.value) +class ContributorEntity(BaseModel): + id: Optional[str] + first_name: Optional[str] + last_name: Optional[str] + email: str + user_role: UserRole + + class Config: + extra = Extra.ignore + + class ProjectEntity(TimedBaseModel): id: Optional[int] team_id: Optional[int] @@ -93,7 +105,7 @@ class ProjectEntity(TimedBaseModel): upload_state: Optional[int] users: Optional[List[Any]] = [] unverified_users: Optional[List[Any]] = [] - contributors: List[Any] = [] + contributors: List[ContributorEntity] = [] settings: List[SettingEntity] = [] classes: List[AnnotationClassEntity] = [] workflows: Optional[List[WorkflowEntity]] = [] @@ -117,13 +129,12 @@ def __copy__(self): team_id=self.team_id, name=self.name, type=self.type, - description=self.description, - instructions_link=self.instructions_link - if self.description - else f"Copy of {self.name}.", + description=f"Copy of {self.name}.", + instructions_link=self.instructions_link, status=self.status, folder_id=self.folder_id, users=self.users, + settings=[s.__copy__() for s in self.settings], upload_state=self.upload_state, ) @@ -151,15 +162,6 @@ class Config: extra = Extra.ignore -class UserEntity(BaseModel): - id: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - email: Optional[str] - picture: Optional[str] - user_role: Optional[int] - - class TeamEntity(BaseModel): id: Optional[int] name: Optional[str] @@ -167,6 +169,6 @@ class TeamEntity(BaseModel): type: Optional[str] user_role: Optional[str] is_default: Optional[bool] - users: Optional[List[UserEntity]] + users: Optional[List[ContributorEntity]] pending_invitations: Optional[List] creator_id: Optional[str] diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index a98314812..929ef13ef 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -100,7 +100,7 @@ def images(self): class UserRole(BaseTitledEnum): - SUPER_ADMIN = "Superadmin", 1 + SUPER_ADMIN = "Superadmin", 1 # noqa ADMIN = "Admin", 2 ANNOTATOR = "Annotator", 3 QA = "QA", 4 diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index a523f1ee4..2a1b38374 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -1,4 +1,3 @@ -import copy import decimal import logging from collections import defaultdict @@ -7,7 +6,7 @@ import lib.core as constances from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ -from lib.core.entities import AnnotationClassEntity +from lib.core.entities import ContributorEntity from lib.core.entities import FolderEntity from lib.core.entities import ProjectEntity from lib.core.entities import SettingEntity @@ -18,7 +17,7 @@ from lib.core.serviceproviders import BaseServiceProvider from lib.core.usecases.base import BaseUseCase from lib.core.usecases.base import BaseUserBasedUseCase -from requests.exceptions import RequestException + logger = logging.getLogger("sa") @@ -207,8 +206,6 @@ def validate_settings(self): self._project.settings.append( SettingEntity(attribute="FrameMode", value=1) ) - else: - frame_mode.value = 1 except ValueError: raise AppValidationException("The FrameRate value should be float") @@ -381,251 +378,6 @@ def execute(self): return self._response -class CloneProjectUseCase(BaseUseCase): - def __init__( - self, - project: ProjectEntity, - project_to_create: ProjectEntity, - service_provider: BaseServiceProvider, - include_annotation_classes: bool = True, - include_settings: bool = True, - include_workflow: bool = True, - include_contributors: bool = False, - ): - super().__init__() - self._project = project - self._project_to_create = project_to_create - self._service_provider = service_provider - self._include_annotation_classes = include_annotation_classes - self._include_settings = include_settings - self._include_workflow = include_workflow - self._include_contributors = include_contributors - - def validate_project_name(self): - if self._project_to_create.name: - if ( - len( - set(self._project_to_create.name).intersection( - constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES - ) - ) - > 0 - ): - self._project_to_create.name = "".join( - "_" - if char in constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES - else char - for char in self._project_to_create.name - ) - logger.warning( - "New folder name has special characters. Special characters will be replaced by underscores." - ) - condition = Condition("name", self._project_to_create.name, EQ) - for project in self._service_provider.projects.list(condition).data: - if project.name == self._project_to_create.name: - logger.error("There are duplicated names.") - raise AppValidationException( - f"Project name {self._project_to_create.name} is not unique. " - f"To use SDK please make project names unique." - ) - - def _copy_annotation_classes( - self, annotation_classes_entity_mapping: dict, project: ProjectEntity - ): - annotation_classes = self._service_provider.annotation_classes.list( - Condition("project_id", self._project.id, EQ) - ).data - for annotation_class in annotation_classes: - annotation_class_copy = copy.copy(annotation_class) - annotation_classes_entity_mapping[ - annotation_class.id - ] = self._service_provider.annotation_classes.create( - project.id, annotation_class_copy - ).data - - def _copy_include_contributors(self, to_project: ProjectEntity): - from_project = self._service_provider.projects.get(uuid=self._project.id).data - users = [] - for user in from_project.users: - users.append( - {"user_id": user.get("user_id"), "user_role": user.get("user_role")} - ) - - for user in from_project.unverified_users: - users.append( - {"user_id": user.get("email"), "user_role": user.get("user_role")} - ) - if users: - self._service_provider.projects.share(to_project, users) - - def _copy_settings(self, to_project: ProjectEntity): - new_settings = [] - for setting in self._service_provider.projects.list_settings( - self._project - ).data: - if setting.attribute == "WorkflowType" and not self._include_workflow: - continue - for new_setting in self._service_provider.projects.list_settings( - to_project - ).data: - if new_setting.attribute == setting.attribute: - setting_copy = copy.copy(setting) - setting_copy.id = new_setting.id - setting_copy.project_id = to_project.id - new_settings.append(setting_copy) - - self._service_provider.projects.set_settings(to_project, new_settings) - - def _copy_workflow( - self, annotation_classes_entity_mapping: dict, to_project: ProjectEntity - ): - existing_workflow_ids = list( - map( - lambda i: i.id, - self._service_provider.projects.list_workflows(to_project).data, - ) - ) - for workflow in self._service_provider.projects.list_workflows( - self._project - ).data: - workflow_data = copy.copy(workflow) - workflow_data.project_id = to_project.id - if workflow.class_id not in annotation_classes_entity_mapping: - continue - workflow_data.class_id = annotation_classes_entity_mapping[ - workflow.class_id - ]["id"] - self._service_provider.projects.set_workflow(to_project, workflow_data) - workflows = self._service_provider.projects.list_workflows(to_project).data - new_workflow = next( - ( - work_flow - for work_flow in workflows - if work_flow.id not in existing_workflow_ids - ), - None, - ) - workflow_attributes = [] - for attribute in workflow_data.attribute: - for annotation_attribute in annotation_classes_entity_mapping[ - workflow.class_id - ]["attribute_groups"]: - if ( - attribute["attribute"]["attribute_group"]["name"] - == annotation_attribute["name"] - ): - for annotation_attribute_value in annotation_attribute[ - "attributes" - ]: - if ( - annotation_attribute_value.get("name") - == attribute["attribute"]["name"] - ): - workflow_attributes.append( - { - "workflow_id": new_workflow.id, - "attribute_id": annotation_attribute_value[ - "id" - ], - } - ) - break - if workflow_attributes: - self._service_provider.projects.set_project_workflow_attributes( - project=to_project, - attributes=workflow_attributes, - ) - - def execute(self): - if self.is_valid(): - if self._project_to_create.type in ( - constances.ProjectType.PIXEL.value, - constances.ProjectType.VECTOR.value, - ): - self._project_to_create.upload_state = ( - constances.UploadState.INITIAL.value - ) - - self._project_to_create.status = constances.ProjectStatus.NotStarted.value - - project = self._service_provider.projects.create( - self._project_to_create - ).data - logger.info( - f"Created project {self._project_to_create.name} with type" - f" {constances.ProjectType.get_name(self._project_to_create.type)}." - ) - # annotation_classes_entity_mapping = defaultdict(dict) - annotation_classes_entity_mapping = defaultdict(AnnotationClassEntity) - annotation_classes_created = False - if self._include_settings: - logger.info( - f"Cloning settings from {self._project.name} to {self._project_to_create.name}." - ) - try: - self._copy_settings(project) - except (AppException, RequestException) as e: - logger.info( - f"Failed to clone settings from {self._project.name} to {self._project_to_create.name}." - ) - logger.debug(str(e), exc_info=True) - - if self._include_contributors: - logger.info( - f"Cloning contributors from {self._project.name} to {self._project_to_create.name}." - ) - try: - self._copy_include_contributors(project) - except (AppException, RequestException) as e: - logger.warning( - f"Failed to clone contributors from {self._project.name} to {self._project_to_create.name}." - ) - logger.debug(str(e), exc_info=True) - - if self._include_annotation_classes: - logger.info( - f"Cloning annotation classes from {self._project.name} to {self._project_to_create.name}." - ) - try: - self._copy_annotation_classes( - annotation_classes_entity_mapping, project - ) - annotation_classes_created = True - except (AppException, RequestException) as e: - logger.warning( - f"Failed to clone annotation classes from {self._project.name} to {self._project_to_create.name}." - ) - logger.debug(str(e), exc_info=True) - - if self._include_workflow: - if self._project.type in ( - constances.ProjectType.DOCUMENT.value, - constances.ProjectType.VIDEO.value, - ): - logger.warning( - "Workflow copy is deprecated for " - f"{constances.ProjectType.get_name(self._project_to_create.type)} projects." - ) - elif not annotation_classes_created: - logger.info( - f"Skipping the workflow clone from {self._project.name} to {self._project_to_create.name}." - ) - else: - logger.info( - f"Cloning workflow from {self._project.name} to {self._project_to_create.name}." - ) - try: - self._copy_workflow(annotation_classes_entity_mapping, project) - except (AppException, RequestException) as e: - logger.warning( - f"Failed to workflow from {self._project.name} to {self._project_to_create.name}." - ) - logger.debug(str(e), exc_info=True) - - self._response.data = self._service_provider.projects.get(project.id).data - return self._response - - class UnShareProjectUseCase(BaseUseCase): def __init__( self, @@ -918,7 +670,7 @@ def execute(self): return self._response -class AddContributorsToProject(BaseUserBasedUseCase): +class AddContributorsToProject(BaseUseCase): """ Returns tuple of lists (added, skipped) """ @@ -927,28 +679,25 @@ def __init__( self, team: TeamEntity, project: ProjectEntity, - emails: list, - role: str, + contributors: List[ContributorEntity], service_provider: BaseServiceProvider, ): - super().__init__(emails) + super().__init__() self._team = team self._project = project - self._role = role + self._contributors = contributors self._service_provider = service_provider - @property - def user_role(self): - return constances.UserRole.get_value(self._role) - def validate_emails(self): - emails = list(set(self._emails)) - len_unique, len_provided = len(emails), len(self._emails) + email_entity_map = {} + for c in self._contributors: + email_entity_map[c.email] = c + len_unique, len_provided = len(email_entity_map), len(self._contributors) if len_unique < len_provided: logger.info( f"Dropping duplicates. Found {len_unique}/{len_provided} unique users." ) - self._emails = emails + self._contributors = email_entity_map.values() def execute(self): if self.is_valid(): @@ -966,27 +715,37 @@ def execute(self): if user["user_role"] > constances.UserRole.ADMIN.value: project_users.add(user["email"]) - to_add = list(team_users.intersection(self._emails) - project_users) - to_skip = list(set(self._emails).difference(to_add)) + role_email_map = defaultdict(list) + to_skip = [] + to_add = [] + for contributor in self._contributors: + role_email_map[contributor.user_role].append(contributor.email) + for role, emails in role_email_map.items(): + _to_add = list(team_users.intersection(emails) - project_users) + to_add.extend(_to_add) + to_skip.extend(list(set(emails).difference(_to_add))) + if _to_add: + response = self._service_provider.projects.share( + project=self._project, + users=[ + dict( + user_id=user_id, + user_role=constances.UserRole.get_value(role), + ) + for user_id in _to_add + ], + ) + if response and not response.data.get("invalidUsers"): + logger.info( + f"Added {len(_to_add)}/{len(emails)} " + f"contributors to the project {self._project.name} with the {role} role." + ) if to_skip: logger.warning( - f"Skipped {len(to_skip)}/{len(self._emails)} " + f"Skipped {len(to_skip)}/{len(self._contributors)} " "contributors that are out of the team scope or already have access to the project." ) - if to_add: - response = self._service_provider.projects.share( - project=self._project, - users=[ - dict(user_id=user_id, user_role=self.user_role) - for user_id in to_add - ], - ) - if response and not response.data.get("invalidUsers"): - logger.info( - f"Added {len(to_add)}/{len(self._emails)} " - f"contributors to the project {self._project.name} with the {self._role} role." - ) self._response.data = to_add, to_skip return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index a1d3efef2..38c0d8c4b 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -16,6 +16,7 @@ from lib.core.entities import AttachmentEntity from lib.core.entities import BaseItemEntity from lib.core.entities import ConfigEntity +from lib.core.entities import ContributorEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity from lib.core.entities import MLModelEntity @@ -109,26 +110,6 @@ def update(self, entity: ProjectEntity) -> Response: ) return use_case.execute() - def clone( - self, - project: ProjectEntity, - new_project: ProjectEntity, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, - ): - use_case = usecases.CloneProjectUseCase( - project=project, - service_provider=self.service_provider, - project_to_create=new_project, - include_contributors=copy_contributors, - include_settings=copy_settings, - include_workflow=copy_workflow, - include_annotation_classes=copy_annotation_classes, - ) - return use_case.execute() - def set_settings(self, project: ProjectEntity, settings: List[SettingEntity]): use_case = usecases.UpdateSettingsUseCase( to_update=settings, @@ -157,13 +138,17 @@ def set_workflows(self, project: ProjectEntity, steps: List): ) return use_case.execute() - def add_contributors(self, project: ProjectEntity, team, emails: list, role: str): + def add_contributors( + self, + team: TeamEntity, + project: ProjectEntity, + contributors: List[ContributorEntity], + ): project = self.get_metadata(project).data use_case = usecases.AddContributorsToProject( team=team, project=project, - emails=emails, - role=role, + contributors=contributors, service_provider=self.service_provider, ) return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py index cf092cb96..6c5746183 100644 --- a/src/superannotate/lib/infrastructure/services/http_client.py +++ b/src/superannotate/lib/infrastructure/services/http_client.py @@ -207,6 +207,6 @@ def serialize_response( data["data"] = data_json return content_type(**data) except json.decoder.JSONDecodeError: - data['_error'] = response.content + data["_error"] = response.content data["reason"] = response.reason return content_type(**data) diff --git a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py index 922581f53..6ce93ea28 100644 --- a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py +++ b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py @@ -8,8 +8,7 @@ sa = SAClient() -class \ - TestAnnotationUploadVector(BaseTestCase): +class TestAnnotationUploadVector(BaseTestCase): PROJECT_NAME = "Test-Upload_annotations_from_folder_to_project" PROJECT_DESCRIPTION = "Desc" PROJECT_TYPE = "Vector" @@ -123,7 +122,7 @@ def test_upload_big_annotations(self): ) == 4 -class TestExporeExportUploadVector(BaseTestCase): +class TestExportUploadVector(BaseTestCase): PROJECT_NAME = "Test-TestExporeExportUploadVector" PROJECT_DESCRIPTION = "Desc" PROJECT_TYPE = "Vector" diff --git a/tests/integration/projects/test_clone_project.py b/tests/integration/projects/test_clone_project.py index 4021c77a6..29dbf28c8 100644 --- a/tests/integration/projects/test_clone_project.py +++ b/tests/integration/projects/test_clone_project.py @@ -4,6 +4,7 @@ import pytest import src.superannotate.lib.core as constances from src.superannotate import SAClient +from superannotate import AppException from tests import DATA_SET_PATH sa = SAClient() @@ -16,6 +17,39 @@ class TestCloneProject(TestCase): PROJECT_TYPE = "Vector" IMAGE_QUALITY = "original" PATH_TO_URLS = "attach_urls.csv" + ANNOTATION_CLASSES = [ + { + "name": "tall", + "is_multiselect": 0, + "attributes": [{"name": "yes"}, {"name": "no"}], + }, + { + "name": "age", + "is_multiselect": 0, + "attributes": [{"name": "young"}, {"name": "old"}], + }, + ] + WORKFLOWS = [ + { + "step": 1, + "className": "rrr", + "tool": 3, + "attribute": [ + { + "attribute": { + "name": "young", + "attribute_group": {"name": "age"}, + } + }, + { + "attribute": { + "name": "yes", + "attribute_group": {"name": "tall"}, + } + }, + ], + } + ] def setUp(self, *args, **kwargs): self.tearDown() @@ -27,8 +61,8 @@ def tearDown(self) -> None: sa.delete_project(self.PROJECT_NAME_1) sa.delete_project(self.PROJECT_NAME_2) - def test_create_like_project(self): - _, _, _ = sa.attach_items( + def test_clone_project(self): + sa.attach_items( self.PROJECT_NAME_1, os.path.join(DATA_SET_PATH, self.PATH_TO_URLS), ) @@ -37,18 +71,7 @@ def test_create_like_project(self): self.PROJECT_NAME_1, "rrr", "#FFAAFF", - [ - { - "name": "tall", - "is_multiselect": 0, - "attributes": [{"name": "yes"}, {"name": "no"}], - }, - { - "name": "age", - "is_multiselect": 0, - "attributes": [{"name": "young"}, {"name": "old"}], - }, - ], + self.ANNOTATION_CLASSES, ) sa.set_project_default_image_quality_in_editor( @@ -56,56 +79,35 @@ def test_create_like_project(self): ) sa.set_project_workflow( self.PROJECT_NAME_1, - [ - { - "step": 1, - "className": "rrr", - "tool": 3, - "attribute": [ - { - "attribute": { - "name": "young", - "attribute_group": {"name": "age"}, - } - }, - { - "attribute": { - "name": "yes", - "attribute_group": {"name": "tall"}, - } - }, - ], - } - ], + self.WORKFLOWS, ) new_project = sa.clone_project( - self.PROJECT_NAME_2, - self.PROJECT_NAME_1, + project_name=self.PROJECT_NAME_2, + from_project=self.PROJECT_NAME_1, copy_contributors=True, + copy_workflow=True, copy_annotation_classes=True, ) self.assertEqual( - new_project["upload_state"], constances.UploadState.INITIAL.name + new_project["upload_state"], constances.UploadState.EXTERNAL.name ) - new_settings = sa.get_project_settings(self.PROJECT_NAME_2) - image_quality = None for setting in new_settings: - if setting["attribute"].lower() == "imagequality": - image_quality = setting["value"] + if setting["attribute"].lower() == "imageQuality".lower(): + self.assertEqual(setting["value"], self.IMAGE_QUALITY) break - self.assertEqual(image_quality, self.IMAGE_QUALITY) - self.assertEqual(new_project["description"], self.PROJECT_DESCRIPTION) + self.assertEqual(new_project["description"], f"Copy of {self.PROJECT_NAME_1}.") self.assertEqual(new_project["type"].lower(), "vector") ann_classes = sa.search_annotation_classes(self.PROJECT_NAME_2) self.assertEqual(len(ann_classes), 1) self.assertEqual(ann_classes[0]["name"], "rrr") + # TODO add annotation class attributes assets + # self.assertEqual(ann_classes[0]["attributes"], "rrr") new_workflow = sa.get_project_workflow(self.PROJECT_NAME_2) self.assertEqual(len(new_workflow), 1) self.assertEqual(new_workflow[0]["className"], "rrr") self.assertEqual(new_workflow[0]["tool"], 3) - self.assertEqual(len(new_workflow[0]["attribute"]), 2) self.assertEqual(new_workflow[0]["attribute"][0]["attribute"]["name"], "young") self.assertEqual( new_workflow[0]["attribute"][0]["attribute"]["attribute_group"]["name"], @@ -176,3 +178,91 @@ def test_create_like_project(self): "Workflow copy is deprecated for Document projects.", self._caplog.text ) assert new_project["status"], constances.ProjectStatus.NotStarted.name + + +class TestCloneVideoProject(TestCase): + PROJECT_NAME_1 = "test_create_like_video_project_1" + PROJECT_NAME_2 = "test_create_like_video_project_2" + PROJECT_TYPE = "Video" + PROJECT_DESCRIPTION = "desc" + + def setUp(self, *args, **kwargs): + self.tearDown() + + def tearDown(self) -> None: + for i in (self.PROJECT_NAME_1, self.PROJECT_NAME_2): + try: + sa.delete_project(i) + except AppException: + ... + + def test_clone_video_project(self): + self._project_1 = sa.create_project( + self.PROJECT_NAME_1, + self.PROJECT_DESCRIPTION, + self.PROJECT_TYPE, + ) + new_project = sa.clone_project( + project_name=self.PROJECT_NAME_2, + from_project=self.PROJECT_NAME_1, + ) + self.assertEqual( + new_project["upload_state"], constances.UploadState.EXTERNAL.name + ) + self.assertEqual(new_project["name"], self.PROJECT_NAME_2) + self.assertEqual(new_project["type"].lower(), "video") + self.assertEqual(new_project["description"], f"Copy of {self.PROJECT_NAME_1}.") + + def test_clone_video_project_frame_mode_on(self): + self._project_1 = sa.create_project( + self.PROJECT_NAME_1, + self.PROJECT_DESCRIPTION, + self.PROJECT_TYPE, + settings=[{"attribute": "FrameRate", "value": 3}], + ) + new_project = sa.clone_project( + project_name=self.PROJECT_NAME_2, + from_project=self.PROJECT_NAME_1, + copy_settings=True, + ) + + self.assertEqual(new_project["type"].lower(), "video") + self.assertEqual(new_project["name"], self.PROJECT_NAME_2) + + new_settings = sa.get_project_settings(self.PROJECT_NAME_2) + for s in new_settings: + if s["attribute"] == "FrameRate": + assert s["value"] == 3 + elif s["attribute"] == "FrameMode": + assert s["value"] + + def test_clone_video_project_frame_mode_off(self): + self._project_1 = sa.create_project( + self.PROJECT_NAME_1, + self.PROJECT_DESCRIPTION, + self.PROJECT_TYPE, + ) + sa.clone_project( + project_name=self.PROJECT_NAME_2, + from_project=self.PROJECT_NAME_1, + copy_settings=True, + ) + + new_settings = sa.get_project_settings(self.PROJECT_NAME_2) + for s in new_settings: + if s["attribute"] == "FrameMode": + assert not s["value"] + + def test_clone_video_project_via_copy_workflow(self): + self._project_1 = sa.create_project( + self.PROJECT_NAME_1, self.PROJECT_DESCRIPTION, self.PROJECT_TYPE + ) + + with self.assertRaisesRegexp( + AppException, "Workflow is not supported in Video project." + ): + sa.clone_project( + project_name=self.PROJECT_NAME_2, + from_project=self.PROJECT_NAME_1, + copy_workflow=True, + ) diff --git a/tests/integration/projects/test_create_project.py b/tests/integration/projects/test_create_project.py index fbcde602e..825dcc25b 100644 --- a/tests/integration/projects/test_create_project.py +++ b/tests/integration/projects/test_create_project.py @@ -1,36 +1,16 @@ import copy from unittest import TestCase +import src.superannotate.lib.core as constances from src.superannotate import AppException from src.superannotate import SAClient -sa = SAClient() - -class BaseTestCase(TestCase): - PROJECT_1 = "project_1" - PROJECT_2 = "project_2" +sa = SAClient() - def setUp(self, *args, **kwargs): - self.tearDown() - def tearDown(self) -> None: - try: - for project_name in (self.PROJECT_1, self.PROJECT_2): - projects = sa.search_projects(project_name, return_metadata=True) - for project in projects: - try: - sa.delete_project(project) - except Exception: - pass - except Exception as e: - print(str(e)) - - -class TestSearchProjectVector(BaseTestCase): - PROJECT_1 = "project_1TestSearchProject" - PROJECT_2 = "project_2TestSearchProject" - PROJECT_TYPE = "Vector" +class ProjectCreateBaseTestCase(TestCase): + PROJECT = "test_vector_project" CLASSES = [ { "type": 1, @@ -50,7 +30,6 @@ class TestSearchProjectVector(BaseTestCase): ], } ] - WORKFLOWS = [ { "step": 1, @@ -92,38 +71,46 @@ class TestSearchProjectVector(BaseTestCase): }, ] - @property - def projects(self): - return self.PROJECT_2, self.PROJECT_1 + def setUp(self, *args, **kwargs): + self.tearDown() + + def tearDown(self) -> None: + try: + sa.delete_project(self.PROJECT) + except AppException: + ... + + +class TestCreateVectorProject(ProjectCreateBaseTestCase): + PROJECT_TYPE = "Vector" - def test_created_project(self): - # check datetime - project = sa.create_project(self.PROJECT_1, "desc", self.PROJECT_TYPE) + def test_create_project_datetime(self): + project = sa.create_project(self.PROJECT, "desc", self.PROJECT_TYPE) metadata = sa.get_project_metadata(project["name"]) assert "Z" not in metadata["createdAt"] - def test_create_project_wrong_type(self): + def test_create_project_with_wrong_type(self): with self.assertRaisesRegexp( AppException, "Available values are 'Vector', 'Pixel', 'Video', 'Document', 'Tiled', 'Other', 'PointCloud'.", ): - sa.create_project(self.PROJECT_1, "desc", "wrong_type") + sa.create_project(self.PROJECT, "desc", "wrong_type") def test_create_project_with_settings(self): sa.create_project( - self.PROJECT_1, + self.PROJECT, "desc", self.PROJECT_TYPE, [{"attribute": "ImageQuality", "value": "original"}], ) - project = sa.get_project_metadata(self.PROJECT_1, include_settings=True) + project = sa.get_project_metadata(self.PROJECT, include_settings=True) for setting in project["settings"]: if setting["attribute"] == "ImageQuality": assert setting["value"] == "original" - def test_create_with_classes_and_workflows(self): + def test_create_project_with_classes_and_workflows(self): project = sa.create_project( - self.PROJECT_1, + self.PROJECT, "desc", self.PROJECT_TYPE, classes=self.CLASSES, @@ -132,7 +119,6 @@ def test_create_with_classes_and_workflows(self): assert len(project["classes"]) == 1 assert len(project["classes"][0]["attribute_groups"]) == 1 assert len(project["classes"][0]["attribute_groups"][0]["attributes"]) == 3 - assert len(project["workflows"]) == 2 assert project["workflows"][0]["className"] == self.CLASSES[0]["name"] assert project["workflows"][0]["attribute"][0]["attribute"]["name"] == "Car" @@ -140,28 +126,21 @@ def test_create_with_classes_and_workflows(self): assert project["workflows"][1]["attribute"][0]["attribute"]["name"] == "Track" assert project["workflows"][1]["attribute"][1]["attribute"]["name"] == "Bus" - def test_create_with_workflow_without_classes(self): + def test_create_project_with_workflow_without_classes(self): with self.assertRaisesRegexp( AppException, "Project with workflows can not be created without classes." ): sa.create_project( - self.PROJECT_1, "desc", self.PROJECT_TYPE, workflows=self.WORKFLOWS + self.PROJECT, "desc", self.PROJECT_TYPE, workflows=self.WORKFLOWS ) - def test_create_wrong_project_type(self): - with self.assertRaisesRegexp( - AppException, "Workflow is not supported in Video project." - ): - sa.create_project(self.PROJECT_1, "desc", "Video", workflows=self.WORKFLOWS) - - def test_create_with_workflow_wrong_classes(self): - # with self.assertRaisesRegexp didnt work + def test_create_project_with_workflow_and_wrong_classes(self): try: workflows = copy.copy(self.WORKFLOWS) workflows[0]["className"] = "1" workflows[1]["className"] = "2" sa.create_project( - self.PROJECT_1, + self.PROJECT, "desc", self.PROJECT_TYPE, classes=self.CLASSES, @@ -171,23 +150,38 @@ def test_create_with_workflow_wrong_classes(self): assert str(e) == "There are no [1, 2] classes created in the project." -class TestSearchProjectVideo(BaseTestCase): - PROJECT_1 = "project_1TestSearchProjectVideo" - PROJECT_2 = "project_2TestSearchProjectVideo" +class TestCreateVideoProject(ProjectCreateBaseTestCase): + PROJECT = "test_video_project" PROJECT_TYPE = "Video" - @property - def projects(self): - return self.PROJECT_2, self.PROJECT_1 + def test_create_wrong_video_project_with_workflow(self): + with self.assertRaisesRegexp( + AppException, "Workflow is not supported in Video project." + ): + sa.create_project( + self.PROJECT, "desc", self.PROJECT_TYPE, workflows=self.WORKFLOWS + ) - def test_create_project_with_settings(self): + def test_create_video_project_frame_mode_off(self): + sa.create_project( + self.PROJECT, + "desc", + self.PROJECT_TYPE, + ) + project = sa.get_project_metadata(self.PROJECT, include_settings=True) + self.assertEqual(project["upload_state"], constances.UploadState.EXTERNAL.name) + for setting in project["settings"]: + if setting["attribute"] == "FrameMode": + assert not setting["value"] + + def test_create_video_project_frame_mode_on(self): sa.create_project( - self.PROJECT_1, + self.PROJECT, "desc", self.PROJECT_TYPE, [{"attribute": "FrameRate", "value": 1.0}], ) - project = sa.get_project_metadata(self.PROJECT_1, include_settings=True) + project = sa.get_project_metadata(self.PROJECT, include_settings=True) for setting in project["settings"]: if setting["attribute"] == "FrameRate": - assert setting["value"] == 1 + assert setting["value"] == 1.0 diff --git a/tests/integration/settings/test_settings.py b/tests/integration/settings/test_settings.py index fd6fdc50e..65c640a58 100644 --- a/tests/integration/settings/test_settings.py +++ b/tests/integration/settings/test_settings.py @@ -82,24 +82,6 @@ def test_create_from_metadata(self): else: raise Exception("Test failed") - def test_clone_project(self): - sa.create_project( - self.PROJECT_NAME, - self.PROJECT_DESCRIPTION, - self.PROJECT_TYPE, - [{"attribute": "ImageQuality", "value": "original"}], - ) - sa.clone_project( - self.SECOND_PROJECT_NAME, self.PROJECT_NAME, copy_settings=True - ) - settings = sa.get_project_settings(self.SECOND_PROJECT_NAME) - for setting in settings: - if setting["attribute"] == "ImageQuality": - assert setting["value"] == "original" - break - else: - raise Exception("Test failed") - def test_frame_rate_invalid_range_value(self): with self.assertRaisesRegexp( AppException, "FrameRate is available only for Video projects" diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 741f6c28d..dcb353d70 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -6,7 +6,6 @@ from unittest.mock import patch import superannotate.lib.core as constants -from superannotate.lib.app.interface.types import validate_arguments from superannotate import AppException from superannotate import SAClient @@ -53,12 +52,14 @@ def test_init_via_config_json_invalid_json(self): config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path), patch( - "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path + "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path ): with open(f"{config_dir}/config.json", "w") as config_json: json.dump({"token": "INVALID_TOKEN"}, config_json) for kwargs in ({}, {"config_path": f"{config_dir}/config.json"}): - with self.assertRaisesRegexp(AppException, r"(\s+)token(\s+)Invalid token."): + with self.assertRaisesRegexp( + AppException, r"(\s+)token(\s+)Invalid token." + ): SAClient(**kwargs) @patch("lib.core.usecases.GetTeamUseCase") @@ -106,7 +107,7 @@ def test_init_via_config_ini_invalid_token(self): config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path), patch( - "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path + "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path ): with open(f"{config_dir}/config.ini", "w") as config_ini: config_parser = ConfigParser() @@ -118,7 +119,9 @@ def test_init_via_config_ini_invalid_token(self): config_parser.write(config_ini) for kwargs in ({}, {"config_path": f"{config_dir}/config.ini"}): - with self.assertRaisesRegexp(AppException, r"(\s+)SA_TOKEN(\s+)Invalid token."): + with self.assertRaisesRegexp( + AppException, r"(\s+)SA_TOKEN(\s+)Invalid token." + ): SAClient(**kwargs) def test_invalid_config_path(self):