diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 16771f2e2..a6c453961 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -74,6 +74,9 @@ ______ .. autofunction:: superannotate.query .. autofunction:: superannotate.search_items +.. autofunction:: superannotate.attach_items +.. autofunction:: superannotate.copy_items +.. autofunction:: superannotate.move_items .. autofunction:: superannotate.get_item_metadata ---------- diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index e0b6ff051..5c95a9322 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -25,6 +25,7 @@ attach_document_urls_to_project, ) from superannotate.lib.app.interface.sdk_interface import attach_image_urls_to_project +from superannotate.lib.app.interface.sdk_interface import attach_items from superannotate.lib.app.interface.sdk_interface import ( attach_items_from_integrated_storage, ) @@ -34,6 +35,7 @@ from superannotate.lib.app.interface.sdk_interface import consensus from superannotate.lib.app.interface.sdk_interface import copy_image from superannotate.lib.app.interface.sdk_interface import copy_images +from superannotate.lib.app.interface.sdk_interface import copy_items from superannotate.lib.app.interface.sdk_interface import create_annotation_class from superannotate.lib.app.interface.sdk_interface import ( create_annotation_classes_from_classes_json, @@ -71,6 +73,7 @@ from superannotate.lib.app.interface.sdk_interface import init from superannotate.lib.app.interface.sdk_interface import invite_contributors_to_team from superannotate.lib.app.interface.sdk_interface import move_images +from superannotate.lib.app.interface.sdk_interface import move_items from superannotate.lib.app.interface.sdk_interface import pin_image from superannotate.lib.app.interface.sdk_interface import prepare_export from superannotate.lib.app.interface.sdk_interface import query @@ -175,6 +178,9 @@ "get_item_metadata", "search_items", "query", + "attach_items", + "copy_items", + "move_items", # Image Section "copy_images", "move_images", diff --git a/src/superannotate/lib/app/helpers.py b/src/superannotate/lib/app/helpers.py index f50d0ec74..39c2e12ab 100644 --- a/src/superannotate/lib/app/helpers.py +++ b/src/superannotate/lib/app/helpers.py @@ -8,6 +8,7 @@ import boto3 import pandas as pd +from superannotate.lib.app.exceptions import AppException from superannotate.lib.app.exceptions import PathError from superannotate.lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from superannotate.lib.core import PIXEL_ANNOTATION_POSTFIX @@ -168,3 +169,33 @@ def get_paths_and_duplicated_from_csv(csv_path): else: duplicate_images.append(temp) return images_to_upload, duplicate_images + + +def get_name_url_duplicated_from_csv(csv_path): + image_data = pd.read_csv(csv_path, dtype=str) + if "url" not in image_data.columns: + raise AppException("Column 'url' is required") + image_data = image_data[~image_data["url"].isnull()] + if "name" in image_data.columns: + image_data["name"] = ( + image_data["name"] + .fillna("") + .apply(lambda cell: cell if str(cell).strip() else str(uuid.uuid4())) + ) + else: + image_data["name"] = [str(uuid.uuid4()) for _ in range(len(image_data.index))] + + image_data = pd.DataFrame(image_data, columns=["name", "url"]) + img_names_urls = image_data.to_dict(orient="records") + duplicate_images = [] + seen = [] + images_to_upload = [] + for i in img_names_urls: + temp = i["name"] + i["name"] = i["name"].strip() + if i["name"] not in seen: + seen.append(i["name"]) + images_to_upload.append(i) + else: + duplicate_images.append(temp) + return images_to_upload, duplicate_images diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 7f75b5da9..9bf5a613d 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -1,3 +1,4 @@ +import collections import io import json import os @@ -17,10 +18,13 @@ from lib.app.annotation_helpers import add_annotation_point_to_json from lib.app.helpers import extract_project_folder from lib.app.helpers import get_annotation_paths +from lib.app.helpers import get_name_url_duplicated_from_csv from lib.app.helpers import get_paths_and_duplicated_from_csv from lib.app.interface.types import AnnotationStatuses from lib.app.interface.types import AnnotationType from lib.app.interface.types import AnnotatorRole +from lib.app.interface.types import AttachmentArg +from lib.app.interface.types import AttachmentDict from lib.app.interface.types import ClassType from lib.app.interface.types import EmailStr from lib.app.interface.types import ImageQualityChoices @@ -36,6 +40,7 @@ from lib.app.serializers import SettingsSerializer from lib.app.serializers import TeamSerializer from lib.core import LIMITED_FUNCTIONS +from lib.core.entities import AttachmentEntity from lib.core.entities.integrations import IntegrationEntity from lib.core.entities.project_entities import AnnotationClassEntity from lib.core.enums import ImageQuality @@ -297,6 +302,7 @@ def search_images( "We're deprecating the search_images function. Please use search_items instead. Learn more." "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.search_items" ) + logger.warning(warning_msg) warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) project = Controller.get_default()._get_project(project_name) @@ -587,7 +593,12 @@ def copy_images( :return: list of skipped image names :rtype: list of strs """ - + warning_msg = ( + "We're deprecating the copy_images function. Please use copy_items instead. Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.copy_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, source_folder_name = extract_project_folder(source_project) to_project_name, destination_folder_name = extract_project_folder( @@ -595,7 +606,7 @@ def copy_images( ) if project_name != to_project_name: raise AppException( - "Source and destination projects should be the same for copy_images" + "Source and destination projects should be the same" ) if not image_names: images = ( @@ -651,6 +662,12 @@ def move_images( :return: list of skipped image names :rtype: list of strs """ + warning_msg = ( + "We're deprecating the move_images function. Please use move_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.move_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, source_folder_name = extract_project_folder(source_project) project = Controller.get_default().get_project_metadata(project_name).data @@ -1810,6 +1827,12 @@ def attach_image_urls_to_project( :return: list of linked image names, list of failed image names, list of duplicate image names :rtype: tuple """ + warning_msg = ( + "We're deprecating the attach_image_urls_to_project function. Please use attach_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) project = Controller.get_default().get_project_metadata(project_name).data project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") @@ -1877,6 +1900,12 @@ def attach_video_urls_to_project( :return: attached videos, failed videos, skipped videos :rtype: (list, list, list) """ + warning_msg = ( + "We're deprecating the attach_video_urls_to_project function. Please use attach_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) project = Controller.get_default().get_project_metadata(project_name).data project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") @@ -2479,8 +2508,10 @@ def search_images_all_folders( :param project: project name :type project: str + :param image_name_prefix: image name prefix for search :type image_name_prefix: str + :param annotation_status: if not None, annotation statuses of images to filter, should be one of NotStarted InProgress QualityCheck Returned Completed Skipped :type annotation_status: str @@ -2735,6 +2766,12 @@ def attach_document_urls_to_project( :return: list of attached documents, list of not attached documents, list of skipped documents :rtype: tuple """ + warning_msg = ( + "We're deprecating the attach_document_urls_to_project function. Please use attach_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) project = Controller.get_default().get_project_metadata(project_name).data project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") @@ -3087,3 +3124,127 @@ def search_items( if response.errors: raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) + + +@Trackable +@validate_arguments +def attach_items( + project: Union[NotEmptyStr, dict], + attachments: AttachmentArg, + annotation_status: Optional[AnnotationStatuses] = "NotStarted" +): + attachments = attachments.data + project_name, folder_name = extract_project_folder(project) + if attachments and isinstance(attachments[0], AttachmentDict): + unique_attachments = set(attachments) + duplicate_attachments = [item for item, count in collections.Counter(attachments).items() if count > 1] + else: + unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv(attachments) + if duplicate_attachments: + logger.info("Dropping duplicates.") + unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments) + uploaded, fails, duplicated = [], [], [] + if unique_attachments: + logger.info(f"Attaching {len(unique_attachments)} file(s) to project {project}.") + response = Controller.get_default().attach_items( + project_name=project_name, + folder_name=folder_name, + attachments=unique_attachments, + annotation_status=annotation_status, + ) + if response.errors: + raise AppException(response.errors) + uploaded, duplicated = response.data + uploaded = [i["name"] for i in uploaded] + fails = [ + attachment.name + for attachment in unique_attachments + if attachment.name not in uploaded and attachment.name not in duplicated + ] + return uploaded, fails, duplicated + + +@Trackable +@validate_arguments +def copy_items( + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, + include_annotations: Optional[StrictBool] = True, +): + """Copy images in bulk between folders in a project + + :param source: project name or folder path to select items from (e.g., “project1/folder1”). + :type source: str + + :param destination: project name (root) or folder path to place copied items. + :type destination: str + + :param items: names of items to copy. If None, all items from the source directory will be copied. + :type itmes: list of str + + :param include_annotations: enables annotations copy + :type include_annotations: bool + + :return: list of skipped item names + :rtype: list of strs + """ + + project_name, source_folder = extract_project_folder(source) + + to_project_name, destination_folder = extract_project_folder(destination) + if project_name != to_project_name: + raise AppException( + "Source and destination projects should be the same for copy_images" + ) + + response = Controller.get_default().copy_items( + project_name=project_name, + from_folder=source_folder, + to_folder=destination_folder, + items=items, + include_annotations=include_annotations, + ) + if response.errors: + raise AppException(response.errors) + + return response.data + + +@Trackable +@validate_arguments +def move_items( + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, +): + """Copy images in bulk between folders in a project + + :param source: project name or folder path to pick items from (e.g., “project1/folder1”). + :type source: str + + :param destination: project name (root) or folder path to move items to. + :type destination: str + + :param items: names of items to move. If None, all items from the source directory will be moved. + :type items: list of str + + :return: list of skipped item names + :rtype: list of strs + """ + + project_name, source_folder = extract_project_folder(source) + to_project_name, destination_folder = extract_project_folder(destination) + if project_name != to_project_name: + raise AppException( + "Source and destination projects should be the same" + ) + response = Controller.get_default().move_items( + project_name=project_name, + from_folder=source_folder, + to_folder=destination_folder, + items=items, + ) + if response.errors: + raise AppException(response.errors) + return response.data diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index 3d571d0ca..e47e8841d 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -1,4 +1,7 @@ +import uuid from functools import wraps +from pathlib import Path +from typing import Optional from typing import Union from lib.core.enums import AnnotationStatus @@ -8,7 +11,13 @@ from lib.core.enums import UserRole from lib.core.exceptions import AppException from lib.infrastructure.validators import wrap_error +from pydantic import BaseModel +from pydantic import conlist from pydantic import constr +from pydantic import Extra +from pydantic import Field +from pydantic import parse_obj_as +from pydantic import root_validator from pydantic import StrictStr from pydantic import validate_arguments as pydantic_validate_arguments from pydantic import ValidationError @@ -22,7 +31,9 @@ class EmailStr(StrictStr): def validate(cls, value: Union[str]) -> Union[str]: try: constr( - regex=r"^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" + regex=r"^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)" + r"*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}" + r"[a-zA-Z0-9])?)*$" ).validate(value) except StrRegexError: raise ValueError("Invalid email") @@ -79,6 +90,44 @@ def validate(cls, value: Union[str]) -> Union[str]: return value +class AttachmentDict(BaseModel): + url: StrictStr + name: Optional[StrictStr] = Field(default_factory=lambda: str(uuid.uuid4())) + + class Config: + extra = Extra.ignore + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return self.url == other.url and self.name.strip() == other.name.strip() + + +AttachmentArgType = Union[NotEmptyStr, Path, conlist(AttachmentDict, min_items=1)] + + +class AttachmentArg(BaseModel): + __root__: AttachmentArgType + + def __getitem__(self, index): + return self.__root__[index] + + @property + def data(self): + return self.__root__ + + @root_validator(pre=True) + def validate_root(cls, values): + try: + parse_obj_as(AttachmentArgType, values["__root__"]) + except ValidationError: + raise ValueError( + "The value must be str, path, or list of dicts with the required 'url' and optional 'name' keys" + ) + return values + + class ImageQualityChoices(StrictStr): VALID_CHOICES = ["compressed", "original"] diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index e6eed4cac..43543e042 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1029,3 +1029,47 @@ def search_items(**kwargs): "recursive": bool(recursive), }, } + + +def move_items(**kwargs): + project = kwargs["project"] + project_name, _ = extract_project_folder(project) + project = Controller.get_default().get_project_metadata(project_name).data["project"] + items = kwargs["items"] + return { + "event_name": "move_items", + "properties": { + "project_type": ProjectType.get_name(project.project_type), + "items_count": len(items) if items else None + }, + } + + +def copy_items(**kwargs): + project = kwargs["project"] + project_name, _ = extract_project_folder(project) + project = Controller.get_default().get_project_metadata(project_name).data["project"] + items = kwargs["items"] + return { + "event_name": "copy_items", + "properties": { + "project_type": ProjectType.get_name(project.project_type), + "items_count": len(items) if items else None, + "include_annotations": kwargs["include_annotations"] + }, + } + + +def attach_items(**kwargs): + project = kwargs["project"] + project_name, _ = extract_project_folder(project) + project = Controller.get_default().get_project_metadata(project_name).data["project"] + attachments = kwargs["attachments"] + return { + "event_name": "copy_items", + "properties": { + "project_type": ProjectType.get_name(project.project_type), + "attachments": "scv" if isinstance(attachments, (str, Path)) else "dict", + "annotation_status": kwargs["annotation_status"] + }, + } diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 5a9a8e87e..1b1fed04c 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -61,6 +61,7 @@ ALREADY_EXISTING_FILES_WARNING = ( "{} already existing file(s) found that won't be uploaded." ) + ATTACHING_FILES_MESSAGE = "Attaching {} file(s) to project {}." ATTACHING_UPLOAD_STATE_ERROR = "You cannot attach URLs in this type of project. Please attach it in an external storage project." diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index 88425761a..78fad0ca9 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -1,3 +1,4 @@ +from lib.core.entities.base import AttachmentEntity from lib.core.entities.base import BaseEntity as TmpBaseEntity from lib.core.entities.integrations import IntegrationEntity from lib.core.entities.items import DocumentEntity @@ -33,6 +34,8 @@ "Entity", "VideoEntity", "DocumentEntity", + # Utils + "AttachmentEntity", # project "ProjectEntity", "ProjectSettingEntity", diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index b733cda88..ffd46f2d4 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -1,3 +1,4 @@ +import uuid from datetime import datetime from typing import Optional @@ -25,3 +26,11 @@ class BaseEntity(TimedBaseModel): class Config: extra = Extra.allow + + +class AttachmentEntity(BaseModel): + name: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4())) + url: str + + class Config: + extra = Extra.ignore diff --git a/src/superannotate/lib/core/exceptions.py b/src/superannotate/lib/core/exceptions.py index e6331dc87..4228e5cb2 100644 --- a/src/superannotate/lib/core/exceptions.py +++ b/src/superannotate/lib/core/exceptions.py @@ -7,12 +7,18 @@ class AppException(Exception): def __init__(self, message): super().__init__(message) - self.message = message + self.message = str(message) def __str__(self): return self.message +class BackendError(AppException): + """ + Backend Error + """ + + class AppValidationException(AppException): """ App validation exception diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 9da0dc83f..111dfae1b 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -116,13 +116,13 @@ def get_upload_token( def update_image(self, image_id: int, team_id: int, project_id: int, data: dict): raise NotImplementedError - def copy_images_between_folders_transaction( + def copy_items_between_folders_transaction( self, team_id: int, project_id: int, from_folder_id: int, to_folder_id: int, - images: List[str], + items: List[str], include_annotations: bool = False, include_pin: bool = False, ) -> int: @@ -151,6 +151,9 @@ def get_progress( ) -> Tuple[int, int]: raise NotImplementedError + def await_progress(self, project_id: int, team_id: int, poll_id: int, items_count) -> Tuple[int, int]: + raise NotImplementedError + def set_images_statuses_bulk( self, image_names: List[str], diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index a35a835e4..165a1b0f5 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -579,7 +579,7 @@ def execute(self): return self._response for i in range(0, len(images_to_copy), self.CHUNK_SIZE): - poll_id = self._backend_service.copy_images_between_folders_transaction( + poll_id = self._backend_service.copy_items_between_folders_transaction( team_id=self._project.team_id, project_id=self._project.uuid, from_folder_id=self._from_folder.uuid, diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 00bcffdce..5988b5890 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -4,6 +4,7 @@ import superannotate.lib.core as constances from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ +from lib.core.entities import AttachmentEntity from lib.core.entities import DocumentEntity from lib.core.entities import Entity from lib.core.entities import FolderEntity @@ -12,6 +13,8 @@ from lib.core.entities import TmpImageEntity from lib.core.entities import VideoEntity from lib.core.exceptions import AppException +from lib.core.exceptions import AppValidationException +from lib.core.exceptions import BackendError from lib.core.reporter import Reporter from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response @@ -22,12 +25,12 @@ class GetItem(BaseReportableUseCae): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - items: BaseReadOnlyRepository, - item_name: str, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + items: BaseReadOnlyRepository, + item_name: str, ): super().__init__(reporter) self._project = project @@ -40,8 +43,8 @@ def serialize_entity(entity: Entity, project: ProjectEntity): if project.upload_state != constances.UploadState.EXTERNAL.value: entity.url = None if project.project_type in ( - constances.ProjectType.VECTOR.value, - constances.ProjectType.PIXEL.value, + constances.ProjectType.VECTOR.value, + constances.ProjectType.PIXEL.value, ): tmp_entity = entity if project.project_type == constances.ProjectType.VECTOR.value: @@ -59,10 +62,10 @@ def serialize_entity(entity: Entity, project: ProjectEntity): def execute(self) -> Response: if self.is_valid(): condition = ( - Condition("name", self._item_name, EQ) - & Condition("team_id", self._project.team_id, EQ) - & Condition("project_id", self._project.uuid, EQ) - & Condition("folder_id", self._folder.uuid, EQ) + Condition("name", self._item_name, EQ) + & Condition("team_id", self._project.team_id, EQ) + & Condition("project_id", self._project.uuid, EQ) + & Condition("folder_id", self._folder.uuid, EQ) ) entity = self._items.get_one(condition) if entity: @@ -75,12 +78,12 @@ def execute(self) -> Response: class QueryEntities(BaseReportableUseCae): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - backend_service_provider: SuperannotateServiceProvider, - query: str, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + backend_service_provider: SuperannotateServiceProvider, + query: str, ): super().__init__(reporter) self._project = project @@ -121,14 +124,14 @@ def execute(self) -> Response: class ListItems(BaseReportableUseCae): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - items: BaseReadOnlyRepository, - search_condition: Condition, - folders: BaseReadOnlyRepository, - recursive: bool = False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + items: BaseReadOnlyRepository, + search_condition: Condition, + folders: BaseReadOnlyRepository, + recursive: bool = False, ): super().__init__(reporter) self._project = project @@ -178,3 +181,285 @@ def execute(self) -> Response: ) self._response.data = items return self._response + + +class AttachItems(BaseReportableUseCae): + CHUNK_SIZE = 500 + + def __init__( + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + attachments: List[AttachmentEntity], + annotation_status: str, + backend_service_provider: SuperannotateServiceProvider, + upload_state_code: int = constances.UploadState.EXTERNAL.value, + ): + super().__init__(reporter) + self._project = project + self._folder = folder + self._attachments = attachments + self._annotation_status_code = constances.AnnotationStatus.get_value(annotation_status) + self._upload_state_code = upload_state_code + self._backend_service = backend_service_provider + self._attachments_count = None + + @property + def attachments_count(self): + if not self._attachments_count: + self._attachments_count = len(self._attachments) + return self._attachments_count + + def validate_limitations(self): + attachments_count = self.attachments_count + response = self._backend_service.get_limitations( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._folder.uuid, + ) + if not response.ok: + raise AppValidationException(response.error) + if attachments_count > response.data.folder_limit.remaining_image_count: + raise AppValidationException(constances.ATTACH_FOLDER_LIMIT_ERROR_MESSAGE) + elif attachments_count > response.data.project_limit.remaining_image_count: + raise AppValidationException(constances.ATTACH_PROJECT_LIMIT_ERROR_MESSAGE) + elif ( + response.data.user_limit + and attachments_count > response.data.user_limit.remaining_image_count + ): + raise AppValidationException(constances.ATTACH_USER_LIMIT_ERROR_MESSAGE) + + def validate_upload_state(self): + if self._project.upload_state == constances.UploadState.BASIC.value: + raise AppValidationException(constances.ATTACHING_UPLOAD_STATE_ERROR) + + @staticmethod + def generate_meta(): + return { + "width": None, + "height": None + } + + def execute(self) -> Response: + if self.is_valid(): + duplications = [] + attached = [] + for i in range(0, self.attachments_count, self.CHUNK_SIZE): + attachments = self._attachments[i: i + self.CHUNK_SIZE] # noqa: E203 + response = self._backend_service.get_bulk_images( + project_id=self._project.uuid, + team_id=self._project.team_id, + folder_id=self._folder.uuid, + images=[attachment.name for attachment in attachments], + ) + if isinstance(response, dict) and "error" in response: + raise AppException(response["error"]) + duplications.extend([image["name"] for image in response]) + to_upload = [] + to_upload_meta = {} + for attachment in attachments: + if attachment.name not in duplications: + to_upload.append({"name": attachment.name, "url": attachment.url}) + to_upload_meta[attachment.name] = self.generate_meta() + if to_upload: + backend_response = self._backend_service.attach_files( + project_id=self._project.uuid, + folder_id=self._folder.uuid, + team_id=self._project.team_id, + files=to_upload, + annotation_status_code=self._annotation_status_code, + upload_state_code=self._upload_state_code, + meta=to_upload_meta + ) + if "error" in backend_response: + self._response.errors = AppException(backend_response["error"]) + else: + attached.extend(backend_response) + self._response.data = attached, duplications + return self._response + + +class CopyItems(BaseReportableUseCae): + """ + Copy items in bulk between folders in a project. + Return skipped item names. + """ + + CHUNK_SIZE = 1000 + + def __init__( + self, + reporter: Reporter, + project: ProjectEntity, + from_folder: FolderEntity, + to_folder: FolderEntity, + item_names: List[str], + items: BaseReadOnlyRepository, + backend_service_provider: SuperannotateServiceProvider, + include_annotations: bool, + ): + super().__init__(reporter) + self._project = project + self._from_folder = from_folder + self._to_folder = to_folder + self._item_names = item_names + self._items = items + self._backend_service = backend_service_provider + self._include_annotations = include_annotations + + def _validate_limitations(self, items_count): + response = self._backend_service.get_limitations( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._to_folder.uuid, + ) + if not response.ok: + raise AppValidationException(response.error) + if items_count > response.data.folder_limit.remaining_image_count: + raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE) + if items_count > response.data.project_limit.remaining_image_count: + raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE) + + def validate_project_type(self): + if self._project.project_type in constances.LIMITED_FUNCTIONS: + raise AppValidationException( + constances.LIMITED_FUNCTIONS[self._project.project_type] + ) + + def validate_item_names(self): + if self._item_names: + self._item_names = list(set(self._item_names)) + + def execute(self): + if self.is_valid(): + skipped_images, duplications = [], [] + if not self._item_names: + condition = ( + Condition("team_id", self._project.team_id, EQ) + & Condition("project_id", self._project.uuid, EQ) + & Condition("folder_id", self._from_folder.uuid, EQ) + ) + items = self._items.get_all(condition) + items_to_copy = [item.name for item in items] + else: + items = self._backend_service.get_bulk_images( + project_id=self._project.uuid, + team_id=self._project.team_id, + folder_id=self._to_folder.uuid, + images=self._item_names, + ) + duplications = [item["name"] for item in items] + items_to_copy = set(self._item_names) - set(duplications) + skipped_images = duplications + try: + self._validate_limitations(len(items_to_copy)) + except AppValidationException as e: + self._response.errors = e + return self._response + if items_to_copy: + for i in range(0, len(items_to_copy), self.CHUNK_SIZE): + chunk_to_copy = items_to_copy[i: i + self.CHUNK_SIZE] # noqa: E203 + poll_id = self._backend_service.copy_items_between_folders_transaction( + team_id=self._project.team_id, + project_id=self._project.uuid, + from_folder_id=self._from_folder.uuid, + to_folder_id=self._to_folder.uuid, + items=chunk_to_copy, + include_annotations=self._include_annotations, + ) + if not poll_id: + skipped_images.append(chunk_to_copy) + continue + try: + self._backend_service.await_progress( + self._project.uuid, + self._project.team_id, + poll_id=poll_id, + items_count=len(chunk_to_copy) + ) + except BackendError as e: + self._response.errors = AppException(e) + return self._response + self.reporter.log_info( + f"Copied {len(items_to_copy)}/{len(items)} items(s) from " + f"{self._project.name}{'' if self._from_folder.is_root else f'/{self._from_folder.name}'} to " + f"{self._project.name}{'' if self._to_folder.is_root else f'/{self._to_folder.name}'}" + ) + self._response.data = skipped_images + return self._response + + +class MoveItems(BaseReportableUseCae): + CHUNK_SIZE = 1000 + + def __init__( + self, + reporter: Reporter, + project: ProjectEntity, + from_folder: FolderEntity, + to_folder: FolderEntity, + item_names: List[str], + items: BaseReadOnlyRepository, + backend_service_provider: SuperannotateServiceProvider, + ): + super().__init__(reporter) + self._project = project + self._from_folder = from_folder + self._to_folder = to_folder + self._item_names = item_names + self._items = items + self._backend_service = backend_service_provider + + def validate_item_names(self): + if self._item_names: + self._item_names = list(set(self._item_names)) + + def _validate_limitations(self, items_count): + response = self._backend_service.get_limitations( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._to_folder.uuid, + ) + if not response.ok: + raise AppValidationException(response.error) + if items_count > response.data.folder_limit.remaining_image_count: + raise AppValidationException(constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE) + if items_count > response.data.project_limit.remaining_image_count: + raise AppValidationException(constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE) + + def execute(self): + if self.is_valid(): + if not self._item_names: + condition = ( + Condition("team_id", self._project.team_id, EQ) + & Condition("project_id", self._project.uuid, EQ) + & Condition("folder_id", self._from_folder.uuid, EQ) + ) + items = [item.name for item in self._items.get_all(condition)] + else: + items = self._item_names + try: + self._validate_limitations(len(items)) + except AppValidationException as e: + self._response.errors = e + return self._response + moved_images = [] + for i in range(0, len(items), self.CHUNK_SIZE): + moved_images.extend( + self._backend_service.move_images_between_folders( + team_id=self._project.team_id, + project_id=self._project.uuid, + from_folder_id=self._from_folder.uuid, + to_folder_id=self._to_folder.uuid, + images=items[i : i + self.CHUNK_SIZE], # noqa: E203 + ) + ) + self.reporter.log_info( + f"Moved {len(moved_images)}/{len(items)} items(s) from " + f"{self._project.name}{'' if self._from_folder.is_root else f'/{self._from_folder.name}'} to " + f"{self._project.name}{'' if self._to_folder.is_root else f'/{self._to_folder.name}'}" + ) + + self._response.data = list(set(items) - set(moved_images)) + return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index ab8dab295..f255eb586 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -15,6 +15,7 @@ from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import AnnotationClassEntity +from lib.core.entities import AttachmentEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity from lib.core.entities import MLModelEntity @@ -289,11 +290,11 @@ def get_folder_name(name: str = None): return "root" def search_project( - self, - name: str = None, - include_complete_image_count=False, - statuses: Union[List[str], Tuple[str]] = (), - **kwargs, + self, + name: str = None, + include_complete_image_count=False, + statuses: Union[List[str], Tuple[str]] = (), + **kwargs, ) -> Response: condition = Condition.get_empty_condition() if name: @@ -315,14 +316,14 @@ def search_project( return use_case.execute() def create_project( - self, - name: str, - description: str, - project_type: str, - settings: Iterable = tuple(), - annotation_classes: Iterable = tuple(), - workflows: Iterable = tuple(), - **extra_kwargs + self, + name: str, + description: str, + project_type: str, + settings: Iterable = tuple(), + annotation_classes: Iterable = tuple(), + workflows: Iterable = tuple(), + **extra_kwargs ) -> Response: try: @@ -368,14 +369,14 @@ def update_project(self, name: str, project_data: dict) -> Response: return use_case.execute() def upload_image_to_project( - self, - project_name: str, - folder_name: str, - image_name: str, - image: Union[str, io.BytesIO] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + image_name: str, + image: Union[str, io.BytesIO] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -403,13 +404,13 @@ def upload_image_to_project( ).execute() def upload_images_to_project( - self, - project_name: str, - folder_name: str, - paths: List[str], - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + paths: List[str], + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -429,16 +430,16 @@ def upload_images_to_project( ) def upload_images_from_folder_to_project( - self, - project_name: str, - folder_name: str, - folder_path: str, - extensions: Optional[List[str]] = None, - annotation_status: str = None, - exclude_file_patterns: Optional[List[str]] = None, - recursive_sub_folders: Optional[bool] = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + folder_path: str, + extensions: Optional[List[str]] = None, + annotation_status: str = None, + exclude_file_patterns: Optional[List[str]] = None, + recursive_sub_folders: Optional[bool] = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -461,14 +462,14 @@ def upload_images_from_folder_to_project( ) def clone_project( - self, - name: str, - from_name: str, - project_description: str, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, + self, + name: str, + from_name: str, + project_description: str, + copy_annotation_classes=True, + copy_settings=True, + copy_workflow=True, + copy_contributors=False, ): project = self._get_project(from_name) @@ -493,12 +494,12 @@ def clone_project( return use_case.execute() def interactive_attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, + self, + project_name: str, + files: List[ImageEntity], + folder_name: str = None, + annotation_status: str = None, + upload_state_code: int = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -533,7 +534,7 @@ def get_folder(self, project_name: str, folder_name: str): return use_case.execute() def search_folders( - self, project_name: str, folder_name: str = None, include_users=False, **kwargs + self, project_name: str, folder_name: str = None, include_users=False, **kwargs ): condition = Condition.get_empty_condition() if kwargs: @@ -563,12 +564,12 @@ def delete_folders(self, project_name: str, folder_names: List[str]): return use_case.execute() def prepare_export( - self, - project_name: str, - folder_names: List[str], - include_fuse: bool, - only_pinned: bool, - annotation_statuses: List[str] = None, + self, + project_name: str, + folder_names: List[str], + include_fuse: bool, + only_pinned: bool, + annotation_statuses: List[str] = None, ): project = self._get_project(project_name) @@ -600,11 +601,11 @@ def search_team_contributors(self, **kwargs): return use_case.execute() def search_images( - self, - project_name: str, - folder_path: str = None, - annotation_status: str = None, - image_name_prefix: str = None, + self, + project_name: str, + folder_path: str = None, + annotation_status: str = None, + image_name_prefix: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -619,7 +620,7 @@ def search_images( return use_case.execute() def _get_image( - self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, + self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, ) -> ImageEntity: response = usecases.GetImageUseCase( service=self._backend_client, @@ -633,7 +634,7 @@ def _get_image( return response.data def get_image( - self, project_name: str, image_name: str, folder_path: str = None + self, project_name: str, image_name: str, folder_path: str = None ) -> ImageEntity: project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -644,18 +645,18 @@ def update_folder(self, project_name: str, folder_name: str, folder_data: dict): folder = self._get_folder(project, folder_name) for field, value in folder_data.items(): setattr(folder, field, value) - use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder,) + use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder, ) return use_case.execute() def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, + copy_annotation_status: bool = False, + move: bool = False, ): from_project = self._get_project(from_project_name) to_project = self._get_project(to_project_name) @@ -678,12 +679,12 @@ def copy_image( return use_case.execute() def copy_image_annotation_classes( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, ): from_project = self._get_project(from_project_name) from_folder = self._get_folder(from_project, from_folder_name) @@ -718,7 +719,7 @@ def copy_image_annotation_classes( return use_case.execute() def update_image( - self, project_name: str, image_name: str, folder_name: str = None, **kwargs + self, project_name: str, image_name: str, folder_name: str = None, **kwargs ): image = self.get_image( project_name=project_name, image_name=image_name, folder_path=folder_name @@ -729,13 +730,13 @@ def update_image( return use_case.execute() def bulk_copy_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], - include_annotations: bool, - include_pin: bool, + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], + include_annotations: bool, + include_pin: bool, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -752,11 +753,11 @@ def bulk_copy_images( return use_case.execute() def bulk_move_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -771,13 +772,13 @@ def bulk_move_images( return use_case.execute() def get_project_metadata( - self, - project_name: str, - include_annotation_classes: bool = False, - include_settings: bool = False, - include_workflow: bool = False, - include_contributors: bool = False, - include_complete_image_count: bool = False, + self, + project_name: str, + include_annotation_classes: bool = False, + include_settings: bool = False, + include_workflow: bool = False, + include_contributors: bool = False, + include_complete_image_count: bool = False, ): project = self._get_project(project_name) @@ -861,11 +862,11 @@ def get_image_metadata(self, project_name: str, folder_name: str, image_name: st return use_case.execute() def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, + self, + project_name: str, + folder_name: str, + image_names: list, + annotation_status: str, ): project_entity = self._get_project(project_name) folder_entity = self._get_folder(project_entity, folder_name) @@ -883,7 +884,7 @@ def set_images_annotation_statuses( return use_case.execute() def delete_images( - self, project_name: str, folder_name: str, image_names: List[str] = None, + self, project_name: str, folder_name: str, image_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -898,7 +899,7 @@ def delete_images( return use_case.execute() def assign_images( - self, project_name: str, folder_name: str, image_names: list, user: str + self, project_name: str, folder_name: str, image_names: list, user: str ): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) @@ -961,7 +962,7 @@ def un_share_project(self, project_name: str, user_id: str): return use_case.execute() def download_image_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -978,14 +979,6 @@ def download_image_annotations( ) return use_case.execute() - @staticmethod - def get_image_from_s3(s3_bucket, image_path: str): - use_case = usecases.GetS3ImageUseCase( - s3_bucket=s3_bucket, image_path=image_path - ) - use_case.execute() - return use_case.execute() - def get_exports(self, project_name: str, return_metadata: bool): project = self._get_project(project_name) @@ -997,7 +990,7 @@ def get_exports(self, project_name: str, return_metadata: bool): return use_case.execute() def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool + self, project_name: str, folder_name: str, with_all_subfolders: bool ): project = self._get_project(project_name) @@ -1013,12 +1006,12 @@ def get_project_image_count( return use_case.execute() def create_annotation_class( - self, - project_name: str, - name: str, - color: str, - attribute_groups: List[dict], - class_type: str, + self, + project_name: str, + name: str, + color: str, + attribute_groups: List[dict], + class_type: str, ): project = self._get_project(project_name) annotation_classes = AnnotationClassRepository( @@ -1080,15 +1073,15 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) return use_case.execute() def download_image( - self, - project_name: str, - image_name: str, - download_path: str, - folder_name: str = None, - image_variant: str = None, - include_annotations: bool = None, - include_fuse: bool = None, - include_overlay: bool = None, + self, + project_name: str, + image_name: str, + download_path: str, + folder_name: str = None, + image_variant: str = None, + include_annotations: bool = None, + include_fuse: bool = None, + include_overlay: bool = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1128,13 +1121,13 @@ def set_project_workflow(self, project_name: str, steps: list): return use_case.execute() def upload_annotations_from_folder( - self, - project_name: str, - folder_name: str, - annotation_paths: List[str], - client_s3_bucket=None, - is_pre_annotations: bool = False, - folder_path: str = None, + self, + project_name: str, + folder_name: str, + annotation_paths: List[str], + client_s3_bucket=None, + is_pre_annotations: bool = False, + folder_path: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1160,13 +1153,13 @@ def upload_annotations_from_folder( return use_case.execute() def upload_image_annotations( - self, - project_name: str, - folder_name: str, - image_name: str, - annotations: dict, - mask: io.BytesIO = None, - verbose: bool = True, + self, + project_name: str, + folder_name: str, + image_name: str, + annotations: dict, + mask: io.BytesIO = None, + verbose: bool = True, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1208,12 +1201,12 @@ def delete_model(self, model_id: int): return use_case.execute() def download_export( - self, - project_name: str, - export_name: str, - folder_path: str, - extract_zip_contents: bool, - to_s3_bucket: bool, + self, + project_name: str, + export_name: str, + folder_path: str, + extract_zip_contents: bool, + to_s3_bucket: bool, ): project = self._get_project(project_name) return usecases.DownloadExportUseCase( @@ -1244,14 +1237,14 @@ def download_ml_model(self, model_data: dict, download_path: str): return use_case.execute() def benchmark( - self, - project_name: str, - ground_truth_folder_name: str, - folder_names: List[str], - export_root: str, - image_list: List[str], - annot_type: str, - show_plots: bool, + self, + project_name: str, + ground_truth_folder_name: str, + folder_names: List[str], + export_root: str, + image_list: List[str], + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1287,13 +1280,13 @@ def benchmark( return use_case.execute() def consensus( - self, - project_name: str, - folder_names: list, - export_path: str, - image_list: list, - annot_type: str, - show_plots: bool, + self, + project_name: str, + folder_names: list, + export_path: str, + image_list: list, + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1327,7 +1320,7 @@ def consensus( return use_case.execute() def run_prediction( - self, project_name: str, images_list: list, model_name: str, folder_name: str + self, project_name: str, images_list: list, model_name: str, folder_name: str ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1345,7 +1338,7 @@ def run_prediction( return use_case.execute() def list_images( - self, project_name: str, annotation_status: str = None, name_prefix: str = None, + self, project_name: str, annotation_status: str = None, name_prefix: str = None, ): project = self._get_project(project_name) @@ -1358,12 +1351,12 @@ def list_images( return use_case.execute() def search_models( - self, - name: str, - model_type: str = None, - project_id: int = None, - task: str = None, - include_global: bool = True, + self, + name: str, + model_type: str = None, + project_id: int = None, + task: str = None, + include_global: bool = True, ): ml_models_repo = MLModelRepository( service=self._backend_client, team_id=self.team_id @@ -1386,10 +1379,10 @@ def search_models( return use_case.execute() def delete_annotations( - self, - project_name: str, - folder_name: str, - image_names: Optional[List[str]] = None, + self, + project_name: str, + folder_name: str, + image_names: Optional[List[str]] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1403,7 +1396,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = False ): use_case = usecases.ValidateAnnotationUseCase( project_type, @@ -1440,17 +1433,17 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): return use_case.execute() def upload_videos( - self, - project_name: str, - folder_name: str, - paths: List[str], - start_time: float, - extensions: List[str] = None, - exclude_file_patterns: List[str] = None, - end_time: Optional[float] = None, - target_fps: Optional[int] = None, - annotation_status: Optional[str] = None, - image_quality_in_editor: Optional[str] = None, + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1476,7 +1469,7 @@ def upload_videos( return use_case.execute() def get_annotations( - self, project_name: str, folder_name: str, item_names: List[str], logging=True + self, project_name: str, folder_name: str, item_names: List[str], logging=True ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1491,7 +1484,7 @@ def get_annotations( return use_case.execute() def get_annotations_per_frame( - self, project_name: str, folder_name: str, video_name: str, fps: int + self, project_name: str, folder_name: str, video_name: str, fps: int ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1508,7 +1501,7 @@ def get_annotations_per_frame( return use_case.execute() def upload_priority_scores( - self, project_name, folder_name, scores, project_folder_name + self, project_name, folder_name, scores, project_folder_name ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1532,11 +1525,11 @@ def get_integrations(self): return use_cae.execute() def attach_integrations( - self, - project_name: str, - folder_name: str, - integration: IntegrationEntity, - folder_path: str, + self, + project_name: str, + folder_name: str, + integration: IntegrationEntity, + folder_path: str, ): team = self.team_data.data project = self._get_project(project_name) @@ -1579,15 +1572,15 @@ def get_item(self, project_name: str, folder_name: str, item_name: str): return use_case.execute() def list_items( - self, - project_name: str, - folder_name: str, - name_contains: str = None, - annotation_status: str = None, - annotator_email: str = None, - qa_email: str = None, - recursive: bool = False, - **kwargs, + self, + project_name: str, + folder_name: str, + name_contains: str = None, + annotation_status: str = None, + annotator_email: str = None, + qa_email: str = None, + recursive: bool = False, + **kwargs, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1617,3 +1610,69 @@ def list_items( ) return use_case.execute() + + def attach_items( + self, + project_name: str, + folder_name: str, + attachments: List[AttachmentEntity], + annotation_status: str + ): + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + + use_case = usecases.AttachItems( + reporter=self.default_reporter, + project=project, + folder=folder, + attachments=attachments, + annotation_status=annotation_status, + backend_service_provider=self.backend_client + ) + return use_case.execute() + + def copy_items( + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, + include_annotations: bool = False, + ): + project = self._get_project(project_name) + from_folder = self._get_folder(project, from_folder) + to_folder = self._get_folder(project, to_folder) + + use_case = usecases.CopyItems( + self.default_reporter, + project=project, + from_folder=from_folder, + to_folder=to_folder, + item_names=items, + items=self.items, + backend_service_provider=self.backend_client, + include_annotations=include_annotations + ) + return use_case.execute() + + def move_items( + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, + ): + project = self._get_project(project_name) + from_folder = self._get_folder(project, from_folder) + to_folder = self._get_folder(project, to_folder) + + use_case = usecases.MoveItems( + self.default_reporter, + project=project, + from_folder=from_folder, + to_folder=to_folder, + item_names=items, + items=self.items, + backend_service_provider=self.backend_client, + ) + return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 98e467da5..e642b6a3c 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -1,5 +1,6 @@ import asyncio import json +import time from contextlib import contextmanager from datetime import datetime from typing import Dict @@ -12,6 +13,7 @@ import lib.core as constance import requests.packages.urllib3 from lib.core.exceptions import AppException +from lib.core.exceptions import BackendError from lib.core.reporter import Reporter from lib.core.service_types import DownloadMLModelAuthData from lib.core.service_types import ServiceResponse @@ -589,13 +591,13 @@ def update_image(self, image_id: int, team_id: int, project_id: int, data: dict) ) return res.ok - def copy_images_between_folders_transaction( + def copy_items_between_folders_transaction( self, team_id: int, project_id: int, from_folder_id: int, to_folder_id: int, - images: List[str], + items: List[str], include_annotations: bool = False, include_pin: bool = False, ) -> int: @@ -609,7 +611,7 @@ def copy_images_between_folders_transaction( params={"team_id": team_id, "project_id": project_id}, data={ "is_folder_copy": False, - "image_names": images, + "image_names": items, "destination_folder_id": to_folder_id, "source_folder_id": from_folder_id, "include_annotations": include_annotations, @@ -654,6 +656,18 @@ def get_progress( ).json() return res["done"], res["skipped"] + def await_progress(self, project_id: int, team_id: int, poll_id: int, items_count): + try: + await_time = items_count * 0.3 + timeout_start = time.time() + while time.time() < timeout_start + await_time: + done_count, skipped_count = self.get_progress(project_id, team_id, poll_id) + if done_count + skipped_count == items_count: + break + time.sleep(4) + except (AppException, Exception) as e: + raise BackendError(e) + def get_duplicated_images( self, project_id: int, team_id: int, folder_id: int, images: List[str] ) -> List[str]: diff --git a/tests/integration/integrations/test_get_integrations.py b/tests/integration/integrations/test_get_integrations.py index 0dd62d5f0..56673a9cc 100644 --- a/tests/integration/integrations/test_get_integrations.py +++ b/tests/integration/integrations/test_get_integrations.py @@ -12,7 +12,7 @@ class TestGetIntegrations(BaseTestCase): TEST_FOLDER_NAME = "test_folder" PROJECT_DESCRIPTION = "desc" PROJECT_TYPE = "Vector" - EXAMPLE_IMAGE = "example_image_1.jpg" + EXAMPLE_IMAGE = "egit xample_image_1.jpg" @property def folder_path(self): diff --git a/tests/integration/items/test_attach_items.py b/tests/integration/items/test_attach_items.py new file mode 100644 index 000000000..3351b356d --- /dev/null +++ b/tests/integration/items/test_attach_items.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path + +import src.superannotate as sa +from tests.integration.base import BaseTestCase + +import pytest + + +class TestAttachItemsVector(BaseTestCase): + PROJECT_NAME = "TestAttachItemsVector" + PROJECT_DESCRIPTION = "TestAttachItemsVector" + PROJECT_TYPE = "Vector" + CSV_PATH = "data_set/attach_urls.csv" + ATTACHED_IMAGE_NAME = "6022a74d5384c50017c366b3" + ATTACHMENT_LIST = [ + { + "url": "https://drive.google.com/uc?export=download&id=1vwfCpTzcjxoEA4hhDxqapPOVvLVeS7ZS", + "name": "6022a74d5384c50017c366b3" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V", + "name": "6022a74b5384c50017c366ad" + }, + { + "url": "1SfGcn9hdkVM35ZP0S93eStsE7Ti4GtHU", + "path": "123" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1geS2YtQiTYuiduEirKVYxBujHJaIWA3V", + "name": "6022a74b5384c50017c366ad" + }, + ] + + @property + def scv_path(self): + return os.path.join(Path(__file__).parent.parent.parent, self.CSV_PATH) + + def test_attached_items_csv(self): + uploaded, _, _ = sa.attach_items(self.PROJECT_NAME, self.scv_path) + assert len(uploaded) == 7 + uploaded, _, duplicated = sa.attach_items(self.PROJECT_NAME, self.scv_path) + assert len(uploaded) == 2 + assert len(duplicated) == 5 + + def test_attached_items_list_of_dict(self): + uploaded, _, _ = sa.attach_items(self.PROJECT_NAME, self.ATTACHMENT_LIST) + assert len(uploaded) == 3 + uploaded, _, duplicated = sa.attach_items(self.PROJECT_NAME, self.ATTACHMENT_LIST) + assert len(uploaded) == 1 + assert len(duplicated) == 2 + + +class TestCopyItems(BaseTestCase): + PROJECT_NAME = "TestCopyItemsVector" + PROJECT_DESCRIPTION = "TestCopyItemsVector" + PROJECT_TYPE = "Vector" + IMAGE_NAME ="test_image" + FOLDER_1 = "folder_1" + FOLDER_2 = "folder_2" + CSV_PATH = "data_set/attach_urls.csv" + + @property + def scv_path(self): + return os.path.join(Path(__file__).parent.parent.parent, self.CSV_PATH) + + def test_copy_items_from_root(self): + uploaded, _, _ = sa.attach_items(self.PROJECT_NAME, self.scv_path) + assert len(uploaded) == 7 + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + skipped_items = sa.copy_items(self.PROJECT_NAME, f"{self.PROJECT_NAME}/{self.FOLDER_1}") + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 7 + + def test_copy_items_from_folder(self): + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + sa.create_folder(self.PROJECT_NAME, self.FOLDER_2) + uploaded, _, _ = sa.attach_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}", self.scv_path) + assert len(uploaded) == 7 + skipped_items = sa.copy_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}", f"{self.PROJECT_NAME}/{self.FOLDER_2}") + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_2}")) == 7 + + def test_copy_item_with_annotations(self): + uploaded, _, _ = sa.attach_items( + self.PROJECT_NAME, [ + {"url": "https://drive.google.com/uc?export=download&id=1vwfCpTzcjxoEA4hhDxqapPOVvLVeS7ZS", + "name": self.IMAGE_NAME} + ] + ) + assert len(uploaded) == 1 + sa.create_annotation_class(self.PROJECT_NAME, "test_class", "#FF0000") + sa.add_annotation_bbox_to_image(self.PROJECT_NAME, self.IMAGE_NAME, [1, 2, 3, 4], "test_class") + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + skipped_items = sa.copy_items( + self.PROJECT_NAME, f"{self.PROJECT_NAME}/{self.FOLDER_1}", include_annotations=True + ) + annotations = sa.get_annotations(f"{self.PROJECT_NAME}/{self.FOLDER_1}") + assert len(annotations) == 1 + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 1 \ No newline at end of file diff --git a/tests/integration/items/test_copy_items.py b/tests/integration/items/test_copy_items.py new file mode 100644 index 000000000..555b12b3a --- /dev/null +++ b/tests/integration/items/test_copy_items.py @@ -0,0 +1,55 @@ +import os +from pathlib import Path + +import src.superannotate as sa +from tests.integration.base import BaseTestCase + + +class TestCopyItems(BaseTestCase): + PROJECT_NAME = "TestCopyItemsVector" + PROJECT_DESCRIPTION = "TestCopyItemsVector" + PROJECT_TYPE = "Vector" + IMAGE_NAME ="test_image" + FOLDER_1 = "folder_1" + FOLDER_2 = "folder_2" + CSV_PATH = "data_set/attach_urls.csv" + + @property + def scv_path(self): + return os.path.join(Path(__file__).parent.parent.parent, self.CSV_PATH) + + def test_copy_items_from_root(self): + uploaded, _, _ = sa.attach_items(self.PROJECT_NAME, self.scv_path) + assert len(uploaded) == 7 + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + skipped_items = sa.copy_items(self.PROJECT_NAME, f"{self.PROJECT_NAME}/{self.FOLDER_1}") + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 7 + + def test_copy_items_from_folder(self): + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + sa.create_folder(self.PROJECT_NAME, self.FOLDER_2) + uploaded, _, _ = sa.attach_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}", self.scv_path) + assert len(uploaded) == 7 + skipped_items = sa.copy_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}", f"{self.PROJECT_NAME}/{self.FOLDER_2}") + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_2}")) == 7 + + def test_copy_item_with_annotations(self): + uploaded, _, _ = sa.attach_items( + self.PROJECT_NAME, [ + {"url": "https://drive.google.com/uc?export=download&id=1vwfCpTzcjxoEA4hhDxqapPOVvLVeS7ZS", + "name": self.IMAGE_NAME} + ] + ) + assert len(uploaded) == 1 + sa.create_annotation_class(self.PROJECT_NAME, "test_class", "#FF0000") + sa.add_annotation_bbox_to_image(self.PROJECT_NAME, self.IMAGE_NAME, [1, 2, 3, 4], "test_class") + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + skipped_items = sa.copy_items( + self.PROJECT_NAME, f"{self.PROJECT_NAME}/{self.FOLDER_1}", include_annotations=True + ) + annotations = sa.get_annotations(f"{self.PROJECT_NAME}/{self.FOLDER_1}") + assert len(annotations) == 1 + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 1 \ No newline at end of file diff --git a/tests/integration/items/test_move_items.py b/tests/integration/items/test_move_items.py new file mode 100644 index 000000000..daf970161 --- /dev/null +++ b/tests/integration/items/test_move_items.py @@ -0,0 +1,56 @@ +import os +from pathlib import Path + +import src.superannotate as sa +from tests.integration.base import BaseTestCase + + +class TestMoveItems(BaseTestCase): + PROJECT_NAME = "TestCopyItemsVector" + PROJECT_DESCRIPTION = "TestCopyItemsVector" + PROJECT_TYPE = "Vector" + IMAGE_NAME = "test_image" + FOLDER_1 = "folder_1" + FOLDER_2 = "folder_2" + CSV_PATH = "data_set/attach_urls.csv" + + @property + def scv_path(self): + return os.path.join(Path(__file__).parent.parent.parent, self.CSV_PATH) + + def test_move_items_from_root(self): + uploaded, _, _ = sa.attach_items(self.PROJECT_NAME, self.scv_path) + assert len(uploaded) == 7 + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + skipped_items = sa.move_items(self.PROJECT_NAME, f"{self.PROJECT_NAME}/{self.FOLDER_1}") + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 7 + + def test_move_items_from_folder(self): + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + sa.create_folder(self.PROJECT_NAME, self.FOLDER_2) + uploaded, _, _ = sa.attach_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}", self.scv_path) + assert len(uploaded) == 7 + skipped_items = sa.move_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}", f"{self.PROJECT_NAME}/{self.FOLDER_2}") + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_2}")) == 7 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 0 + + def test_move_item_with_annotations(self): + uploaded, _, _ = sa.attach_items( + self.PROJECT_NAME, [ + {"url": "https://drive.google.com/uc?export=download&id=1vwfCpTzcjxoEA4hhDxqapPOVvLVeS7ZS", + "name": self.IMAGE_NAME} + ] + ) + assert len(uploaded) == 1 + sa.create_annotation_class(self.PROJECT_NAME, "test_class", "#FF0000") + sa.add_annotation_bbox_to_image(self.PROJECT_NAME, self.IMAGE_NAME, [1, 2, 3, 4], "test_class") + sa.create_folder(self.PROJECT_NAME, self.FOLDER_1) + skipped_items = sa.move_items( + self.PROJECT_NAME, f"{self.PROJECT_NAME}/{self.FOLDER_1}" + ) + annotations = sa.get_annotations(f"{self.PROJECT_NAME}/{self.FOLDER_1}") + assert len(annotations) == 1 + assert len(skipped_items) == 0 + assert len(sa.search_items(f"{self.PROJECT_NAME}/{self.FOLDER_1}")) == 1