From 57fe0820f6de42259b6158db83c19607df7d17da Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 10 May 2022 10:10:18 +0400 Subject: [PATCH 01/59] client initial changes --- src/superannotate/__init__.py | 194 +- src/superannotate/lib/__init__.py | 12 - .../lib/app/interface/base_interface.py | 134 +- .../lib/app/interface/cli_interface.py | 35 +- .../lib/app/interface/sdk_interface.py | 6157 +++++++++-------- src/superannotate/lib/app/interface/types.py | 4 +- src/superannotate/lib/app/mixp/decorators.py | 6 +- src/superannotate/lib/core/reporter.py | 40 + src/superannotate/lib/core/usecases/models.py | 3 +- .../lib/infrastructure/controller.py | 625 +- src/superannotate/version.py | 1 - tests/convertors/test_consensus.py | 3 +- tests/convertors/test_conversion.py | 3 +- .../test_json_version_conversion.py | 3 +- tests/convertors/test_project_converter.py | 3 +- .../aggregations/test_df_processing.py | 3 +- .../test_docuement_annotation_to_df.py | 3 +- .../test_video_annotation_to_df.py | 3 +- .../test_create_annotation_class.py | 3 +- .../annotations/test_annotation_adding.py | 4 +- .../annotations/test_annotation_class_new.py | 3 +- .../annotations/test_annotation_classes.py | 3 +- .../annotations/test_annotation_delete.py | 3 +- .../test_annotation_upload_pixel.py | 3 +- .../test_annotation_upload_vector.py | 3 +- .../test_annotations_pre_processing.py | 5 +- .../test_annotations_upload_status_change.py | 3 +- .../annotations/test_get_annotations.py | 9 +- .../test_get_annotations_per_frame.py | 5 +- .../test_missing_annotation_upload.py | 3 +- .../annotations/test_preannotation_upload.py | 3 +- .../test_text_annotation_upload.py | 3 +- ...est_uopload_annotations_without_classes.py | 3 +- ...load_annotations_from_folder_to_project.py | 3 +- .../test_video_annotation_upload.py | 3 +- tests/integration/base.py | 5 +- .../classes/test_create_annotation_class.py | 3 +- .../classes/test_tag_annotation_classes.py | 3 +- tests/integration/folders/test_folders.py | 3 +- .../integrations/test_get_integrations.py | 3 +- tests/integration/items/test_attach_items.py | 3 +- tests/integration/items/test_copy_items.py | 3 +- .../items/test_get_item_metadata.py | 3 +- tests/integration/items/test_move_items.py | 3 +- tests/integration/items/test_saqul_query.py | 3 +- tests/integration/items/test_search_items.py | 3 +- .../items/test_set_annotation_statuses.py | 3 +- .../test_add_contributors_to_project.py | 18 +- .../projects/test_basic_project.py | 3 +- .../projects/test_clone_project.py | 3 +- .../projects/test_create_project.py | 3 +- .../projects/test_project_rename.py | 3 +- tests/integration/settings/test_settings.py | 3 +- tests/integration/test_assign_images.py | 3 +- .../integration/test_attach_document_urls.py | 3 +- tests/integration/test_attach_video_urls.py | 3 +- tests/integration/test_basic_images.py | 3 +- tests/integration/test_benchmark.py | 3 +- tests/integration/test_cli.py | 41 +- .../integration/test_create_from_full_info.py | 3 +- .../test_depricated_functions_document.py | 19 +- .../test_depricated_functions_video.py | 18 +- .../test_duplicate_image_upload.py | 4 +- tests/integration/test_export_import.py | 3 +- tests/integration/test_export_upload_s3.py | 3 +- tests/integration/test_fuse_gen.py | 3 +- tests/integration/test_get_exports.py | 3 +- tests/integration/test_image_quality.py | 3 +- tests/integration/test_interface.py | 3 +- tests/integration/test_limitations.py | 3 +- tests/integration/test_ml_funcs.py | 3 +- tests/integration/test_recursive_folder.py | 3 +- .../test_recursive_folder_pixel.py | 3 +- .../test_single_annotation_download.py | 3 +- tests/integration/test_single_image_upload.py | 3 +- tests/integration/test_team_metadata.py | 5 +- tests/integration/test_upload_images.py | 3 +- .../test_upload_priority_scores.py | 3 +- .../integration/test_validate_upload_state.py | 3 +- tests/integration/test_video.py | 3 +- tests/unit/test_controller_init.py | 225 +- tests/unit/test_validators.py | 3 +- 82 files changed, 3853 insertions(+), 3893 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index d55efa2b0..ca15d6962 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -5,220 +5,30 @@ import requests import superannotate.lib.core as constances from packaging.version import parse -from superannotate.lib import get_default_controller from superannotate.lib.app.analytics.class_analytics import class_distribution from superannotate.lib.app.exceptions import AppException from superannotate.lib.app.input_converters.conversion import convert_json_version from superannotate.lib.app.input_converters.conversion import convert_project_type from superannotate.lib.app.input_converters.conversion import export_annotation from superannotate.lib.app.input_converters.conversion import import_annotation -from superannotate.lib.app.interface.sdk_interface import add_annotation_bbox_to_image -from superannotate.lib.app.interface.sdk_interface import ( - add_annotation_comment_to_image, -) -from superannotate.lib.app.interface.sdk_interface import add_annotation_point_to_image -from superannotate.lib.app.interface.sdk_interface import add_contributors_to_project -from superannotate.lib.app.interface.sdk_interface import aggregate_annotations_as_df -from superannotate.lib.app.interface.sdk_interface import assign_folder -from superannotate.lib.app.interface.sdk_interface import assign_images -from superannotate.lib.app.interface.sdk_interface import ( - attach_document_urls_to_project, -) -from superannotate.lib.app.interface.sdk_interface import attach_image_urls_to_project -from superannotate.lib.app.interface.sdk_interface import attach_items -from superannotate.lib.app.interface.sdk_interface import ( - attach_items_from_integrated_storage, -) -from superannotate.lib.app.interface.sdk_interface import attach_video_urls_to_project -from superannotate.lib.app.interface.sdk_interface import benchmark -from superannotate.lib.app.interface.sdk_interface import clone_project -from superannotate.lib.app.interface.sdk_interface import consensus -from superannotate.lib.app.interface.sdk_interface import copy_image -from superannotate.lib.app.interface.sdk_interface import copy_images -from superannotate.lib.app.interface.sdk_interface import copy_items -from superannotate.lib.app.interface.sdk_interface import create_annotation_class -from superannotate.lib.app.interface.sdk_interface import ( - create_annotation_classes_from_classes_json, -) -from superannotate.lib.app.interface.sdk_interface import create_folder -from superannotate.lib.app.interface.sdk_interface import create_project -from superannotate.lib.app.interface.sdk_interface import create_project_from_metadata -from superannotate.lib.app.interface.sdk_interface import delete_annotation_class -from superannotate.lib.app.interface.sdk_interface import delete_annotations -from superannotate.lib.app.interface.sdk_interface import delete_folders -from superannotate.lib.app.interface.sdk_interface import delete_images -from superannotate.lib.app.interface.sdk_interface import delete_project -from superannotate.lib.app.interface.sdk_interface import ( - download_annotation_classes_json, -) -from superannotate.lib.app.interface.sdk_interface import download_export -from superannotate.lib.app.interface.sdk_interface import download_image -from superannotate.lib.app.interface.sdk_interface import download_image_annotations -from superannotate.lib.app.interface.sdk_interface import download_model -from superannotate.lib.app.interface.sdk_interface import get_annotations -from superannotate.lib.app.interface.sdk_interface import get_annotations_per_frame -from superannotate.lib.app.interface.sdk_interface import get_exports -from superannotate.lib.app.interface.sdk_interface import get_folder_metadata -from superannotate.lib.app.interface.sdk_interface import get_integrations -from superannotate.lib.app.interface.sdk_interface import get_item_metadata -from superannotate.lib.app.interface.sdk_interface import get_project_image_count -from superannotate.lib.app.interface.sdk_interface import get_project_metadata -from superannotate.lib.app.interface.sdk_interface import get_project_settings -from superannotate.lib.app.interface.sdk_interface import get_project_workflow -from superannotate.lib.app.interface.sdk_interface import get_team_metadata -from superannotate.lib.app.interface.sdk_interface import init -from superannotate.lib.app.interface.sdk_interface import invite_contributors_to_team -from superannotate.lib.app.interface.sdk_interface import move_images -from superannotate.lib.app.interface.sdk_interface import move_items -from superannotate.lib.app.interface.sdk_interface import pin_image -from superannotate.lib.app.interface.sdk_interface import prepare_export -from superannotate.lib.app.interface.sdk_interface import query -from superannotate.lib.app.interface.sdk_interface import rename_project -from superannotate.lib.app.interface.sdk_interface import run_prediction -from superannotate.lib.app.interface.sdk_interface import search_annotation_classes -from superannotate.lib.app.interface.sdk_interface import search_folders -from superannotate.lib.app.interface.sdk_interface import search_items -from superannotate.lib.app.interface.sdk_interface import search_models -from superannotate.lib.app.interface.sdk_interface import search_projects -from superannotate.lib.app.interface.sdk_interface import search_team_contributors -from superannotate.lib.app.interface.sdk_interface import set_annotation_statuses -from superannotate.lib.app.interface.sdk_interface import set_auth_token -from superannotate.lib.app.interface.sdk_interface import set_image_annotation_status -from superannotate.lib.app.interface.sdk_interface import set_images_annotation_statuses -from superannotate.lib.app.interface.sdk_interface import ( - set_project_default_image_quality_in_editor, -) -from superannotate.lib.app.interface.sdk_interface import set_project_workflow -from superannotate.lib.app.interface.sdk_interface import share_project -from superannotate.lib.app.interface.sdk_interface import unassign_folder -from superannotate.lib.app.interface.sdk_interface import unassign_images -from superannotate.lib.app.interface.sdk_interface import ( - upload_annotations_from_folder_to_project, -) -from superannotate.lib.app.interface.sdk_interface import upload_image_annotations -from superannotate.lib.app.interface.sdk_interface import upload_image_to_project -from superannotate.lib.app.interface.sdk_interface import ( - upload_images_from_folder_to_project, -) -from superannotate.lib.app.interface.sdk_interface import upload_images_to_project -from superannotate.lib.app.interface.sdk_interface import ( - upload_preannotations_from_folder_to_project, -) -from superannotate.lib.app.interface.sdk_interface import upload_priority_scores -from superannotate.lib.app.interface.sdk_interface import upload_video_to_project -from superannotate.lib.app.interface.sdk_interface import ( - upload_videos_from_folder_to_project, -) -from superannotate.lib.app.interface.sdk_interface import validate_annotations +from superannotate.lib.app.interface.sdk_interface import SAClient from superannotate.logger import get_default_logger from superannotate.version import __version__ -controller = get_default_controller() - - __all__ = [ "__version__", - "controller", + "SAClient", "constances", # Utils "AppException", - "validate_annotations", - # - "init", - "set_auth_token", # analytics "class_distribution", - "aggregate_annotations_as_df", - "get_exports", - # annotations - "get_annotations", - "get_annotations_per_frame", - # integrations - "get_integrations", - "attach_items_from_integrated_storage", # converters "convert_json_version", "import_annotation", "export_annotation", "convert_project_type", - # Teams Section - "get_team_metadata", - "search_team_contributors", - # Projects Section - "create_project_from_metadata", - "get_project_settings", - "get_project_metadata", - "get_project_workflow", - "set_project_workflow", - "search_projects", - "create_project", - "clone_project", - "share_project", - "delete_project", - "rename_project", - "upload_priority_scores", - # Images Section - "copy_image", - # Folders Section - "create_folder", - "get_folder_metadata", - "delete_folders", - "search_folders", - "assign_folder", - "unassign_folder", - # Items Section - "get_item_metadata", - "search_items", - "query", - "attach_items", - "copy_items", - "move_items", - "set_annotation_statuses", - # Image Section - "copy_images", - "move_images", - "delete_images", - "download_image", - "pin_image", - "get_project_image_count", - "assign_images", - "unassign_images", - "download_image_annotations", - "delete_annotations", - "upload_image_to_project", - "upload_image_annotations", - "upload_images_from_folder_to_project", - "attach_image_urls_to_project", - "attach_video_urls_to_project", - "attach_document_urls_to_project", - # Video Section - "upload_videos_from_folder_to_project", - # Annotation Section - "create_annotation_class", - "delete_annotation_class", - "prepare_export", - "download_export", - "set_images_annotation_statuses", - "add_annotation_bbox_to_image", - "add_annotation_point_to_image", - "add_annotation_comment_to_image", - "search_annotation_classes", - "create_annotation_classes_from_classes_json", - "upload_annotations_from_folder_to_project", - "upload_preannotations_from_folder_to_project", - "download_annotation_classes_json", - "set_project_default_image_quality_in_editor", - "run_prediction", - "search_models", - "download_model", - "set_image_annotation_status", - "benchmark", - "consensus", - "upload_video_to_project", - "upload_images_to_project", - "add_contributors_to_project", - "invite_contributors_to_team", ] __author__ = "Superannotate" diff --git a/src/superannotate/lib/__init__.py b/src/superannotate/lib/__init__.py index 42f7f7243..e69de29bb 100644 --- a/src/superannotate/lib/__init__.py +++ b/src/superannotate/lib/__init__.py @@ -1,12 +0,0 @@ -import os -import sys - - -sys.path.append(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -def get_default_controller(): - from lib.infrastructure.controller import Controller - - return Controller.get_default() diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index ba9fc8e2b..76a09245d 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -1,14 +1,128 @@ -from lib import get_default_controller -from lib.infrastructure.repositories import ConfigRepository +import functools +import sys +from abc import ABC +from abc import abstractmethod +from inspect import signature +from typing import Iterable +from typing import Sized +from mixpanel import Mixpanel -class BaseInterfaceFacade: - def __init__(self): - self._config_path = None - self._controller = get_default_controller() +from lib.app.helpers import extract_project_folder +from lib.core.reporter import Session +from version import __version__ + + +class BaseInterfaceFacade(ABC): + @property + @abstractmethod + def host(self): + raise NotImplementedError + + @property + @abstractmethod + def token(self): + raise NotImplementedError @property - def controller(self): - if not ConfigRepository().get_one("token"): - raise Exception("Config does not exists!") - return self._controller + @abstractmethod + def logger(self): + raise NotImplementedError + + +class Tracker: + TEAM_DATA = None + INITIAL_EVENT = {"event_name": "SDK init", "properties": {}} + INITIAL_LOGGED = False + + @staticmethod + def get_mp_instance() -> Mixpanel: + # if "api.annotate.online" in get_default_controller()._backend_url: + # return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") + return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") + + @staticmethod + def get_default_payload(team_name, user_id, project_name=None): + return { + "SDK": True, + "Paid": True, + "Team": team_name, + "Team Owner": user_id, + "Project Name": project_name, + "Project Role": "Admin", + "Version": __version__, + } + + def __init__(self, function): + self.function = function + functools.update_wrapper(self, function) + + @staticmethod + def extract_arguments(function, *args, **kwargs) -> dict: + bound_arguments = signature(function).bind(*args, **kwargs) + bound_arguments.apply_defaults() + return dict(bound_arguments.arguments) + + @staticmethod + def default_parser(function_name: str, kwargs: dict) -> tuple: + properties = {} + for key, value in kwargs.items(): + if key == "self": + continue + elif key == "project": + properties["project_name"], properties["folder_name"] = extract_project_folder(value) + elif isinstance(value, str) and key == "project": + properties["project_name"] = value.split() + if isinstance(value, (str, int, float, bool, str)): + properties[key] = value + elif isinstance(value, dict): + properties[key] = value.keys() + elif isinstance(value, Sized): + properties[key] = len(value) + elif isinstance(value, Iterable): + properties[key] = "N/A" + else: + properties[key] = str(value) + return function_name, properties + + def track(self, args, kwargs, success: bool, session): + try: + function_name = self.function.__name__ if self.function else "" + arguments = self.extract_arguments(self.function, *args, **kwargs) + event_name, properties = self.default_parser(function_name, arguments) + + user_id = self.team_data.creator_id + team_name = self.team_data.name + properties["Success"] = success + + default = self.get_default_payload( + team_name=team_name, + user_id=user_id, + project_name=properties.pop("project_name", None), + ) + if "pytest" not in sys.modules: + self.get_mp_instance().track(user_id, event_name, {**default, **properties, **session.data}) + except Exception: + raise + + def __get__(self, instance, owner): + if instance is None: + return self + d = self + mfactory = lambda self, *args, **kw: d(self, *args, **kw) + mfactory.__name__ = self.function.__name__ + self.team_data = instance.controller.team_data.data + return mfactory.__get__(instance, owner) + + def __call__(self, *args, **kwargs): + success = True + try: + with Session(self.function.__name__) as session: + result = self.function(*args, **kwargs) + except Exception as e: + success = False + raise e + else: + return result + finally: + self.track(args=args, kwargs=kwargs, success=success, session=session) diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index 793517d02..27a9fbaae 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -9,17 +9,8 @@ from lib.app.helpers import split_project_path from lib.app.input_converters.conversion import import_annotation from lib.app.interface.base_interface import BaseInterfaceFacade -from lib.app.interface.sdk_interface import attach_document_urls_to_project -from lib.app.interface.sdk_interface import attach_image_urls_to_project -from lib.app.interface.sdk_interface import attach_video_urls_to_project -from lib.app.interface.sdk_interface import create_folder -from lib.app.interface.sdk_interface import create_project -from lib.app.interface.sdk_interface import upload_annotations_from_folder_to_project -from lib.app.interface.sdk_interface import upload_images_from_folder_to_project -from lib.app.interface.sdk_interface import upload_preannotations_from_folder_to_project -from lib.app.interface.sdk_interface import upload_videos_from_folder_to_project +from lib.app.interface.sdk_interface import SAClient from lib.core.entities import ConfigEntity -from lib.infrastructure.controller import Controller from lib.infrastructure.repositories import ConfigRepository @@ -68,13 +59,13 @@ def create_project(self, name: str, description: str, type: str): """ To create a new project """ - create_project(name, description, type) + SAClient().create_project(name, description, type) def create_folder(self, project: str, name: str): """ To create a new folder """ - create_folder(project, name) + SAClient().create_folder(project, name) sys.exit(0) def upload_images( @@ -97,7 +88,7 @@ def upload_images( """ if not isinstance(extensions, list): extensions = extensions.split(",") - upload_images_from_folder_to_project( + SAClient().upload_images_from_folder_to_project( project, folder_path=folder, extensions=extensions, @@ -120,12 +111,12 @@ def export_project( folders = None if folder_name: folders = [folder_name] - export_res = Controller.get_default().prepare_export( + export_res = SAClient().prepare_export( project_name, folders, include_fuse, False, annotation_statuses ) export_name = export_res.data["name"] - use_case = Controller.get_default().download_export( + use_case = SAClient().download_export( project_name=project_name, export_name=export_name, folder_path=folder, @@ -186,7 +177,7 @@ def _upload_annotations( project_folder_name = project project_name, folder_name = split_project_path(project) project = ( - Controller.get_default() + SAClient() .get_project_metadata(project_name=project_name) .data ) @@ -213,11 +204,11 @@ def _upload_annotations( ) annotations_path = temp_dir if pre: - upload_preannotations_from_folder_to_project( + SAClient().upload_preannotations_from_folder_to_project( project_folder_name, annotations_path ) else: - upload_annotations_from_folder_to_project( + SAClient().upload_annotations_from_folder_to_project( project_folder_name, annotations_path ) sys.exit(0) @@ -229,7 +220,7 @@ def attach_image_urls( To attach image URLs to project use: """ - attach_image_urls_to_project( + SAClient().attach_image_urls_to_project( project=project, attachments=attachments, annotation_status=annotation_status, @@ -239,7 +230,7 @@ def attach_image_urls( def attach_video_urls( self, project: str, attachments: str, annotation_status: Optional[Any] = None ): - attach_video_urls_to_project( + SAClient().attach_video_urls_to_project( project=project, attachments=attachments, annotation_status=annotation_status, @@ -250,7 +241,7 @@ def attach_video_urls( def attach_document_urls( project: str, attachments: str, annotation_status: Optional[Any] = None ): - attach_document_urls_to_project( + SAClient().attach_document_urls_to_project( project=project, attachments=attachments, annotation_status=annotation_status, @@ -279,7 +270,7 @@ def upload_videos( end-time specifies time (in seconds) up to which to extract frames. If it is not specified, then up to end is assumed. """ - upload_videos_from_folder_to_project( + SAClient().upload_videos_from_folder_to_project( project=project, folder_path=folder, extensions=extensions, diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 346040ea3..cc5d5944a 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -12,6 +12,12 @@ from typing import Union import boto3 +from pydantic import StrictBool +from pydantic import conlist +from pydantic import parse_obj_as +from pydantic.error_wrappers import ValidationError +from tqdm import tqdm + import lib.core as constances from lib.app.annotation_helpers import add_annotation_bbox_to_json from lib.app.annotation_helpers import add_annotation_comment_to_json @@ -20,6 +26,8 @@ from lib.app.helpers import get_annotation_paths from lib.app.helpers import get_name_url_duplicated_from_csv from lib.app.helpers import get_paths_and_duplicated_from_csv +from lib.app.interface.base_interface import BaseInterfaceFacade +from lib.app.interface.base_interface import Tracker from lib.app.interface.types import AnnotationStatuses from lib.app.interface.types import AnnotationType from lib.app.interface.types import AnnotatorRole @@ -33,7 +41,6 @@ from lib.app.interface.types import ProjectTypes from lib.app.interface.types import Setting from lib.app.interface.types import validate_arguments -from lib.app.mixp.decorators import Trackable from lib.app.serializers import BaseSerializer from lib.app.serializers import FolderSerializer from lib.app.serializers import ImageSerializer @@ -52,3134 +59,3148 @@ from lib.core.types import PriorityScore from lib.core.types import Project from lib.infrastructure.controller import Controller -from pydantic import conlist -from pydantic import parse_obj_as -from pydantic import StrictBool -from pydantic.error_wrappers import ValidationError +from lib.infrastructure.repositories import ConfigRepository from superannotate.logger import get_default_logger -from tqdm import tqdm logger = get_default_logger() -@validate_arguments -def init(path_to_config_json: Optional[str] = None, token: str = None): - """ - Initializes and authenticates to SuperAnnotate platform using the config file. - If not initialized then $HOME/.superannotate/config.json - will be used. - - :param path_to_config_json: Location to config JSON file - :type path_to_config_json: str or Path - - :param token: Team token - :type token: str - """ - Controller.set_default(Controller(config_path=path_to_config_json, token=token)) - - -@validate_arguments -def set_auth_token(token: str): - Controller.get_default().set_token(token) - - -@Trackable -def get_team_metadata(): - """Returns team metadata - - :return: team metadata - :rtype: dict - """ - response = Controller.get_default().get_team() - return TeamSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def search_team_contributors( - email: EmailStr = None, - first_name: NotEmptyStr = None, - last_name: NotEmptyStr = None, - return_metadata: bool = True, -): - """Search for contributors in the team - - :param email: filter by email - :type email: str - :param first_name: filter by first name - :type first_name: str - :param last_name: filter by last name - :type last_name: str - :param return_metadata: return metadata of contributors instead of names - :type return_metadata: bool - - :return: metadata of found users - :rtype: list of dicts - """ - - contributors = ( - Controller.get_default() - .search_team_contributors( - email=email, first_name=first_name, last_name=last_name - ) - .data - ) - if not return_metadata: - return [contributor["email"] for contributor in contributors] - return contributors - - -@Trackable -@validate_arguments -def search_projects( - name: Optional[NotEmptyStr] = None, - return_metadata: bool = False, - include_complete_image_count: bool = False, - status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, -): - """ - Project name based case-insensitive search for projects. - If **name** is None, all the projects will be returned. - - :param name: search string - :type name: str - - :param return_metadata: return metadata of projects instead of names - :type return_metadata: bool - - :param include_complete_image_count: return projects that have completed images and include the number of completed images in response. - :type include_complete_image_count: bool - - :param status: search projects via project status - :type status: str - - :return: project names or metadatas - :rtype: list of strs or dicts - """ - statuses = [] - if status: - if isinstance(status, (list, tuple, set)): - statuses = list(status) +class SAClient(BaseInterfaceFacade): + def __init__( + self, + token: str = None, + host=constances.BACKEND_URL, + config_path: str = constances.CONFIG_FILE_LOCATION, + ): + env_token = os.environ.get("SA_TOKEN") + version = os.environ.get("SA_VERSION", "v1") + ssl_verify = bool(os.environ.get("SA_SSL", True)) + if token: + token = Controller.validate_token(token=token) + elif env_token: + host = os.environ.get("SA_UTR", constances.BACKEND_URL) + + token = Controller.validate_token(env_token) else: - statuses = [status] - result = ( - Controller.get_default() - .search_project( + config_path = str(config_path) + if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + raise AppException( + f"SuperAnnotate config file {str(config_path)} not found." + f" Please provide correct config file location to sa.init() or use " + f"CLI's superannotate init to generate default location config file." + ) + config_repo = ConfigRepository(config_path) + token, host, ssl_verify = ( + Controller.validate_token(config_repo.get_one("token").value), + config_repo.get_one("main_endpoint").value, + config_repo.get_one("ssl_verify").value, + ) + self._host = host + self._token = token + self.controller = Controller(token, host, ssl_verify, version) + + @property + def host(self): + return self._host + + @property + def token(self): + return self._token + + @property + def logger(self): + pass + + def get_team_metadata(self): + """Returns team metadata + + :return: team metadata + :rtype: dict + """ + response = self.controller.get_team() + return TeamSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def search_team_contributors( + self, + email: EmailStr = None, + first_name: NotEmptyStr = None, + last_name: NotEmptyStr = None, + return_metadata: bool = True, + ): + """Search for contributors in the team + + :param email: filter by email + :type email: str + :param first_name: filter by first name + :type first_name: str + :param last_name: filter by last name + :type last_name: str + :param return_metadata: return metadata of contributors instead of names + :type return_metadata: bool + + :return: metadata of found users + :rtype: list of dicts + """ + + contributors = self.controller.search_team_contributors( + email=email, first_name=first_name, last_name=last_name + ).data + + if not return_metadata: + return [contributor["email"] for contributor in contributors] + return contributors + + @Tracker + @validate_arguments + def search_projects( + self, + name: Optional[NotEmptyStr] = None, + return_metadata: bool = False, + include_complete_image_count: bool = False, + status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, + ): + """ + Project name based case-insensitive search for projects. + If **name** is None, all the projects will be returned. + + :param name: search string + :type name: str + + :param return_metadata: return metadata of projects instead of names + :type return_metadata: bool + + :param include_complete_image_count: return projects that have completed images and include the number of completed images in response. + :type include_complete_image_count: bool + + :param status: search projects via project status + :type status: str + + :return: project names or metadatas + :rtype: list of strs or dicts + """ + statuses = [] + if status: + if isinstance(status, (list, tuple, set)): + statuses = list(status) + else: + statuses = [status] + result = self.controller.search_project( name=name, include_complete_image_count=include_complete_image_count, - statuses=statuses, - ) - .data - ) - - if return_metadata: - return [ - ProjectSerializer(project).serialize( - exclude={ - "annotation_classes", - "workflows", - "settings", - "contributors", - "classes", - } - ) - for project in result - ] - else: - return [project.name for project in result] - - -@Trackable -@validate_arguments -def create_project( - project_name: NotEmptyStr, - project_description: NotEmptyStr, - project_type: NotEmptyStr, - settings: List[Setting] = None, -): - """Create a new project in the team. - - :param project_name: the new project's name - :type project_name: str - - :param project_description: the new project's description - :type project_description: str - - :param project_type: the new project type, Vector or Pixel. - :type project_type: str - - :param settings: list of settings objects - :type settings: list of dicts - - :return: dict object metadata the new project - :rtype: dict - """ - if settings: - settings = parse_obj_as(List[SettingEntity], settings) - else: - settings = [] - response = Controller.get_default().create_project( - name=project_name, - description=project_description, - project_type=project_type, - settings=settings, - ) - if response.errors: - raise AppException(response.errors) - - return ProjectSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def create_project_from_metadata(project_metadata: Project): - """Create a new project in the team using project metadata object dict. - Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) - Non-mandatory keys: "workflow", "settings" and "annotation_classes". - - :return: dict object metadata the new project - :rtype: dict - """ - project_metadata = project_metadata.dict() - response = Controller.get_default().create_project( - name=project_metadata["name"], - description=project_metadata.get("description"), - project_type=project_metadata["type"], - settings=parse_obj_as( - List[SettingEntity], project_metadata.get("settings", []) - ), - classes=project_metadata.get("classes", []), - workflows=project_metadata.get("workflows", []), - instructions_link=project_metadata.get("instructions_link"), - ) - if response.errors: - raise AppException(response.errors) - return ProjectSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def clone_project( - project_name: Union[NotEmptyStr, dict], - from_project: Union[NotEmptyStr, dict], - project_description: Optional[NotEmptyStr] = None, - copy_annotation_classes: Optional[StrictBool] = True, - copy_settings: Optional[StrictBool] = True, - copy_workflow: Optional[StrictBool] = True, - copy_contributors: Optional[StrictBool] = False, -): - """Create a new project in the team using annotation classes and settings from from_project. - - :param project_name: new project's name - :type project_name: str - :param from_project: the name of the project being used for duplication - :type from_project: str - :param project_description: the new project's description. If None, from_project's - description will be used - :type project_description: str - :param copy_annotation_classes: enables copying annotation classes - :type copy_annotation_classes: bool - :param copy_settings: enables copying project settings - :type copy_settings: bool - :param copy_workflow: enables copying project workflow - :type copy_workflow: bool - :param copy_contributors: enables copying project contributors - :type copy_contributors: bool - - :return: dict object metadata of the new project - :rtype: dict - """ - response = Controller.get_default().clone_project( - name=project_name, - from_name=from_project, - project_description=project_description, - copy_annotation_classes=copy_annotation_classes, - copy_settings=copy_settings, - copy_workflow=copy_workflow, - copy_contributors=copy_contributors, - ) - if response.errors: - raise AppException(response.errors) - return ProjectSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def create_folder(project: NotEmptyStr, folder_name: NotEmptyStr): - """Create a new folder in the project. - - :param project: project name - :type project: str - :param folder_name: the new folder's name - :type folder_name: str - - :return: dict object metadata the new folder - :rtype: dict - """ - - res = Controller.get_default().create_folder( - project=project, folder_name=folder_name - ) - if res.data: - folder = res.data - logger.info(f"Folder {folder.name} created in project {project}") - return folder.to_dict() - if res.errors: - raise AppException(res.errors) - - -@Trackable -@validate_arguments -def delete_project(project: Union[NotEmptyStr, dict]): - """Deletes the project + statuses=statuses + ).data + + if return_metadata: + return [ + ProjectSerializer(project).serialize( + exclude={ + "annotation_classes", + "workflows", + "settings", + "contributors", + "classes", + } + ) + for project in result + ] + else: + return [project.name for project in result] + + @Tracker + @validate_arguments + def create_project( + self, + project_name: NotEmptyStr, + project_description: NotEmptyStr, + project_type: NotEmptyStr, + settings: List[Setting] = None, + ): + """Create a new project in the team. + + :param project_name: the new project's name + :type project_name: str + + :param project_description: the new project's description + :type project_description: str + + :param project_type: the new project type, Vector or Pixel. + :type project_type: str + + :param settings: list of settings objects + :type settings: list of dicts + + :return: dict object metadata the new project + :rtype: dict + """ + if settings: + settings = parse_obj_as(List[SettingEntity], settings) + else: + settings = [] + response = self.controller.create_project( + name=project_name, + description=project_description, + project_type=project_type, + settings=settings, + ) + if response.errors: + raise AppException(response.errors) + + return ProjectSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def create_project_from_metadata( + self, project_metadata: Project): + """Create a new project in the team using project metadata object dict. + Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) + Non-mandatory keys: "workflow", "settings" and "annotation_classes". + + :return: dict object metadata the new project + :rtype: dict + """ + project_metadata = project_metadata.dict() + response = self.controller.create_project( + name=project_metadata["name"], + description=project_metadata.get("description"), + project_type=project_metadata["type"], + settings=parse_obj_as( + List[SettingEntity], project_metadata.get("settings", []) + ), + classes=project_metadata.get("classes", []), + workflows=project_metadata.get("workflows", []), + instructions_link=project_metadata.get("instructions_link"), + ) + if response.errors: + raise AppException(response.errors) + return ProjectSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def clone_project( + self, + project_name: Union[NotEmptyStr, dict], + from_project: Union[NotEmptyStr, dict], + project_description: Optional[NotEmptyStr] = None, + copy_annotation_classes: Optional[StrictBool] = True, + copy_settings: Optional[StrictBool] = True, + copy_workflow: Optional[StrictBool] = True, + copy_contributors: Optional[StrictBool] = False, + ): + """Create a new project in the team using annotation classes and settings from from_project. + + :param project_name: new project's name + :type project_name: str + :param from_project: the name of the project being used for duplication + :type from_project: str + :param project_description: the new project's description. If None, from_project's + description will be used + :type project_description: str + :param copy_annotation_classes: enables copying annotation classes + :type copy_annotation_classes: bool + :param copy_settings: enables copying project settings + :type copy_settings: bool + :param copy_workflow: enables copying project workflow + :type copy_workflow: bool + :param copy_contributors: enables copying project contributors + :type copy_contributors: bool + + :return: dict object metadata of the new project + :rtype: dict + """ + response = self.controller.clone_project( + name=project_name, + from_name=from_project, + project_description=project_description, + copy_annotation_classes=copy_annotation_classes, + copy_settings=copy_settings, + copy_workflow=copy_workflow, + copy_contributors=copy_contributors, + ) + if response.errors: + raise AppException(response.errors) + return ProjectSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def create_folder( + self, project: NotEmptyStr, folder_name: NotEmptyStr): + """Create a new folder in the project. + + :param project: project name + :type project: str + :param folder_name: the new folder's name + :type folder_name: str + + :return: dict object metadata the new folder + :rtype: dict + """ + + res = self.controller.create_folder( + project=project, folder_name=folder_name + ) + if res.data: + folder = res.data + logger.info(f"Folder {folder.name} created in project {project}") + return folder.to_dict() + if res.errors: + raise AppException(res.errors) + + @Tracker + @validate_arguments + def delete_project( + self, project: Union[NotEmptyStr, dict]): + """Deletes the project + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + """ + name = project + if isinstance(project, dict): + name = project["name"] + self.controller.delete_project(name=name) + + @Tracker + @validate_arguments + def rename_project( + self, project: NotEmptyStr, new_name: NotEmptyStr): + """Renames the project :param project: project name or folder path (e.g., "project1/folder1") :type project: str - """ - name = project - if isinstance(project, dict): - name = project["name"] - Controller.get_default().delete_project(name=name) - - -@Trackable -@validate_arguments -def rename_project(project: NotEmptyStr, new_name: NotEmptyStr): - """Renames the project - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param new_name: project's new name - :type new_name: str - """ - - response = Controller.get_default().update_project( - name=project, project_data={"name": new_name} - ) - if response.errors: - raise AppException(response.errors) - logger.info("Successfully renamed project %s to %s.", project, response.data.name) - return ProjectSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def get_folder_metadata(project: NotEmptyStr, folder_name: NotEmptyStr): - """Returns folder metadata - - :param project: project name - :type project: str - :param folder_name: folder's name - :type folder_name: str - - :return: metadata of folder - :rtype: dict - """ - result = ( - Controller.get_default() - .get_folder(project_name=project, folder_name=folder_name) - .data - ) - if not result: - raise AppException("Folder not found.") - return FolderSerializer(result).serialize() - - -@Trackable -@validate_arguments -def delete_folders(project: NotEmptyStr, folder_names: List[NotEmptyStr]): - """Delete folder in project. - - :param project: project name - :type project: str - :param folder_names: to be deleted folders' names - :type folder_names: list of strs - """ - - res = Controller.get_default().delete_folders( - project_name=project, folder_names=folder_names - ) - if res.errors: - raise AppException(res.errors) - logger.info(f"Folders {folder_names} deleted in project {project}") - - -@Trackable -@validate_arguments -def search_folders( - project: NotEmptyStr, - folder_name: Optional[NotEmptyStr] = None, - return_metadata: Optional[StrictBool] = False, -): - """Folder name based case-insensitive search for folders in project. - - :param project: project name - :type project: str - :param folder_name: the new folder's name - :type folder_name: str. If None, all the folders in the project will be returned. - :param return_metadata: return metadata of folders instead of names - :type return_metadata: bool - - :return: folder names or metadatas - :rtype: list of strs or dicts - """ - - response = Controller.get_default().search_folders( - project_name=project, folder_name=folder_name, include_users=return_metadata - ) - if response.errors: - raise AppException(response.errors) - data = response.data - if return_metadata: - return [FolderSerializer(folder).serialize() for folder in data] - return [folder.name for folder in data] - - -@Trackable -@validate_arguments -def copy_image( - source_project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = False, - copy_annotation_status: Optional[StrictBool] = False, - copy_pin: Optional[StrictBool] = False, -): - """Copy image to a project. The image's project is the same as destination - project then the name will be changed to _()., - where is the next available number deducted from project image list. - - :param source_project: project name plus optional subfolder in the project (e.g., "project1/folder1") or - metadata of the project of source project - :type source_project: str or dict - :param image_name: image name - :type image_name: str - :param destination_project: project name or metadata of the project of destination project - :type destination_project: str or dict - :param include_annotations: enables annotations copy - :type include_annotations: bool - :param copy_annotation_status: enables annotations status copy - :type copy_annotation_status: bool - :param copy_pin: enables image pin status copy - :type copy_pin: bool - """ - source_project_name, source_folder_name = extract_project_folder(source_project) - - destination_project, destination_folder = extract_project_folder( - destination_project - ) - source_project_metadata = ( - Controller.get_default().get_project_metadata(source_project_name).data - ) - destination_project_metadata = ( - Controller.get_default().get_project_metadata(destination_project).data - ) - - if destination_project_metadata["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ] or source_project_metadata["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[source_project_metadata["project"].type]) - - response = Controller.get_default().copy_image( - from_project_name=source_project_name, - from_folder_name=source_folder_name, - to_project_name=destination_project, - to_folder_name=destination_folder, - image_name=image_name, - copy_annotation_status=copy_annotation_status, - ) - if response.errors: - raise AppException(response.errors) - - if include_annotations: - Controller.get_default().copy_image_annotation_classes( - from_project_name=source_project_name, - from_folder_name=source_folder_name, - to_folder_name=destination_folder, - to_project_name=destination_project, - image_name=image_name, + :param new_name: project's new name + :type new_name: str + """ + + response = self.controller.update_project( + name=project, project_data={"name": new_name} ) - if copy_pin: - Controller.get_default().update_image( - project_name=destination_project, - folder_name=destination_folder, - image_name=image_name, - is_pinned=1, - ) - logger.info( - f"Copied image {source_project}/{image_name}" - f" to {destination_project}/{destination_folder}." - ) - - -@Trackable -@validate_arguments -def copy_images( - source_project: Union[NotEmptyStr, dict], - image_names: Optional[List[NotEmptyStr]], - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = True, - copy_pin: Optional[StrictBool] = True, -): - """Copy images in bulk between folders in a project - - :param source_project: project name or folder path (e.g., "project1/folder1") - :type source_project: str` - :param image_names: image names. If None, all images from source project will be copied - :type image_names: list of str - :param destination_project: project name or folder path (e.g., "project1/folder2") - :type destination_project: str - :param include_annotations: enables annotations copy - :type include_annotations: bool - :param copy_pin: enables image pin status copy - :type copy_pin: bool - :return: list of skipped image names - :rtype: list of strs - """ - warning_msg = ( - "We're deprecating the copy_images function. Please use copy_items instead. Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.copy_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, source_folder_name = extract_project_folder(source_project) - - to_project_name, destination_folder_name = extract_project_folder( - destination_project - ) - if project_name != to_project_name: - raise AppException("Source and destination projects should be the same") - if not image_names: - images = ( - Controller.get_default() - .search_images(project_name=project_name, folder_path=source_folder_name) - .data - ) - image_names = [image.name for image in images] - - res = Controller.get_default().bulk_copy_images( - project_name=project_name, - from_folder_name=source_folder_name, - to_folder_name=destination_folder_name, - image_names=image_names, - include_annotations=include_annotations, - include_pin=copy_pin, - ) - if res.errors: - raise AppException(res.errors) - skipped_images = res.data - done_count = len(image_names) - len(skipped_images) - message_postfix = "{from_path} to {to_path}." - message_prefix = "Copied images from " - if done_count > 1 or done_count == 0: - message_prefix = f"Copied {done_count}/{len(image_names)} images from " - elif done_count == 1: - message_prefix = "Copied an image from " - logger.info( - message_prefix - + message_postfix.format(from_path=source_project, to_path=destination_project) - ) - - return skipped_images - - -@Trackable -@validate_arguments -def move_images( - source_project: Union[NotEmptyStr, dict], - image_names: Optional[List[NotEmptyStr]], - destination_project: Union[NotEmptyStr, dict], - *args, - **kwargs, -): - """Move images in bulk between folders in a project - - :param source_project: project name or folder path (e.g., "project1/folder1") - :type source_project: str - :param image_names: image names. If None, all images from source project will be moved - :type image_names: list of str - :param destination_project: project name or folder path (e.g., "project1/folder2") - :type destination_project: str - :return: list of skipped image names - :rtype: list of strs - """ - warning_msg = ( - "We're deprecating the move_images function. Please use move_items instead. Learn more." - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.move_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, source_folder_name = extract_project_folder(source_project) - - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - - to_project_name, destination_folder_name = extract_project_folder( - destination_project - ) - - if project_name != to_project_name: - raise AppException( - "Source and destination projects should be the same for move_images" - ) - - if not image_names: - images = Controller.get_default().search_images( - project_name=project_name, folder_path=source_folder_name - ) - images = images.data - image_names = [image.name for image in images] - - response = Controller.get_default().bulk_move_images( - project_name=project_name, - from_folder_name=source_folder_name, - to_folder_name=destination_folder_name, - image_names=image_names, - ) - if response.errors: - raise AppException(response.errors) - moved_images = response.data - moved_count = len(moved_images) - message_postfix = "{from_path} to {to_path}." - message_prefix = "Moved images from " - if moved_count > 1 or moved_count == 0: - message_prefix = f"Moved {moved_count}/{len(image_names)} images from " - elif moved_count == 1: - message_prefix = "Moved an image from" - - logger.info( - message_prefix - + message_postfix.format(from_path=source_project, to_path=destination_project) - ) - - return list(set(image_names) - set(moved_images)) - - -@Trackable -@validate_arguments -def get_project_metadata( - project: Union[NotEmptyStr, dict], - include_annotation_classes: Optional[StrictBool] = False, - include_settings: Optional[StrictBool] = False, - include_workflow: Optional[StrictBool] = False, - include_contributors: Optional[StrictBool] = False, - include_complete_image_count: Optional[StrictBool] = False, -): - """Returns project metadata - - :param project: project name - :type project: str - :param include_annotation_classes: enables project annotation classes output under - the key "annotation_classes" - :type include_annotation_classes: bool - :param include_settings: enables project settings output under - the key "settings" - :type include_settings: bool - :param include_workflow: enables project workflow output under - the key "workflow" - :type include_workflow: bool - :param include_contributors: enables project contributors output under - the key "contributors" - :type include_contributors: bool - - :param include_complete_image_count: enables project complete image count output under - the key "completed_images_count" - :type include_complete_image_count: bool - - :return: metadata of project - :rtype: dict - """ - project_name, folder_name = extract_project_folder(project) - response = ( - Controller.get_default() - .get_project_metadata( - project_name, - include_annotation_classes, - include_settings, - include_workflow, - include_contributors, - include_complete_image_count, + if response.errors: + raise AppException(response.errors) + logger.info("Successfully renamed project %s to %s.", project, response.data.name) + return ProjectSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def get_folder_metadata( + self, project: NotEmptyStr, folder_name: NotEmptyStr): + """Returns folder metadata + + :param project: project name + :type project: str + :param folder_name: folder's name + :type folder_name: str + + :return: metadata of folder + :rtype: dict + """ + result = ( + self.controller + .get_folder(project_name=project, folder_name=folder_name) + .data ) - .data - ) + if not result: + raise AppException("Folder not found.") + return FolderSerializer(result).serialize() - metadata = ProjectSerializer(response["project"]).serialize() + @Tracker + @validate_arguments + def delete_folders( + self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): + """Delete folder in project. - for elem in "classes", "workflows", "contributors": - if response.get(elem): - metadata[elem] = [ - BaseSerializer(attribute).serialize() for attribute in response[elem] - ] - return metadata - - -@Trackable -@validate_arguments -def get_project_settings(project: Union[NotEmptyStr, dict]): - """Gets project's settings. - - Return value example: [{ "attribute" : "Brightness", "value" : 10, ...},...] - - :param project: project name or metadata - :type project: str or dict - - :return: project settings - :rtype: list of dicts - """ - project_name, folder_name = extract_project_folder(project) - settings = Controller.get_default().get_project_settings(project_name=project_name) - settings = [ - SettingsSerializer(attribute).serialize() for attribute in settings.data - ] - return settings - - -@Trackable -@validate_arguments -def get_project_workflow(project: Union[str, dict]): - """Gets project's workflow. - - Return value example: [{ "step" : , "className" : , "tool" : , ...},...] - - :param project: project name or metadata - :type project: str or dict - - :return: project workflow - :rtype: list of dicts - """ - project_name, folder_name = extract_project_folder(project) - workflow = Controller.get_default().get_project_workflow(project_name=project_name) - if workflow.errors: - raise AppException(workflow.errors) - return workflow.data - - -@Trackable -@validate_arguments -def search_annotation_classes( - project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None -): - """Searches annotation classes by name_prefix (case-insensitive) - - :param project: project name - :type project: str - :param name_contains: search string. Returns those classes, - where the given string is found anywhere within its name. If None, all annotation classes will be returned. - :type name_prefix: str - - :return: annotation classes of the project - :rtype: list of dicts - """ - project_name, folder_name = extract_project_folder(project) - classes = Controller.get_default().search_annotation_classes( - project_name, name_contains - ) - classes = [BaseSerializer(attribute).serialize() for attribute in classes.data] - return classes - - -@Trackable -@validate_arguments -def set_project_default_image_quality_in_editor( - project: Union[NotEmptyStr, dict], image_quality_in_editor: Optional[str], -): - """Sets project's default image quality in editor setting. - - :param project: project name or metadata - :type project: str or dict - :param image_quality_in_editor: new setting value, should be "original" or "compressed" - :type image_quality_in_editor: str - """ - project_name, folder_name = extract_project_folder(project) - image_quality_in_editor = ImageQuality.get_value(image_quality_in_editor) - - response = Controller.get_default().set_project_settings( - project_name=project_name, - new_settings=[{"attribute": "ImageQuality", "value": image_quality_in_editor}], - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def pin_image( - project: Union[NotEmptyStr, dict], image_name: str, pin: Optional[StrictBool] = True -): - """Pins (or unpins) image - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param pin: sets to pin if True, else unpins image - :type pin: bool - """ - project_name, folder_name = extract_project_folder(project) - Controller.get_default().update_image( - project_name=project_name, - image_name=image_name, - folder_name=folder_name, - is_pinned=int(pin), - ) - - -@Trackable -@validate_arguments -def set_images_annotation_statuses( - project: Union[NotEmptyStr, dict], - annotation_status: NotEmptyStr, - image_names: Optional[List[NotEmptyStr]] = None, -): - """Sets annotation statuses of images - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_names: image names. If None, all the images in the project will be used - :type image_names: list of str - :param annotation_status: annotation status to set, - should be one of NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().set_images_annotation_statuses( - project_name, folder_name, image_names, annotation_status - ) - if response.errors: - raise AppException(response.errors) - logger.info("Annotations status of images changed") - - -@Trackable -@validate_arguments -def delete_images( - project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None -): - """Delete images in project. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_names: to be deleted images' names. If None, all the images will be deleted - :type image_names: list of strs - """ - project_name, folder_name = extract_project_folder(project) - - if not isinstance(image_names, list) and image_names is not None: - raise AppException("Image_names should be a list of str or None.") - - response = Controller.get_default().delete_images( - project_name=project_name, folder_name=folder_name, image_names=image_names - ) - if response.errors: - raise AppException(response.errors) - - logger.info( - f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" - ) - - -@Trackable -@validate_arguments -def assign_images(project: Union[NotEmptyStr, dict], image_names: List[str], user: str): - """Assigns images to a user. The assignment role, QA or Annotator, will - be deduced from the user's role in the project. With SDK, the user can be - assigned to a role in the project with the share_project function. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_names: list of image names to assign - :type image_names: list of str - :param user: user email - :type user: str - """ - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - - contributors = ( - Controller.get_default() - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users - ) - contributor = None - for c in contributors: - if c["user_id"] == user: - contributor = user - - if not contributor: - logger.warning( - f"Skipping {user}. {user} is not a verified contributor for the {project_name}" - ) - return - - response = Controller.get_default().assign_images( - project_name, folder_name, image_names, user - ) - if not response.errors: - logger.info(f"Assign images to user {user}") - else: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def unassign_images(project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr]): - """Removes assignment of given images for all assignees.With SDK, - the user can be assigned to a role in the project with the share_project - function. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_names: list of image unassign - :type image_names: list of str - """ - project_name, folder_name = extract_project_folder(project) - - response = Controller.get_default().un_assign_images( - project_name=project_name, folder_name=folder_name, image_names=image_names - ) - if response.errors: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def unassign_folder(project_name: NotEmptyStr, folder_name: NotEmptyStr): - """Removes assignment of given folder for all assignees. - With SDK, the user can be assigned to a role in the project - with the share_project function. - - :param project_name: project name - :type project_name: str - :param folder_name: folder name to remove assignees - :type folder_name: str - """ - response = Controller.get_default().un_assign_folder( - project_name=project_name, folder_name=folder_name - ) - if response.errors: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def assign_folder( - project_name: NotEmptyStr, folder_name: NotEmptyStr, users: List[NotEmptyStr] -): - """Assigns folder to users. With SDK, the user can be - assigned to a role in the project with the share_project function. - - :param project_name: project name or metadata of the project - :type project_name: str or dict - :param folder_name: folder name to assign - :type folder_name: str - :param users: list of user emails - :type users: list of str - """ - - contributors = ( - Controller.get_default() - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users - ) - verified_users = [i["user_id"] for i in contributors] - verified_users = set(users).intersection(set(verified_users)) - unverified_contributor = set(users) - verified_users - - for user in unverified_contributor: - logger.warning( - f"Skipping {user} from assignees. {user} is not a verified contributor for the {project_name}" - ) - - if not verified_users: - return - - response = Controller.get_default().assign_folder( - project_name=project_name, folder_name=folder_name, users=list(verified_users) - ) - - if response.errors: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def share_project( - project_name: NotEmptyStr, user: Union[str, dict], user_role: NotEmptyStr -): - """Share project with user. - - :param project_name: project name - :type project_name: str - :param user: user email or metadata of the user to share project with - :type user: str or dict - :param user_role: user role to apply, one of Admin , Annotator , QA , Customer , Viewer - :type user_role: str - """ - warning_msg = ( - "The share_project function is deprecated and will be removed with the coming release, " - "please use add_contributors_to_project instead." - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - if isinstance(user, dict): - user_id = user["id"] - else: - response = Controller.get_default().search_team_contributors(email=user) - if not response.data: - raise AppException(f"User {user} not found.") - user_id = response.data[0]["id"] - response = Controller.get_default().share_project( - project_name=project_name, user_id=user_id, user_role=user_role - ) - if response.errors: - raise AppException(response.errors) - - -@validate_arguments -def upload_images_from_folder_to_project( - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[List[NotEmptyStr], Tuple[NotEmptyStr]] - ] = constances.DEFAULT_IMAGE_EXTENSIONS, - annotation_status="NotStarted", - from_s3_bucket=None, - exclude_file_patterns: Optional[ - Iterable[NotEmptyStr] - ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders: Optional[StrictBool] = False, - image_quality_in_editor: Optional[str] = None, -): - """Uploads all images with given extensions from folder_path to the project. - Sets status of all the uploaded images to set_status if it is not None. - - If an image with existing name already exists in the project it won't be uploaded, - and its path will be appended to the third member of return value of this - function. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str or dict - - :param folder_path: from which folder to upload the images - :type folder_path: Path-like (str or Path) - - :param extensions: tuple or list of filename extensions to include from folder - :type extensions: tuple or list of strs - - :param annotation_status: value to set the annotation statuses of the uploaded images - NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - - :param exclude_file_patterns: filename patterns to exclude from uploading, - default value is to exclude SuperAnnotate export related ["___save.png", "___fuse.png"] - :type exclude_file_patterns: list or tuple of strs - - :param recursive_subfolders: enable recursive subfolder parsing - :type recursive_subfolders: bool - - :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. - Can be either "compressed" or "original". If None then the default value in project settings will be used. - :type image_quality_in_editor: str - - :return: uploaded, could-not-upload, existing-images filepaths - :rtype: tuple (3 members) of list of strs - """ - - project_name, folder_name = extract_project_folder(project) - if recursive_subfolders: - logger.info( - "When using recursive subfolder parsing same name images in different subfolders will overwrite each other." - ) - if not isinstance(extensions, (list, tuple)): - print(extensions) - raise AppException( - "extensions should be a list or a tuple in upload_images_from_folder_to_project" - ) - elif len(extensions) < 1: - return [], [], [] - - if exclude_file_patterns: - exclude_file_patterns = list(exclude_file_patterns) + list( - constances.DEFAULT_FILE_EXCLUDE_PATTERNS - ) - exclude_file_patterns = list(set(exclude_file_patterns)) - - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - logger.info( - "Uploading all images with extensions %s from %s to project %s. Excluded file patterns are: %s.", - extensions, - folder_path, - project_folder_name, - exclude_file_patterns, - ) - - use_case = Controller.get_default().upload_images_from_folder_to_project( - project_name=project_name, - folder_name=folder_name, - folder_path=folder_path, - extensions=extensions, - annotation_status=annotation_status, - from_s3_bucket=from_s3_bucket, - exclude_file_patterns=exclude_file_patterns, - recursive_sub_folders=recursive_subfolders, - image_quality_in_editor=image_quality_in_editor, - ) - images_to_upload, duplicates = use_case.images_to_upload - if len(duplicates): - logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) - ) - logger.info( - "Uploading %s images to project %s.", len(images_to_upload), project_folder_name - ) - if not images_to_upload: - return [], [], duplicates - if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: - for _ in use_case.execute(): - progress_bar.update(1) - return use_case.data - raise AppException(use_case.response.errors) - - -@Trackable -@validate_arguments -def get_project_image_count( - project: Union[NotEmptyStr, dict], with_all_subfolders: Optional[StrictBool] = False -): - """Returns number of images in the project. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param with_all_subfolders: enables recursive folder counting - :type with_all_subfolders: bool - - :return: number of images in the project - :rtype: int - """ - - project_name, folder_name = extract_project_folder(project) - - response = Controller.get_default().get_project_image_count( - project_name=project_name, - folder_name=folder_name, - with_all_subfolders=with_all_subfolders, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def download_image_annotations( - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Union[str, Path], -): - """Downloads annotations of the image (JSON and mask if pixel type project) - to local_dir_path. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param local_dir_path: local directory path to download to - :type local_dir_path: Path-like (str or Path) - - :return: paths of downloaded annotations - :rtype: tuple - """ - project_name, folder_name = extract_project_folder(project) - res = Controller.get_default().download_image_annotations( - project_name=project_name, - folder_name=folder_name, - image_name=image_name, - destination=local_dir_path, - ) - if res.errors: - raise AppException(res.errors) - return res.data - - -@Trackable -@validate_arguments -def get_exports(project: NotEmptyStr, return_metadata: Optional[StrictBool] = False): - """Get all prepared exports of the project. - - :param project: project name - :type project: str - :param return_metadata: return metadata of images instead of names - :type return_metadata: bool - - :return: names or metadata objects of the all prepared exports of the project - :rtype: list of strs or dicts - """ - response = Controller.get_default().get_exports( - project_name=project, return_metadata=return_metadata - ) - return response.data - - -@Trackable -@validate_arguments -def prepare_export( - project: Union[NotEmptyStr, dict], - folder_names: Optional[List[NotEmptyStr]] = None, - annotation_statuses: Optional[List[AnnotationStatuses]] = None, - include_fuse: Optional[StrictBool] = False, - only_pinned=False, -): - """Prepare annotations and classes.json for export. Original and fused images for images with - annotations can be included with include_fuse flag. - - :param project: project name - :type project: str - :param folder_names: names of folders to include in the export. If None, whole project will be exported - :type folder_names: list of str - :param annotation_statuses: images with which status to include, if None, - ["NotStarted", "InProgress", "QualityCheck", "Returned", "Completed", "Skipped"] will be chose - list elements should be one of NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_statuses: list of strs - :param include_fuse: enables fuse images in the export - :type include_fuse: bool - :param only_pinned: enable only pinned output in export. This option disables all other types of output. - :type only_pinned: bool - - :return: metadata object of the prepared export - :rtype: dict - """ - project_name, folder_name = extract_project_folder(project) - if folder_names is None: - folders = [folder_name] if folder_name else [] - else: - folders = folder_names - if not annotation_statuses: - annotation_statuses = [ - constances.AnnotationStatus.NOT_STARTED.name, - constances.AnnotationStatus.IN_PROGRESS.name, - constances.AnnotationStatus.QUALITY_CHECK.name, - constances.AnnotationStatus.RETURNED.name, - constances.AnnotationStatus.COMPLETED.name, - constances.AnnotationStatus.SKIPPED.name, - ] - response = Controller.get_default().prepare_export( - project_name=project_name, - folder_names=folders, - include_fuse=include_fuse, - only_pinned=only_pinned, - annotation_statuses=annotation_statuses, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def upload_videos_from_folder_to_project( - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[Tuple[NotEmptyStr], List[NotEmptyStr]] - ] = constances.DEFAULT_VIDEO_EXTENSIONS, - exclude_file_patterns: Optional[List[NotEmptyStr]] = (), - recursive_subfolders: Optional[StrictBool] = False, - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, -): - """Uploads image frames from all videos with given extensions from folder_path to the project. - Sets status of all the uploaded images to set_status if it is not None. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param folder_path: from which folder to upload the videos - :type folder_path: Path-like (str or Path) - :param extensions: tuple or list of filename extensions to include from folder - :type extensions: tuple or list of strs - :param exclude_file_patterns: filename patterns to exclude from uploading - :type exclude_file_patterns: listlike of strs - :param recursive_subfolders: enable recursive subfolder parsing - :type recursive_subfolders: bool - :param target_fps: how many frames per second need to extract from the video (approximate). - If None, all frames will be uploaded - :type target_fps: float - :param start_time: Time (in seconds) from which to start extracting frames - :type start_time: float - :param end_time: Time (in seconds) up to which to extract frames. If None up to end - :type end_time: float - :param annotation_status: value to set the annotation statuses of the uploaded images - NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. - Can be either "compressed" or "original". If None then the default value in project settings will be used. - :type image_quality_in_editor: str - - :return: uploaded and not-uploaded video frame images' filenames - :rtype: tuple of list of strs - """ - - project_name, folder_name = extract_project_folder(project) - - video_paths = [] - for extension in extensions: - if not recursive_subfolders: - video_paths += list(Path(folder_path).glob(f"*.{extension.lower()}")) - if os.name != "nt": - video_paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) - else: - logger.warning( - "When using recursive subfolder parsing same name videos " - "in different subfolders will overwrite each other." - ) - video_paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) - if os.name != "nt": - video_paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) - - video_paths = [str(path) for path in video_paths] - response = Controller.get_default().upload_videos( - project_name=project_name, - folder_name=folder_name, - paths=video_paths, - target_fps=target_fps, - start_time=start_time, - exclude_file_patterns=exclude_file_patterns, - end_time=end_time, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def upload_video_to_project( - project: Union[NotEmptyStr, dict], - video_path: Union[NotEmptyStr, Path], - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, -): - """Uploads image frames from video to platform. Uploaded images will have - names "_.jpg". - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param video_path: video to upload - :type video_path: Path-like (str or Path) - :param target_fps: how many frames per second need to extract from the video (approximate). - If None, all frames will be uploaded - :type target_fps: float - :param start_time: Time (in seconds) from which to start extracting frames - :type start_time: float - :param end_time: Time (in seconds) up to which to extract frames. If None up to end - :type end_time: float - :param annotation_status: value to set the annotation statuses of the uploaded - video frames NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. - Can be either "compressed" or "original". If None then the default value in project settings will be used. - :type image_quality_in_editor: str - - :return: filenames of uploaded images - :rtype: list of strs - """ - - project_name, folder_name = extract_project_folder(project) - - response = Controller.get_default().upload_videos( - project_name=project_name, - folder_name=folder_name, - paths=[video_path], - target_fps=target_fps, - start_time=start_time, - end_time=end_time, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def create_annotation_class( - project: Union[Project, NotEmptyStr], - name: NotEmptyStr, - color: NotEmptyStr, - attribute_groups: Optional[List[AttributeGroup]] = None, - class_type: ClassType = "object", -): - """Create annotation class in project - - :param project: project name - :type project: str - :param name: name for the class - :type name: str - :param color: RGB hex color value, e.g., "#FFFFAA" - :type color: str - :param attribute_groups: example: - [ { "name": "tall", "is_multiselect": 0, "attributes": [ { "name": "yes" }, { "name": "no" } ] }, - { "name": "age", "is_multiselect": 0, "attributes": [ { "name": "young" }, { "name": "old" } ] } ] - :type attribute_groups: list of dicts - :param class_type: class type - :type class_type: str - - :return: new class metadata - :rtype: dict - """ - if isinstance(project, Project): - project = project.dict() - attribute_groups = ( - list(map(lambda x: x.dict(), attribute_groups)) if attribute_groups else [] - ) - response = Controller.get_default().create_annotation_class( - project_name=project, - name=name, - color=color, - attribute_groups=attribute_groups, - class_type=class_type, - ) - if response.errors: - raise AppException(response.errors) - return BaseSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def delete_annotation_class( - project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] -): - """Deletes annotation class from project - - :param project: project name - :type project: str - :param annotation_class: annotation class name or metadata - :type annotation_class: str or dict - """ - Controller.get_default().delete_annotation_class( - project_name=project, annotation_class_name=annotation_class - ) - - -@Trackable -@validate_arguments -def download_annotation_classes_json(project: NotEmptyStr, folder: Union[str, Path]): - """Downloads project classes.json to folder - - :param project: project name - :type project: str - :param folder: folder to download to - :type folder: Path-like (str or Path) - - :return: path of the download file - :rtype: str - """ - response = Controller.get_default().download_annotation_classes( - project_name=project, download_path=folder - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def create_annotation_classes_from_classes_json( - project: Union[NotEmptyStr, dict], - classes_json: Union[List[AnnotationClassEntity], str, Path], - from_s3_bucket=False, -): - """Creates annotation classes in project from a SuperAnnotate format - annotation classes.json. - - :param project: project name - :type project: str - :param classes_json: JSON itself or path to the JSON file - :type classes_json: list or Path-like (str or Path) - :param from_s3_bucket: AWS S3 bucket to use. If None then classes_json is in local filesystem - :type from_s3_bucket: str - - :return: list of created annotation class metadatas - :rtype: list of dicts - """ - if isinstance(classes_json, str) or isinstance(classes_json, Path): - if from_s3_bucket: - from_session = boto3.Session() - from_s3 = from_session.resource("s3") - file = io.BytesIO() - from_s3_object = from_s3.Object(from_s3_bucket, classes_json) - from_s3_object.download_fileobj(file) - file.seek(0) - data = file - else: - data = open(classes_json) - classes_json = json.load(data) - try: - annotation_classes = parse_obj_as(List[AnnotationClassEntity], classes_json) - except ValidationError: - raise AppException("Couldn't validate annotation classes.") - logger.info(f"Creating annotation classes in project {project}.") - response = Controller.get_default().create_annotation_classes( - project_name=project, annotation_classes=annotation_classes, - ) - if response.errors: - raise AppException(response.errors) - return [BaseSerializer(i).serialize() for i in response.data] - - -@Trackable -@validate_arguments -def download_export( - project: Union[NotEmptyStr, dict], - export: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - extract_zip_contents: Optional[StrictBool] = True, - to_s3_bucket=None, -): - """Download prepared export. - - WARNING: Starting from version 1.9.0 :ref:`download_export ` additionally - requires :py:obj:`project` as first argument. - - :param project: project name - :type project: str - :param export: export name - :type export: str, dict - :param folder_path: where to download the export - :type folder_path: Path-like (str or Path) - :param extract_zip_contents: if False then a zip file will be downloaded, - if True the zip file will be extracted at folder_path - :type extract_zip_contents: bool - :param to_s3_bucket: AWS S3 bucket to use for download. If None then folder_path is in local filesystem. - :type to_s3_bucket: Bucket object - """ - project_name, folder_name = extract_project_folder(project) - export_name = export["name"] if isinstance(export, dict) else export - - use_case = Controller.get_default().download_export( - project_name=project_name, - export_name=export_name, - folder_path=folder_path, - extract_zip_contents=extract_zip_contents, - to_s3_bucket=to_s3_bucket, - ) - if use_case.is_valid(): - if to_s3_bucket: - with tqdm( - total=use_case.get_upload_files_count(), desc="Uploading" - ) as progress_bar: - for _ in use_case.execute(): - progress_bar.update() - progress_bar.close() - else: - for _ in use_case.execute(): - continue - logger.info(use_case.response.data) - else: - raise AppException(use_case.response.errors) + :param project: project name + :type project: str + :param folder_names: to be deleted folders' names + :type folder_names: list of strs + """ + + res = self.controller.delete_folders( + project_name=project, folder_names=folder_names + ) + if res.errors: + raise AppException(res.errors) + logger.info(f"Folders {folder_names} deleted in project {project}") + + @Tracker + @validate_arguments + def search_folders( + self, + project: NotEmptyStr, + folder_name: Optional[NotEmptyStr] = None, + return_metadata: Optional[StrictBool] = False, + ): + """Folder name based case-insensitive search for folders in project. + :param project: project name + :type project: str + :param folder_name: the new folder's name + :type folder_name: str. If None, all the folders in the project will be returned. + :param return_metadata: return metadata of folders instead of names + :type return_metadata: bool -@Trackable -@validate_arguments -def set_image_annotation_status( - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - annotation_status: NotEmptyStr, -): - """Sets the image annotation status - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param annotation_status: annotation status to set, - should be one of NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :return: metadata of the updated image - :rtype: dict - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().set_images_annotation_statuses( - project_name, folder_name, [image_name], annotation_status - ) - if response.errors: - raise AppException(response.errors) - image = ( - Controller.get_default().get_item(project_name, folder_name, image_name).data - ) - return BaseSerializer(image).serialize() - - -@Trackable -@validate_arguments -def set_project_workflow(project: Union[NotEmptyStr, dict], new_workflow: List[dict]): - """Sets project's workflow. - - new_workflow example: [{ "step" : , "className" : , "tool" : , - "attribute":[{"attribute" : {"name" : , "attribute_group" : {"name": }}}, - ...] - },...] - - :param project: project name or metadata - :type project: str or dict - :param new_workflow: new workflow list of dicts - :type new_workflow: list of dicts - """ - project_name, _ = extract_project_folder(project) - response = Controller.get_default().set_project_workflow( - project_name=project_name, steps=new_workflow - ) - if response.errors: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def download_image( - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Optional[Union[str, Path]] = "./", - include_annotations: Optional[StrictBool] = False, - include_fuse: Optional[StrictBool] = False, - include_overlay: Optional[StrictBool] = False, - variant: Optional[str] = "original", -): - """Downloads the image (and annotation if not None) to local_dir_path - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param local_dir_path: where to download the image - :type local_dir_path: Path-like (str or Path) - :param include_annotations: enables annotation download with the image - :type include_annotations: bool - :param include_fuse: enables fuse image download with the image - :type include_fuse: bool - :param include_overlay: enables overlay image download with the image - :type include_overlay: bool - :param variant: which resolution to download, can be 'original' or 'lores' - (low resolution used in web editor) - :type variant: str - - :return: paths of downloaded image and annotations if included - :rtype: tuple - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().download_image( - project_name=project_name, - folder_name=folder_name, - image_name=image_name, - download_path=str(local_dir_path), - image_variant=variant, - include_annotations=include_annotations, - include_fuse=include_fuse, - include_overlay=include_overlay, - ) - if response.errors: - raise AppException(response.errors) - logger.info(f"Downloaded image {image_name} to {local_dir_path} ") - return response.data - - -@Trackable -@validate_arguments -def attach_image_urls_to_project( - project: Union[NotEmptyStr, dict], - attachments: Union[str, Path], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", -): - """Link images on external storage to SuperAnnotate. - - :param project: project name or project folder path - :type project: str or dict - :param attachments: path to csv file on attachments metadata - :type attachments: Path-like (str or Path) - :param annotation_status: value to set the annotation statuses of the linked images: NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :return: list of linked image names, list of failed image names, list of duplicate image names - :rtype: tuple - """ - warning_msg = ( - "We're deprecating the attach_image_urls_to_project function. Please use attach_items instead. Learn more." - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException( - constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( - constances.ProjectType.get_name(project["project"].type) - ) + :return: folder names or metadatas + :rtype: list of strs or dicts + """ + + response = self.controller.search_folders( + project_name=project, folder_name=folder_name, include_users=return_metadata + ) + if response.errors: + raise AppException(response.errors) + data = response.data + if return_metadata: + return [FolderSerializer(folder).serialize() for folder in data] + return [folder.name for folder in data] + + @Tracker + @validate_arguments + def copy_image( + self, + source_project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + destination_project: Union[NotEmptyStr, dict], + include_annotations: Optional[StrictBool] = False, + copy_annotation_status: Optional[StrictBool] = False, + copy_pin: Optional[StrictBool] = False, + ): + """Copy image to a project. The image's project is the same as destination + project then the name will be changed to _()., + where is the next available number deducted from project image list. + + :param source_project: project name plus optional subfolder in the project (e.g., "project1/folder1") or + metadata of the project of source project + :type source_project: str or dict + :param image_name: image name + :type image_name: str + :param destination_project: project name or metadata of the project of destination project + :type destination_project: str or dict + :param include_annotations: enables annotations copy + :type include_annotations: bool + :param copy_annotation_status: enables annotations status copy + :type copy_annotation_status: bool + :param copy_pin: enables image pin status copy + :type copy_pin: bool + """ + source_project_name, source_folder_name = extract_project_folder(source_project) + + destination_project, destination_folder = extract_project_folder( + destination_project + ) + source_project_metadata = ( + self.controller.get_project_metadata(source_project_name).data ) - images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) - use_case = Controller.get_default().interactive_attach_urls( - project_name=project_name, - folder_name=folder_name, - files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 - annotation_status=annotation_status, - ) - if len(duplicate_images): - logger.warning( - constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) + destination_project_metadata = ( + self.controller.get_project_metadata(destination_project).data ) - if use_case.is_valid(): - logger.info( - constances.ATTACHING_FILES_MESSAGE.format( - len(images_to_upload), project_folder_name - ) + if destination_project_metadata["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ] or source_project_metadata["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[source_project_metadata["project"].type]) + + response = self.controller.copy_image( + from_project_name=source_project_name, + from_folder_name=source_folder_name, + to_project_name=destination_project, + to_folder_name=destination_folder, + image_name=image_name, + copy_annotation_status=copy_annotation_status, ) - with tqdm( - total=use_case.attachments_count, desc="Attaching urls" - ) as progress_bar: - for attached in use_case.execute(): - progress_bar.update(attached) - uploaded, duplications = use_case.data - uploaded = [i["name"] for i in uploaded] - duplications.extend(duplicate_images) - failed_images = [ - image["name"] - for image in images_to_upload - if image["name"] not in uploaded + duplications - ] - return uploaded, failed_images, duplications - raise AppException(use_case.response.errors) - - -@Trackable -@validate_arguments -def attach_video_urls_to_project( - project: Union[NotEmptyStr, dict], - attachments: Union[str, Path], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", -): - """Link videos on external storage to SuperAnnotate. - - :param project: project name or project folder path - :type project: str or dict - :param attachments: path to csv file on attachments metadata - :type attachments: Path-like (str or Path) - :param annotation_status: value to set the annotation statuses of the linked videos: NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :return: attached videos, failed videos, skipped videos - :rtype: (list, list, list) - """ - warning_msg = ( - "We're deprecating the attach_video_urls_to_project function. Please use attach_items instead. Learn more." - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - if project["project"].type != constances.ProjectType.VIDEO.value: - raise AppException( - constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( - constances.ProjectType.get_name(project["project"].type) + if response.errors: + raise AppException(response.errors) + + if include_annotations: + self.controller.copy_image_annotation_classes( + from_project_name=source_project_name, + from_folder_name=source_folder_name, + to_folder_name=destination_folder, + to_project_name=destination_project, + image_name=image_name, ) + if copy_pin: + self.controller.update_image( + project_name=destination_project, + folder_name=destination_folder, + image_name=image_name, + is_pinned=1, + ) + logger.info( + f"Copied image {source_project}/{image_name}" + f" to {destination_project}/{destination_folder}." ) - images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) - use_case = Controller.get_default().interactive_attach_urls( - project_name=project_name, - folder_name=folder_name, - files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 - annotation_status=annotation_status, - ) - if len(duplicate_images): - logger.warning( - constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) + @Tracker + @validate_arguments + def copy_images( + self, + source_project: Union[NotEmptyStr, dict], + image_names: Optional[List[NotEmptyStr]], + destination_project: Union[NotEmptyStr, dict], + include_annotations: Optional[StrictBool] = True, + copy_pin: Optional[StrictBool] = True, + ): + """Copy images in bulk between folders in a project + + :param source_project: project name or folder path (e.g., "project1/folder1") + :type source_project: str` + :param image_names: image names. If None, all images from source project will be copied + :type image_names: list of str + :param destination_project: project name or folder path (e.g., "project1/folder2") + :type destination_project: str + :param include_annotations: enables annotations copy + :type include_annotations: bool + :param copy_pin: enables image pin status copy + :type copy_pin: bool + :return: list of skipped image names + :rtype: list of strs + """ + warning_msg = ( + "We're deprecating the copy_images function. Please use copy_items instead. Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.copy_items" ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) + project_name, source_folder_name = extract_project_folder(source_project) - if use_case.is_valid(): - logger.info( - constances.ATTACHING_FILES_MESSAGE.format( - len(images_to_upload), project_folder_name + to_project_name, destination_folder_name = extract_project_folder( + destination_project + ) + if project_name != to_project_name: + raise AppException("Source and destination projects should be the same") + if not image_names: + images = ( + self.controller + .search_images(project_name=project_name, folder_path=source_folder_name) + .data ) + image_names = [image.name for image in images] + + res = self.controller.bulk_copy_images( + project_name=project_name, + from_folder_name=source_folder_name, + to_folder_name=destination_folder_name, + image_names=image_names, + include_annotations=include_annotations, + include_pin=copy_pin, ) - with tqdm( - total=use_case.attachments_count, desc="Attaching urls" - ) as progress_bar: - for attached in use_case.execute(): - progress_bar.update(attached) - uploaded, duplications = use_case.data - uploaded = [i["name"] for i in uploaded] - duplications.extend(duplicate_images) - failed_images = [ - image["name"] - for image in images_to_upload - if image["name"] not in uploaded + duplications - ] - return uploaded, failed_images, duplications - raise AppException(use_case.response.errors) - - -@Trackable -@validate_arguments -def upload_annotations_from_folder_to_project( - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, -): - """Finds and uploads all JSON files in the folder_path as annotations to the project. - - The JSON files should follow specific naming convention. For Vector - projects they should be named "___objects.json" (e.g., if - image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects - JSON file should be named "___pixel.json" and also second mask - image file should be present with the name "___save.png". In both cases - image with should be already present on the platform. - - Existing annotations will be overwritten. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str or dict - :param folder_path: from which folder to upload annotations - :type folder_path: str or dict - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - :param recursive_subfolders: enable recursive subfolder parsing - :type recursive_subfolders: bool - - :return: paths to annotations uploaded, could-not-upload, missing-images - :rtype: tuple of list of strs - """ - - project_name, folder_name = extract_project_folder(project) - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - if recursive_subfolders: + if res.errors: + raise AppException(res.errors) + skipped_images = res.data + done_count = len(image_names) - len(skipped_images) + message_postfix = "{from_path} to {to_path}." + message_prefix = "Copied images from " + if done_count > 1 or done_count == 0: + message_prefix = f"Copied {done_count}/{len(image_names)} images from " + elif done_count == 1: + message_prefix = "Copied an image from " logger.info( - "When using recursive subfolder parsing same name annotations in different " - "subfolders will overwrite each other.", - ) - logger.info( - "The JSON files should follow a specific naming convention, matching file names already present " - "on the platform. Existing annotations will be overwritten" - ) - - annotation_paths = get_annotation_paths( - folder_path, from_s3_bucket, recursive_subfolders - ) - - logger.info( - f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}." - ) - response = Controller.get_default().upload_annotations_from_folder( - project_name=project_name, - folder_name=folder_name, - annotation_paths=annotation_paths, # noqa: E203 - client_s3_bucket=from_s3_bucket, - folder_path=folder_path, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def upload_preannotations_from_folder_to_project( - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, -): - """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. - - The JSON files should follow specific naming convention. For Vector - projects they should be named "___objects.json" (e.g., if - image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects - JSON file should be named "___pixel.json" and also second mask - image file should be present with the name "___save.png". In both cases - image with should be already present on the platform. - - Existing pre-annotations will be overwritten. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param folder_path: from which folder to upload the pre-annotations - :type folder_path: Path-like (str or Path) - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - :param recursive_subfolders: enable recursive subfolder parsing - :type recursive_subfolders: bool - - :return: paths to pre-annotations uploaded and could-not-upload - :rtype: tuple of list of strs - """ - project_name, folder_name = extract_project_folder(project) - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - if recursive_subfolders: - logger.info( - "When using recursive subfolder parsing same name annotations in different " - "subfolders will overwrite each other.", - ) - logger.info( - "The JSON files should follow a specific naming convention, matching file names already present " - "on the platform. Existing annotations will be overwritten" - ) - annotation_paths = get_annotation_paths( - folder_path, from_s3_bucket, recursive_subfolders - ) - logger.info( - f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}." - ) - response = Controller.get_default().upload_annotations_from_folder( - project_name=project_name, - folder_name=folder_name, - annotation_paths=annotation_paths, # noqa: E203 - client_s3_bucket=from_s3_bucket, - folder_path=folder_path, - is_pre_annotations=True, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def upload_image_annotations( - project: Union[NotEmptyStr, dict], - image_name: str, - annotation_json: Union[str, Path, dict], - mask: Optional[Union[str, Path, bytes]] = None, - verbose: Optional[StrictBool] = True, -): - """Upload annotations from JSON (also mask for pixel annotations) - to the image. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param annotation_json: annotations in SuperAnnotate format JSON dict or path to JSON file - :type annotation_json: dict or Path-like (str or Path) - :param mask: BytesIO object or filepath to mask annotation for pixel projects in SuperAnnotate format - :type mask: BytesIO or Path-like (str or Path) - """ - - project_name, folder_name = extract_project_folder(project) - - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - - if not mask: - if not isinstance(annotation_json, dict): - mask_path = str(annotation_json).replace("___pixel.json", "___save.png") - else: - mask_path = f"{image_name}___save.png" - if os.path.exists(mask_path): - mask = open(mask_path, "rb").read() - elif isinstance(mask, str) or isinstance(mask, Path): - if os.path.exists(mask): - mask = open(mask, "rb").read() - - if not isinstance(annotation_json, dict): - if verbose: - logger.info("Uploading annotations from %s.", annotation_json) - annotation_json = json.load(open(annotation_json)) - response = Controller.get_default().upload_image_annotations( - project_name=project_name, - folder_name=folder_name, - image_name=image_name, - annotations=annotation_json, - mask=mask, - verbose=verbose, - ) - if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def download_model(model: MLModel, output_dir: Union[str, Path]): - """Downloads the neural network and related files - which are the .pth/pkl. .json, .yaml, classes_mapper.json - - :param model: the model that needs to be downloaded - :type model: dict - :param output_dir: the directiory in which the files will be saved - :type output_dir: str - :return: the metadata of the model - :rtype: dict - """ - res = Controller.get_default().download_ml_model( - model_data=model.dict(), download_path=output_dir - ) - if res.errors: - logger.error("\n".join([str(error) for error in res.errors])) - else: - return BaseSerializer(res.data).serialize() - - -@Trackable -@validate_arguments -def benchmark( - project: Union[NotEmptyStr, dict], - gt_folder: str, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[str, Path]] = None, - image_list=None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots=False, -): - """Computes benchmark score for each instance of given images that are present both gt_project_name project and projects in folder_names list: - - :param project: project name or metadata of the project - :type project: str or dict - :param gt_folder: project folder name that contains the ground truth annotations - :type gt_folder: str - :param folder_names: list of folder names in the project for which the scores will be computed - :type folder_names: list of str - :param export_root: root export path of the projects - :type export_root: Path-like (str or Path) - :param image_list: List of image names from the projects list that must be used. If None, then all images from the projects list will be used. Default: None - :type image_list: list - :param annot_type: Type of annotation instances to consider. Available candidates are: ["bbox", "polygon", "point"] - :type annot_type: str - :param show_plots: If True, show plots based on results of consensus computation. Default: False - :type show_plots: bool - - :return: Pandas DateFrame with columns (creatorEmail, QA, imageName, instanceId, className, area, attribute, folderName, score) - :rtype: pandas DataFrame - """ - project_name = project - if isinstance(project, dict): - project_name = project["name"] - - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - - if not export_root: - with tempfile.TemporaryDirectory() as temp_dir: - response = Controller.get_default().benchmark( - project_name=project_name, - ground_truth_folder_name=gt_folder, - folder_names=folder_names, - export_root=temp_dir, - image_list=image_list, - annot_type=annot_type, - show_plots=show_plots, + message_prefix + + message_postfix.format(from_path=source_project, to_path=destination_project) + ) + + return skipped_images + + @Tracker + @validate_arguments + def move_images( + self, + source_project: Union[NotEmptyStr, dict], + image_names: Optional[List[NotEmptyStr]], + destination_project: Union[NotEmptyStr, dict], + *args, + **kwargs, + ): + """Move images in bulk between folders in a project + + :param source_project: project name or folder path (e.g., "project1/folder1") + :type source_project: str + :param image_names: image names. If None, all images from source project will be moved + :type image_names: list of str + :param destination_project: project name or folder path (e.g., "project1/folder2") + :type destination_project: str + :return: list of skipped image names + :rtype: list of strs + """ + warning_msg = ( + "We're deprecating the move_images function. Please use move_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.move_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) + project_name, source_folder_name = extract_project_folder(source_project) + + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + + to_project_name, destination_folder_name = extract_project_folder( + destination_project + ) + + if project_name != to_project_name: + raise AppException( + "Source and destination projects should be the same for move_images" + ) + + if not image_names: + images = self.controller.search_images( + project_name=project_name, folder_path=source_folder_name ) + images = images.data + image_names = [image.name for image in images] - else: - response = Controller.get_default().benchmark( + response = self.controller.bulk_move_images( project_name=project_name, - ground_truth_folder_name=gt_folder, - folder_names=folder_names, - export_root=export_root, - image_list=image_list, - annot_type=annot_type, - show_plots=show_plots, + from_folder_name=source_folder_name, + to_folder_name=destination_folder_name, + image_names=image_names, ) if response.errors: raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def consensus( - project: NotEmptyStr, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[NotEmptyStr, Path]] = None, - image_list: Optional[List[NotEmptyStr]] = None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots: Optional[StrictBool] = False, -): - """Computes consensus score for each instance of given images that are present in at least 2 of the given projects: - - :param project: project name - :type project: str - :param folder_names: list of folder names in the project for which the scores will be computed - :type folder_names: list of str - :param export_root: root export path of the projects - :type export_root: Path-like (str or Path) - :param image_list: List of image names from the projects list that must be used. If None, then all images from the projects list will be used. Default: None - :type image_list: list - :param annot_type: Type of annotation instances to consider. Available candidates are: ["bbox", "polygon", "point"] - :type annot_type: str - :param show_plots: If True, show plots based on results of consensus computation. Default: False - :type show_plots: bool - - :return: Pandas DateFrame with columns (creatorEmail, QA, imageName, instanceId, className, area, attribute, folderName, score) - :rtype: pandas DataFrame - """ - - if export_root is None: - with tempfile.TemporaryDirectory() as temp_dir: - export_root = temp_dir - response = Controller.get_default().consensus( - project_name=project, - folder_names=folder_names, - export_path=export_root, - image_list=image_list, - annot_type=annot_type, - show_plots=show_plots, + moved_images = response.data + moved_count = len(moved_images) + message_postfix = "{from_path} to {to_path}." + message_prefix = "Moved images from " + if moved_count > 1 or moved_count == 0: + message_prefix = f"Moved {moved_count}/{len(image_names)} images from " + elif moved_count == 1: + message_prefix = "Moved an image from" + + logger.info( + message_prefix + + message_postfix.format(from_path=source_project, to_path=destination_project) + ) + + return list(set(image_names) - set(moved_images)) + + @Tracker + @validate_arguments + def get_project_metadata( + self, + project: Union[NotEmptyStr, dict], + include_annotation_classes: Optional[StrictBool] = False, + include_settings: Optional[StrictBool] = False, + include_workflow: Optional[StrictBool] = False, + include_contributors: Optional[StrictBool] = False, + include_complete_image_count: Optional[StrictBool] = False, + ): + """Returns project metadata + + :param project: project name + :type project: str + :param include_annotation_classes: enables project annotation classes output under + the key "annotation_classes" + :type include_annotation_classes: bool + :param include_settings: enables project settings output under + the key "settings" + :type include_settings: bool + :param include_workflow: enables project workflow output under + the key "workflow" + :type include_workflow: bool + :param include_contributors: enables project contributors output under + the key "contributors" + :type include_contributors: bool + + :param include_complete_image_count: enables project complete image count output under + the key "completed_images_count" + :type include_complete_image_count: bool + + :return: metadata of project + :rtype: dict + """ + project_name, folder_name = extract_project_folder(project) + response = ( + self.controller.get_project_metadata( + project_name, + include_annotation_classes, + include_settings, + include_workflow, + include_contributors, + include_complete_image_count, ) + .data + ) - else: - response = Controller.get_default().consensus( - project_name=project, - folder_names=folder_names, - export_path=export_root, - image_list=image_list, - annot_type=annot_type, - show_plots=show_plots, + metadata = ProjectSerializer(response["project"]).serialize() + + for elem in "classes", "workflows", "contributors": + if response.get(elem): + metadata[elem] = [ + BaseSerializer(attribute).serialize() for attribute in response[elem] + ] + return metadata + + @Tracker + @validate_arguments + def get_project_settings( + self, project: Union[NotEmptyStr, dict]): + """Gets project's settings. + + Return value example: [{ "attribute" : "Brightness", "value" : 10, ...},...] + + :param project: project name or metadata + :type project: str or dict + + :return: project settings + :rtype: list of dicts + """ + project_name, folder_name = extract_project_folder(project) + settings = self.controller.get_project_settings(project_name=project_name) + settings = [ + SettingsSerializer(attribute).serialize() for attribute in settings.data + ] + return settings + + @Tracker + @validate_arguments + def get_project_workflow( + self, project: Union[str, dict]): + """Gets project's workflow. + + Return value example: [{ "step" : , "className" : , "tool" : , ...},...] + + :param project: project name or metadata + :type project: str or dict + + :return: project workflow + :rtype: list of dicts + """ + project_name, folder_name = extract_project_folder(project) + workflow = self.controller.get_project_workflow(project_name=project_name) + if workflow.errors: + raise AppException(workflow.errors) + return workflow.data + + @Tracker + @validate_arguments + def search_annotation_classes( + self, + project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None + ): + """Searches annotation classes by name_prefix (case-insensitive) + + :param project: project name + :type project: str + :param name_contains: search string. Returns those classes, + where the given string is found anywhere within its name. If None, all annotation classes will be returned. + :type name_prefix: str + + :return: annotation classes of the project + :rtype: list of dicts + """ + project_name, folder_name = extract_project_folder(project) + classes = self.controller.search_annotation_classes( + project_name, name_contains + ) + classes = [BaseSerializer(attribute).serialize() for attribute in classes.data] + return classes + + @Tracker + @validate_arguments + def set_project_default_image_quality_in_editor( + self, + project: Union[NotEmptyStr, dict], + image_quality_in_editor: Optional[str], + ): + """Sets project's default image quality in editor setting. + + :param project: project name or metadata + :type project: str or dict + :param image_quality_in_editor: new setting value, should be "original" or "compressed" + :type image_quality_in_editor: str + """ + project_name, folder_name = extract_project_folder(project) + image_quality_in_editor = ImageQuality.get_value(image_quality_in_editor) + + response = self.controller.set_project_settings( + project_name=project_name, + new_settings=[{"attribute": "ImageQuality", "value": image_quality_in_editor}], ) if response.errors: raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def run_prediction( - project: Union[NotEmptyStr, dict], - images_list: List[NotEmptyStr], - model: Union[NotEmptyStr, dict], -): - """This function runs smart prediction on given list of images from a given project using the neural network of your choice - - :param project: the project in which the target images are uploaded. - :type project: str or dict - :param images_list: the list of image names on which smart prediction has to be run - :type images_list: list of str - :param model: the name of the model that should be used for running smart prediction - :type model: str or dict - :return: tupe of two lists, list of images on which the prediction has succeded and failed respectively - :rtype: tuple - """ - project_name = None - folder_name = None - if isinstance(project, dict): - project_name = project["name"] - if isinstance(project, str): - project_name, folder_name = extract_project_folder(project) + return response.data - model_name = model - if isinstance(model, dict): - model_name = model["name"] - - response = Controller.get_default().run_prediction( - project_name=project_name, - images_list=images_list, - model_name=model_name, - folder_name=folder_name, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def add_annotation_bbox_to_image( - project: NotEmptyStr, - image_name: NotEmptyStr, - bbox: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, -): - """Add a bounding box annotation to image annotations - - annotation_class_attributes has the form - [ {"name" : "" }, "groupName" : ""} ], ... ] - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param bbox: 4 element list of top-left x,y and bottom-right x, y coordinates - :type bbox: list of floats - :param annotation_class_name: annotation class name - :type annotation_class_name: str - :param annotation_class_attributes: list of annotation class attributes - :type annotation_class_attributes: list of 2 element dicts - :param error: if not None, marks annotation as error (True) or no-error (False) - :type error: bool - """ - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - response = Controller.get_default().get_annotations( - project_name=project_name, - folder_name=folder_name, - item_names=[image_name], - logging=False, - ) - if response.errors: - raise AppException(response.errors) - if response.data: - annotations = response.data[0] - else: - annotations = {} - annotations = add_annotation_bbox_to_json( - annotations, - bbox, - annotation_class_name, - annotation_class_attributes, - error, - image_name, - ) - - Controller.get_default().upload_image_annotations( - project_name, folder_name, image_name, annotations - ) - - -@Trackable -@validate_arguments -def add_annotation_point_to_image( - project: NotEmptyStr, - image_name: NotEmptyStr, - point: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, -): - """Add a point annotation to image annotations - - annotation_class_attributes has the form [ {"name" : "", "groupName" : ""}, ... ] - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param point: [x,y] list of coordinates - :type point: list of floats - :param annotation_class_name: annotation class name - :type annotation_class_name: str - :param annotation_class_attributes: list of annotation class attributes - :type annotation_class_attributes: list of 2 element dicts - :param error: if not None, marks annotation as error (True) or no-error (False) - :type error: bool - """ - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - response = Controller.get_default().get_annotations( - project_name=project_name, - folder_name=folder_name, - item_names=[image_name], - logging=False, - ) - if response.errors: - raise AppException(response.errors) - if response.data: - annotations = response.data[0] - else: - annotations = {} - annotations = add_annotation_point_to_json( - annotations, - point, - annotation_class_name, - image_name, - annotation_class_attributes, - error, - ) - Controller.get_default().upload_image_annotations( - project_name, folder_name, image_name, annotations - ) - - -@Trackable -@validate_arguments -def add_annotation_comment_to_image( - project: NotEmptyStr, - image_name: NotEmptyStr, - comment_text: NotEmptyStr, - comment_coords: List[float], - comment_author: EmailStr, - resolved: Optional[StrictBool] = False, -): - """Add a comment to SuperAnnotate format annotation JSON - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param comment_text: comment text - :type comment_text: str - :param comment_coords: [x, y] coords - :type comment_coords: list - :param comment_author: comment author email - :type comment_author: str - :param resolved: comment resolve status - :type resolved: bool - """ - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - response = Controller.get_default().get_annotations( - project_name=project_name, - folder_name=folder_name, - item_names=[image_name], - logging=False, - ) - if response.errors: - raise AppException(response.errors) - if response.data: - annotations = response.data[0] - else: - annotations = {} - annotations = add_annotation_comment_to_json( - annotations, - comment_text, - comment_coords, - comment_author, - resolved=resolved, - image_name=image_name, - ) - Controller.get_default().upload_image_annotations( - project_name, folder_name, image_name, annotations - ) - - -@Trackable -@validate_arguments -def upload_image_to_project( - project: NotEmptyStr, - img, - image_name: Optional[NotEmptyStr] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[NotEmptyStr] = None, -): - """Uploads image (io.BytesIO() or filepath to image) to project. - Sets status of the uploaded image to set_status if it is not None. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param img: image to upload - :type img: io.BytesIO() or Path-like (str or Path) - :param image_name: image name to set on platform. If None and img is filepath, - image name will be set to filename of the path - :type image_name: str - :param annotation_status: value to set the annotation statuses of the uploaded image NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. - Can be either "compressed" or "original". If None then the default value in project settings will be used. - :type image_quality_in_editor: str - """ - project_name, folder_name = extract_project_folder(project) - - response = Controller.get_default().upload_image_to_project( - project_name=project_name, - folder_name=folder_name, - image_name=image_name, - image=img, - annotation_status=annotation_status, - from_s3_bucket=from_s3_bucket, - image_quality_in_editor=image_quality_in_editor, - ) - if response.errors: - raise AppException(response.errors) - - -def search_models( - name: Optional[NotEmptyStr] = None, - type_: Optional[NotEmptyStr] = None, - project_id: Optional[int] = None, - task: Optional[NotEmptyStr] = None, - include_global: Optional[StrictBool] = True, -): - """Search for ML models. - - :param name: search string - :type name: str - :param type_: ml model type string - :type type_: str - :param project_id: project id - :type project_id: int - :param task: training task - :type task: str - :param include_global: include global ml models - :type include_global: bool - - :return: ml model metadata - :rtype: list of dicts - """ - res = Controller.get_default().search_models( - name=name, - model_type=type_, - project_id=project_id, - task=task, - include_global=include_global, - ) - return res.data - - -@Trackable -@validate_arguments -def upload_images_to_project( - project: NotEmptyStr, - img_paths: List[NotEmptyStr], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[ImageQualityChoices] = None, -): - """Uploads all images given in list of path objects in img_paths to the project. - Sets status of all the uploaded images to set_status if it is not None. - - If an image with existing name already exists in the project it won't be uploaded, - and its path will be appended to the third member of return value of this - function. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param img_paths: list of Path-like (str or Path) objects to upload - :type img_paths: list - :param annotation_status: value to set the annotation statuses of the uploaded images NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. - Can be either "compressed" or "original". If None then the default value in project settings will be used. - :type image_quality_in_editor: str - - :return: uploaded, could-not-upload, existing-images filepaths - :rtype: tuple (3 members) of list of strs - """ - project_name, folder_name = extract_project_folder(project) - - use_case = Controller.get_default().upload_images_to_project( - project_name=project_name, - folder_name=folder_name, - paths=img_paths, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - from_s3_bucket=from_s3_bucket, - ) - - images_to_upload, duplicates = use_case.images_to_upload - if len(duplicates): - logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) - ) - logger.info(f"Uploading {len(images_to_upload)} images to project {project}.") - uploaded, failed_images, duplications = [], [], duplicates - if not images_to_upload: - return uploaded, failed_images, duplications - if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: - for _ in use_case.execute(): - progress_bar.update(1) - uploaded, failed_images, duplications = use_case.data - if duplications: - logger.info(f"Duplicated images {', '.join(duplications)}") - return uploaded, failed_images, duplications - raise AppException(use_case.response.errors) - - -@Trackable -@validate_arguments -def aggregate_annotations_as_df( - project_root: Union[NotEmptyStr, Path], - project_type: ProjectTypes, - folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, -): - """Aggregate annotations as pandas dataframe from project root. - - :param project_root: the export path of the project - :type project_root: Path-like (str or Path) - - :param project_type: the project type, Vector/Pixel, Video or Document - :type project_type: str - - :param folder_names: Aggregate the specified folders from project_root. - If None aggregate all folders in the project_root - :type folder_names: list of Pathlike (str or Path) objects - - :return: DataFrame on annotations - :rtype: pandas DataFrame - """ - if project_type in ( - constances.ProjectType.VECTOR.name, - constances.ProjectType.PIXEL.name, + @Tracker + @validate_arguments + def pin_image( + self, + project: Union[NotEmptyStr, dict], image_name: str, pin: Optional[StrictBool] = True ): - from superannotate.lib.app.analytics.common import ( - aggregate_image_annotations_as_df, - ) + """Pins (or unpins) image - return aggregate_image_annotations_as_df( - project_root=project_root, - include_classes_wo_annotations=False, - include_comments=True, - include_tags=True, - folder_names=folder_names, + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param pin: sets to pin if True, else unpins image + :type pin: bool + """ + project_name, folder_name = extract_project_folder(project) + self.controller.update_image( + project_name=project_name, + image_name=image_name, + folder_name=folder_name, + is_pinned=int(pin), ) - elif project_type in ( - constances.ProjectType.VIDEO.name, - constances.ProjectType.DOCUMENT.name, + + @Tracker + @validate_arguments + def set_images_annotation_statuses( + self, + project: Union[NotEmptyStr, dict], + annotation_status: NotEmptyStr, + image_names: Optional[List[NotEmptyStr]] = None, ): - from superannotate.lib.app.analytics.aggregators import DataAggregator + """Sets annotation statuses of images - return DataAggregator( - project_type=project_type, - project_root=project_root, - folder_names=folder_names, - ).aggregate_annotations_as_df() - - -@Trackable -@validate_arguments -def delete_annotations( - project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None -): - """ - Delete image annotations from a given list of images. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_names: image names. If None, all image annotations from a given project/folder will be deleted. - :type image_names: list of strs - """ - - project_name, folder_name = extract_project_folder(project) - - response = Controller.get_default().delete_annotations( - project_name=project_name, folder_name=folder_name, image_names=image_names - ) - if response.errors: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def attach_document_urls_to_project( - project: Union[NotEmptyStr, dict], - attachments: Union[Path, NotEmptyStr], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", -): - """Link documents on external storage to SuperAnnotate. - - :param project: project name or project folder path - :type project: str or dict - :param attachments: path to csv file on attachments metadata - :type attachments: Path-like (str or Path) - :param annotation_status: value to set the annotation statuses of the linked documents: NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :return: list of attached documents, list of not attached documents, list of skipped documents - :rtype: tuple - """ - warning_msg = ( - "We're deprecating the attach_document_urls_to_project function. Please use attach_items instead. Learn more." - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, folder_name = extract_project_folder(project) - project = Controller.get_default().get_project_metadata(project_name).data - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - if project["project"].type != constances.ProjectType.DOCUMENT.value: - raise AppException( - constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( - constances.ProjectType.get_name(project["project"].type) - ) + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_names: image names. If None, all the images in the project will be used + :type image_names: list of str + :param annotation_status: annotation status to set, + should be one of NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.set_images_annotation_statuses( + project_name, folder_name, image_names, annotation_status ) + if response.errors: + raise AppException(response.errors) + logger.info("Annotations status of images changed") + + @Tracker + @validate_arguments + def delete_images( + self, + project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None + ): + """Delete images in project. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_names: to be deleted images' names. If None, all the images will be deleted + :type image_names: list of strs + """ + project_name, folder_name = extract_project_folder(project) - images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) + if not isinstance(image_names, list) and image_names is not None: + raise AppException("Image_names should be a list of str or None.") - use_case = Controller.get_default().interactive_attach_urls( - project_name=project_name, - folder_name=folder_name, - files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 - annotation_status=annotation_status, - ) - if len(duplicate_images): - logger.warning( - constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) + response = self.controller.delete_images( + project_name=project_name, folder_name=folder_name, image_names=image_names ) - if use_case.is_valid(): + if response.errors: + raise AppException(response.errors) + logger.info( - constances.ATTACHING_FILES_MESSAGE.format( - len(images_to_upload), project_folder_name - ) + f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) - with tqdm( - total=use_case.attachments_count, desc="Attaching urls" - ) as progress_bar: - for attached in use_case.execute(): - progress_bar.update(attached) - uploaded, duplications = use_case.data - uploaded = [i["name"] for i in uploaded] - duplications.extend(duplicate_images) - failed_images = [ - image["name"] - for image in images_to_upload - if image["name"] not in uploaded + duplications - ] - return uploaded, failed_images, duplications - raise AppException(use_case.response.errors) + @Tracker + @validate_arguments + def assign_images( + self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str): + """Assigns images to a user. The assignment role, QA or Annotator, will + be deduced from the user's role in the project. With SDK, the user can be + assigned to a role in the project with the share_project function. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_names: list of image names to assign + :type image_names: list of str + :param user: user email + :type user: str + """ + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + + contributors = ( + self.controller + .get_project_metadata(project_name=project_name, include_contributors=True) + .data["project"] + .users + ) + contributor = None + for c in contributors: + if c["user_id"] == user: + contributor = user -@Trackable -@validate_arguments -def validate_annotations( - project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] -): - """Validates given annotation JSON. + if not contributor: + logger.warning( + f"Skipping {user}. {user} is not a verified contributor for the {project_name}" + ) + return - :param project_type: The project type Vector, Pixel, Video or Document - :type project_type: str + response = self.controller.assign_images( + project_name, folder_name, image_names, user + ) + if not response.errors: + logger.info(f"Assign images to user {user}") + else: + raise AppException(response.errors) - :param annotations_json: path to annotation JSON - :type annotations_json: Path-like (str or Path) + @Tracker + @validate_arguments + def unassign_images( + self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr]): + """Removes assignment of given images for all assignees.With SDK, + the user can be assigned to a role in the project with the share_project + function. - :return: The success of the validation - :rtype: bool + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_names: list of image unassign + :type image_names: list of str """ - with open(annotations_json) as file: - annotation_data = json.loads(file.read()) - response = Controller.validate_annotations( - project_type, annotation_data, allow_extra=False + project_name, folder_name = extract_project_folder(project) + + response = self.controller.un_assign_images( + project_name=project_name, folder_name=folder_name, image_names=image_names ) if response.errors: raise AppException(response.errors) - is_valid, _ = response.data - if is_valid: - return True - print(response.report) - return False - - -@Trackable -@validate_arguments -def add_contributors_to_project( - project: NotEmptyStr, emails: conlist(EmailStr, min_items=1), role: AnnotatorRole -) -> Tuple[List[str], List[str]]: - """Add contributors to project. - - :param project: project name - :type project: str - - :param emails: users email - :type emails: list - - :param role: user role to apply, one of Admin , Annotator , QA - :type role: str - - :return: lists of added, skipped contributors of the project - :rtype: tuple (2 members) of lists of strs - """ - response = Controller.get_default().add_contributors_to_project( - project_name=project, emails=emails, role=role - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def invite_contributors_to_team( - emails: conlist(EmailStr, min_items=1), admin: StrictBool = False -) -> Tuple[List[str], List[str]]: - """Invites contributors to the team. - - :param emails: list of contributor emails - :type emails: list - - :param admin: enables admin privileges for the contributor - :type admin: bool - - :return: lists of invited, skipped contributors of the team - :rtype: tuple (2 members) of lists of strs - """ - response = Controller.get_default().invite_contributors_to_team( - emails=emails, set_admin=admin - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def get_annotations(project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): - """Returns annotations for the given list of items. - - :param project: project name or folder path (e.g., “project1/folder1”). - :type project: str - - :param items: item names. If None all items in the project will be exported - :type items: list of strs - - :return: list of annotations - :rtype: list of strs - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().get_annotations( - project_name, folder_name, items - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def get_annotations_per_frame(project: NotEmptyStr, video: NotEmptyStr, fps: int = 1): - """Returns per frame annotations for the given video. - - - :param project: project name or folder path (e.g., “project1/folder1”). - :type project: str - - :param video: video name - :type video: str - - :param fps: how many frames per second needs to be extracted from the video. - Will extract 1 frame per second by default. - :type fps: str - - :return: list of annotation objects - :rtype: list of dicts - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().get_annotations_per_frame( - project_name, folder_name, video_name=video, fps=fps - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def upload_priority_scores(project: NotEmptyStr, scores: List[PriorityScore]): - """Returns per frame annotations for the given video. - - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str - - :param scores: list of score objects - :type scores: list of dicts - - :return: lists of uploaded, skipped items - :rtype: tuple (2 members) of lists of strs - """ - project_name, folder_name = extract_project_folder(project) - project_folder_name = project - response = Controller.get_default().upload_priority_scores( - project_name, folder_name, scores, project_folder_name - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def get_integrations(): - """Get all integrations per team - - :return: metadata objects of all integrations of the team. - :rtype: list of dicts - """ - response = Controller.get_default().get_integrations() - if response.errors: - raise AppException(response.errors) - integrations = response.data - return BaseSerializer.serialize_iterable(integrations, ("name", "type", "root")) - - -@Trackable -@validate_arguments -def attach_items_from_integrated_storage( - project: NotEmptyStr, - integration: Union[NotEmptyStr, IntegrationEntity], - folder_path: Optional[NotEmptyStr] = None, -): - """Link images from integrated external storage to SuperAnnotate. - - :param project: project name or folder path where items should be attached (e.g., “project1/folder1”). - :type project: str - - :param integration: existing integration name or metadata dict to pull items from. - Mandatory keys in integration metadata’s dict is “name”. - :type integration: str or dict - - :param folder_path: Points to an exact folder/directory within given storage. - If None, items are fetched from the root directory. - :type folder_path: str - """ - project_name, folder_name = extract_project_folder(project) - if isinstance(integration, str): - integration = IntegrationEntity(name=integration) - response = Controller.get_default().attach_integrations( - project_name, folder_name, integration, folder_path - ) - if response.errors: - raise AppException(response.errors) - - -@Trackable -@validate_arguments -def query(project: NotEmptyStr, query: Optional[NotEmptyStr]): - """Return items that satisfy the given query. - Query syntax should be in SuperAnnotate query language(https://doc.superannotate.com/docs/query-search-1). - - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str - - :param query: SAQuL query string. - :type query: str - - :return: queried items’ metadata list - :rtype: list of dicts - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().query_entities(project_name, folder_name, query) - if response.errors: - raise AppException(response.errors) - return BaseSerializer.serialize_iterable(response.data) - - -@Trackable -@validate_arguments -def get_item_metadata( - project: NotEmptyStr, item_name: NotEmptyStr, -): - """Returns item metadata - - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str - - :param item_name: item name - :type item_name: str - - :return: metadata of item - :rtype: dict - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().get_item(project_name, folder_name, item_name) - if response.errors: - raise AppException(response.errors) - return BaseSerializer(response.data).serialize() - - -@Trackable -@validate_arguments -def search_items( - project: NotEmptyStr, - name_contains: NotEmptyStr = None, - annotation_status: Optional[AnnotationStatuses] = None, - annotator_email: Optional[NotEmptyStr] = None, - qa_email: Optional[NotEmptyStr] = None, - recursive: bool = False, -): - """Search items by filtering criteria. - - - :param project: project name or folder path (e.g., “project1/folder1”). - If recursive=False=True, then only the project name is required. - :type project: str - - :param name_contains: Returns those items, where the given string is found anywhere within an item’s name. - If None, all items returned, in accordance with the recursive=False parameter. - :type name_contains: str - - :param annotation_status: if not None, filters items by annotation status. - Values are: - “NotStarted” - “InProgress” - “QualityCheck” - “Returned” - “Completed” - “Skipped” - :type annotation_status: str - - - :param annotator_email: returns those items’ names that are assigned to the specified annotator. - If None, all items are returned. Strict equal. - :type annotator_email: str - - :param qa_email: returns those items’ names that are assigned to the specified QA. - If None, all items are returned. Strict equal. - :type qa_email: str - - :param recursive: search in the project’s root and all of its folders. - If False search only in the project’s root or given directory. - :type recursive: bool - - :return: items' metadata - :rtype: list of dicts - """ - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().list_items( - project_name, - folder_name, - name_contains=name_contains, - annotation_status=annotation_status, - annotator_email=annotator_email, - qa_email=qa_email, - recursive=recursive, - ) - if response.errors: - raise AppException(response.errors) - return BaseSerializer.serialize_iterable(response.data) - - -@Trackable -@validate_arguments -def attach_items( - project: Union[NotEmptyStr, dict], - attachments: AttachmentArg, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", -): - """Link items from external storage to SuperAnnotate using URLs. - - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str - - :param attachments: path to CSV file or list of dicts containing attachments URLs. - :type project: path-like (str or Path) or list of dicts - - :param annotation_status: value to set the annotation statuses of the linked items - “NotStarted” - “InProgress” - “QualityCheck” - “Returned” - “Completed” - “Skipped” - :type annotation_status: str - - :return: None - """ - attachments = attachments.data - project_name, folder_name = extract_project_folder(project) - if attachments and isinstance(attachments[0], AttachmentDict): - unique_attachments = set(attachments) - duplicate_attachments = [ - item - for item, count in collections.Counter(attachments).items() - if count > 1 - ] - else: - unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv( - attachments - ) - if duplicate_attachments: - logger.info("Dropping duplicates.") - unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments) - uploaded, fails, duplicated = [], [], [] - if unique_attachments: - logger.info( - f"Attaching {len(unique_attachments)} file(s) to project {project}." - ) - response = Controller.get_default().attach_items( - project_name=project_name, - folder_name=folder_name, - attachments=unique_attachments, - annotation_status=annotation_status, + + @Tracker + @validate_arguments + def unassign_folder( + self, project_name: NotEmptyStr, folder_name: NotEmptyStr): + """Removes assignment of given folder for all assignees. + With SDK, the user can be assigned to a role in the project + with the share_project function. + + :param project_name: project name + :type project_name: str + :param folder_name: folder name to remove assignees + :type folder_name: str + """ + response = self.controller.un_assign_folder( + project_name=project_name, folder_name=folder_name ) if response.errors: raise AppException(response.errors) - uploaded, duplicated = response.data - uploaded = [i["name"] for i in uploaded] - fails = [ - attachment.name - for attachment in unique_attachments - if attachment.name not in uploaded and attachment.name not in duplicated - ] - return uploaded, fails, duplicated - - -@Trackable -@validate_arguments -def copy_items( - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, - include_annotations: Optional[StrictBool] = True, -): - """Copy images in bulk between folders in a project - - :param source: project name or folder path to select items from (e.g., “project1/folder1”). - :type source: str - - :param destination: project name (root) or folder path to place copied items. - :type destination: str - - :param items: names of items to copy. If None, all items from the source directory will be copied. - :type itmes: list of str - - :param include_annotations: enables annotations copy - :type include_annotations: bool - - :return: list of skipped item names - :rtype: list of strs - """ - - project_name, source_folder = extract_project_folder(source) - - to_project_name, destination_folder = extract_project_folder(destination) - if project_name != to_project_name: - raise AppException("Source and destination projects should be the same") - - response = Controller.get_default().copy_items( - project_name=project_name, - from_folder=source_folder, - to_folder=destination_folder, - items=items, - include_annotations=include_annotations, - ) - if response.errors: - raise AppException(response.errors) - - return response.data - - -@Trackable -@validate_arguments -def move_items( - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, -): - """Copy images in bulk between folders in a project - - :param source: project name or folder path to pick items from (e.g., “project1/folder1”). - :type source: str - - :param destination: project name (root) or folder path to move items to. - :type destination: str - - :param items: names of items to move. If None, all items from the source directory will be moved. - :type items: list of str - - :return: list of skipped item names - :rtype: list of strs - """ - - project_name, source_folder = extract_project_folder(source) - to_project_name, destination_folder = extract_project_folder(destination) - if project_name != to_project_name: - raise AppException("Source and destination projects should be the same") - response = Controller.get_default().move_items( - project_name=project_name, - from_folder=source_folder, - to_folder=destination_folder, - items=items, - ) - if response.errors: - raise AppException(response.errors) - return response.data - - -@Trackable -@validate_arguments -def set_annotation_statuses( - project: Union[NotEmptyStr, dict], - annotation_status: AnnotationStatuses, - item_names: Optional[List[NotEmptyStr]] = None, -): - """Sets annotation statuses of items - - :param project: project name or folder path (e.g., “project1/folder1”). - :type project: str - - :param annotation_status: annotation status to set, should be one of. - “NotStarted” - “InProgress” - “QualityCheck” - “Returned” - “Completed” - “Skipped” - :type annotation_status: str - - :param item_names: item names to set the mentioned status for. If None, all the items in the project will be used. - :type item_names: str - - :return: None - """ - - project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().set_annotation_statuses( - project_name=project_name, - folder_name=folder_name, - annotation_status=annotation_status, - item_names=item_names, - ) - if response.errors: - raise AppException(response.errors) - return response.data + + @Tracker + @validate_arguments + def assign_folder( + self, + project_name: NotEmptyStr, folder_name: NotEmptyStr, users: List[NotEmptyStr] + ): + """Assigns folder to users. With SDK, the user can be + assigned to a role in the project with the share_project function. + + :param project_name: project name or metadata of the project + :type project_name: str or dict + :param folder_name: folder name to assign + :type folder_name: str + :param users: list of user emails + :type users: list of str + """ + + contributors = ( + self.controller + .get_project_metadata(project_name=project_name, include_contributors=True) + .data["project"] + .users + ) + verified_users = [i["user_id"] for i in contributors] + verified_users = set(users).intersection(set(verified_users)) + unverified_contributor = set(users) - verified_users + + for user in unverified_contributor: + logger.warning( + f"Skipping {user} from assignees. {user} is not a verified contributor for the {project_name}" + ) + + if not verified_users: + return + + response = self.controller.assign_folder( + project_name=project_name, folder_name=folder_name, users=list(verified_users) + ) + + if response.errors: + raise AppException(response.errors) + + @Tracker + @validate_arguments + def share_project( + self, + project_name: NotEmptyStr, user: Union[str, dict], user_role: NotEmptyStr + ): + """Share project with user. + + :param project_name: project name + :type project_name: str + :param user: user email or metadata of the user to share project with + :type user: str or dict + :param user_role: user role to apply, one of Admin , Annotator , QA , Customer , Viewer + :type user_role: str + """ + warning_msg = ( + "The share_project function is deprecated and will be removed with the coming release, " + "please use add_contributors_to_project instead." + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) + if isinstance(user, dict): + user_id = user["id"] + else: + response = self.controller.search_team_contributors(email=user) + if not response.data: + raise AppException(f"User {user} not found.") + user_id = response.data[0]["id"] + response = self.controller.share_project( + project_name=project_name, user_id=user_id, user_role=user_role + ) + if response.errors: + raise AppException(response.errors) + + @Tracker + @validate_arguments + def upload_images_from_folder_to_project( + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[List[NotEmptyStr], Tuple[NotEmptyStr]] + ] = constances.DEFAULT_IMAGE_EXTENSIONS, + annotation_status="NotStarted", + from_s3_bucket=None, + exclude_file_patterns: Optional[ + Iterable[NotEmptyStr] + ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders: Optional[StrictBool] = False, + image_quality_in_editor: Optional[str] = None, + ): + """Uploads all images with given extensions from folder_path to the project. + Sets status of all the uploaded images to set_status if it is not None. + + If an image with existing name already exists in the project it won't be uploaded, + and its path will be appended to the third member of return value of this + function. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str or dict + + :param folder_path: from which folder to upload the images + :type folder_path: Path-like (str or Path) + + :param extensions: tuple or list of filename extensions to include from folder + :type extensions: tuple or list of strs + + :param annotation_status: value to set the annotation statuses of the uploaded images + NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + + :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem + :type from_s3_bucket: str + + :param exclude_file_patterns: filename patterns to exclude from uploading, + default value is to exclude SuperAnnotate export related ["___save.png", "___fuse.png"] + :type exclude_file_patterns: list or tuple of strs + + :param recursive_subfolders: enable recursive subfolder parsing + :type recursive_subfolders: bool + + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. + Can be either "compressed" or "original". If None then the default value in project settings will be used. + :type image_quality_in_editor: str + + :return: uploaded, could-not-upload, existing-images filepaths + :rtype: tuple (3 members) of list of strs + """ + + project_name, folder_name = extract_project_folder(project) + if recursive_subfolders: + logger.info( + "When using recursive subfolder parsing same name images in different subfolders will overwrite each other." + ) + if not isinstance(extensions, (list, tuple)): + print(extensions) + raise AppException( + "extensions should be a list or a tuple in upload_images_from_folder_to_project" + ) + elif len(extensions) < 1: + return [], [], [] + + if exclude_file_patterns: + exclude_file_patterns = list(exclude_file_patterns) + list( + constances.DEFAULT_FILE_EXCLUDE_PATTERNS + ) + exclude_file_patterns = list(set(exclude_file_patterns)) + + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + + logger.info( + "Uploading all images with extensions %s from %s to project %s. Excluded file patterns are: %s.", + extensions, + folder_path, + project_folder_name, + exclude_file_patterns, + ) + + use_case = self.controller.upload_images_from_folder_to_project( + project_name=project_name, + folder_name=folder_name, + folder_path=folder_path, + extensions=extensions, + annotation_status=annotation_status, + from_s3_bucket=from_s3_bucket, + exclude_file_patterns=exclude_file_patterns, + recursive_sub_folders=recursive_subfolders, + image_quality_in_editor=image_quality_in_editor, + ) + images_to_upload, duplicates = use_case.images_to_upload + if len(duplicates): + logger.warning( + "%s already existing images found that won't be uploaded.", len(duplicates) + ) + logger.info( + "Uploading %s images to project %s.", len(images_to_upload), project_folder_name + ) + if not images_to_upload: + return [], [], duplicates + if use_case.is_valid(): + with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + for _ in use_case.execute(): + progress_bar.update(1) + return use_case.data + raise AppException(use_case.response.errors) + + @Tracker + @validate_arguments + def get_project_image_count( + self, + project: Union[NotEmptyStr, dict], with_all_subfolders: Optional[StrictBool] = False + ): + """Returns number of images in the project. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param with_all_subfolders: enables recursive folder counting + :type with_all_subfolders: bool + + :return: number of images in the project + :rtype: int + """ + + project_name, folder_name = extract_project_folder(project) + + response = self.controller.get_project_image_count( + project_name=project_name, + folder_name=folder_name, + with_all_subfolders=with_all_subfolders, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def download_image_annotations( + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Union[str, Path], + ): + """Downloads annotations of the image (JSON and mask if pixel type project) + to local_dir_path. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param local_dir_path: local directory path to download to + :type local_dir_path: Path-like (str or Path) + + :return: paths of downloaded annotations + :rtype: tuple + """ + project_name, folder_name = extract_project_folder(project) + res = self.controller.download_image_annotations( + project_name=project_name, + folder_name=folder_name, + image_name=image_name, + destination=local_dir_path, + ) + if res.errors: + raise AppException(res.errors) + return res.data + + @Tracker + @validate_arguments + def get_exports( + self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False): + """Get all prepared exports of the project. + + :param project: project name + :type project: str + :param return_metadata: return metadata of images instead of names + :type return_metadata: bool + + :return: names or metadata objects of the all prepared exports of the project + :rtype: list of strs or dicts + """ + response = self.controller.get_exports( + project_name=project, return_metadata=return_metadata + ) + return response.data + + @Tracker + @validate_arguments + def prepare_export( + self, + project: Union[NotEmptyStr, dict], + folder_names: Optional[List[NotEmptyStr]] = None, + annotation_statuses: Optional[List[AnnotationStatuses]] = None, + include_fuse: Optional[StrictBool] = False, + only_pinned=False, + ): + """Prepare annotations and classes.json for export. Original and fused images for images with + annotations can be included with include_fuse flag. + + :param project: project name + :type project: str + :param folder_names: names of folders to include in the export. If None, whole project will be exported + :type folder_names: list of str + :param annotation_statuses: images with which status to include, if None, + ["NotStarted", "InProgress", "QualityCheck", "Returned", "Completed", "Skipped"] will be chose + list elements should be one of NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_statuses: list of strs + :param include_fuse: enables fuse images in the export + :type include_fuse: bool + :param only_pinned: enable only pinned output in export. This option disables all other types of output. + :type only_pinned: bool + + :return: metadata object of the prepared export + :rtype: dict + """ + project_name, folder_name = extract_project_folder(project) + if folder_names is None: + folders = [folder_name] if folder_name else [] + else: + folders = folder_names + if not annotation_statuses: + annotation_statuses = [ + constances.AnnotationStatus.NOT_STARTED.name, + constances.AnnotationStatus.IN_PROGRESS.name, + constances.AnnotationStatus.QUALITY_CHECK.name, + constances.AnnotationStatus.RETURNED.name, + constances.AnnotationStatus.COMPLETED.name, + constances.AnnotationStatus.SKIPPED.name, + ] + response = self.controller.prepare_export( + project_name=project_name, + folder_names=folders, + include_fuse=include_fuse, + only_pinned=only_pinned, + annotation_statuses=annotation_statuses, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def upload_videos_from_folder_to_project( + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[Tuple[NotEmptyStr], List[NotEmptyStr]] + ] = constances.DEFAULT_VIDEO_EXTENSIONS, + exclude_file_patterns: Optional[List[NotEmptyStr]] = (), + recursive_subfolders: Optional[StrictBool] = False, + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, + ): + """Uploads image frames from all videos with given extensions from folder_path to the project. + Sets status of all the uploaded images to set_status if it is not None. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param folder_path: from which folder to upload the videos + :type folder_path: Path-like (str or Path) + :param extensions: tuple or list of filename extensions to include from folder + :type extensions: tuple or list of strs + :param exclude_file_patterns: filename patterns to exclude from uploading + :type exclude_file_patterns: listlike of strs + :param recursive_subfolders: enable recursive subfolder parsing + :type recursive_subfolders: bool + :param target_fps: how many frames per second need to extract from the video (approximate). + If None, all frames will be uploaded + :type target_fps: float + :param start_time: Time (in seconds) from which to start extracting frames + :type start_time: float + :param end_time: Time (in seconds) up to which to extract frames. If None up to end + :type end_time: float + :param annotation_status: value to set the annotation statuses of the uploaded images + NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. + Can be either "compressed" or "original". If None then the default value in project settings will be used. + :type image_quality_in_editor: str + + :return: uploaded and not-uploaded video frame images' filenames + :rtype: tuple of list of strs + """ + + project_name, folder_name = extract_project_folder(project) + + video_paths = [] + for extension in extensions: + if not recursive_subfolders: + video_paths += list(Path(folder_path).glob(f"*.{extension.lower()}")) + if os.name != "nt": + video_paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) + else: + logger.warning( + "When using recursive subfolder parsing same name videos " + "in different subfolders will overwrite each other." + ) + video_paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) + if os.name != "nt": + video_paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) + + video_paths = [str(path) for path in video_paths] + response = self.controller.upload_videos( + project_name=project_name, + folder_name=folder_name, + paths=video_paths, + target_fps=target_fps, + start_time=start_time, + exclude_file_patterns=exclude_file_patterns, + end_time=end_time, + annotation_status=annotation_status, + image_quality_in_editor=image_quality_in_editor, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def upload_video_to_project( + self, + project: Union[NotEmptyStr, dict], + video_path: Union[NotEmptyStr, Path], + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, + ): + """Uploads image frames from video to platform. Uploaded images will have + names "_.jpg". + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param video_path: video to upload + :type video_path: Path-like (str or Path) + :param target_fps: how many frames per second need to extract from the video (approximate). + If None, all frames will be uploaded + :type target_fps: float + :param start_time: Time (in seconds) from which to start extracting frames + :type start_time: float + :param end_time: Time (in seconds) up to which to extract frames. If None up to end + :type end_time: float + :param annotation_status: value to set the annotation statuses of the uploaded + video frames NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. + Can be either "compressed" or "original". If None then the default value in project settings will be used. + :type image_quality_in_editor: str + + :return: filenames of uploaded images + :rtype: list of strs + """ + + project_name, folder_name = extract_project_folder(project) + + response = self.controller.upload_videos( + project_name=project_name, + folder_name=folder_name, + paths=[video_path], + target_fps=target_fps, + start_time=start_time, + end_time=end_time, + annotation_status=annotation_status, + image_quality_in_editor=image_quality_in_editor, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def create_annotation_class( + self, + project: Union[Project, NotEmptyStr], + name: NotEmptyStr, + color: NotEmptyStr, + attribute_groups: Optional[List[AttributeGroup]] = None, + class_type: ClassType = "object", + ): + """Create annotation class in project + + :param project: project name + :type project: str + :param name: name for the class + :type name: str + :param color: RGB hex color value, e.g., "#FFFFAA" + :type color: str + :param attribute_groups: example: + [ { "name": "tall", "is_multiselect": 0, "attributes": [ { "name": "yes" }, { "name": "no" } ] }, + { "name": "age", "is_multiselect": 0, "attributes": [ { "name": "young" }, { "name": "old" } ] } ] + :type attribute_groups: list of dicts + :param class_type: class type + :type class_type: str + + :return: new class metadata + :rtype: dict + """ + if isinstance(project, Project): + project = project.dict() + attribute_groups = ( + list(map(lambda x: x.dict(), attribute_groups)) if attribute_groups else [] + ) + response = self.controller.create_annotation_class( + project_name=project, + name=name, + color=color, + attribute_groups=attribute_groups, + class_type=class_type, + ) + if response.errors: + raise AppException(response.errors) + return BaseSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def delete_annotation_class( + self, + project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] + ): + """Deletes annotation class from project + + :param project: project name + :type project: str + :param annotation_class: annotation class name or metadata + :type annotation_class: str or dict + """ + self.controller.delete_annotation_class( + project_name=project, annotation_class_name=annotation_class + ) + + @Tracker + @validate_arguments + def download_annotation_classes_json( + self, project: NotEmptyStr, folder: Union[str, Path]): + """Downloads project classes.json to folder + + :param project: project name + :type project: str + :param folder: folder to download to + :type folder: Path-like (str or Path) + + :return: path of the download file + :rtype: str + """ + response = self.controller.download_annotation_classes( + project_name=project, download_path=folder + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def create_annotation_classes_from_classes_json( + self, + project: Union[NotEmptyStr, dict], + classes_json: Union[List[AnnotationClassEntity], str, Path], + from_s3_bucket=False, + ): + """Creates annotation classes in project from a SuperAnnotate format + annotation classes.json. + + :param project: project name + :type project: str + :param classes_json: JSON itself or path to the JSON file + :type classes_json: list or Path-like (str or Path) + :param from_s3_bucket: AWS S3 bucket to use. If None then classes_json is in local filesystem + :type from_s3_bucket: str + + :return: list of created annotation class metadatas + :rtype: list of dicts + """ + if isinstance(classes_json, str) or isinstance(classes_json, Path): + if from_s3_bucket: + from_session = boto3.Session() + from_s3 = from_session.resource("s3") + file = io.BytesIO() + from_s3_object = from_s3.Object(from_s3_bucket, classes_json) + from_s3_object.download_fileobj(file) + file.seek(0) + data = file + else: + data = open(classes_json) + classes_json = json.load(data) + try: + annotation_classes = parse_obj_as(List[AnnotationClassEntity], classes_json) + except ValidationError: + raise AppException("Couldn't validate annotation classes.") + logger.info(f"Creating annotation classes in project {project}.") + response = self.controller.create_annotation_classes( + project_name=project, annotation_classes=annotation_classes, + ) + if response.errors: + raise AppException(response.errors) + return [BaseSerializer(i).serialize() for i in response.data] + + @Tracker + @validate_arguments + def download_export( + self, + project: Union[NotEmptyStr, dict], + export: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + extract_zip_contents: Optional[StrictBool] = True, + to_s3_bucket=None, + ): + """Download prepared export. + + WARNING: Starting from version 1.9.0 :ref:`download_export ` additionally + requires :py:obj:`project` as first argument. + + :param project: project name + :type project: str + :param export: export name + :type export: str, dict + :param folder_path: where to download the export + :type folder_path: Path-like (str or Path) + :param extract_zip_contents: if False then a zip file will be downloaded, + if True the zip file will be extracted at folder_path + :type extract_zip_contents: bool + :param to_s3_bucket: AWS S3 bucket to use for download. If None then folder_path is in local filesystem. + :type to_s3_bucket: Bucket object + """ + project_name, folder_name = extract_project_folder(project) + export_name = export["name"] if isinstance(export, dict) else export + + use_case = self.controller.download_export( + project_name=project_name, + export_name=export_name, + folder_path=folder_path, + extract_zip_contents=extract_zip_contents, + to_s3_bucket=to_s3_bucket, + ) + if use_case.is_valid(): + if to_s3_bucket: + with tqdm( + total=use_case.get_upload_files_count(), desc="Uploading" + ) as progress_bar: + for _ in use_case.execute(): + progress_bar.update() + progress_bar.close() + else: + for _ in use_case.execute(): + continue + logger.info(use_case.response.data) + else: + raise AppException(use_case.response.errors) + + @Tracker + @validate_arguments + def set_image_annotation_status( + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + annotation_status: NotEmptyStr, + ): + """Sets the image annotation status + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param annotation_status: annotation status to set, + should be one of NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + + :return: metadata of the updated image + :rtype: dict + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.set_images_annotation_statuses( + project_name, folder_name, [image_name], annotation_status + ) + if response.errors: + raise AppException(response.errors) + image = ( + self.controller.get_item(project_name, folder_name, image_name).data + ) + return BaseSerializer(image).serialize() + + @Tracker + @validate_arguments + def set_project_workflow( + self, project: Union[NotEmptyStr, dict], new_workflow: List[dict]): + """Sets project's workflow. + + new_workflow example: [{ "step" : , "className" : , "tool" : , + "attribute":[{"attribute" : {"name" : , "attribute_group" : {"name": }}}, + ...] + },...] + + :param project: project name or metadata + :type project: str or dict + :param new_workflow: new workflow list of dicts + :type new_workflow: list of dicts + """ + project_name, _ = extract_project_folder(project) + response = self.controller.set_project_workflow( + project_name=project_name, steps=new_workflow + ) + if response.errors: + raise AppException(response.errors) + + @Tracker + @validate_arguments + def download_image( + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Optional[Union[str, Path]] = "./", + include_annotations: Optional[StrictBool] = False, + include_fuse: Optional[StrictBool] = False, + include_overlay: Optional[StrictBool] = False, + variant: Optional[str] = "original", + ): + """Downloads the image (and annotation if not None) to local_dir_path + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param local_dir_path: where to download the image + :type local_dir_path: Path-like (str or Path) + :param include_annotations: enables annotation download with the image + :type include_annotations: bool + :param include_fuse: enables fuse image download with the image + :type include_fuse: bool + :param include_overlay: enables overlay image download with the image + :type include_overlay: bool + :param variant: which resolution to download, can be 'original' or 'lores' + (low resolution used in web editor) + :type variant: str + + :return: paths of downloaded image and annotations if included + :rtype: tuple + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.download_image( + project_name=project_name, + folder_name=folder_name, + image_name=image_name, + download_path=str(local_dir_path), + image_variant=variant, + include_annotations=include_annotations, + include_fuse=include_fuse, + include_overlay=include_overlay, + ) + if response.errors: + raise AppException(response.errors) + logger.info(f"Downloaded image {image_name} to {local_dir_path} ") + return response.data + + @Tracker + @validate_arguments + def attach_image_urls_to_project( + self, + project: Union[NotEmptyStr, dict], + attachments: Union[str, Path], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + ): + """Link images on external storage to SuperAnnotate. + + :param project: project name or project folder path + :type project: str or dict + :param attachments: path to csv file on attachments metadata + :type attachments: Path-like (str or Path) + :param annotation_status: value to set the annotation statuses of the linked images: NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + + :return: list of linked image names, list of failed image names, list of duplicate image names + :rtype: tuple + """ + warning_msg = ( + "We're deprecating the attach_image_urls_to_project function. Please use attach_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException( + constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( + constances.ProjectType.get_name(project["project"].type) + ) + ) + images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) + use_case = self.controller.interactive_attach_urls( + project_name=project_name, + folder_name=folder_name, + files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 + annotation_status=annotation_status, + ) + if len(duplicate_images): + logger.warning( + constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) + ) + + if use_case.is_valid(): + logger.info( + constances.ATTACHING_FILES_MESSAGE.format( + len(images_to_upload), project_folder_name + ) + ) + with tqdm( + total=use_case.attachments_count, desc="Attaching urls" + ) as progress_bar: + for attached in use_case.execute(): + progress_bar.update(attached) + uploaded, duplications = use_case.data + uploaded = [i["name"] for i in uploaded] + duplications.extend(duplicate_images) + failed_images = [ + image["name"] + for image in images_to_upload + if image["name"] not in uploaded + duplications + ] + return uploaded, failed_images, duplications + raise AppException(use_case.response.errors) + + @Tracker + @validate_arguments + def attach_video_urls_to_project( + self, + project: Union[NotEmptyStr, dict], + attachments: Union[str, Path], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + ): + """Link videos on external storage to SuperAnnotate. + + :param project: project name or project folder path + :type project: str or dict + :param attachments: path to csv file on attachments metadata + :type attachments: Path-like (str or Path) + :param annotation_status: value to set the annotation statuses of the linked videos: NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + + :return: attached videos, failed videos, skipped videos + :rtype: (list, list, list) + """ + warning_msg = ( + "We're deprecating the attach_video_urls_to_project function. Please use attach_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + + if project["project"].type != constances.ProjectType.VIDEO.value: + raise AppException( + constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( + constances.ProjectType.get_name(project["project"].type) + ) + ) + + images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) + use_case = self.controller.interactive_attach_urls( + project_name=project_name, + folder_name=folder_name, + files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 + annotation_status=annotation_status, + ) + if len(duplicate_images): + logger.warning( + constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) + ) + + if use_case.is_valid(): + logger.info( + constances.ATTACHING_FILES_MESSAGE.format( + len(images_to_upload), project_folder_name + ) + ) + with tqdm( + total=use_case.attachments_count, desc="Attaching urls" + ) as progress_bar: + for attached in use_case.execute(): + progress_bar.update(attached) + uploaded, duplications = use_case.data + uploaded = [i["name"] for i in uploaded] + duplications.extend(duplicate_images) + failed_images = [ + image["name"] + for image in images_to_upload + if image["name"] not in uploaded + duplications + ] + return uploaded, failed_images, duplications + raise AppException(use_case.response.errors) + + @Tracker + @validate_arguments + def upload_annotations_from_folder_to_project( + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, + ): + """Finds and uploads all JSON files in the folder_path as annotations to the project. + + The JSON files should follow specific naming convention. For Vector + projects they should be named "___objects.json" (e.g., if + image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects + JSON file should be named "___pixel.json" and also second mask + image file should be present with the name "___save.png". In both cases + image with should be already present on the platform. + + Existing annotations will be overwritten. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str or dict + :param folder_path: from which folder to upload annotations + :type folder_path: str or dict + :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem + :type from_s3_bucket: str + :param recursive_subfolders: enable recursive subfolder parsing + :type recursive_subfolders: bool + + :return: paths to annotations uploaded, could-not-upload, missing-images + :rtype: tuple of list of strs + """ + + project_name, folder_name = extract_project_folder(project) + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + + if recursive_subfolders: + logger.info( + "When using recursive subfolder parsing same name annotations in different " + "subfolders will overwrite each other.", + ) + logger.info( + "The JSON files should follow a specific naming convention, matching file names already present " + "on the platform. Existing annotations will be overwritten" + ) + + annotation_paths = get_annotation_paths( + folder_path, from_s3_bucket, recursive_subfolders + ) + + logger.info( + f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}." + ) + response = self.controller.upload_annotations_from_folder( + project_name=project_name, + folder_name=folder_name, + annotation_paths=annotation_paths, # noqa: E203 + client_s3_bucket=from_s3_bucket, + folder_path=folder_path, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def upload_preannotations_from_folder_to_project( + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, + ): + """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. + + The JSON files should follow specific naming convention. For Vector + projects they should be named "___objects.json" (e.g., if + image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects + JSON file should be named "___pixel.json" and also second mask + image file should be present with the name "___save.png". In both cases + image with should be already present on the platform. + + Existing pre-annotations will be overwritten. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param folder_path: from which folder to upload the pre-annotations + :type folder_path: Path-like (str or Path) + :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem + :type from_s3_bucket: str + :param recursive_subfolders: enable recursive subfolder parsing + :type recursive_subfolders: bool + + :return: paths to pre-annotations uploaded and could-not-upload + :rtype: tuple of list of strs + """ + project_name, folder_name = extract_project_folder(project) + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + if recursive_subfolders: + logger.info( + "When using recursive subfolder parsing same name annotations in different " + "subfolders will overwrite each other.", + ) + logger.info( + "The JSON files should follow a specific naming convention, matching file names already present " + "on the platform. Existing annotations will be overwritten" + ) + annotation_paths = get_annotation_paths( + folder_path, from_s3_bucket, recursive_subfolders + ) + logger.info( + f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}." + ) + response = self.controller.upload_annotations_from_folder( + project_name=project_name, + folder_name=folder_name, + annotation_paths=annotation_paths, # noqa: E203 + client_s3_bucket=from_s3_bucket, + folder_path=folder_path, + is_pre_annotations=True, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def upload_image_annotations( + self, + project: Union[NotEmptyStr, dict], + image_name: str, + annotation_json: Union[str, Path, dict], + mask: Optional[Union[str, Path, bytes]] = None, + verbose: Optional[StrictBool] = True, + ): + """Upload annotations from JSON (also mask for pixel annotations) + to the image. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param annotation_json: annotations in SuperAnnotate format JSON dict or path to JSON file + :type annotation_json: dict or Path-like (str or Path) + :param mask: BytesIO object or filepath to mask annotation for pixel projects in SuperAnnotate format + :type mask: BytesIO or Path-like (str or Path) + """ + + project_name, folder_name = extract_project_folder(project) + + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + + if not mask: + if not isinstance(annotation_json, dict): + mask_path = str(annotation_json).replace("___pixel.json", "___save.png") + else: + mask_path = f"{image_name}___save.png" + if os.path.exists(mask_path): + mask = open(mask_path, "rb").read() + elif isinstance(mask, str) or isinstance(mask, Path): + if os.path.exists(mask): + mask = open(mask, "rb").read() + + if not isinstance(annotation_json, dict): + if verbose: + logger.info("Uploading annotations from %s.", annotation_json) + annotation_json = json.load(open(annotation_json)) + response = self.controller.upload_image_annotations( + project_name=project_name, + folder_name=folder_name, + image_name=image_name, + annotations=annotation_json, + mask=mask, + verbose=verbose, + ) + if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: + raise AppException(response.errors) + + @Tracker + @validate_arguments + def download_model( + self, model: MLModel, output_dir: Union[str, Path]): + """Downloads the neural network and related files + which are the .pth/pkl. .json, .yaml, classes_mapper.json + + :param model: the model that needs to be downloaded + :type model: dict + :param output_dir: the directiory in which the files will be saved + :type output_dir: str + :return: the metadata of the model + :rtype: dict + """ + res = self.controller.download_ml_model( + model_data=model.dict(), download_path=output_dir + ) + if res.errors: + logger.error("\n".join([str(error) for error in res.errors])) + else: + return BaseSerializer(res.data).serialize() + + @Tracker + @validate_arguments + def benchmark( + self, + project: Union[NotEmptyStr, dict], + gt_folder: str, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[str, Path]] = None, + image_list=None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots=False, + ): + """Computes benchmark score for each instance of given images that are present both gt_project_name project and projects in folder_names list: + + :param project: project name or metadata of the project + :type project: str or dict + :param gt_folder: project folder name that contains the ground truth annotations + :type gt_folder: str + :param folder_names: list of folder names in the project for which the scores will be computed + :type folder_names: list of str + :param export_root: root export path of the projects + :type export_root: Path-like (str or Path) + :param image_list: List of image names from the projects list that must be used. If None, then all images from the projects list will be used. Default: None + :type image_list: list + :param annot_type: Type of annotation instances to consider. Available candidates are: ["bbox", "polygon", "point"] + :type annot_type: str + :param show_plots: If True, show plots based on results of consensus computation. Default: False + :type show_plots: bool + + :return: Pandas DateFrame with columns (creatorEmail, QA, imageName, instanceId, className, area, attribute, folderName, score) + :rtype: pandas DataFrame + """ + project_name = project + if isinstance(project, dict): + project_name = project["name"] + + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + + if not export_root: + with tempfile.TemporaryDirectory() as temp_dir: + response = self.controller.benchmark( + project_name=project_name, + ground_truth_folder_name=gt_folder, + folder_names=folder_names, + export_root=temp_dir, + image_list=image_list, + annot_type=annot_type, + show_plots=show_plots, + ) + + else: + response = self.controller.benchmark( + project_name=project_name, + ground_truth_folder_name=gt_folder, + folder_names=folder_names, + export_root=export_root, + image_list=image_list, + annot_type=annot_type, + show_plots=show_plots, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def consensus( + self, + project: NotEmptyStr, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[NotEmptyStr, Path]] = None, + image_list: Optional[List[NotEmptyStr]] = None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots: Optional[StrictBool] = False, + ): + """Computes consensus score for each instance of given images that are present in at least 2 of the given projects: + + :param project: project name + :type project: str + :param folder_names: list of folder names in the project for which the scores will be computed + :type folder_names: list of str + :param export_root: root export path of the projects + :type export_root: Path-like (str or Path) + :param image_list: List of image names from the projects list that must be used. If None, then all images from the projects list will be used. Default: None + :type image_list: list + :param annot_type: Type of annotation instances to consider. Available candidates are: ["bbox", "polygon", "point"] + :type annot_type: str + :param show_plots: If True, show plots based on results of consensus computation. Default: False + :type show_plots: bool + + :return: Pandas DateFrame with columns (creatorEmail, QA, imageName, instanceId, className, area, attribute, folderName, score) + :rtype: pandas DataFrame + """ + + if export_root is None: + with tempfile.TemporaryDirectory() as temp_dir: + export_root = temp_dir + response = self.controller.consensus( + project_name=project, + folder_names=folder_names, + export_path=export_root, + image_list=image_list, + annot_type=annot_type, + show_plots=show_plots, + ) + + else: + response = self.controller.consensus( + project_name=project, + folder_names=folder_names, + export_path=export_root, + image_list=image_list, + annot_type=annot_type, + show_plots=show_plots, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def run_prediction( + self, + project: Union[NotEmptyStr, dict], + images_list: List[NotEmptyStr], + model: Union[NotEmptyStr, dict], + ): + """This function runs smart prediction on given list of images from a given project using the neural network of your choice + + :param project: the project in which the target images are uploaded. + :type project: str or dict + :param images_list: the list of image names on which smart prediction has to be run + :type images_list: list of str + :param model: the name of the model that should be used for running smart prediction + :type model: str or dict + :return: tupe of two lists, list of images on which the prediction has succeded and failed respectively + :rtype: tuple + """ + project_name = None + folder_name = None + if isinstance(project, dict): + project_name = project["name"] + if isinstance(project, str): + project_name, folder_name = extract_project_folder(project) + + model_name = model + if isinstance(model, dict): + model_name = model["name"] + + response = self.controller.run_prediction( + project_name=project_name, + images_list=images_list, + model_name=model_name, + folder_name=folder_name, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def add_annotation_bbox_to_image( + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + bbox: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, + ): + """Add a bounding box annotation to image annotations + + annotation_class_attributes has the form + [ {"name" : "" }, "groupName" : ""} ], ... ] + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param bbox: 4 element list of top-left x,y and bottom-right x, y coordinates + :type bbox: list of floats + :param annotation_class_name: annotation class name + :type annotation_class_name: str + :param annotation_class_attributes: list of annotation class attributes + :type annotation_class_attributes: list of 2 element dicts + :param error: if not None, marks annotation as error (True) or no-error (False) + :type error: bool + """ + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + response = self.controller.get_annotations( + project_name=project_name, + folder_name=folder_name, + item_names=[image_name], + logging=False, + ) + if response.errors: + raise AppException(response.errors) + if response.data: + annotations = response.data[0] + else: + annotations = {} + annotations = add_annotation_bbox_to_json( + annotations, + bbox, + annotation_class_name, + annotation_class_attributes, + error, + image_name, + ) + + self.controller.upload_image_annotations( + project_name, folder_name, image_name, annotations + ) + + @Tracker + @validate_arguments + def add_annotation_point_to_image( + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + point: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, + ): + """Add a point annotation to image annotations + + annotation_class_attributes has the form [ {"name" : "", "groupName" : ""}, ... ] + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param point: [x,y] list of coordinates + :type point: list of floats + :param annotation_class_name: annotation class name + :type annotation_class_name: str + :param annotation_class_attributes: list of annotation class attributes + :type annotation_class_attributes: list of 2 element dicts + :param error: if not None, marks annotation as error (True) or no-error (False) + :type error: bool + """ + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + response = self.controller.get_annotations( + project_name=project_name, + folder_name=folder_name, + item_names=[image_name], + logging=False, + ) + if response.errors: + raise AppException(response.errors) + if response.data: + annotations = response.data[0] + else: + annotations = {} + annotations = add_annotation_point_to_json( + annotations, + point, + annotation_class_name, + image_name, + annotation_class_attributes, + error, + ) + self.controller.upload_image_annotations( + project_name, folder_name, image_name, annotations + ) + + @Tracker + @validate_arguments + def add_annotation_comment_to_image( + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + comment_text: NotEmptyStr, + comment_coords: List[float], + comment_author: EmailStr, + resolved: Optional[StrictBool] = False, + ): + """Add a comment to SuperAnnotate format annotation JSON + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_name: image name + :type image_name: str + :param comment_text: comment text + :type comment_text: str + :param comment_coords: [x, y] coords + :type comment_coords: list + :param comment_author: comment author email + :type comment_author: str + :param resolved: comment resolve status + :type resolved: bool + """ + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + if project["project"].type in [ + constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + ]: + raise AppException(LIMITED_FUNCTIONS[project["project"].type]) + response = self.controller.get_annotations( + project_name=project_name, + folder_name=folder_name, + item_names=[image_name], + logging=False, + ) + if response.errors: + raise AppException(response.errors) + if response.data: + annotations = response.data[0] + else: + annotations = {} + annotations = add_annotation_comment_to_json( + annotations, + comment_text, + comment_coords, + comment_author, + resolved=resolved, + image_name=image_name, + ) + self.controller.upload_image_annotations( + project_name, folder_name, image_name, annotations + ) + + @Tracker + @validate_arguments + def upload_image_to_project( + self, + project: NotEmptyStr, + img, + image_name: Optional[NotEmptyStr] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[NotEmptyStr] = None, + ): + """Uploads image (io.BytesIO() or filepath to image) to project. + Sets status of the uploaded image to set_status if it is not None. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param img: image to upload + :type img: io.BytesIO() or Path-like (str or Path) + :param image_name: image name to set on platform. If None and img is filepath, + image name will be set to filename of the path + :type image_name: str + :param annotation_status: value to set the annotation statuses of the uploaded image NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem + :type from_s3_bucket: str + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. + Can be either "compressed" or "original". If None then the default value in project settings will be used. + :type image_quality_in_editor: str + """ + project_name, folder_name = extract_project_folder(project) + + response = self.controller.upload_image_to_project( + project_name=project_name, + folder_name=folder_name, + image_name=image_name, + image=img, + annotation_status=annotation_status, + from_s3_bucket=from_s3_bucket, + image_quality_in_editor=image_quality_in_editor, + ) + if response.errors: + raise AppException(response.errors) + + def search_models( + self, + name: Optional[NotEmptyStr] = None, + type_: Optional[NotEmptyStr] = None, + project_id: Optional[int] = None, + task: Optional[NotEmptyStr] = None, + include_global: Optional[StrictBool] = True, + ): + """Search for ML models. + + :param name: search string + :type name: str + :param type_: ml model type string + :type type_: str + :param project_id: project id + :type project_id: int + :param task: training task + :type task: str + :param include_global: include global ml models + :type include_global: bool + + :return: ml model metadata + :rtype: list of dicts + """ + res = self.controller.search_models( + name=name, + model_type=type_, + project_id=project_id, + task=task, + include_global=include_global, + ) + return res.data + + @Tracker + @validate_arguments + def upload_images_to_project( + self, + project: NotEmptyStr, + img_paths: List[NotEmptyStr], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[ImageQualityChoices] = None, + ): + """Uploads all images given in list of path objects in img_paths to the project. + Sets status of all the uploaded images to set_status if it is not None. + + If an image with existing name already exists in the project it won't be uploaded, + and its path will be appended to the third member of return value of this + function. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param img_paths: list of Path-like (str or Path) objects to upload + :type img_paths: list + :param annotation_status: value to set the annotation statuses of the uploaded images NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem + :type from_s3_bucket: str + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. + Can be either "compressed" or "original". If None then the default value in project settings will be used. + :type image_quality_in_editor: str + + :return: uploaded, could-not-upload, existing-images filepaths + :rtype: tuple (3 members) of list of strs + """ + project_name, folder_name = extract_project_folder(project) + + use_case = self.controller.upload_images_to_project( + project_name=project_name, + folder_name=folder_name, + paths=img_paths, + annotation_status=annotation_status, + image_quality_in_editor=image_quality_in_editor, + from_s3_bucket=from_s3_bucket, + ) + + images_to_upload, duplicates = use_case.images_to_upload + if len(duplicates): + logger.warning( + "%s already existing images found that won't be uploaded.", len(duplicates) + ) + logger.info(f"Uploading {len(images_to_upload)} images to project {project}.") + uploaded, failed_images, duplications = [], [], duplicates + if not images_to_upload: + return uploaded, failed_images, duplications + if use_case.is_valid(): + with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + for _ in use_case.execute(): + progress_bar.update(1) + uploaded, failed_images, duplications = use_case.data + if duplications: + logger.info(f"Duplicated images {', '.join(duplications)}") + return uploaded, failed_images, duplications + raise AppException(use_case.response.errors) + + @Tracker + @validate_arguments + def aggregate_annotations_as_df( + self, + project_root: Union[NotEmptyStr, Path], + project_type: ProjectTypes, + folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, + ): + """Aggregate annotations as pandas dataframe from project root. + + :param project_root: the export path of the project + :type project_root: Path-like (str or Path) + + :param project_type: the project type, Vector/Pixel, Video or Document + :type project_type: str + + :param folder_names: Aggregate the specified folders from project_root. + If None aggregate all folders in the project_root + :type folder_names: list of Pathlike (str or Path) objects + + :return: DataFrame on annotations + :rtype: pandas DataFrame + """ + if project_type in ( + constances.ProjectType.VECTOR.name, + constances.ProjectType.PIXEL.name, + ): + from superannotate.lib.app.analytics.common import ( + aggregate_image_annotations_as_df, + ) + + return aggregate_image_annotations_as_df( + project_root=project_root, + include_classes_wo_annotations=False, + include_comments=True, + include_tags=True, + folder_names=folder_names, + ) + elif project_type in ( + constances.ProjectType.VIDEO.name, + constances.ProjectType.DOCUMENT.name, + ): + from superannotate.lib.app.analytics.aggregators import DataAggregator + + return DataAggregator( + project_type=project_type, + project_root=project_root, + folder_names=folder_names, + ).aggregate_annotations_as_df() + + @Tracker + @validate_arguments + def delete_annotations( + self, + project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None + ): + """ + Delete image annotations from a given list of images. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param image_names: image names. If None, all image annotations from a given project/folder will be deleted. + :type image_names: list of strs + """ + + project_name, folder_name = extract_project_folder(project) + + response = self.controller.delete_annotations( + project_name=project_name, folder_name=folder_name, image_names=image_names + ) + if response.errors: + raise AppException(response.errors) + + @Tracker + @validate_arguments + def attach_document_urls_to_project( + self, + project: Union[NotEmptyStr, dict], + attachments: Union[Path, NotEmptyStr], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + ): + """Link documents on external storage to SuperAnnotate. + + :param project: project name or project folder path + :type project: str or dict + :param attachments: path to csv file on attachments metadata + :type attachments: Path-like (str or Path) + :param annotation_status: value to set the annotation statuses of the linked documents: NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + + :return: list of attached documents, list of not attached documents, list of skipped documents + :rtype: tuple + """ + warning_msg = ( + "We're deprecating the attach_document_urls_to_project function. Please use attach_items instead. Learn more." + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + + if project["project"].type != constances.ProjectType.DOCUMENT.value: + raise AppException( + constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( + constances.ProjectType.get_name(project["project"].type) + ) + ) + + images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) + + use_case = self.controller.interactive_attach_urls( + project_name=project_name, + folder_name=folder_name, + files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 + annotation_status=annotation_status, + ) + if len(duplicate_images): + logger.warning( + constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) + ) + if use_case.is_valid(): + logger.info( + constances.ATTACHING_FILES_MESSAGE.format( + len(images_to_upload), project_folder_name + ) + ) + with tqdm( + total=use_case.attachments_count, desc="Attaching urls" + ) as progress_bar: + for attached in use_case.execute(): + progress_bar.update(attached) + uploaded, duplications = use_case.data + uploaded = [i["name"] for i in uploaded] + duplications.extend(duplicate_images) + failed_images = [ + image["name"] + for image in images_to_upload + if image["name"] not in uploaded + duplications + ] + return uploaded, failed_images, duplications + raise AppException(use_case.response.errors) + + @Tracker + @validate_arguments + def validate_annotations( + self, + project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] + ): + """Validates given annotation JSON. + + :param project_type: The project type Vector, Pixel, Video or Document + :type project_type: str + + :param annotations_json: path to annotation JSON + :type annotations_json: Path-like (str or Path) + + :return: The success of the validation + :rtype: bool + """ + with open(annotations_json) as file: + annotation_data = json.loads(file.read()) + response = Controller.validate_annotations( + project_type, annotation_data, allow_extra=False + ) + if response.errors: + raise AppException(response.errors) + is_valid, _ = response.data + if is_valid: + return True + print(response.report) + return False + + @Tracker + @validate_arguments + def add_contributors_to_project( + self, + project: NotEmptyStr, emails: conlist(EmailStr, min_items=1), role: AnnotatorRole + ) -> Tuple[List[str], List[str]]: + """Add contributors to project. + + :param project: project name + :type project: str + + :param emails: users email + :type emails: list + + :param role: user role to apply, one of Admin , Annotator , QA + :type role: str + + :return: lists of added, skipped contributors of the project + :rtype: tuple (2 members) of lists of strs + """ + response = self.controller.add_contributors_to_project( + project_name=project, emails=emails, role=role + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def invite_contributors_to_team( + self, + emails: conlist(EmailStr, min_items=1), admin: StrictBool = False + ) -> Tuple[List[str], List[str]]: + """Invites contributors to the team. + + :param emails: list of contributor emails + :type emails: list + + :param admin: enables admin privileges for the contributor + :type admin: bool + + :return: lists of invited, skipped contributors of the team + :rtype: tuple (2 members) of lists of strs + """ + response = self.controller.invite_contributors_to_team( + emails=emails, set_admin=admin + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def get_annotations( + self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): + """Returns annotations for the given list of items. + + :param project: project name or folder path (e.g., “project1/folder1”). + :type project: str + + :param items: item names. If None all items in the project will be exported + :type items: list of strs + + :return: list of annotations + :rtype: list of strs + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.get_annotations( + project_name, folder_name, items + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def get_annotations_per_frame( + self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1): + """Returns per frame annotations for the given video. + + + :param project: project name or folder path (e.g., “project1/folder1”). + :type project: str + + :param video: video name + :type video: str + + :param fps: how many frames per second needs to be extracted from the video. + Will extract 1 frame per second by default. + :type fps: str + + :return: list of annotation objects + :rtype: list of dicts + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.get_annotations_per_frame( + project_name, folder_name, video_name=video, fps=fps + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def upload_priority_scores( + self, project: NotEmptyStr, scores: List[PriorityScore]): + """Returns per frame annotations for the given video. + + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str + + :param scores: list of score objects + :type scores: list of dicts + + :return: lists of uploaded, skipped items + :rtype: tuple (2 members) of lists of strs + """ + project_name, folder_name = extract_project_folder(project) + project_folder_name = project + response = self.controller.upload_priority_scores( + project_name, folder_name, scores, project_folder_name + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def get_integrations( + self): + """Get all integrations per team + + :return: metadata objects of all integrations of the team. + :rtype: list of dicts + """ + response = self.controller.get_integrations() + if response.errors: + raise AppException(response.errors) + integrations = response.data + return BaseSerializer.serialize_iterable(integrations, ("name", "type", "root")) + + @Tracker + @validate_arguments + def attach_items_from_integrated_storage( + self, + project: NotEmptyStr, + integration: Union[NotEmptyStr, IntegrationEntity], + folder_path: Optional[NotEmptyStr] = None, + ): + """Link images from integrated external storage to SuperAnnotate. + + :param project: project name or folder path where items should be attached (e.g., “project1/folder1”). + :type project: str + + :param integration: existing integration name or metadata dict to pull items from. + Mandatory keys in integration metadata’s dict is “name”. + :type integration: str or dict + + :param folder_path: Points to an exact folder/directory within given storage. + If None, items are fetched from the root directory. + :type folder_path: str + """ + project_name, folder_name = extract_project_folder(project) + if isinstance(integration, str): + integration = IntegrationEntity(name=integration) + response = self.controller.attach_integrations( + project_name, folder_name, integration, folder_path + ) + if response.errors: + raise AppException(response.errors) + + @Tracker + @validate_arguments + def query( + self, project: NotEmptyStr, query: Optional[NotEmptyStr]): + """Return items that satisfy the given query. + Query syntax should be in SuperAnnotate query language(https://doc.superannotate.com/docs/query-search-1). + + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str + + :param query: SAQuL query string. + :type query: str + + :return: queried items’ metadata list + :rtype: list of dicts + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.query_entities(project_name, folder_name, query) + if response.errors: + raise AppException(response.errors) + return BaseSerializer.serialize_iterable(response.data) + + @Tracker + @validate_arguments + def get_item_metadata( + self, + project: NotEmptyStr, item_name: NotEmptyStr, + ): + """Returns item metadata + + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str + + :param item_name: item name + :type item_name: str + + :return: metadata of item + :rtype: dict + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.get_item(project_name, folder_name, item_name) + if response.errors: + raise AppException(response.errors) + return BaseSerializer(response.data).serialize() + + @Tracker + @validate_arguments + def search_items( + self, + project: NotEmptyStr, + name_contains: NotEmptyStr = None, + annotation_status: Optional[AnnotationStatuses] = None, + annotator_email: Optional[NotEmptyStr] = None, + qa_email: Optional[NotEmptyStr] = None, + recursive: bool = False, + ): + """Search items by filtering criteria. + + + :param project: project name or folder path (e.g., “project1/folder1”). + If recursive=False=True, then only the project name is required. + :type project: str + + :param name_contains: Returns those items, where the given string is found anywhere within an item’s name. + If None, all items returned, in accordance with the recursive=False parameter. + :type name_contains: str + + :param annotation_status: if not None, filters items by annotation status. + Values are: + “NotStarted” + “InProgress” + “QualityCheck” + “Returned” + “Completed” + “Skipped” + :type annotation_status: str + + + :param annotator_email: returns those items’ names that are assigned to the specified annotator. + If None, all items are returned. Strict equal. + :type annotator_email: str + + :param qa_email: returns those items’ names that are assigned to the specified QA. + If None, all items are returned. Strict equal. + :type qa_email: str + + :param recursive: search in the project’s root and all of its folders. + If False search only in the project’s root or given directory. + :type recursive: bool + + :return: items' metadata + :rtype: list of dicts + """ + project_name, folder_name = extract_project_folder(project) + response = self.controller.list_items( + project_name, + folder_name, + name_contains=name_contains, + annotation_status=annotation_status, + annotator_email=annotator_email, + qa_email=qa_email, + recursive=recursive, + ) + if response.errors: + raise AppException(response.errors) + return BaseSerializer.serialize_iterable(response.data) + + @Tracker + @validate_arguments + def attach_items( + self, + project: Union[NotEmptyStr, dict], + attachments: AttachmentArg, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + ): + """Link items from external storage to SuperAnnotate using URLs. + + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str + + :param attachments: path to CSV file or list of dicts containing attachments URLs. + :type project: path-like (str or Path) or list of dicts + + :param annotation_status: value to set the annotation statuses of the linked items + “NotStarted” + “InProgress” + “QualityCheck” + “Returned” + “Completed” + “Skipped” + :type annotation_status: str + + :return: None + """ + attachments = attachments.data + project_name, folder_name = extract_project_folder(project) + if attachments and isinstance(attachments[0], AttachmentDict): + unique_attachments = set(attachments) + duplicate_attachments = [ + item + for item, count in collections.Counter(attachments).items() + if count > 1 + ] + else: + unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv( + attachments + ) + if duplicate_attachments: + logger.info("Dropping duplicates.") + unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments) + uploaded, fails, duplicated = [], [], [] + if unique_attachments: + logger.info( + f"Attaching {len(unique_attachments)} file(s) to project {project}." + ) + response = self.controller.attach_items( + project_name=project_name, + folder_name=folder_name, + attachments=unique_attachments, + annotation_status=annotation_status, + ) + if response.errors: + raise AppException(response.errors) + uploaded, duplicated = response.data + uploaded = [i["name"] for i in uploaded] + fails = [ + attachment.name + for attachment in unique_attachments + if attachment.name not in uploaded and attachment.name not in duplicated + ] + return uploaded, fails, duplicated + + @Tracker + @validate_arguments + def copy_items( + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, + include_annotations: Optional[StrictBool] = True, + ): + """Copy images in bulk between folders in a project + + :param source: project name or folder path to select items from (e.g., “project1/folder1”). + :type source: str + + :param destination: project name (root) or folder path to place copied items. + :type destination: str + + :param items: names of items to copy. If None, all items from the source directory will be copied. + :type items: list of str + + :param include_annotations: enables annotations copy + :type include_annotations: bool + + :return: list of skipped item names + :rtype: list of strs + """ + + project_name, source_folder = extract_project_folder(source) + + to_project_name, destination_folder = extract_project_folder(destination) + if project_name != to_project_name: + raise AppException("Source and destination projects should be the same") + + response = self.controller.copy_items( + project_name=project_name, + from_folder=source_folder, + to_folder=destination_folder, + items=items, + include_annotations=include_annotations, + ) + if response.errors: + raise AppException(response.errors) + + return response.data + + @Tracker + @validate_arguments + def move_items( + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, + ): + """Copy images in bulk between folders in a project + + :param source: project name or folder path to pick items from (e.g., “project1/folder1”). + :type source: str + + :param destination: project name (root) or folder path to move items to. + :type destination: str + + :param items: names of items to move. If None, all items from the source directory will be moved. + :type items: list of str + + :return: list of skipped item names + :rtype: list of strs + """ + + project_name, source_folder = extract_project_folder(source) + to_project_name, destination_folder = extract_project_folder(destination) + if project_name != to_project_name: + raise AppException("Source and destination projects should be the same") + response = self.controller.move_items( + project_name=project_name, + from_folder=source_folder, + to_folder=destination_folder, + items=items, + ) + if response.errors: + raise AppException(response.errors) + return response.data + + @Tracker + @validate_arguments + def set_annotation_statuses( + self, + project: Union[NotEmptyStr, dict], + annotation_status: AnnotationStatuses, + item_names: Optional[List[NotEmptyStr]] = None, + ): + """Sets annotation statuses of items + + :param project: project name or folder path (e.g., “project1/folder1”). + :type project: str + + :param annotation_status: annotation status to set, should be one of. + “NotStarted” + “InProgress” + “QualityCheck” + “Returned” + “Completed” + “Skipped” + :type annotation_status: str + + :param item_names: item names to set the mentioned status for. If None, all the items in the project will be used. + :type item_names: str + + :return: None + """ + + project_name, folder_name = extract_project_folder(project) + response = self.controller.set_annotation_statuses( + project_name=project_name, + folder_name=folder_name, + annotation_status=annotation_status, + item_names=item_names, + ) + if response.errors: + raise AppException(response.errors) + return response.data diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index c262d0776..cb936313f 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -182,9 +182,9 @@ def validate(cls, value: Union[str]) -> Union[str]: def validate_arguments(func): @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(self, *args, **kwargs): try: - return pydantic_validate_arguments(func)(*args, **kwargs) + return pydantic_validate_arguments(func)(self, *args, **kwargs) except ValidationError as e: raise AppException(wrap_error(e)) diff --git a/src/superannotate/lib/app/mixp/decorators.py b/src/superannotate/lib/app/mixp/decorators.py index 8a0f86a64..0ef6d8a81 100644 --- a/src/superannotate/lib/app/mixp/decorators.py +++ b/src/superannotate/lib/app/mixp/decorators.py @@ -2,7 +2,7 @@ import sys from inspect import signature -from lib import get_default_controller + from mixpanel import Mixpanel from superannotate.logger import get_default_logger from version import __version__ @@ -13,8 +13,8 @@ def get_mp_instance() -> Mixpanel: - if "api.annotate.online" in get_default_controller()._backend_url: - return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") + # if "api.annotate.online" in get_default_controller()._backend_url: + # return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 75193342a..88eb8e90d 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -1,3 +1,4 @@ +import uuid from collections import defaultdict from typing import Union @@ -5,6 +6,39 @@ from superannotate.logger import get_default_logger +class Session: + def __init__(self, pk: str): + self.pk = pk + self._uuid = str(uuid.uuid4()) + self._data_dict = {} + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + if type is not None: + return False + + @property + def data(self): + return self._data_dict + + def __setitem__(self, key, item): + self._data_dict[key] = item + + def __getitem__(self, key): + return self._data_dict[key] + + def __repr__(self): + return repr(self._data_dict) + + def __delitem__(self, key): + del self._data_dict[key] + + def clear(self): + return self._data_dict.clear() + + class Reporter: def __init__( self, @@ -12,6 +46,7 @@ def __init__( log_warning: bool = True, disable_progress_bar: bool = False, log_debug: bool = True, + session: Session = None ): self.logger = get_default_logger() self._log_info = log_info @@ -23,6 +58,7 @@ def __init__( self.debug_messages = [] self.custom_messages = defaultdict(set) self.progress_bar = None + self.session = session def disable_warnings(self): self._log_warning = False @@ -91,6 +127,10 @@ def messages(self): for key, values in self.custom_messages.items(): yield f"{key} [{', '.join(values)}]" + def track(self, key, value): + if self.session: + self.session[key] = value + class Progress: def __init__(self, iterations: Union[int, range], description: str = "Processing"): diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 9a2861811..c078ea615 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -74,8 +74,7 @@ def validate_fuse(self): def validate_folder_names(self): if self._folder_names: condition = ( - Condition("team_id", self._project.team_id, EQ) & - Condition("project_id", self._project.id, EQ) + Condition("team_id", self._project.team_id, EQ) & Condition("project_id", self._project.id, EQ) ) existing_folders = {folder.name for folder in self._folders.get_all(condition)} folder_names_set = set(self._folder_names) diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 3ea2835dd..98168445e 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -2,7 +2,6 @@ import io import os from abc import ABCMeta -from os.path import expanduser from pathlib import Path from typing import Iterable from typing import List @@ -10,10 +9,12 @@ from typing import Tuple from typing import Union +from superannotate_schemas.validators import AnnotationValidators + import lib.core as constances from lib.core import usecases -from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ +from lib.core.conditions import Condition from lib.core.entities import AnnotationClassEntity from lib.core.entities import AttachmentEntity from lib.core.entities import FolderEntity @@ -24,10 +25,10 @@ from lib.core.entities.integrations import IntegrationEntity from lib.core.exceptions import AppException from lib.core.reporter import Reporter +from lib.core.reporter import Session from lib.core.response import Response from lib.infrastructure.helpers import timed_lru_cache from lib.infrastructure.repositories import AnnotationClassRepository -from lib.infrastructure.repositories import ConfigRepository from lib.infrastructure.repositories import FolderRepository from lib.infrastructure.repositories import ImageRepository from lib.infrastructure.repositories import IntegrationRepository @@ -40,7 +41,6 @@ from lib.infrastructure.repositories import WorkflowRepository from lib.infrastructure.services import SuperannotateBackendService from superannotate.logger import get_default_logger -from superannotate_schemas.validators import AnnotationValidators def build_condition(**kwargs) -> Condition: @@ -52,14 +52,21 @@ def build_condition(**kwargs) -> Condition: class BaseController(metaclass=ABCMeta): - def __init__(self, config_path: str = None, token: str = None): - self._team_data = None - self._token = None - self._backend_url = None - self._ssl_verify = False - self._config_path = None - self._backend_client = None + SESSIONS = {} + + def __init__(self, token: str, host: str, ssl_verify: bool, version: str): + self._version = version self._logger = get_default_logger() + self._testing = os.getenv("SA_TESTING", "False").lower() in ("true", "1", "t") + self._token = token + self._backend_client = SuperannotateBackendService( + api_url=host, + auth_token=token, + logger=self._logger, + verify_ssl=ssl_verify, + testing=self._testing, + ) + self._team_data = None self._s3_upload_auth_data = None self._projects = None self._folders = None @@ -71,75 +78,24 @@ def __init__(self, config_path: str = None, token: str = None): self._user_id = None self._team_name = None self._reporter = None - self._testing = os.getenv("SA_TESTING", "False").lower() in ("true", "1", "t") - self._ssl_verify = not self._testing - self._backend_url = os.environ.get("SA_URL", constances.BACKEND_URL) - - if token: - self._token = self._validate_token(token) - elif config_path: - config_path = expanduser(config_path) - self.retrieve_configs(Path(config_path), raise_exception=True) - else: - env_token = os.environ.get("SA_TOKEN") - if env_token: - self._token = self._validate_token(os.environ.get("SA_TOKEN")) - else: - config_path = expanduser(constances.CONFIG_FILE_LOCATION) - self.retrieve_configs(Path(config_path), raise_exception=False) - self.initialize_backend_client() - - def retrieve_configs(self, path: Path, raise_exception=True): - - token, backend_url, ssl_verify = None, None, None - if not Path(path).is_file() or not os.access(path, os.R_OK): - if raise_exception: - raise AppException( - f"SuperAnnotate config file {str(path)} not found." - f" Please provide correct config file location to sa.init() or use " - f"CLI's superannotate init to generate default location config file." - ) + + def get_session(self, pk: str): try: - config_repo = ConfigRepository(str(path)) - token, backend_url, ssl_verify = ( - self._validate_token(config_repo.get_one("token").value), - config_repo.get_one("main_endpoint").value, - config_repo.get_one("ssl_verify").value, - ) - except Exception: - if raise_exception: - raise AppException( - f"Incorrect config file: token is not present in the config file {path}" - ) - self._token = token - self._backend_url = backend_url or self._backend_url - self._ssl_verify = ssl_verify if ssl_verify is not None else True + return self.SESSIONS[pk] + except KeyError: + self.SESSIONS[pk] = Session(pk) + return self.SESSIONS[pk] @staticmethod - def _validate_token(token: str): + def validate_token(token: str): try: int(token.split("=")[-1]) except ValueError: raise AppException("Invalid token.") return token - def initialize_backend_client(self): - if not self._token: - raise AppException("Team token not provided") - self._backend_client = SuperannotateBackendService( - api_url=self._backend_url, - auth_token=self._token, - logger=self._logger, - verify_ssl=self._ssl_verify, - testing=self._testing, - ) - self._backend_client.get_session.cache_clear() - return self._backend_client - @property def backend_client(self): - if not self._backend_client: - self.initialize_backend_client() return self._backend_client @property @@ -154,12 +110,6 @@ def team_name(self): _, self._team_name = self.get_team() return self._team_name - def set_token(self, token: str, backend_url: str = constances.BACKEND_URL): - self._token = self._validate_token(token) - if backend_url: - self._backend_url = backend_url - self.initialize_backend_client() - @property def projects(self): if not self._projects: @@ -211,14 +161,31 @@ def get_integrations_repo(self, team_id: int): @property def team_id(self) -> int: if not self._token: - raise AppException( - f"Invalid credentials provided in the {self._config_path}." - ) + raise AppException("Invalid credentials provided.") return int(self._token.split("=")[-1]) - @property - def default_reporter(self): - return Reporter() + @staticmethod + def get_default_reporter( + log_info: bool = True, + log_warning: bool = True, + disable_progress_bar: bool = False, + log_debug: bool = True + ) -> Reporter: + import inspect + session = None + loop_limit = 16 + current_frame = inspect.currentframe() + while loop_limit: + loop_limit -= 1 + try: + session = current_frame.f_locals['session'] + if session: + break + except KeyError: + pass + finally: + current_frame = current_frame.f_back + return Reporter(log_info, log_warning, disable_progress_bar, log_debug, session) @timed_lru_cache(seconds=3600) def get_auth_data(self, project_id: int, team_id: int, folder_id: int): @@ -250,19 +217,6 @@ def annotation_validators(self) -> AnnotationValidators: class Controller(BaseController): DEFAULT = None - def __init__(self, config_path: str = None, token: str = None): - super().__init__(config_path, token) - self._team = None - - @classmethod - def get_default(cls): - if not cls.DEFAULT: - try: - cls.DEFAULT = Controller() - except Exception: - pass - return cls.DEFAULT - @classmethod def set_default(cls, obj): cls.DEFAULT = obj @@ -299,11 +253,11 @@ def get_folder_name(name: str = None): return "root" def search_project( - self, - name: str = None, - include_complete_image_count=False, - statuses: Union[List[str], Tuple[str]] = (), - **kwargs, + self, + name: str = None, + include_complete_image_count=False, + statuses: Union[List[str], Tuple[str]] = (), + **kwargs, ) -> Response: condition = Condition.get_empty_condition() if name: @@ -324,14 +278,14 @@ def search_project( return use_case.execute() def create_project( - self, - name: str, - description: str, - project_type: str, - settings: Iterable[SettingEntity] = None, - classes: Iterable = tuple(), - workflows: Iterable = tuple(), - **extra_kwargs, + self, + name: str, + description: str, + project_type: str, + settings: Iterable[SettingEntity] = None, + classes: Iterable = tuple(), + workflows: Iterable = tuple(), + **extra_kwargs, ) -> Response: try: @@ -374,14 +328,14 @@ def update_project(self, name: str, project_data: dict) -> Response: return use_case.execute() def upload_image_to_project( - self, - project_name: str, - folder_name: str, - image_name: str, - image: Union[str, io.BytesIO] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + image_name: str, + image: Union[str, io.BytesIO] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -409,13 +363,13 @@ def upload_image_to_project( ).execute() def upload_images_to_project( - self, - project_name: str, - folder_name: str, - paths: List[str], - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + paths: List[str], + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -435,16 +389,16 @@ def upload_images_to_project( ) def upload_images_from_folder_to_project( - self, - project_name: str, - folder_name: str, - folder_path: str, - extensions: Optional[List[str]] = None, - annotation_status: str = None, - exclude_file_patterns: Optional[List[str]] = None, - recursive_sub_folders: Optional[bool] = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + folder_path: str, + extensions: Optional[List[str]] = None, + annotation_status: str = None, + exclude_file_patterns: Optional[List[str]] = None, + recursive_sub_folders: Optional[bool] = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -467,23 +421,25 @@ def upload_images_from_folder_to_project( ) def clone_project( - self, - name: str, - from_name: str, - project_description: str, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, + self, + name: str, + from_name: str, + project_description: str, + copy_annotation_classes=True, + copy_settings=True, + copy_workflow=True, + copy_contributors=False, ): project = self._get_project(from_name) project_to_create = copy.copy(project) + reporter = self.get_default_reporter() + reporter.track("external", project.upload_state == constances.UploadState.EXTERNAL.value) project_to_create.name = name if project_description is not None: project_to_create.description = project_description use_case = usecases.CloneProjectUseCase( - reporter=self.default_reporter, + reporter=reporter, project=project, project_to_create=project_to_create, projects=self.projects, @@ -499,12 +455,12 @@ def clone_project( return use_case.execute() def interactive_attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, + self, + project_name: str, + files: List[ImageEntity], + folder_name: str = None, + annotation_status: str = None, + upload_state_code: int = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -539,7 +495,7 @@ def get_folder(self, project_name: str, folder_name: str): return use_case.execute() def search_folders( - self, project_name: str, folder_name: str = None, include_users=False, **kwargs + self, project_name: str, folder_name: str = None, include_users=False, **kwargs ): condition = build_condition(**kwargs) project = self._get_project(project_name) @@ -566,12 +522,12 @@ def delete_folders(self, project_name: str, folder_names: List[str]): return use_case.execute() def prepare_export( - self, - project_name: str, - folder_names: List[str], - include_fuse: bool, - only_pinned: bool, - annotation_statuses: List[str] = None, + self, + project_name: str, + folder_names: List[str], + include_fuse: bool, + only_pinned: bool, + annotation_statuses: List[str] = None, ): project = self._get_project(project_name) @@ -596,11 +552,11 @@ def search_team_contributors(self, **kwargs): return use_case.execute() def search_images( - self, - project_name: str, - folder_path: str = None, - annotation_status: str = None, - image_name_prefix: str = None, + self, + project_name: str, + folder_path: str = None, + annotation_status: str = None, + image_name_prefix: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -615,7 +571,7 @@ def search_images( return use_case.execute() def _get_image( - self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, + self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, ) -> ImageEntity: response = usecases.GetImageUseCase( service=self._backend_client, @@ -629,7 +585,7 @@ def _get_image( return response.data def get_image( - self, project_name: str, image_name: str, folder_path: str = None + self, project_name: str, image_name: str, folder_path: str = None ) -> ImageEntity: project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -640,18 +596,18 @@ def update_folder(self, project_name: str, folder_name: str, folder_data: dict): folder = self._get_folder(project, folder_name) for field, value in folder_data.items(): setattr(folder, field, value) - use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder,) + use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder, ) return use_case.execute() def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, + copy_annotation_status: bool = False, + move: bool = False, ): from_project = self._get_project(from_project_name) to_project = self._get_project(to_project_name) @@ -674,12 +630,12 @@ def copy_image( return use_case.execute() def copy_image_annotation_classes( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, ): from_project = self._get_project(from_project_name) from_folder = self._get_folder(from_project, from_folder_name) @@ -714,7 +670,7 @@ def copy_image_annotation_classes( return use_case.execute() def update_image( - self, project_name: str, image_name: str, folder_name: str = None, **kwargs + self, project_name: str, image_name: str, folder_name: str = None, **kwargs ): image = self.get_image( project_name=project_name, image_name=image_name, folder_path=folder_name @@ -725,13 +681,13 @@ def update_image( return use_case.execute() def bulk_copy_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], - include_annotations: bool, - include_pin: bool, + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], + include_annotations: bool, + include_pin: bool, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -748,11 +704,11 @@ def bulk_copy_images( return use_case.execute() def bulk_move_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -767,13 +723,13 @@ def bulk_move_images( return use_case.execute() def get_project_metadata( - self, - project_name: str, - include_annotation_classes: bool = False, - include_settings: bool = False, - include_workflow: bool = False, - include_contributors: bool = False, - include_complete_image_count: bool = False, + self, + project_name: str, + include_annotation_classes: bool = False, + include_settings: bool = False, + include_workflow: bool = False, + include_contributors: bool = False, + include_complete_image_count: bool = False, ): project = self._get_project(project_name) @@ -848,11 +804,11 @@ def set_project_settings(self, project_name: str, new_settings: List[dict]): return use_case.execute() def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, + self, + project_name: str, + folder_name: str, + image_names: list, + annotation_status: str, ): project_entity = self._get_project(project_name) folder_entity = self._get_folder(project_entity, folder_name) @@ -870,7 +826,7 @@ def set_images_annotation_statuses( return use_case.execute() def delete_images( - self, project_name: str, folder_name: str, image_names: List[str] = None, + self, project_name: str, folder_name: str, image_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -885,7 +841,7 @@ def delete_images( return use_case.execute() def assign_images( - self, project_name: str, folder_name: str, image_names: list, user: str + self, project_name: str, folder_name: str, image_names: list, user: str ): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) @@ -948,7 +904,7 @@ def un_share_project(self, project_name: str, user_id: str): return use_case.execute() def download_image_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -976,7 +932,7 @@ def get_exports(self, project_name: str, return_metadata: bool): return use_case.execute() def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool + self, project_name: str, folder_name: str, with_all_subfolders: bool ): project = self._get_project(project_name) @@ -992,12 +948,12 @@ def get_project_image_count( return use_case.execute() def create_annotation_class( - self, - project_name: str, - name: str, - color: str, - attribute_groups: List[dict], - class_type: str, + self, + project_name: str, + name: str, + color: str, + attribute_groups: List[dict], + class_type: str, ): project = self._get_project(project_name) annotation_classes = AnnotationClassRepository( @@ -1059,15 +1015,15 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) return use_case.execute() def download_image( - self, - project_name: str, - image_name: str, - download_path: str, - folder_name: str = None, - image_variant: str = None, - include_annotations: bool = None, - include_fuse: bool = None, - include_overlay: bool = None, + self, + project_name: str, + image_name: str, + download_path: str, + folder_name: str = None, + image_variant: str = None, + include_annotations: bool = None, + include_fuse: bool = None, + include_overlay: bool = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1107,13 +1063,13 @@ def set_project_workflow(self, project_name: str, steps: list): return use_case.execute() def upload_annotations_from_folder( - self, - project_name: str, - folder_name: str, - annotation_paths: List[str], - client_s3_bucket=None, - is_pre_annotations: bool = False, - folder_path: str = None, + self, + project_name: str, + folder_name: str, + annotation_paths: List[str], + client_s3_bucket=None, + is_pre_annotations: bool = False, + folder_path: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1133,19 +1089,19 @@ def upload_annotations_from_folder( "data", [] ), validators=self.annotation_validators, - reporter=Reporter(log_info=False, log_warning=False), + reporter=self.get_default_reporter(log_info=False, log_warning=False), folder_path=folder_path, ) return use_case.execute() def upload_image_annotations( - self, - project_name: str, - folder_name: str, - image_name: str, - annotations: dict, - mask: io.BytesIO = None, - verbose: bool = True, + self, + project_name: str, + folder_name: str, + image_name: str, + annotations: dict, + mask: io.BytesIO = None, + verbose: bool = True, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1169,7 +1125,7 @@ def upload_image_annotations( backend_service_provider=self._backend_client, mask=mask, verbose=verbose, - reporter=self.default_reporter, + reporter=self.get_default_reporter(), validators=self.annotation_validators, ) return use_case.execute() @@ -1187,12 +1143,12 @@ def delete_model(self, model_id: int): return use_case.execute() def download_export( - self, - project_name: str, - export_name: str, - folder_path: str, - extract_zip_contents: bool, - to_s3_bucket: bool, + self, + project_name: str, + export_name: str, + folder_path: str, + extract_zip_contents: bool, + to_s3_bucket: bool, ): project = self._get_project(project_name) return usecases.DownloadExportUseCase( @@ -1223,17 +1179,16 @@ def download_ml_model(self, model_data: dict, download_path: str): return use_case.execute() def benchmark( - self, - project_name: str, - ground_truth_folder_name: str, - folder_names: List[str], - export_root: str, - image_list: List[str], - annot_type: str, - show_plots: bool, + self, + project_name: str, + ground_truth_folder_name: str, + folder_names: List[str], + export_root: str, + image_list: List[str], + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) - export_response = self.prepare_export( project.name, folder_names=folder_names, @@ -1266,13 +1221,13 @@ def benchmark( return use_case.execute() def consensus( - self, - project_name: str, - folder_names: list, - export_path: str, - image_list: list, - annot_type: str, - show_plots: bool, + self, + project_name: str, + folder_names: list, + export_path: str, + image_list: list, + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1306,7 +1261,7 @@ def consensus( return use_case.execute() def run_prediction( - self, project_name: str, images_list: list, model_name: str, folder_name: str + self, project_name: str, images_list: list, model_name: str, folder_name: str ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1324,7 +1279,7 @@ def run_prediction( return use_case.execute() def list_images( - self, project_name: str, annotation_status: str = None, name_prefix: str = None, + self, project_name: str, annotation_status: str = None, name_prefix: str = None, ): project = self._get_project(project_name) @@ -1337,12 +1292,12 @@ def list_images( return use_case.execute() def search_models( - self, - name: str, - model_type: str = None, - project_id: int = None, - task: str = None, - include_global: bool = True, + self, + name: str, + model_type: str = None, + project_id: int = None, + task: str = None, + include_global: bool = True, ): ml_models_repo = MLModelRepository( service=self._backend_client, team_id=self.team_id @@ -1365,10 +1320,10 @@ def search_models( return use_case.execute() def delete_annotations( - self, - project_name: str, - folder_name: str, - image_names: Optional[List[str]] = None, + self, + project_name: str, + folder_name: str, + image_names: Optional[List[str]] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1382,7 +1337,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = False ): use_case = usecases.ValidateAnnotationUseCase( project_type, @@ -1398,7 +1353,7 @@ def add_contributors_to_project(self, project_name: str, emails: list, role: str project_name=project_name, include_contributors=True ) use_case = usecases.AddContributorsToProject( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), team=team.data, project=project.data["project"], emails=emails, @@ -1410,7 +1365,7 @@ def add_contributors_to_project(self, project_name: str, emails: list, role: str def invite_contributors_to_team(self, emails: list, set_admin: bool): team = self.get_team() use_case = usecases.InviteContributorsToTeam( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), team=team.data, emails=emails, set_admin=set_admin, @@ -1419,23 +1374,23 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): return use_case.execute() def upload_videos( - self, - project_name: str, - folder_name: str, - paths: List[str], - start_time: float, - extensions: List[str] = None, - exclude_file_patterns: List[str] = None, - end_time: Optional[float] = None, - target_fps: Optional[int] = None, - annotation_status: Optional[str] = None, - image_quality_in_editor: Optional[str] = None, + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.UploadVideosAsImages( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), service=self.backend_client, project=project, folder=folder, @@ -1455,12 +1410,12 @@ def upload_videos( return use_case.execute() def get_annotations( - self, project_name: str, folder_name: str, item_names: List[str], logging=True + self, project_name: str, folder_name: str, item_names: List[str], logging=True ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.GetAnnotations( - reporter=Reporter(log_info=logging, log_debug=logging), + reporter=self.get_default_reporter(log_info=logging, log_debug=logging), project=project, folder=folder, images=self.images, @@ -1470,13 +1425,13 @@ def get_annotations( return use_case.execute() def get_annotations_per_frame( - self, project_name: str, folder_name: str, video_name: str, fps: int + self, project_name: str, folder_name: str, video_name: str, fps: int ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.GetVideoAnnotationsPerFrame( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, images=self.images, @@ -1487,12 +1442,12 @@ def get_annotations_per_frame( return use_case.execute() def upload_priority_scores( - self, project_name, folder_name, scores, project_folder_name + self, project_name, folder_name, scores, project_folder_name ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.UploadPriorityScoresUseCase( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, scores=scores, @@ -1504,24 +1459,24 @@ def upload_priority_scores( def get_integrations(self): team = self.team_data.data use_cae = usecases.GetIntegrations( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), team=self.team_data.data, integrations=self.get_integrations_repo(team_id=team.uuid), ) return use_cae.execute() def attach_integrations( - self, - project_name: str, - folder_name: str, - integration: IntegrationEntity, - folder_path: str, + self, + project_name: str, + folder_name: str, + integration: IntegrationEntity, + folder_path: str, ): team = self.team_data.data project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.AttachIntegrations( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), team=self.team_data.data, backend_service=self.backend_client, project=project, @@ -1537,7 +1492,7 @@ def query_entities(self, project_name: str, folder_name: str, query: str = None) folder = self._get_folder(project, folder_name) use_case = usecases.QueryEntities( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, query=query, @@ -1549,7 +1504,7 @@ def get_item(self, project_name: str, folder_name: str, item_name: str): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.GetItem( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, item_name=item_name, @@ -1558,15 +1513,15 @@ def get_item(self, project_name: str, folder_name: str, item_name: str): return use_case.execute() def list_items( - self, - project_name: str, - folder_name: str, - name_contains: str = None, - annotation_status: str = None, - annotator_email: str = None, - qa_email: str = None, - recursive: bool = False, - **kwargs, + self, + project_name: str, + folder_name: str, + name_contains: str = None, + annotation_status: str = None, + annotator_email: str = None, + qa_email: str = None, + recursive: bool = False, + **kwargs, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1586,7 +1541,7 @@ def list_items( search_condition &= build_condition(**kwargs) use_case = usecases.ListItems( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, recursive=recursive, @@ -1598,17 +1553,17 @@ def list_items( return use_case.execute() def attach_items( - self, - project_name: str, - folder_name: str, - attachments: List[AttachmentEntity], - annotation_status: str, + self, + project_name: str, + folder_name: str, + attachments: List[AttachmentEntity], + annotation_status: str, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.AttachItems( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, attachments=attachments, @@ -1618,19 +1573,19 @@ def attach_items( return use_case.execute() def copy_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, - include_annotations: bool = False, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, + include_annotations: bool = False, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) to_folder = self._get_folder(project, to_folder) use_case = usecases.CopyItems( - self.default_reporter, + self.get_default_reporter(), project=project, from_folder=from_folder, to_folder=to_folder, @@ -1642,18 +1597,18 @@ def copy_items( return use_case.execute() def move_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) to_folder = self._get_folder(project, to_folder) use_case = usecases.MoveItems( - self.default_reporter, + self.get_default_reporter(), project=project, from_folder=from_folder, to_folder=to_folder, @@ -1664,17 +1619,17 @@ def move_items( return use_case.execute() def set_annotation_statuses( - self, - project_name: str, - folder_name: str, - annotation_status: str, - item_names: List[str] = None, + self, + project_name: str, + folder_name: str, + annotation_status: str, + item_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.SetAnnotationStatues( - self.default_reporter, + self.get_default_reporter(), project=project, folder=folder, annotation_status=annotation_status, diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 300a46621..a44846ae0 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1,2 +1 @@ __version__ = "4.3.3b1" - diff --git a/tests/convertors/test_consensus.py b/tests/convertors/test_consensus.py index 4418bcbf4..52244e062 100644 --- a/tests/convertors/test_consensus.py +++ b/tests/convertors/test_consensus.py @@ -3,7 +3,8 @@ import time from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/convertors/test_conversion.py b/tests/convertors/test_conversion.py index d9766885e..390420945 100644 --- a/tests/convertors/test_conversion.py +++ b/tests/convertors/test_conversion.py @@ -6,7 +6,8 @@ from unittest import TestCase import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() class TestCocoSplit(TestCase): diff --git a/tests/convertors/test_json_version_conversion.py b/tests/convertors/test_json_version_conversion.py index aa8982156..0b3927446 100644 --- a/tests/convertors/test_json_version_conversion.py +++ b/tests/convertors/test_json_version_conversion.py @@ -5,7 +5,8 @@ from pathlib import Path from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() class TestVersionConversion(TestCase): diff --git a/tests/convertors/test_project_converter.py b/tests/convertors/test_project_converter.py index 411a07183..109021b46 100644 --- a/tests/convertors/test_project_converter.py +++ b/tests/convertors/test_project_converter.py @@ -5,7 +5,8 @@ from pathlib import Path from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() class TestCocoSplit(TestCase): diff --git a/tests/integration/aggregations/test_df_processing.py b/tests/integration/aggregations/test_df_processing.py index 4be91039d..0cd64f720 100644 --- a/tests/integration/aggregations/test_df_processing.py +++ b/tests/integration/aggregations/test_df_processing.py @@ -4,7 +4,8 @@ import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/aggregations/test_docuement_annotation_to_df.py b/tests/integration/aggregations/test_docuement_annotation_to_df.py index cab6a5c7d..12a2118f2 100644 --- a/tests/integration/aggregations/test_docuement_annotation_to_df.py +++ b/tests/integration/aggregations/test_docuement_annotation_to_df.py @@ -5,7 +5,8 @@ from unittest import mock from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate.logger import get_default_logger diff --git a/tests/integration/aggregations/test_video_annotation_to_df.py b/tests/integration/aggregations/test_video_annotation_to_df.py index 65b8e16b7..918259040 100644 --- a/tests/integration/aggregations/test_video_annotation_to_df.py +++ b/tests/integration/aggregations/test_video_annotation_to_df.py @@ -5,7 +5,8 @@ from unittest import mock from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate.logger import get_default_logger diff --git a/tests/integration/annotation_classes/test_create_annotation_class.py b/tests/integration/annotation_classes/test_create_annotation_class.py index d42187ebd..770b1ac46 100644 --- a/tests/integration/annotation_classes/test_create_annotation_class.py +++ b/tests/integration/annotation_classes/test_create_annotation_class.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_annotation_adding.py b/tests/integration/annotations/test_annotation_adding.py index b154c76db..8c4ed4964 100644 --- a/tests/integration/annotations/test_annotation_adding.py +++ b/tests/integration/annotations/test_annotation_adding.py @@ -3,9 +3,11 @@ import tempfile from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient from tests.integration.base import BaseTestCase +sa = SAClient() + class TestAnnotationAdding(BaseTestCase): PROJECT_NAME = "test_annotations_adding" diff --git a/tests/integration/annotations/test_annotation_class_new.py b/tests/integration/annotations/test_annotation_class_new.py index 8831874a8..c5bba28c6 100644 --- a/tests/integration/annotations/test_annotation_class_new.py +++ b/tests/integration/annotations/test_annotation_class_new.py @@ -2,7 +2,8 @@ from pathlib import Path from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_annotation_classes.py b/tests/integration/annotations/test_annotation_classes.py index c0584fa49..06a2ad789 100644 --- a/tests/integration/annotations/test_annotation_classes.py +++ b/tests/integration/annotations/test_annotation_classes.py @@ -1,7 +1,8 @@ from urllib.parse import urlparse import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_annotation_delete.py b/tests/integration/annotations/test_annotation_delete.py index 9c852fb24..db1f2fb38 100644 --- a/tests/integration/annotations/test_annotation_delete.py +++ b/tests/integration/annotations/test_annotation_delete.py @@ -3,7 +3,8 @@ import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_annotation_upload_pixel.py b/tests/integration/annotations/test_annotation_upload_pixel.py index db974e15e..08d349e46 100644 --- a/tests/integration/annotations/test_annotation_upload_pixel.py +++ b/tests/integration/annotations/test_annotation_upload_pixel.py @@ -3,7 +3,8 @@ from pathlib import Path from unittest.mock import patch -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase import pytest diff --git a/tests/integration/annotations/test_annotation_upload_vector.py b/tests/integration/annotations/test_annotation_upload_vector.py index e7d0cef74..d0f00bd52 100644 --- a/tests/integration/annotations/test_annotation_upload_vector.py +++ b/tests/integration/annotations/test_annotation_upload_vector.py @@ -7,7 +7,8 @@ from unittest.mock import patch from unittest.mock import MagicMock -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_annotations_pre_processing.py b/tests/integration/annotations/test_annotations_pre_processing.py index ed82a9379..574fce0ac 100644 --- a/tests/integration/annotations/test_annotations_pre_processing.py +++ b/tests/integration/annotations/test_annotations_pre_processing.py @@ -8,7 +8,8 @@ from unittest.mock import patch from unittest.mock import MagicMock -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from superannotate_schemas.schemas.base import CreationTypeEnum from tests.integration.base import BaseTestCase @@ -45,7 +46,7 @@ def test_annotation_last_action_and_creation_type(self, reporter): self.assertEqual(instance["creationType"], CreationTypeEnum.PRE_ANNOTATION.value) self.assertEqual( type(annotation["metadata"]["lastAction"]["email"]), - type(sa.get_default_controller().team_data.data.creator_id) + type(sa.controller.team_data.data.creator_id) ) self.assertEqual( type(annotation["metadata"]["lastAction"]["timestamp"]), diff --git a/tests/integration/annotations/test_annotations_upload_status_change.py b/tests/integration/annotations/test_annotations_upload_status_change.py index f64c3e095..d2facc855 100644 --- a/tests/integration/annotations/test_annotations_upload_status_change.py +++ b/tests/integration/annotations/test_annotations_upload_status_change.py @@ -5,7 +5,8 @@ from unittest.mock import patch from unittest.mock import MagicMock -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() import src.superannotate.lib.core as constances from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index 1d4cddc1d..7e099d199 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -8,7 +8,8 @@ from pydantic import parse_obj_as from superannotate_schemas.schemas.internal import VectorAnnotation -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase @@ -26,7 +27,6 @@ def folder_path(self): @pytest.mark.flaky(reruns=3) def test_get_annotations(self): - sa.init() sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) @@ -46,7 +46,6 @@ def test_get_annotations(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_order(self): - sa.init() sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) @@ -65,7 +64,6 @@ def test_get_annotations_order(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_from_folder(self): - sa.init() sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) sa.upload_images_from_folder_to_project( @@ -87,7 +85,6 @@ def test_get_annotations_from_folder(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_all(self): - sa.init() sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) @@ -129,7 +126,6 @@ def annotations_path(self): return os.path.join(self.folder_path, self.ANNOTATIONS_PATH) def test_video_annotation_upload_root(self): - sa.init() sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) _, _, _ = sa.attach_video_urls_to_project( @@ -141,7 +137,6 @@ def test_video_annotation_upload_root(self): self.assertEqual(len(annotations), 2) def test_video_annotation_upload_folder(self): - sa.init() sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) sa.create_folder(self.PROJECT_NAME, "folder") path = f"{self.PROJECT_NAME}/folder" diff --git a/tests/integration/annotations/test_get_annotations_per_frame.py b/tests/integration/annotations/test_get_annotations_per_frame.py index 565d38464..f0a789a87 100644 --- a/tests/integration/annotations/test_get_annotations_per_frame.py +++ b/tests/integration/annotations/test_get_annotations_per_frame.py @@ -3,9 +3,11 @@ from os.path import dirname from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient from tests.integration.base import BaseTestCase +sa = SAClient() + class TestGetAnnotations(BaseTestCase): PROJECT_NAME = "test attach video urls" @@ -35,7 +37,6 @@ def annotations_path(self): return os.path.join(self.folder_path, self.ANNOTATIONS_PATH) def test_video_annotation_upload(self): - sa.init() sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) _, _, _ = sa.attach_video_urls_to_project( diff --git a/tests/integration/annotations/test_missing_annotation_upload.py b/tests/integration/annotations/test_missing_annotation_upload.py index e509ddc3c..73728b65c 100644 --- a/tests/integration/annotations/test_missing_annotation_upload.py +++ b/tests/integration/annotations/test_missing_annotation_upload.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_preannotation_upload.py b/tests/integration/annotations/test_preannotation_upload.py index 6a5e00e44..d16c35b53 100644 --- a/tests/integration/annotations/test_preannotation_upload.py +++ b/tests/integration/annotations/test_preannotation_upload.py @@ -2,7 +2,8 @@ import tempfile from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_text_annotation_upload.py b/tests/integration/annotations/test_text_annotation_upload.py index 98368c31c..6091804ec 100644 --- a/tests/integration/annotations/test_text_annotation_upload.py +++ b/tests/integration/annotations/test_text_annotation_upload.py @@ -3,7 +3,8 @@ import json from pathlib import Path import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_uopload_annotations_without_classes.py b/tests/integration/annotations/test_uopload_annotations_without_classes.py index c9cc17db9..aa9c03d3c 100644 --- a/tests/integration/annotations/test_uopload_annotations_without_classes.py +++ b/tests/integration/annotations/test_uopload_annotations_without_classes.py @@ -8,7 +8,8 @@ from unittest.mock import patch from unittest.mock import MagicMock -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py index 175f95d0f..f46a392df 100644 --- a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py +++ b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py @@ -5,7 +5,8 @@ import json import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/annotations/test_video_annotation_upload.py b/tests/integration/annotations/test_video_annotation_upload.py index 8f3f23a6f..e95ac297f 100644 --- a/tests/integration/annotations/test_video_annotation_upload.py +++ b/tests/integration/annotations/test_video_annotation_upload.py @@ -4,7 +4,8 @@ from pathlib import Path import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase from src.superannotate.lib.core.data_handlers import VideoFormatHandler from lib.core.reporter import Reporter diff --git a/tests/integration/base.py b/tests/integration/base.py index c49899957..e12af6cd4 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -1,6 +1,9 @@ from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient + + +sa = SAClient() class BaseTestCase(TestCase): diff --git a/tests/integration/classes/test_create_annotation_class.py b/tests/integration/classes/test_create_annotation_class.py index 779c412ce..7c20f3fa9 100644 --- a/tests/integration/classes/test_create_annotation_class.py +++ b/tests/integration/classes/test_create_annotation_class.py @@ -2,7 +2,8 @@ from os.path import dirname import tempfile -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase from tests import DATA_SET_PATH diff --git a/tests/integration/classes/test_tag_annotation_classes.py b/tests/integration/classes/test_tag_annotation_classes.py index 5c098e9e9..e86556c0e 100644 --- a/tests/integration/classes/test_tag_annotation_classes.py +++ b/tests/integration/classes/test_tag_annotation_classes.py @@ -1,6 +1,7 @@ import tempfile -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/folders/test_folders.py b/tests/integration/folders/test_folders.py index 433a17896..1e235fe84 100644 --- a/tests/integration/folders/test_folders.py +++ b/tests/integration/folders/test_folders.py @@ -4,7 +4,8 @@ import time from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase from tests import DATA_SET_PATH diff --git a/tests/integration/integrations/test_get_integrations.py b/tests/integration/integrations/test_get_integrations.py index 56673a9cc..e0800e2e2 100644 --- a/tests/integration/integrations/test_get_integrations.py +++ b/tests/integration/integrations/test_get_integrations.py @@ -2,7 +2,8 @@ from os.path import dirname import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/items/test_attach_items.py b/tests/integration/items/test_attach_items.py index 39adfe579..0fc6b18e3 100644 --- a/tests/integration/items/test_attach_items.py +++ b/tests/integration/items/test_attach_items.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/items/test_copy_items.py b/tests/integration/items/test_copy_items.py index 18a01faf2..1c2e627c3 100644 --- a/tests/integration/items/test_copy_items.py +++ b/tests/integration/items/test_copy_items.py @@ -2,7 +2,8 @@ from collections import Counter from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/items/test_get_item_metadata.py b/tests/integration/items/test_get_item_metadata.py index 1dff00cd9..190c02628 100644 --- a/tests/integration/items/test_get_item_metadata.py +++ b/tests/integration/items/test_get_item_metadata.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/items/test_move_items.py b/tests/integration/items/test_move_items.py index cf408665f..109704f28 100644 --- a/tests/integration/items/test_move_items.py +++ b/tests/integration/items/test_move_items.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/items/test_saqul_query.py b/tests/integration/items/test_saqul_query.py index c4b0c1ff7..7d8c4a7bd 100644 --- a/tests/integration/items/test_saqul_query.py +++ b/tests/integration/items/test_saqul_query.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/items/test_search_items.py b/tests/integration/items/test_search_items.py index 6ae414237..5854c29c4 100644 --- a/tests/integration/items/test_search_items.py +++ b/tests/integration/items/test_search_items.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() import src.superannotate.lib.core as constances from tests.integration.base import BaseTestCase from tests.integration.items import IMAGE_EXPECTED_KEYS diff --git a/tests/integration/items/test_set_annotation_statuses.py b/tests/integration/items/test_set_annotation_statuses.py index d18d4d7f5..1ab0423ce 100644 --- a/tests/integration/items/test_set_annotation_statuses.py +++ b/tests/integration/items/test_set_annotation_statuses.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from src.superannotate.lib.core.usecases import SetAnnotationStatues from tests.integration.base import BaseTestCase diff --git a/tests/integration/projects/test_add_contributors_to_project.py b/tests/integration/projects/test_add_contributors_to_project.py index e5629f183..492aada02 100644 --- a/tests/integration/projects/test_add_contributors_to_project.py +++ b/tests/integration/projects/test_add_contributors_to_project.py @@ -1,17 +1,17 @@ import random import string -from unittest.mock import patch from unittest.mock import MagicMock from unittest.mock import PropertyMock +from unittest.mock import patch -import src.superannotate as sa -from src.superannotate import controller -from src.superannotate.lib.core.entities import TeamEntity +from src.superannotate import SAClient from src.superannotate.lib.core.entities import ProjectEntity +from src.superannotate.lib.core.entities import TeamEntity from src.superannotate.lib.core.entities import UserEntity - from tests.integration.base import BaseTestCase +sa = SAClient() + class TestProject(BaseTestCase): PROJECT_NAME = "add_contributors_to_project" @@ -41,14 +41,14 @@ def test_add_contributors(self, client, get_project_metadata_mock, get_team_mock project_data = MagicMock() get_team_mock.return_value = team_data team_data.data = TeamEntity( - uuid=controller.team_id, + uuid=sa.controller.team_id, users=team_users, pending_invitations=pending_users ) get_project_metadata_mock.return_value = project_data project_data.data = dict( project=ProjectEntity( - uuid=controller.team_id, + uuid=sa.controller.team_id, users=project_users, unverified_users=unverified_users, ) @@ -69,7 +69,7 @@ def test_invite_contributors(self, client, get_team_mock): team_data = MagicMock() get_team_mock.return_value = team_data team_data.data = TeamEntity( - uuid=controller.team_id, + uuid=sa.controller.team_id, users=team_users, pending_invitations=pending_users ) @@ -79,4 +79,4 @@ def test_invite_contributors(self, client, get_team_mock): self.assertEqual(len(skipped), 5) def test_(self): - sa.search_team_contributors(email="vaghinak@superannotate.com", first_name="Vaghinak") \ No newline at end of file + sa.search_team_contributors(email="vaghinak@superannotate.com", first_name="Vaghinak") diff --git a/tests/integration/projects/test_basic_project.py b/tests/integration/projects/test_basic_project.py index f7264c196..92446fcc4 100644 --- a/tests/integration/projects/test_basic_project.py +++ b/tests/integration/projects/test_basic_project.py @@ -5,7 +5,8 @@ import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests import DATA_SET_PATH from tests.integration.base import BaseTestCase diff --git a/tests/integration/projects/test_clone_project.py b/tests/integration/projects/test_clone_project.py index c4f77841c..bdd830ac0 100644 --- a/tests/integration/projects/test_clone_project.py +++ b/tests/integration/projects/test_clone_project.py @@ -1,7 +1,8 @@ import os from unittest import TestCase import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests import DATA_SET_PATH from src.superannotate import constances diff --git a/tests/integration/projects/test_create_project.py b/tests/integration/projects/test_create_project.py index decd336da..89a70d42b 100644 --- a/tests/integration/projects/test_create_project.py +++ b/tests/integration/projects/test_create_project.py @@ -1,6 +1,7 @@ from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() class BaseTestCase(TestCase): diff --git a/tests/integration/projects/test_project_rename.py b/tests/integration/projects/test_project_rename.py index 5bbe501e5..3977369da 100644 --- a/tests/integration/projects/test_project_rename.py +++ b/tests/integration/projects/test_project_rename.py @@ -1,4 +1,5 @@ -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/settings/test_settings.py b/tests/integration/settings/test_settings.py index 420e444b0..7f3aff239 100644 --- a/tests/integration/settings/test_settings.py +++ b/tests/integration/settings/test_settings.py @@ -1,6 +1,7 @@ from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException diff --git a/tests/integration/test_assign_images.py b/tests/integration/test_assign_images.py index ab0b8e567..590495955 100644 --- a/tests/integration/test_assign_images.py +++ b/tests/integration/test_assign_images.py @@ -2,7 +2,8 @@ import pytest from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_attach_document_urls.py b/tests/integration/test_attach_document_urls.py index 2f819c1b5..60f31eeba 100644 --- a/tests/integration/test_attach_document_urls.py +++ b/tests/integration/test_attach_document_urls.py @@ -2,7 +2,8 @@ from os.path import dirname from os.path import join -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException import src.superannotate.lib.core as constances from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_attach_video_urls.py b/tests/integration/test_attach_video_urls.py index 96a84f4e7..77d54fc76 100644 --- a/tests/integration/test_attach_video_urls.py +++ b/tests/integration/test_attach_video_urls.py @@ -1,7 +1,8 @@ import os from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException import src.superannotate.lib.core as constances from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_basic_images.py b/tests/integration/test_basic_images.py index be4b05eff..552854ba3 100644 --- a/tests/integration/test_basic_images.py +++ b/tests/integration/test_basic_images.py @@ -5,7 +5,8 @@ import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_benchmark.py b/tests/integration/test_benchmark.py index 5d1df2964..3c626acbe 100644 --- a/tests/integration/test_benchmark.py +++ b/tests/integration/test_benchmark.py @@ -3,7 +3,8 @@ from os.path import dirname import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index d821c48a9..07fe2f735 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -7,7 +7,10 @@ from pathlib import Path from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +from src.superannotate import __version__ + +sa = SAClient() try: CLI_VERSION = pkg_resources.get_distribution("superannotate").version @@ -96,8 +99,8 @@ def _create_project(self, project_type="Vector"): shell=True, ) - # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - # reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + # reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_create_folder(self): self._create_project() subprocess.run( @@ -110,8 +113,8 @@ def test_create_folder(self): ) self.assertEqual(self.FOLDER_NAME, folder["name"]) - # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - # reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + # reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_upload_images(self): self._create_project() subprocess.run( @@ -124,8 +127,8 @@ def test_upload_images(self): ) self.assertEqual(1, len(sa.search_items(self.PROJECT_NAME))) - # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - # reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + # reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_upload_export(self): self._create_project() with tempfile.TemporaryDirectory() as temp_dir: @@ -140,8 +143,8 @@ def test_upload_export(self): self.assertEqual(len(list(test_dir.glob("*.jpg"))), 0) self.assertEqual(len(list(test_dir.glob("*.png"))), 0) - # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - # reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + # reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_vector_pre_annotation_folder_upload_download_cli(self): self._create_project() @@ -169,8 +172,8 @@ def test_vector_pre_annotation_folder_upload_download_cli(self): ) # tod add test - @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_vector_annotation_folder_upload_download_cli(self): self._create_project() sa.create_annotation_classes_from_classes_json( @@ -202,8 +205,8 @@ def test_vector_annotation_folder_upload_download_cli(self): count_out = len(list(Path(temp_dir).glob("*.json"))) self.assertEqual(count_in, count_out) - @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_attach_image_urls(self): self._create_project() subprocess.run( @@ -216,8 +219,8 @@ def test_attach_image_urls(self): self.assertEqual(3, len(sa.search_items(self.PROJECT_NAME))) - # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - # reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + # reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_attach_video_urls(self): self._create_project("Video") subprocess.run( @@ -229,8 +232,8 @@ def test_attach_video_urls(self): ) # self.assertEqual(3, len(sa.search_items(self.PROJECT_NAME))) - # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - # reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + # @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + # reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_upload_videos(self): self._create_project() subprocess.run( @@ -243,8 +246,8 @@ def test_upload_videos(self): ) self.assertEqual(5, len(sa.search_items(self.PROJECT_NAME))) - @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != sa.__version__, - reason=f"Updated package version from {CLI_VERSION} to {sa.__version__}") + @pytest.mark.skipif(CLI_VERSION and CLI_VERSION != __version__, + reason=f"Updated package version from {CLI_VERSION} to {__version__}") def test_attach_document_urls(self): self._create_project("Document") subprocess.run( diff --git a/tests/integration/test_create_from_full_info.py b/tests/integration/test_create_from_full_info.py index f76393b04..fd4e24f19 100644 --- a/tests/integration/test_create_from_full_info.py +++ b/tests/integration/test_create_from_full_info.py @@ -2,7 +2,8 @@ from os.path import dirname from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() class TestCloneProject(TestCase): diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py index 96c9ec420..90be0c24e 100644 --- a/tests/integration/test_depricated_functions_document.py +++ b/tests/integration/test_depricated_functions_document.py @@ -3,7 +3,8 @@ import pytest from unittest import TestCase -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from src.superannotate.lib.core import LIMITED_FUNCTIONS from src.superannotate.lib.core import INVALID_PROJECT_TYPE_TO_PROCESS @@ -135,14 +136,6 @@ def test_deprecated_functions(self): sa.move_images(self.PROJECT_NAME, [self.UPLOAD_IMAGE_NAME], self.PROJECT_NAME_2) except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.class_distribution(self.video_export_path, [self.PROJECT_NAME]) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) - try: - sa.convert_project_type(self.video_export_path, "./") - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) try: sa.prepare_export(self.PROJECT_NAME, include_fuse=True, only_pinned=True) except AppException as e: @@ -166,14 +159,6 @@ def test_deprecated_functions(self): sa.assign_images(self.PROJECT_NAME, [self.UPLOAD_IMAGE_NAME], "some user") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.export_annotation( - "input_dir", "fromSuperAnnotate/panoptic_test", "COCO", "panoptic_test", self.PROJECT_TYPE, - "panoptic_segmentation" - ) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: sa.set_project_default_image_quality_in_editor(self.PROJECT_NAME,"original") except AppException as e: diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py index 75bb7f88f..2fe286087 100644 --- a/tests/integration/test_depricated_functions_video.py +++ b/tests/integration/test_depricated_functions_video.py @@ -3,7 +3,8 @@ from unittest import TestCase import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from src.superannotate.lib.core import LIMITED_FUNCTIONS from src.superannotate.lib.core import INVALID_PROJECT_TYPE_TO_PROCESS @@ -130,14 +131,6 @@ def test_deprecated_functions(self): sa.set_project_default_image_quality_in_editor(self.PROJECT_NAME, "original") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) - try: - sa.class_distribution(self.video_export_path, [self.PROJECT_NAME]) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) - try: - sa.convert_project_type(self.video_export_path, "./") - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) try: sa.prepare_export(self.PROJECT_NAME, include_fuse=True, only_pinned=True) except AppException as e: @@ -161,10 +154,3 @@ def test_deprecated_functions(self): sa.assign_images(self.PROJECT_NAME, [self.UPLOAD_IMAGE_NAME], "some@user.com") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.export_annotation( - "input_dir", "fromSuperAnnotate/panoptic_test", "COCO", "panoptic_test", self.PROJECT_TYPE, - "panoptic_segmentation" - ) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) diff --git a/tests/integration/test_duplicate_image_upload.py b/tests/integration/test_duplicate_image_upload.py index 13730f913..9e8439422 100644 --- a/tests/integration/test_duplicate_image_upload.py +++ b/tests/integration/test_duplicate_image_upload.py @@ -1,9 +1,11 @@ import os from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient from tests.integration.base import BaseTestCase +sa = SAClient() + class TestDuplicateImage(BaseTestCase): PROJECT_NAME = "duplicate_image" diff --git a/tests/integration/test_export_import.py b/tests/integration/test_export_import.py index d5370369f..ff1b91fd4 100644 --- a/tests/integration/test_export_import.py +++ b/tests/integration/test_export_import.py @@ -2,7 +2,8 @@ import tempfile from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_export_upload_s3.py b/tests/integration/test_export_upload_s3.py index 14a0414f9..a2e764da3 100644 --- a/tests/integration/test_export_upload_s3.py +++ b/tests/integration/test_export_upload_s3.py @@ -3,7 +3,8 @@ from os.path import dirname import boto3 -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_fuse_gen.py b/tests/integration/test_fuse_gen.py index 45d5d4405..ffde96e1f 100644 --- a/tests/integration/test_fuse_gen.py +++ b/tests/integration/test_fuse_gen.py @@ -5,7 +5,8 @@ from unittest import TestCase import pytest import numpy as np -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from PIL import Image diff --git a/tests/integration/test_get_exports.py b/tests/integration/test_get_exports.py index 47a382af5..592d1d038 100644 --- a/tests/integration/test_get_exports.py +++ b/tests/integration/test_get_exports.py @@ -3,7 +3,8 @@ import tempfile from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_image_quality.py b/tests/integration/test_image_quality.py index 84af247be..89a748090 100644 --- a/tests/integration/test_image_quality.py +++ b/tests/integration/test_image_quality.py @@ -4,7 +4,8 @@ from os.path import dirname import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase from src.superannotate import AppException diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 462321991..6a6650d06 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -4,7 +4,8 @@ import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_limitations.py b/tests/integration/test_limitations.py index 2c69cfbd1..eda27b13c 100644 --- a/tests/integration/test_limitations.py +++ b/tests/integration/test_limitations.py @@ -2,7 +2,8 @@ from unittest.mock import patch from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from src.superannotate.lib.core import UPLOAD_FOLDER_LIMIT_ERROR_MESSAGE from src.superannotate.lib.core import UPLOAD_PROJECT_LIMIT_ERROR_MESSAGE diff --git a/tests/integration/test_ml_funcs.py b/tests/integration/test_ml_funcs.py index f9d7b3536..941f3d1bf 100644 --- a/tests/integration/test_ml_funcs.py +++ b/tests/integration/test_ml_funcs.py @@ -4,7 +4,8 @@ from os.path import dirname import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_recursive_folder.py b/tests/integration/test_recursive_folder.py index d7d29649e..d5c466dc9 100644 --- a/tests/integration/test_recursive_folder.py +++ b/tests/integration/test_recursive_folder.py @@ -5,7 +5,8 @@ from pathlib import Path import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_recursive_folder_pixel.py b/tests/integration/test_recursive_folder_pixel.py index 308ac882a..552416a85 100644 --- a/tests/integration/test_recursive_folder_pixel.py +++ b/tests/integration/test_recursive_folder_pixel.py @@ -1,4 +1,5 @@ -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_single_annotation_download.py b/tests/integration/test_single_annotation_download.py index 97343dd53..fb00b50c5 100644 --- a/tests/integration/test_single_annotation_download.py +++ b/tests/integration/test_single_annotation_download.py @@ -6,7 +6,8 @@ from os.path import dirname import pytest -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_single_image_upload.py b/tests/integration/test_single_image_upload.py index 5959497c0..a709686b5 100644 --- a/tests/integration/test_single_image_upload.py +++ b/tests/integration/test_single_image_upload.py @@ -2,7 +2,8 @@ import os from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_team_metadata.py b/tests/integration/test_team_metadata.py index a3564f789..f471c07c0 100644 --- a/tests/integration/test_team_metadata.py +++ b/tests/integration/test_team_metadata.py @@ -1,6 +1,9 @@ -import src.superannotate as sa +from src.superannotate import SAClient + from tests.integration.base import BaseTestCase +sa = SAClient() + class TestTeam(BaseTestCase): PROJECT_NAME = "test_team" diff --git a/tests/integration/test_upload_images.py b/tests/integration/test_upload_images.py index bba4d14a3..99967505b 100644 --- a/tests/integration/test_upload_images.py +++ b/tests/integration/test_upload_images.py @@ -1,7 +1,8 @@ import os from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index 70cb75329..ead902e0f 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_validate_upload_state.py b/tests/integration/test_validate_upload_state.py index e67e84106..bed6c49cd 100644 --- a/tests/integration/test_validate_upload_state.py +++ b/tests/integration/test_validate_upload_state.py @@ -1,7 +1,8 @@ import os from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate import AppException from src.superannotate.lib.core import ATTACHING_UPLOAD_STATE_ERROR from src.superannotate.lib.core import UPLOADING_UPLOAD_STATE_ERROR diff --git a/tests/integration/test_video.py b/tests/integration/test_video.py index 08452ee3a..2e7944cda 100644 --- a/tests/integration/test_video.py +++ b/tests/integration/test_video.py @@ -1,7 +1,8 @@ import os from os.path import dirname -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from src.superannotate.lib.core.plugin import VideoPlugin from tests.integration.base import BaseTestCase import pytest diff --git a/tests/unit/test_controller_init.py b/tests/unit/test_controller_init.py index f9aece145..756163e82 100644 --- a/tests/unit/test_controller_init.py +++ b/tests/unit/test_controller_init.py @@ -1,111 +1,114 @@ -import os -from os.path import join -import json -import pkg_resources -import tempfile -import pytest -from unittest import TestCase -from unittest.mock import mock_open -from unittest.mock import patch - - -from src.superannotate.lib.app.interface.cli_interface import CLIFacade -from tests.utils.helpers import catch_prints - - -try: - CLI_VERSION = pkg_resources.get_distribution("superannotate").version -except Exception: - CLI_VERSION = None - - -class CLITest(TestCase): - CONFIG_FILE_DATA = '{"main_endpoint": "https://amazonaws.com:3000","token": "c9c55ct=6085","ssl_verify": false}' - - @pytest.mark.skip(reason="Need to adjust") - @patch('builtins.input') - def test_init_update(self, input_mock): - input_mock.side_effect = ["y", "token"] - with patch('builtins.open', mock_open(read_data=self.CONFIG_FILE_DATA)) as config_file: - try: - with catch_prints() as out: - cli = CLIFacade() - cli.init() - except SystemExit: - input_mock.assert_called_with("Input the team SDK token from https://app.superannotate.com/team : ") - config_file().write.assert_called_once_with( - json.dumps( - {"main_endpoint": "https://api.devsuperannotate.com", "ssl_verify": False, "token": "token"}, - indent=4 - ) - ) - self.assertEqual(out.getvalue().strip(), "Configuration file successfully updated.") - - @pytest.mark.skip(reason="Need to adjust") - @patch('builtins.input') - def test_init_create(self, input_mock): - input_mock.side_effect = ["token"] - with patch('builtins.open', mock_open(read_data="{}")) as config_file: - try: - with catch_prints() as out: - cli = CLIFacade() - cli.init() - except SystemExit: - input_mock.assert_called_with("Input the team SDK token from https://app.superannotate.com/team : ") - config_file().write.assert_called_once_with( - json.dumps( - {"token": "token"}, - indent=4 - ) - ) - self.assertEqual(out.getvalue().strip(), "Configuration file successfully created.") - - -class SKDInitTest(TestCase): - TEST_TOKEN = "toke=123" - - VALID_JSON = { - "token": "a"*28 + "=1234" - } - INVALID_JSON = { - "token": "a" * 28 + "=1234asd" - } - FILE_NAME = "config.json" - FILE_NAME_2 = "config.json" - - def test_env_flow(self): - import superannotate as sa - os.environ.update({"SA_TOKEN": self.TEST_TOKEN}) - sa.init() - self.assertEqual(sa.get_default_controller()._token, self.TEST_TOKEN) - - def test_init_via_config_file(self): - with tempfile.TemporaryDirectory() as temp_dir: - token_path = f"{temp_dir}/config.json" - with open(token_path, "w") as temp_config: - json.dump({"token": self.TEST_TOKEN}, temp_config) - temp_config.close() - import src.superannotate as sa - sa.init(token_path) - - @patch("lib.infrastructure.controller.Controller.retrieve_configs") - def test_init_default_configs_open(self, retrieve_configs): - import src.superannotate as sa - try: - sa.init() - except Exception: - self.assertTrue(retrieve_configs.call_args[0], sa.constances.CONFIG_FILE_LOCATION) - - def test_init(self): - with tempfile.TemporaryDirectory() as temp_dir: - path = join(temp_dir, self.FILE_NAME) - with open(path, "w") as config: - json.dump(self.VALID_JSON, config) - import src.superannotate as sa - sa.init(path) - self.assertEqual(sa.get_default_controller().team_id, 1234) - - def test_(self): - import src.superannotate as sa - sa.init("~/.superannotate/prod_config.json") - sa.search_projects() \ No newline at end of file +# import os +# from os.path import join +# import json +# import pkg_resources +# import tempfile +# import pytest +# from unittest import TestCase +# from unittest.mock import mock_open +# from unittest.mock import patch +# +# +# from src.superannotate.lib.app.interface.cli_interface import CLIFacade +# from tests.utils.helpers import catch_prints +# +# +# try: +# CLI_VERSION = pkg_resources.get_distribution("superannotate").version +# except Exception: +# CLI_VERSION = None +# +# +# class CLITest(TestCase): +# CONFIG_FILE_DATA = '{"main_endpoint": "https://amazonaws.com:3000","token": "c9c55ct=6085","ssl_verify": false}' +# +# @pytest.mark.skip(reason="Need to adjust") +# @patch('builtins.input') +# def test_init_update(self, input_mock): +# input_mock.side_effect = ["y", "token"] +# with patch('builtins.open', mock_open(read_data=self.CONFIG_FILE_DATA)) as config_file: +# try: +# with catch_prints() as out: +# cli = CLIFacade() +# cli.init() +# except SystemExit: +# input_mock.assert_called_with("Input the team SDK token from https://app.superannotate.com/team : ") +# config_file().write.assert_called_once_with( +# json.dumps( +# {"main_endpoint": "https://api.devsuperannotate.com", "ssl_verify": False, "token": "token"}, +# indent=4 +# ) +# ) +# self.assertEqual(out.getvalue().strip(), "Configuration file successfully updated.") +# +# @pytest.mark.skip(reason="Need to adjust") +# @patch('builtins.input') +# def test_init_create(self, input_mock): +# input_mock.side_effect = ["token"] +# with patch('builtins.open', mock_open(read_data="{}")) as config_file: +# try: +# with catch_prints() as out: +# cli = CLIFacade() +# cli.init() +# except SystemExit: +# input_mock.assert_called_with("Input the team SDK token from https://app.superannotate.com/team : ") +# config_file().write.assert_called_once_with( +# json.dumps( +# {"token": "token"}, +# indent=4 +# ) +# ) +# self.assertEqual(out.getvalue().strip(), "Configuration file successfully created.") +# +# +# class SKDInitTest(TestCase): +# TEST_TOKEN = "toke=123" +# +# VALID_JSON = { +# "token": "a"*28 + "=1234" +# } +# INVALID_JSON = { +# "token": "a" * 28 + "=1234asd" +# } +# FILE_NAME = "config.json" +# FILE_NAME_2 = "config.json" +# +# def test_env_flow(self): +# import superannotate as sa +# os.environ.update({"SA_TOKEN": self.TEST_TOKEN}) +# self.assertEqual(sa.controller.token, self.TEST_TOKEN) +# +# def test_init_via_config_file(self): +# with tempfile.TemporaryDirectory() as temp_dir: +# token_path = f"{temp_dir}/config.json" +# with open(token_path, "w") as temp_config: +# json.dump({"token": self.TEST_TOKEN}, temp_config) +# temp_config.close() +# from src.superannotate import SAClient +# sa = SAClient() +# sa.init(token_path) +# +# @patch("lib.infrastructure.controller.Controller.retrieve_configs") +# def test_init_default_configs_open(self, retrieve_configs): +# from src.superannotate import SAClient +# sa = SAClient() +# try: +# sa.init() +# except Exception: +# self.assertTrue(retrieve_configs.call_args[0], sa.constances.CONFIG_FILE_LOCATION) +# +# def test_init(self): +# with tempfile.TemporaryDirectory() as temp_dir: +# path = join(temp_dir, self.FILE_NAME) +# with open(path, "w") as config: +# json.dump(self.VALID_JSON, config) +# from src.superannotate import SAClient +# sa = SAClient() +# sa.init(path) +# self.assertEqual(sa.get_default_controller().team_id, 1234) +# +# def test_(self): +# from src.superannotate import SAClient +# sa = SAClient() +# sa.init("~/.superannotate/prod_config.json") +# sa.search_projects() \ No newline at end of file diff --git a/tests/unit/test_validators.py b/tests/unit/test_validators.py index 3cbb613ea..eeed05e77 100644 --- a/tests/unit/test_validators.py +++ b/tests/unit/test_validators.py @@ -7,7 +7,8 @@ from pydantic import ValidationError -import src.superannotate as sa +from src.superannotate import SAClient +sa = SAClient() from superannotate_schemas.validators import AnnotationValidators From 0a29bff23629f2a88556cc9f7237b54a6fa7f295 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 24 May 2022 10:50:53 +0400 Subject: [PATCH 02/59] SAClient initial changes --- pytest.ini | 2 +- src/superannotate/__init__.py | 9 +- .../lib/app/input_converters/conversion.py | 3 +- .../lib/app/interface/base_interface.py | 16 +- .../lib/app/interface/cli_interface.py | 97 +++-- .../lib/app/interface/sdk_interface.py | 376 ++---------------- src/superannotate/lib/app/mixp/decorators.py | 1 - src/superannotate/lib/core/reporter.py | 2 +- src/superannotate/lib/core/usecases/models.py | 2 +- .../lib/infrastructure/controller.py | 17 +- src/superannotate/logger.py | 2 +- .../annotations/test_download_annotations.py | 7 +- .../test_video_annotation_upload.py | 5 +- .../projects/test_clone_project.py | 2 +- tests/integration/test_cli.py | 18 +- .../test_depricated_functions_document.py | 11 - tests/integration/test_get_exports.py | 9 +- tests/integration/test_interface.py | 8 +- 18 files changed, 136 insertions(+), 451 deletions(-) diff --git a/pytest.ini b/pytest.ini index a50732786..de724250b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.7 log_cli=true python_files = test_*.py -addopts = -n auto --dist=loadscope +;addopts = -n auto --dist=loadscope diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index dd3ffdef5..eca931970 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -11,6 +11,9 @@ from superannotate.lib.app.input_converters.conversion import export_annotation from superannotate.lib.app.input_converters.conversion import import_annotation from superannotate.lib.app.interface.sdk_interface import SAClient +from superannotate.lib.core import PACKAGE_VERSION_INFO_MESSAGE +from superannotate.lib.core import PACKAGE_VERSION_MAJOR_UPGRADE +from superannotate.lib.core import PACKAGE_VERSION_UPGRADE from superannotate.logger import get_default_logger from superannotate.version import __version__ @@ -39,7 +42,7 @@ def log_version_info(): local_version = parse(__version__) if local_version.is_prerelease: - logger.info(constances.PACKAGE_VERSION_INFO_MESSAGE.format(__version__)) + logger.info(PACKAGE_VERSION_INFO_MESSAGE.format(__version__)) req = requests.get("https://pypi.python.org/pypi/superannotate/json") if req.ok: releases = req.json().get("releases", []) @@ -50,13 +53,13 @@ def log_version_info(): pip_version = max(pip_version, ver) if pip_version.major > local_version.major: logger.warning( - constances.PACKAGE_VERSION_MAJOR_UPGRADE.format( + PACKAGE_VERSION_MAJOR_UPGRADE.format( local_version, pip_version ) ) elif pip_version > local_version: logger.warning( - constances.PACKAGE_VERSION_UPGRADE.format(local_version, pip_version) + PACKAGE_VERSION_UPGRADE.format(local_version, pip_version) ) diff --git a/src/superannotate/lib/app/input_converters/conversion.py b/src/superannotate/lib/app/input_converters/conversion.py index f9d1a4199..0bb412325 100644 --- a/src/superannotate/lib/app/input_converters/conversion.py +++ b/src/superannotate/lib/app/input_converters/conversion.py @@ -5,6 +5,7 @@ from pathlib import Path from lib.app.exceptions import AppException +from lib.app.interface.base_interface import Tracker from lib.app.mixp.decorators import Trackable from lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE from lib.core import LIMITED_FUNCTIONS @@ -135,7 +136,7 @@ def _passes_converter_sanity(args, direction): ) -@Trackable +@Tracker def export_annotation( input_dir, output_dir, diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 76a09245d..21dd5a84a 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -3,13 +3,13 @@ from abc import ABC from abc import abstractmethod from inspect import signature +from types import FunctionType from typing import Iterable from typing import Sized -from mixpanel import Mixpanel - from lib.app.helpers import extract_project_folder from lib.core.reporter import Session +from mixpanel import Mixpanel from version import __version__ @@ -126,3 +126,15 @@ def __call__(self, *args, **kwargs): return result finally: self.track(args=args, kwargs=kwargs, success=success, session=session) + + +class TrackableMeta(type): + def __new__(mcs, name, bases, attrs): + for attr_name, attr_value in attrs.iteritems(): + if isinstance(attr_value, FunctionType): + attrs[attr_name] = mcs.decorate(attr_value) + return super().__new__(mcs, name, bases, attrs) + + @staticmethod + def decorate(func): + return Tracker(func) diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index e7a15d100..26f90b467 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -8,13 +8,12 @@ from lib import __file__ as lib_path from lib.app.helpers import split_project_path from lib.app.input_converters.conversion import import_annotation -from lib.app.interface.base_interface import BaseInterfaceFacade from lib.app.interface.sdk_interface import SAClient from lib.core.entities import ConfigEntity from lib.infrastructure.repositories import ConfigRepository -class CLIFacade(BaseInterfaceFacade): +class CLIFacade: """ With SuperAnnotate CLI, basic tasks can be accomplished using shell commands: superannotatecli <--arg1 val1> <--arg2 val2> [--optional_arg3 val3] [--optional_arg4] ... @@ -26,7 +25,7 @@ def version(): To show the version of the current SDK installation """ with open( - f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" + f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" ) as f: version = f.read().rstrip()[15:-1] print(version) @@ -41,7 +40,7 @@ def init(): config = repo.get_one(uuid=constances.TOKEN_UUID) if config: if not input( - f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " + f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " ).lower() in ("y", "yes"): return token = input( @@ -59,24 +58,24 @@ def create_project(self, name: str, description: str, type: str): """ To create a new project """ - create_project(name, description, type) + SAClient().create_project(name, description, type) def create_folder(self, project: str, name: str): """ To create a new folder """ - create_folder(project, name) + SAClient().create_folder(project, name) sys.exit(0) def upload_images( - self, - project: str, - folder: str, - extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, - set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, - exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders=False, - image_quality_in_editor=None, + self, + project: str, + folder: str, + extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, + set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, + exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders=False, + image_quality_in_editor=None, ): """ To upload images from folder to project use: @@ -100,15 +99,17 @@ def upload_images( sys.exit(0) def export_project( - self, - project, - folder, - include_fuse=False, - disable_extract_zip_contents=False, - annotation_statuses=None, + self, + project, + folder, + include_fuse=False, + disable_extract_zip_contents=False, + annotation_statuses=None, ): project_name, folder_name = split_project_path(project) folders = None + if not annotation_statuses: + annotation_statuses = [] if folder_name: folders = [folder_name] export_res = SAClient().prepare_export( @@ -116,17 +117,17 @@ def export_project( ) export_name = export_res.data["name"] - use_case = SAClient().download_export( + SAClient().download_export( project_name=project_name, export_name=export_name, folder_path=folder, extract_zip_contents=not disable_extract_zip_contents, - to_s3_bucket=False, + to_s3_bucket= False, ) sys.exit(0) def upload_preannotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload preannotations from folder to project use @@ -147,7 +148,7 @@ def upload_preannotations( sys.exit(0) def upload_annotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload annotations from folder to project use @@ -168,15 +169,11 @@ def upload_annotations( sys.exit(0) def _upload_annotations( - self, project, folder, format, dataset_name, task, pre=True + self, project, folder, format, dataset_name, task, pre=True ): project_folder_name = project project_name, folder_name = split_project_path(project) - project = ( - SAClient() - .get_project_metadata(project_name=project_name) - .data - ) + project = SAClient().controller.get_project_metadata(project_name=project_name).data if not format: format = "SuperAnnotate" if not dataset_name and format == "COCO": @@ -210,16 +207,16 @@ def _upload_annotations( sys.exit(0) def attach_image_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): """ To attach image URLs to project use: """ - SAClient().attach_image_urls_to_project( + SAClient().attach_items( project=project, attachments=attachments, annotation_status=annotation_status, @@ -227,12 +224,12 @@ def attach_image_urls( sys.exit(0) def attach_video_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): - SAClient().attach_video_urls_to_project( + SAClient().attach_items( project=project, attachments=attachments, annotation_status=annotation_status, @@ -241,7 +238,7 @@ def attach_video_urls( @staticmethod def attach_document_urls( - project: str, attachments: str, annotation_status: Optional[Any] = None + project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" ): SAClient().attach_items( project=project, @@ -251,15 +248,15 @@ def attach_document_urls( sys.exit(0) def upload_videos( - self, - project, - folder, - target_fps=None, - recursive=False, - extensions=constances.DEFAULT_VIDEO_EXTENSIONS, - set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, - start_time=0.0, - end_time=None, + self, + project, + folder, + target_fps=None, + recursive=False, + extensions=constances.DEFAULT_VIDEO_EXTENSIONS, + set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, + start_time=0.0, + end_time=None, ): """ To upload videos from folder to project use diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 8ae5a6e0b..ddd8f9bcd 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -5,6 +5,7 @@ import tempfile import warnings from pathlib import Path +from typing import Callable from typing import Iterable from typing import List from typing import Optional @@ -12,12 +13,6 @@ from typing import Union import boto3 -from pydantic import StrictBool -from pydantic import conlist -from pydantic import parse_obj_as -from pydantic.error_wrappers import ValidationError -from tqdm import tqdm - import lib.core as constances from lib.app.annotation_helpers import add_annotation_bbox_to_json from lib.app.annotation_helpers import add_annotation_comment_to_json @@ -25,9 +20,7 @@ from lib.app.helpers import extract_project_folder from lib.app.helpers import get_annotation_paths from lib.app.helpers import get_name_url_duplicated_from_csv -from lib.app.helpers import get_paths_and_duplicated_from_csv from lib.app.interface.base_interface import BaseInterfaceFacade -from lib.app.interface.base_interface import Tracker from lib.app.interface.types import AnnotationStatuses from lib.app.interface.types import AnnotationType from lib.app.interface.types import AnnotatorRole @@ -59,7 +52,12 @@ from lib.core.types import Project from lib.infrastructure.controller import Controller from lib.infrastructure.repositories import ConfigRepository +from pydantic import conlist +from pydantic import parse_obj_as +from pydantic import StrictBool +from pydantic.error_wrappers import ValidationError from superannotate.logger import get_default_logger +from tqdm import tqdm logger = get_default_logger() @@ -119,7 +117,6 @@ def get_team_metadata(self): response = self.controller.get_team() return TeamSerializer(response.data).serialize() - @Tracker @validate_arguments def search_team_contributors( self, @@ -151,7 +148,6 @@ def search_team_contributors( return [contributor["email"] for contributor in contributors] return contributors - @Tracker @validate_arguments def search_projects( self, @@ -207,7 +203,6 @@ def search_projects( else: return [project.name for project in result] - @Tracker @validate_arguments def create_project( self, @@ -248,7 +243,6 @@ def create_project( return ProjectSerializer(response.data).serialize() - @Tracker @validate_arguments def create_project_from_metadata( self, project_metadata: Project): @@ -275,7 +269,6 @@ def create_project_from_metadata( raise AppException(response.errors) return ProjectSerializer(response.data).serialize() - @Tracker @validate_arguments def clone_project( self, @@ -321,7 +314,6 @@ def clone_project( raise AppException(response.errors) return ProjectSerializer(response.data).serialize() - @Tracker @validate_arguments def create_folder( self, project: NotEmptyStr, folder_name: NotEmptyStr): @@ -346,7 +338,6 @@ def create_folder( if res.errors: raise AppException(res.errors) - @Tracker @validate_arguments def delete_project( self, project: Union[NotEmptyStr, dict]): @@ -360,7 +351,6 @@ def delete_project( name = project["name"] self.controller.delete_project(name=name) - @Tracker @validate_arguments def rename_project( self, project: NotEmptyStr, new_name: NotEmptyStr): @@ -380,7 +370,6 @@ def rename_project( logger.info("Successfully renamed project %s to %s.", project, response.data.name) return ProjectSerializer(response.data).serialize() - @Tracker @validate_arguments def get_folder_metadata( self, project: NotEmptyStr, folder_name: NotEmptyStr): @@ -403,7 +392,6 @@ def get_folder_metadata( raise AppException("Folder not found.") return FolderSerializer(result).serialize() - @Tracker @validate_arguments def delete_folders( self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): @@ -422,7 +410,6 @@ def delete_folders( raise AppException(res.errors) logger.info(f"Folders {folder_names} deleted in project {project}") - @Tracker @validate_arguments def search_folders( self, @@ -453,7 +440,6 @@ def search_folders( return [FolderSerializer(folder).serialize() for folder in data] return [folder.name for folder in data] - @Tracker @validate_arguments def copy_image( self, @@ -534,154 +520,6 @@ def copy_image( f" to {destination_project}/{destination_folder}." ) - @Tracker - @validate_arguments - def copy_images( - self, - source_project: Union[NotEmptyStr, dict], - image_names: Optional[List[NotEmptyStr]], - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = True, - copy_pin: Optional[StrictBool] = True, - ): - """Copy images in bulk between folders in a project - - :param source_project: project name or folder path (e.g., "project1/folder1") - :type source_project: str` - :param image_names: image names. If None, all images from source project will be copied - :type image_names: list of str - :param destination_project: project name or folder path (e.g., "project1/folder2") - :type destination_project: str - :param include_annotations: enables annotations copy - :type include_annotations: bool - :param copy_pin: enables image pin status copy - :type copy_pin: bool - :return: list of skipped image names - :rtype: list of strs - """ - warning_msg = ( - "We're deprecating the copy_images function. Please use copy_items instead. Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.copy_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, source_folder_name = extract_project_folder(source_project) - - to_project_name, destination_folder_name = extract_project_folder( - destination_project - ) - if project_name != to_project_name: - raise AppException("Source and destination projects should be the same") - if not image_names: - images = ( - self.controller - .search_images(project_name=project_name, folder_path=source_folder_name) - .data - ) - image_names = [image.name for image in images] - - res = self.controller.bulk_copy_images( - project_name=project_name, - from_folder_name=source_folder_name, - to_folder_name=destination_folder_name, - image_names=image_names, - include_annotations=include_annotations, - include_pin=copy_pin, - ) - if res.errors: - raise AppException(res.errors) - skipped_images = res.data - done_count = len(image_names) - len(skipped_images) - message_postfix = "{from_path} to {to_path}." - message_prefix = "Copied images from " - if done_count > 1 or done_count == 0: - message_prefix = f"Copied {done_count}/{len(image_names)} images from " - elif done_count == 1: - message_prefix = "Copied an image from " - logger.info( - message_prefix - + message_postfix.format(from_path=source_project, to_path=destination_project) - ) - - return skipped_images - - @Tracker - @validate_arguments - def move_images( - self, - source_project: Union[NotEmptyStr, dict], - image_names: Optional[List[NotEmptyStr]], - destination_project: Union[NotEmptyStr, dict], - *args, - **kwargs, - ): - """Move images in bulk between folders in a project - - :param source_project: project name or folder path (e.g., "project1/folder1") - :type source_project: str - :param image_names: image names. If None, all images from source project will be moved - :type image_names: list of str - :param destination_project: project name or folder path (e.g., "project1/folder2") - :type destination_project: str - :return: list of skipped image names - :rtype: list of strs - """ - warning_msg = ( - "We're deprecating the move_images function. Please use move_items instead. Learn more." - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.move_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, source_folder_name = extract_project_folder(source_project) - - project = self.controller.get_project_metadata(project_name).data - if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].type]) - - to_project_name, destination_folder_name = extract_project_folder( - destination_project - ) - - if project_name != to_project_name: - raise AppException( - "Source and destination projects should be the same for move_images" - ) - - if not image_names: - images = self.controller.search_images( - project_name=project_name, folder_path=source_folder_name - ) - images = images.data - image_names = [image.name for image in images] - - response = self.controller.bulk_move_images( - project_name=project_name, - from_folder_name=source_folder_name, - to_folder_name=destination_folder_name, - image_names=image_names, - ) - if response.errors: - raise AppException(response.errors) - moved_images = response.data - moved_count = len(moved_images) - message_postfix = "{from_path} to {to_path}." - message_prefix = "Moved images from " - if moved_count > 1 or moved_count == 0: - message_prefix = f"Moved {moved_count}/{len(image_names)} images from " - elif moved_count == 1: - message_prefix = "Moved an image from" - - logger.info( - message_prefix - + message_postfix.format(from_path=source_project, to_path=destination_project) - ) - - return list(set(image_names) - set(moved_images)) - - @Tracker @validate_arguments def get_project_metadata( self, @@ -738,7 +576,6 @@ def get_project_metadata( ] return metadata - @Tracker @validate_arguments def get_project_settings( self, project: Union[NotEmptyStr, dict]): @@ -759,7 +596,6 @@ def get_project_settings( ] return settings - @Tracker @validate_arguments def get_project_workflow( self, project: Union[str, dict]): @@ -779,7 +615,6 @@ def get_project_workflow( raise AppException(workflow.errors) return workflow.data - @Tracker @validate_arguments def search_annotation_classes( self, @@ -803,7 +638,6 @@ def search_annotation_classes( classes = [BaseSerializer(attribute).serialize() for attribute in classes.data] return classes - @Tracker @validate_arguments def set_project_default_image_quality_in_editor( self, @@ -828,7 +662,6 @@ def set_project_default_image_quality_in_editor( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def pin_image( self, @@ -851,7 +684,6 @@ def pin_image( is_pinned=int(pin), ) - @Tracker @validate_arguments def set_images_annotation_statuses( self, @@ -869,6 +701,13 @@ def set_images_annotation_statuses( should be one of NotStarted InProgress QualityCheck Returned Completed Skipped :type annotation_status: str """ + warning_msg = ( + "We're deprecating the set_images_annotation_statuses function. Please use set_annotation_statuses instead. " + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.set_annotation_statuses" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) response = self.controller.set_images_annotation_statuses( project_name, folder_name, image_names, annotation_status @@ -877,7 +716,6 @@ def set_images_annotation_statuses( raise AppException(response.errors) logger.info("Annotations status of images changed") - @Tracker @validate_arguments def delete_images( self, @@ -905,7 +743,6 @@ def delete_images( f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) - @Tracker @validate_arguments def assign_images( self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str): @@ -954,7 +791,6 @@ def assign_images( else: raise AppException(response.errors) - @Tracker @validate_arguments def unassign_images( self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr]): @@ -975,7 +811,6 @@ def unassign_images( if response.errors: raise AppException(response.errors) - @Tracker @validate_arguments def unassign_folder( self, project_name: NotEmptyStr, folder_name: NotEmptyStr): @@ -994,7 +829,6 @@ def unassign_folder( if response.errors: raise AppException(response.errors) - @Tracker @validate_arguments def assign_folder( self, @@ -1036,41 +870,6 @@ def assign_folder( if response.errors: raise AppException(response.errors) - @Tracker - @validate_arguments - def share_project( - self, - project_name: NotEmptyStr, user: Union[str, dict], user_role: NotEmptyStr - ): - """Share project with user. - - :param project_name: project name - :type project_name: str - :param user: user email or metadata of the user to share project with - :type user: str or dict - :param user_role: user role to apply, one of Admin , Annotator , QA , Customer , Viewer - :type user_role: str - """ - warning_msg = ( - "The share_project function is deprecated and will be removed with the coming release, " - "please use add_contributors_to_project instead." - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - if isinstance(user, dict): - user_id = user["id"] - else: - response = self.controller.search_team_contributors(email=user) - if not response.data: - raise AppException(f"User {user} not found.") - user_id = response.data[0]["id"] - response = self.controller.share_project( - project_name=project_name, user_id=user_id, user_role=user_role - ) - if response.errors: - raise AppException(response.errors) - - @Tracker @validate_arguments def upload_images_from_folder_to_project( self, @@ -1182,7 +981,6 @@ def upload_images_from_folder_to_project( return use_case.data raise AppException(use_case.response.errors) - @Tracker @validate_arguments def get_project_image_count( self, @@ -1210,7 +1008,6 @@ def get_project_image_count( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def download_image_annotations( self, @@ -1242,7 +1039,6 @@ def download_image_annotations( raise AppException(res.errors) return res.data - @Tracker @validate_arguments def get_exports( self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False): @@ -1261,7 +1057,6 @@ def get_exports( ) return response.data - @Tracker @validate_arguments def prepare_export( self, @@ -1315,7 +1110,6 @@ def prepare_export( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def upload_videos_from_folder_to_project( self, @@ -1396,7 +1190,6 @@ def upload_videos_from_folder_to_project( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def upload_video_to_project( self, @@ -1449,7 +1242,6 @@ def upload_video_to_project( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def create_annotation_class( self, @@ -1493,7 +1285,6 @@ def create_annotation_class( raise AppException(response.errors) return BaseSerializer(response.data).serialize() - @Tracker @validate_arguments def delete_annotation_class( self, @@ -1510,7 +1301,6 @@ def delete_annotation_class( project_name=project, annotation_class_name=annotation_class ) - @Tracker @validate_arguments def download_annotation_classes_json( self, project: NotEmptyStr, folder: Union[str, Path]): @@ -1531,7 +1321,6 @@ def download_annotation_classes_json( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def create_annotation_classes_from_classes_json( self, @@ -1576,7 +1365,6 @@ def create_annotation_classes_from_classes_json( raise AppException(response.errors) return [BaseSerializer(i).serialize() for i in response.data] - @Tracker @validate_arguments def download_export( self, @@ -1606,29 +1394,17 @@ def download_export( project_name, folder_name = extract_project_folder(project) export_name = export["name"] if isinstance(export, dict) else export - use_case = self.controller.download_export( + response = self.controller.download_export( project_name=project_name, export_name=export_name, folder_path=folder_path, extract_zip_contents=extract_zip_contents, to_s3_bucket=to_s3_bucket, ) - if use_case.is_valid(): - if to_s3_bucket: - with tqdm( - total=use_case.get_upload_files_count(), desc="Uploading" - ) as progress_bar: - for _ in use_case.execute(): - progress_bar.update() - progress_bar.close() - else: - for _ in use_case.execute(): - continue - logger.info(use_case.response.data) - else: - raise AppException(use_case.response.errors) + if response.errors: + raise AppException(response.errors) + logger.info(response.data) - @Tracker @validate_arguments def set_image_annotation_status( self, @@ -1649,6 +1425,13 @@ def set_image_annotation_status( :return: metadata of the updated image :rtype: dict """ + warning_msg = ( + "We're deprecating the set_image_annotation_status function. Please use set_annotation_statuses instead. " + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.set_annotation_statuses" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) response = self.controller.set_images_annotation_statuses( project_name, folder_name, [image_name], annotation_status @@ -1660,7 +1443,6 @@ def set_image_annotation_status( ) return BaseSerializer(image).serialize() - @Tracker @validate_arguments def set_project_workflow( self, project: Union[NotEmptyStr, dict], new_workflow: List[dict]): @@ -1683,7 +1465,6 @@ def set_project_workflow( if response.errors: raise AppException(response.errors) - @Tracker @validate_arguments def download_image( self, @@ -1732,8 +1513,6 @@ def download_image( logger.info(f"Downloaded image {image_name} to {local_dir_path} ") return response.data - - @Tracker @validate_arguments def upload_annotations_from_folder_to_project( self, @@ -1797,7 +1576,6 @@ def upload_annotations_from_folder_to_project( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def upload_preannotations_from_folder_to_project( self, @@ -1864,7 +1642,6 @@ def upload_preannotations_from_folder_to_project( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def upload_image_annotations( self, @@ -1922,7 +1699,6 @@ def upload_image_annotations( if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: raise AppException(response.errors) - @Tracker @validate_arguments def download_model( self, model: MLModel, output_dir: Union[str, Path]): @@ -1944,7 +1720,6 @@ def download_model( else: return BaseSerializer(res.data).serialize() - @Tracker @validate_arguments def benchmark( self, @@ -2013,7 +1788,6 @@ def benchmark( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def consensus( self, @@ -2068,7 +1842,6 @@ def consensus( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def run_prediction( self, @@ -2108,7 +1881,6 @@ def run_prediction( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def add_annotation_bbox_to_image( self, @@ -2169,7 +1941,6 @@ def add_annotation_bbox_to_image( project_name, folder_name, image_name, annotations ) - @Tracker @validate_arguments def add_annotation_point_to_image( self, @@ -2228,7 +1999,6 @@ def add_annotation_point_to_image( project_name, folder_name, image_name, annotations ) - @Tracker @validate_arguments def add_annotation_comment_to_image( self, @@ -2285,7 +2055,6 @@ def add_annotation_comment_to_image( project_name, folder_name, image_name, annotations ) - @Tracker @validate_arguments def upload_image_to_project( self, @@ -2328,7 +2097,6 @@ def upload_image_to_project( if response.errors: raise AppException(response.errors) - @Trackable @validate_arguments def search_models( self, @@ -2363,7 +2131,6 @@ def search_models( ) return res.data - @Tracker @validate_arguments def upload_images_to_project( self, @@ -2425,7 +2192,6 @@ def upload_images_to_project( return uploaded, failed_images, duplications raise AppException(use_case.response.errors) - @Tracker @validate_arguments def aggregate_annotations_as_df( self, @@ -2475,7 +2241,6 @@ def aggregate_annotations_as_df( folder_names=folder_names, ).aggregate_annotations_as_df() - @Tracker @validate_arguments def delete_annotations( self, @@ -2493,83 +2258,11 @@ def delete_annotations( project_name, folder_name = extract_project_folder(project) response = self.controller.delete_annotations( - project_name=project_name, folder_name=folder_name, image_names=image_names + project_name=project_name, folder_name=folder_name, item_names=image_names ) if response.errors: raise AppException(response.errors) - @Tracker - @validate_arguments - def attach_document_urls_to_project( - self, - project: Union[NotEmptyStr, dict], - attachments: Union[Path, NotEmptyStr], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - ): - """Link documents on external storage to SuperAnnotate. - - :param project: project name or project folder path - :type project: str or dict - :param attachments: path to csv file on attachments metadata - :type attachments: Path-like (str or Path) - :param annotation_status: value to set the annotation statuses of the linked documents: NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :return: list of attached documents, list of not attached documents, list of skipped documents - :rtype: tuple - """ - warning_msg = ( - "We're deprecating the attach_document_urls_to_project function. Please use attach_items instead. Learn more." - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, folder_name = extract_project_folder(project) - project = self.controller.get_project_metadata(project_name).data - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - if project["project"].type != constances.ProjectType.DOCUMENT.value: - raise AppException( - constances.INVALID_PROJECT_TYPE_TO_PROCESS.format( - constances.ProjectType.get_name(project["project"].type) - ) - ) - - images_to_upload, duplicate_images = get_paths_and_duplicated_from_csv(attachments) - - use_case = self.controller.interactive_attach_urls( - project_name=project_name, - folder_name=folder_name, - files=ImageSerializer.deserialize(images_to_upload), # noqa: E203 - annotation_status=annotation_status, - ) - if len(duplicate_images): - logger.warning( - constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images)) - ) - if use_case.is_valid(): - logger.info( - constances.ATTACHING_FILES_MESSAGE.format( - len(images_to_upload), project_folder_name - ) - ) - with tqdm( - total=use_case.attachments_count, desc="Attaching urls" - ) as progress_bar: - for attached in use_case.execute(): - progress_bar.update(attached) - uploaded, duplications = use_case.data - uploaded = [i["name"] for i in uploaded] - duplications.extend(duplicate_images) - failed_images = [ - image["name"] - for image in images_to_upload - if image["name"] not in uploaded + duplications - ] - return uploaded, failed_images, duplications - raise AppException(use_case.response.errors) - - @Tracker @validate_arguments def validate_annotations( self, @@ -2599,7 +2292,6 @@ def validate_annotations( print(response.report) return False - @Tracker @validate_arguments def add_contributors_to_project( self, @@ -2626,7 +2318,6 @@ def add_contributors_to_project( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def invite_contributors_to_team( self, @@ -2650,7 +2341,6 @@ def invite_contributors_to_team( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def get_annotations( self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): @@ -2673,7 +2363,6 @@ def get_annotations( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def get_annotations_per_frame( self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1): @@ -2701,7 +2390,6 @@ def get_annotations_per_frame( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def upload_priority_scores( self, project: NotEmptyStr, scores: List[PriorityScore]): @@ -2725,7 +2413,6 @@ def upload_priority_scores( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def get_integrations( self): @@ -2740,7 +2427,6 @@ def get_integrations( integrations = response.data return BaseSerializer.serialize_iterable(integrations, ("name", "type", "root")) - @Tracker @validate_arguments def attach_items_from_integrated_storage( self, @@ -2770,7 +2456,6 @@ def attach_items_from_integrated_storage( if response.errors: raise AppException(response.errors) - @Tracker @validate_arguments def query( self, project: NotEmptyStr, query: Optional[NotEmptyStr]): @@ -2792,7 +2477,6 @@ def query( raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) - @Tracker @validate_arguments def get_item_metadata( self, @@ -2815,7 +2499,6 @@ def get_item_metadata( raise AppException(response.errors) return BaseSerializer(response.data).serialize() - @Tracker @validate_arguments def search_items( self, @@ -2878,7 +2561,6 @@ def search_items( raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) - @Tracker @validate_arguments def attach_items( self, @@ -2943,7 +2625,6 @@ def attach_items( ] return uploaded, fails, duplicated - @Tracker @validate_arguments def copy_items( self, @@ -2988,7 +2669,6 @@ def copy_items( return response.data - @Tracker @validate_arguments def move_items( self, @@ -3025,7 +2705,6 @@ def move_items( raise AppException(response.errors) return response.data - @Tracker @validate_arguments def set_annotation_statuses( self, @@ -3062,7 +2741,6 @@ def set_annotation_statuses( raise AppException(response.errors) return response.data - @Trackable @validate_arguments def download_annotations( self, @@ -3095,7 +2773,7 @@ def download_annotations( :rtype: str """ project_name, folder_name = extract_project_folder(project) - response = Controller.get_default().download_annotations( + response = self.controller.download_annotations( project_name=project_name, folder_name=folder_name, destination=path, @@ -3105,4 +2783,4 @@ def download_annotations( ) if response.errors: raise AppException(response.errors) - return response.data \ No newline at end of file + return response.data diff --git a/src/superannotate/lib/app/mixp/decorators.py b/src/superannotate/lib/app/mixp/decorators.py index 0ef6d8a81..5f2cc60ec 100644 --- a/src/superannotate/lib/app/mixp/decorators.py +++ b/src/superannotate/lib/app/mixp/decorators.py @@ -2,7 +2,6 @@ import sys from inspect import signature - from mixpanel import Mixpanel from superannotate.logger import get_default_logger from version import __version__ diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 32073bb3b..db8c24395 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -33,7 +33,7 @@ def init_spin(self): class Session: - def __init__(self, pk: str): + def __init__(self, pk: int): self.pk = pk self._uuid = str(uuid.uuid4()) self._data_dict = {} diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 7a290890a..402fbdf0d 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -8,7 +8,6 @@ import boto3 import lib.core as constances -from lib.core.enums import ProjectType import pandas as pd import requests from botocore.exceptions import ClientError @@ -21,6 +20,7 @@ from lib.core.entities import MLModelEntity from lib.core.entities import ProjectEntity from lib.core.enums import ExportStatus +from lib.core.enums import ProjectType from lib.core.exceptions import AppException from lib.core.exceptions import AppValidationException from lib.core.reporter import Reporter diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 2e957ae98..f54652fed 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1,6 +1,7 @@ import copy import io import os +import threading from abc import ABCMeta from pathlib import Path from typing import Callable @@ -10,12 +11,10 @@ from typing import Tuple from typing import Union -from superannotate_schemas.validators import AnnotationValidators - import lib.core as constances from lib.core import usecases -from lib.core.conditions import CONDITION_EQ as EQ from lib.core.conditions import Condition +from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import AnnotationClassEntity from lib.core.entities import AttachmentEntity from lib.core.entities import FolderEntity @@ -42,6 +41,7 @@ from lib.infrastructure.repositories import WorkflowRepository from lib.infrastructure.services import SuperannotateBackendService from superannotate.logger import get_default_logger +from superannotate_schemas.validators import AnnotationValidators def build_condition(**kwargs) -> Condition: @@ -80,11 +80,12 @@ def __init__(self, token: str, host: str, ssl_verify: bool, version: str): self._team_name = None self._reporter = None - def get_session(self, pk: str): + def get_session(self): + pk = threading.get_ident() try: return self.SESSIONS[pk] except KeyError: - self.SESSIONS[pk] = Session(pk) + self.SESSIONS[threading.get_ident()] = Session(pk) return self.SESSIONS[pk] @staticmethod @@ -1149,7 +1150,7 @@ def download_export( folder_path=folder_path, extract_zip_contents=extract_zip_contents, to_s3_bucket=to_s3_bucket, - reporter=self.default_reporter, + reporter=self.get_default_reporter(), ) return use_case.execute() @@ -1198,7 +1199,7 @@ def benchmark( folder_path=export_root, extract_zip_contents=True, to_s3_bucket=False, - reporter=self.default_reporter, + reporter=self.get_default_reporter(), ).execute() if response.errors: raise AppException(response.errors) @@ -1644,7 +1645,7 @@ def download_annotations( folder = self._get_folder(project, folder_name) use_case = usecases.DownloadAnnotations( - reporter=self.default_reporter, + reporter=self.get_default_reporter(), project=project, folder=folder, destination=destination, diff --git a/src/superannotate/logger.py b/src/superannotate/logger.py index 3345c636f..3af9ce1e0 100644 --- a/src/superannotate/logger.py +++ b/src/superannotate/logger.py @@ -4,7 +4,7 @@ from logging.handlers import RotatingFileHandler from os.path import expanduser -from superannotate import constances +import superannotate.lib.core as constances default_logger = None diff --git a/tests/integration/annotations/test_download_annotations.py b/tests/integration/annotations/test_download_annotations.py index 4223b9010..48fb3587e 100644 --- a/tests/integration/annotations/test_download_annotations.py +++ b/tests/integration/annotations/test_download_annotations.py @@ -5,10 +5,13 @@ import pytest -import src.superannotate as sa +from src.superannotate import SAClient from tests.integration.base import BaseTestCase +sa = SAClient() + + class TestDownloadAnnotations(BaseTestCase): PROJECT_NAME = "Test-download_annotations" FOLDER_NAME = "FOLDER_NAME" @@ -24,7 +27,6 @@ def folder_path(self): @pytest.mark.flaky(reruns=3) def test_download_annotations(self): - sa.init() sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) @@ -45,7 +47,6 @@ def test_download_annotations(self): @pytest.mark.flaky(reruns=3) def test_download_annotations_from_folders(self): - sa.init() sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME_2) sa.create_annotation_classes_from_classes_json( diff --git a/tests/integration/annotations/test_video_annotation_upload.py b/tests/integration/annotations/test_video_annotation_upload.py index 27cf33ae1..44795738c 100644 --- a/tests/integration/annotations/test_video_annotation_upload.py +++ b/tests/integration/annotations/test_video_annotation_upload.py @@ -4,9 +4,12 @@ from pathlib import Path import pytest + from src.superannotate import SAClient +from src.superannotate.lib.core.reporter import Reporter + sa = SAClient() -from tests.integration.base import BaseTestCase + from src.superannotate.lib.core.data_handlers import VideoFormatHandler from tests.integration.base import BaseTestCase diff --git a/tests/integration/projects/test_clone_project.py b/tests/integration/projects/test_clone_project.py index 4601f4e25..5b8c87952 100644 --- a/tests/integration/projects/test_clone_project.py +++ b/tests/integration/projects/test_clone_project.py @@ -4,7 +4,7 @@ from src.superannotate import SAClient sa = SAClient() from tests import DATA_SET_PATH -from src.superannotate import constances +import src.superannotate.lib.core as constances class TestCloneProject(TestCase): diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index c15b41bb7..cd333ee31 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -4,12 +4,9 @@ from pathlib import Path from unittest import TestCase -from src.superannotate import SAClient -from src.superannotate import __version__ - import pkg_resources -import src.superannotate as sa +from src.superannotate import SAClient from src.superannotate.lib.app.interface.cli_interface import CLIFacade sa = SAClient() @@ -136,22 +133,23 @@ def test_vector_pre_annotation_folder_upload_download_cli(self): sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.vector_folder_path}/classes/classes.json" ) - self.safe_run(self._cli.upload_images, self.PROJECT_NAME, folder=str(self.convertor_data_path), + self.safe_run(self._cli.upload_images, project=self.PROJECT_NAME, folder=str(self.convertor_data_path), extensions="jpg", set_annotation_status="QualityCheck") - self.safe_run(self._cli.upload_preannotations, self.PROJECT_NAME, folder=str(self.convertor_data_path), + self.safe_run(self._cli.upload_preannotations, project=self.PROJECT_NAME, folder=str(self.convertor_data_path), format="COCO", dataset_name="instances_test") - # tod add test def test_vector_annotation_folder_upload_download_cli(self): self._create_project() sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.vector_folder_path}/classes/classes.json" ) - self.safe_run(self._cli.upload_images, self.PROJECT_NAME, str(self.convertor_data_path), extensions="jpg", + self.safe_run(self._cli.upload_images, project=self.PROJECT_NAME, folder=str(self.convertor_data_path), + extensions="jpg", set_annotation_status="QualityCheck") - self.safe_run(self._cli.upload_annotations, self.PROJECT_NAME, str(self.convertor_data_path), format="COCO", + self.safe_run(self._cli.upload_annotations, project=self.PROJECT_NAME, folder=str(self.convertor_data_path), + format="COCO", dataset_name="instances_test") count_in = len(list(self.vector_folder_path.glob("*.json"))) @@ -175,7 +173,7 @@ def test_attach_video_urls(self): def test_upload_videos(self): self._create_project() self.safe_run(self._cli.upload_videos, self.PROJECT_NAME, str(self.video_folder_path)) - self.assertEqual(5, len(sa.search_items(self.PROJECT_NAME))) + self.assertEqual(121, len(sa.search_items(self.PROJECT_NAME))) def test_attach_document_urls(self): self._create_project("Document") diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py index 77c80e8c7..58bc213a5 100644 --- a/tests/integration/test_depricated_functions_document.py +++ b/tests/integration/test_depricated_functions_document.py @@ -123,17 +123,6 @@ def test_deprecated_functions(self): sa.get_project_workflow(self.PROJECT_NAME) except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.class_distribution(self.video_export_path, [self.PROJECT_NAME]) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) - try: - sa.convert_project_type(self.video_export_path, "./") - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE_DOCUMENT_VIDEO, str(e)) - sa.move_images(self.PROJECT_NAME, [self.UPLOAD_IMAGE_NAME], self.PROJECT_NAME_2) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.prepare_export(self.PROJECT_NAME, include_fuse=True, only_pinned=True) except AppException as e: diff --git a/tests/integration/test_get_exports.py b/tests/integration/test_get_exports.py index 592d1d038..66f9f7c4b 100644 --- a/tests/integration/test_get_exports.py +++ b/tests/integration/test_get_exports.py @@ -4,10 +4,11 @@ from os.path import dirname from src.superannotate import SAClient -sa = SAClient() -from src.superannotate import AppException +from src.superannotate import export_annotation from tests.integration.base import BaseTestCase +sa = SAClient() + class TestGetExports(BaseTestCase): PROJECT_NAME = "get_exports" @@ -61,7 +62,7 @@ def test_convert_pixel_exported_data(self): with tempfile.TemporaryDirectory() as tmp_dir: sa.download_export(self.PROJECT_NAME, export["name"], tmp_dir) with tempfile.TemporaryDirectory() as converted_data_tmp_dir: - sa.export_annotation( + export_annotation( tmp_dir, converted_data_tmp_dir, "COCO", "export", "Pixel", "panoptic_segmentation" ) - self.assertEqual(1, len(list(glob.glob(converted_data_tmp_dir + "/*.json")))) \ No newline at end of file + self.assertEqual(1, len(list(glob.glob(converted_data_tmp_dir + "/*.json")))) diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 6a8786446..9739ddca8 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -4,11 +4,13 @@ import pytest -from src.superannotate import SAClient -sa = SAClient() from src.superannotate import AppException +from src.superannotate import SAClient +from src.superannotate import export_annotation from tests.integration.base import BaseTestCase +sa = SAClient() + class TestInterface(BaseTestCase): PROJECT_NAME = "Interface test" @@ -266,7 +268,7 @@ def test_export_annotation(self): ) sa.download_export(self.PROJECT_NAME, result, export_dir, True) with tempfile.TemporaryDirectory() as convert_path: - sa.export_annotation( + export_annotation( export_dir, convert_path, "COCO", "data_set_name", "Pixel", "panoptic_segmentation" ) pass From a2f19f3318731cb7f4001c7460ffa87a5beebf93 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 24 May 2022 12:42:31 +0400 Subject: [PATCH 03/59] fix import issue --- src/superannotate/__init__.py | 33 +++++++++---------- .../lib/app/interface/cli_interface.py | 2 +- .../lib/app/interface/sdk_interface.py | 4 +-- .../test_get_annotations_per_frame.py | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index eca931970..448a787a5 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -1,22 +1,22 @@ -import logging.config import os import sys +sys.path.append(os.path.split(os.path.realpath(__file__))[0]) -import requests -from packaging.version import parse -from superannotate.lib.app.analytics.class_analytics import class_distribution -from superannotate.lib.app.exceptions import AppException -from superannotate.lib.app.input_converters.conversion import convert_json_version -from superannotate.lib.app.input_converters.conversion import convert_project_type -from superannotate.lib.app.input_converters.conversion import export_annotation -from superannotate.lib.app.input_converters.conversion import import_annotation -from superannotate.lib.app.interface.sdk_interface import SAClient -from superannotate.lib.core import PACKAGE_VERSION_INFO_MESSAGE -from superannotate.lib.core import PACKAGE_VERSION_MAJOR_UPGRADE -from superannotate.lib.core import PACKAGE_VERSION_UPGRADE -from superannotate.logger import get_default_logger -from superannotate.version import __version__ - +import logging.config # noqa +import requests # noqa +from packaging.version import parse # noqa +from superannotate.lib.app.analytics.class_analytics import class_distribution # noqa +from superannotate.lib.app.exceptions import AppException # noqa +from superannotate.lib.app.input_converters.conversion import convert_json_version # noqa +from superannotate.lib.app.input_converters.conversion import convert_project_type # noqa +from superannotate.lib.app.input_converters.conversion import export_annotation # noqa +from superannotate.lib.app.input_converters.conversion import import_annotation # noqa +from superannotate.lib.app.interface.sdk_interface import SAClient # noqa +from superannotate.lib.core import PACKAGE_VERSION_INFO_MESSAGE # noqa +from superannotate.lib.core import PACKAGE_VERSION_MAJOR_UPGRADE # noqa +from superannotate.lib.core import PACKAGE_VERSION_UPGRADE # noqa +from superannotate.logger import get_default_logger # noqa +from superannotate.version import __version__ # noqa __all__ = [ "__version__", @@ -34,7 +34,6 @@ __author__ = "Superannotate" -sys.path.append(os.path.split(os.path.realpath(__file__))[0]) logging.getLogger("botocore").setLevel(logging.CRITICAL) logger = get_default_logger() diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index 26f90b467..c0576ccce 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -122,7 +122,7 @@ def export_project( export_name=export_name, folder_path=folder, extract_zip_contents=not disable_extract_zip_contents, - to_s3_bucket= False, + to_s3_bucket=False, ) sys.exit(0) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index ddd8f9bcd..4e6177c79 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -66,16 +66,16 @@ class SAClient(BaseInterfaceFacade): def __init__( self, token: str = None, - host=constances.BACKEND_URL, config_path: str = constances.CONFIG_FILE_LOCATION, ): + host = constances.BACKEND_URL env_token = os.environ.get("SA_TOKEN") version = os.environ.get("SA_VERSION", "v1") ssl_verify = bool(os.environ.get("SA_SSL", True)) if token: token = Controller.validate_token(token=token) elif env_token: - host = os.environ.get("SA_UTR", constances.BACKEND_URL) + host = os.environ.get("SA_URL", constances.BACKEND_URL) token = Controller.validate_token(env_token) else: diff --git a/tests/integration/annotations/test_get_annotations_per_frame.py b/tests/integration/annotations/test_get_annotations_per_frame.py index 18be86750..16bfaa96c 100644 --- a/tests/integration/annotations/test_get_annotations_per_frame.py +++ b/tests/integration/annotations/test_get_annotations_per_frame.py @@ -38,7 +38,7 @@ def annotations_path(self): return os.path.join(self.folder_path, self.ANNOTATIONS_PATH) def test_video_annotation_upload(self): - sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) + # sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) _, _, _ = sa.attach_items( self.PROJECT_NAME, From 34f76a189e51bad199b8742c3d435b9905a02f53 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 25 May 2022 16:27:59 +0400 Subject: [PATCH 04/59] Update by comments --- src/superannotate/lib/app/interface/base_interface.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 21dd5a84a..5c5a8b9c6 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -71,9 +71,7 @@ def default_parser(function_name: str, kwargs: dict) -> tuple: continue elif key == "project": properties["project_name"], properties["folder_name"] = extract_project_folder(value) - elif isinstance(value, str) and key == "project": - properties["project_name"] = value.split() - if isinstance(value, (str, int, float, bool, str)): + elif isinstance(value, (str, int, float, bool, str)): properties[key] = value elif isinstance(value, dict): properties[key] = value.keys() From 2631f627ee64f2c510830998a6dde211026e312f Mon Sep 17 00:00:00 2001 From: VavoTK Date: Mon, 30 May 2022 14:59:54 +0400 Subject: [PATCH 05/59] adding assign items and un_assign items functionality --- .../lib/app/interface/sdk_interface.py | 861 ++++++++++-------- .../lib/core/serviceproviders.py | 15 + src/superannotate/lib/core/usecases/items.py | 70 ++ .../lib/infrastructure/controller.py | 550 ++++++----- .../lib/infrastructure/services.py | 38 + 5 files changed, 892 insertions(+), 642 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 4e6177c79..a00f41982 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -64,9 +64,9 @@ class SAClient(BaseInterfaceFacade): def __init__( - self, - token: str = None, - config_path: str = constances.CONFIG_FILE_LOCATION, + self, + token: str = None, + config_path: str = constances.CONFIG_FILE_LOCATION, ): host = constances.BACKEND_URL env_token = os.environ.get("SA_TOKEN") @@ -119,11 +119,11 @@ def get_team_metadata(self): @validate_arguments def search_team_contributors( - self, - email: EmailStr = None, - first_name: NotEmptyStr = None, - last_name: NotEmptyStr = None, - return_metadata: bool = True, + self, + email: EmailStr = None, + first_name: NotEmptyStr = None, + last_name: NotEmptyStr = None, + return_metadata: bool = True, ): """Search for contributors in the team @@ -150,11 +150,11 @@ def search_team_contributors( @validate_arguments def search_projects( - self, - name: Optional[NotEmptyStr] = None, - return_metadata: bool = False, - include_complete_image_count: bool = False, - status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, + self, + name: Optional[NotEmptyStr] = None, + return_metadata: bool = False, + include_complete_image_count: bool = False, + status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, ): """ Project name based case-insensitive search for projects. @@ -184,7 +184,7 @@ def search_projects( result = self.controller.search_project( name=name, include_complete_image_count=include_complete_image_count, - statuses=statuses + statuses=statuses, ).data if return_metadata: @@ -205,11 +205,11 @@ def search_projects( @validate_arguments def create_project( - self, - project_name: NotEmptyStr, - project_description: NotEmptyStr, - project_type: NotEmptyStr, - settings: List[Setting] = None, + self, + project_name: NotEmptyStr, + project_description: NotEmptyStr, + project_type: NotEmptyStr, + settings: List[Setting] = None, ): """Create a new project in the team. @@ -244,8 +244,7 @@ def create_project( return ProjectSerializer(response.data).serialize() @validate_arguments - def create_project_from_metadata( - self, project_metadata: Project): + def create_project_from_metadata(self, project_metadata: Project): """Create a new project in the team using project metadata object dict. Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) Non-mandatory keys: "workflow", "settings" and "annotation_classes". @@ -271,14 +270,14 @@ def create_project_from_metadata( @validate_arguments def clone_project( - self, - project_name: Union[NotEmptyStr, dict], - from_project: Union[NotEmptyStr, dict], - project_description: Optional[NotEmptyStr] = None, - copy_annotation_classes: Optional[StrictBool] = True, - copy_settings: Optional[StrictBool] = True, - copy_workflow: Optional[StrictBool] = True, - copy_contributors: Optional[StrictBool] = False, + self, + project_name: Union[NotEmptyStr, dict], + from_project: Union[NotEmptyStr, dict], + project_description: Optional[NotEmptyStr] = None, + copy_annotation_classes: Optional[StrictBool] = True, + copy_settings: Optional[StrictBool] = True, + copy_workflow: Optional[StrictBool] = True, + copy_contributors: Optional[StrictBool] = False, ): """Create a new project in the team using annotation classes and settings from from_project. @@ -315,8 +314,7 @@ def clone_project( return ProjectSerializer(response.data).serialize() @validate_arguments - def create_folder( - self, project: NotEmptyStr, folder_name: NotEmptyStr): + def create_folder(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Create a new folder in the project. :param project: project name @@ -328,9 +326,7 @@ def create_folder( :rtype: dict """ - res = self.controller.create_folder( - project=project, folder_name=folder_name - ) + res = self.controller.create_folder(project=project, folder_name=folder_name) if res.data: folder = res.data logger.info(f"Folder {folder.name} created in project {project}") @@ -339,12 +335,11 @@ def create_folder( raise AppException(res.errors) @validate_arguments - def delete_project( - self, project: Union[NotEmptyStr, dict]): + def delete_project(self, project: Union[NotEmptyStr, dict]): """Deletes the project - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str """ name = project if isinstance(project, dict): @@ -352,8 +347,7 @@ def delete_project( self.controller.delete_project(name=name) @validate_arguments - def rename_project( - self, project: NotEmptyStr, new_name: NotEmptyStr): + def rename_project(self, project: NotEmptyStr, new_name: NotEmptyStr): """Renames the project :param project: project name or folder path (e.g., "project1/folder1") @@ -367,12 +361,13 @@ def rename_project( ) if response.errors: raise AppException(response.errors) - logger.info("Successfully renamed project %s to %s.", project, response.data.name) + logger.info( + "Successfully renamed project %s to %s.", project, response.data.name + ) return ProjectSerializer(response.data).serialize() @validate_arguments - def get_folder_metadata( - self, project: NotEmptyStr, folder_name: NotEmptyStr): + def get_folder_metadata(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Returns folder metadata :param project: project name @@ -383,18 +378,15 @@ def get_folder_metadata( :return: metadata of folder :rtype: dict """ - result = ( - self.controller - .get_folder(project_name=project, folder_name=folder_name) - .data - ) + result = self.controller.get_folder( + project_name=project, folder_name=folder_name + ).data if not result: raise AppException("Folder not found.") return FolderSerializer(result).serialize() @validate_arguments - def delete_folders( - self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): + def delete_folders(self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): """Delete folder in project. :param project: project name @@ -412,10 +404,10 @@ def delete_folders( @validate_arguments def search_folders( - self, - project: NotEmptyStr, - folder_name: Optional[NotEmptyStr] = None, - return_metadata: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + folder_name: Optional[NotEmptyStr] = None, + return_metadata: Optional[StrictBool] = False, ): """Folder name based case-insensitive search for folders in project. @@ -442,13 +434,13 @@ def search_folders( @validate_arguments def copy_image( - self, - source_project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = False, - copy_annotation_status: Optional[StrictBool] = False, - copy_pin: Optional[StrictBool] = False, + self, + source_project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + destination_project: Union[NotEmptyStr, dict], + include_annotations: Optional[StrictBool] = False, + copy_annotation_status: Optional[StrictBool] = False, + copy_pin: Optional[StrictBool] = False, ): """Copy image to a project. The image's project is the same as destination project then the name will be changed to _()., @@ -473,12 +465,12 @@ def copy_image( destination_project, destination_folder = extract_project_folder( destination_project ) - source_project_metadata = ( - self.controller.get_project_metadata(source_project_name).data - ) - destination_project_metadata = ( - self.controller.get_project_metadata(destination_project).data - ) + source_project_metadata = self.controller.get_project_metadata( + source_project_name + ).data + destination_project_metadata = self.controller.get_project_metadata( + destination_project + ).data if destination_project_metadata["project"].type in [ constances.ProjectType.VIDEO.value, @@ -487,7 +479,9 @@ def copy_image( constances.ProjectType.VIDEO.value, constances.ProjectType.DOCUMENT.value, ]: - raise AppException(LIMITED_FUNCTIONS[source_project_metadata["project"].type]) + raise AppException( + LIMITED_FUNCTIONS[source_project_metadata["project"].type] + ) response = self.controller.copy_image( from_project_name=source_project_name, @@ -522,13 +516,13 @@ def copy_image( @validate_arguments def get_project_metadata( - self, - project: Union[NotEmptyStr, dict], - include_annotation_classes: Optional[StrictBool] = False, - include_settings: Optional[StrictBool] = False, - include_workflow: Optional[StrictBool] = False, - include_contributors: Optional[StrictBool] = False, - include_complete_image_count: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + include_annotation_classes: Optional[StrictBool] = False, + include_settings: Optional[StrictBool] = False, + include_workflow: Optional[StrictBool] = False, + include_contributors: Optional[StrictBool] = False, + include_complete_image_count: Optional[StrictBool] = False, ): """Returns project metadata @@ -555,30 +549,27 @@ def get_project_metadata( :rtype: dict """ project_name, folder_name = extract_project_folder(project) - response = ( - self.controller.get_project_metadata( - project_name, - include_annotation_classes, - include_settings, - include_workflow, - include_contributors, - include_complete_image_count, - ) - .data - ) + response = self.controller.get_project_metadata( + project_name, + include_annotation_classes, + include_settings, + include_workflow, + include_contributors, + include_complete_image_count, + ).data metadata = ProjectSerializer(response["project"]).serialize() for elem in "classes", "workflows", "contributors": if response.get(elem): metadata[elem] = [ - BaseSerializer(attribute).serialize() for attribute in response[elem] + BaseSerializer(attribute).serialize() + for attribute in response[elem] ] return metadata @validate_arguments - def get_project_settings( - self, project: Union[NotEmptyStr, dict]): + def get_project_settings(self, project: Union[NotEmptyStr, dict]): """Gets project's settings. Return value example: [{ "attribute" : "Brightness", "value" : 10, ...},...] @@ -597,8 +588,7 @@ def get_project_settings( return settings @validate_arguments - def get_project_workflow( - self, project: Union[str, dict]): + def get_project_workflow(self, project: Union[str, dict]): """Gets project's workflow. Return value example: [{ "step" : , "className" : , "tool" : , ...},...] @@ -617,8 +607,7 @@ def get_project_workflow( @validate_arguments def search_annotation_classes( - self, - project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None + self, project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None ): """Searches annotation classes by name_prefix (case-insensitive) @@ -632,17 +621,15 @@ def search_annotation_classes( :rtype: list of dicts """ project_name, folder_name = extract_project_folder(project) - classes = self.controller.search_annotation_classes( - project_name, name_contains - ) + classes = self.controller.search_annotation_classes(project_name, name_contains) classes = [BaseSerializer(attribute).serialize() for attribute in classes.data] return classes @validate_arguments def set_project_default_image_quality_in_editor( - self, - project: Union[NotEmptyStr, dict], - image_quality_in_editor: Optional[str], + self, + project: Union[NotEmptyStr, dict], + image_quality_in_editor: Optional[str], ): """Sets project's default image quality in editor setting. @@ -656,7 +643,9 @@ def set_project_default_image_quality_in_editor( response = self.controller.set_project_settings( project_name=project_name, - new_settings=[{"attribute": "ImageQuality", "value": image_quality_in_editor}], + new_settings=[ + {"attribute": "ImageQuality", "value": image_quality_in_editor} + ], ) if response.errors: raise AppException(response.errors) @@ -664,8 +653,10 @@ def set_project_default_image_quality_in_editor( @validate_arguments def pin_image( - self, - project: Union[NotEmptyStr, dict], image_name: str, pin: Optional[StrictBool] = True + self, + project: Union[NotEmptyStr, dict], + image_name: str, + pin: Optional[StrictBool] = True, ): """Pins (or unpins) image @@ -686,10 +677,10 @@ def pin_image( @validate_arguments def set_images_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: NotEmptyStr, - image_names: Optional[List[NotEmptyStr]] = None, + self, + project: Union[NotEmptyStr, dict], + annotation_status: NotEmptyStr, + image_names: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of images @@ -718,8 +709,7 @@ def set_images_annotation_statuses( @validate_arguments def delete_images( - self, - project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None + self, project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None ): """Delete images in project. @@ -743,9 +733,77 @@ def delete_images( f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) + @validate_arguments + def assign_items( + self, project: Union[NotEmptyStr, dict], item_names: List[str], user: str + ): + """Assigns items to a user. The assignment role, QA or Annotator, will + be deduced from the user's role in the project. The type of the objects` image, video or text + will be deduced from the project type. With SDK, the user can be + assigned to a role in the project with the share_project function. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param item_names: list of item names to assign + :type item_names: list of str + :param user: user email + :type user: str + """ + + project_name, folder_name = extract_project_folder(project) + project = self.controller.get_project_metadata(project_name).data + + contributors = ( + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users + ) + contributor = None + for c in contributors: + if c["user_id"] == user: + contributor = user + + if not contributor: + logger.warning( + f"Skipping {user}. {user} is not a verified contributor for the {project_name}" + ) + return + + response = self.controller.assign_items( + project_name, folder_name, item_names, user + ) + if not response.errors: + logger.info(f"Assign items to user {user}") + else: + raise AppException(response.errors) + + @validate_arguments + def unassign_items( + self, project: Union[NotEmptyStr, dict], item_names: List[NotEmptyStr] + ): + """Removes assignment of given items for all assignees. With SDK, + the user can be assigned to a role in the project with the share_project + function. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param item_names: list of items to unassign + :type item_names: list of str + """ + project_name, folder_name = extract_project_folder(project) + + response = self.controller.un_assign_items( + project_name=project_name, folder_name=folder_name, item_names=item_names + ) + if response.errors: + raise AppException(response.errors) + @validate_arguments def assign_images( - self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str): + self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str + ): """Assigns images to a user. The assignment role, QA or Annotator, will be deduced from the user's role in the project. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -767,10 +825,11 @@ def assign_images( raise AppException(LIMITED_FUNCTIONS[project["project"].type]) contributors = ( - self.controller - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users ) contributor = None for c in contributors: @@ -793,27 +852,27 @@ def assign_images( @validate_arguments def unassign_images( - self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr]): - """Removes assignment of given images for all assignees.With SDK, + self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr] + ): + """Removes assignment of given images for all assignees. With SDK, the user can be assigned to a role in the project with the share_project function. :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param image_names: list of image unassign + :param image_names: list of images to unassign :type image_names: list of str """ project_name, folder_name = extract_project_folder(project) - response = self.controller.un_assign_images( + response = self.controller.un_assign_items( project_name=project_name, folder_name=folder_name, image_names=image_names ) if response.errors: raise AppException(response.errors) @validate_arguments - def unassign_folder( - self, project_name: NotEmptyStr, folder_name: NotEmptyStr): + def unassign_folder(self, project_name: NotEmptyStr, folder_name: NotEmptyStr): """Removes assignment of given folder for all assignees. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -831,8 +890,10 @@ def unassign_folder( @validate_arguments def assign_folder( - self, - project_name: NotEmptyStr, folder_name: NotEmptyStr, users: List[NotEmptyStr] + self, + project_name: NotEmptyStr, + folder_name: NotEmptyStr, + users: List[NotEmptyStr], ): """Assigns folder to users. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -846,10 +907,11 @@ def assign_folder( """ contributors = ( - self.controller - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users ) verified_users = [i["user_id"] for i in contributors] verified_users = set(users).intersection(set(verified_users)) @@ -864,7 +926,9 @@ def assign_folder( return response = self.controller.assign_folder( - project_name=project_name, folder_name=folder_name, users=list(verified_users) + project_name=project_name, + folder_name=folder_name, + users=list(verified_users), ) if response.errors: @@ -872,19 +936,19 @@ def assign_folder( @validate_arguments def upload_images_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[List[NotEmptyStr], Tuple[NotEmptyStr]] - ] = constances.DEFAULT_IMAGE_EXTENSIONS, - annotation_status="NotStarted", - from_s3_bucket=None, - exclude_file_patterns: Optional[ - Iterable[NotEmptyStr] - ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders: Optional[StrictBool] = False, - image_quality_in_editor: Optional[str] = None, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[List[NotEmptyStr], Tuple[NotEmptyStr]] + ] = constances.DEFAULT_IMAGE_EXTENSIONS, + annotation_status="NotStarted", + from_s3_bucket=None, + exclude_file_patterns: Optional[ + Iterable[NotEmptyStr] + ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders: Optional[StrictBool] = False, + image_quality_in_editor: Optional[str] = None, ): """Uploads all images with given extensions from folder_path to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -967,15 +1031,20 @@ def upload_images_from_folder_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) + "%s already existing images found that won't be uploaded.", + len(duplicates), ) logger.info( - "Uploading %s images to project %s.", len(images_to_upload), project_folder_name + "Uploading %s images to project %s.", + len(images_to_upload), + project_folder_name, ) if not images_to_upload: return [], [], duplicates if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: for _ in use_case.execute(): progress_bar.update(1) return use_case.data @@ -983,8 +1052,9 @@ def upload_images_from_folder_to_project( @validate_arguments def get_project_image_count( - self, - project: Union[NotEmptyStr, dict], with_all_subfolders: Optional[StrictBool] = False + self, + project: Union[NotEmptyStr, dict], + with_all_subfolders: Optional[StrictBool] = False, ): """Returns number of images in the project. @@ -1010,10 +1080,10 @@ def get_project_image_count( @validate_arguments def download_image_annotations( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Union[str, Path], + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Union[str, Path], ): """Downloads annotations of the image (JSON and mask if pixel type project) to local_dir_path. @@ -1041,7 +1111,8 @@ def download_image_annotations( @validate_arguments def get_exports( - self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False): + self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False + ): """Get all prepared exports of the project. :param project: project name @@ -1059,12 +1130,12 @@ def get_exports( @validate_arguments def prepare_export( - self, - project: Union[NotEmptyStr, dict], - folder_names: Optional[List[NotEmptyStr]] = None, - annotation_statuses: Optional[List[AnnotationStatuses]] = None, - include_fuse: Optional[StrictBool] = False, - only_pinned=False, + self, + project: Union[NotEmptyStr, dict], + folder_names: Optional[List[NotEmptyStr]] = None, + annotation_statuses: Optional[List[AnnotationStatuses]] = None, + include_fuse: Optional[StrictBool] = False, + only_pinned=False, ): """Prepare annotations and classes.json for export. Original and fused images for images with annotations can be included with include_fuse flag. @@ -1112,19 +1183,19 @@ def prepare_export( @validate_arguments def upload_videos_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[Tuple[NotEmptyStr], List[NotEmptyStr]] - ] = constances.DEFAULT_VIDEO_EXTENSIONS, - exclude_file_patterns: Optional[List[NotEmptyStr]] = (), - recursive_subfolders: Optional[StrictBool] = False, - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[Tuple[NotEmptyStr], List[NotEmptyStr]] + ] = constances.DEFAULT_VIDEO_EXTENSIONS, + exclude_file_patterns: Optional[List[NotEmptyStr]] = (), + recursive_subfolders: Optional[StrictBool] = False, + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads image frames from all videos with given extensions from folder_path to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -1164,7 +1235,9 @@ def upload_videos_from_folder_to_project( if not recursive_subfolders: video_paths += list(Path(folder_path).glob(f"*.{extension.lower()}")) if os.name != "nt": - video_paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) + video_paths += list( + Path(folder_path).glob(f"*.{extension.upper()}") + ) else: logger.warning( "When using recursive subfolder parsing same name videos " @@ -1172,7 +1245,9 @@ def upload_videos_from_folder_to_project( ) video_paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) if os.name != "nt": - video_paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) + video_paths += list( + Path(folder_path).rglob(f"*.{extension.upper()}") + ) video_paths = [str(path) for path in video_paths] response = self.controller.upload_videos( @@ -1192,14 +1267,14 @@ def upload_videos_from_folder_to_project( @validate_arguments def upload_video_to_project( - self, - project: Union[NotEmptyStr, dict], - video_path: Union[NotEmptyStr, Path], - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: Union[NotEmptyStr, dict], + video_path: Union[NotEmptyStr, Path], + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads image frames from video to platform. Uploaded images will have names "_.jpg". @@ -1244,12 +1319,12 @@ def upload_video_to_project( @validate_arguments def create_annotation_class( - self, - project: Union[Project, NotEmptyStr], - name: NotEmptyStr, - color: NotEmptyStr, - attribute_groups: Optional[List[AttributeGroup]] = None, - class_type: ClassType = "object", + self, + project: Union[Project, NotEmptyStr], + name: NotEmptyStr, + color: NotEmptyStr, + attribute_groups: Optional[List[AttributeGroup]] = None, + class_type: ClassType = "object", ): """Create annotation class in project @@ -1287,8 +1362,7 @@ def create_annotation_class( @validate_arguments def delete_annotation_class( - self, - project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] + self, project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] ): """Deletes annotation class from project @@ -1303,7 +1377,8 @@ def delete_annotation_class( @validate_arguments def download_annotation_classes_json( - self, project: NotEmptyStr, folder: Union[str, Path]): + self, project: NotEmptyStr, folder: Union[str, Path] + ): """Downloads project classes.json to folder :param project: project name @@ -1323,10 +1398,10 @@ def download_annotation_classes_json( @validate_arguments def create_annotation_classes_from_classes_json( - self, - project: Union[NotEmptyStr, dict], - classes_json: Union[List[AnnotationClassEntity], str, Path], - from_s3_bucket=False, + self, + project: Union[NotEmptyStr, dict], + classes_json: Union[List[AnnotationClassEntity], str, Path], + from_s3_bucket=False, ): """Creates annotation classes in project from a SuperAnnotate format annotation classes.json. @@ -1359,7 +1434,8 @@ def create_annotation_classes_from_classes_json( raise AppException("Couldn't validate annotation classes.") logger.info(f"Creating annotation classes in project {project}.") response = self.controller.create_annotation_classes( - project_name=project, annotation_classes=annotation_classes, + project_name=project, + annotation_classes=annotation_classes, ) if response.errors: raise AppException(response.errors) @@ -1367,12 +1443,12 @@ def create_annotation_classes_from_classes_json( @validate_arguments def download_export( - self, - project: Union[NotEmptyStr, dict], - export: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - extract_zip_contents: Optional[StrictBool] = True, - to_s3_bucket=None, + self, + project: Union[NotEmptyStr, dict], + export: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + extract_zip_contents: Optional[StrictBool] = True, + to_s3_bucket=None, ): """Download prepared export. @@ -1407,10 +1483,10 @@ def download_export( @validate_arguments def set_image_annotation_status( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - annotation_status: NotEmptyStr, + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + annotation_status: NotEmptyStr, ): """Sets the image annotation status @@ -1438,14 +1514,13 @@ def set_image_annotation_status( ) if response.errors: raise AppException(response.errors) - image = ( - self.controller.get_item(project_name, folder_name, image_name).data - ) + image = self.controller.get_item(project_name, folder_name, image_name).data return BaseSerializer(image).serialize() @validate_arguments def set_project_workflow( - self, project: Union[NotEmptyStr, dict], new_workflow: List[dict]): + self, project: Union[NotEmptyStr, dict], new_workflow: List[dict] + ): """Sets project's workflow. new_workflow example: [{ "step" : , "className" : , "tool" : , @@ -1467,14 +1542,14 @@ def set_project_workflow( @validate_arguments def download_image( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Optional[Union[str, Path]] = "./", - include_annotations: Optional[StrictBool] = False, - include_fuse: Optional[StrictBool] = False, - include_overlay: Optional[StrictBool] = False, - variant: Optional[str] = "original", + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Optional[Union[str, Path]] = "./", + include_annotations: Optional[StrictBool] = False, + include_fuse: Optional[StrictBool] = False, + include_overlay: Optional[StrictBool] = False, + variant: Optional[str] = "original", ): """Downloads the image (and annotation if not None) to local_dir_path @@ -1515,11 +1590,11 @@ def download_image( @validate_arguments def upload_annotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, ): """Finds and uploads all JSON files in the folder_path as annotations to the project. @@ -1578,11 +1653,11 @@ def upload_annotations_from_folder_to_project( @validate_arguments def upload_preannotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, ): """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. @@ -1644,12 +1719,12 @@ def upload_preannotations_from_folder_to_project( @validate_arguments def upload_image_annotations( - self, - project: Union[NotEmptyStr, dict], - image_name: str, - annotation_json: Union[str, Path, dict], - mask: Optional[Union[str, Path, bytes]] = None, - verbose: Optional[StrictBool] = True, + self, + project: Union[NotEmptyStr, dict], + image_name: str, + annotation_json: Union[str, Path, dict], + mask: Optional[Union[str, Path, bytes]] = None, + verbose: Optional[StrictBool] = True, ): """Upload annotations from JSON (also mask for pixel annotations) to the image. @@ -1700,8 +1775,7 @@ def upload_image_annotations( raise AppException(response.errors) @validate_arguments - def download_model( - self, model: MLModel, output_dir: Union[str, Path]): + def download_model(self, model: MLModel, output_dir: Union[str, Path]): """Downloads the neural network and related files which are the .pth/pkl. .json, .yaml, classes_mapper.json @@ -1722,14 +1796,14 @@ def download_model( @validate_arguments def benchmark( - self, - project: Union[NotEmptyStr, dict], - gt_folder: str, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[str, Path]] = None, - image_list=None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots=False, + self, + project: Union[NotEmptyStr, dict], + gt_folder: str, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[str, Path]] = None, + image_list=None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots=False, ): """Computes benchmark score for each instance of given images that are present both gt_project_name project and projects in folder_names list: @@ -1790,13 +1864,13 @@ def benchmark( @validate_arguments def consensus( - self, - project: NotEmptyStr, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[NotEmptyStr, Path]] = None, - image_list: Optional[List[NotEmptyStr]] = None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[NotEmptyStr, Path]] = None, + image_list: Optional[List[NotEmptyStr]] = None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots: Optional[StrictBool] = False, ): """Computes consensus score for each instance of given images that are present in at least 2 of the given projects: @@ -1844,10 +1918,10 @@ def consensus( @validate_arguments def run_prediction( - self, - project: Union[NotEmptyStr, dict], - images_list: List[NotEmptyStr], - model: Union[NotEmptyStr, dict], + self, + project: Union[NotEmptyStr, dict], + images_list: List[NotEmptyStr], + model: Union[NotEmptyStr, dict], ): """This function runs smart prediction on given list of images from a given project using the neural network of your choice @@ -1883,13 +1957,13 @@ def run_prediction( @validate_arguments def add_annotation_bbox_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - bbox: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + bbox: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, ): """Add a bounding box annotation to image annotations @@ -1943,13 +2017,13 @@ def add_annotation_bbox_to_image( @validate_arguments def add_annotation_point_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - point: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + point: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, ): """Add a point annotation to image annotations @@ -2001,13 +2075,13 @@ def add_annotation_point_to_image( @validate_arguments def add_annotation_comment_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - comment_text: NotEmptyStr, - comment_coords: List[float], - comment_author: EmailStr, - resolved: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + comment_text: NotEmptyStr, + comment_coords: List[float], + comment_author: EmailStr, + resolved: Optional[StrictBool] = False, ): """Add a comment to SuperAnnotate format annotation JSON @@ -2057,13 +2131,13 @@ def add_annotation_comment_to_image( @validate_arguments def upload_image_to_project( - self, - project: NotEmptyStr, - img, - image_name: Optional[NotEmptyStr] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[NotEmptyStr] = None, + self, + project: NotEmptyStr, + img, + image_name: Optional[NotEmptyStr] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[NotEmptyStr] = None, ): """Uploads image (io.BytesIO() or filepath to image) to project. Sets status of the uploaded image to set_status if it is not None. @@ -2099,28 +2173,28 @@ def upload_image_to_project( @validate_arguments def search_models( - self, - name: Optional[NotEmptyStr] = None, - type_: Optional[NotEmptyStr] = None, - project_id: Optional[int] = None, - task: Optional[NotEmptyStr] = None, - include_global: Optional[StrictBool] = True, + self, + name: Optional[NotEmptyStr] = None, + type_: Optional[NotEmptyStr] = None, + project_id: Optional[int] = None, + task: Optional[NotEmptyStr] = None, + include_global: Optional[StrictBool] = True, ): """Search for ML models. - :param name: search string - :type name: str - :param type_: ml model type string - :type type_: str - :param project_id: project id - :type project_id: int - :param task: training task - :type task: str - :param include_global: include global ml models - :type include_global: bool - - :return: ml model metadata - :rtype: list of dicts + :param name: search string + :type name: str + :param type_: ml model type string + :type type_: str + :param project_id: project id + :type project_id: int + :param task: training task + :type task: str + :param include_global: include global ml models + :type include_global: bool + + :return: ml model metadata + :rtype: list of dicts """ res = self.controller.search_models( name=name, @@ -2133,12 +2207,12 @@ def search_models( @validate_arguments def upload_images_to_project( - self, - project: NotEmptyStr, - img_paths: List[NotEmptyStr], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: NotEmptyStr, + img_paths: List[NotEmptyStr], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads all images given in list of path objects in img_paths to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -2176,14 +2250,17 @@ def upload_images_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) + "%s already existing images found that won't be uploaded.", + len(duplicates), ) logger.info(f"Uploading {len(images_to_upload)} images to project {project}.") uploaded, failed_images, duplications = [], [], duplicates if not images_to_upload: return uploaded, failed_images, duplications if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: for _ in use_case.execute(): progress_bar.update(1) uploaded, failed_images, duplications = use_case.data @@ -2194,10 +2271,10 @@ def upload_images_to_project( @validate_arguments def aggregate_annotations_as_df( - self, - project_root: Union[NotEmptyStr, Path], - project_type: ProjectTypes, - folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, + self, + project_root: Union[NotEmptyStr, Path], + project_type: ProjectTypes, + folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, ): """Aggregate annotations as pandas dataframe from project root. @@ -2215,8 +2292,8 @@ def aggregate_annotations_as_df( :rtype: pandas DataFrame """ if project_type in ( - constances.ProjectType.VECTOR.name, - constances.ProjectType.PIXEL.name, + constances.ProjectType.VECTOR.name, + constances.ProjectType.PIXEL.name, ): from superannotate.lib.app.analytics.common import ( aggregate_image_annotations_as_df, @@ -2230,8 +2307,8 @@ def aggregate_annotations_as_df( folder_names=folder_names, ) elif project_type in ( - constances.ProjectType.VIDEO.name, - constances.ProjectType.DOCUMENT.name, + constances.ProjectType.VIDEO.name, + constances.ProjectType.DOCUMENT.name, ): from superannotate.lib.app.analytics.aggregators import DataAggregator @@ -2243,8 +2320,7 @@ def aggregate_annotations_as_df( @validate_arguments def delete_annotations( - self, - project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None + self, project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None ): """ Delete image annotations from a given list of images. @@ -2265,20 +2341,19 @@ def delete_annotations( @validate_arguments def validate_annotations( - self, - project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] + self, project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] ): """Validates given annotation JSON. - :param project_type: The project type Vector, Pixel, Video or Document - :type project_type: str + :param project_type: The project type Vector, Pixel, Video or Document + :type project_type: str - :param annotations_json: path to annotation JSON - :type annotations_json: Path-like (str or Path) + :param annotations_json: path to annotation JSON + :type annotations_json: Path-like (str or Path) - :return: The success of the validation - :rtype: bool - """ + :return: The success of the validation + :rtype: bool + """ with open(annotations_json) as file: annotation_data = json.loads(file.read()) response = Controller.validate_annotations( @@ -2294,8 +2369,10 @@ def validate_annotations( @validate_arguments def add_contributors_to_project( - self, - project: NotEmptyStr, emails: conlist(EmailStr, min_items=1), role: AnnotatorRole + self, + project: NotEmptyStr, + emails: conlist(EmailStr, min_items=1), + role: AnnotatorRole, ) -> Tuple[List[str], List[str]]: """Add contributors to project. @@ -2320,8 +2397,7 @@ def add_contributors_to_project( @validate_arguments def invite_contributors_to_team( - self, - emails: conlist(EmailStr, min_items=1), admin: StrictBool = False + self, emails: conlist(EmailStr, min_items=1), admin: StrictBool = False ) -> Tuple[List[str], List[str]]: """Invites contributors to the team. @@ -2343,7 +2419,8 @@ def invite_contributors_to_team( @validate_arguments def get_annotations( - self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): + self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None + ): """Returns annotations for the given list of items. :param project: project name or folder path (e.g., “project1/folder1”). @@ -2356,16 +2433,15 @@ def get_annotations( :rtype: list of strs """ project_name, folder_name = extract_project_folder(project) - response = self.controller.get_annotations( - project_name, folder_name, items - ) + response = self.controller.get_annotations(project_name, folder_name, items) if response.errors: raise AppException(response.errors) return response.data @validate_arguments def get_annotations_per_frame( - self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1): + self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1 + ): """Returns per frame annotations for the given video. @@ -2391,8 +2467,7 @@ def get_annotations_per_frame( return response.data @validate_arguments - def upload_priority_scores( - self, project: NotEmptyStr, scores: List[PriorityScore]): + def upload_priority_scores(self, project: NotEmptyStr, scores: List[PriorityScore]): """Returns per frame annotations for the given video. :param project: project name or folder path (e.g., “project1/folder1”) @@ -2414,8 +2489,7 @@ def upload_priority_scores( return response.data @validate_arguments - def get_integrations( - self): + def get_integrations(self): """Get all integrations per team :return: metadata objects of all integrations of the team. @@ -2429,10 +2503,10 @@ def get_integrations( @validate_arguments def attach_items_from_integrated_storage( - self, - project: NotEmptyStr, - integration: Union[NotEmptyStr, IntegrationEntity], - folder_path: Optional[NotEmptyStr] = None, + self, + project: NotEmptyStr, + integration: Union[NotEmptyStr, IntegrationEntity], + folder_path: Optional[NotEmptyStr] = None, ): """Link images from integrated external storage to SuperAnnotate. @@ -2457,8 +2531,7 @@ def attach_items_from_integrated_storage( raise AppException(response.errors) @validate_arguments - def query( - self, project: NotEmptyStr, query: Optional[NotEmptyStr]): + def query(self, project: NotEmptyStr, query: Optional[NotEmptyStr]): """Return items that satisfy the given query. Query syntax should be in SuperAnnotate query language(https://doc.superannotate.com/docs/query-search-1). @@ -2479,8 +2552,9 @@ def query( @validate_arguments def get_item_metadata( - self, - project: NotEmptyStr, item_name: NotEmptyStr, + self, + project: NotEmptyStr, + item_name: NotEmptyStr, ): """Returns item metadata @@ -2501,13 +2575,13 @@ def get_item_metadata( @validate_arguments def search_items( - self, - project: NotEmptyStr, - name_contains: NotEmptyStr = None, - annotation_status: Optional[AnnotationStatuses] = None, - annotator_email: Optional[NotEmptyStr] = None, - qa_email: Optional[NotEmptyStr] = None, - recursive: bool = False, + self, + project: NotEmptyStr, + name_contains: NotEmptyStr = None, + annotation_status: Optional[AnnotationStatuses] = None, + annotator_email: Optional[NotEmptyStr] = None, + qa_email: Optional[NotEmptyStr] = None, + recursive: bool = False, ): """Search items by filtering criteria. @@ -2563,30 +2637,30 @@ def search_items( @validate_arguments def attach_items( - self, - project: Union[NotEmptyStr, dict], - attachments: AttachmentArg, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", + self, + project: Union[NotEmptyStr, dict], + attachments: AttachmentArg, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", ): """Link items from external storage to SuperAnnotate using URLs. - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str - :param attachments: path to CSV file or list of dicts containing attachments URLs. - :type attachments: path-like (str or Path) or list of dicts + :param attachments: path to CSV file or list of dicts containing attachments URLs. + :type attachments: path-like (str or Path) or list of dicts - :param annotation_status: value to set the annotation statuses of the linked items - “NotStarted” - “InProgress” - “QualityCheck” - “Returned” - “Completed” - “Skipped” - :type annotation_status: str + :param annotation_status: value to set the annotation statuses of the linked items + “NotStarted” + “InProgress” + “QualityCheck” + “Returned” + “Completed” + “Skipped” + :type annotation_status: str - :return: None - """ + :return: None + """ attachments = attachments.data project_name, folder_name = extract_project_folder(project) if attachments and isinstance(attachments[0], AttachmentDict): @@ -2597,9 +2671,10 @@ def attach_items( if count > 1 ] else: - unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv( - attachments - ) + ( + unique_attachments, + duplicate_attachments, + ) = get_name_url_duplicated_from_csv(attachments) if duplicate_attachments: logger.info("Dropping duplicates.") unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments) @@ -2627,11 +2702,11 @@ def attach_items( @validate_arguments def copy_items( - self, - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, - include_annotations: Optional[StrictBool] = True, + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, + include_annotations: Optional[StrictBool] = True, ): """Copy images in bulk between folders in a project @@ -2671,10 +2746,10 @@ def copy_items( @validate_arguments def move_items( - self, - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, ): """Move images in bulk between folders in a project @@ -2707,10 +2782,10 @@ def move_items( @validate_arguments def set_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: AnnotationStatuses, - item_names: Optional[List[NotEmptyStr]] = None, + self, + project: Union[NotEmptyStr, dict], + annotation_status: AnnotationStatuses, + item_names: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of items @@ -2743,12 +2818,12 @@ def set_annotation_statuses( @validate_arguments def download_annotations( - self, - project: Union[NotEmptyStr, dict], - path: Union[str, Path] = None, - items: Optional[List[NotEmptyStr]] = None, - recursive: bool = False, - callback: Callable = None, + self, + project: Union[NotEmptyStr, dict], + path: Union[str, Path] = None, + items: Optional[List[NotEmptyStr]] = None, + recursive: bool = False, + callback: Callable = None, ): """Downloads annotation JSON files of the selected items to the local directory. diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index ea120e4ea..1e06b7cf6 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -171,6 +171,16 @@ def assign_images( ): raise NotImplementedError + def assign_items( + self, + team_id: int, + project_id: int, + folder_name: str, + user: str, + item_names: list, + ): + raise NotImplementedError + def get_bulk_images( self, project_id: int, team_id: int, folder_id: int, images: List[str] ) -> List[dict]: @@ -191,6 +201,11 @@ def un_assign_images( ): raise NotImplementedError + def un_assign_items( + self, team_id: int, project_id: int, folder_name: str, item_names: list, + ): + raise NotImplementedError + def un_share_project( self, team_id: int, project_id: int, user_id: str, ): diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 0bb2fa294..4e32a6ffc 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -187,6 +187,76 @@ def execute(self) -> Response: return self._response +class AssignItemsUseCase(BaseUseCase): + CHUNK_SIZE = 500 + + def __init__( + self, + service: SuperannotateServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + item_names: list, + user: str, + ): + super().__init__() + self._project = project + self._folder = folder + self._item_names = item_names + self._user = user + self._service = service + + def execute(self): + if self.is_valid(): + for i in range(0, len(self._image_names), self.CHUNK_SIZE): + is_assigned = self._service.assign_items( + team_id=self._project.team_id, + project_id=self._project.id, + folder_name=self._folder.name, + user=self._user, + item_names=self._item_names[ + i : i + self.CHUNK_SIZE # noqa: E203 + ], + ) + if not is_assigned: + self._response.errors = AppException( + f"Cant assign {', '.join(self._item_names[i: i + self.CHUNK_SIZE])}" + ) + continue + return self._response + + +class UnAssignItemsUseCase(BaseUseCase): + CHUNK_SIZE = 500 + + def __init__( + self,a + service: SuperannotateServiceProvider, + project_entity: ProjectEntity, + folder: FolderEntity, + item_names: list, + ): + super().__init__() + self._project_entity = project_entity + self._folder = folder + self._item_names = item_names + self._service = service + + def execute(self): + # todo handling to backend side + for i in range(0, len(self._item_names, self.CHUNK_SIZE): + is_un_assigned = self._service.un_assign_items( + team_id=self._project_entity.team_id, + project_id=self._project_entity.id, + folder_name=self._folder.name, + item_names=self._item_names[i : i + self.CHUNK_SIZE], # noqa: E203 + ) + if not is_un_assigned: + self._response.errors = AppException( + f"Cant un assign {', '.join(self._item_names[i: i + self.CHUNK_SIZE])}" + ) + + return self._response + class AttachItems(BaseReportableUseCae): CHUNK_SIZE = 500 diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f54652fed..ff7099d2a 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -168,26 +168,27 @@ def team_id(self) -> int: @staticmethod def get_default_reporter( - log_info: bool = True, - log_warning: bool = True, - disable_progress_bar: bool = False, - log_debug: bool = True + log_info: bool = True, + log_warning: bool = True, + disable_progress_bar: bool = False, + log_debug: bool = True, ) -> Reporter: import inspect - session = None - loop_limit = 16 - current_frame = inspect.currentframe() - while loop_limit: - loop_limit -= 1 - try: - session = current_frame.f_locals['session'] - if session: - break - except KeyError: - pass - finally: - current_frame = current_frame.f_back - return Reporter(log_info, log_warning, disable_progress_bar, log_debug, session) + + # session = None + # loop_limit = 16 + # current_frame = inspect.currentframe() + # while loop_limit: + # loop_limit -= 1 + # try: + # session = current_frame.f_locals["session"] + # if session: + # break + # except KeyError: + # pass + # finally: + # current_frame = current_frame.f_back + return Reporter(log_info, log_warning, disable_progress_bar, log_debug) @timed_lru_cache(seconds=3600) def get_auth_data(self, project_id: int, team_id: int, folder_id: int): @@ -255,11 +256,11 @@ def get_folder_name(name: str = None): return "root" def search_project( - self, - name: str = None, - include_complete_image_count=False, - statuses: Union[List[str], Tuple[str]] = (), - **kwargs, + self, + name: str = None, + include_complete_image_count=False, + statuses: Union[List[str], Tuple[str]] = (), + **kwargs, ) -> Response: condition = Condition.get_empty_condition() if name: @@ -275,19 +276,21 @@ def search_project( condition &= build_condition(**kwargs) use_case = usecases.GetProjectsUseCase( - condition=condition, projects=self.projects, team_id=self.team_id, + condition=condition, + projects=self.projects, + team_id=self.team_id, ) return use_case.execute() def create_project( - self, - name: str, - description: str, - project_type: str, - settings: Iterable[SettingEntity] = None, - classes: Iterable = tuple(), - workflows: Iterable = tuple(), - **extra_kwargs, + self, + name: str, + description: str, + project_type: str, + settings: Iterable[SettingEntity] = None, + classes: Iterable = tuple(), + workflows: Iterable = tuple(), + **extra_kwargs, ) -> Response: try: @@ -320,7 +323,9 @@ def create_project( def delete_project(self, name: str): use_case = usecases.DeleteProjectUseCase( - project_name=name, team_id=self.team_id, projects=self.projects, + project_name=name, + team_id=self.team_id, + projects=self.projects, ) return use_case.execute() @@ -330,14 +335,14 @@ def update_project(self, name: str, project_data: dict) -> Response: return use_case.execute() def upload_image_to_project( - self, - project_name: str, - folder_name: str, - image_name: str, - image: Union[str, io.BytesIO] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + image_name: str, + image: Union[str, io.BytesIO] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -365,13 +370,13 @@ def upload_image_to_project( ).execute() def upload_images_to_project( - self, - project_name: str, - folder_name: str, - paths: List[str], - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + paths: List[str], + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -391,16 +396,16 @@ def upload_images_to_project( ) def upload_images_from_folder_to_project( - self, - project_name: str, - folder_name: str, - folder_path: str, - extensions: Optional[List[str]] = None, - annotation_status: str = None, - exclude_file_patterns: Optional[List[str]] = None, - recursive_sub_folders: Optional[bool] = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + folder_path: str, + extensions: Optional[List[str]] = None, + annotation_status: str = None, + exclude_file_patterns: Optional[List[str]] = None, + recursive_sub_folders: Optional[bool] = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -423,20 +428,22 @@ def upload_images_from_folder_to_project( ) def clone_project( - self, - name: str, - from_name: str, - project_description: str, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, + self, + name: str, + from_name: str, + project_description: str, + copy_annotation_classes=True, + copy_settings=True, + copy_workflow=True, + copy_contributors=False, ): project = self._get_project(from_name) project_to_create = copy.copy(project) reporter = self.get_default_reporter() - reporter.track("external", project.upload_state == constances.UploadState.EXTERNAL.value) + reporter.track( + "external", project.upload_state == constances.UploadState.EXTERNAL.value + ) project_to_create.name = name if project_description is not None: project_to_create.description = project_description @@ -457,12 +464,12 @@ def clone_project( return use_case.execute() def interactive_attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, + self, + project_name: str, + files: List[ImageEntity], + folder_name: str = None, + annotation_status: str = None, + upload_state_code: int = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -482,7 +489,9 @@ def create_folder(self, project: str, folder_name: str): name=folder_name, project_id=project.id, team_id=project.team_id ) use_case = usecases.CreateFolderUseCase( - project=project, folder=folder, folders=self.folders, + project=project, + folder=folder, + folders=self.folders, ) return use_case.execute() @@ -497,7 +506,7 @@ def get_folder(self, project_name: str, folder_name: str): return use_case.execute() def search_folders( - self, project_name: str, folder_name: str = None, include_users=False, **kwargs + self, project_name: str, folder_name: str = None, include_users=False, **kwargs ): condition = build_condition(**kwargs) project = self._get_project(project_name) @@ -524,12 +533,12 @@ def delete_folders(self, project_name: str, folder_names: List[str]): return use_case.execute() def prepare_export( - self, - project_name: str, - folder_names: List[str], - include_fuse: bool, - only_pinned: bool, - annotation_statuses: List[str] = None, + self, + project_name: str, + folder_names: List[str], + include_fuse: bool, + only_pinned: bool, + annotation_statuses: List[str] = None, ): project = self._get_project(project_name) @@ -554,11 +563,11 @@ def search_team_contributors(self, **kwargs): return use_case.execute() def search_images( - self, - project_name: str, - folder_path: str = None, - annotation_status: str = None, - image_name_prefix: str = None, + self, + project_name: str, + folder_path: str = None, + annotation_status: str = None, + image_name_prefix: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -573,7 +582,10 @@ def search_images( return use_case.execute() def _get_image( - self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, + self, + project: ProjectEntity, + image_name: str, + folder: FolderEntity = None, ) -> ImageEntity: response = usecases.GetImageUseCase( service=self._backend_client, @@ -587,7 +599,7 @@ def _get_image( return response.data def get_image( - self, project_name: str, image_name: str, folder_path: str = None + self, project_name: str, image_name: str, folder_path: str = None ) -> ImageEntity: project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -598,18 +610,21 @@ def update_folder(self, project_name: str, folder_name: str, folder_data: dict): folder = self._get_folder(project, folder_name) for field, value in folder_data.items(): setattr(folder, field, value) - use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder, ) + use_case = usecases.UpdateFolderUseCase( + folders=self.folders, + folder=folder, + ) return use_case.execute() def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, + copy_annotation_status: bool = False, + move: bool = False, ): from_project = self._get_project(from_project_name) to_project = self._get_project(to_project_name) @@ -632,12 +647,12 @@ def copy_image( return use_case.execute() def copy_image_annotation_classes( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, ): from_project = self._get_project(from_project_name) from_folder = self._get_folder(from_project, from_folder_name) @@ -672,7 +687,7 @@ def copy_image_annotation_classes( return use_case.execute() def update_image( - self, project_name: str, image_name: str, folder_name: str = None, **kwargs + self, project_name: str, image_name: str, folder_name: str = None, **kwargs ): image = self.get_image( project_name=project_name, image_name=image_name, folder_path=folder_name @@ -683,13 +698,13 @@ def update_image( return use_case.execute() def bulk_copy_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], - include_annotations: bool, - include_pin: bool, + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], + include_annotations: bool, + include_pin: bool, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -706,11 +721,11 @@ def bulk_copy_images( return use_case.execute() def bulk_move_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -725,13 +740,13 @@ def bulk_move_images( return use_case.execute() def get_project_metadata( - self, - project_name: str, - include_annotation_classes: bool = False, - include_settings: bool = False, - include_workflow: bool = False, - include_contributors: bool = False, - include_complete_image_count: bool = False, + self, + project_name: str, + include_annotation_classes: bool = False, + include_settings: bool = False, + include_workflow: bool = False, + include_contributors: bool = False, + include_complete_image_count: bool = False, ): project = self._get_project(project_name) @@ -806,11 +821,11 @@ def set_project_settings(self, project_name: str, new_settings: List[dict]): return use_case.execute() def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, + self, + project_name: str, + folder_name: str, + image_names: list, + annotation_status: str, ): project_entity = self._get_project(project_name) folder_entity = self._get_folder(project_entity, folder_name) @@ -828,7 +843,10 @@ def set_images_annotation_statuses( return use_case.execute() def delete_images( - self, project_name: str, folder_name: str, image_names: List[str] = None, + self, + project_name: str, + folder_name: str, + image_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -842,8 +860,33 @@ def delete_images( ) return use_case.execute() + def assign_items( + self, project_name: str, folder_name: str, item_names: list, user: str + ): + project_entity = self._get_project(project_name) + folder = self._get_folder(project_entity, folder_name) + use_case = usecases.AssignItemsUseCase( + project=project_entity, + service=self._backend_client, + folder=folder, + item_names=item_names, + user=user, + ) + return use_case.execute() + + def un_assign_items(self, project_name, folder_name, item_names): + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + use_case = usecases.UnAssignItemsUseCase( + project_entity=project, + service=self._backend_client, + folder=folder, + item_names=item_names, + ) + return use_case.execute() + def assign_images( - self, project_name: str, folder_name: str, image_names: list, user: str + self, project_name: str, folder_name: str, image_names: list, user: str ): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) @@ -871,7 +914,9 @@ def un_assign_folder(self, project_name: str, folder_name: str): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) use_case = usecases.UnAssignFolderUseCase( - service=self._backend_client, project_entity=project_entity, folder=folder, + service=self._backend_client, + project_entity=project_entity, + folder=folder, ) return use_case.execute() @@ -896,7 +941,7 @@ def un_share_project(self, project_name: str, user_id: str): return use_case.execute() def download_image_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -924,7 +969,7 @@ def get_exports(self, project_name: str, return_metadata: bool): return use_case.execute() def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool + self, project_name: str, folder_name: str, with_all_subfolders: bool ): project = self._get_project(project_name) @@ -940,12 +985,12 @@ def get_project_image_count( return use_case.execute() def create_annotation_class( - self, - project_name: str, - name: str, - color: str, - attribute_groups: List[dict], - class_type: str, + self, + project_name: str, + name: str, + color: str, + attribute_groups: List[dict], + class_type: str, ): project = self._get_project(project_name) annotation_classes = AnnotationClassRepository( @@ -966,7 +1011,8 @@ def delete_annotation_class(self, project_name: str, annotation_class_name: str) use_case = usecases.DeleteAnnotationClassUseCase( annotation_class_name=annotation_class_name, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), project_name=project_name, ) @@ -977,7 +1023,8 @@ def get_annotation_class(self, project_name: str, annotation_class_name: str): use_case = usecases.GetAnnotationClassUseCase( annotation_class_name=annotation_class_name, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), ) return use_case.execute() @@ -986,7 +1033,8 @@ def download_annotation_classes(self, project_name: str, download_path: str): project = self._get_project(project_name) use_case = usecases.DownloadAnnotationClassesUseCase( annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), download_path=download_path, project_name=project_name, @@ -999,7 +1047,8 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) use_case = usecases.CreateAnnotationClassesUseCase( service=self._backend_client, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), annotation_classes=annotation_classes, project=project, @@ -1007,15 +1056,15 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) return use_case.execute() def download_image( - self, - project_name: str, - image_name: str, - download_path: str, - folder_name: str = None, - image_variant: str = None, - include_annotations: bool = None, - include_fuse: bool = None, - include_overlay: bool = None, + self, + project_name: str, + image_name: str, + download_path: str, + folder_name: str = None, + image_variant: str = None, + include_annotations: bool = None, + include_fuse: bool = None, + include_overlay: bool = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1055,13 +1104,13 @@ def set_project_workflow(self, project_name: str, steps: list): return use_case.execute() def upload_annotations_from_folder( - self, - project_name: str, - folder_name: str, - annotation_paths: List[str], - client_s3_bucket=None, - is_pre_annotations: bool = False, - folder_path: str = None, + self, + project_name: str, + folder_name: str, + annotation_paths: List[str], + client_s3_bucket=None, + is_pre_annotations: bool = False, + folder_path: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1087,13 +1136,13 @@ def upload_annotations_from_folder( return use_case.execute() def upload_image_annotations( - self, - project_name: str, - folder_name: str, - image_name: str, - annotations: dict, - mask: io.BytesIO = None, - verbose: bool = True, + self, + project_name: str, + folder_name: str, + image_name: str, + annotations: dict, + mask: io.BytesIO = None, + verbose: bool = True, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1135,12 +1184,12 @@ def delete_model(self, model_id: int): return use_case.execute() def download_export( - self, - project_name: str, - export_name: str, - folder_path: str, - extract_zip_contents: bool, - to_s3_bucket: bool, + self, + project_name: str, + export_name: str, + folder_path: str, + extract_zip_contents: bool, + to_s3_bucket: bool, ): project = self._get_project(project_name) use_case = usecases.DownloadExportUseCase( @@ -1173,14 +1222,14 @@ def download_ml_model(self, model_data: dict, download_path: str): return use_case.execute() def benchmark( - self, - project_name: str, - ground_truth_folder_name: str, - folder_names: List[str], - export_root: str, - image_list: List[str], - annot_type: str, - show_plots: bool, + self, + project_name: str, + ground_truth_folder_name: str, + folder_names: List[str], + export_root: str, + image_list: List[str], + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) export_response = self.prepare_export( @@ -1215,13 +1264,13 @@ def benchmark( return use_case.execute() def consensus( - self, - project_name: str, - folder_names: list, - export_path: str, - image_list: list, - annot_type: str, - show_plots: bool, + self, + project_name: str, + folder_names: list, + export_path: str, + image_list: list, + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1254,7 +1303,7 @@ def consensus( return use_case.execute() def run_prediction( - self, project_name: str, images_list: list, model_name: str, folder_name: str + self, project_name: str, images_list: list, model_name: str, folder_name: str ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1272,7 +1321,10 @@ def run_prediction( return use_case.execute() def list_images( - self, project_name: str, annotation_status: str = None, name_prefix: str = None, + self, + project_name: str, + annotation_status: str = None, + name_prefix: str = None, ): project = self._get_project(project_name) @@ -1285,12 +1337,12 @@ def list_images( return use_case.execute() def search_models( - self, - name: str, - model_type: str = None, - project_id: int = None, - task: str = None, - include_global: bool = True, + self, + name: str, + model_type: str = None, + project_id: int = None, + task: str = None, + include_global: bool = True, ): ml_models_repo = MLModelRepository( service=self._backend_client, team_id=self.team_id @@ -1313,10 +1365,10 @@ def search_models( return use_case.execute() def delete_annotations( - self, - project_name: str, - folder_name: str, - item_names: Optional[List[str]] = None, + self, + project_name: str, + folder_name: str, + item_names: Optional[List[str]] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1330,7 +1382,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = False ): use_case = usecases.ValidateAnnotationUseCase( project_type, @@ -1367,17 +1419,17 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): return use_case.execute() def upload_videos( - self, - project_name: str, - folder_name: str, - paths: List[str], - start_time: float, - extensions: List[str] = None, - exclude_file_patterns: List[str] = None, - end_time: Optional[float] = None, - target_fps: Optional[int] = None, - annotation_status: Optional[str] = None, - image_quality_in_editor: Optional[str] = None, + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1403,7 +1455,7 @@ def upload_videos( return use_case.execute() def get_annotations( - self, project_name: str, folder_name: str, item_names: List[str], logging=True + self, project_name: str, folder_name: str, item_names: List[str], logging=True ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1418,7 +1470,7 @@ def get_annotations( return use_case.execute() def get_annotations_per_frame( - self, project_name: str, folder_name: str, video_name: str, fps: int + self, project_name: str, folder_name: str, video_name: str, fps: int ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1435,7 +1487,7 @@ def get_annotations_per_frame( return use_case.execute() def upload_priority_scores( - self, project_name, folder_name, scores, project_folder_name + self, project_name, folder_name, scores, project_folder_name ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1459,11 +1511,11 @@ def get_integrations(self): return use_cae.execute() def attach_integrations( - self, - project_name: str, - folder_name: str, - integration: IntegrationEntity, - folder_path: str, + self, + project_name: str, + folder_name: str, + integration: IntegrationEntity, + folder_path: str, ): team = self.team_data.data project = self._get_project(project_name) @@ -1506,15 +1558,15 @@ def get_item(self, project_name: str, folder_name: str, item_name: str): return use_case.execute() def list_items( - self, - project_name: str, - folder_name: str, - name_contains: str = None, - annotation_status: str = None, - annotator_email: str = None, - qa_email: str = None, - recursive: bool = False, - **kwargs, + self, + project_name: str, + folder_name: str, + name_contains: str = None, + annotation_status: str = None, + annotator_email: str = None, + qa_email: str = None, + recursive: bool = False, + **kwargs, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1546,11 +1598,11 @@ def list_items( return use_case.execute() def attach_items( - self, - project_name: str, - folder_name: str, - attachments: List[AttachmentEntity], - annotation_status: str, + self, + project_name: str, + folder_name: str, + attachments: List[AttachmentEntity], + annotation_status: str, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1566,12 +1618,12 @@ def attach_items( return use_case.execute() def copy_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, - include_annotations: bool = False, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, + include_annotations: bool = False, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) @@ -1590,11 +1642,11 @@ def copy_items( return use_case.execute() def move_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) @@ -1612,11 +1664,11 @@ def move_items( return use_case.execute() def set_annotation_statuses( - self, - project_name: str, - folder_name: str, - annotation_status: str, - item_names: List[str] = None, + self, + project_name: str, + folder_name: str, + annotation_status: str, + item_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 9d27ae698..507d101e8 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -215,6 +215,7 @@ class SuperannotateBackendService(BaseBackendService): URL_MOVE_IMAGES_FROM_FOLDER = "image/move" URL_GET_COPY_PROGRESS = "images/copy-image-progress" URL_ASSIGN_IMAGES = "images/editAssignment/" + URL_ASSIGN_ITEMS = "images/editAssignment/" URL_ASSIGN_FOLDER = "folder/editAssignment" URL_GET_EXPORTS = "exports" URL_GET_CLASS = "class/{}" @@ -748,6 +749,43 @@ def un_assign_images( ) return res.ok + def assign_items( + self, + team_id: int, + project_id: int, + folder_name: str, + user: str, + item_names: list, + ): + assign_items_url = urljoin(self.api_url, self.URL_ASSIGN_ITEMS) + res = self._request( + assign_items_url, + "put", + params={"team_id": team_id, "project_id": project_id}, + data={ + "image_names": item_names, + "assign_user_id": user, + "folder_name": folder_name, + }, + ) + return res.ok + + def un_assign_items( + self, team_id: int, project_id: int, folder_name: str, item_names: List[str], + ): + un_assign_items_url = urljoin(self.api_url, self.URL_ASSIGN_ITEMS) + res = self._request( + un_assign_items_url, + "put", + params={"team_id": team_id, "project_id": project_id}, + data={ + "image_names": item_names, + "remove_user_ids": ["all"], + "folder_name": folder_name, + }, + ) + return res.ok + def un_assign_folder( self, team_id: int, project_id: int, folder_name: str, ): From 1e24db4053f47c480c7e7becfcade960d7bf76e5 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 30 May 2022 12:15:24 +0400 Subject: [PATCH 06/59] Added mixpanel, changed docs --- .pre-commit-config.yaml | 2 +- docs/source/superannotate.sdk.rst | 140 +-- pytest.ini | 2 +- src/superannotate/__init__.py | 20 +- .../lib/app/analytics/class_analytics.py | 10 +- src/superannotate/lib/app/common.py | 10 +- .../lib/app/input_converters/__init__.py | 12 + .../lib/app/input_converters/conversion.py | 79 +- .../coco_converters/coco_converter.py | 6 +- .../lib/app/interface/base_interface.py | 163 ++-- .../lib/app/interface/cli_interface.py | 89 +- .../lib/app/interface/sdk_interface.py | 892 ++++++++---------- src/superannotate/lib/app/mixp/decorators.py | 134 --- src/superannotate/lib/core/__init__.py | 7 +- src/superannotate/lib/core/data_handlers.py | 2 +- .../lib/core/entities/project_entities.py | 10 +- src/superannotate/lib/core/plugin.py | 5 +- src/superannotate/lib/core/reporter.py | 27 +- src/superannotate/lib/core/repositories.py | 6 +- .../lib/core/serviceproviders.py | 22 +- .../lib/core/usecases/annotations.py | 31 +- src/superannotate/lib/core/usecases/base.py | 4 +- .../lib/core/usecases/folders.py | 4 +- src/superannotate/lib/core/usecases/images.py | 19 +- .../lib/core/usecases/integrations.py | 6 +- src/superannotate/lib/core/usecases/items.py | 32 +- src/superannotate/lib/core/usecases/models.py | 8 +- .../lib/core/usecases/projects.py | 27 +- .../lib/infrastructure/controller.py | 523 +++++----- .../lib/infrastructure/services.py | 21 +- .../lib/infrastructure/stream_data_handler.py | 11 +- .../test_annotations_pre_processing.py | 2 +- .../annotations/test_download_annotations.py | 11 + .../annotations/test_get_annotations.py | 2 +- .../mixpanel/test_individual_fuinctions.py | 30 + .../mixpanel/test_mixpanel_decorator.py | 139 +++ tests/integration/test_assign_images.py | 0 .../integration/test_attach_document_urls.py | 17 - tests/integration/test_attach_video_urls.py | 0 tests/unit/test_validators.py | 4 +- 40 files changed, 1310 insertions(+), 1219 deletions(-) delete mode 100644 src/superannotate/lib/app/mixp/decorators.py create mode 100644 tests/integration/mixpanel/test_individual_fuinctions.py create mode 100644 tests/integration/mixpanel/test_mixpanel_decorator.py delete mode 100644 tests/integration/test_assign_images.py delete mode 100644 tests/integration/test_attach_document_urls.py delete mode 100644 tests/integration/test_attach_video_urls.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3982e77cc..b65ed2b11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - '--application-directories' - app - repo: 'https://github.com/python/black' - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black name: Code Formatter (black) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 224765118..3d2888431 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -11,7 +11,7 @@ Remote functions Initialization and authentication _________________________________ -.. autofunction:: superannotate.init +.. automethod:: superannotate.SAClient.__init__ .. _ref_projects: @@ -20,61 +20,61 @@ Projects ________ .. _ref_search_projects: -.. autofunction:: superannotate.search_projects -.. autofunction:: superannotate.create_project -.. autofunction:: superannotate.create_project_from_metadata -.. autofunction:: superannotate.clone_project -.. autofunction:: superannotate.delete_project -.. autofunction:: superannotate.rename_project +.. automethod:: superannotate.SAClient.search_projects +.. automethod:: superannotate.SAClient.create_project +.. automethod:: superannotate.SAClient.create_project_from_metadata +.. automethod:: superannotate.SAClient.clone_project +.. automethod:: superannotate.SAClient.delete_project +.. automethod:: superannotate.SAClient.rename_project .. _ref_get_project_metadata: -.. autofunction:: superannotate.get_project_metadata -.. autofunction:: superannotate.get_project_image_count -.. autofunction:: superannotate.search_folders -.. autofunction:: superannotate.get_folder_metadata -.. autofunction:: superannotate.create_folder -.. autofunction:: superannotate.delete_folders -.. autofunction:: superannotate.upload_images_to_project -.. autofunction:: superannotate.attach_items_from_integrated_storage -.. autofunction:: superannotate.upload_image_to_project -.. autofunction:: superannotate.delete_annotations +.. automethod:: superannotate.SAClient.get_project_metadata +.. automethod:: superannotate.SAClient.get_project_image_count +.. automethod:: superannotate.SAClient.search_folders +.. automethod:: superannotate.SAClient.get_folder_metadata +.. automethod:: superannotate.SAClient.create_folder +.. automethod:: superannotate.SAClient.delete_folders +.. automethod:: superannotate.SAClient.upload_images_to_project +.. automethod:: superannotate.SAClient.attach_items_from_integrated_storage +.. automethod:: superannotate.SAClient.upload_image_to_project +.. automethod:: superannotate.SAClient.delete_annotations .. _ref_upload_images_from_folder_to_project: -.. autofunction:: superannotate.upload_images_from_folder_to_project -.. autofunction:: superannotate.upload_video_to_project -.. autofunction:: superannotate.upload_videos_from_folder_to_project +.. automethod:: superannotate.SAClient.upload_images_from_folder_to_project +.. automethod:: superannotate.SAClient.upload_video_to_project +.. automethod:: superannotate.SAClient.upload_videos_from_folder_to_project .. _ref_upload_annotations_from_folder_to_project: -.. autofunction:: superannotate.upload_annotations_from_folder_to_project -.. autofunction:: superannotate.upload_preannotations_from_folder_to_project -.. autofunction:: superannotate.add_contributors_to_project -.. autofunction:: superannotate.get_project_settings -.. autofunction:: superannotate.set_project_default_image_quality_in_editor -.. autofunction:: superannotate.get_project_workflow -.. autofunction:: superannotate.set_project_workflow +.. automethod:: superannotate.SAClient.upload_annotations_from_folder_to_project +.. automethod:: superannotate.SAClient.upload_preannotations_from_folder_to_project +.. automethod:: superannotate.SAClient.add_contributors_to_project +.. automethod:: superannotate.SAClient.get_project_settings +.. automethod:: superannotate.SAClient.set_project_default_image_quality_in_editor +.. automethod:: superannotate.SAClient.get_project_workflow +.. automethod:: superannotate.SAClient.set_project_workflow ---------- Exports _______ -.. autofunction:: superannotate.prepare_export -.. autofunction:: superannotate.get_annotations -.. autofunction:: superannotate.get_annotations_per_frame +.. automethod:: superannotate.SAClient.prepare_export +.. automethod:: superannotate.SAClient.get_annotations +.. automethod:: superannotate.SAClient.get_annotations_per_frame .. _ref_download_export: -.. autofunction:: superannotate.download_export -.. autofunction:: superannotate.get_exports +.. automethod:: superannotate.SAClient.download_export +.. automethod:: superannotate.SAClient.get_exports ---------- Items ______ -.. autofunction:: superannotate.query -.. autofunction:: superannotate.search_items -.. autofunction:: superannotate.download_annotations -.. autofunction:: superannotate.attach_items -.. autofunction:: superannotate.copy_items -.. autofunction:: superannotate.move_items -.. autofunction:: superannotate.get_item_metadata -.. autofunction:: superannotate.set_annotation_statuses +.. automethod:: superannotate.SAClient.query +.. automethod:: superannotate.SAClient.search_items +.. automethod:: superannotate.SAClient.download_annotations +.. automethod:: superannotate.SAClient.attach_items +.. automethod:: superannotate.SAClient.copy_items +.. automethod:: superannotate.SAClient.move_items +.. automethod:: superannotate.SAClient.get_item_metadata +.. automethod:: superannotate.SAClient.set_annotation_statuses ---------- @@ -83,50 +83,50 @@ ______ .. _ref_search_images: -.. autofunction:: superannotate.download_image -.. autofunction:: superannotate.set_image_annotation_status -.. autofunction:: superannotate.set_images_annotation_statuses -.. autofunction:: superannotate.download_image_annotations -.. autofunction:: superannotate.upload_image_annotations -.. autofunction:: superannotate.copy_image -.. autofunction:: superannotate.pin_image -.. autofunction:: superannotate.assign_images -.. autofunction:: superannotate.delete_images -.. autofunction:: superannotate.add_annotation_bbox_to_image -.. autofunction:: superannotate.add_annotation_point_to_image -.. autofunction:: superannotate.add_annotation_comment_to_image -.. autofunction:: superannotate.upload_priority_scores +.. automethod:: superannotate.SAClient.download_image +.. automethod:: superannotate.SAClient.set_image_annotation_status +.. automethod:: superannotate.SAClient.set_images_annotation_statuses +.. automethod:: superannotate.SAClient.download_image_annotations +.. automethod:: superannotate.SAClient.upload_image_annotations +.. automethod:: superannotate.SAClient.copy_image +.. automethod:: superannotate.SAClient.pin_image +.. automethod:: superannotate.SAClient.assign_images +.. automethod:: superannotate.SAClient.delete_images +.. automethod:: superannotate.SAClient.add_annotation_bbox_to_image +.. automethod:: superannotate.SAClient.add_annotation_point_to_image +.. automethod:: superannotate.SAClient.add_annotation_comment_to_image +.. automethod:: superannotate.SAClient.upload_priority_scores ---------- Annotation Classes __________________ -.. autofunction:: superannotate.create_annotation_class +.. automethod:: superannotate.SAClient.create_annotation_class .. _ref_create_annotation_classes_from_classes_json: -.. autofunction:: superannotate.create_annotation_classes_from_classes_json -.. autofunction:: superannotate.search_annotation_classes -.. autofunction:: superannotate.download_annotation_classes_json -.. autofunction:: superannotate.delete_annotation_class +.. automethod:: superannotate.SAClient.create_annotation_classes_from_classes_json +.. automethod:: superannotate.SAClient.search_annotation_classes +.. automethod:: superannotate.SAClient.download_annotation_classes_json +.. automethod:: superannotate.SAClient.delete_annotation_class ---------- Team _________________ -.. autofunction:: superannotate.get_team_metadata -.. autofunction:: superannotate.get_integrations -.. autofunction:: superannotate.invite_contributors_to_team -.. autofunction:: superannotate.search_team_contributors +.. automethod:: superannotate.SAClient.get_team_metadata +.. automethod:: superannotate.SAClient.get_integrations +.. automethod:: superannotate.SAClient.invite_contributors_to_team +.. automethod:: superannotate.SAClient.search_team_contributors ---------- Neural Network _______________ -.. autofunction:: superannotate.download_model -.. autofunction:: superannotate.run_prediction -.. autofunction:: superannotate.search_models +.. automethod:: superannotate.SAClient.download_model +.. automethod:: superannotate.SAClient.run_prediction +.. automethod:: superannotate.SAClient.search_models ---------- @@ -196,7 +196,7 @@ Export metadata example: Integration metadata -_______________ +______________________ Integration metadata example: @@ -383,8 +383,8 @@ Working with annotations ________________________ .. _ref_aggregate_annotations_as_df: -.. autofunction:: superannotate.validate_annotations -.. autofunction:: superannotate.aggregate_annotations_as_df +.. automethod:: superannotate.SAClient.validate_annotations +.. automethod:: superannotate.SAClient.aggregate_annotations_as_df ---------- @@ -398,5 +398,5 @@ _____________________________________________________________ Utility functions -------------------------------- -.. autofunction:: superannotate.consensus -.. autofunction:: superannotate.benchmark \ No newline at end of file +.. autofunction:: superannotate.SAClient.consensus +.. autofunction:: superannotate.SAClient.benchmark \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index de724250b..86c2d4c63 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.7 log_cli=true python_files = test_*.py -;addopts = -n auto --dist=loadscope +addopts = -n auto --dist=loadscope \ No newline at end of file diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 448a787a5..f6bfa0eed 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -1,5 +1,6 @@ import os import sys + sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import logging.config # noqa @@ -7,10 +8,10 @@ from packaging.version import parse # noqa from superannotate.lib.app.analytics.class_analytics import class_distribution # noqa from superannotate.lib.app.exceptions import AppException # noqa -from superannotate.lib.app.input_converters.conversion import convert_json_version # noqa -from superannotate.lib.app.input_converters.conversion import convert_project_type # noqa -from superannotate.lib.app.input_converters.conversion import export_annotation # noqa -from superannotate.lib.app.input_converters.conversion import import_annotation # noqa +from superannotate.lib.app.input_converters import convert_json_version +from superannotate.lib.app.input_converters import convert_project_type # noqa +from superannotate.lib.app.input_converters import export_annotation # noqa +from superannotate.lib.app.input_converters import import_annotation # noqa from superannotate.lib.app.interface.sdk_interface import SAClient # noqa from superannotate.lib.core import PACKAGE_VERSION_INFO_MESSAGE # noqa from superannotate.lib.core import PACKAGE_VERSION_MAJOR_UPGRADE # noqa @@ -18,6 +19,9 @@ from superannotate.logger import get_default_logger # noqa from superannotate.version import __version__ # noqa + +SESSIONS = {} + __all__ = [ "__version__", "SAClient", @@ -52,14 +56,10 @@ def log_version_info(): pip_version = max(pip_version, ver) if pip_version.major > local_version.major: logger.warning( - PACKAGE_VERSION_MAJOR_UPGRADE.format( - local_version, pip_version - ) + PACKAGE_VERSION_MAJOR_UPGRADE.format(local_version, pip_version) ) elif pip_version > local_version: - logger.warning( - PACKAGE_VERSION_UPGRADE.format(local_version, pip_version) - ) + logger.warning(PACKAGE_VERSION_UPGRADE.format(local_version, pip_version)) if not os.environ.get("SA_VERSION_CHECK", "True").lower() == "false": diff --git a/src/superannotate/lib/app/analytics/class_analytics.py b/src/superannotate/lib/app/analytics/class_analytics.py index 36a50bb45..712e9da58 100644 --- a/src/superannotate/lib/app/analytics/class_analytics.py +++ b/src/superannotate/lib/app/analytics/class_analytics.py @@ -2,7 +2,7 @@ import pandas as pd import plotly.express as px -from lib.app.mixp.decorators import Trackable +from lib.app.interface.base_interface import Tracker from superannotate.lib.app.exceptions import AppException from superannotate.lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE from superannotate.logger import get_default_logger @@ -12,7 +12,7 @@ logger = get_default_logger() -@Trackable +@Tracker def class_distribution(export_root, project_names, visualize=False): """Aggregate distribution of classes across multiple projects. @@ -60,7 +60,11 @@ def class_distribution(export_root, project_names, visualize=False): df = df.sort_values(["count"], ascending=False) if visualize: - fig = px.bar(df, x="className", y="count",) + fig = px.bar( + df, + x="className", + y="count", + ) fig.update_traces(hovertemplate="%{x}: %{y}") fig.update_yaxes(title_text="Instance Count") fig.update_xaxes(title_text="") diff --git a/src/superannotate/lib/app/common.py b/src/superannotate/lib/app/common.py index de05b04f6..23a9b9d45 100644 --- a/src/superannotate/lib/app/common.py +++ b/src/superannotate/lib/app/common.py @@ -8,15 +8,13 @@ def hex_to_rgb(hex_string): - """Converts HEX values to RGB values - """ + """Converts HEX values to RGB values""" h = hex_string.lstrip("#") return tuple(int(h[i : i + 2], 16) for i in (0, 2, 4)) def blue_color_generator(n, hex_values=True): - """ Blue colors generator for SuperAnnotate blue mask. - """ + """Blue colors generator for SuperAnnotate blue mask.""" hex_colors = [] for i in range(n + 1): int_color = i * 15 @@ -27,7 +25,9 @@ def blue_color_generator(n, hex_values=True): hex_color = ( "#" + "{:02x}".format(bgr_color[2]) - + "{:02x}".format(bgr_color[1],) + + "{:02x}".format( + bgr_color[1], + ) + "{:02x}".format(bgr_color[0]) ) if hex_values: diff --git a/src/superannotate/lib/app/input_converters/__init__.py b/src/superannotate/lib/app/input_converters/__init__.py index e69de29bb..1acd1e490 100644 --- a/src/superannotate/lib/app/input_converters/__init__.py +++ b/src/superannotate/lib/app/input_converters/__init__.py @@ -0,0 +1,12 @@ +from .conversion import convert_json_version +from .conversion import convert_project_type +from .conversion import export_annotation +from .conversion import import_annotation + + +__all__ = [ + "convert_json_version", + "convert_project_type", + "export_annotation", + "import_annotation", +] diff --git a/src/superannotate/lib/app/input_converters/conversion.py b/src/superannotate/lib/app/input_converters/conversion.py index 0bb412325..812714089 100644 --- a/src/superannotate/lib/app/input_converters/conversion.py +++ b/src/superannotate/lib/app/input_converters/conversion.py @@ -6,7 +6,6 @@ from lib.app.exceptions import AppException from lib.app.interface.base_interface import Tracker -from lib.app.mixp.decorators import Trackable from lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE from lib.core import LIMITED_FUNCTIONS from lib.core.enums import ProjectType @@ -146,41 +145,41 @@ def export_annotation( task="object_detection", ): """ - Converts SuperAnnotate annotation format to the other annotation formats. Currently available (project_type, task) combinations for converter - presented below: - - ============== ====================== - From SA to COCO - -------------------------------------- - project_type task - ============== ====================== - Pixel panoptic_segmentation - Pixel instance_segmentation - Vector instance_segmentation - Vector object_detection - Vector keypoint_detection - ============== ====================== - - :param input_dir: Path to the dataset folder that you want to convert. - :type input_dir: Pathlike(str or Path) - :param output_dir: Path to the folder, where you want to have converted dataset. - :type output_dir: Pathlike(str or Path) - :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"] - :type dataset_format: str - :param dataset_name: Will be used to create json file in the output_dir. - :type dataset_name: str - :param project_type: SuperAnnotate project type is either 'Vector' or 'Pixel' (Default: 'Vector') - 'Vector' project creates ___objects.json for each image. - 'Pixel' project creates ___pixel.jsons and ___save.png annotation mask for each image. - :type project_type: str - :param task: Task can be one of the following: ['panoptic_segmentation', 'instance_segmentation', - 'keypoint_detection', 'object_detection']. (Default: "object_detection"). - 'keypoint_detection' can be used to converts keypoints from/to available annotation format. - 'panoptic_segmentation' will use panoptic mask for each image to generate bluemask for SuperAnnotate annotation format and use bluemask to generate panoptic mask for invert conversion. Panoptic masks should be in the input folder. - 'instance_segmentation' 'Pixel' project_type converts instance masks and 'Vector' project_type generates bounding boxes and polygons from instance masks. Masks should be in the input folder if it is 'Pixel' project_type. - 'object_detection' converts objects from/to available annotation format - :type task: str - """ + Converts SuperAnnotate annotation format to the other annotation formats. Currently available (project_type, task) combinations for converter + presented below: + + ============== ====================== + From SA to COCO + -------------------------------------- + project_type task + ============== ====================== + Pixel panoptic_segmentation + Pixel instance_segmentation + Vector instance_segmentation + Vector object_detection + Vector keypoint_detection + ============== ====================== + + :param input_dir: Path to the dataset folder that you want to convert. + :type input_dir: Pathlike(str or Path) + :param output_dir: Path to the folder, where you want to have converted dataset. + :type output_dir: Pathlike(str or Path) + :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"] + :type dataset_format: str + :param dataset_name: Will be used to create json file in the output_dir. + :type dataset_name: str + :param project_type: SuperAnnotate project type is either 'Vector' or 'Pixel' (Default: 'Vector') + 'Vector' project creates ___objects.json for each image. + 'Pixel' project creates ___pixel.jsons and ___save.png annotation mask for each image. + :type project_type: str + :param task: Task can be one of the following: ['panoptic_segmentation', 'instance_segmentation', + 'keypoint_detection', 'object_detection']. (Default: "object_detection"). + 'keypoint_detection' can be used to converts keypoints from/to available annotation format. + 'panoptic_segmentation' will use panoptic mask for each image to generate bluemask for SuperAnnotate annotation format and use bluemask to generate panoptic mask for invert conversion. Panoptic masks should be in the input folder. + 'instance_segmentation' 'Pixel' project_type converts instance masks and 'Vector' project_type generates bounding boxes and polygons from instance masks. Masks should be in the input folder if it is 'Pixel' project_type. + 'object_detection' converts objects from/to available annotation format + :type task: str + """ if project_type in [ ProjectType.VIDEO.name, @@ -224,7 +223,7 @@ def export_annotation( export_from_sa(args) -@Trackable +@Tracker def import_annotation( input_dir, output_dir, @@ -408,9 +407,9 @@ def import_annotation( import_to_sa(args) -@Trackable +@Tracker def convert_project_type(input_dir, output_dir): - """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. + """Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. :param input_dir: Path to the dataset folder that you want to convert. :type input_dir: Pathlike(str or Path) @@ -436,7 +435,7 @@ def convert_project_type(input_dir, output_dir): sa_convert_project_type(input_dir, output_dir) -@Trackable +@Tracker def convert_json_version(input_dir, output_dir, version=2): """ Converts SuperAnnotate JSON versions. Newest JSON version is 2. diff --git a/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py b/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py index 4c5834986..81fa39461 100644 --- a/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py +++ b/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py @@ -137,9 +137,9 @@ def convert_from_old_sa_to_new(self, old_json_data, project_type): def _parse_json_into_common_format(self, sa_annotation_json, fpath): """ - If the annotation format ever changes this function will handle it and - return something optimal for the converters. Additionally, if anything - important is absent from the current json, this function fills it. + If the annotation format ever changes this function will handle it and + return something optimal for the converters. Additionally, if anything + important is absent from the current json, this function fills it. """ if isinstance(sa_annotation_json, list): sa_annotation_json = self.convert_from_old_sa_to_new( diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 5c5a8b9c6..3904a97a6 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -1,19 +1,65 @@ import functools +import os import sys -from abc import ABC from abc import abstractmethod from inspect import signature +from pathlib import Path from types import FunctionType from typing import Iterable from typing import Sized +import lib.core as constants from lib.app.helpers import extract_project_folder +from lib.app.interface.types import validate_arguments +from lib.core.exceptions import AppException from lib.core.reporter import Session +from lib.infrastructure.controller import Controller +from lib.infrastructure.repositories import ConfigRepository from mixpanel import Mixpanel from version import __version__ -class BaseInterfaceFacade(ABC): +class BaseInterfaceFacade: + REGISTRY = [] + + def __init__( + self, + token: str = None, + config_path: str = constants.CONFIG_PATH, + ): + host = constants.BACKEND_URL + env_token = os.environ.get("SA_TOKEN") + version = os.environ.get("SA_VERSION", "v1") + ssl_verify = bool(os.environ.get("SA_SSL", True)) + if token: + token = Controller.validate_token(token=token) + elif env_token: + host = os.environ.get("SA_URL", constants.BACKEND_URL) + + token = Controller.validate_token(env_token) + else: + config_path = os.path.expanduser(str(config_path)) + if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + raise AppException( + f"SuperAnnotate config file {str(config_path)} not found." + f" Please provide correct config file location to sa.init() or use " + f"CLI's superannotate init to generate default location config file." + ) + config_repo = ConfigRepository(config_path) + token, host, ssl_verify = ( + Controller.validate_token(config_repo.get_one("token").value), + config_repo.get_one("main_endpoint").value, + config_repo.get_one("ssl_verify").value, + ) + self._host = host + self._token = token + self.controller = Controller(token, host, ssl_verify, version) + + def __new__(cls, *args, **kwargs): + obj = super().__new__(cls, *args, **kwargs) + cls.REGISTRY.append(obj) + return obj + @property @abstractmethod def host(self): @@ -31,32 +77,38 @@ def logger(self): class Tracker: - TEAM_DATA = None - INITIAL_EVENT = {"event_name": "SDK init", "properties": {}} - INITIAL_LOGGED = False - - @staticmethod - def get_mp_instance() -> Mixpanel: - # if "api.annotate.online" in get_default_controller()._backend_url: - # return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") - return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") + def get_mp_instance(self) -> Mixpanel: + if self.client: + if self.client.host == constants.BACKEND_URL: + return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") + else: + return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") @staticmethod - def get_default_payload(team_name, user_id, project_name=None): + def get_default_payload(team_name, user_id): return { "SDK": True, - "Paid": True, "Team": team_name, "Team Owner": user_id, - "Project Name": project_name, - "Project Role": "Admin", "Version": __version__, } def __init__(self, function): self.function = function + self._client = None functools.update_wrapper(self, function) + @property + def client(self): + if not self._client: + if BaseInterfaceFacade.REGISTRY: + self._client = BaseInterfaceFacade.REGISTRY[-1] + else: + from lib.app.interface.sdk_interface import SAClient + + self._client = SAClient() + return self._client + @staticmethod def extract_arguments(function, *args, **kwargs) -> dict: bound_arguments = signature(function).bind(*args, **kwargs) @@ -69,12 +121,16 @@ def default_parser(function_name: str, kwargs: dict) -> tuple: for key, value in kwargs.items(): if key == "self": continue + elif value is None: + properties[key] = value elif key == "project": - properties["project_name"], properties["folder_name"] = extract_project_folder(value) - elif isinstance(value, (str, int, float, bool, str)): + properties["project_name"], folder_name = extract_project_folder(value) + if folder_name: + properties["folder_name"] = folder_name + elif isinstance(value, (str, int, float, bool)): properties[key] = value elif isinstance(value, dict): - properties[key] = value.keys() + properties[key] = list(value.keys()) elif isinstance(value, Sized): properties[key] = len(value) elif isinstance(value, Iterable): @@ -83,56 +139,51 @@ def default_parser(function_name: str, kwargs: dict) -> tuple: properties[key] = str(value) return function_name, properties - def track(self, args, kwargs, success: bool, session): - try: - function_name = self.function.__name__ if self.function else "" - arguments = self.extract_arguments(self.function, *args, **kwargs) - event_name, properties = self.default_parser(function_name, arguments) - - user_id = self.team_data.creator_id - team_name = self.team_data.name - properties["Success"] = success - - default = self.get_default_payload( - team_name=team_name, - user_id=user_id, - project_name=properties.pop("project_name", None), - ) - if "pytest" not in sys.modules: - self.get_mp_instance().track(user_id, event_name, {**default, **properties, **session.data}) - except Exception: - raise - - def __get__(self, instance, owner): - if instance is None: - return self - d = self - mfactory = lambda self, *args, **kw: d(self, *args, **kw) - mfactory.__name__ = self.function.__name__ - self.team_data = instance.controller.team_data.data - return mfactory.__get__(instance, owner) + def _track(self, user_id: str, event_name: str, data: dict): + if "pytest" not in sys.modules: + self.get_mp_instance().track(user_id, event_name, data) + + def _track_method(self, args, kwargs, success: bool): + function_name = self.function.__name__ if self.function else "" + arguments = self.extract_arguments(self.function, *args, **kwargs) + event_name, properties = self.default_parser(function_name, arguments) + + user_id = self.client.controller.team_data.creator_id + team_name = self.client.controller.team_data.name + + properties["Success"] = success + default = self.get_default_payload(team_name=team_name, user_id=user_id) + self._track( + user_id, + event_name, + {**default, **properties, **Session.get_current_session().data}, + ) + + def __get__(self, obj, owner=None): + if obj is not None: + self._client = obj + tmp = functools.partial(self.__call__, obj) + functools.update_wrapper(tmp, self.function) + return tmp + return self def __call__(self, *args, **kwargs): success = True try: - with Session(self.function.__name__) as session: - result = self.function(*args, **kwargs) + result = self.function(*args, **kwargs) except Exception as e: success = False raise e else: return result finally: - self.track(args=args, kwargs=kwargs, success=success, session=session) + self._track_method(args=args, kwargs=kwargs, success=success) class TrackableMeta(type): def __new__(mcs, name, bases, attrs): - for attr_name, attr_value in attrs.iteritems(): + for attr_name, attr_value in attrs.items(): if isinstance(attr_value, FunctionType): - attrs[attr_name] = mcs.decorate(attr_value) - return super().__new__(mcs, name, bases, attrs) - - @staticmethod - def decorate(func): - return Tracker(func) + attrs[attr_name] = Tracker(validate_arguments(attr_value)) + tmp = super().__new__(mcs, name, bases, attrs) + return tmp diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index c0576ccce..54f6f90e0 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -25,7 +25,7 @@ def version(): To show the version of the current SDK installation """ with open( - f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" + f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" ) as f: version = f.read().rstrip()[15:-1] print(version) @@ -40,7 +40,7 @@ def init(): config = repo.get_one(uuid=constances.TOKEN_UUID) if config: if not input( - f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " + f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " ).lower() in ("y", "yes"): return token = input( @@ -68,14 +68,14 @@ def create_folder(self, project: str, name: str): sys.exit(0) def upload_images( - self, - project: str, - folder: str, - extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, - set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, - exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders=False, - image_quality_in_editor=None, + self, + project: str, + folder: str, + extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, + set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, + exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders=False, + image_quality_in_editor=None, ): """ To upload images from folder to project use: @@ -99,12 +99,12 @@ def upload_images( sys.exit(0) def export_project( - self, - project, - folder, - include_fuse=False, - disable_extract_zip_contents=False, - annotation_statuses=None, + self, + project, + folder, + include_fuse=False, + disable_extract_zip_contents=False, + annotation_statuses=None, ): project_name, folder_name = split_project_path(project) folders = None @@ -113,13 +113,16 @@ def export_project( if folder_name: folders = [folder_name] export_res = SAClient().prepare_export( - project_name, folders, include_fuse, False, annotation_statuses + project=project_name, + folder_names=folders, + include_fuse=include_fuse, + annotation_statuses=annotation_statuses, ) - export_name = export_res.data["name"] + export_name = export_res["name"] SAClient().download_export( - project_name=project_name, - export_name=export_name, + project=project_name, + export=export_name, folder_path=folder, extract_zip_contents=not disable_extract_zip_contents, to_s3_bucket=False, @@ -127,7 +130,7 @@ def export_project( sys.exit(0) def upload_preannotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload preannotations from folder to project use @@ -148,7 +151,7 @@ def upload_preannotations( sys.exit(0) def upload_annotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload annotations from folder to project use @@ -169,11 +172,13 @@ def upload_annotations( sys.exit(0) def _upload_annotations( - self, project, folder, format, dataset_name, task, pre=True + self, project, folder, format, dataset_name, task, pre=True ): project_folder_name = project project_name, folder_name = split_project_path(project) - project = SAClient().controller.get_project_metadata(project_name=project_name).data + project = ( + SAClient().controller.get_project_metadata(project_name=project_name).data + ) if not format: format = "SuperAnnotate" if not dataset_name and format == "COCO": @@ -207,10 +212,10 @@ def _upload_annotations( sys.exit(0) def attach_image_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): """ To attach image URLs to project use: @@ -224,10 +229,10 @@ def attach_image_urls( sys.exit(0) def attach_video_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): SAClient().attach_items( project=project, @@ -238,7 +243,7 @@ def attach_video_urls( @staticmethod def attach_document_urls( - project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" + project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" ): SAClient().attach_items( project=project, @@ -248,15 +253,15 @@ def attach_document_urls( sys.exit(0) def upload_videos( - self, - project, - folder, - target_fps=None, - recursive=False, - extensions=constances.DEFAULT_VIDEO_EXTENSIONS, - set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, - start_time=0.0, - end_time=None, + self, + project, + folder, + target_fps=None, + recursive=False, + extensions=constances.DEFAULT_VIDEO_EXTENSIONS, + set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, + start_time=0.0, + end_time=None, ): """ To upload videos from folder to project use diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 4e6177c79..59ef32313 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -21,6 +21,7 @@ from lib.app.helpers import get_annotation_paths from lib.app.helpers import get_name_url_duplicated_from_csv from lib.app.interface.base_interface import BaseInterfaceFacade +from lib.app.interface.base_interface import TrackableMeta from lib.app.interface.types import AnnotationStatuses from lib.app.interface.types import AnnotationType from lib.app.interface.types import AnnotatorRole @@ -33,7 +34,6 @@ from lib.app.interface.types import ProjectStatusEnum from lib.app.interface.types import ProjectTypes from lib.app.interface.types import Setting -from lib.app.interface.types import validate_arguments from lib.app.serializers import BaseSerializer from lib.app.serializers import FolderSerializer from lib.app.serializers import ProjectSerializer @@ -51,7 +51,6 @@ from lib.core.types import PriorityScore from lib.core.types import Project from lib.infrastructure.controller import Controller -from lib.infrastructure.repositories import ConfigRepository from pydantic import conlist from pydantic import parse_obj_as from pydantic import StrictBool @@ -62,52 +61,7 @@ logger = get_default_logger() -class SAClient(BaseInterfaceFacade): - def __init__( - self, - token: str = None, - config_path: str = constances.CONFIG_FILE_LOCATION, - ): - host = constances.BACKEND_URL - env_token = os.environ.get("SA_TOKEN") - version = os.environ.get("SA_VERSION", "v1") - ssl_verify = bool(os.environ.get("SA_SSL", True)) - if token: - token = Controller.validate_token(token=token) - elif env_token: - host = os.environ.get("SA_URL", constances.BACKEND_URL) - - token = Controller.validate_token(env_token) - else: - config_path = str(config_path) - if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): - raise AppException( - f"SuperAnnotate config file {str(config_path)} not found." - f" Please provide correct config file location to sa.init() or use " - f"CLI's superannotate init to generate default location config file." - ) - config_repo = ConfigRepository(config_path) - token, host, ssl_verify = ( - Controller.validate_token(config_repo.get_one("token").value), - config_repo.get_one("main_endpoint").value, - config_repo.get_one("ssl_verify").value, - ) - self._host = host - self._token = token - self.controller = Controller(token, host, ssl_verify, version) - - @property - def host(self): - return self._host - - @property - def token(self): - return self._token - - @property - def logger(self): - pass - +class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta): def get_team_metadata(self): """Returns team metadata @@ -117,13 +71,12 @@ def get_team_metadata(self): response = self.controller.get_team() return TeamSerializer(response.data).serialize() - @validate_arguments def search_team_contributors( - self, - email: EmailStr = None, - first_name: NotEmptyStr = None, - last_name: NotEmptyStr = None, - return_metadata: bool = True, + self, + email: EmailStr = None, + first_name: NotEmptyStr = None, + last_name: NotEmptyStr = None, + return_metadata: bool = True, ): """Search for contributors in the team @@ -148,13 +101,12 @@ def search_team_contributors( return [contributor["email"] for contributor in contributors] return contributors - @validate_arguments def search_projects( - self, - name: Optional[NotEmptyStr] = None, - return_metadata: bool = False, - include_complete_image_count: bool = False, - status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, + self, + name: Optional[NotEmptyStr] = None, + return_metadata: bool = False, + include_complete_image_count: bool = False, + status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, ): """ Project name based case-insensitive search for projects. @@ -184,7 +136,7 @@ def search_projects( result = self.controller.search_project( name=name, include_complete_image_count=include_complete_image_count, - statuses=statuses + statuses=statuses, ).data if return_metadata: @@ -203,13 +155,12 @@ def search_projects( else: return [project.name for project in result] - @validate_arguments def create_project( - self, - project_name: NotEmptyStr, - project_description: NotEmptyStr, - project_type: NotEmptyStr, - settings: List[Setting] = None, + self, + project_name: NotEmptyStr, + project_description: NotEmptyStr, + project_type: NotEmptyStr, + settings: List[Setting] = None, ): """Create a new project in the team. @@ -243,9 +194,7 @@ def create_project( return ProjectSerializer(response.data).serialize() - @validate_arguments - def create_project_from_metadata( - self, project_metadata: Project): + def create_project_from_metadata(self, project_metadata: Project): """Create a new project in the team using project metadata object dict. Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) Non-mandatory keys: "workflow", "settings" and "annotation_classes". @@ -269,16 +218,15 @@ def create_project_from_metadata( raise AppException(response.errors) return ProjectSerializer(response.data).serialize() - @validate_arguments def clone_project( - self, - project_name: Union[NotEmptyStr, dict], - from_project: Union[NotEmptyStr, dict], - project_description: Optional[NotEmptyStr] = None, - copy_annotation_classes: Optional[StrictBool] = True, - copy_settings: Optional[StrictBool] = True, - copy_workflow: Optional[StrictBool] = True, - copy_contributors: Optional[StrictBool] = False, + self, + project_name: Union[NotEmptyStr, dict], + from_project: Union[NotEmptyStr, dict], + project_description: Optional[NotEmptyStr] = None, + copy_annotation_classes: Optional[StrictBool] = True, + copy_settings: Optional[StrictBool] = True, + copy_workflow: Optional[StrictBool] = True, + copy_contributors: Optional[StrictBool] = False, ): """Create a new project in the team using annotation classes and settings from from_project. @@ -314,9 +262,7 @@ def clone_project( raise AppException(response.errors) return ProjectSerializer(response.data).serialize() - @validate_arguments - def create_folder( - self, project: NotEmptyStr, folder_name: NotEmptyStr): + def create_folder(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Create a new folder in the project. :param project: project name @@ -328,9 +274,7 @@ def create_folder( :rtype: dict """ - res = self.controller.create_folder( - project=project, folder_name=folder_name - ) + res = self.controller.create_folder(project=project, folder_name=folder_name) if res.data: folder = res.data logger.info(f"Folder {folder.name} created in project {project}") @@ -338,22 +282,18 @@ def create_folder( if res.errors: raise AppException(res.errors) - @validate_arguments - def delete_project( - self, project: Union[NotEmptyStr, dict]): + def delete_project(self, project: Union[NotEmptyStr, dict]): """Deletes the project - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str """ name = project if isinstance(project, dict): name = project["name"] self.controller.delete_project(name=name) - @validate_arguments - def rename_project( - self, project: NotEmptyStr, new_name: NotEmptyStr): + def rename_project(self, project: NotEmptyStr, new_name: NotEmptyStr): """Renames the project :param project: project name or folder path (e.g., "project1/folder1") @@ -367,12 +307,12 @@ def rename_project( ) if response.errors: raise AppException(response.errors) - logger.info("Successfully renamed project %s to %s.", project, response.data.name) + logger.info( + "Successfully renamed project %s to %s.", project, response.data.name + ) return ProjectSerializer(response.data).serialize() - @validate_arguments - def get_folder_metadata( - self, project: NotEmptyStr, folder_name: NotEmptyStr): + def get_folder_metadata(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Returns folder metadata :param project: project name @@ -383,18 +323,14 @@ def get_folder_metadata( :return: metadata of folder :rtype: dict """ - result = ( - self.controller - .get_folder(project_name=project, folder_name=folder_name) - .data - ) + result = self.controller.get_folder( + project_name=project, folder_name=folder_name + ).data if not result: raise AppException("Folder not found.") return FolderSerializer(result).serialize() - @validate_arguments - def delete_folders( - self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): + def delete_folders(self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): """Delete folder in project. :param project: project name @@ -410,12 +346,11 @@ def delete_folders( raise AppException(res.errors) logger.info(f"Folders {folder_names} deleted in project {project}") - @validate_arguments def search_folders( - self, - project: NotEmptyStr, - folder_name: Optional[NotEmptyStr] = None, - return_metadata: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + folder_name: Optional[NotEmptyStr] = None, + return_metadata: Optional[StrictBool] = False, ): """Folder name based case-insensitive search for folders in project. @@ -440,15 +375,14 @@ def search_folders( return [FolderSerializer(folder).serialize() for folder in data] return [folder.name for folder in data] - @validate_arguments def copy_image( - self, - source_project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = False, - copy_annotation_status: Optional[StrictBool] = False, - copy_pin: Optional[StrictBool] = False, + self, + source_project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + destination_project: Union[NotEmptyStr, dict], + include_annotations: Optional[StrictBool] = False, + copy_annotation_status: Optional[StrictBool] = False, + copy_pin: Optional[StrictBool] = False, ): """Copy image to a project. The image's project is the same as destination project then the name will be changed to _()., @@ -473,12 +407,12 @@ def copy_image( destination_project, destination_folder = extract_project_folder( destination_project ) - source_project_metadata = ( - self.controller.get_project_metadata(source_project_name).data - ) - destination_project_metadata = ( - self.controller.get_project_metadata(destination_project).data - ) + source_project_metadata = self.controller.get_project_metadata( + source_project_name + ).data + destination_project_metadata = self.controller.get_project_metadata( + destination_project + ).data if destination_project_metadata["project"].type in [ constances.ProjectType.VIDEO.value, @@ -487,7 +421,9 @@ def copy_image( constances.ProjectType.VIDEO.value, constances.ProjectType.DOCUMENT.value, ]: - raise AppException(LIMITED_FUNCTIONS[source_project_metadata["project"].type]) + raise AppException( + LIMITED_FUNCTIONS[source_project_metadata["project"].type] + ) response = self.controller.copy_image( from_project_name=source_project_name, @@ -520,15 +456,14 @@ def copy_image( f" to {destination_project}/{destination_folder}." ) - @validate_arguments def get_project_metadata( - self, - project: Union[NotEmptyStr, dict], - include_annotation_classes: Optional[StrictBool] = False, - include_settings: Optional[StrictBool] = False, - include_workflow: Optional[StrictBool] = False, - include_contributors: Optional[StrictBool] = False, - include_complete_image_count: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + include_annotation_classes: Optional[StrictBool] = False, + include_settings: Optional[StrictBool] = False, + include_workflow: Optional[StrictBool] = False, + include_contributors: Optional[StrictBool] = False, + include_complete_image_count: Optional[StrictBool] = False, ): """Returns project metadata @@ -555,30 +490,26 @@ def get_project_metadata( :rtype: dict """ project_name, folder_name = extract_project_folder(project) - response = ( - self.controller.get_project_metadata( - project_name, - include_annotation_classes, - include_settings, - include_workflow, - include_contributors, - include_complete_image_count, - ) - .data - ) + response = self.controller.get_project_metadata( + project_name, + include_annotation_classes, + include_settings, + include_workflow, + include_contributors, + include_complete_image_count, + ).data metadata = ProjectSerializer(response["project"]).serialize() for elem in "classes", "workflows", "contributors": if response.get(elem): metadata[elem] = [ - BaseSerializer(attribute).serialize() for attribute in response[elem] + BaseSerializer(attribute).serialize() + for attribute in response[elem] ] return metadata - @validate_arguments - def get_project_settings( - self, project: Union[NotEmptyStr, dict]): + def get_project_settings(self, project: Union[NotEmptyStr, dict]): """Gets project's settings. Return value example: [{ "attribute" : "Brightness", "value" : 10, ...},...] @@ -596,9 +527,7 @@ def get_project_settings( ] return settings - @validate_arguments - def get_project_workflow( - self, project: Union[str, dict]): + def get_project_workflow(self, project: Union[str, dict]): """Gets project's workflow. Return value example: [{ "step" : , "className" : , "tool" : , ...},...] @@ -615,10 +544,8 @@ def get_project_workflow( raise AppException(workflow.errors) return workflow.data - @validate_arguments def search_annotation_classes( - self, - project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None + self, project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None ): """Searches annotation classes by name_prefix (case-insensitive) @@ -632,17 +559,14 @@ def search_annotation_classes( :rtype: list of dicts """ project_name, folder_name = extract_project_folder(project) - classes = self.controller.search_annotation_classes( - project_name, name_contains - ) + classes = self.controller.search_annotation_classes(project_name, name_contains) classes = [BaseSerializer(attribute).serialize() for attribute in classes.data] return classes - @validate_arguments def set_project_default_image_quality_in_editor( - self, - project: Union[NotEmptyStr, dict], - image_quality_in_editor: Optional[str], + self, + project: Union[NotEmptyStr, dict], + image_quality_in_editor: Optional[str], ): """Sets project's default image quality in editor setting. @@ -656,16 +580,19 @@ def set_project_default_image_quality_in_editor( response = self.controller.set_project_settings( project_name=project_name, - new_settings=[{"attribute": "ImageQuality", "value": image_quality_in_editor}], + new_settings=[ + {"attribute": "ImageQuality", "value": image_quality_in_editor} + ], ) if response.errors: raise AppException(response.errors) return response.data - @validate_arguments def pin_image( - self, - project: Union[NotEmptyStr, dict], image_name: str, pin: Optional[StrictBool] = True + self, + project: Union[NotEmptyStr, dict], + image_name: str, + pin: Optional[StrictBool] = True, ): """Pins (or unpins) image @@ -684,12 +611,11 @@ def pin_image( is_pinned=int(pin), ) - @validate_arguments def set_images_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: NotEmptyStr, - image_names: Optional[List[NotEmptyStr]] = None, + self, + project: Union[NotEmptyStr, dict], + annotation_status: NotEmptyStr, + image_names: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of images @@ -716,10 +642,8 @@ def set_images_annotation_statuses( raise AppException(response.errors) logger.info("Annotations status of images changed") - @validate_arguments def delete_images( - self, - project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None + self, project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None ): """Delete images in project. @@ -743,9 +667,9 @@ def delete_images( f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) - @validate_arguments def assign_images( - self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str): + self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str + ): """Assigns images to a user. The assignment role, QA or Annotator, will be deduced from the user's role in the project. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -767,10 +691,11 @@ def assign_images( raise AppException(LIMITED_FUNCTIONS[project["project"].type]) contributors = ( - self.controller - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users ) contributor = None for c in contributors: @@ -791,9 +716,9 @@ def assign_images( else: raise AppException(response.errors) - @validate_arguments def unassign_images( - self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr]): + self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr] + ): """Removes assignment of given images for all assignees.With SDK, the user can be assigned to a role in the project with the share_project function. @@ -811,9 +736,7 @@ def unassign_images( if response.errors: raise AppException(response.errors) - @validate_arguments - def unassign_folder( - self, project_name: NotEmptyStr, folder_name: NotEmptyStr): + def unassign_folder(self, project_name: NotEmptyStr, folder_name: NotEmptyStr): """Removes assignment of given folder for all assignees. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -829,10 +752,11 @@ def unassign_folder( if response.errors: raise AppException(response.errors) - @validate_arguments def assign_folder( - self, - project_name: NotEmptyStr, folder_name: NotEmptyStr, users: List[NotEmptyStr] + self, + project_name: NotEmptyStr, + folder_name: NotEmptyStr, + users: List[NotEmptyStr], ): """Assigns folder to users. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -846,10 +770,11 @@ def assign_folder( """ contributors = ( - self.controller - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users ) verified_users = [i["user_id"] for i in contributors] verified_users = set(users).intersection(set(verified_users)) @@ -864,27 +789,28 @@ def assign_folder( return response = self.controller.assign_folder( - project_name=project_name, folder_name=folder_name, users=list(verified_users) + project_name=project_name, + folder_name=folder_name, + users=list(verified_users), ) if response.errors: raise AppException(response.errors) - @validate_arguments def upload_images_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[List[NotEmptyStr], Tuple[NotEmptyStr]] - ] = constances.DEFAULT_IMAGE_EXTENSIONS, - annotation_status="NotStarted", - from_s3_bucket=None, - exclude_file_patterns: Optional[ - Iterable[NotEmptyStr] - ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders: Optional[StrictBool] = False, - image_quality_in_editor: Optional[str] = None, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[List[NotEmptyStr], Tuple[NotEmptyStr]] + ] = constances.DEFAULT_IMAGE_EXTENSIONS, + annotation_status="NotStarted", + from_s3_bucket=None, + exclude_file_patterns: Optional[ + Iterable[NotEmptyStr] + ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders: Optional[StrictBool] = False, + image_quality_in_editor: Optional[str] = None, ): """Uploads all images with given extensions from folder_path to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -967,24 +893,29 @@ def upload_images_from_folder_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) + "%s already existing images found that won't be uploaded.", + len(duplicates), ) logger.info( - "Uploading %s images to project %s.", len(images_to_upload), project_folder_name + "Uploading %s images to project %s.", + len(images_to_upload), + project_folder_name, ) if not images_to_upload: return [], [], duplicates if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: for _ in use_case.execute(): progress_bar.update(1) return use_case.data raise AppException(use_case.response.errors) - @validate_arguments def get_project_image_count( - self, - project: Union[NotEmptyStr, dict], with_all_subfolders: Optional[StrictBool] = False + self, + project: Union[NotEmptyStr, dict], + with_all_subfolders: Optional[StrictBool] = False, ): """Returns number of images in the project. @@ -1008,12 +939,11 @@ def get_project_image_count( raise AppException(response.errors) return response.data - @validate_arguments def download_image_annotations( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Union[str, Path], + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Union[str, Path], ): """Downloads annotations of the image (JSON and mask if pixel type project) to local_dir_path. @@ -1039,9 +969,9 @@ def download_image_annotations( raise AppException(res.errors) return res.data - @validate_arguments def get_exports( - self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False): + self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False + ): """Get all prepared exports of the project. :param project: project name @@ -1057,14 +987,13 @@ def get_exports( ) return response.data - @validate_arguments def prepare_export( - self, - project: Union[NotEmptyStr, dict], - folder_names: Optional[List[NotEmptyStr]] = None, - annotation_statuses: Optional[List[AnnotationStatuses]] = None, - include_fuse: Optional[StrictBool] = False, - only_pinned=False, + self, + project: Union[NotEmptyStr, dict], + folder_names: Optional[List[NotEmptyStr]] = None, + annotation_statuses: Optional[List[AnnotationStatuses]] = None, + include_fuse: Optional[StrictBool] = False, + only_pinned=False, ): """Prepare annotations and classes.json for export. Original and fused images for images with annotations can be included with include_fuse flag. @@ -1110,21 +1039,20 @@ def prepare_export( raise AppException(response.errors) return response.data - @validate_arguments def upload_videos_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[Tuple[NotEmptyStr], List[NotEmptyStr]] - ] = constances.DEFAULT_VIDEO_EXTENSIONS, - exclude_file_patterns: Optional[List[NotEmptyStr]] = (), - recursive_subfolders: Optional[StrictBool] = False, - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[Tuple[NotEmptyStr], List[NotEmptyStr]] + ] = constances.DEFAULT_VIDEO_EXTENSIONS, + exclude_file_patterns: Optional[List[NotEmptyStr]] = (), + recursive_subfolders: Optional[StrictBool] = False, + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads image frames from all videos with given extensions from folder_path to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -1164,7 +1092,9 @@ def upload_videos_from_folder_to_project( if not recursive_subfolders: video_paths += list(Path(folder_path).glob(f"*.{extension.lower()}")) if os.name != "nt": - video_paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) + video_paths += list( + Path(folder_path).glob(f"*.{extension.upper()}") + ) else: logger.warning( "When using recursive subfolder parsing same name videos " @@ -1172,7 +1102,9 @@ def upload_videos_from_folder_to_project( ) video_paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) if os.name != "nt": - video_paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) + video_paths += list( + Path(folder_path).rglob(f"*.{extension.upper()}") + ) video_paths = [str(path) for path in video_paths] response = self.controller.upload_videos( @@ -1190,16 +1122,15 @@ def upload_videos_from_folder_to_project( raise AppException(response.errors) return response.data - @validate_arguments def upload_video_to_project( - self, - project: Union[NotEmptyStr, dict], - video_path: Union[NotEmptyStr, Path], - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: Union[NotEmptyStr, dict], + video_path: Union[NotEmptyStr, Path], + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads image frames from video to platform. Uploaded images will have names "_.jpg". @@ -1242,14 +1173,13 @@ def upload_video_to_project( raise AppException(response.errors) return response.data - @validate_arguments def create_annotation_class( - self, - project: Union[Project, NotEmptyStr], - name: NotEmptyStr, - color: NotEmptyStr, - attribute_groups: Optional[List[AttributeGroup]] = None, - class_type: ClassType = "object", + self, + project: Union[Project, NotEmptyStr], + name: NotEmptyStr, + color: NotEmptyStr, + attribute_groups: Optional[List[AttributeGroup]] = None, + class_type: ClassType = "object", ): """Create annotation class in project @@ -1285,10 +1215,8 @@ def create_annotation_class( raise AppException(response.errors) return BaseSerializer(response.data).serialize() - @validate_arguments def delete_annotation_class( - self, - project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] + self, project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] ): """Deletes annotation class from project @@ -1301,9 +1229,9 @@ def delete_annotation_class( project_name=project, annotation_class_name=annotation_class ) - @validate_arguments def download_annotation_classes_json( - self, project: NotEmptyStr, folder: Union[str, Path]): + self, project: NotEmptyStr, folder: Union[str, Path] + ): """Downloads project classes.json to folder :param project: project name @@ -1321,12 +1249,11 @@ def download_annotation_classes_json( raise AppException(response.errors) return response.data - @validate_arguments def create_annotation_classes_from_classes_json( - self, - project: Union[NotEmptyStr, dict], - classes_json: Union[List[AnnotationClassEntity], str, Path], - from_s3_bucket=False, + self, + project: Union[NotEmptyStr, dict], + classes_json: Union[List[AnnotationClassEntity], str, Path], + from_s3_bucket=False, ): """Creates annotation classes in project from a SuperAnnotate format annotation classes.json. @@ -1359,20 +1286,20 @@ def create_annotation_classes_from_classes_json( raise AppException("Couldn't validate annotation classes.") logger.info(f"Creating annotation classes in project {project}.") response = self.controller.create_annotation_classes( - project_name=project, annotation_classes=annotation_classes, + project_name=project, + annotation_classes=annotation_classes, ) if response.errors: raise AppException(response.errors) return [BaseSerializer(i).serialize() for i in response.data] - @validate_arguments def download_export( - self, - project: Union[NotEmptyStr, dict], - export: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - extract_zip_contents: Optional[StrictBool] = True, - to_s3_bucket=None, + self, + project: Union[NotEmptyStr, dict], + export: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + extract_zip_contents: Optional[StrictBool] = True, + to_s3_bucket=None, ): """Download prepared export. @@ -1405,12 +1332,11 @@ def download_export( raise AppException(response.errors) logger.info(response.data) - @validate_arguments def set_image_annotation_status( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - annotation_status: NotEmptyStr, + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + annotation_status: NotEmptyStr, ): """Sets the image annotation status @@ -1438,14 +1364,12 @@ def set_image_annotation_status( ) if response.errors: raise AppException(response.errors) - image = ( - self.controller.get_item(project_name, folder_name, image_name).data - ) + image = self.controller.get_item(project_name, folder_name, image_name).data return BaseSerializer(image).serialize() - @validate_arguments def set_project_workflow( - self, project: Union[NotEmptyStr, dict], new_workflow: List[dict]): + self, project: Union[NotEmptyStr, dict], new_workflow: List[dict] + ): """Sets project's workflow. new_workflow example: [{ "step" : , "className" : , "tool" : , @@ -1465,16 +1389,15 @@ def set_project_workflow( if response.errors: raise AppException(response.errors) - @validate_arguments def download_image( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Optional[Union[str, Path]] = "./", - include_annotations: Optional[StrictBool] = False, - include_fuse: Optional[StrictBool] = False, - include_overlay: Optional[StrictBool] = False, - variant: Optional[str] = "original", + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Optional[Union[str, Path]] = "./", + include_annotations: Optional[StrictBool] = False, + include_fuse: Optional[StrictBool] = False, + include_overlay: Optional[StrictBool] = False, + variant: Optional[str] = "original", ): """Downloads the image (and annotation if not None) to local_dir_path @@ -1513,13 +1436,12 @@ def download_image( logger.info(f"Downloaded image {image_name} to {local_dir_path} ") return response.data - @validate_arguments def upload_annotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, ): """Finds and uploads all JSON files in the folder_path as annotations to the project. @@ -1576,13 +1498,12 @@ def upload_annotations_from_folder_to_project( raise AppException(response.errors) return response.data - @validate_arguments def upload_preannotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, ): """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. @@ -1642,14 +1563,13 @@ def upload_preannotations_from_folder_to_project( raise AppException(response.errors) return response.data - @validate_arguments def upload_image_annotations( - self, - project: Union[NotEmptyStr, dict], - image_name: str, - annotation_json: Union[str, Path, dict], - mask: Optional[Union[str, Path, bytes]] = None, - verbose: Optional[StrictBool] = True, + self, + project: Union[NotEmptyStr, dict], + image_name: str, + annotation_json: Union[str, Path, dict], + mask: Optional[Union[str, Path, bytes]] = None, + verbose: Optional[StrictBool] = True, ): """Upload annotations from JSON (also mask for pixel annotations) to the image. @@ -1699,9 +1619,7 @@ def upload_image_annotations( if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: raise AppException(response.errors) - @validate_arguments - def download_model( - self, model: MLModel, output_dir: Union[str, Path]): + def download_model(self, model: MLModel, output_dir: Union[str, Path]): """Downloads the neural network and related files which are the .pth/pkl. .json, .yaml, classes_mapper.json @@ -1720,16 +1638,15 @@ def download_model( else: return BaseSerializer(res.data).serialize() - @validate_arguments def benchmark( - self, - project: Union[NotEmptyStr, dict], - gt_folder: str, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[str, Path]] = None, - image_list=None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots=False, + self, + project: Union[NotEmptyStr, dict], + gt_folder: str, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[str, Path]] = None, + image_list=None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots=False, ): """Computes benchmark score for each instance of given images that are present both gt_project_name project and projects in folder_names list: @@ -1788,15 +1705,14 @@ def benchmark( raise AppException(response.errors) return response.data - @validate_arguments def consensus( - self, - project: NotEmptyStr, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[NotEmptyStr, Path]] = None, - image_list: Optional[List[NotEmptyStr]] = None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[NotEmptyStr, Path]] = None, + image_list: Optional[List[NotEmptyStr]] = None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots: Optional[StrictBool] = False, ): """Computes consensus score for each instance of given images that are present in at least 2 of the given projects: @@ -1842,12 +1758,11 @@ def consensus( raise AppException(response.errors) return response.data - @validate_arguments def run_prediction( - self, - project: Union[NotEmptyStr, dict], - images_list: List[NotEmptyStr], - model: Union[NotEmptyStr, dict], + self, + project: Union[NotEmptyStr, dict], + images_list: List[NotEmptyStr], + model: Union[NotEmptyStr, dict], ): """This function runs smart prediction on given list of images from a given project using the neural network of your choice @@ -1881,15 +1796,14 @@ def run_prediction( raise AppException(response.errors) return response.data - @validate_arguments def add_annotation_bbox_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - bbox: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + bbox: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, ): """Add a bounding box annotation to image annotations @@ -1941,15 +1855,14 @@ def add_annotation_bbox_to_image( project_name, folder_name, image_name, annotations ) - @validate_arguments def add_annotation_point_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - point: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + point: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, ): """Add a point annotation to image annotations @@ -1999,15 +1912,14 @@ def add_annotation_point_to_image( project_name, folder_name, image_name, annotations ) - @validate_arguments def add_annotation_comment_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - comment_text: NotEmptyStr, - comment_coords: List[float], - comment_author: EmailStr, - resolved: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + comment_text: NotEmptyStr, + comment_coords: List[float], + comment_author: EmailStr, + resolved: Optional[StrictBool] = False, ): """Add a comment to SuperAnnotate format annotation JSON @@ -2055,15 +1967,14 @@ def add_annotation_comment_to_image( project_name, folder_name, image_name, annotations ) - @validate_arguments def upload_image_to_project( - self, - project: NotEmptyStr, - img, - image_name: Optional[NotEmptyStr] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[NotEmptyStr] = None, + self, + project: NotEmptyStr, + img, + image_name: Optional[NotEmptyStr] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[NotEmptyStr] = None, ): """Uploads image (io.BytesIO() or filepath to image) to project. Sets status of the uploaded image to set_status if it is not None. @@ -2097,30 +2008,29 @@ def upload_image_to_project( if response.errors: raise AppException(response.errors) - @validate_arguments def search_models( - self, - name: Optional[NotEmptyStr] = None, - type_: Optional[NotEmptyStr] = None, - project_id: Optional[int] = None, - task: Optional[NotEmptyStr] = None, - include_global: Optional[StrictBool] = True, + self, + name: Optional[NotEmptyStr] = None, + type_: Optional[NotEmptyStr] = None, + project_id: Optional[int] = None, + task: Optional[NotEmptyStr] = None, + include_global: Optional[StrictBool] = True, ): """Search for ML models. - :param name: search string - :type name: str - :param type_: ml model type string - :type type_: str - :param project_id: project id - :type project_id: int - :param task: training task - :type task: str - :param include_global: include global ml models - :type include_global: bool - - :return: ml model metadata - :rtype: list of dicts + :param name: search string + :type name: str + :param type_: ml model type string + :type type_: str + :param project_id: project id + :type project_id: int + :param task: training task + :type task: str + :param include_global: include global ml models + :type include_global: bool + + :return: ml model metadata + :rtype: list of dicts """ res = self.controller.search_models( name=name, @@ -2131,14 +2041,13 @@ def search_models( ) return res.data - @validate_arguments def upload_images_to_project( - self, - project: NotEmptyStr, - img_paths: List[NotEmptyStr], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: NotEmptyStr, + img_paths: List[NotEmptyStr], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads all images given in list of path objects in img_paths to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -2176,14 +2085,17 @@ def upload_images_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) + "%s already existing images found that won't be uploaded.", + len(duplicates), ) logger.info(f"Uploading {len(images_to_upload)} images to project {project}.") uploaded, failed_images, duplications = [], [], duplicates if not images_to_upload: return uploaded, failed_images, duplications if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: for _ in use_case.execute(): progress_bar.update(1) uploaded, failed_images, duplications = use_case.data @@ -2192,12 +2104,11 @@ def upload_images_to_project( return uploaded, failed_images, duplications raise AppException(use_case.response.errors) - @validate_arguments def aggregate_annotations_as_df( - self, - project_root: Union[NotEmptyStr, Path], - project_type: ProjectTypes, - folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, + self, + project_root: Union[NotEmptyStr, Path], + project_type: ProjectTypes, + folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, ): """Aggregate annotations as pandas dataframe from project root. @@ -2215,8 +2126,8 @@ def aggregate_annotations_as_df( :rtype: pandas DataFrame """ if project_type in ( - constances.ProjectType.VECTOR.name, - constances.ProjectType.PIXEL.name, + constances.ProjectType.VECTOR.name, + constances.ProjectType.PIXEL.name, ): from superannotate.lib.app.analytics.common import ( aggregate_image_annotations_as_df, @@ -2230,8 +2141,8 @@ def aggregate_annotations_as_df( folder_names=folder_names, ) elif project_type in ( - constances.ProjectType.VIDEO.name, - constances.ProjectType.DOCUMENT.name, + constances.ProjectType.VIDEO.name, + constances.ProjectType.DOCUMENT.name, ): from superannotate.lib.app.analytics.aggregators import DataAggregator @@ -2241,10 +2152,8 @@ def aggregate_annotations_as_df( folder_names=folder_names, ).aggregate_annotations_as_df() - @validate_arguments def delete_annotations( - self, - project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None + self, project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None ): """ Delete image annotations from a given list of images. @@ -2263,22 +2172,20 @@ def delete_annotations( if response.errors: raise AppException(response.errors) - @validate_arguments def validate_annotations( - self, - project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] + self, project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] ): """Validates given annotation JSON. - :param project_type: The project type Vector, Pixel, Video or Document - :type project_type: str + :param project_type: The project type Vector, Pixel, Video or Document + :type project_type: str - :param annotations_json: path to annotation JSON - :type annotations_json: Path-like (str or Path) + :param annotations_json: path to annotation JSON + :type annotations_json: Path-like (str or Path) - :return: The success of the validation - :rtype: bool - """ + :return: The success of the validation + :rtype: bool + """ with open(annotations_json) as file: annotation_data = json.loads(file.read()) response = Controller.validate_annotations( @@ -2292,10 +2199,11 @@ def validate_annotations( print(response.report) return False - @validate_arguments def add_contributors_to_project( - self, - project: NotEmptyStr, emails: conlist(EmailStr, min_items=1), role: AnnotatorRole + self, + project: NotEmptyStr, + emails: conlist(EmailStr, min_items=1), + role: AnnotatorRole, ) -> Tuple[List[str], List[str]]: """Add contributors to project. @@ -2318,10 +2226,8 @@ def add_contributors_to_project( raise AppException(response.errors) return response.data - @validate_arguments def invite_contributors_to_team( - self, - emails: conlist(EmailStr, min_items=1), admin: StrictBool = False + self, emails: conlist(EmailStr, min_items=1), admin: StrictBool = False ) -> Tuple[List[str], List[str]]: """Invites contributors to the team. @@ -2341,9 +2247,9 @@ def invite_contributors_to_team( raise AppException(response.errors) return response.data - @validate_arguments def get_annotations( - self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): + self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None + ): """Returns annotations for the given list of items. :param project: project name or folder path (e.g., “project1/folder1”). @@ -2356,16 +2262,14 @@ def get_annotations( :rtype: list of strs """ project_name, folder_name = extract_project_folder(project) - response = self.controller.get_annotations( - project_name, folder_name, items - ) + response = self.controller.get_annotations(project_name, folder_name, items) if response.errors: raise AppException(response.errors) return response.data - @validate_arguments def get_annotations_per_frame( - self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1): + self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1 + ): """Returns per frame annotations for the given video. @@ -2390,9 +2294,7 @@ def get_annotations_per_frame( raise AppException(response.errors) return response.data - @validate_arguments - def upload_priority_scores( - self, project: NotEmptyStr, scores: List[PriorityScore]): + def upload_priority_scores(self, project: NotEmptyStr, scores: List[PriorityScore]): """Returns per frame annotations for the given video. :param project: project name or folder path (e.g., “project1/folder1”) @@ -2413,9 +2315,7 @@ def upload_priority_scores( raise AppException(response.errors) return response.data - @validate_arguments - def get_integrations( - self): + def get_integrations(self): """Get all integrations per team :return: metadata objects of all integrations of the team. @@ -2427,12 +2327,11 @@ def get_integrations( integrations = response.data return BaseSerializer.serialize_iterable(integrations, ("name", "type", "root")) - @validate_arguments def attach_items_from_integrated_storage( - self, - project: NotEmptyStr, - integration: Union[NotEmptyStr, IntegrationEntity], - folder_path: Optional[NotEmptyStr] = None, + self, + project: NotEmptyStr, + integration: Union[NotEmptyStr, IntegrationEntity], + folder_path: Optional[NotEmptyStr] = None, ): """Link images from integrated external storage to SuperAnnotate. @@ -2444,7 +2343,7 @@ def attach_items_from_integrated_storage( :type integration: str or dict :param folder_path: Points to an exact folder/directory within given storage. - If None, items are fetched from the root directory. + If None, items are fetched from the root directory. :type folder_path: str """ project_name, folder_name = extract_project_folder(project) @@ -2456,9 +2355,7 @@ def attach_items_from_integrated_storage( if response.errors: raise AppException(response.errors) - @validate_arguments - def query( - self, project: NotEmptyStr, query: Optional[NotEmptyStr]): + def query(self, project: NotEmptyStr, query: Optional[NotEmptyStr]): """Return items that satisfy the given query. Query syntax should be in SuperAnnotate query language(https://doc.superannotate.com/docs/query-search-1). @@ -2477,10 +2374,10 @@ def query( raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) - @validate_arguments def get_item_metadata( - self, - project: NotEmptyStr, item_name: NotEmptyStr, + self, + project: NotEmptyStr, + item_name: NotEmptyStr, ): """Returns item metadata @@ -2499,15 +2396,14 @@ def get_item_metadata( raise AppException(response.errors) return BaseSerializer(response.data).serialize() - @validate_arguments def search_items( - self, - project: NotEmptyStr, - name_contains: NotEmptyStr = None, - annotation_status: Optional[AnnotationStatuses] = None, - annotator_email: Optional[NotEmptyStr] = None, - qa_email: Optional[NotEmptyStr] = None, - recursive: bool = False, + self, + project: NotEmptyStr, + name_contains: NotEmptyStr = None, + annotation_status: Optional[AnnotationStatuses] = None, + annotator_email: Optional[NotEmptyStr] = None, + qa_email: Optional[NotEmptyStr] = None, + recursive: bool = False, ): """Search items by filtering criteria. @@ -2561,32 +2457,29 @@ def search_items( raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) - @validate_arguments def attach_items( - self, - project: Union[NotEmptyStr, dict], - attachments: AttachmentArg, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", + self, + project: Union[NotEmptyStr, dict], + attachments: AttachmentArg, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", ): """Link items from external storage to SuperAnnotate using URLs. - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str - :param attachments: path to CSV file or list of dicts containing attachments URLs. - :type attachments: path-like (str or Path) or list of dicts + :param attachments: path to CSV file or list of dicts containing attachments URLs. + :type attachments: path-like (str or Path) or list of dicts - :param annotation_status: value to set the annotation statuses of the linked items + :param annotation_status: value to set the annotation statuses of the linked items “NotStarted” “InProgress” “QualityCheck” “Returned” “Completed” “Skipped” - :type annotation_status: str - - :return: None - """ + :type annotation_status: str + """ attachments = attachments.data project_name, folder_name = extract_project_folder(project) if attachments and isinstance(attachments[0], AttachmentDict): @@ -2597,9 +2490,10 @@ def attach_items( if count > 1 ] else: - unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv( - attachments - ) + ( + unique_attachments, + duplicate_attachments, + ) = get_name_url_duplicated_from_csv(attachments) if duplicate_attachments: logger.info("Dropping duplicates.") unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments) @@ -2625,13 +2519,12 @@ def attach_items( ] return uploaded, fails, duplicated - @validate_arguments def copy_items( - self, - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, - include_annotations: Optional[StrictBool] = True, + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, + include_annotations: Optional[StrictBool] = True, ): """Copy images in bulk between folders in a project @@ -2669,12 +2562,11 @@ def copy_items( return response.data - @validate_arguments def move_items( - self, - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, ): """Move images in bulk between folders in a project @@ -2705,12 +2597,11 @@ def move_items( raise AppException(response.errors) return response.data - @validate_arguments def set_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: AnnotationStatuses, - item_names: Optional[List[NotEmptyStr]] = None, + self, + project: Union[NotEmptyStr, dict], + annotation_status: AnnotationStatuses, + item_names: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of items @@ -2741,14 +2632,13 @@ def set_annotation_statuses( raise AppException(response.errors) return response.data - @validate_arguments def download_annotations( - self, - project: Union[NotEmptyStr, dict], - path: Union[str, Path] = None, - items: Optional[List[NotEmptyStr]] = None, - recursive: bool = False, - callback: Callable = None, + self, + project: Union[NotEmptyStr, dict], + path: Union[str, Path] = None, + items: Optional[List[NotEmptyStr]] = None, + recursive: bool = False, + callback: Callable = None, ): """Downloads annotation JSON files of the selected items to the local directory. diff --git a/src/superannotate/lib/app/mixp/decorators.py b/src/superannotate/lib/app/mixp/decorators.py deleted file mode 100644 index 5f2cc60ec..000000000 --- a/src/superannotate/lib/app/mixp/decorators.py +++ /dev/null @@ -1,134 +0,0 @@ -import functools -import sys -from inspect import signature - -from mixpanel import Mixpanel -from superannotate.logger import get_default_logger -from version import __version__ - -from .utils import parsers - -logger = get_default_logger() - - -def get_mp_instance() -> Mixpanel: - # if "api.annotate.online" in get_default_controller()._backend_url: - # return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") - return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") - - -def get_default(team_name, user_id, project_name=None): - return { - "SDK": True, - "Paid": True, - "Team": team_name, - "Team Owner": user_id, - "Project Name": project_name, - "Project Role": "Admin", - "Version": __version__, - } - - -class Trackable: - TEAM_DATA = None - INITIAL_EVENT = {"event_name": "SDK init", "properties": {}} - INITIAL_LOGGED = False - - def __init__(self, function, initial=False): - self.function = function - self._success = False - self._initial = initial - if initial: - self.track() - functools.update_wrapper(self, function) - - @staticmethod - def extract_arguments(function, *args, **kwargs) -> dict: - bound_arguments = signature(function).bind(*args, **kwargs) - bound_arguments.apply_defaults() - return dict(bound_arguments.arguments) - - @property - def team(self): - return get_default_controller().get_team() - - @staticmethod - def default_parser(function_name: str, kwargs: dict): - properties = {} - for key, value in kwargs: - if isinstance(value, (str, int, float, bool, str)): - properties[key] = value - elif isinstance(value, (list, set, tuple)): - properties[key] = len(value) - elif isinstance(value, dict): - properties[key] = value.keys() - elif hasattr(value, "__len__"): - properties[key] = len(value) - else: - properties[key] = str(value) - return {"event_name": function_name, "properties": properties} - - def track(self, *args, **kwargs): - try: - function_name = self.function.__name__ if self.function else "" - if self._initial: - data = self.INITIAL_EVENT - Trackable.INITIAL_LOGGED = True - self._success = True - else: - data = {} - arguments = self.extract_arguments(self.function, *args, **kwargs) - if hasattr(parsers, function_name): - try: - data = getattr(parsers, function_name)(**arguments) - except Exception: - pass - else: - data = self.default_parser(function_name, arguments) - event_name = data.get("event_name",) - properties = data.get("properties", {}) - team_data = self.team.data - user_id = team_data.creator_id - team_name = team_data.name - properties["Success"] = self._success - default = get_default( - team_name=team_name, - user_id=user_id, - project_name=properties.get("project_name", None), - ) - properties.pop("project_name", None) - properties = {**default, **properties} - - if "pytest" not in sys.modules: - get_mp_instance().track(user_id, event_name, properties) - except Exception: - pass - - def __call__(self, *args, **kwargs): - try: - controller = get_default_controller() - if controller: - self.__class__.TEAM_DATA = controller.get_team() - result = self.function(*args, **kwargs) - self._success = True - else: - raise Exception( - "SuperAnnotate config file not found." - " Please provide correct config file location to sa.init() or use " - "CLI's superannotate init to generate default location config file." - ) - except Exception as e: - self._success = False - logger.debug(str(e), exc_info=True) - raise e - else: - return result - finally: - try: - self.track(*args, **kwargs) - except Exception: - pass - - -if __name__ == "lib.app.mixp.decorators" and not Trackable.INITIAL_LOGGED: - Trackable(None, initial=True) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 1b1fed04c..c30d67469 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -1,3 +1,4 @@ +from os.path import expanduser from pathlib import Path from superannotate.lib.core.enums import AnnotationStatus @@ -12,8 +13,9 @@ from superannotate.lib.core.enums import UserRole -CONFIG_FILE_LOCATION = str(Path.home() / ".superannotate" / "config.json") -LOG_FILE_LOCATION = str(Path.home() / ".superannotate" / "sa.log") +CONFIG_PATH = "~/.superannotate/config.json" +CONFIG_FILE_LOCATION = expanduser(CONFIG_PATH) +LOG_FILE_LOCATION = expanduser("~/.superannotate/sa.log") BACKEND_URL = "https://api.annotate.online" DEFAULT_IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "tif", "tiff", "webp", "bmp"] @@ -141,6 +143,7 @@ ImageQuality, AnnotationStatus, CONFIG_FILE_LOCATION, + CONFIG_PATH, BACKEND_URL, DEFAULT_IMAGE_EXTENSIONS, DEFAULT_FILE_EXCLUDE_PATTERNS, diff --git a/src/superannotate/lib/core/data_handlers.py b/src/superannotate/lib/core/data_handlers.py index c0a1c442c..1eb77e684 100644 --- a/src/superannotate/lib/core/data_handlers.py +++ b/src/superannotate/lib/core/data_handlers.py @@ -266,7 +266,7 @@ def safe_time(timestamp): return "0" if str(timestamp) == "0.0" else timestamp def convert_timestamp(timestamp): - return timestamp / 10 ** 6 if timestamp else "0" + return timestamp / 10**6 if timestamp else "0" editor_data = { "instances": [], diff --git a/src/superannotate/lib/core/entities/project_entities.py b/src/superannotate/lib/core/entities/project_entities.py index 4e50080f3..244bca0c9 100644 --- a/src/superannotate/lib/core/entities/project_entities.py +++ b/src/superannotate/lib/core/entities/project_entities.py @@ -40,7 +40,10 @@ def to_dict(self): class BaseTimedEntity(BaseEntity): def __init__( - self, uuid: Any = None, createdAt: str = None, updatedAt: str = None, + self, + uuid: Any = None, + createdAt: str = None, + updatedAt: str = None, ): super().__init__(uuid) self.createdAt = createdAt @@ -222,7 +225,10 @@ def to_dict(self): class ImageInfoEntity(BaseEntity): def __init__( - self, uuid=None, width: float = None, height: float = None, + self, + uuid=None, + width: float = None, + height: float = None, ): super().__init__(uuid), self.width = width diff --git a/src/superannotate/lib/core/plugin.py b/src/superannotate/lib/core/plugin.py index 06f77115e..9ef6fec30 100644 --- a/src/superannotate/lib/core/plugin.py +++ b/src/superannotate/lib/core/plugin.py @@ -254,7 +254,10 @@ def frames_generator( @staticmethod def get_extractable_frames( - video_path: str, start_time, end_time, target_fps: float, + video_path: str, + start_time, + end_time, + target_fps: float, ): total = VideoPlugin.get_frames_count(video_path) total_with_fps = sum( diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index db8c24395..f93781170 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -2,7 +2,6 @@ import sys import threading import time -import uuid from collections import defaultdict from typing import Union @@ -33,9 +32,8 @@ def init_spin(self): class Session: - def __init__(self, pk: int): - self.pk = pk - self._uuid = str(uuid.uuid4()) + def __init__(self): + self.pk = threading.get_ident() self._data_dict = {} def __enter__(self): @@ -45,10 +43,26 @@ def __exit__(self, type, value, traceback): if type is not None: return False + def __del__(self): + globs = globals() + if "SESSIONS" in globs and globs["SESSIONS"].get(self.pk): + del globs["SESSIONS"][self.pk] + @property def data(self): return self._data_dict + @staticmethod + def get_current_session(): + globs = globals() + if not globs.get("SESSIONS") or not globs["SESSIONS"].get( + threading.get_ident() + ): + session = Session() + globals().update({"SESSIONS": {session.pk: session}}) + return session + return globs["SESSIONS"][threading.get_ident()] + def __setitem__(self, key, item): self._data_dict[key] = item @@ -58,9 +72,6 @@ def __getitem__(self, key): def __repr__(self): return repr(self._data_dict) - def __delitem__(self, key): - del self._data_dict[key] - def clear(self): return self._data_dict.clear() @@ -72,7 +83,7 @@ def __init__( log_warning: bool = True, disable_progress_bar: bool = False, log_debug: bool = True, - session: Session = None + session: Session = None, ): self.logger = get_default_logger() self._log_info = log_info diff --git a/src/superannotate/lib/core/repositories.py b/src/superannotate/lib/core/repositories.py index a94258a5c..f60c7b44e 100644 --- a/src/superannotate/lib/core/repositories.py +++ b/src/superannotate/lib/core/repositories.py @@ -67,7 +67,11 @@ def __init__(self, service: SuperannotateServiceProvider, project: ProjectEntity class BaseS3Repository(BaseManageableRepository): def __init__( - self, access_key: str, secret_key: str, session_token: str, bucket: str, + self, + access_key: str, + secret_key: str, + session_token: str, + bucket: str, ): self._session = boto3.Session( aws_access_key_id=access_key, diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index ea120e4ea..1861e8ea1 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -106,7 +106,11 @@ def get_download_token( raise NotImplementedError def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, ) -> dict: raise NotImplementedError @@ -177,7 +181,10 @@ def get_bulk_images( raise NotImplementedError def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, + team_id: int, + project_id: int, + folder_name: str, ): raise NotImplementedError @@ -187,12 +194,19 @@ def assign_folder( raise NotImplementedError def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: list, + self, + team_id: int, + project_id: int, + folder_name: str, + image_names: list, ): raise NotImplementedError def un_share_project( - self, team_id: int, project_id: int, user_id: str, + self, + team_id: int, + project_id: int, + user_id: str, ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 8b1666bba..c553db8a5 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -33,7 +33,7 @@ from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.types import PriorityScore -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.images import GetBulkImages from lib.core.usecases.images import ValidateAnnotationUseCase from lib.core.video_convertor import VideoFrameGenerator @@ -46,7 +46,7 @@ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -class UploadAnnotationsUseCase(BaseReportableUseCae): +class UploadAnnotationsUseCase(BaseReportableUseCase): MAX_WORKERS = 10 CHUNK_SIZE = 100 AUTH_DATA_CHUNK_SIZE = 500 @@ -312,7 +312,7 @@ def execute(self): return self._response -class UploadAnnotationUseCase(BaseReportableUseCae): +class UploadAnnotationUseCase(BaseReportableUseCase): def __init__( self, project: ProjectEntity, @@ -444,7 +444,10 @@ def prepare_annotations( handlers_chain.attach(LastActionHandler(team.creator_id)) return handlers_chain.handle(annotations) - def clean_json(self, json_data: dict,) -> Tuple[bool, dict]: + def clean_json( + self, + json_data: dict, + ) -> Tuple[bool, dict]: use_case = ValidateAnnotationUseCase( constances.ProjectType.get_name(self._project.type), annotation=json_data, @@ -500,7 +503,7 @@ def execute(self): return self._response -class GetAnnotations(BaseReportableUseCae): +class GetAnnotations(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -584,7 +587,7 @@ def execute(self): return self._response -class GetVideoAnnotationsPerFrame(BaseReportableUseCae): +class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -645,7 +648,7 @@ def execute(self): return self._response -class UploadPriorityScoresUseCase(BaseReportableUseCae): +class UploadPriorityScoresUseCase(BaseReportableUseCase): CHUNK_SIZE = 100 def __init__( @@ -733,7 +736,7 @@ def execute(self): return self._response -class DownloadAnnotations(BaseReportableUseCae): +class DownloadAnnotations(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -793,8 +796,9 @@ def get_postfix(self): def download_annotation_classes(self, path: str): classes = self._classes.get_all() - os.mkdir(f"{path}/classes") - with open(f"{path}/classes/classes.json", "w+") as file: + classes_path = Path(path) / "classes" + classes_path.mkdir(parents=True, exist_ok=True) + with open(classes_path / "classes.json", "w+") as file: json.dump([i.dict() for i in classes], file, indent=4) @staticmethod @@ -810,13 +814,12 @@ def coroutine_wrapper(coroutine): def execute(self): if self.is_valid(): - export_prefix = f"{self._project.name}{f'/{self._folder.name}' if not self._folder.is_root else ''}" export_path = str( self.destination - / Path(f"{export_prefix} {datetime.now().strftime('%B %d %Y %H_%M')}") + / Path(f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}") ) self.reporter.log_info( - f"Downloading the annotations of the requested items to {export_path} \nThis might take a while…" + f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" ) self.reporter.start_spinner() folders = [] @@ -869,7 +872,7 @@ def execute(self): self.reporter.stop_spinner() self.reporter.log_info( - f"SA-PYTHON-SDK - INFO - Downloaded annotations for {self.get_items_count(export_path)} items." + f"Downloaded annotations for {self.get_items_count(export_path)} items." ) self.download_annotation_classes(export_path) self._response.data = os.path.abspath(export_path) diff --git a/src/superannotate/lib/core/usecases/base.py b/src/superannotate/lib/core/usecases/base.py index 3f3e3750f..97ed22ee8 100644 --- a/src/superannotate/lib/core/usecases/base.py +++ b/src/superannotate/lib/core/usecases/base.py @@ -58,13 +58,13 @@ def execute(self) -> Iterable: raise NotImplementedError -class BaseReportableUseCae(BaseUseCase, metaclass=ABCMeta): +class BaseReportableUseCase(BaseUseCase, metaclass=ABCMeta): def __init__(self, reporter: Reporter): super().__init__() self.reporter = reporter -class BaseUserBasedUseCase(BaseReportableUseCae, metaclass=ABCMeta): +class BaseUserBasedUseCase(BaseReportableUseCase, metaclass=ABCMeta): """ class contain validation of unique emails """ diff --git a/src/superannotate/lib/core/usecases/folders.py b/src/superannotate/lib/core/usecases/folders.py index a9733652f..b98c79e88 100644 --- a/src/superannotate/lib/core/usecases/folders.py +++ b/src/superannotate/lib/core/usecases/folders.py @@ -142,7 +142,9 @@ def execute(self): class UpdateFolderUseCase(BaseUseCase): def __init__( - self, folders: BaseManageableRepository, folder: FolderEntity, + self, + folders: BaseManageableRepository, + folder: FolderEntity, ): super().__init__() self._folders = folders diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 31b4eeba0..41611999e 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -42,7 +42,7 @@ from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseInteractiveUseCase -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.projects import GetAnnotationClassesUseCase from PIL import UnidentifiedImageError @@ -581,7 +581,11 @@ def execute(self): image = ImagePlugin(io.BytesIO(file.read())) images = [ - Image("fuse", f"{self._image_path}___fuse.png", image.get_empty(),) + Image( + "fuse", + f"{self._image_path}___fuse.png", + image.get_empty(), + ) ] if self._generate_overlay: images.append( @@ -711,7 +715,9 @@ def execute(self): class GetS3ImageUseCase(BaseUseCase): def __init__( - self, s3_bucket, image_path: str, + self, + s3_bucket, + image_path: str, ): super().__init__() self._s3_bucket = s3_bucket @@ -1536,7 +1542,8 @@ def execute(self) -> Response: image_bytes = ( GetImageBytesUseCase( - image=image, backend_service_provider=self._backend_service, + image=image, + backend_service_provider=self._backend_service, ) .execute() .data @@ -1876,7 +1883,7 @@ def execute(self): return self._response -class GetImageAnnotationsUseCase(BaseReportableUseCae): +class GetImageAnnotationsUseCase(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -2438,7 +2445,7 @@ def execute(self) -> Response: return self._response -class UploadVideosAsImages(BaseReportableUseCae): +class UploadVideosAsImages(BaseReportableUseCase): def __init__( self, reporter: Reporter, diff --git a/src/superannotate/lib/core/usecases/integrations.py b/src/superannotate/lib/core/usecases/integrations.py index c61379de0..4f0c42d1e 100644 --- a/src/superannotate/lib/core/usecases/integrations.py +++ b/src/superannotate/lib/core/usecases/integrations.py @@ -9,10 +9,10 @@ from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases import BaseReportableUseCae +from lib.core.usecases import BaseReportableUseCase -class GetIntegrations(BaseReportableUseCae): +class GetIntegrations(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -32,7 +32,7 @@ def execute(self) -> Response: return self._response -class AttachIntegrations(BaseReportableUseCae): +class AttachIntegrations(BaseReportableUseCase): def __init__( self, reporter: Reporter, diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 0bb2fa294..4110cd340 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -18,10 +18,10 @@ from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase -class GetItem(BaseReportableUseCae): +class GetItem(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -74,7 +74,7 @@ def execute(self) -> Response: return self._response -class QueryEntities(BaseReportableUseCae): +class QueryEntities(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -125,7 +125,7 @@ def execute(self) -> Response: return self._response -class ListItems(BaseReportableUseCae): +class ListItems(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -187,7 +187,7 @@ def execute(self) -> Response: return self._response -class AttachItems(BaseReportableUseCae): +class AttachItems(BaseReportableUseCase): CHUNK_SIZE = 500 def __init__( @@ -288,7 +288,7 @@ def execute(self) -> Response: return self._response -class CopyItems(BaseReportableUseCae): +class CopyItems(BaseReportableUseCase): """ Copy items in bulk between folders in a project. Return skipped item names. @@ -362,13 +362,15 @@ def execute(self): if items_to_copy: for i in range(0, len(items_to_copy), self.CHUNK_SIZE): chunk_to_copy = items_to_copy[i : i + self.CHUNK_SIZE] # noqa: E203 - poll_id = self._backend_service.copy_items_between_folders_transaction( - team_id=self._project.team_id, - project_id=self._project.id, - from_folder_id=self._from_folder.uuid, - to_folder_id=self._to_folder.uuid, - items=chunk_to_copy, - include_annotations=self._include_annotations, + poll_id = ( + self._backend_service.copy_items_between_folders_transaction( + team_id=self._project.team_id, + project_id=self._project.id, + from_folder_id=self._from_folder.uuid, + to_folder_id=self._to_folder.uuid, + items=chunk_to_copy, + include_annotations=self._include_annotations, + ) ) if not poll_id: skipped_items.extend(chunk_to_copy) @@ -404,7 +406,7 @@ def execute(self): return self._response -class MoveItems(BaseReportableUseCae): +class MoveItems(BaseReportableUseCase): CHUNK_SIZE = 1000 def __init__( @@ -479,7 +481,7 @@ def execute(self): return self._response -class SetAnnotationStatues(BaseReportableUseCae): +class SetAnnotationStatues(BaseReportableUseCase): CHUNK_SIZE = 500 ERROR_MESSAGE = "Failed to change status" diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 402fbdf0d..3d6ec6ff9 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -26,7 +26,7 @@ from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.images import GetBulkImages from superannotate.logger import get_default_logger @@ -179,7 +179,7 @@ def execute(self): return self._response -class DownloadExportUseCase(BaseReportableUseCae): +class DownloadExportUseCase(BaseReportableUseCase): def __init__( self, service: SuperannotateServiceProvider, @@ -635,7 +635,9 @@ def execute(self): class SearchMLModels(BaseUseCase): def __init__( - self, ml_models_repo: BaseManageableRepository, condition: Condition, + self, + ml_models_repo: BaseManageableRepository, + condition: Condition, ): super().__init__() self._ml_models = ml_models_repo diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index fb97e87de..8461619f9 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -20,7 +20,7 @@ from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.base import BaseUserBasedUseCase from requests.exceptions import RequestException @@ -31,7 +31,10 @@ class GetProjectsUseCase(BaseUseCase): def __init__( - self, condition: Condition, team_id: int, projects: BaseManageableRepository, + self, + condition: Condition, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() self._condition = condition @@ -47,7 +50,10 @@ def execute(self): class GetProjectByNameUseCase(BaseUseCase): def __init__( - self, name: str, team_id: int, projects: BaseManageableRepository, + self, + name: str, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() self._name = name @@ -300,7 +306,10 @@ def execute(self): class DeleteProjectUseCase(BaseUseCase): def __init__( - self, project_name: str, team_id: int, projects: BaseManageableRepository, + self, + project_name: str, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() @@ -310,7 +319,9 @@ def __init__( def execute(self): use_case = GetProjectByNameUseCase( - name=self._project_name, team_id=self._team_id, projects=self._projects, + name=self._project_name, + team_id=self._team_id, + projects=self._projects, ) project_response = use_case.execute() if project_response.data: @@ -373,7 +384,7 @@ def execute(self): return self._response -class CloneProjectUseCase(BaseReportableUseCae): +class CloneProjectUseCase(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -696,7 +707,9 @@ def execute(self): class GetAnnotationClassesUseCase(BaseUseCase): def __init__( - self, classes: BaseManageableRepository, condition: Condition = None, + self, + classes: BaseManageableRepository, + condition: Condition = None, ): super().__init__() self._classes = classes diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f54652fed..f5639ec4e 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1,7 +1,6 @@ import copy import io import os -import threading from abc import ABCMeta from pathlib import Path from typing import Callable @@ -25,7 +24,6 @@ from lib.core.entities.integrations import IntegrationEntity from lib.core.exceptions import AppException from lib.core.reporter import Reporter -from lib.core.reporter import Session from lib.core.response import Response from lib.infrastructure.helpers import timed_lru_cache from lib.infrastructure.repositories import AnnotationClassRepository @@ -80,14 +78,6 @@ def __init__(self, token: str, host: str, ssl_verify: bool, version: str): self._team_name = None self._reporter = None - def get_session(self): - pk = threading.get_ident() - try: - return self.SESSIONS[pk] - except KeyError: - self.SESSIONS[threading.get_ident()] = Session(pk) - return self.SESSIONS[pk] - @staticmethod def validate_token(token: str): try: @@ -140,7 +130,7 @@ def teams(self): @property def team_data(self): if not self._team_data: - self._team_data = self.get_team() + self._team_data = self.get_team().data return self._team_data @property @@ -168,19 +158,20 @@ def team_id(self) -> int: @staticmethod def get_default_reporter( - log_info: bool = True, - log_warning: bool = True, - disable_progress_bar: bool = False, - log_debug: bool = True + log_info: bool = True, + log_warning: bool = True, + disable_progress_bar: bool = False, + log_debug: bool = True, ) -> Reporter: import inspect + session = None loop_limit = 16 current_frame = inspect.currentframe() while loop_limit: loop_limit -= 1 try: - session = current_frame.f_locals['session'] + session = current_frame.f_locals["session"] if session: break except KeyError: @@ -255,11 +246,11 @@ def get_folder_name(name: str = None): return "root" def search_project( - self, - name: str = None, - include_complete_image_count=False, - statuses: Union[List[str], Tuple[str]] = (), - **kwargs, + self, + name: str = None, + include_complete_image_count=False, + statuses: Union[List[str], Tuple[str]] = (), + **kwargs, ) -> Response: condition = Condition.get_empty_condition() if name: @@ -275,19 +266,21 @@ def search_project( condition &= build_condition(**kwargs) use_case = usecases.GetProjectsUseCase( - condition=condition, projects=self.projects, team_id=self.team_id, + condition=condition, + projects=self.projects, + team_id=self.team_id, ) return use_case.execute() def create_project( - self, - name: str, - description: str, - project_type: str, - settings: Iterable[SettingEntity] = None, - classes: Iterable = tuple(), - workflows: Iterable = tuple(), - **extra_kwargs, + self, + name: str, + description: str, + project_type: str, + settings: Iterable[SettingEntity] = None, + classes: Iterable = tuple(), + workflows: Iterable = tuple(), + **extra_kwargs, ) -> Response: try: @@ -320,7 +313,9 @@ def create_project( def delete_project(self, name: str): use_case = usecases.DeleteProjectUseCase( - project_name=name, team_id=self.team_id, projects=self.projects, + project_name=name, + team_id=self.team_id, + projects=self.projects, ) return use_case.execute() @@ -330,14 +325,14 @@ def update_project(self, name: str, project_data: dict) -> Response: return use_case.execute() def upload_image_to_project( - self, - project_name: str, - folder_name: str, - image_name: str, - image: Union[str, io.BytesIO] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + image_name: str, + image: Union[str, io.BytesIO] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -365,13 +360,13 @@ def upload_image_to_project( ).execute() def upload_images_to_project( - self, - project_name: str, - folder_name: str, - paths: List[str], - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + paths: List[str], + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -391,16 +386,16 @@ def upload_images_to_project( ) def upload_images_from_folder_to_project( - self, - project_name: str, - folder_name: str, - folder_path: str, - extensions: Optional[List[str]] = None, - annotation_status: str = None, - exclude_file_patterns: Optional[List[str]] = None, - recursive_sub_folders: Optional[bool] = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + folder_path: str, + extensions: Optional[List[str]] = None, + annotation_status: str = None, + exclude_file_patterns: Optional[List[str]] = None, + recursive_sub_folders: Optional[bool] = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -423,20 +418,22 @@ def upload_images_from_folder_to_project( ) def clone_project( - self, - name: str, - from_name: str, - project_description: str, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, + self, + name: str, + from_name: str, + project_description: str, + copy_annotation_classes=True, + copy_settings=True, + copy_workflow=True, + copy_contributors=False, ): project = self._get_project(from_name) project_to_create = copy.copy(project) reporter = self.get_default_reporter() - reporter.track("external", project.upload_state == constances.UploadState.EXTERNAL.value) + reporter.track( + "external", project.upload_state == constances.UploadState.EXTERNAL.value + ) project_to_create.name = name if project_description is not None: project_to_create.description = project_description @@ -457,12 +454,12 @@ def clone_project( return use_case.execute() def interactive_attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, + self, + project_name: str, + files: List[ImageEntity], + folder_name: str = None, + annotation_status: str = None, + upload_state_code: int = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -482,7 +479,9 @@ def create_folder(self, project: str, folder_name: str): name=folder_name, project_id=project.id, team_id=project.team_id ) use_case = usecases.CreateFolderUseCase( - project=project, folder=folder, folders=self.folders, + project=project, + folder=folder, + folders=self.folders, ) return use_case.execute() @@ -497,7 +496,7 @@ def get_folder(self, project_name: str, folder_name: str): return use_case.execute() def search_folders( - self, project_name: str, folder_name: str = None, include_users=False, **kwargs + self, project_name: str, folder_name: str = None, include_users=False, **kwargs ): condition = build_condition(**kwargs) project = self._get_project(project_name) @@ -524,12 +523,12 @@ def delete_folders(self, project_name: str, folder_names: List[str]): return use_case.execute() def prepare_export( - self, - project_name: str, - folder_names: List[str], - include_fuse: bool, - only_pinned: bool, - annotation_statuses: List[str] = None, + self, + project_name: str, + folder_names: List[str], + include_fuse: bool, + only_pinned: bool, + annotation_statuses: List[str] = None, ): project = self._get_project(project_name) @@ -554,11 +553,11 @@ def search_team_contributors(self, **kwargs): return use_case.execute() def search_images( - self, - project_name: str, - folder_path: str = None, - annotation_status: str = None, - image_name_prefix: str = None, + self, + project_name: str, + folder_path: str = None, + annotation_status: str = None, + image_name_prefix: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -573,7 +572,10 @@ def search_images( return use_case.execute() def _get_image( - self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, + self, + project: ProjectEntity, + image_name: str, + folder: FolderEntity = None, ) -> ImageEntity: response = usecases.GetImageUseCase( service=self._backend_client, @@ -587,7 +589,7 @@ def _get_image( return response.data def get_image( - self, project_name: str, image_name: str, folder_path: str = None + self, project_name: str, image_name: str, folder_path: str = None ) -> ImageEntity: project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -598,18 +600,21 @@ def update_folder(self, project_name: str, folder_name: str, folder_data: dict): folder = self._get_folder(project, folder_name) for field, value in folder_data.items(): setattr(folder, field, value) - use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder, ) + use_case = usecases.UpdateFolderUseCase( + folders=self.folders, + folder=folder, + ) return use_case.execute() def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, + copy_annotation_status: bool = False, + move: bool = False, ): from_project = self._get_project(from_project_name) to_project = self._get_project(to_project_name) @@ -632,12 +637,12 @@ def copy_image( return use_case.execute() def copy_image_annotation_classes( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, ): from_project = self._get_project(from_project_name) from_folder = self._get_folder(from_project, from_folder_name) @@ -672,7 +677,7 @@ def copy_image_annotation_classes( return use_case.execute() def update_image( - self, project_name: str, image_name: str, folder_name: str = None, **kwargs + self, project_name: str, image_name: str, folder_name: str = None, **kwargs ): image = self.get_image( project_name=project_name, image_name=image_name, folder_path=folder_name @@ -683,13 +688,13 @@ def update_image( return use_case.execute() def bulk_copy_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], - include_annotations: bool, - include_pin: bool, + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], + include_annotations: bool, + include_pin: bool, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -706,11 +711,11 @@ def bulk_copy_images( return use_case.execute() def bulk_move_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -725,13 +730,13 @@ def bulk_move_images( return use_case.execute() def get_project_metadata( - self, - project_name: str, - include_annotation_classes: bool = False, - include_settings: bool = False, - include_workflow: bool = False, - include_contributors: bool = False, - include_complete_image_count: bool = False, + self, + project_name: str, + include_annotation_classes: bool = False, + include_settings: bool = False, + include_workflow: bool = False, + include_contributors: bool = False, + include_complete_image_count: bool = False, ): project = self._get_project(project_name) @@ -806,11 +811,11 @@ def set_project_settings(self, project_name: str, new_settings: List[dict]): return use_case.execute() def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, + self, + project_name: str, + folder_name: str, + image_names: list, + annotation_status: str, ): project_entity = self._get_project(project_name) folder_entity = self._get_folder(project_entity, folder_name) @@ -828,7 +833,10 @@ def set_images_annotation_statuses( return use_case.execute() def delete_images( - self, project_name: str, folder_name: str, image_names: List[str] = None, + self, + project_name: str, + folder_name: str, + image_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -843,7 +851,7 @@ def delete_images( return use_case.execute() def assign_images( - self, project_name: str, folder_name: str, image_names: list, user: str + self, project_name: str, folder_name: str, image_names: list, user: str ): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) @@ -871,7 +879,9 @@ def un_assign_folder(self, project_name: str, folder_name: str): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) use_case = usecases.UnAssignFolderUseCase( - service=self._backend_client, project_entity=project_entity, folder=folder, + service=self._backend_client, + project_entity=project_entity, + folder=folder, ) return use_case.execute() @@ -896,7 +906,7 @@ def un_share_project(self, project_name: str, user_id: str): return use_case.execute() def download_image_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -924,7 +934,7 @@ def get_exports(self, project_name: str, return_metadata: bool): return use_case.execute() def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool + self, project_name: str, folder_name: str, with_all_subfolders: bool ): project = self._get_project(project_name) @@ -940,12 +950,12 @@ def get_project_image_count( return use_case.execute() def create_annotation_class( - self, - project_name: str, - name: str, - color: str, - attribute_groups: List[dict], - class_type: str, + self, + project_name: str, + name: str, + color: str, + attribute_groups: List[dict], + class_type: str, ): project = self._get_project(project_name) annotation_classes = AnnotationClassRepository( @@ -966,7 +976,8 @@ def delete_annotation_class(self, project_name: str, annotation_class_name: str) use_case = usecases.DeleteAnnotationClassUseCase( annotation_class_name=annotation_class_name, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), project_name=project_name, ) @@ -977,7 +988,8 @@ def get_annotation_class(self, project_name: str, annotation_class_name: str): use_case = usecases.GetAnnotationClassUseCase( annotation_class_name=annotation_class_name, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), ) return use_case.execute() @@ -986,7 +998,8 @@ def download_annotation_classes(self, project_name: str, download_path: str): project = self._get_project(project_name) use_case = usecases.DownloadAnnotationClassesUseCase( annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), download_path=download_path, project_name=project_name, @@ -999,7 +1012,8 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) use_case = usecases.CreateAnnotationClassesUseCase( service=self._backend_client, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), annotation_classes=annotation_classes, project=project, @@ -1007,15 +1021,15 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) return use_case.execute() def download_image( - self, - project_name: str, - image_name: str, - download_path: str, - folder_name: str = None, - image_variant: str = None, - include_annotations: bool = None, - include_fuse: bool = None, - include_overlay: bool = None, + self, + project_name: str, + image_name: str, + download_path: str, + folder_name: str = None, + image_variant: str = None, + include_annotations: bool = None, + include_fuse: bool = None, + include_overlay: bool = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1055,13 +1069,13 @@ def set_project_workflow(self, project_name: str, steps: list): return use_case.execute() def upload_annotations_from_folder( - self, - project_name: str, - folder_name: str, - annotation_paths: List[str], - client_s3_bucket=None, - is_pre_annotations: bool = False, - folder_path: str = None, + self, + project_name: str, + folder_name: str, + annotation_paths: List[str], + client_s3_bucket=None, + is_pre_annotations: bool = False, + folder_path: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1069,7 +1083,7 @@ def upload_annotations_from_folder( project=project, folder=folder, images=self.images, - team=self.team_data.data, + team=self.team_data, annotation_paths=annotation_paths, backend_service_provider=self._backend_client, annotation_classes=AnnotationClassRepository( @@ -1087,13 +1101,13 @@ def upload_annotations_from_folder( return use_case.execute() def upload_image_annotations( - self, - project_name: str, - folder_name: str, - image_name: str, - annotations: dict, - mask: io.BytesIO = None, - verbose: bool = True, + self, + project_name: str, + folder_name: str, + image_name: str, + annotations: dict, + mask: io.BytesIO = None, + verbose: bool = True, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1105,7 +1119,7 @@ def upload_image_annotations( project=project, folder=folder, images=self.images, - team=self.team_data.data, + team=self.team_data, annotation_classes=AnnotationClassRepository( service=self._backend_client, project=project ).get_all(), @@ -1135,12 +1149,12 @@ def delete_model(self, model_id: int): return use_case.execute() def download_export( - self, - project_name: str, - export_name: str, - folder_path: str, - extract_zip_contents: bool, - to_s3_bucket: bool, + self, + project_name: str, + export_name: str, + folder_path: str, + extract_zip_contents: bool, + to_s3_bucket: bool, ): project = self._get_project(project_name) use_case = usecases.DownloadExportUseCase( @@ -1173,14 +1187,14 @@ def download_ml_model(self, model_data: dict, download_path: str): return use_case.execute() def benchmark( - self, - project_name: str, - ground_truth_folder_name: str, - folder_names: List[str], - export_root: str, - image_list: List[str], - annot_type: str, - show_plots: bool, + self, + project_name: str, + ground_truth_folder_name: str, + folder_names: List[str], + export_root: str, + image_list: List[str], + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) export_response = self.prepare_export( @@ -1215,13 +1229,13 @@ def benchmark( return use_case.execute() def consensus( - self, - project_name: str, - folder_names: list, - export_path: str, - image_list: list, - annot_type: str, - show_plots: bool, + self, + project_name: str, + folder_names: list, + export_path: str, + image_list: list, + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1254,7 +1268,7 @@ def consensus( return use_case.execute() def run_prediction( - self, project_name: str, images_list: list, model_name: str, folder_name: str + self, project_name: str, images_list: list, model_name: str, folder_name: str ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1272,7 +1286,10 @@ def run_prediction( return use_case.execute() def list_images( - self, project_name: str, annotation_status: str = None, name_prefix: str = None, + self, + project_name: str, + annotation_status: str = None, + name_prefix: str = None, ): project = self._get_project(project_name) @@ -1285,12 +1302,12 @@ def list_images( return use_case.execute() def search_models( - self, - name: str, - model_type: str = None, - project_id: int = None, - task: str = None, - include_global: bool = True, + self, + name: str, + model_type: str = None, + project_id: int = None, + task: str = None, + include_global: bool = True, ): ml_models_repo = MLModelRepository( service=self._backend_client, team_id=self.team_id @@ -1313,10 +1330,10 @@ def search_models( return use_case.execute() def delete_annotations( - self, - project_name: str, - folder_name: str, - item_names: Optional[List[str]] = None, + self, + project_name: str, + folder_name: str, + item_names: Optional[List[str]] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1330,7 +1347,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = False ): use_case = usecases.ValidateAnnotationUseCase( project_type, @@ -1367,17 +1384,17 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): return use_case.execute() def upload_videos( - self, - project_name: str, - folder_name: str, - paths: List[str], - start_time: float, - extensions: List[str] = None, - exclude_file_patterns: List[str] = None, - end_time: Optional[float] = None, - target_fps: Optional[int] = None, - annotation_status: Optional[str] = None, - image_quality_in_editor: Optional[str] = None, + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1403,7 +1420,7 @@ def upload_videos( return use_case.execute() def get_annotations( - self, project_name: str, folder_name: str, item_names: List[str], logging=True + self, project_name: str, folder_name: str, item_names: List[str], logging=True ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1418,7 +1435,7 @@ def get_annotations( return use_case.execute() def get_annotations_per_frame( - self, project_name: str, folder_name: str, video_name: str, fps: int + self, project_name: str, folder_name: str, video_name: str, fps: int ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1435,7 +1452,7 @@ def get_annotations_per_frame( return use_case.execute() def upload_priority_scores( - self, project_name, folder_name, scores, project_folder_name + self, project_name, folder_name, scores, project_folder_name ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1450,27 +1467,27 @@ def upload_priority_scores( return use_case.execute() def get_integrations(self): - team = self.team_data.data + team = self.team_data use_cae = usecases.GetIntegrations( reporter=self.get_default_reporter(), - team=self.team_data.data, + team=self.team_data, integrations=self.get_integrations_repo(team_id=team.uuid), ) return use_cae.execute() def attach_integrations( - self, - project_name: str, - folder_name: str, - integration: IntegrationEntity, - folder_path: str, + self, + project_name: str, + folder_name: str, + integration: IntegrationEntity, + folder_path: str, ): - team = self.team_data.data + team = self.team_data project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.AttachIntegrations( reporter=self.get_default_reporter(), - team=self.team_data.data, + team=self.team_data, backend_service=self.backend_client, project=project, folder=folder, @@ -1506,15 +1523,15 @@ def get_item(self, project_name: str, folder_name: str, item_name: str): return use_case.execute() def list_items( - self, - project_name: str, - folder_name: str, - name_contains: str = None, - annotation_status: str = None, - annotator_email: str = None, - qa_email: str = None, - recursive: bool = False, - **kwargs, + self, + project_name: str, + folder_name: str, + name_contains: str = None, + annotation_status: str = None, + annotator_email: str = None, + qa_email: str = None, + recursive: bool = False, + **kwargs, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1546,11 +1563,11 @@ def list_items( return use_case.execute() def attach_items( - self, - project_name: str, - folder_name: str, - attachments: List[AttachmentEntity], - annotation_status: str, + self, + project_name: str, + folder_name: str, + attachments: List[AttachmentEntity], + annotation_status: str, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1566,12 +1583,12 @@ def attach_items( return use_case.execute() def copy_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, - include_annotations: bool = False, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, + include_annotations: bool = False, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) @@ -1590,11 +1607,11 @@ def copy_items( return use_case.execute() def move_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) @@ -1612,11 +1629,11 @@ def move_items( return use_case.execute() def set_annotation_statuses( - self, - project_name: str, - folder_name: str, - annotation_status: str, - item_names: List[str] = None, + self, + project_name: str, + folder_name: str, + annotation_status: str, + item_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 9d27ae698..4ddc12351 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -70,7 +70,7 @@ def __init__( @property def assets_provider_url(self): if self.api_url != constance.BACKEND_URL: - return "https://assets-provider.devsuperannotate.com/api/v1/" + return "https://sa-assets-provider.us-west-2.elasticbeanstalk.com/api/v1/" return "https://assets-provider.superannotate.com/api/v1/" @timed_lru_cache(seconds=360) @@ -233,7 +233,7 @@ class SuperannotateBackendService(BaseBackendService): URL_DELETE_ANNOTATIONS = "annotations/remove" URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus" URL_GET_LIMITS = "project/{}/limitationDetails" - URL_GET_ANNOTATIONS = "images/annotations/stream" + URL_GET_ANNOTATIONS = "annotations/stream" URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" URL_GET_INTEGRATIONS = "integrations" URL_ATTACH_INTEGRATIONS = "image/integration/create" @@ -299,7 +299,11 @@ def get_download_token( return response.json() def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, ): download_token_url = urljoin( self.api_url, @@ -733,7 +737,11 @@ def assign_images( return res.ok def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: List[str], + self, + team_id: int, + project_id: int, + folder_name: str, + image_names: List[str], ): un_assign_images_url = urljoin(self.api_url, self.URL_ASSIGN_IMAGES) res = self._request( @@ -749,7 +757,10 @@ def un_assign_images( return res.ok def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, + team_id: int, + project_id: int, + folder_name: str, ): un_assign_folder_url = urljoin(self.api_url, self.URL_ASSIGN_FOLDER) res = self._request( diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 3be1cf8d7..844c5fe36 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -19,7 +19,7 @@ def __init__( self._headers = headers self._annotations = [] self._reporter = reporter - self._callback = callback + self._callback: Callable = callback self._map_function = map_function async def fetch( @@ -88,9 +88,10 @@ async def get_data( return self._annotations @staticmethod - def _store_annotation(path, postfix, annotation: dict): + def _store_annotation(path, postfix, annotation: dict, callback: Callable = None): os.makedirs(path, exist_ok=True) with open(f"{path}/{annotation['metadata']['name']}{postfix}", "w") as file: + annotation = callback(annotation) if callback else annotation json.dump(annotation, file) def _process_data(self, data): @@ -122,7 +123,8 @@ async def download_data( self._store_annotation( download_path, postfix, - self._callback(annotation) if self._callback else annotation, + annotation, + self._callback, ) else: async for annotation in self.fetch( @@ -131,5 +133,6 @@ async def download_data( self._store_annotation( download_path, postfix, - self._callback(annotation) if self._callback else annotation, + annotation, + self._callback ) diff --git a/tests/integration/annotations/test_annotations_pre_processing.py b/tests/integration/annotations/test_annotations_pre_processing.py index 574fce0ac..9653d658c 100644 --- a/tests/integration/annotations/test_annotations_pre_processing.py +++ b/tests/integration/annotations/test_annotations_pre_processing.py @@ -46,7 +46,7 @@ def test_annotation_last_action_and_creation_type(self, reporter): self.assertEqual(instance["creationType"], CreationTypeEnum.PRE_ANNOTATION.value) self.assertEqual( type(annotation["metadata"]["lastAction"]["email"]), - type(sa.controller.team_data.data.creator_id) + type(sa.controller.team_data.creator_id) ) self.assertEqual( type(annotation["metadata"]["lastAction"]["timestamp"]), diff --git a/tests/integration/annotations/test_download_annotations.py b/tests/integration/annotations/test_download_annotations.py index 48fb3587e..1083d2c76 100644 --- a/tests/integration/annotations/test_download_annotations.py +++ b/tests/integration/annotations/test_download_annotations.py @@ -62,3 +62,14 @@ def test_download_annotations_from_folders(self): with tempfile.TemporaryDirectory() as temp_dir: annotations_path = sa.download_annotations(f"{self.PROJECT_NAME}", temp_dir) self.assertEqual(len(os.listdir(annotations_path)), 5) + + @pytest.mark.flaky(reruns=3) + def test_download_annotations_from_folders(self): + sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) + sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME_2) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" + ) + with tempfile.TemporaryDirectory() as temp_dir: + annotations_path = sa.download_annotations(f"{self.PROJECT_NAME}", temp_dir) + self.assertEqual(len(os.listdir(annotations_path)), 1) \ No newline at end of file diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index 4871c3ca0..9dfae7102 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -25,7 +25,7 @@ class TestGetAnnotations(BaseTestCase): def folder_path(self): return os.path.join(Path(__file__).parent.parent.parent, self.TEST_FOLDER_PATH) - @pytest.mark.flaky(reruns=3) + # @pytest.mark.flaky(reruns=3) def test_get_annotations(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" diff --git a/tests/integration/mixpanel/test_individual_fuinctions.py b/tests/integration/mixpanel/test_individual_fuinctions.py new file mode 100644 index 000000000..685dede23 --- /dev/null +++ b/tests/integration/mixpanel/test_individual_fuinctions.py @@ -0,0 +1,30 @@ +import os +from unittest import TestCase +from unittest.mock import patch + +from src.superannotate import AppException +from src.superannotate import __version__ +from src.superannotate import class_distribution +from tests import DATA_SET_PATH + + +class TestDocumentUrls(TestCase): + PROJECT_NAME = "TEST_MIX" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PATH = "data_set" + + @property + def folder_path(self): + return os.path.join(DATA_SET_PATH, self.TEST_FOLDER_PATH) + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_get_team_metadata(self, track_method): + try: + class_distribution(os.path.join(self.folder_path, "sample_project_vector"), "test") + except AppException: + pass + data = list(track_method.call_args)[0][2] + assert not data["Success"] + assert data["Version"] == __version__ + assert data["project_names"] == "test" diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py new file mode 100644 index 000000000..f99b57768 --- /dev/null +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -0,0 +1,139 @@ +import copy +import threading +from unittest import TestCase +from unittest.mock import patch + +from src.superannotate import SAClient +from src.superannotate import AppException +from src.superannotate import __version__ + + +class TestDocumentUrls(TestCase): + CLIENT = SAClient() + TEAM_DATA = CLIENT.get_team_metadata() + BLANK_PAYLOAD = { + "SDK": True, + "Team": TEAM_DATA["name"], + "Team Owner": TEAM_DATA["creator_id"], + "Version": __version__, + "Success": True + } + PROJECT_NAME = "TEST_MIX" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PATH = "data_set" + + @classmethod + def setUpClass(cls) -> None: + cls.tearDownClass() + print(cls.PROJECT_NAME) + cls._project = cls.CLIENT.create_project( + cls.PROJECT_NAME, cls.PROJECT_DESCRIPTION, cls.PROJECT_TYPE + ) + + @classmethod + def tearDownClass(cls) -> None: + cls._safe_delete_project(cls.PROJECT_NAME) + + + @classmethod + def _safe_delete_project(cls, project_name): + projects = cls.CLIENT.search_projects(project_name, return_metadata=True) + for project in projects: + try: + cls.CLIENT.delete_project(project) + except Exception: + raise + + @property + def default_payload(self): + return copy.copy(self.BLANK_PAYLOAD) + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_get_team_metadata(self, track_method): + team = self.CLIENT.get_team_metadata() + team_owner = team["creator_id"] + result = list(track_method.call_args)[0] + payload = self.default_payload + assert result[0] == team_owner + assert result[1] == "get_team_metadata" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_search_team_contributors(self, track_method): + kwargs = { + "email": "user@supernnotate.com", + "first_name": "first_name", + "last_name": "last_name", + "return_metadata": False} + self.CLIENT.search_team_contributors(**kwargs) + result = list(track_method.call_args)[0] + payload = self.default_payload + payload.update(kwargs) + assert result[1] == "search_team_contributors" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_search_projects(self, track_method): + kwargs = { + "name": self.PROJECT_NAME, + "include_complete_image_count": True, + "status": "NotStarted", + "return_metadata": False} + self.CLIENT.search_projects(**kwargs) + result = list(track_method.call_args)[0] + payload = self.default_payload + payload.update(kwargs) + assert result[1] == "search_projects" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_create_project(self, track_method): + kwargs = { + "project_name": self.PROJECT_NAME, + "project_description": self.PROJECT_DESCRIPTION, + "project_type": self.PROJECT_TYPE, + "settings": {"a": 1, "b": 2} + } + try: + self.CLIENT.create_project(**kwargs) + except AppException: + pass + result = list(track_method.call_args)[0] + payload = self.default_payload + payload["Success"] = False + payload.update(kwargs) + payload["settings"] = list(kwargs["settings"].keys()) + assert result[1] == "create_project" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_create_project_multi_thread(self, track_method): + project_1 = self.PROJECT_NAME + "_1" + project_2 = self.PROJECT_NAME + "_2" + try: + kwargs_1 = { + "project_name": project_1, + "project_description": self.PROJECT_DESCRIPTION, + "project_type": self.PROJECT_TYPE, + } + kwargs_2 = { + "project_name": project_2, + "project_description": self.PROJECT_DESCRIPTION, + "project_type": self.PROJECT_TYPE, + } + thread_1 = threading.Thread(target=self.CLIENT.create_project, kwargs=kwargs_1) + thread_2 = threading.Thread(target=self.CLIENT.create_project, kwargs=kwargs_2) + thread_1.start() + thread_2.start() + thread_1.join() + thread_2.join() + r1, r2 = track_method.call_args_list + r1_pr_name = r1[0][2].pop("project_name") + r2_pr_name = r2[0][2].pop("project_name") + assert r1_pr_name == project_1 + assert r2_pr_name == project_2 + assert r1[0][2] == r2[0][2] + finally: + self._safe_delete_project(project_1) + self._safe_delete_project(project_2) \ No newline at end of file diff --git a/tests/integration/test_assign_images.py b/tests/integration/test_assign_images.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/integration/test_attach_document_urls.py b/tests/integration/test_attach_document_urls.py deleted file mode 100644 index 64d434728..000000000 --- a/tests/integration/test_attach_document_urls.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -from os.path import dirname -from os.path import join - -from src.superannotate import SAClient -sa = SAClient() -from src.superannotate import AppException -import src.superannotate.lib.core as constances -from tests.integration.base import BaseTestCase - - -class TestDocumentUrls(BaseTestCase): - PROJECT_NAME = "document attach urls" - PATH_TO_URLS = "csv_files/text_urls.csv" - PATH_TO_50K_URLS = "501_urls.csv" - PROJECT_DESCRIPTION = "desc" - PROJECT_TYPE = "Document" diff --git a/tests/integration/test_attach_video_urls.py b/tests/integration/test_attach_video_urls.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/test_validators.py b/tests/unit/test_validators.py index eeed05e77..a10421564 100644 --- a/tests/unit/test_validators.py +++ b/tests/unit/test_validators.py @@ -3,8 +3,8 @@ import tempfile from os.path import dirname from unittest import TestCase -from unittest.mock import patch - +`from unittest.mock import patch +` from pydantic import ValidationError from src.superannotate import SAClient From 5d73eb969006b583c9b2f0bb4ad908d59d55d0e4 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Wed, 1 Jun 2022 15:59:51 +0400 Subject: [PATCH 07/59] moved verificatioon logic inside the usecases isValid, changed 'Response' to have a printable __str__ and it's status is str, Ok if everything is fine --- .pre-commit-config.yaml | 2 +- src/superannotate/__init__.py | 18 +++-- .../lib/app/analytics/aggregators.py | 2 +- .../lib/app/analytics/class_analytics.py | 6 +- src/superannotate/lib/app/common.py | 10 +-- .../lib/app/input_converters/conversion.py | 72 ++++++++--------- .../coco_converters/coco_converter.py | 6 +- .../lib/app/interface/base_interface.py | 9 ++- .../lib/app/interface/cli_interface.py | 78 ++++++++++--------- .../lib/app/interface/sdk_interface.py | 24 +----- src/superannotate/lib/core/data_handlers.py | 2 +- .../lib/core/entities/project_entities.py | 10 ++- src/superannotate/lib/core/plugin.py | 5 +- src/superannotate/lib/core/reporter.py | 2 +- src/superannotate/lib/core/repositories.py | 6 +- src/superannotate/lib/core/response.py | 4 +- .../lib/core/serviceproviders.py | 28 +++++-- .../lib/core/usecases/annotations.py | 5 +- .../lib/core/usecases/folders.py | 4 +- src/superannotate/lib/core/usecases/images.py | 13 +++- src/superannotate/lib/core/usecases/items.py | 46 +++++++---- src/superannotate/lib/core/usecases/models.py | 4 +- .../lib/core/usecases/projects.py | 23 ++++-- .../lib/infrastructure/controller.py | 2 +- .../lib/infrastructure/services.py | 23 +++++- 25 files changed, 247 insertions(+), 157 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3982e77cc..b65ed2b11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - '--application-directories' - app - repo: 'https://github.com/python/black' - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black name: Code Formatter (black) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 448a787a5..58be88e9d 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -1,14 +1,20 @@ import os import sys + sys.path.append(os.path.split(os.path.realpath(__file__))[0]) +from superannotate.lib.app.input_converters.conversion import ( + convert_json_version, +) # noqa +from superannotate.lib.app.input_converters.conversion import ( + convert_project_type, +) # noqa + import logging.config # noqa import requests # noqa from packaging.version import parse # noqa from superannotate.lib.app.analytics.class_analytics import class_distribution # noqa from superannotate.lib.app.exceptions import AppException # noqa -from superannotate.lib.app.input_converters.conversion import convert_json_version # noqa -from superannotate.lib.app.input_converters.conversion import convert_project_type # noqa from superannotate.lib.app.input_converters.conversion import export_annotation # noqa from superannotate.lib.app.input_converters.conversion import import_annotation # noqa from superannotate.lib.app.interface.sdk_interface import SAClient # noqa @@ -52,14 +58,10 @@ def log_version_info(): pip_version = max(pip_version, ver) if pip_version.major > local_version.major: logger.warning( - PACKAGE_VERSION_MAJOR_UPGRADE.format( - local_version, pip_version - ) + PACKAGE_VERSION_MAJOR_UPGRADE.format(local_version, pip_version) ) elif pip_version > local_version: - logger.warning( - PACKAGE_VERSION_UPGRADE.format(local_version, pip_version) - ) + logger.warning(PACKAGE_VERSION_UPGRADE.format(local_version, pip_version)) if not os.environ.get("SA_VERSION_CHECK", "True").lower() == "false": diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index 9e92c4a00..9df04d7f0 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -1,5 +1,6 @@ import copy import json +from dataclasses import dataclass from pathlib import Path from typing import List from typing import Optional @@ -7,7 +8,6 @@ import lib.core as constances import pandas as pd -from dataclasses import dataclass from lib.app.exceptions import AppException from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from lib.core import PIXEL_ANNOTATION_POSTFIX diff --git a/src/superannotate/lib/app/analytics/class_analytics.py b/src/superannotate/lib/app/analytics/class_analytics.py index 36a50bb45..167e05b84 100644 --- a/src/superannotate/lib/app/analytics/class_analytics.py +++ b/src/superannotate/lib/app/analytics/class_analytics.py @@ -60,7 +60,11 @@ def class_distribution(export_root, project_names, visualize=False): df = df.sort_values(["count"], ascending=False) if visualize: - fig = px.bar(df, x="className", y="count",) + fig = px.bar( + df, + x="className", + y="count", + ) fig.update_traces(hovertemplate="%{x}: %{y}") fig.update_yaxes(title_text="Instance Count") fig.update_xaxes(title_text="") diff --git a/src/superannotate/lib/app/common.py b/src/superannotate/lib/app/common.py index de05b04f6..23a9b9d45 100644 --- a/src/superannotate/lib/app/common.py +++ b/src/superannotate/lib/app/common.py @@ -8,15 +8,13 @@ def hex_to_rgb(hex_string): - """Converts HEX values to RGB values - """ + """Converts HEX values to RGB values""" h = hex_string.lstrip("#") return tuple(int(h[i : i + 2], 16) for i in (0, 2, 4)) def blue_color_generator(n, hex_values=True): - """ Blue colors generator for SuperAnnotate blue mask. - """ + """Blue colors generator for SuperAnnotate blue mask.""" hex_colors = [] for i in range(n + 1): int_color = i * 15 @@ -27,7 +25,9 @@ def blue_color_generator(n, hex_values=True): hex_color = ( "#" + "{:02x}".format(bgr_color[2]) - + "{:02x}".format(bgr_color[1],) + + "{:02x}".format( + bgr_color[1], + ) + "{:02x}".format(bgr_color[0]) ) if hex_values: diff --git a/src/superannotate/lib/app/input_converters/conversion.py b/src/superannotate/lib/app/input_converters/conversion.py index 0bb412325..b49c00457 100644 --- a/src/superannotate/lib/app/input_converters/conversion.py +++ b/src/superannotate/lib/app/input_converters/conversion.py @@ -146,41 +146,41 @@ def export_annotation( task="object_detection", ): """ - Converts SuperAnnotate annotation format to the other annotation formats. Currently available (project_type, task) combinations for converter - presented below: - - ============== ====================== - From SA to COCO - -------------------------------------- - project_type task - ============== ====================== - Pixel panoptic_segmentation - Pixel instance_segmentation - Vector instance_segmentation - Vector object_detection - Vector keypoint_detection - ============== ====================== - - :param input_dir: Path to the dataset folder that you want to convert. - :type input_dir: Pathlike(str or Path) - :param output_dir: Path to the folder, where you want to have converted dataset. - :type output_dir: Pathlike(str or Path) - :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"] - :type dataset_format: str - :param dataset_name: Will be used to create json file in the output_dir. - :type dataset_name: str - :param project_type: SuperAnnotate project type is either 'Vector' or 'Pixel' (Default: 'Vector') - 'Vector' project creates ___objects.json for each image. - 'Pixel' project creates ___pixel.jsons and ___save.png annotation mask for each image. - :type project_type: str - :param task: Task can be one of the following: ['panoptic_segmentation', 'instance_segmentation', - 'keypoint_detection', 'object_detection']. (Default: "object_detection"). - 'keypoint_detection' can be used to converts keypoints from/to available annotation format. - 'panoptic_segmentation' will use panoptic mask for each image to generate bluemask for SuperAnnotate annotation format and use bluemask to generate panoptic mask for invert conversion. Panoptic masks should be in the input folder. - 'instance_segmentation' 'Pixel' project_type converts instance masks and 'Vector' project_type generates bounding boxes and polygons from instance masks. Masks should be in the input folder if it is 'Pixel' project_type. - 'object_detection' converts objects from/to available annotation format - :type task: str - """ + Converts SuperAnnotate annotation format to the other annotation formats. Currently available (project_type, task) combinations for converter + presented below: + + ============== ====================== + From SA to COCO + -------------------------------------- + project_type task + ============== ====================== + Pixel panoptic_segmentation + Pixel instance_segmentation + Vector instance_segmentation + Vector object_detection + Vector keypoint_detection + ============== ====================== + + :param input_dir: Path to the dataset folder that you want to convert. + :type input_dir: Pathlike(str or Path) + :param output_dir: Path to the folder, where you want to have converted dataset. + :type output_dir: Pathlike(str or Path) + :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"] + :type dataset_format: str + :param dataset_name: Will be used to create json file in the output_dir. + :type dataset_name: str + :param project_type: SuperAnnotate project type is either 'Vector' or 'Pixel' (Default: 'Vector') + 'Vector' project creates ___objects.json for each image. + 'Pixel' project creates ___pixel.jsons and ___save.png annotation mask for each image. + :type project_type: str + :param task: Task can be one of the following: ['panoptic_segmentation', 'instance_segmentation', + 'keypoint_detection', 'object_detection']. (Default: "object_detection"). + 'keypoint_detection' can be used to converts keypoints from/to available annotation format. + 'panoptic_segmentation' will use panoptic mask for each image to generate bluemask for SuperAnnotate annotation format and use bluemask to generate panoptic mask for invert conversion. Panoptic masks should be in the input folder. + 'instance_segmentation' 'Pixel' project_type converts instance masks and 'Vector' project_type generates bounding boxes and polygons from instance masks. Masks should be in the input folder if it is 'Pixel' project_type. + 'object_detection' converts objects from/to available annotation format + :type task: str + """ if project_type in [ ProjectType.VIDEO.name, @@ -410,7 +410,7 @@ def import_annotation( @Trackable def convert_project_type(input_dir, output_dir): - """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. + """Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. :param input_dir: Path to the dataset folder that you want to convert. :type input_dir: Pathlike(str or Path) diff --git a/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py b/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py index 4c5834986..81fa39461 100644 --- a/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py +++ b/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py @@ -137,9 +137,9 @@ def convert_from_old_sa_to_new(self, old_json_data, project_type): def _parse_json_into_common_format(self, sa_annotation_json, fpath): """ - If the annotation format ever changes this function will handle it and - return something optimal for the converters. Additionally, if anything - important is absent from the current json, this function fills it. + If the annotation format ever changes this function will handle it and + return something optimal for the converters. Additionally, if anything + important is absent from the current json, this function fills it. """ if isinstance(sa_annotation_json, list): sa_annotation_json = self.convert_from_old_sa_to_new( diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 5c5a8b9c6..c59338a8d 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -70,7 +70,10 @@ def default_parser(function_name: str, kwargs: dict) -> tuple: if key == "self": continue elif key == "project": - properties["project_name"], properties["folder_name"] = extract_project_folder(value) + ( + properties["project_name"], + properties["folder_name"], + ) = extract_project_folder(value) elif isinstance(value, (str, int, float, bool, str)): properties[key] = value elif isinstance(value, dict): @@ -99,7 +102,9 @@ def track(self, args, kwargs, success: bool, session): project_name=properties.pop("project_name", None), ) if "pytest" not in sys.modules: - self.get_mp_instance().track(user_id, event_name, {**default, **properties, **session.data}) + self.get_mp_instance().track( + user_id, event_name, {**default, **properties, **session.data} + ) except Exception: raise diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index c0576ccce..fa05ab81f 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -25,7 +25,7 @@ def version(): To show the version of the current SDK installation """ with open( - f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" + f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" ) as f: version = f.read().rstrip()[15:-1] print(version) @@ -40,7 +40,7 @@ def init(): config = repo.get_one(uuid=constances.TOKEN_UUID) if config: if not input( - f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " + f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " ).lower() in ("y", "yes"): return token = input( @@ -68,14 +68,14 @@ def create_folder(self, project: str, name: str): sys.exit(0) def upload_images( - self, - project: str, - folder: str, - extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, - set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, - exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders=False, - image_quality_in_editor=None, + self, + project: str, + folder: str, + extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, + set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, + exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders=False, + image_quality_in_editor=None, ): """ To upload images from folder to project use: @@ -99,12 +99,12 @@ def upload_images( sys.exit(0) def export_project( - self, - project, - folder, - include_fuse=False, - disable_extract_zip_contents=False, - annotation_statuses=None, + self, + project, + folder, + include_fuse=False, + disable_extract_zip_contents=False, + annotation_statuses=None, ): project_name, folder_name = split_project_path(project) folders = None @@ -127,7 +127,7 @@ def export_project( sys.exit(0) def upload_preannotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload preannotations from folder to project use @@ -148,7 +148,7 @@ def upload_preannotations( sys.exit(0) def upload_annotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload annotations from folder to project use @@ -169,11 +169,13 @@ def upload_annotations( sys.exit(0) def _upload_annotations( - self, project, folder, format, dataset_name, task, pre=True + self, project, folder, format, dataset_name, task, pre=True ): project_folder_name = project project_name, folder_name = split_project_path(project) - project = SAClient().controller.get_project_metadata(project_name=project_name).data + project = ( + SAClient().controller.get_project_metadata(project_name=project_name).data + ) if not format: format = "SuperAnnotate" if not dataset_name and format == "COCO": @@ -207,10 +209,10 @@ def _upload_annotations( sys.exit(0) def attach_image_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): """ To attach image URLs to project use: @@ -224,10 +226,10 @@ def attach_image_urls( sys.exit(0) def attach_video_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): SAClient().attach_items( project=project, @@ -238,7 +240,7 @@ def attach_video_urls( @staticmethod def attach_document_urls( - project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" + project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" ): SAClient().attach_items( project=project, @@ -248,15 +250,15 @@ def attach_document_urls( sys.exit(0) def upload_videos( - self, - project, - folder, - target_fps=None, - recursive=False, - extensions=constances.DEFAULT_VIDEO_EXTENSIONS, - set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, - start_time=0.0, - end_time=None, + self, + project, + folder, + target_fps=None, + recursive=False, + extensions=constances.DEFAULT_VIDEO_EXTENSIONS, + set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, + start_time=0.0, + end_time=None, ): """ To upload videos from folder to project use diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index a00f41982..6bca8bd0d 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -751,30 +751,12 @@ def assign_items( """ project_name, folder_name = extract_project_folder(project) - project = self.controller.get_project_metadata(project_name).data - - contributors = ( - self.controller.get_project_metadata( - project_name=project_name, include_contributors=True - ) - .data["project"] - .users - ) - contributor = None - for c in contributors: - if c["user_id"] == user: - contributor = user - - if not contributor: - logger.warning( - f"Skipping {user}. {user} is not a verified contributor for the {project_name}" - ) - return response = self.controller.assign_items( - project_name, folder_name, item_names, user + project, folder_name, item_names, user ) - if not response.errors: + + if not response.errors and response.status == 'Ok': logger.info(f"Assign items to user {user}") else: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/data_handlers.py b/src/superannotate/lib/core/data_handlers.py index c0a1c442c..1eb77e684 100644 --- a/src/superannotate/lib/core/data_handlers.py +++ b/src/superannotate/lib/core/data_handlers.py @@ -266,7 +266,7 @@ def safe_time(timestamp): return "0" if str(timestamp) == "0.0" else timestamp def convert_timestamp(timestamp): - return timestamp / 10 ** 6 if timestamp else "0" + return timestamp / 10**6 if timestamp else "0" editor_data = { "instances": [], diff --git a/src/superannotate/lib/core/entities/project_entities.py b/src/superannotate/lib/core/entities/project_entities.py index 4e50080f3..244bca0c9 100644 --- a/src/superannotate/lib/core/entities/project_entities.py +++ b/src/superannotate/lib/core/entities/project_entities.py @@ -40,7 +40,10 @@ def to_dict(self): class BaseTimedEntity(BaseEntity): def __init__( - self, uuid: Any = None, createdAt: str = None, updatedAt: str = None, + self, + uuid: Any = None, + createdAt: str = None, + updatedAt: str = None, ): super().__init__(uuid) self.createdAt = createdAt @@ -222,7 +225,10 @@ def to_dict(self): class ImageInfoEntity(BaseEntity): def __init__( - self, uuid=None, width: float = None, height: float = None, + self, + uuid=None, + width: float = None, + height: float = None, ): super().__init__(uuid), self.width = width diff --git a/src/superannotate/lib/core/plugin.py b/src/superannotate/lib/core/plugin.py index 06f77115e..9ef6fec30 100644 --- a/src/superannotate/lib/core/plugin.py +++ b/src/superannotate/lib/core/plugin.py @@ -254,7 +254,10 @@ def frames_generator( @staticmethod def get_extractable_frames( - video_path: str, start_time, end_time, target_fps: float, + video_path: str, + start_time, + end_time, + target_fps: float, ): total = VideoPlugin.get_frames_count(video_path) total_with_fps = sum( diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index db8c24395..4d425ae56 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -72,7 +72,7 @@ def __init__( log_warning: bool = True, disable_progress_bar: bool = False, log_debug: bool = True, - session: Session = None + session: Session = None, ): self.logger = get_default_logger() self._log_info = log_info diff --git a/src/superannotate/lib/core/repositories.py b/src/superannotate/lib/core/repositories.py index a94258a5c..f60c7b44e 100644 --- a/src/superannotate/lib/core/repositories.py +++ b/src/superannotate/lib/core/repositories.py @@ -67,7 +67,11 @@ def __init__(self, service: SuperannotateServiceProvider, project: ProjectEntity class BaseS3Repository(BaseManageableRepository): def __init__( - self, access_key: str, secret_key: str, session_token: str, bucket: str, + self, + access_key: str, + secret_key: str, + session_token: str, + bucket: str, ): self._session = boto3.Session( aws_access_key_id=access_key, diff --git a/src/superannotate/lib/core/response.py b/src/superannotate/lib/core/response.py index ddc5e413e..86aab3020 100644 --- a/src/superannotate/lib/core/response.py +++ b/src/superannotate/lib/core/response.py @@ -8,6 +8,8 @@ def __init__(self, status: str = None, data: Union[dict, list] = None): self._report = [] self._errors = [] + def __str__(self): + return f"Response object with status:{self.status}, data : {self.data}, errors: {self.errors} " @property def data(self): return self._data @@ -30,7 +32,7 @@ def report_messages(self): @property def status(self): - return self.data + return self._status @status.setter def status(self, value): diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 1e06b7cf6..7a9868912 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -106,7 +106,11 @@ def get_download_token( raise NotImplementedError def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, ) -> dict: raise NotImplementedError @@ -187,7 +191,10 @@ def get_bulk_images( raise NotImplementedError def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, + team_id: int, + project_id: int, + folder_name: str, ): raise NotImplementedError @@ -197,17 +204,28 @@ def assign_folder( raise NotImplementedError def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: list, + self, + team_id: int, + project_id: int, + folder_name: str, + image_names: list, ): raise NotImplementedError def un_assign_items( - self, team_id: int, project_id: int, folder_name: str, item_names: list, + self, + team_id: int, + project_id: int, + folder_name: str, + item_names: list, ): raise NotImplementedError def un_share_project( - self, team_id: int, project_id: int, user_id: str, + self, + team_id: int, + project_id: int, + user_id: str, ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 8b1666bba..ba0dc171a 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -444,7 +444,10 @@ def prepare_annotations( handlers_chain.attach(LastActionHandler(team.creator_id)) return handlers_chain.handle(annotations) - def clean_json(self, json_data: dict,) -> Tuple[bool, dict]: + def clean_json( + self, + json_data: dict, + ) -> Tuple[bool, dict]: use_case = ValidateAnnotationUseCase( constances.ProjectType.get_name(self._project.type), annotation=json_data, diff --git a/src/superannotate/lib/core/usecases/folders.py b/src/superannotate/lib/core/usecases/folders.py index a9733652f..b98c79e88 100644 --- a/src/superannotate/lib/core/usecases/folders.py +++ b/src/superannotate/lib/core/usecases/folders.py @@ -142,7 +142,9 @@ def execute(self): class UpdateFolderUseCase(BaseUseCase): def __init__( - self, folders: BaseManageableRepository, folder: FolderEntity, + self, + folders: BaseManageableRepository, + folder: FolderEntity, ): super().__init__() self._folders = folders diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 31b4eeba0..2f012c094 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -581,7 +581,11 @@ def execute(self): image = ImagePlugin(io.BytesIO(file.read())) images = [ - Image("fuse", f"{self._image_path}___fuse.png", image.get_empty(),) + Image( + "fuse", + f"{self._image_path}___fuse.png", + image.get_empty(), + ) ] if self._generate_overlay: images.append( @@ -711,7 +715,9 @@ def execute(self): class GetS3ImageUseCase(BaseUseCase): def __init__( - self, s3_bucket, image_path: str, + self, + s3_bucket, + image_path: str, ): super().__init__() self._s3_bucket = s3_bucket @@ -1536,7 +1542,8 @@ def execute(self) -> Response: image_bytes = ( GetImageBytesUseCase( - image=image, backend_service_provider=self._backend_service, + image=image, + backend_service_provider=self._backend_service, ) .execute() .data diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 4e32a6ffc..85918eec0 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -19,7 +19,10 @@ from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseUseCase +from superannotate.logger import get_default_logger +logger = get_default_logger() class GetItem(BaseReportableUseCae): def __init__( @@ -205,23 +208,37 @@ def __init__( self._user = user self._service = service + def is_valid(self, ): + + if not super().is_valid(): + return False + + for c in self._project.users: + if c["user_id"] == self._user: + return True + + logger.warning( + f"Skipping {self._user}. {self._user} is not a verified contributor for the {self._project.name}" + ) + return False + def execute(self): if self.is_valid(): - for i in range(0, len(self._image_names), self.CHUNK_SIZE): + for i in range(0, len(self._item_names), self.CHUNK_SIZE): is_assigned = self._service.assign_items( team_id=self._project.team_id, project_id=self._project.id, folder_name=self._folder.name, user=self._user, - item_names=self._item_names[ - i : i + self.CHUNK_SIZE # noqa: E203 - ], + item_names=self._item_names[i : i + self.CHUNK_SIZE], # noqa: E203 ) if not is_assigned: self._response.errors = AppException( f"Cant assign {', '.join(self._item_names[i: i + self.CHUNK_SIZE])}" ) continue + else: + self._response.status = 'Ok' return self._response @@ -229,7 +246,7 @@ class UnAssignItemsUseCase(BaseUseCase): CHUNK_SIZE = 500 def __init__( - self,a + self, service: SuperannotateServiceProvider, project_entity: ProjectEntity, folder: FolderEntity, @@ -243,7 +260,7 @@ def __init__( def execute(self): # todo handling to backend side - for i in range(0, len(self._item_names, self.CHUNK_SIZE): + for i in range(0, len(self._item_names), self.CHUNK_SIZE): is_un_assigned = self._service.un_assign_items( team_id=self._project_entity.team_id, project_id=self._project_entity.id, @@ -257,6 +274,7 @@ def execute(self): return self._response + class AttachItems(BaseReportableUseCae): CHUNK_SIZE = 500 @@ -432,13 +450,15 @@ def execute(self): if items_to_copy: for i in range(0, len(items_to_copy), self.CHUNK_SIZE): chunk_to_copy = items_to_copy[i : i + self.CHUNK_SIZE] # noqa: E203 - poll_id = self._backend_service.copy_items_between_folders_transaction( - team_id=self._project.team_id, - project_id=self._project.id, - from_folder_id=self._from_folder.uuid, - to_folder_id=self._to_folder.uuid, - items=chunk_to_copy, - include_annotations=self._include_annotations, + poll_id = ( + self._backend_service.copy_items_between_folders_transaction( + team_id=self._project.team_id, + project_id=self._project.id, + from_folder_id=self._from_folder.uuid, + to_folder_id=self._to_folder.uuid, + items=chunk_to_copy, + include_annotations=self._include_annotations, + ) ) if not poll_id: skipped_items.extend(chunk_to_copy) diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 402fbdf0d..abca29e8d 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -635,7 +635,9 @@ def execute(self): class SearchMLModels(BaseUseCase): def __init__( - self, ml_models_repo: BaseManageableRepository, condition: Condition, + self, + ml_models_repo: BaseManageableRepository, + condition: Condition, ): super().__init__() self._ml_models = ml_models_repo diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index fb97e87de..c2240e33c 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -31,7 +31,10 @@ class GetProjectsUseCase(BaseUseCase): def __init__( - self, condition: Condition, team_id: int, projects: BaseManageableRepository, + self, + condition: Condition, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() self._condition = condition @@ -47,7 +50,10 @@ def execute(self): class GetProjectByNameUseCase(BaseUseCase): def __init__( - self, name: str, team_id: int, projects: BaseManageableRepository, + self, + name: str, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() self._name = name @@ -300,7 +306,10 @@ def execute(self): class DeleteProjectUseCase(BaseUseCase): def __init__( - self, project_name: str, team_id: int, projects: BaseManageableRepository, + self, + project_name: str, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() @@ -310,7 +319,9 @@ def __init__( def execute(self): use_case = GetProjectByNameUseCase( - name=self._project_name, team_id=self._team_id, projects=self._projects, + name=self._project_name, + team_id=self._team_id, + projects=self._projects, ) project_response = use_case.execute() if project_response.data: @@ -696,7 +707,9 @@ def execute(self): class GetAnnotationClassesUseCase(BaseUseCase): def __init__( - self, classes: BaseManageableRepository, condition: Condition = None, + self, + classes: BaseManageableRepository, + condition: Condition = None, ): super().__init__() self._classes = classes diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index ff7099d2a..8a04a710a 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -863,7 +863,7 @@ def delete_images( def assign_items( self, project_name: str, folder_name: str, item_names: list, user: str ): - project_entity = self._get_project(project_name) + project_entity = self.get_project_metadata(project_name, include_contributors = True).data['project'] folder = self._get_folder(project_entity, folder_name) use_case = usecases.AssignItemsUseCase( project=project_entity, diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 507d101e8..95e3c896e 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -300,7 +300,11 @@ def get_download_token( return response.json() def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, ): download_token_url = urljoin( self.api_url, @@ -734,7 +738,11 @@ def assign_images( return res.ok def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: List[str], + self, + team_id: int, + project_id: int, + folder_name: str, + image_names: List[str], ): un_assign_images_url = urljoin(self.api_url, self.URL_ASSIGN_IMAGES) res = self._request( @@ -771,7 +779,11 @@ def assign_items( return res.ok def un_assign_items( - self, team_id: int, project_id: int, folder_name: str, item_names: List[str], + self, + team_id: int, + project_id: int, + folder_name: str, + item_names: List[str], ): un_assign_items_url = urljoin(self.api_url, self.URL_ASSIGN_ITEMS) res = self._request( @@ -787,7 +799,10 @@ def un_assign_items( return res.ok def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, + team_id: int, + project_id: int, + folder_name: str, ): un_assign_folder_url = urljoin(self.api_url, self.URL_ASSIGN_FOLDER) res = self._request( From 610e47047a79c4fb7c94d9e22a3ffe86896c5586 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Thu, 2 Jun 2022 11:26:10 +0400 Subject: [PATCH 08/59] conforming to general validation logic. Added 'Skippable Validation Exception' to make sure sdk runtime doesn't exit' --- src/superannotate/lib/app/interface/sdk_interface.py | 4 ++-- src/superannotate/lib/core/exceptions.py | 5 +++++ src/superannotate/lib/core/usecases/base.py | 5 ++++- src/superannotate/lib/core/usecases/items.py | 11 ++++------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 6bca8bd0d..a22119c14 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -756,9 +756,9 @@ def assign_items( project, folder_name, item_names, user ) - if not response.errors and response.status == 'Ok': + if not response.errors: logger.info(f"Assign items to user {user}") - else: + elif response.status != "Skip": raise AppException(response.errors) @validate_arguments diff --git a/src/superannotate/lib/core/exceptions.py b/src/superannotate/lib/core/exceptions.py index 4228e5cb2..b14acac5d 100644 --- a/src/superannotate/lib/core/exceptions.py +++ b/src/superannotate/lib/core/exceptions.py @@ -25,6 +25,11 @@ class AppValidationException(AppException): """ +class SkippableAppValidationException(AppValidationException): + """ + App validation exception + """ + class ImageProcessingException(AppException): """ App validation exception diff --git a/src/superannotate/lib/core/usecases/base.py b/src/superannotate/lib/core/usecases/base.py index 3f3e3750f..d7ae21730 100644 --- a/src/superannotate/lib/core/usecases/base.py +++ b/src/superannotate/lib/core/usecases/base.py @@ -4,7 +4,7 @@ from typing import Iterable from typing import List -from lib.core.exceptions import AppValidationException +from lib.core.exceptions import AppValidationException, SkippableAppValidationException from lib.core.reporter import Reporter from lib.core.response import Response @@ -24,6 +24,9 @@ def _validate(self): if name.startswith("validate_"): method = getattr(self, name) method() + except SkippableAppValidationException as e: + self._response.errors = e + self._response.status = 'Skip' except AppValidationException as e: self._response.errors = e diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 85918eec0..6261d5990 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -12,7 +12,7 @@ from lib.core.entities import TmpImageEntity from lib.core.entities import VideoEntity from lib.core.exceptions import AppException -from lib.core.exceptions import AppValidationException +from lib.core.exceptions import AppValidationException, SkippableAppValidationException from lib.core.exceptions import BackendError from lib.core.reporter import Reporter from lib.core.repositories import BaseReadOnlyRepository @@ -208,10 +208,7 @@ def __init__( self._user = user self._service = service - def is_valid(self, ): - - if not super().is_valid(): - return False + def validate_user(self, ): for c in self._project.users: if c["user_id"] == self._user: @@ -220,7 +217,8 @@ def is_valid(self, ): logger.warning( f"Skipping {self._user}. {self._user} is not a verified contributor for the {self._project.name}" ) - return False + + raise SkippableAppValidationException(f"{self._user} is not a verified contributor for the {self._project.name}") def execute(self): if self.is_valid(): @@ -244,7 +242,6 @@ def execute(self): class UnAssignItemsUseCase(BaseUseCase): CHUNK_SIZE = 500 - def __init__( self, service: SuperannotateServiceProvider, From 229a4b2990356b9ee054b77aa3dca278b0329822 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 30 May 2022 12:15:24 +0400 Subject: [PATCH 09/59] Added mixpanel, changed docs --- .pre-commit-config.yaml | 2 +- docs/source/superannotate.sdk.rst | 140 +-- pytest.ini | 2 +- src/superannotate/__init__.py | 20 +- .../lib/app/analytics/class_analytics.py | 10 +- src/superannotate/lib/app/common.py | 10 +- .../lib/app/input_converters/__init__.py | 12 + .../lib/app/input_converters/conversion.py | 79 +- .../coco_converters/coco_converter.py | 6 +- .../lib/app/interface/base_interface.py | 159 ++-- .../lib/app/interface/cli_interface.py | 89 +- .../lib/app/interface/sdk_interface.py | 892 ++++++++---------- src/superannotate/lib/app/mixp/decorators.py | 134 --- src/superannotate/lib/core/__init__.py | 7 +- src/superannotate/lib/core/data_handlers.py | 2 +- .../lib/core/entities/project_entities.py | 10 +- src/superannotate/lib/core/plugin.py | 5 +- src/superannotate/lib/core/reporter.py | 27 +- src/superannotate/lib/core/repositories.py | 6 +- .../lib/core/serviceproviders.py | 22 +- .../lib/core/usecases/annotations.py | 31 +- src/superannotate/lib/core/usecases/base.py | 4 +- .../lib/core/usecases/folders.py | 4 +- src/superannotate/lib/core/usecases/images.py | 19 +- .../lib/core/usecases/integrations.py | 6 +- src/superannotate/lib/core/usecases/items.py | 32 +- src/superannotate/lib/core/usecases/models.py | 8 +- .../lib/core/usecases/projects.py | 27 +- .../lib/infrastructure/controller.py | 523 +++++----- .../lib/infrastructure/services.py | 21 +- .../lib/infrastructure/stream_data_handler.py | 11 +- .../test_annotations_pre_processing.py | 2 +- .../annotations/test_download_annotations.py | 11 + .../annotations/test_get_annotations.py | 2 +- .../mixpanel/test_individual_fuinctions.py | 30 + .../mixpanel/test_mixpanel_decorator.py | 139 +++ .../integration/test_attach_document_urls.py | 17 - tests/integration/test_attach_video_urls.py | 0 .../test_client.py} | 0 tests/unit/test_validators.py | 4 +- 40 files changed, 1306 insertions(+), 1219 deletions(-) delete mode 100644 src/superannotate/lib/app/mixp/decorators.py create mode 100644 tests/integration/mixpanel/test_individual_fuinctions.py create mode 100644 tests/integration/mixpanel/test_mixpanel_decorator.py delete mode 100644 tests/integration/test_attach_document_urls.py delete mode 100644 tests/integration/test_attach_video_urls.py rename tests/{integration/test_assign_images.py => unit/test_client.py} (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3982e77cc..b65ed2b11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - '--application-directories' - app - repo: 'https://github.com/python/black' - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black name: Code Formatter (black) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 224765118..3d2888431 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -11,7 +11,7 @@ Remote functions Initialization and authentication _________________________________ -.. autofunction:: superannotate.init +.. automethod:: superannotate.SAClient.__init__ .. _ref_projects: @@ -20,61 +20,61 @@ Projects ________ .. _ref_search_projects: -.. autofunction:: superannotate.search_projects -.. autofunction:: superannotate.create_project -.. autofunction:: superannotate.create_project_from_metadata -.. autofunction:: superannotate.clone_project -.. autofunction:: superannotate.delete_project -.. autofunction:: superannotate.rename_project +.. automethod:: superannotate.SAClient.search_projects +.. automethod:: superannotate.SAClient.create_project +.. automethod:: superannotate.SAClient.create_project_from_metadata +.. automethod:: superannotate.SAClient.clone_project +.. automethod:: superannotate.SAClient.delete_project +.. automethod:: superannotate.SAClient.rename_project .. _ref_get_project_metadata: -.. autofunction:: superannotate.get_project_metadata -.. autofunction:: superannotate.get_project_image_count -.. autofunction:: superannotate.search_folders -.. autofunction:: superannotate.get_folder_metadata -.. autofunction:: superannotate.create_folder -.. autofunction:: superannotate.delete_folders -.. autofunction:: superannotate.upload_images_to_project -.. autofunction:: superannotate.attach_items_from_integrated_storage -.. autofunction:: superannotate.upload_image_to_project -.. autofunction:: superannotate.delete_annotations +.. automethod:: superannotate.SAClient.get_project_metadata +.. automethod:: superannotate.SAClient.get_project_image_count +.. automethod:: superannotate.SAClient.search_folders +.. automethod:: superannotate.SAClient.get_folder_metadata +.. automethod:: superannotate.SAClient.create_folder +.. automethod:: superannotate.SAClient.delete_folders +.. automethod:: superannotate.SAClient.upload_images_to_project +.. automethod:: superannotate.SAClient.attach_items_from_integrated_storage +.. automethod:: superannotate.SAClient.upload_image_to_project +.. automethod:: superannotate.SAClient.delete_annotations .. _ref_upload_images_from_folder_to_project: -.. autofunction:: superannotate.upload_images_from_folder_to_project -.. autofunction:: superannotate.upload_video_to_project -.. autofunction:: superannotate.upload_videos_from_folder_to_project +.. automethod:: superannotate.SAClient.upload_images_from_folder_to_project +.. automethod:: superannotate.SAClient.upload_video_to_project +.. automethod:: superannotate.SAClient.upload_videos_from_folder_to_project .. _ref_upload_annotations_from_folder_to_project: -.. autofunction:: superannotate.upload_annotations_from_folder_to_project -.. autofunction:: superannotate.upload_preannotations_from_folder_to_project -.. autofunction:: superannotate.add_contributors_to_project -.. autofunction:: superannotate.get_project_settings -.. autofunction:: superannotate.set_project_default_image_quality_in_editor -.. autofunction:: superannotate.get_project_workflow -.. autofunction:: superannotate.set_project_workflow +.. automethod:: superannotate.SAClient.upload_annotations_from_folder_to_project +.. automethod:: superannotate.SAClient.upload_preannotations_from_folder_to_project +.. automethod:: superannotate.SAClient.add_contributors_to_project +.. automethod:: superannotate.SAClient.get_project_settings +.. automethod:: superannotate.SAClient.set_project_default_image_quality_in_editor +.. automethod:: superannotate.SAClient.get_project_workflow +.. automethod:: superannotate.SAClient.set_project_workflow ---------- Exports _______ -.. autofunction:: superannotate.prepare_export -.. autofunction:: superannotate.get_annotations -.. autofunction:: superannotate.get_annotations_per_frame +.. automethod:: superannotate.SAClient.prepare_export +.. automethod:: superannotate.SAClient.get_annotations +.. automethod:: superannotate.SAClient.get_annotations_per_frame .. _ref_download_export: -.. autofunction:: superannotate.download_export -.. autofunction:: superannotate.get_exports +.. automethod:: superannotate.SAClient.download_export +.. automethod:: superannotate.SAClient.get_exports ---------- Items ______ -.. autofunction:: superannotate.query -.. autofunction:: superannotate.search_items -.. autofunction:: superannotate.download_annotations -.. autofunction:: superannotate.attach_items -.. autofunction:: superannotate.copy_items -.. autofunction:: superannotate.move_items -.. autofunction:: superannotate.get_item_metadata -.. autofunction:: superannotate.set_annotation_statuses +.. automethod:: superannotate.SAClient.query +.. automethod:: superannotate.SAClient.search_items +.. automethod:: superannotate.SAClient.download_annotations +.. automethod:: superannotate.SAClient.attach_items +.. automethod:: superannotate.SAClient.copy_items +.. automethod:: superannotate.SAClient.move_items +.. automethod:: superannotate.SAClient.get_item_metadata +.. automethod:: superannotate.SAClient.set_annotation_statuses ---------- @@ -83,50 +83,50 @@ ______ .. _ref_search_images: -.. autofunction:: superannotate.download_image -.. autofunction:: superannotate.set_image_annotation_status -.. autofunction:: superannotate.set_images_annotation_statuses -.. autofunction:: superannotate.download_image_annotations -.. autofunction:: superannotate.upload_image_annotations -.. autofunction:: superannotate.copy_image -.. autofunction:: superannotate.pin_image -.. autofunction:: superannotate.assign_images -.. autofunction:: superannotate.delete_images -.. autofunction:: superannotate.add_annotation_bbox_to_image -.. autofunction:: superannotate.add_annotation_point_to_image -.. autofunction:: superannotate.add_annotation_comment_to_image -.. autofunction:: superannotate.upload_priority_scores +.. automethod:: superannotate.SAClient.download_image +.. automethod:: superannotate.SAClient.set_image_annotation_status +.. automethod:: superannotate.SAClient.set_images_annotation_statuses +.. automethod:: superannotate.SAClient.download_image_annotations +.. automethod:: superannotate.SAClient.upload_image_annotations +.. automethod:: superannotate.SAClient.copy_image +.. automethod:: superannotate.SAClient.pin_image +.. automethod:: superannotate.SAClient.assign_images +.. automethod:: superannotate.SAClient.delete_images +.. automethod:: superannotate.SAClient.add_annotation_bbox_to_image +.. automethod:: superannotate.SAClient.add_annotation_point_to_image +.. automethod:: superannotate.SAClient.add_annotation_comment_to_image +.. automethod:: superannotate.SAClient.upload_priority_scores ---------- Annotation Classes __________________ -.. autofunction:: superannotate.create_annotation_class +.. automethod:: superannotate.SAClient.create_annotation_class .. _ref_create_annotation_classes_from_classes_json: -.. autofunction:: superannotate.create_annotation_classes_from_classes_json -.. autofunction:: superannotate.search_annotation_classes -.. autofunction:: superannotate.download_annotation_classes_json -.. autofunction:: superannotate.delete_annotation_class +.. automethod:: superannotate.SAClient.create_annotation_classes_from_classes_json +.. automethod:: superannotate.SAClient.search_annotation_classes +.. automethod:: superannotate.SAClient.download_annotation_classes_json +.. automethod:: superannotate.SAClient.delete_annotation_class ---------- Team _________________ -.. autofunction:: superannotate.get_team_metadata -.. autofunction:: superannotate.get_integrations -.. autofunction:: superannotate.invite_contributors_to_team -.. autofunction:: superannotate.search_team_contributors +.. automethod:: superannotate.SAClient.get_team_metadata +.. automethod:: superannotate.SAClient.get_integrations +.. automethod:: superannotate.SAClient.invite_contributors_to_team +.. automethod:: superannotate.SAClient.search_team_contributors ---------- Neural Network _______________ -.. autofunction:: superannotate.download_model -.. autofunction:: superannotate.run_prediction -.. autofunction:: superannotate.search_models +.. automethod:: superannotate.SAClient.download_model +.. automethod:: superannotate.SAClient.run_prediction +.. automethod:: superannotate.SAClient.search_models ---------- @@ -196,7 +196,7 @@ Export metadata example: Integration metadata -_______________ +______________________ Integration metadata example: @@ -383,8 +383,8 @@ Working with annotations ________________________ .. _ref_aggregate_annotations_as_df: -.. autofunction:: superannotate.validate_annotations -.. autofunction:: superannotate.aggregate_annotations_as_df +.. automethod:: superannotate.SAClient.validate_annotations +.. automethod:: superannotate.SAClient.aggregate_annotations_as_df ---------- @@ -398,5 +398,5 @@ _____________________________________________________________ Utility functions -------------------------------- -.. autofunction:: superannotate.consensus -.. autofunction:: superannotate.benchmark \ No newline at end of file +.. autofunction:: superannotate.SAClient.consensus +.. autofunction:: superannotate.SAClient.benchmark \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index de724250b..86c2d4c63 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.7 log_cli=true python_files = test_*.py -;addopts = -n auto --dist=loadscope +addopts = -n auto --dist=loadscope \ No newline at end of file diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 448a787a5..f6bfa0eed 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -1,5 +1,6 @@ import os import sys + sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import logging.config # noqa @@ -7,10 +8,10 @@ from packaging.version import parse # noqa from superannotate.lib.app.analytics.class_analytics import class_distribution # noqa from superannotate.lib.app.exceptions import AppException # noqa -from superannotate.lib.app.input_converters.conversion import convert_json_version # noqa -from superannotate.lib.app.input_converters.conversion import convert_project_type # noqa -from superannotate.lib.app.input_converters.conversion import export_annotation # noqa -from superannotate.lib.app.input_converters.conversion import import_annotation # noqa +from superannotate.lib.app.input_converters import convert_json_version +from superannotate.lib.app.input_converters import convert_project_type # noqa +from superannotate.lib.app.input_converters import export_annotation # noqa +from superannotate.lib.app.input_converters import import_annotation # noqa from superannotate.lib.app.interface.sdk_interface import SAClient # noqa from superannotate.lib.core import PACKAGE_VERSION_INFO_MESSAGE # noqa from superannotate.lib.core import PACKAGE_VERSION_MAJOR_UPGRADE # noqa @@ -18,6 +19,9 @@ from superannotate.logger import get_default_logger # noqa from superannotate.version import __version__ # noqa + +SESSIONS = {} + __all__ = [ "__version__", "SAClient", @@ -52,14 +56,10 @@ def log_version_info(): pip_version = max(pip_version, ver) if pip_version.major > local_version.major: logger.warning( - PACKAGE_VERSION_MAJOR_UPGRADE.format( - local_version, pip_version - ) + PACKAGE_VERSION_MAJOR_UPGRADE.format(local_version, pip_version) ) elif pip_version > local_version: - logger.warning( - PACKAGE_VERSION_UPGRADE.format(local_version, pip_version) - ) + logger.warning(PACKAGE_VERSION_UPGRADE.format(local_version, pip_version)) if not os.environ.get("SA_VERSION_CHECK", "True").lower() == "false": diff --git a/src/superannotate/lib/app/analytics/class_analytics.py b/src/superannotate/lib/app/analytics/class_analytics.py index 36a50bb45..712e9da58 100644 --- a/src/superannotate/lib/app/analytics/class_analytics.py +++ b/src/superannotate/lib/app/analytics/class_analytics.py @@ -2,7 +2,7 @@ import pandas as pd import plotly.express as px -from lib.app.mixp.decorators import Trackable +from lib.app.interface.base_interface import Tracker from superannotate.lib.app.exceptions import AppException from superannotate.lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE from superannotate.logger import get_default_logger @@ -12,7 +12,7 @@ logger = get_default_logger() -@Trackable +@Tracker def class_distribution(export_root, project_names, visualize=False): """Aggregate distribution of classes across multiple projects. @@ -60,7 +60,11 @@ def class_distribution(export_root, project_names, visualize=False): df = df.sort_values(["count"], ascending=False) if visualize: - fig = px.bar(df, x="className", y="count",) + fig = px.bar( + df, + x="className", + y="count", + ) fig.update_traces(hovertemplate="%{x}: %{y}") fig.update_yaxes(title_text="Instance Count") fig.update_xaxes(title_text="") diff --git a/src/superannotate/lib/app/common.py b/src/superannotate/lib/app/common.py index de05b04f6..23a9b9d45 100644 --- a/src/superannotate/lib/app/common.py +++ b/src/superannotate/lib/app/common.py @@ -8,15 +8,13 @@ def hex_to_rgb(hex_string): - """Converts HEX values to RGB values - """ + """Converts HEX values to RGB values""" h = hex_string.lstrip("#") return tuple(int(h[i : i + 2], 16) for i in (0, 2, 4)) def blue_color_generator(n, hex_values=True): - """ Blue colors generator for SuperAnnotate blue mask. - """ + """Blue colors generator for SuperAnnotate blue mask.""" hex_colors = [] for i in range(n + 1): int_color = i * 15 @@ -27,7 +25,9 @@ def blue_color_generator(n, hex_values=True): hex_color = ( "#" + "{:02x}".format(bgr_color[2]) - + "{:02x}".format(bgr_color[1],) + + "{:02x}".format( + bgr_color[1], + ) + "{:02x}".format(bgr_color[0]) ) if hex_values: diff --git a/src/superannotate/lib/app/input_converters/__init__.py b/src/superannotate/lib/app/input_converters/__init__.py index e69de29bb..1acd1e490 100644 --- a/src/superannotate/lib/app/input_converters/__init__.py +++ b/src/superannotate/lib/app/input_converters/__init__.py @@ -0,0 +1,12 @@ +from .conversion import convert_json_version +from .conversion import convert_project_type +from .conversion import export_annotation +from .conversion import import_annotation + + +__all__ = [ + "convert_json_version", + "convert_project_type", + "export_annotation", + "import_annotation", +] diff --git a/src/superannotate/lib/app/input_converters/conversion.py b/src/superannotate/lib/app/input_converters/conversion.py index 0bb412325..812714089 100644 --- a/src/superannotate/lib/app/input_converters/conversion.py +++ b/src/superannotate/lib/app/input_converters/conversion.py @@ -6,7 +6,6 @@ from lib.app.exceptions import AppException from lib.app.interface.base_interface import Tracker -from lib.app.mixp.decorators import Trackable from lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE from lib.core import LIMITED_FUNCTIONS from lib.core.enums import ProjectType @@ -146,41 +145,41 @@ def export_annotation( task="object_detection", ): """ - Converts SuperAnnotate annotation format to the other annotation formats. Currently available (project_type, task) combinations for converter - presented below: - - ============== ====================== - From SA to COCO - -------------------------------------- - project_type task - ============== ====================== - Pixel panoptic_segmentation - Pixel instance_segmentation - Vector instance_segmentation - Vector object_detection - Vector keypoint_detection - ============== ====================== - - :param input_dir: Path to the dataset folder that you want to convert. - :type input_dir: Pathlike(str or Path) - :param output_dir: Path to the folder, where you want to have converted dataset. - :type output_dir: Pathlike(str or Path) - :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"] - :type dataset_format: str - :param dataset_name: Will be used to create json file in the output_dir. - :type dataset_name: str - :param project_type: SuperAnnotate project type is either 'Vector' or 'Pixel' (Default: 'Vector') - 'Vector' project creates ___objects.json for each image. - 'Pixel' project creates ___pixel.jsons and ___save.png annotation mask for each image. - :type project_type: str - :param task: Task can be one of the following: ['panoptic_segmentation', 'instance_segmentation', - 'keypoint_detection', 'object_detection']. (Default: "object_detection"). - 'keypoint_detection' can be used to converts keypoints from/to available annotation format. - 'panoptic_segmentation' will use panoptic mask for each image to generate bluemask for SuperAnnotate annotation format and use bluemask to generate panoptic mask for invert conversion. Panoptic masks should be in the input folder. - 'instance_segmentation' 'Pixel' project_type converts instance masks and 'Vector' project_type generates bounding boxes and polygons from instance masks. Masks should be in the input folder if it is 'Pixel' project_type. - 'object_detection' converts objects from/to available annotation format - :type task: str - """ + Converts SuperAnnotate annotation format to the other annotation formats. Currently available (project_type, task) combinations for converter + presented below: + + ============== ====================== + From SA to COCO + -------------------------------------- + project_type task + ============== ====================== + Pixel panoptic_segmentation + Pixel instance_segmentation + Vector instance_segmentation + Vector object_detection + Vector keypoint_detection + ============== ====================== + + :param input_dir: Path to the dataset folder that you want to convert. + :type input_dir: Pathlike(str or Path) + :param output_dir: Path to the folder, where you want to have converted dataset. + :type output_dir: Pathlike(str or Path) + :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"] + :type dataset_format: str + :param dataset_name: Will be used to create json file in the output_dir. + :type dataset_name: str + :param project_type: SuperAnnotate project type is either 'Vector' or 'Pixel' (Default: 'Vector') + 'Vector' project creates ___objects.json for each image. + 'Pixel' project creates ___pixel.jsons and ___save.png annotation mask for each image. + :type project_type: str + :param task: Task can be one of the following: ['panoptic_segmentation', 'instance_segmentation', + 'keypoint_detection', 'object_detection']. (Default: "object_detection"). + 'keypoint_detection' can be used to converts keypoints from/to available annotation format. + 'panoptic_segmentation' will use panoptic mask for each image to generate bluemask for SuperAnnotate annotation format and use bluemask to generate panoptic mask for invert conversion. Panoptic masks should be in the input folder. + 'instance_segmentation' 'Pixel' project_type converts instance masks and 'Vector' project_type generates bounding boxes and polygons from instance masks. Masks should be in the input folder if it is 'Pixel' project_type. + 'object_detection' converts objects from/to available annotation format + :type task: str + """ if project_type in [ ProjectType.VIDEO.name, @@ -224,7 +223,7 @@ def export_annotation( export_from_sa(args) -@Trackable +@Tracker def import_annotation( input_dir, output_dir, @@ -408,9 +407,9 @@ def import_annotation( import_to_sa(args) -@Trackable +@Tracker def convert_project_type(input_dir, output_dir): - """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. + """Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. :param input_dir: Path to the dataset folder that you want to convert. :type input_dir: Pathlike(str or Path) @@ -436,7 +435,7 @@ def convert_project_type(input_dir, output_dir): sa_convert_project_type(input_dir, output_dir) -@Trackable +@Tracker def convert_json_version(input_dir, output_dir, version=2): """ Converts SuperAnnotate JSON versions. Newest JSON version is 2. diff --git a/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py b/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py index 4c5834986..81fa39461 100644 --- a/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py +++ b/src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py @@ -137,9 +137,9 @@ def convert_from_old_sa_to_new(self, old_json_data, project_type): def _parse_json_into_common_format(self, sa_annotation_json, fpath): """ - If the annotation format ever changes this function will handle it and - return something optimal for the converters. Additionally, if anything - important is absent from the current json, this function fills it. + If the annotation format ever changes this function will handle it and + return something optimal for the converters. Additionally, if anything + important is absent from the current json, this function fills it. """ if isinstance(sa_annotation_json, list): sa_annotation_json = self.convert_from_old_sa_to_new( diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 5c5a8b9c6..4dfd4f1fb 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -1,19 +1,61 @@ import functools +import os import sys -from abc import ABC from abc import abstractmethod from inspect import signature +from pathlib import Path from types import FunctionType from typing import Iterable from typing import Sized +import lib.core as constants from lib.app.helpers import extract_project_folder +from lib.app.interface.types import validate_arguments +from lib.core.exceptions import AppException from lib.core.reporter import Session +from lib.infrastructure.controller import Controller +from lib.infrastructure.repositories import ConfigRepository from mixpanel import Mixpanel from version import __version__ -class BaseInterfaceFacade(ABC): +class BaseInterfaceFacade: + REGISTRY = [] + + def __init__( + self, + token: str = None, + config_path: str = constants.CONFIG_PATH, + ): + host = constants.BACKEND_URL + env_token = os.environ.get("SA_TOKEN") + version = os.environ.get("SA_VERSION", "v1") + ssl_verify = bool(os.environ.get("SA_SSL", True)) + if token: + token = Controller.validate_token(token=token) + elif env_token: + host = os.environ.get("SA_URL", constants.BACKEND_URL) + + token = Controller.validate_token(env_token) + else: + config_path = os.path.expanduser(str(config_path)) + if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + raise AppException( + f"SuperAnnotate config file {str(config_path)} not found." + f" Please provide correct config file location to sa.init() or use " + f"CLI's superannotate init to generate default location config file." + ) + config_repo = ConfigRepository(config_path) + token, host, ssl_verify = ( + Controller.validate_token(config_repo.get_one("token").value), + config_repo.get_one("main_endpoint").value, + config_repo.get_one("ssl_verify").value, + ) + self._host = host + self._token = token + self.controller = Controller(token, host, ssl_verify, version) + BaseInterfaceFacade.REGISTRY.append(self) + @property @abstractmethod def host(self): @@ -31,32 +73,38 @@ def logger(self): class Tracker: - TEAM_DATA = None - INITIAL_EVENT = {"event_name": "SDK init", "properties": {}} - INITIAL_LOGGED = False - - @staticmethod - def get_mp_instance() -> Mixpanel: - # if "api.annotate.online" in get_default_controller()._backend_url: - # return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") - return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") + def get_mp_instance(self) -> Mixpanel: + if self.client: + if self.client.host == constants.BACKEND_URL: + return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") + else: + return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") @staticmethod - def get_default_payload(team_name, user_id, project_name=None): + def get_default_payload(team_name, user_id): return { "SDK": True, - "Paid": True, "Team": team_name, "Team Owner": user_id, - "Project Name": project_name, - "Project Role": "Admin", "Version": __version__, } def __init__(self, function): self.function = function + self._client = None functools.update_wrapper(self, function) + @property + def client(self): + if not self._client: + if BaseInterfaceFacade.REGISTRY: + self._client = BaseInterfaceFacade.REGISTRY[-1] + else: + from lib.app.interface.sdk_interface import SAClient + + self._client = SAClient() + return self._client + @staticmethod def extract_arguments(function, *args, **kwargs) -> dict: bound_arguments = signature(function).bind(*args, **kwargs) @@ -69,12 +117,16 @@ def default_parser(function_name: str, kwargs: dict) -> tuple: for key, value in kwargs.items(): if key == "self": continue + elif value is None: + properties[key] = value elif key == "project": - properties["project_name"], properties["folder_name"] = extract_project_folder(value) - elif isinstance(value, (str, int, float, bool, str)): + properties["project_name"], folder_name = extract_project_folder(value) + if folder_name: + properties["folder_name"] = folder_name + elif isinstance(value, (str, int, float, bool)): properties[key] = value elif isinstance(value, dict): - properties[key] = value.keys() + properties[key] = list(value.keys()) elif isinstance(value, Sized): properties[key] = len(value) elif isinstance(value, Iterable): @@ -83,56 +135,51 @@ def default_parser(function_name: str, kwargs: dict) -> tuple: properties[key] = str(value) return function_name, properties - def track(self, args, kwargs, success: bool, session): - try: - function_name = self.function.__name__ if self.function else "" - arguments = self.extract_arguments(self.function, *args, **kwargs) - event_name, properties = self.default_parser(function_name, arguments) - - user_id = self.team_data.creator_id - team_name = self.team_data.name - properties["Success"] = success - - default = self.get_default_payload( - team_name=team_name, - user_id=user_id, - project_name=properties.pop("project_name", None), - ) - if "pytest" not in sys.modules: - self.get_mp_instance().track(user_id, event_name, {**default, **properties, **session.data}) - except Exception: - raise - - def __get__(self, instance, owner): - if instance is None: - return self - d = self - mfactory = lambda self, *args, **kw: d(self, *args, **kw) - mfactory.__name__ = self.function.__name__ - self.team_data = instance.controller.team_data.data - return mfactory.__get__(instance, owner) + def _track(self, user_id: str, event_name: str, data: dict): + if "pytest" not in sys.modules: + self.get_mp_instance().track(user_id, event_name, data) + + def _track_method(self, args, kwargs, success: bool): + function_name = self.function.__name__ if self.function else "" + arguments = self.extract_arguments(self.function, *args, **kwargs) + event_name, properties = self.default_parser(function_name, arguments) + + user_id = self.client.controller.team_data.creator_id + team_name = self.client.controller.team_data.name + + properties["Success"] = success + default = self.get_default_payload(team_name=team_name, user_id=user_id) + self._track( + user_id, + event_name, + {**default, **properties, **Session.get_current_session().data}, + ) + + def __get__(self, obj, owner=None): + if obj is not None: + self._client = obj + tmp = functools.partial(self.__call__, obj) + functools.update_wrapper(tmp, self.function) + return tmp + return self def __call__(self, *args, **kwargs): success = True try: - with Session(self.function.__name__) as session: - result = self.function(*args, **kwargs) + result = self.function(*args, **kwargs) except Exception as e: success = False raise e else: return result finally: - self.track(args=args, kwargs=kwargs, success=success, session=session) + self._track_method(args=args, kwargs=kwargs, success=success) class TrackableMeta(type): def __new__(mcs, name, bases, attrs): - for attr_name, attr_value in attrs.iteritems(): + for attr_name, attr_value in attrs.items(): if isinstance(attr_value, FunctionType): - attrs[attr_name] = mcs.decorate(attr_value) - return super().__new__(mcs, name, bases, attrs) - - @staticmethod - def decorate(func): - return Tracker(func) + attrs[attr_name] = Tracker(validate_arguments(attr_value)) + tmp = super().__new__(mcs, name, bases, attrs) + return tmp diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index c0576ccce..54f6f90e0 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -25,7 +25,7 @@ def version(): To show the version of the current SDK installation """ with open( - f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" + f"{os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(lib_path))))}/version.py" ) as f: version = f.read().rstrip()[15:-1] print(version) @@ -40,7 +40,7 @@ def init(): config = repo.get_one(uuid=constances.TOKEN_UUID) if config: if not input( - f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " + f"File {repo.config_path} exists. Do you want to overwrite? [y/n] : " ).lower() in ("y", "yes"): return token = input( @@ -68,14 +68,14 @@ def create_folder(self, project: str, name: str): sys.exit(0) def upload_images( - self, - project: str, - folder: str, - extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, - set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, - exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders=False, - image_quality_in_editor=None, + self, + project: str, + folder: str, + extensions: str = constances.DEFAULT_IMAGE_EXTENSIONS, + set_annotation_status: str = constances.AnnotationStatus.NOT_STARTED.name, + exclude_file_patterns=constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders=False, + image_quality_in_editor=None, ): """ To upload images from folder to project use: @@ -99,12 +99,12 @@ def upload_images( sys.exit(0) def export_project( - self, - project, - folder, - include_fuse=False, - disable_extract_zip_contents=False, - annotation_statuses=None, + self, + project, + folder, + include_fuse=False, + disable_extract_zip_contents=False, + annotation_statuses=None, ): project_name, folder_name = split_project_path(project) folders = None @@ -113,13 +113,16 @@ def export_project( if folder_name: folders = [folder_name] export_res = SAClient().prepare_export( - project_name, folders, include_fuse, False, annotation_statuses + project=project_name, + folder_names=folders, + include_fuse=include_fuse, + annotation_statuses=annotation_statuses, ) - export_name = export_res.data["name"] + export_name = export_res["name"] SAClient().download_export( - project_name=project_name, - export_name=export_name, + project=project_name, + export=export_name, folder_path=folder, extract_zip_contents=not disable_extract_zip_contents, to_s3_bucket=False, @@ -127,7 +130,7 @@ def export_project( sys.exit(0) def upload_preannotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload preannotations from folder to project use @@ -148,7 +151,7 @@ def upload_preannotations( sys.exit(0) def upload_annotations( - self, project, folder, dataset_name=None, task=None, format=None + self, project, folder, dataset_name=None, task=None, format=None ): """ To upload annotations from folder to project use @@ -169,11 +172,13 @@ def upload_annotations( sys.exit(0) def _upload_annotations( - self, project, folder, format, dataset_name, task, pre=True + self, project, folder, format, dataset_name, task, pre=True ): project_folder_name = project project_name, folder_name = split_project_path(project) - project = SAClient().controller.get_project_metadata(project_name=project_name).data + project = ( + SAClient().controller.get_project_metadata(project_name=project_name).data + ) if not format: format = "SuperAnnotate" if not dataset_name and format == "COCO": @@ -207,10 +212,10 @@ def _upload_annotations( sys.exit(0) def attach_image_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): """ To attach image URLs to project use: @@ -224,10 +229,10 @@ def attach_image_urls( sys.exit(0) def attach_video_urls( - self, - project: str, - attachments: str, - annotation_status: Optional[Any] = "NotStarted", + self, + project: str, + attachments: str, + annotation_status: Optional[Any] = "NotStarted", ): SAClient().attach_items( project=project, @@ -238,7 +243,7 @@ def attach_video_urls( @staticmethod def attach_document_urls( - project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" + project: str, attachments: str, annotation_status: Optional[Any] = "NotStarted" ): SAClient().attach_items( project=project, @@ -248,15 +253,15 @@ def attach_document_urls( sys.exit(0) def upload_videos( - self, - project, - folder, - target_fps=None, - recursive=False, - extensions=constances.DEFAULT_VIDEO_EXTENSIONS, - set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, - start_time=0.0, - end_time=None, + self, + project, + folder, + target_fps=None, + recursive=False, + extensions=constances.DEFAULT_VIDEO_EXTENSIONS, + set_annotation_status=constances.AnnotationStatus.NOT_STARTED.name, + start_time=0.0, + end_time=None, ): """ To upload videos from folder to project use diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 4e6177c79..59ef32313 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -21,6 +21,7 @@ from lib.app.helpers import get_annotation_paths from lib.app.helpers import get_name_url_duplicated_from_csv from lib.app.interface.base_interface import BaseInterfaceFacade +from lib.app.interface.base_interface import TrackableMeta from lib.app.interface.types import AnnotationStatuses from lib.app.interface.types import AnnotationType from lib.app.interface.types import AnnotatorRole @@ -33,7 +34,6 @@ from lib.app.interface.types import ProjectStatusEnum from lib.app.interface.types import ProjectTypes from lib.app.interface.types import Setting -from lib.app.interface.types import validate_arguments from lib.app.serializers import BaseSerializer from lib.app.serializers import FolderSerializer from lib.app.serializers import ProjectSerializer @@ -51,7 +51,6 @@ from lib.core.types import PriorityScore from lib.core.types import Project from lib.infrastructure.controller import Controller -from lib.infrastructure.repositories import ConfigRepository from pydantic import conlist from pydantic import parse_obj_as from pydantic import StrictBool @@ -62,52 +61,7 @@ logger = get_default_logger() -class SAClient(BaseInterfaceFacade): - def __init__( - self, - token: str = None, - config_path: str = constances.CONFIG_FILE_LOCATION, - ): - host = constances.BACKEND_URL - env_token = os.environ.get("SA_TOKEN") - version = os.environ.get("SA_VERSION", "v1") - ssl_verify = bool(os.environ.get("SA_SSL", True)) - if token: - token = Controller.validate_token(token=token) - elif env_token: - host = os.environ.get("SA_URL", constances.BACKEND_URL) - - token = Controller.validate_token(env_token) - else: - config_path = str(config_path) - if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): - raise AppException( - f"SuperAnnotate config file {str(config_path)} not found." - f" Please provide correct config file location to sa.init() or use " - f"CLI's superannotate init to generate default location config file." - ) - config_repo = ConfigRepository(config_path) - token, host, ssl_verify = ( - Controller.validate_token(config_repo.get_one("token").value), - config_repo.get_one("main_endpoint").value, - config_repo.get_one("ssl_verify").value, - ) - self._host = host - self._token = token - self.controller = Controller(token, host, ssl_verify, version) - - @property - def host(self): - return self._host - - @property - def token(self): - return self._token - - @property - def logger(self): - pass - +class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta): def get_team_metadata(self): """Returns team metadata @@ -117,13 +71,12 @@ def get_team_metadata(self): response = self.controller.get_team() return TeamSerializer(response.data).serialize() - @validate_arguments def search_team_contributors( - self, - email: EmailStr = None, - first_name: NotEmptyStr = None, - last_name: NotEmptyStr = None, - return_metadata: bool = True, + self, + email: EmailStr = None, + first_name: NotEmptyStr = None, + last_name: NotEmptyStr = None, + return_metadata: bool = True, ): """Search for contributors in the team @@ -148,13 +101,12 @@ def search_team_contributors( return [contributor["email"] for contributor in contributors] return contributors - @validate_arguments def search_projects( - self, - name: Optional[NotEmptyStr] = None, - return_metadata: bool = False, - include_complete_image_count: bool = False, - status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, + self, + name: Optional[NotEmptyStr] = None, + return_metadata: bool = False, + include_complete_image_count: bool = False, + status: Optional[Union[ProjectStatusEnum, List[ProjectStatusEnum]]] = None, ): """ Project name based case-insensitive search for projects. @@ -184,7 +136,7 @@ def search_projects( result = self.controller.search_project( name=name, include_complete_image_count=include_complete_image_count, - statuses=statuses + statuses=statuses, ).data if return_metadata: @@ -203,13 +155,12 @@ def search_projects( else: return [project.name for project in result] - @validate_arguments def create_project( - self, - project_name: NotEmptyStr, - project_description: NotEmptyStr, - project_type: NotEmptyStr, - settings: List[Setting] = None, + self, + project_name: NotEmptyStr, + project_description: NotEmptyStr, + project_type: NotEmptyStr, + settings: List[Setting] = None, ): """Create a new project in the team. @@ -243,9 +194,7 @@ def create_project( return ProjectSerializer(response.data).serialize() - @validate_arguments - def create_project_from_metadata( - self, project_metadata: Project): + def create_project_from_metadata(self, project_metadata: Project): """Create a new project in the team using project metadata object dict. Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) Non-mandatory keys: "workflow", "settings" and "annotation_classes". @@ -269,16 +218,15 @@ def create_project_from_metadata( raise AppException(response.errors) return ProjectSerializer(response.data).serialize() - @validate_arguments def clone_project( - self, - project_name: Union[NotEmptyStr, dict], - from_project: Union[NotEmptyStr, dict], - project_description: Optional[NotEmptyStr] = None, - copy_annotation_classes: Optional[StrictBool] = True, - copy_settings: Optional[StrictBool] = True, - copy_workflow: Optional[StrictBool] = True, - copy_contributors: Optional[StrictBool] = False, + self, + project_name: Union[NotEmptyStr, dict], + from_project: Union[NotEmptyStr, dict], + project_description: Optional[NotEmptyStr] = None, + copy_annotation_classes: Optional[StrictBool] = True, + copy_settings: Optional[StrictBool] = True, + copy_workflow: Optional[StrictBool] = True, + copy_contributors: Optional[StrictBool] = False, ): """Create a new project in the team using annotation classes and settings from from_project. @@ -314,9 +262,7 @@ def clone_project( raise AppException(response.errors) return ProjectSerializer(response.data).serialize() - @validate_arguments - def create_folder( - self, project: NotEmptyStr, folder_name: NotEmptyStr): + def create_folder(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Create a new folder in the project. :param project: project name @@ -328,9 +274,7 @@ def create_folder( :rtype: dict """ - res = self.controller.create_folder( - project=project, folder_name=folder_name - ) + res = self.controller.create_folder(project=project, folder_name=folder_name) if res.data: folder = res.data logger.info(f"Folder {folder.name} created in project {project}") @@ -338,22 +282,18 @@ def create_folder( if res.errors: raise AppException(res.errors) - @validate_arguments - def delete_project( - self, project: Union[NotEmptyStr, dict]): + def delete_project(self, project: Union[NotEmptyStr, dict]): """Deletes the project - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str """ name = project if isinstance(project, dict): name = project["name"] self.controller.delete_project(name=name) - @validate_arguments - def rename_project( - self, project: NotEmptyStr, new_name: NotEmptyStr): + def rename_project(self, project: NotEmptyStr, new_name: NotEmptyStr): """Renames the project :param project: project name or folder path (e.g., "project1/folder1") @@ -367,12 +307,12 @@ def rename_project( ) if response.errors: raise AppException(response.errors) - logger.info("Successfully renamed project %s to %s.", project, response.data.name) + logger.info( + "Successfully renamed project %s to %s.", project, response.data.name + ) return ProjectSerializer(response.data).serialize() - @validate_arguments - def get_folder_metadata( - self, project: NotEmptyStr, folder_name: NotEmptyStr): + def get_folder_metadata(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Returns folder metadata :param project: project name @@ -383,18 +323,14 @@ def get_folder_metadata( :return: metadata of folder :rtype: dict """ - result = ( - self.controller - .get_folder(project_name=project, folder_name=folder_name) - .data - ) + result = self.controller.get_folder( + project_name=project, folder_name=folder_name + ).data if not result: raise AppException("Folder not found.") return FolderSerializer(result).serialize() - @validate_arguments - def delete_folders( - self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): + def delete_folders(self, project: NotEmptyStr, folder_names: List[NotEmptyStr]): """Delete folder in project. :param project: project name @@ -410,12 +346,11 @@ def delete_folders( raise AppException(res.errors) logger.info(f"Folders {folder_names} deleted in project {project}") - @validate_arguments def search_folders( - self, - project: NotEmptyStr, - folder_name: Optional[NotEmptyStr] = None, - return_metadata: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + folder_name: Optional[NotEmptyStr] = None, + return_metadata: Optional[StrictBool] = False, ): """Folder name based case-insensitive search for folders in project. @@ -440,15 +375,14 @@ def search_folders( return [FolderSerializer(folder).serialize() for folder in data] return [folder.name for folder in data] - @validate_arguments def copy_image( - self, - source_project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = False, - copy_annotation_status: Optional[StrictBool] = False, - copy_pin: Optional[StrictBool] = False, + self, + source_project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + destination_project: Union[NotEmptyStr, dict], + include_annotations: Optional[StrictBool] = False, + copy_annotation_status: Optional[StrictBool] = False, + copy_pin: Optional[StrictBool] = False, ): """Copy image to a project. The image's project is the same as destination project then the name will be changed to _()., @@ -473,12 +407,12 @@ def copy_image( destination_project, destination_folder = extract_project_folder( destination_project ) - source_project_metadata = ( - self.controller.get_project_metadata(source_project_name).data - ) - destination_project_metadata = ( - self.controller.get_project_metadata(destination_project).data - ) + source_project_metadata = self.controller.get_project_metadata( + source_project_name + ).data + destination_project_metadata = self.controller.get_project_metadata( + destination_project + ).data if destination_project_metadata["project"].type in [ constances.ProjectType.VIDEO.value, @@ -487,7 +421,9 @@ def copy_image( constances.ProjectType.VIDEO.value, constances.ProjectType.DOCUMENT.value, ]: - raise AppException(LIMITED_FUNCTIONS[source_project_metadata["project"].type]) + raise AppException( + LIMITED_FUNCTIONS[source_project_metadata["project"].type] + ) response = self.controller.copy_image( from_project_name=source_project_name, @@ -520,15 +456,14 @@ def copy_image( f" to {destination_project}/{destination_folder}." ) - @validate_arguments def get_project_metadata( - self, - project: Union[NotEmptyStr, dict], - include_annotation_classes: Optional[StrictBool] = False, - include_settings: Optional[StrictBool] = False, - include_workflow: Optional[StrictBool] = False, - include_contributors: Optional[StrictBool] = False, - include_complete_image_count: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + include_annotation_classes: Optional[StrictBool] = False, + include_settings: Optional[StrictBool] = False, + include_workflow: Optional[StrictBool] = False, + include_contributors: Optional[StrictBool] = False, + include_complete_image_count: Optional[StrictBool] = False, ): """Returns project metadata @@ -555,30 +490,26 @@ def get_project_metadata( :rtype: dict """ project_name, folder_name = extract_project_folder(project) - response = ( - self.controller.get_project_metadata( - project_name, - include_annotation_classes, - include_settings, - include_workflow, - include_contributors, - include_complete_image_count, - ) - .data - ) + response = self.controller.get_project_metadata( + project_name, + include_annotation_classes, + include_settings, + include_workflow, + include_contributors, + include_complete_image_count, + ).data metadata = ProjectSerializer(response["project"]).serialize() for elem in "classes", "workflows", "contributors": if response.get(elem): metadata[elem] = [ - BaseSerializer(attribute).serialize() for attribute in response[elem] + BaseSerializer(attribute).serialize() + for attribute in response[elem] ] return metadata - @validate_arguments - def get_project_settings( - self, project: Union[NotEmptyStr, dict]): + def get_project_settings(self, project: Union[NotEmptyStr, dict]): """Gets project's settings. Return value example: [{ "attribute" : "Brightness", "value" : 10, ...},...] @@ -596,9 +527,7 @@ def get_project_settings( ] return settings - @validate_arguments - def get_project_workflow( - self, project: Union[str, dict]): + def get_project_workflow(self, project: Union[str, dict]): """Gets project's workflow. Return value example: [{ "step" : , "className" : , "tool" : , ...},...] @@ -615,10 +544,8 @@ def get_project_workflow( raise AppException(workflow.errors) return workflow.data - @validate_arguments def search_annotation_classes( - self, - project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None + self, project: Union[NotEmptyStr, dict], name_contains: Optional[str] = None ): """Searches annotation classes by name_prefix (case-insensitive) @@ -632,17 +559,14 @@ def search_annotation_classes( :rtype: list of dicts """ project_name, folder_name = extract_project_folder(project) - classes = self.controller.search_annotation_classes( - project_name, name_contains - ) + classes = self.controller.search_annotation_classes(project_name, name_contains) classes = [BaseSerializer(attribute).serialize() for attribute in classes.data] return classes - @validate_arguments def set_project_default_image_quality_in_editor( - self, - project: Union[NotEmptyStr, dict], - image_quality_in_editor: Optional[str], + self, + project: Union[NotEmptyStr, dict], + image_quality_in_editor: Optional[str], ): """Sets project's default image quality in editor setting. @@ -656,16 +580,19 @@ def set_project_default_image_quality_in_editor( response = self.controller.set_project_settings( project_name=project_name, - new_settings=[{"attribute": "ImageQuality", "value": image_quality_in_editor}], + new_settings=[ + {"attribute": "ImageQuality", "value": image_quality_in_editor} + ], ) if response.errors: raise AppException(response.errors) return response.data - @validate_arguments def pin_image( - self, - project: Union[NotEmptyStr, dict], image_name: str, pin: Optional[StrictBool] = True + self, + project: Union[NotEmptyStr, dict], + image_name: str, + pin: Optional[StrictBool] = True, ): """Pins (or unpins) image @@ -684,12 +611,11 @@ def pin_image( is_pinned=int(pin), ) - @validate_arguments def set_images_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: NotEmptyStr, - image_names: Optional[List[NotEmptyStr]] = None, + self, + project: Union[NotEmptyStr, dict], + annotation_status: NotEmptyStr, + image_names: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of images @@ -716,10 +642,8 @@ def set_images_annotation_statuses( raise AppException(response.errors) logger.info("Annotations status of images changed") - @validate_arguments def delete_images( - self, - project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None + self, project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None ): """Delete images in project. @@ -743,9 +667,9 @@ def delete_images( f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) - @validate_arguments def assign_images( - self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str): + self, project: Union[NotEmptyStr, dict], image_names: List[str], user: str + ): """Assigns images to a user. The assignment role, QA or Annotator, will be deduced from the user's role in the project. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -767,10 +691,11 @@ def assign_images( raise AppException(LIMITED_FUNCTIONS[project["project"].type]) contributors = ( - self.controller - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users ) contributor = None for c in contributors: @@ -791,9 +716,9 @@ def assign_images( else: raise AppException(response.errors) - @validate_arguments def unassign_images( - self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr]): + self, project: Union[NotEmptyStr, dict], image_names: List[NotEmptyStr] + ): """Removes assignment of given images for all assignees.With SDK, the user can be assigned to a role in the project with the share_project function. @@ -811,9 +736,7 @@ def unassign_images( if response.errors: raise AppException(response.errors) - @validate_arguments - def unassign_folder( - self, project_name: NotEmptyStr, folder_name: NotEmptyStr): + def unassign_folder(self, project_name: NotEmptyStr, folder_name: NotEmptyStr): """Removes assignment of given folder for all assignees. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -829,10 +752,11 @@ def unassign_folder( if response.errors: raise AppException(response.errors) - @validate_arguments def assign_folder( - self, - project_name: NotEmptyStr, folder_name: NotEmptyStr, users: List[NotEmptyStr] + self, + project_name: NotEmptyStr, + folder_name: NotEmptyStr, + users: List[NotEmptyStr], ): """Assigns folder to users. With SDK, the user can be assigned to a role in the project with the share_project function. @@ -846,10 +770,11 @@ def assign_folder( """ contributors = ( - self.controller - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["project"] - .users + self.controller.get_project_metadata( + project_name=project_name, include_contributors=True + ) + .data["project"] + .users ) verified_users = [i["user_id"] for i in contributors] verified_users = set(users).intersection(set(verified_users)) @@ -864,27 +789,28 @@ def assign_folder( return response = self.controller.assign_folder( - project_name=project_name, folder_name=folder_name, users=list(verified_users) + project_name=project_name, + folder_name=folder_name, + users=list(verified_users), ) if response.errors: raise AppException(response.errors) - @validate_arguments def upload_images_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[List[NotEmptyStr], Tuple[NotEmptyStr]] - ] = constances.DEFAULT_IMAGE_EXTENSIONS, - annotation_status="NotStarted", - from_s3_bucket=None, - exclude_file_patterns: Optional[ - Iterable[NotEmptyStr] - ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, - recursive_subfolders: Optional[StrictBool] = False, - image_quality_in_editor: Optional[str] = None, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[List[NotEmptyStr], Tuple[NotEmptyStr]] + ] = constances.DEFAULT_IMAGE_EXTENSIONS, + annotation_status="NotStarted", + from_s3_bucket=None, + exclude_file_patterns: Optional[ + Iterable[NotEmptyStr] + ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + recursive_subfolders: Optional[StrictBool] = False, + image_quality_in_editor: Optional[str] = None, ): """Uploads all images with given extensions from folder_path to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -967,24 +893,29 @@ def upload_images_from_folder_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) + "%s already existing images found that won't be uploaded.", + len(duplicates), ) logger.info( - "Uploading %s images to project %s.", len(images_to_upload), project_folder_name + "Uploading %s images to project %s.", + len(images_to_upload), + project_folder_name, ) if not images_to_upload: return [], [], duplicates if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: for _ in use_case.execute(): progress_bar.update(1) return use_case.data raise AppException(use_case.response.errors) - @validate_arguments def get_project_image_count( - self, - project: Union[NotEmptyStr, dict], with_all_subfolders: Optional[StrictBool] = False + self, + project: Union[NotEmptyStr, dict], + with_all_subfolders: Optional[StrictBool] = False, ): """Returns number of images in the project. @@ -1008,12 +939,11 @@ def get_project_image_count( raise AppException(response.errors) return response.data - @validate_arguments def download_image_annotations( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Union[str, Path], + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Union[str, Path], ): """Downloads annotations of the image (JSON and mask if pixel type project) to local_dir_path. @@ -1039,9 +969,9 @@ def download_image_annotations( raise AppException(res.errors) return res.data - @validate_arguments def get_exports( - self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False): + self, project: NotEmptyStr, return_metadata: Optional[StrictBool] = False + ): """Get all prepared exports of the project. :param project: project name @@ -1057,14 +987,13 @@ def get_exports( ) return response.data - @validate_arguments def prepare_export( - self, - project: Union[NotEmptyStr, dict], - folder_names: Optional[List[NotEmptyStr]] = None, - annotation_statuses: Optional[List[AnnotationStatuses]] = None, - include_fuse: Optional[StrictBool] = False, - only_pinned=False, + self, + project: Union[NotEmptyStr, dict], + folder_names: Optional[List[NotEmptyStr]] = None, + annotation_statuses: Optional[List[AnnotationStatuses]] = None, + include_fuse: Optional[StrictBool] = False, + only_pinned=False, ): """Prepare annotations and classes.json for export. Original and fused images for images with annotations can be included with include_fuse flag. @@ -1110,21 +1039,20 @@ def prepare_export( raise AppException(response.errors) return response.data - @validate_arguments def upload_videos_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[NotEmptyStr, Path], - extensions: Optional[ - Union[Tuple[NotEmptyStr], List[NotEmptyStr]] - ] = constances.DEFAULT_VIDEO_EXTENSIONS, - exclude_file_patterns: Optional[List[NotEmptyStr]] = (), - recursive_subfolders: Optional[StrictBool] = False, - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[NotEmptyStr, Path], + extensions: Optional[ + Union[Tuple[NotEmptyStr], List[NotEmptyStr]] + ] = constances.DEFAULT_VIDEO_EXTENSIONS, + exclude_file_patterns: Optional[List[NotEmptyStr]] = (), + recursive_subfolders: Optional[StrictBool] = False, + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads image frames from all videos with given extensions from folder_path to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -1164,7 +1092,9 @@ def upload_videos_from_folder_to_project( if not recursive_subfolders: video_paths += list(Path(folder_path).glob(f"*.{extension.lower()}")) if os.name != "nt": - video_paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) + video_paths += list( + Path(folder_path).glob(f"*.{extension.upper()}") + ) else: logger.warning( "When using recursive subfolder parsing same name videos " @@ -1172,7 +1102,9 @@ def upload_videos_from_folder_to_project( ) video_paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) if os.name != "nt": - video_paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) + video_paths += list( + Path(folder_path).rglob(f"*.{extension.upper()}") + ) video_paths = [str(path) for path in video_paths] response = self.controller.upload_videos( @@ -1190,16 +1122,15 @@ def upload_videos_from_folder_to_project( raise AppException(response.errors) return response.data - @validate_arguments def upload_video_to_project( - self, - project: Union[NotEmptyStr, dict], - video_path: Union[NotEmptyStr, Path], - target_fps: Optional[int] = None, - start_time: Optional[float] = 0.0, - end_time: Optional[float] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: Union[NotEmptyStr, dict], + video_path: Union[NotEmptyStr, Path], + target_fps: Optional[int] = None, + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads image frames from video to platform. Uploaded images will have names "_.jpg". @@ -1242,14 +1173,13 @@ def upload_video_to_project( raise AppException(response.errors) return response.data - @validate_arguments def create_annotation_class( - self, - project: Union[Project, NotEmptyStr], - name: NotEmptyStr, - color: NotEmptyStr, - attribute_groups: Optional[List[AttributeGroup]] = None, - class_type: ClassType = "object", + self, + project: Union[Project, NotEmptyStr], + name: NotEmptyStr, + color: NotEmptyStr, + attribute_groups: Optional[List[AttributeGroup]] = None, + class_type: ClassType = "object", ): """Create annotation class in project @@ -1285,10 +1215,8 @@ def create_annotation_class( raise AppException(response.errors) return BaseSerializer(response.data).serialize() - @validate_arguments def delete_annotation_class( - self, - project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] + self, project: NotEmptyStr, annotation_class: Union[dict, NotEmptyStr] ): """Deletes annotation class from project @@ -1301,9 +1229,9 @@ def delete_annotation_class( project_name=project, annotation_class_name=annotation_class ) - @validate_arguments def download_annotation_classes_json( - self, project: NotEmptyStr, folder: Union[str, Path]): + self, project: NotEmptyStr, folder: Union[str, Path] + ): """Downloads project classes.json to folder :param project: project name @@ -1321,12 +1249,11 @@ def download_annotation_classes_json( raise AppException(response.errors) return response.data - @validate_arguments def create_annotation_classes_from_classes_json( - self, - project: Union[NotEmptyStr, dict], - classes_json: Union[List[AnnotationClassEntity], str, Path], - from_s3_bucket=False, + self, + project: Union[NotEmptyStr, dict], + classes_json: Union[List[AnnotationClassEntity], str, Path], + from_s3_bucket=False, ): """Creates annotation classes in project from a SuperAnnotate format annotation classes.json. @@ -1359,20 +1286,20 @@ def create_annotation_classes_from_classes_json( raise AppException("Couldn't validate annotation classes.") logger.info(f"Creating annotation classes in project {project}.") response = self.controller.create_annotation_classes( - project_name=project, annotation_classes=annotation_classes, + project_name=project, + annotation_classes=annotation_classes, ) if response.errors: raise AppException(response.errors) return [BaseSerializer(i).serialize() for i in response.data] - @validate_arguments def download_export( - self, - project: Union[NotEmptyStr, dict], - export: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - extract_zip_contents: Optional[StrictBool] = True, - to_s3_bucket=None, + self, + project: Union[NotEmptyStr, dict], + export: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + extract_zip_contents: Optional[StrictBool] = True, + to_s3_bucket=None, ): """Download prepared export. @@ -1405,12 +1332,11 @@ def download_export( raise AppException(response.errors) logger.info(response.data) - @validate_arguments def set_image_annotation_status( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - annotation_status: NotEmptyStr, + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + annotation_status: NotEmptyStr, ): """Sets the image annotation status @@ -1438,14 +1364,12 @@ def set_image_annotation_status( ) if response.errors: raise AppException(response.errors) - image = ( - self.controller.get_item(project_name, folder_name, image_name).data - ) + image = self.controller.get_item(project_name, folder_name, image_name).data return BaseSerializer(image).serialize() - @validate_arguments def set_project_workflow( - self, project: Union[NotEmptyStr, dict], new_workflow: List[dict]): + self, project: Union[NotEmptyStr, dict], new_workflow: List[dict] + ): """Sets project's workflow. new_workflow example: [{ "step" : , "className" : , "tool" : , @@ -1465,16 +1389,15 @@ def set_project_workflow( if response.errors: raise AppException(response.errors) - @validate_arguments def download_image( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - local_dir_path: Optional[Union[str, Path]] = "./", - include_annotations: Optional[StrictBool] = False, - include_fuse: Optional[StrictBool] = False, - include_overlay: Optional[StrictBool] = False, - variant: Optional[str] = "original", + self, + project: Union[NotEmptyStr, dict], + image_name: NotEmptyStr, + local_dir_path: Optional[Union[str, Path]] = "./", + include_annotations: Optional[StrictBool] = False, + include_fuse: Optional[StrictBool] = False, + include_overlay: Optional[StrictBool] = False, + variant: Optional[str] = "original", ): """Downloads the image (and annotation if not None) to local_dir_path @@ -1513,13 +1436,12 @@ def download_image( logger.info(f"Downloaded image {image_name} to {local_dir_path} ") return response.data - @validate_arguments def upload_annotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, ): """Finds and uploads all JSON files in the folder_path as annotations to the project. @@ -1576,13 +1498,12 @@ def upload_annotations_from_folder_to_project( raise AppException(response.errors) return response.data - @validate_arguments def upload_preannotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, + self, + project: Union[NotEmptyStr, dict], + folder_path: Union[str, Path], + from_s3_bucket=None, + recursive_subfolders: Optional[StrictBool] = False, ): """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. @@ -1642,14 +1563,13 @@ def upload_preannotations_from_folder_to_project( raise AppException(response.errors) return response.data - @validate_arguments def upload_image_annotations( - self, - project: Union[NotEmptyStr, dict], - image_name: str, - annotation_json: Union[str, Path, dict], - mask: Optional[Union[str, Path, bytes]] = None, - verbose: Optional[StrictBool] = True, + self, + project: Union[NotEmptyStr, dict], + image_name: str, + annotation_json: Union[str, Path, dict], + mask: Optional[Union[str, Path, bytes]] = None, + verbose: Optional[StrictBool] = True, ): """Upload annotations from JSON (also mask for pixel annotations) to the image. @@ -1699,9 +1619,7 @@ def upload_image_annotations( if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: raise AppException(response.errors) - @validate_arguments - def download_model( - self, model: MLModel, output_dir: Union[str, Path]): + def download_model(self, model: MLModel, output_dir: Union[str, Path]): """Downloads the neural network and related files which are the .pth/pkl. .json, .yaml, classes_mapper.json @@ -1720,16 +1638,15 @@ def download_model( else: return BaseSerializer(res.data).serialize() - @validate_arguments def benchmark( - self, - project: Union[NotEmptyStr, dict], - gt_folder: str, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[str, Path]] = None, - image_list=None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots=False, + self, + project: Union[NotEmptyStr, dict], + gt_folder: str, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[str, Path]] = None, + image_list=None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots=False, ): """Computes benchmark score for each instance of given images that are present both gt_project_name project and projects in folder_names list: @@ -1788,15 +1705,14 @@ def benchmark( raise AppException(response.errors) return response.data - @validate_arguments def consensus( - self, - project: NotEmptyStr, - folder_names: List[NotEmptyStr], - export_root: Optional[Union[NotEmptyStr, Path]] = None, - image_list: Optional[List[NotEmptyStr]] = None, - annot_type: Optional[AnnotationType] = "bbox", - show_plots: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + folder_names: List[NotEmptyStr], + export_root: Optional[Union[NotEmptyStr, Path]] = None, + image_list: Optional[List[NotEmptyStr]] = None, + annot_type: Optional[AnnotationType] = "bbox", + show_plots: Optional[StrictBool] = False, ): """Computes consensus score for each instance of given images that are present in at least 2 of the given projects: @@ -1842,12 +1758,11 @@ def consensus( raise AppException(response.errors) return response.data - @validate_arguments def run_prediction( - self, - project: Union[NotEmptyStr, dict], - images_list: List[NotEmptyStr], - model: Union[NotEmptyStr, dict], + self, + project: Union[NotEmptyStr, dict], + images_list: List[NotEmptyStr], + model: Union[NotEmptyStr, dict], ): """This function runs smart prediction on given list of images from a given project using the neural network of your choice @@ -1881,15 +1796,14 @@ def run_prediction( raise AppException(response.errors) return response.data - @validate_arguments def add_annotation_bbox_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - bbox: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + bbox: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, ): """Add a bounding box annotation to image annotations @@ -1941,15 +1855,14 @@ def add_annotation_bbox_to_image( project_name, folder_name, image_name, annotations ) - @validate_arguments def add_annotation_point_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - point: List[float], - annotation_class_name: NotEmptyStr, - annotation_class_attributes: Optional[List[dict]] = None, - error: Optional[StrictBool] = None, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + point: List[float], + annotation_class_name: NotEmptyStr, + annotation_class_attributes: Optional[List[dict]] = None, + error: Optional[StrictBool] = None, ): """Add a point annotation to image annotations @@ -1999,15 +1912,14 @@ def add_annotation_point_to_image( project_name, folder_name, image_name, annotations ) - @validate_arguments def add_annotation_comment_to_image( - self, - project: NotEmptyStr, - image_name: NotEmptyStr, - comment_text: NotEmptyStr, - comment_coords: List[float], - comment_author: EmailStr, - resolved: Optional[StrictBool] = False, + self, + project: NotEmptyStr, + image_name: NotEmptyStr, + comment_text: NotEmptyStr, + comment_coords: List[float], + comment_author: EmailStr, + resolved: Optional[StrictBool] = False, ): """Add a comment to SuperAnnotate format annotation JSON @@ -2055,15 +1967,14 @@ def add_annotation_comment_to_image( project_name, folder_name, image_name, annotations ) - @validate_arguments def upload_image_to_project( - self, - project: NotEmptyStr, - img, - image_name: Optional[NotEmptyStr] = None, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[NotEmptyStr] = None, + self, + project: NotEmptyStr, + img, + image_name: Optional[NotEmptyStr] = None, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[NotEmptyStr] = None, ): """Uploads image (io.BytesIO() or filepath to image) to project. Sets status of the uploaded image to set_status if it is not None. @@ -2097,30 +2008,29 @@ def upload_image_to_project( if response.errors: raise AppException(response.errors) - @validate_arguments def search_models( - self, - name: Optional[NotEmptyStr] = None, - type_: Optional[NotEmptyStr] = None, - project_id: Optional[int] = None, - task: Optional[NotEmptyStr] = None, - include_global: Optional[StrictBool] = True, + self, + name: Optional[NotEmptyStr] = None, + type_: Optional[NotEmptyStr] = None, + project_id: Optional[int] = None, + task: Optional[NotEmptyStr] = None, + include_global: Optional[StrictBool] = True, ): """Search for ML models. - :param name: search string - :type name: str - :param type_: ml model type string - :type type_: str - :param project_id: project id - :type project_id: int - :param task: training task - :type task: str - :param include_global: include global ml models - :type include_global: bool - - :return: ml model metadata - :rtype: list of dicts + :param name: search string + :type name: str + :param type_: ml model type string + :type type_: str + :param project_id: project id + :type project_id: int + :param task: training task + :type task: str + :param include_global: include global ml models + :type include_global: bool + + :return: ml model metadata + :rtype: list of dicts """ res = self.controller.search_models( name=name, @@ -2131,14 +2041,13 @@ def search_models( ) return res.data - @validate_arguments def upload_images_to_project( - self, - project: NotEmptyStr, - img_paths: List[NotEmptyStr], - annotation_status: Optional[AnnotationStatuses] = "NotStarted", - from_s3_bucket=None, - image_quality_in_editor: Optional[ImageQualityChoices] = None, + self, + project: NotEmptyStr, + img_paths: List[NotEmptyStr], + annotation_status: Optional[AnnotationStatuses] = "NotStarted", + from_s3_bucket=None, + image_quality_in_editor: Optional[ImageQualityChoices] = None, ): """Uploads all images given in list of path objects in img_paths to the project. Sets status of all the uploaded images to set_status if it is not None. @@ -2176,14 +2085,17 @@ def upload_images_to_project( images_to_upload, duplicates = use_case.images_to_upload if len(duplicates): logger.warning( - "%s already existing images found that won't be uploaded.", len(duplicates) + "%s already existing images found that won't be uploaded.", + len(duplicates), ) logger.info(f"Uploading {len(images_to_upload)} images to project {project}.") uploaded, failed_images, duplications = [], [], duplicates if not images_to_upload: return uploaded, failed_images, duplications if use_case.is_valid(): - with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar: + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: for _ in use_case.execute(): progress_bar.update(1) uploaded, failed_images, duplications = use_case.data @@ -2192,12 +2104,11 @@ def upload_images_to_project( return uploaded, failed_images, duplications raise AppException(use_case.response.errors) - @validate_arguments def aggregate_annotations_as_df( - self, - project_root: Union[NotEmptyStr, Path], - project_type: ProjectTypes, - folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, + self, + project_root: Union[NotEmptyStr, Path], + project_type: ProjectTypes, + folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, ): """Aggregate annotations as pandas dataframe from project root. @@ -2215,8 +2126,8 @@ def aggregate_annotations_as_df( :rtype: pandas DataFrame """ if project_type in ( - constances.ProjectType.VECTOR.name, - constances.ProjectType.PIXEL.name, + constances.ProjectType.VECTOR.name, + constances.ProjectType.PIXEL.name, ): from superannotate.lib.app.analytics.common import ( aggregate_image_annotations_as_df, @@ -2230,8 +2141,8 @@ def aggregate_annotations_as_df( folder_names=folder_names, ) elif project_type in ( - constances.ProjectType.VIDEO.name, - constances.ProjectType.DOCUMENT.name, + constances.ProjectType.VIDEO.name, + constances.ProjectType.DOCUMENT.name, ): from superannotate.lib.app.analytics.aggregators import DataAggregator @@ -2241,10 +2152,8 @@ def aggregate_annotations_as_df( folder_names=folder_names, ).aggregate_annotations_as_df() - @validate_arguments def delete_annotations( - self, - project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None + self, project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None ): """ Delete image annotations from a given list of images. @@ -2263,22 +2172,20 @@ def delete_annotations( if response.errors: raise AppException(response.errors) - @validate_arguments def validate_annotations( - self, - project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] + self, project_type: ProjectTypes, annotations_json: Union[NotEmptyStr, Path] ): """Validates given annotation JSON. - :param project_type: The project type Vector, Pixel, Video or Document - :type project_type: str + :param project_type: The project type Vector, Pixel, Video or Document + :type project_type: str - :param annotations_json: path to annotation JSON - :type annotations_json: Path-like (str or Path) + :param annotations_json: path to annotation JSON + :type annotations_json: Path-like (str or Path) - :return: The success of the validation - :rtype: bool - """ + :return: The success of the validation + :rtype: bool + """ with open(annotations_json) as file: annotation_data = json.loads(file.read()) response = Controller.validate_annotations( @@ -2292,10 +2199,11 @@ def validate_annotations( print(response.report) return False - @validate_arguments def add_contributors_to_project( - self, - project: NotEmptyStr, emails: conlist(EmailStr, min_items=1), role: AnnotatorRole + self, + project: NotEmptyStr, + emails: conlist(EmailStr, min_items=1), + role: AnnotatorRole, ) -> Tuple[List[str], List[str]]: """Add contributors to project. @@ -2318,10 +2226,8 @@ def add_contributors_to_project( raise AppException(response.errors) return response.data - @validate_arguments def invite_contributors_to_team( - self, - emails: conlist(EmailStr, min_items=1), admin: StrictBool = False + self, emails: conlist(EmailStr, min_items=1), admin: StrictBool = False ) -> Tuple[List[str], List[str]]: """Invites contributors to the team. @@ -2341,9 +2247,9 @@ def invite_contributors_to_team( raise AppException(response.errors) return response.data - @validate_arguments def get_annotations( - self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): + self, project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None + ): """Returns annotations for the given list of items. :param project: project name or folder path (e.g., “project1/folder1”). @@ -2356,16 +2262,14 @@ def get_annotations( :rtype: list of strs """ project_name, folder_name = extract_project_folder(project) - response = self.controller.get_annotations( - project_name, folder_name, items - ) + response = self.controller.get_annotations(project_name, folder_name, items) if response.errors: raise AppException(response.errors) return response.data - @validate_arguments def get_annotations_per_frame( - self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1): + self, project: NotEmptyStr, video: NotEmptyStr, fps: int = 1 + ): """Returns per frame annotations for the given video. @@ -2390,9 +2294,7 @@ def get_annotations_per_frame( raise AppException(response.errors) return response.data - @validate_arguments - def upload_priority_scores( - self, project: NotEmptyStr, scores: List[PriorityScore]): + def upload_priority_scores(self, project: NotEmptyStr, scores: List[PriorityScore]): """Returns per frame annotations for the given video. :param project: project name or folder path (e.g., “project1/folder1”) @@ -2413,9 +2315,7 @@ def upload_priority_scores( raise AppException(response.errors) return response.data - @validate_arguments - def get_integrations( - self): + def get_integrations(self): """Get all integrations per team :return: metadata objects of all integrations of the team. @@ -2427,12 +2327,11 @@ def get_integrations( integrations = response.data return BaseSerializer.serialize_iterable(integrations, ("name", "type", "root")) - @validate_arguments def attach_items_from_integrated_storage( - self, - project: NotEmptyStr, - integration: Union[NotEmptyStr, IntegrationEntity], - folder_path: Optional[NotEmptyStr] = None, + self, + project: NotEmptyStr, + integration: Union[NotEmptyStr, IntegrationEntity], + folder_path: Optional[NotEmptyStr] = None, ): """Link images from integrated external storage to SuperAnnotate. @@ -2444,7 +2343,7 @@ def attach_items_from_integrated_storage( :type integration: str or dict :param folder_path: Points to an exact folder/directory within given storage. - If None, items are fetched from the root directory. + If None, items are fetched from the root directory. :type folder_path: str """ project_name, folder_name = extract_project_folder(project) @@ -2456,9 +2355,7 @@ def attach_items_from_integrated_storage( if response.errors: raise AppException(response.errors) - @validate_arguments - def query( - self, project: NotEmptyStr, query: Optional[NotEmptyStr]): + def query(self, project: NotEmptyStr, query: Optional[NotEmptyStr]): """Return items that satisfy the given query. Query syntax should be in SuperAnnotate query language(https://doc.superannotate.com/docs/query-search-1). @@ -2477,10 +2374,10 @@ def query( raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) - @validate_arguments def get_item_metadata( - self, - project: NotEmptyStr, item_name: NotEmptyStr, + self, + project: NotEmptyStr, + item_name: NotEmptyStr, ): """Returns item metadata @@ -2499,15 +2396,14 @@ def get_item_metadata( raise AppException(response.errors) return BaseSerializer(response.data).serialize() - @validate_arguments def search_items( - self, - project: NotEmptyStr, - name_contains: NotEmptyStr = None, - annotation_status: Optional[AnnotationStatuses] = None, - annotator_email: Optional[NotEmptyStr] = None, - qa_email: Optional[NotEmptyStr] = None, - recursive: bool = False, + self, + project: NotEmptyStr, + name_contains: NotEmptyStr = None, + annotation_status: Optional[AnnotationStatuses] = None, + annotator_email: Optional[NotEmptyStr] = None, + qa_email: Optional[NotEmptyStr] = None, + recursive: bool = False, ): """Search items by filtering criteria. @@ -2561,32 +2457,29 @@ def search_items( raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) - @validate_arguments def attach_items( - self, - project: Union[NotEmptyStr, dict], - attachments: AttachmentArg, - annotation_status: Optional[AnnotationStatuses] = "NotStarted", + self, + project: Union[NotEmptyStr, dict], + attachments: AttachmentArg, + annotation_status: Optional[AnnotationStatuses] = "NotStarted", ): """Link items from external storage to SuperAnnotate using URLs. - :param project: project name or folder path (e.g., “project1/folder1”) - :type project: str + :param project: project name or folder path (e.g., “project1/folder1”) + :type project: str - :param attachments: path to CSV file or list of dicts containing attachments URLs. - :type attachments: path-like (str or Path) or list of dicts + :param attachments: path to CSV file or list of dicts containing attachments URLs. + :type attachments: path-like (str or Path) or list of dicts - :param annotation_status: value to set the annotation statuses of the linked items + :param annotation_status: value to set the annotation statuses of the linked items “NotStarted” “InProgress” “QualityCheck” “Returned” “Completed” “Skipped” - :type annotation_status: str - - :return: None - """ + :type annotation_status: str + """ attachments = attachments.data project_name, folder_name = extract_project_folder(project) if attachments and isinstance(attachments[0], AttachmentDict): @@ -2597,9 +2490,10 @@ def attach_items( if count > 1 ] else: - unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv( - attachments - ) + ( + unique_attachments, + duplicate_attachments, + ) = get_name_url_duplicated_from_csv(attachments) if duplicate_attachments: logger.info("Dropping duplicates.") unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments) @@ -2625,13 +2519,12 @@ def attach_items( ] return uploaded, fails, duplicated - @validate_arguments def copy_items( - self, - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, - include_annotations: Optional[StrictBool] = True, + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, + include_annotations: Optional[StrictBool] = True, ): """Copy images in bulk between folders in a project @@ -2669,12 +2562,11 @@ def copy_items( return response.data - @validate_arguments def move_items( - self, - source: Union[NotEmptyStr, dict], - destination: Union[NotEmptyStr, dict], - items: Optional[List[NotEmptyStr]] = None, + self, + source: Union[NotEmptyStr, dict], + destination: Union[NotEmptyStr, dict], + items: Optional[List[NotEmptyStr]] = None, ): """Move images in bulk between folders in a project @@ -2705,12 +2597,11 @@ def move_items( raise AppException(response.errors) return response.data - @validate_arguments def set_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: AnnotationStatuses, - item_names: Optional[List[NotEmptyStr]] = None, + self, + project: Union[NotEmptyStr, dict], + annotation_status: AnnotationStatuses, + item_names: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of items @@ -2741,14 +2632,13 @@ def set_annotation_statuses( raise AppException(response.errors) return response.data - @validate_arguments def download_annotations( - self, - project: Union[NotEmptyStr, dict], - path: Union[str, Path] = None, - items: Optional[List[NotEmptyStr]] = None, - recursive: bool = False, - callback: Callable = None, + self, + project: Union[NotEmptyStr, dict], + path: Union[str, Path] = None, + items: Optional[List[NotEmptyStr]] = None, + recursive: bool = False, + callback: Callable = None, ): """Downloads annotation JSON files of the selected items to the local directory. diff --git a/src/superannotate/lib/app/mixp/decorators.py b/src/superannotate/lib/app/mixp/decorators.py deleted file mode 100644 index 5f2cc60ec..000000000 --- a/src/superannotate/lib/app/mixp/decorators.py +++ /dev/null @@ -1,134 +0,0 @@ -import functools -import sys -from inspect import signature - -from mixpanel import Mixpanel -from superannotate.logger import get_default_logger -from version import __version__ - -from .utils import parsers - -logger = get_default_logger() - - -def get_mp_instance() -> Mixpanel: - # if "api.annotate.online" in get_default_controller()._backend_url: - # return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") - return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") - - -def get_default(team_name, user_id, project_name=None): - return { - "SDK": True, - "Paid": True, - "Team": team_name, - "Team Owner": user_id, - "Project Name": project_name, - "Project Role": "Admin", - "Version": __version__, - } - - -class Trackable: - TEAM_DATA = None - INITIAL_EVENT = {"event_name": "SDK init", "properties": {}} - INITIAL_LOGGED = False - - def __init__(self, function, initial=False): - self.function = function - self._success = False - self._initial = initial - if initial: - self.track() - functools.update_wrapper(self, function) - - @staticmethod - def extract_arguments(function, *args, **kwargs) -> dict: - bound_arguments = signature(function).bind(*args, **kwargs) - bound_arguments.apply_defaults() - return dict(bound_arguments.arguments) - - @property - def team(self): - return get_default_controller().get_team() - - @staticmethod - def default_parser(function_name: str, kwargs: dict): - properties = {} - for key, value in kwargs: - if isinstance(value, (str, int, float, bool, str)): - properties[key] = value - elif isinstance(value, (list, set, tuple)): - properties[key] = len(value) - elif isinstance(value, dict): - properties[key] = value.keys() - elif hasattr(value, "__len__"): - properties[key] = len(value) - else: - properties[key] = str(value) - return {"event_name": function_name, "properties": properties} - - def track(self, *args, **kwargs): - try: - function_name = self.function.__name__ if self.function else "" - if self._initial: - data = self.INITIAL_EVENT - Trackable.INITIAL_LOGGED = True - self._success = True - else: - data = {} - arguments = self.extract_arguments(self.function, *args, **kwargs) - if hasattr(parsers, function_name): - try: - data = getattr(parsers, function_name)(**arguments) - except Exception: - pass - else: - data = self.default_parser(function_name, arguments) - event_name = data.get("event_name",) - properties = data.get("properties", {}) - team_data = self.team.data - user_id = team_data.creator_id - team_name = team_data.name - properties["Success"] = self._success - default = get_default( - team_name=team_name, - user_id=user_id, - project_name=properties.get("project_name", None), - ) - properties.pop("project_name", None) - properties = {**default, **properties} - - if "pytest" not in sys.modules: - get_mp_instance().track(user_id, event_name, properties) - except Exception: - pass - - def __call__(self, *args, **kwargs): - try: - controller = get_default_controller() - if controller: - self.__class__.TEAM_DATA = controller.get_team() - result = self.function(*args, **kwargs) - self._success = True - else: - raise Exception( - "SuperAnnotate config file not found." - " Please provide correct config file location to sa.init() or use " - "CLI's superannotate init to generate default location config file." - ) - except Exception as e: - self._success = False - logger.debug(str(e), exc_info=True) - raise e - else: - return result - finally: - try: - self.track(*args, **kwargs) - except Exception: - pass - - -if __name__ == "lib.app.mixp.decorators" and not Trackable.INITIAL_LOGGED: - Trackable(None, initial=True) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 1b1fed04c..c30d67469 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -1,3 +1,4 @@ +from os.path import expanduser from pathlib import Path from superannotate.lib.core.enums import AnnotationStatus @@ -12,8 +13,9 @@ from superannotate.lib.core.enums import UserRole -CONFIG_FILE_LOCATION = str(Path.home() / ".superannotate" / "config.json") -LOG_FILE_LOCATION = str(Path.home() / ".superannotate" / "sa.log") +CONFIG_PATH = "~/.superannotate/config.json" +CONFIG_FILE_LOCATION = expanduser(CONFIG_PATH) +LOG_FILE_LOCATION = expanduser("~/.superannotate/sa.log") BACKEND_URL = "https://api.annotate.online" DEFAULT_IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "tif", "tiff", "webp", "bmp"] @@ -141,6 +143,7 @@ ImageQuality, AnnotationStatus, CONFIG_FILE_LOCATION, + CONFIG_PATH, BACKEND_URL, DEFAULT_IMAGE_EXTENSIONS, DEFAULT_FILE_EXCLUDE_PATTERNS, diff --git a/src/superannotate/lib/core/data_handlers.py b/src/superannotate/lib/core/data_handlers.py index c0a1c442c..1eb77e684 100644 --- a/src/superannotate/lib/core/data_handlers.py +++ b/src/superannotate/lib/core/data_handlers.py @@ -266,7 +266,7 @@ def safe_time(timestamp): return "0" if str(timestamp) == "0.0" else timestamp def convert_timestamp(timestamp): - return timestamp / 10 ** 6 if timestamp else "0" + return timestamp / 10**6 if timestamp else "0" editor_data = { "instances": [], diff --git a/src/superannotate/lib/core/entities/project_entities.py b/src/superannotate/lib/core/entities/project_entities.py index 4e50080f3..244bca0c9 100644 --- a/src/superannotate/lib/core/entities/project_entities.py +++ b/src/superannotate/lib/core/entities/project_entities.py @@ -40,7 +40,10 @@ def to_dict(self): class BaseTimedEntity(BaseEntity): def __init__( - self, uuid: Any = None, createdAt: str = None, updatedAt: str = None, + self, + uuid: Any = None, + createdAt: str = None, + updatedAt: str = None, ): super().__init__(uuid) self.createdAt = createdAt @@ -222,7 +225,10 @@ def to_dict(self): class ImageInfoEntity(BaseEntity): def __init__( - self, uuid=None, width: float = None, height: float = None, + self, + uuid=None, + width: float = None, + height: float = None, ): super().__init__(uuid), self.width = width diff --git a/src/superannotate/lib/core/plugin.py b/src/superannotate/lib/core/plugin.py index 06f77115e..9ef6fec30 100644 --- a/src/superannotate/lib/core/plugin.py +++ b/src/superannotate/lib/core/plugin.py @@ -254,7 +254,10 @@ def frames_generator( @staticmethod def get_extractable_frames( - video_path: str, start_time, end_time, target_fps: float, + video_path: str, + start_time, + end_time, + target_fps: float, ): total = VideoPlugin.get_frames_count(video_path) total_with_fps = sum( diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index db8c24395..f93781170 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -2,7 +2,6 @@ import sys import threading import time -import uuid from collections import defaultdict from typing import Union @@ -33,9 +32,8 @@ def init_spin(self): class Session: - def __init__(self, pk: int): - self.pk = pk - self._uuid = str(uuid.uuid4()) + def __init__(self): + self.pk = threading.get_ident() self._data_dict = {} def __enter__(self): @@ -45,10 +43,26 @@ def __exit__(self, type, value, traceback): if type is not None: return False + def __del__(self): + globs = globals() + if "SESSIONS" in globs and globs["SESSIONS"].get(self.pk): + del globs["SESSIONS"][self.pk] + @property def data(self): return self._data_dict + @staticmethod + def get_current_session(): + globs = globals() + if not globs.get("SESSIONS") or not globs["SESSIONS"].get( + threading.get_ident() + ): + session = Session() + globals().update({"SESSIONS": {session.pk: session}}) + return session + return globs["SESSIONS"][threading.get_ident()] + def __setitem__(self, key, item): self._data_dict[key] = item @@ -58,9 +72,6 @@ def __getitem__(self, key): def __repr__(self): return repr(self._data_dict) - def __delitem__(self, key): - del self._data_dict[key] - def clear(self): return self._data_dict.clear() @@ -72,7 +83,7 @@ def __init__( log_warning: bool = True, disable_progress_bar: bool = False, log_debug: bool = True, - session: Session = None + session: Session = None, ): self.logger = get_default_logger() self._log_info = log_info diff --git a/src/superannotate/lib/core/repositories.py b/src/superannotate/lib/core/repositories.py index a94258a5c..f60c7b44e 100644 --- a/src/superannotate/lib/core/repositories.py +++ b/src/superannotate/lib/core/repositories.py @@ -67,7 +67,11 @@ def __init__(self, service: SuperannotateServiceProvider, project: ProjectEntity class BaseS3Repository(BaseManageableRepository): def __init__( - self, access_key: str, secret_key: str, session_token: str, bucket: str, + self, + access_key: str, + secret_key: str, + session_token: str, + bucket: str, ): self._session = boto3.Session( aws_access_key_id=access_key, diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index ea120e4ea..1861e8ea1 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -106,7 +106,11 @@ def get_download_token( raise NotImplementedError def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, ) -> dict: raise NotImplementedError @@ -177,7 +181,10 @@ def get_bulk_images( raise NotImplementedError def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, + team_id: int, + project_id: int, + folder_name: str, ): raise NotImplementedError @@ -187,12 +194,19 @@ def assign_folder( raise NotImplementedError def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: list, + self, + team_id: int, + project_id: int, + folder_name: str, + image_names: list, ): raise NotImplementedError def un_share_project( - self, team_id: int, project_id: int, user_id: str, + self, + team_id: int, + project_id: int, + user_id: str, ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 8b1666bba..c553db8a5 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -33,7 +33,7 @@ from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.types import PriorityScore -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.images import GetBulkImages from lib.core.usecases.images import ValidateAnnotationUseCase from lib.core.video_convertor import VideoFrameGenerator @@ -46,7 +46,7 @@ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -class UploadAnnotationsUseCase(BaseReportableUseCae): +class UploadAnnotationsUseCase(BaseReportableUseCase): MAX_WORKERS = 10 CHUNK_SIZE = 100 AUTH_DATA_CHUNK_SIZE = 500 @@ -312,7 +312,7 @@ def execute(self): return self._response -class UploadAnnotationUseCase(BaseReportableUseCae): +class UploadAnnotationUseCase(BaseReportableUseCase): def __init__( self, project: ProjectEntity, @@ -444,7 +444,10 @@ def prepare_annotations( handlers_chain.attach(LastActionHandler(team.creator_id)) return handlers_chain.handle(annotations) - def clean_json(self, json_data: dict,) -> Tuple[bool, dict]: + def clean_json( + self, + json_data: dict, + ) -> Tuple[bool, dict]: use_case = ValidateAnnotationUseCase( constances.ProjectType.get_name(self._project.type), annotation=json_data, @@ -500,7 +503,7 @@ def execute(self): return self._response -class GetAnnotations(BaseReportableUseCae): +class GetAnnotations(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -584,7 +587,7 @@ def execute(self): return self._response -class GetVideoAnnotationsPerFrame(BaseReportableUseCae): +class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -645,7 +648,7 @@ def execute(self): return self._response -class UploadPriorityScoresUseCase(BaseReportableUseCae): +class UploadPriorityScoresUseCase(BaseReportableUseCase): CHUNK_SIZE = 100 def __init__( @@ -733,7 +736,7 @@ def execute(self): return self._response -class DownloadAnnotations(BaseReportableUseCae): +class DownloadAnnotations(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -793,8 +796,9 @@ def get_postfix(self): def download_annotation_classes(self, path: str): classes = self._classes.get_all() - os.mkdir(f"{path}/classes") - with open(f"{path}/classes/classes.json", "w+") as file: + classes_path = Path(path) / "classes" + classes_path.mkdir(parents=True, exist_ok=True) + with open(classes_path / "classes.json", "w+") as file: json.dump([i.dict() for i in classes], file, indent=4) @staticmethod @@ -810,13 +814,12 @@ def coroutine_wrapper(coroutine): def execute(self): if self.is_valid(): - export_prefix = f"{self._project.name}{f'/{self._folder.name}' if not self._folder.is_root else ''}" export_path = str( self.destination - / Path(f"{export_prefix} {datetime.now().strftime('%B %d %Y %H_%M')}") + / Path(f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}") ) self.reporter.log_info( - f"Downloading the annotations of the requested items to {export_path} \nThis might take a while…" + f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" ) self.reporter.start_spinner() folders = [] @@ -869,7 +872,7 @@ def execute(self): self.reporter.stop_spinner() self.reporter.log_info( - f"SA-PYTHON-SDK - INFO - Downloaded annotations for {self.get_items_count(export_path)} items." + f"Downloaded annotations for {self.get_items_count(export_path)} items." ) self.download_annotation_classes(export_path) self._response.data = os.path.abspath(export_path) diff --git a/src/superannotate/lib/core/usecases/base.py b/src/superannotate/lib/core/usecases/base.py index 3f3e3750f..97ed22ee8 100644 --- a/src/superannotate/lib/core/usecases/base.py +++ b/src/superannotate/lib/core/usecases/base.py @@ -58,13 +58,13 @@ def execute(self) -> Iterable: raise NotImplementedError -class BaseReportableUseCae(BaseUseCase, metaclass=ABCMeta): +class BaseReportableUseCase(BaseUseCase, metaclass=ABCMeta): def __init__(self, reporter: Reporter): super().__init__() self.reporter = reporter -class BaseUserBasedUseCase(BaseReportableUseCae, metaclass=ABCMeta): +class BaseUserBasedUseCase(BaseReportableUseCase, metaclass=ABCMeta): """ class contain validation of unique emails """ diff --git a/src/superannotate/lib/core/usecases/folders.py b/src/superannotate/lib/core/usecases/folders.py index a9733652f..b98c79e88 100644 --- a/src/superannotate/lib/core/usecases/folders.py +++ b/src/superannotate/lib/core/usecases/folders.py @@ -142,7 +142,9 @@ def execute(self): class UpdateFolderUseCase(BaseUseCase): def __init__( - self, folders: BaseManageableRepository, folder: FolderEntity, + self, + folders: BaseManageableRepository, + folder: FolderEntity, ): super().__init__() self._folders = folders diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 31b4eeba0..41611999e 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -42,7 +42,7 @@ from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseInteractiveUseCase -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.projects import GetAnnotationClassesUseCase from PIL import UnidentifiedImageError @@ -581,7 +581,11 @@ def execute(self): image = ImagePlugin(io.BytesIO(file.read())) images = [ - Image("fuse", f"{self._image_path}___fuse.png", image.get_empty(),) + Image( + "fuse", + f"{self._image_path}___fuse.png", + image.get_empty(), + ) ] if self._generate_overlay: images.append( @@ -711,7 +715,9 @@ def execute(self): class GetS3ImageUseCase(BaseUseCase): def __init__( - self, s3_bucket, image_path: str, + self, + s3_bucket, + image_path: str, ): super().__init__() self._s3_bucket = s3_bucket @@ -1536,7 +1542,8 @@ def execute(self) -> Response: image_bytes = ( GetImageBytesUseCase( - image=image, backend_service_provider=self._backend_service, + image=image, + backend_service_provider=self._backend_service, ) .execute() .data @@ -1876,7 +1883,7 @@ def execute(self): return self._response -class GetImageAnnotationsUseCase(BaseReportableUseCae): +class GetImageAnnotationsUseCase(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -2438,7 +2445,7 @@ def execute(self) -> Response: return self._response -class UploadVideosAsImages(BaseReportableUseCae): +class UploadVideosAsImages(BaseReportableUseCase): def __init__( self, reporter: Reporter, diff --git a/src/superannotate/lib/core/usecases/integrations.py b/src/superannotate/lib/core/usecases/integrations.py index c61379de0..4f0c42d1e 100644 --- a/src/superannotate/lib/core/usecases/integrations.py +++ b/src/superannotate/lib/core/usecases/integrations.py @@ -9,10 +9,10 @@ from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases import BaseReportableUseCae +from lib.core.usecases import BaseReportableUseCase -class GetIntegrations(BaseReportableUseCae): +class GetIntegrations(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -32,7 +32,7 @@ def execute(self) -> Response: return self._response -class AttachIntegrations(BaseReportableUseCae): +class AttachIntegrations(BaseReportableUseCase): def __init__( self, reporter: Reporter, diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 0bb2fa294..4110cd340 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -18,10 +18,10 @@ from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase -class GetItem(BaseReportableUseCae): +class GetItem(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -74,7 +74,7 @@ def execute(self) -> Response: return self._response -class QueryEntities(BaseReportableUseCae): +class QueryEntities(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -125,7 +125,7 @@ def execute(self) -> Response: return self._response -class ListItems(BaseReportableUseCae): +class ListItems(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -187,7 +187,7 @@ def execute(self) -> Response: return self._response -class AttachItems(BaseReportableUseCae): +class AttachItems(BaseReportableUseCase): CHUNK_SIZE = 500 def __init__( @@ -288,7 +288,7 @@ def execute(self) -> Response: return self._response -class CopyItems(BaseReportableUseCae): +class CopyItems(BaseReportableUseCase): """ Copy items in bulk between folders in a project. Return skipped item names. @@ -362,13 +362,15 @@ def execute(self): if items_to_copy: for i in range(0, len(items_to_copy), self.CHUNK_SIZE): chunk_to_copy = items_to_copy[i : i + self.CHUNK_SIZE] # noqa: E203 - poll_id = self._backend_service.copy_items_between_folders_transaction( - team_id=self._project.team_id, - project_id=self._project.id, - from_folder_id=self._from_folder.uuid, - to_folder_id=self._to_folder.uuid, - items=chunk_to_copy, - include_annotations=self._include_annotations, + poll_id = ( + self._backend_service.copy_items_between_folders_transaction( + team_id=self._project.team_id, + project_id=self._project.id, + from_folder_id=self._from_folder.uuid, + to_folder_id=self._to_folder.uuid, + items=chunk_to_copy, + include_annotations=self._include_annotations, + ) ) if not poll_id: skipped_items.extend(chunk_to_copy) @@ -404,7 +406,7 @@ def execute(self): return self._response -class MoveItems(BaseReportableUseCae): +class MoveItems(BaseReportableUseCase): CHUNK_SIZE = 1000 def __init__( @@ -479,7 +481,7 @@ def execute(self): return self._response -class SetAnnotationStatues(BaseReportableUseCae): +class SetAnnotationStatues(BaseReportableUseCase): CHUNK_SIZE = 500 ERROR_MESSAGE = "Failed to change status" diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 402fbdf0d..3d6ec6ff9 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -26,7 +26,7 @@ from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.images import GetBulkImages from superannotate.logger import get_default_logger @@ -179,7 +179,7 @@ def execute(self): return self._response -class DownloadExportUseCase(BaseReportableUseCae): +class DownloadExportUseCase(BaseReportableUseCase): def __init__( self, service: SuperannotateServiceProvider, @@ -635,7 +635,9 @@ def execute(self): class SearchMLModels(BaseUseCase): def __init__( - self, ml_models_repo: BaseManageableRepository, condition: Condition, + self, + ml_models_repo: BaseManageableRepository, + condition: Condition, ): super().__init__() self._ml_models = ml_models_repo diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index fb97e87de..8461619f9 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -20,7 +20,7 @@ from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository from lib.core.serviceproviders import SuperannotateServiceProvider -from lib.core.usecases.base import BaseReportableUseCae +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from lib.core.usecases.base import BaseUserBasedUseCase from requests.exceptions import RequestException @@ -31,7 +31,10 @@ class GetProjectsUseCase(BaseUseCase): def __init__( - self, condition: Condition, team_id: int, projects: BaseManageableRepository, + self, + condition: Condition, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() self._condition = condition @@ -47,7 +50,10 @@ def execute(self): class GetProjectByNameUseCase(BaseUseCase): def __init__( - self, name: str, team_id: int, projects: BaseManageableRepository, + self, + name: str, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() self._name = name @@ -300,7 +306,10 @@ def execute(self): class DeleteProjectUseCase(BaseUseCase): def __init__( - self, project_name: str, team_id: int, projects: BaseManageableRepository, + self, + project_name: str, + team_id: int, + projects: BaseManageableRepository, ): super().__init__() @@ -310,7 +319,9 @@ def __init__( def execute(self): use_case = GetProjectByNameUseCase( - name=self._project_name, team_id=self._team_id, projects=self._projects, + name=self._project_name, + team_id=self._team_id, + projects=self._projects, ) project_response = use_case.execute() if project_response.data: @@ -373,7 +384,7 @@ def execute(self): return self._response -class CloneProjectUseCase(BaseReportableUseCae): +class CloneProjectUseCase(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -696,7 +707,9 @@ def execute(self): class GetAnnotationClassesUseCase(BaseUseCase): def __init__( - self, classes: BaseManageableRepository, condition: Condition = None, + self, + classes: BaseManageableRepository, + condition: Condition = None, ): super().__init__() self._classes = classes diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f54652fed..f5639ec4e 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1,7 +1,6 @@ import copy import io import os -import threading from abc import ABCMeta from pathlib import Path from typing import Callable @@ -25,7 +24,6 @@ from lib.core.entities.integrations import IntegrationEntity from lib.core.exceptions import AppException from lib.core.reporter import Reporter -from lib.core.reporter import Session from lib.core.response import Response from lib.infrastructure.helpers import timed_lru_cache from lib.infrastructure.repositories import AnnotationClassRepository @@ -80,14 +78,6 @@ def __init__(self, token: str, host: str, ssl_verify: bool, version: str): self._team_name = None self._reporter = None - def get_session(self): - pk = threading.get_ident() - try: - return self.SESSIONS[pk] - except KeyError: - self.SESSIONS[threading.get_ident()] = Session(pk) - return self.SESSIONS[pk] - @staticmethod def validate_token(token: str): try: @@ -140,7 +130,7 @@ def teams(self): @property def team_data(self): if not self._team_data: - self._team_data = self.get_team() + self._team_data = self.get_team().data return self._team_data @property @@ -168,19 +158,20 @@ def team_id(self) -> int: @staticmethod def get_default_reporter( - log_info: bool = True, - log_warning: bool = True, - disable_progress_bar: bool = False, - log_debug: bool = True + log_info: bool = True, + log_warning: bool = True, + disable_progress_bar: bool = False, + log_debug: bool = True, ) -> Reporter: import inspect + session = None loop_limit = 16 current_frame = inspect.currentframe() while loop_limit: loop_limit -= 1 try: - session = current_frame.f_locals['session'] + session = current_frame.f_locals["session"] if session: break except KeyError: @@ -255,11 +246,11 @@ def get_folder_name(name: str = None): return "root" def search_project( - self, - name: str = None, - include_complete_image_count=False, - statuses: Union[List[str], Tuple[str]] = (), - **kwargs, + self, + name: str = None, + include_complete_image_count=False, + statuses: Union[List[str], Tuple[str]] = (), + **kwargs, ) -> Response: condition = Condition.get_empty_condition() if name: @@ -275,19 +266,21 @@ def search_project( condition &= build_condition(**kwargs) use_case = usecases.GetProjectsUseCase( - condition=condition, projects=self.projects, team_id=self.team_id, + condition=condition, + projects=self.projects, + team_id=self.team_id, ) return use_case.execute() def create_project( - self, - name: str, - description: str, - project_type: str, - settings: Iterable[SettingEntity] = None, - classes: Iterable = tuple(), - workflows: Iterable = tuple(), - **extra_kwargs, + self, + name: str, + description: str, + project_type: str, + settings: Iterable[SettingEntity] = None, + classes: Iterable = tuple(), + workflows: Iterable = tuple(), + **extra_kwargs, ) -> Response: try: @@ -320,7 +313,9 @@ def create_project( def delete_project(self, name: str): use_case = usecases.DeleteProjectUseCase( - project_name=name, team_id=self.team_id, projects=self.projects, + project_name=name, + team_id=self.team_id, + projects=self.projects, ) return use_case.execute() @@ -330,14 +325,14 @@ def update_project(self, name: str, project_data: dict) -> Response: return use_case.execute() def upload_image_to_project( - self, - project_name: str, - folder_name: str, - image_name: str, - image: Union[str, io.BytesIO] = None, - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + image_name: str, + image: Union[str, io.BytesIO] = None, + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -365,13 +360,13 @@ def upload_image_to_project( ).execute() def upload_images_to_project( - self, - project_name: str, - folder_name: str, - paths: List[str], - annotation_status: str = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + paths: List[str], + annotation_status: str = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -391,16 +386,16 @@ def upload_images_to_project( ) def upload_images_from_folder_to_project( - self, - project_name: str, - folder_name: str, - folder_path: str, - extensions: Optional[List[str]] = None, - annotation_status: str = None, - exclude_file_patterns: Optional[List[str]] = None, - recursive_sub_folders: Optional[bool] = None, - image_quality_in_editor: str = None, - from_s3_bucket=None, + self, + project_name: str, + folder_name: str, + folder_path: str, + extensions: Optional[List[str]] = None, + annotation_status: str = None, + exclude_file_patterns: Optional[List[str]] = None, + recursive_sub_folders: Optional[bool] = None, + image_quality_in_editor: str = None, + from_s3_bucket=None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -423,20 +418,22 @@ def upload_images_from_folder_to_project( ) def clone_project( - self, - name: str, - from_name: str, - project_description: str, - copy_annotation_classes=True, - copy_settings=True, - copy_workflow=True, - copy_contributors=False, + self, + name: str, + from_name: str, + project_description: str, + copy_annotation_classes=True, + copy_settings=True, + copy_workflow=True, + copy_contributors=False, ): project = self._get_project(from_name) project_to_create = copy.copy(project) reporter = self.get_default_reporter() - reporter.track("external", project.upload_state == constances.UploadState.EXTERNAL.value) + reporter.track( + "external", project.upload_state == constances.UploadState.EXTERNAL.value + ) project_to_create.name = name if project_description is not None: project_to_create.description = project_description @@ -457,12 +454,12 @@ def clone_project( return use_case.execute() def interactive_attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, + self, + project_name: str, + files: List[ImageEntity], + folder_name: str = None, + annotation_status: str = None, + upload_state_code: int = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -482,7 +479,9 @@ def create_folder(self, project: str, folder_name: str): name=folder_name, project_id=project.id, team_id=project.team_id ) use_case = usecases.CreateFolderUseCase( - project=project, folder=folder, folders=self.folders, + project=project, + folder=folder, + folders=self.folders, ) return use_case.execute() @@ -497,7 +496,7 @@ def get_folder(self, project_name: str, folder_name: str): return use_case.execute() def search_folders( - self, project_name: str, folder_name: str = None, include_users=False, **kwargs + self, project_name: str, folder_name: str = None, include_users=False, **kwargs ): condition = build_condition(**kwargs) project = self._get_project(project_name) @@ -524,12 +523,12 @@ def delete_folders(self, project_name: str, folder_names: List[str]): return use_case.execute() def prepare_export( - self, - project_name: str, - folder_names: List[str], - include_fuse: bool, - only_pinned: bool, - annotation_statuses: List[str] = None, + self, + project_name: str, + folder_names: List[str], + include_fuse: bool, + only_pinned: bool, + annotation_statuses: List[str] = None, ): project = self._get_project(project_name) @@ -554,11 +553,11 @@ def search_team_contributors(self, **kwargs): return use_case.execute() def search_images( - self, - project_name: str, - folder_path: str = None, - annotation_status: str = None, - image_name_prefix: str = None, + self, + project_name: str, + folder_path: str = None, + annotation_status: str = None, + image_name_prefix: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -573,7 +572,10 @@ def search_images( return use_case.execute() def _get_image( - self, project: ProjectEntity, image_name: str, folder: FolderEntity = None, + self, + project: ProjectEntity, + image_name: str, + folder: FolderEntity = None, ) -> ImageEntity: response = usecases.GetImageUseCase( service=self._backend_client, @@ -587,7 +589,7 @@ def _get_image( return response.data def get_image( - self, project_name: str, image_name: str, folder_path: str = None + self, project_name: str, image_name: str, folder_path: str = None ) -> ImageEntity: project = self._get_project(project_name) folder = self._get_folder(project, folder_path) @@ -598,18 +600,21 @@ def update_folder(self, project_name: str, folder_name: str, folder_data: dict): folder = self._get_folder(project, folder_name) for field, value in folder_data.items(): setattr(folder, field, value) - use_case = usecases.UpdateFolderUseCase(folders=self.folders, folder=folder, ) + use_case = usecases.UpdateFolderUseCase( + folders=self.folders, + folder=folder, + ) return use_case.execute() def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, + copy_annotation_status: bool = False, + move: bool = False, ): from_project = self._get_project(from_project_name) to_project = self._get_project(to_project_name) @@ -632,12 +637,12 @@ def copy_image( return use_case.execute() def copy_image_annotation_classes( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, + self, + from_project_name: str, + from_folder_name: str, + to_project_name: str, + to_folder_name: str, + image_name: str, ): from_project = self._get_project(from_project_name) from_folder = self._get_folder(from_project, from_folder_name) @@ -672,7 +677,7 @@ def copy_image_annotation_classes( return use_case.execute() def update_image( - self, project_name: str, image_name: str, folder_name: str = None, **kwargs + self, project_name: str, image_name: str, folder_name: str = None, **kwargs ): image = self.get_image( project_name=project_name, image_name=image_name, folder_path=folder_name @@ -683,13 +688,13 @@ def update_image( return use_case.execute() def bulk_copy_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], - include_annotations: bool, - include_pin: bool, + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], + include_annotations: bool, + include_pin: bool, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -706,11 +711,11 @@ def bulk_copy_images( return use_case.execute() def bulk_move_images( - self, - project_name: str, - from_folder_name: str, - to_folder_name: str, - image_names: List[str], + self, + project_name: str, + from_folder_name: str, + to_folder_name: str, + image_names: List[str], ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder_name) @@ -725,13 +730,13 @@ def bulk_move_images( return use_case.execute() def get_project_metadata( - self, - project_name: str, - include_annotation_classes: bool = False, - include_settings: bool = False, - include_workflow: bool = False, - include_contributors: bool = False, - include_complete_image_count: bool = False, + self, + project_name: str, + include_annotation_classes: bool = False, + include_settings: bool = False, + include_workflow: bool = False, + include_contributors: bool = False, + include_complete_image_count: bool = False, ): project = self._get_project(project_name) @@ -806,11 +811,11 @@ def set_project_settings(self, project_name: str, new_settings: List[dict]): return use_case.execute() def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, + self, + project_name: str, + folder_name: str, + image_names: list, + annotation_status: str, ): project_entity = self._get_project(project_name) folder_entity = self._get_folder(project_entity, folder_name) @@ -828,7 +833,10 @@ def set_images_annotation_statuses( return use_case.execute() def delete_images( - self, project_name: str, folder_name: str, image_names: List[str] = None, + self, + project_name: str, + folder_name: str, + image_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -843,7 +851,7 @@ def delete_images( return use_case.execute() def assign_images( - self, project_name: str, folder_name: str, image_names: list, user: str + self, project_name: str, folder_name: str, image_names: list, user: str ): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) @@ -871,7 +879,9 @@ def un_assign_folder(self, project_name: str, folder_name: str): project_entity = self._get_project(project_name) folder = self._get_folder(project_entity, folder_name) use_case = usecases.UnAssignFolderUseCase( - service=self._backend_client, project_entity=project_entity, folder=folder, + service=self._backend_client, + project_entity=project_entity, + folder=folder, ) return use_case.execute() @@ -896,7 +906,7 @@ def un_share_project(self, project_name: str, user_id: str): return use_case.execute() def download_image_annotations( - self, project_name: str, folder_name: str, image_name: str, destination: str + self, project_name: str, folder_name: str, image_name: str, destination: str ): project = self._get_project(project_name) folder = self._get_folder(project=project, name=folder_name) @@ -924,7 +934,7 @@ def get_exports(self, project_name: str, return_metadata: bool): return use_case.execute() def get_project_image_count( - self, project_name: str, folder_name: str, with_all_subfolders: bool + self, project_name: str, folder_name: str, with_all_subfolders: bool ): project = self._get_project(project_name) @@ -940,12 +950,12 @@ def get_project_image_count( return use_case.execute() def create_annotation_class( - self, - project_name: str, - name: str, - color: str, - attribute_groups: List[dict], - class_type: str, + self, + project_name: str, + name: str, + color: str, + attribute_groups: List[dict], + class_type: str, ): project = self._get_project(project_name) annotation_classes = AnnotationClassRepository( @@ -966,7 +976,8 @@ def delete_annotation_class(self, project_name: str, annotation_class_name: str) use_case = usecases.DeleteAnnotationClassUseCase( annotation_class_name=annotation_class_name, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), project_name=project_name, ) @@ -977,7 +988,8 @@ def get_annotation_class(self, project_name: str, annotation_class_name: str): use_case = usecases.GetAnnotationClassUseCase( annotation_class_name=annotation_class_name, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), ) return use_case.execute() @@ -986,7 +998,8 @@ def download_annotation_classes(self, project_name: str, download_path: str): project = self._get_project(project_name) use_case = usecases.DownloadAnnotationClassesUseCase( annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), download_path=download_path, project_name=project_name, @@ -999,7 +1012,8 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) use_case = usecases.CreateAnnotationClassesUseCase( service=self._backend_client, annotation_classes_repo=AnnotationClassRepository( - service=self._backend_client, project=project, + service=self._backend_client, + project=project, ), annotation_classes=annotation_classes, project=project, @@ -1007,15 +1021,15 @@ def create_annotation_classes(self, project_name: str, annotation_classes: list) return use_case.execute() def download_image( - self, - project_name: str, - image_name: str, - download_path: str, - folder_name: str = None, - image_variant: str = None, - include_annotations: bool = None, - include_fuse: bool = None, - include_overlay: bool = None, + self, + project_name: str, + image_name: str, + download_path: str, + folder_name: str = None, + image_variant: str = None, + include_annotations: bool = None, + include_fuse: bool = None, + include_overlay: bool = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1055,13 +1069,13 @@ def set_project_workflow(self, project_name: str, steps: list): return use_case.execute() def upload_annotations_from_folder( - self, - project_name: str, - folder_name: str, - annotation_paths: List[str], - client_s3_bucket=None, - is_pre_annotations: bool = False, - folder_path: str = None, + self, + project_name: str, + folder_name: str, + annotation_paths: List[str], + client_s3_bucket=None, + is_pre_annotations: bool = False, + folder_path: str = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1069,7 +1083,7 @@ def upload_annotations_from_folder( project=project, folder=folder, images=self.images, - team=self.team_data.data, + team=self.team_data, annotation_paths=annotation_paths, backend_service_provider=self._backend_client, annotation_classes=AnnotationClassRepository( @@ -1087,13 +1101,13 @@ def upload_annotations_from_folder( return use_case.execute() def upload_image_annotations( - self, - project_name: str, - folder_name: str, - image_name: str, - annotations: dict, - mask: io.BytesIO = None, - verbose: bool = True, + self, + project_name: str, + folder_name: str, + image_name: str, + annotations: dict, + mask: io.BytesIO = None, + verbose: bool = True, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1105,7 +1119,7 @@ def upload_image_annotations( project=project, folder=folder, images=self.images, - team=self.team_data.data, + team=self.team_data, annotation_classes=AnnotationClassRepository( service=self._backend_client, project=project ).get_all(), @@ -1135,12 +1149,12 @@ def delete_model(self, model_id: int): return use_case.execute() def download_export( - self, - project_name: str, - export_name: str, - folder_path: str, - extract_zip_contents: bool, - to_s3_bucket: bool, + self, + project_name: str, + export_name: str, + folder_path: str, + extract_zip_contents: bool, + to_s3_bucket: bool, ): project = self._get_project(project_name) use_case = usecases.DownloadExportUseCase( @@ -1173,14 +1187,14 @@ def download_ml_model(self, model_data: dict, download_path: str): return use_case.execute() def benchmark( - self, - project_name: str, - ground_truth_folder_name: str, - folder_names: List[str], - export_root: str, - image_list: List[str], - annot_type: str, - show_plots: bool, + self, + project_name: str, + ground_truth_folder_name: str, + folder_names: List[str], + export_root: str, + image_list: List[str], + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) export_response = self.prepare_export( @@ -1215,13 +1229,13 @@ def benchmark( return use_case.execute() def consensus( - self, - project_name: str, - folder_names: list, - export_path: str, - image_list: list, - annot_type: str, - show_plots: bool, + self, + project_name: str, + folder_names: list, + export_path: str, + image_list: list, + annot_type: str, + show_plots: bool, ): project = self._get_project(project_name) @@ -1254,7 +1268,7 @@ def consensus( return use_case.execute() def run_prediction( - self, project_name: str, images_list: list, model_name: str, folder_name: str + self, project_name: str, images_list: list, model_name: str, folder_name: str ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1272,7 +1286,10 @@ def run_prediction( return use_case.execute() def list_images( - self, project_name: str, annotation_status: str = None, name_prefix: str = None, + self, + project_name: str, + annotation_status: str = None, + name_prefix: str = None, ): project = self._get_project(project_name) @@ -1285,12 +1302,12 @@ def list_images( return use_case.execute() def search_models( - self, - name: str, - model_type: str = None, - project_id: int = None, - task: str = None, - include_global: bool = True, + self, + name: str, + model_type: str = None, + project_id: int = None, + task: str = None, + include_global: bool = True, ): ml_models_repo = MLModelRepository( service=self._backend_client, team_id=self.team_id @@ -1313,10 +1330,10 @@ def search_models( return use_case.execute() def delete_annotations( - self, - project_name: str, - folder_name: str, - item_names: Optional[List[str]] = None, + self, + project_name: str, + folder_name: str, + item_names: Optional[List[str]] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1330,7 +1347,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = False ): use_case = usecases.ValidateAnnotationUseCase( project_type, @@ -1367,17 +1384,17 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): return use_case.execute() def upload_videos( - self, - project_name: str, - folder_name: str, - paths: List[str], - start_time: float, - extensions: List[str] = None, - exclude_file_patterns: List[str] = None, - end_time: Optional[float] = None, - target_fps: Optional[int] = None, - annotation_status: Optional[str] = None, - image_quality_in_editor: Optional[str] = None, + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1403,7 +1420,7 @@ def upload_videos( return use_case.execute() def get_annotations( - self, project_name: str, folder_name: str, item_names: List[str], logging=True + self, project_name: str, folder_name: str, item_names: List[str], logging=True ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1418,7 +1435,7 @@ def get_annotations( return use_case.execute() def get_annotations_per_frame( - self, project_name: str, folder_name: str, video_name: str, fps: int + self, project_name: str, folder_name: str, video_name: str, fps: int ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1435,7 +1452,7 @@ def get_annotations_per_frame( return use_case.execute() def upload_priority_scores( - self, project_name, folder_name, scores, project_folder_name + self, project_name, folder_name, scores, project_folder_name ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1450,27 +1467,27 @@ def upload_priority_scores( return use_case.execute() def get_integrations(self): - team = self.team_data.data + team = self.team_data use_cae = usecases.GetIntegrations( reporter=self.get_default_reporter(), - team=self.team_data.data, + team=self.team_data, integrations=self.get_integrations_repo(team_id=team.uuid), ) return use_cae.execute() def attach_integrations( - self, - project_name: str, - folder_name: str, - integration: IntegrationEntity, - folder_path: str, + self, + project_name: str, + folder_name: str, + integration: IntegrationEntity, + folder_path: str, ): - team = self.team_data.data + team = self.team_data project = self._get_project(project_name) folder = self._get_folder(project, folder_name) use_case = usecases.AttachIntegrations( reporter=self.get_default_reporter(), - team=self.team_data.data, + team=self.team_data, backend_service=self.backend_client, project=project, folder=folder, @@ -1506,15 +1523,15 @@ def get_item(self, project_name: str, folder_name: str, item_name: str): return use_case.execute() def list_items( - self, - project_name: str, - folder_name: str, - name_contains: str = None, - annotation_status: str = None, - annotator_email: str = None, - qa_email: str = None, - recursive: bool = False, - **kwargs, + self, + project_name: str, + folder_name: str, + name_contains: str = None, + annotation_status: str = None, + annotator_email: str = None, + qa_email: str = None, + recursive: bool = False, + **kwargs, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1546,11 +1563,11 @@ def list_items( return use_case.execute() def attach_items( - self, - project_name: str, - folder_name: str, - attachments: List[AttachmentEntity], - annotation_status: str, + self, + project_name: str, + folder_name: str, + attachments: List[AttachmentEntity], + annotation_status: str, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) @@ -1566,12 +1583,12 @@ def attach_items( return use_case.execute() def copy_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, - include_annotations: bool = False, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, + include_annotations: bool = False, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) @@ -1590,11 +1607,11 @@ def copy_items( return use_case.execute() def move_items( - self, - project_name: str, - from_folder: str, - to_folder: str, - items: List[str] = None, + self, + project_name: str, + from_folder: str, + to_folder: str, + items: List[str] = None, ): project = self._get_project(project_name) from_folder = self._get_folder(project, from_folder) @@ -1612,11 +1629,11 @@ def move_items( return use_case.execute() def set_annotation_statuses( - self, - project_name: str, - folder_name: str, - annotation_status: str, - item_names: List[str] = None, + self, + project_name: str, + folder_name: str, + annotation_status: str, + item_names: List[str] = None, ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 9d27ae698..4ddc12351 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -70,7 +70,7 @@ def __init__( @property def assets_provider_url(self): if self.api_url != constance.BACKEND_URL: - return "https://assets-provider.devsuperannotate.com/api/v1/" + return "https://sa-assets-provider.us-west-2.elasticbeanstalk.com/api/v1/" return "https://assets-provider.superannotate.com/api/v1/" @timed_lru_cache(seconds=360) @@ -233,7 +233,7 @@ class SuperannotateBackendService(BaseBackendService): URL_DELETE_ANNOTATIONS = "annotations/remove" URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus" URL_GET_LIMITS = "project/{}/limitationDetails" - URL_GET_ANNOTATIONS = "images/annotations/stream" + URL_GET_ANNOTATIONS = "annotations/stream" URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" URL_GET_INTEGRATIONS = "integrations" URL_ATTACH_INTEGRATIONS = "image/integration/create" @@ -299,7 +299,11 @@ def get_download_token( return response.json() def get_upload_token( - self, project_id: int, team_id: int, folder_id: int, image_id: int, + self, + project_id: int, + team_id: int, + folder_id: int, + image_id: int, ): download_token_url = urljoin( self.api_url, @@ -733,7 +737,11 @@ def assign_images( return res.ok def un_assign_images( - self, team_id: int, project_id: int, folder_name: str, image_names: List[str], + self, + team_id: int, + project_id: int, + folder_name: str, + image_names: List[str], ): un_assign_images_url = urljoin(self.api_url, self.URL_ASSIGN_IMAGES) res = self._request( @@ -749,7 +757,10 @@ def un_assign_images( return res.ok def un_assign_folder( - self, team_id: int, project_id: int, folder_name: str, + self, + team_id: int, + project_id: int, + folder_name: str, ): un_assign_folder_url = urljoin(self.api_url, self.URL_ASSIGN_FOLDER) res = self._request( diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 3be1cf8d7..844c5fe36 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -19,7 +19,7 @@ def __init__( self._headers = headers self._annotations = [] self._reporter = reporter - self._callback = callback + self._callback: Callable = callback self._map_function = map_function async def fetch( @@ -88,9 +88,10 @@ async def get_data( return self._annotations @staticmethod - def _store_annotation(path, postfix, annotation: dict): + def _store_annotation(path, postfix, annotation: dict, callback: Callable = None): os.makedirs(path, exist_ok=True) with open(f"{path}/{annotation['metadata']['name']}{postfix}", "w") as file: + annotation = callback(annotation) if callback else annotation json.dump(annotation, file) def _process_data(self, data): @@ -122,7 +123,8 @@ async def download_data( self._store_annotation( download_path, postfix, - self._callback(annotation) if self._callback else annotation, + annotation, + self._callback, ) else: async for annotation in self.fetch( @@ -131,5 +133,6 @@ async def download_data( self._store_annotation( download_path, postfix, - self._callback(annotation) if self._callback else annotation, + annotation, + self._callback ) diff --git a/tests/integration/annotations/test_annotations_pre_processing.py b/tests/integration/annotations/test_annotations_pre_processing.py index 574fce0ac..9653d658c 100644 --- a/tests/integration/annotations/test_annotations_pre_processing.py +++ b/tests/integration/annotations/test_annotations_pre_processing.py @@ -46,7 +46,7 @@ def test_annotation_last_action_and_creation_type(self, reporter): self.assertEqual(instance["creationType"], CreationTypeEnum.PRE_ANNOTATION.value) self.assertEqual( type(annotation["metadata"]["lastAction"]["email"]), - type(sa.controller.team_data.data.creator_id) + type(sa.controller.team_data.creator_id) ) self.assertEqual( type(annotation["metadata"]["lastAction"]["timestamp"]), diff --git a/tests/integration/annotations/test_download_annotations.py b/tests/integration/annotations/test_download_annotations.py index 48fb3587e..1083d2c76 100644 --- a/tests/integration/annotations/test_download_annotations.py +++ b/tests/integration/annotations/test_download_annotations.py @@ -62,3 +62,14 @@ def test_download_annotations_from_folders(self): with tempfile.TemporaryDirectory() as temp_dir: annotations_path = sa.download_annotations(f"{self.PROJECT_NAME}", temp_dir) self.assertEqual(len(os.listdir(annotations_path)), 5) + + @pytest.mark.flaky(reruns=3) + def test_download_annotations_from_folders(self): + sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) + sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME_2) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" + ) + with tempfile.TemporaryDirectory() as temp_dir: + annotations_path = sa.download_annotations(f"{self.PROJECT_NAME}", temp_dir) + self.assertEqual(len(os.listdir(annotations_path)), 1) \ No newline at end of file diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index 4871c3ca0..9dfae7102 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -25,7 +25,7 @@ class TestGetAnnotations(BaseTestCase): def folder_path(self): return os.path.join(Path(__file__).parent.parent.parent, self.TEST_FOLDER_PATH) - @pytest.mark.flaky(reruns=3) + # @pytest.mark.flaky(reruns=3) def test_get_annotations(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" diff --git a/tests/integration/mixpanel/test_individual_fuinctions.py b/tests/integration/mixpanel/test_individual_fuinctions.py new file mode 100644 index 000000000..685dede23 --- /dev/null +++ b/tests/integration/mixpanel/test_individual_fuinctions.py @@ -0,0 +1,30 @@ +import os +from unittest import TestCase +from unittest.mock import patch + +from src.superannotate import AppException +from src.superannotate import __version__ +from src.superannotate import class_distribution +from tests import DATA_SET_PATH + + +class TestDocumentUrls(TestCase): + PROJECT_NAME = "TEST_MIX" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PATH = "data_set" + + @property + def folder_path(self): + return os.path.join(DATA_SET_PATH, self.TEST_FOLDER_PATH) + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_get_team_metadata(self, track_method): + try: + class_distribution(os.path.join(self.folder_path, "sample_project_vector"), "test") + except AppException: + pass + data = list(track_method.call_args)[0][2] + assert not data["Success"] + assert data["Version"] == __version__ + assert data["project_names"] == "test" diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py new file mode 100644 index 000000000..f99b57768 --- /dev/null +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -0,0 +1,139 @@ +import copy +import threading +from unittest import TestCase +from unittest.mock import patch + +from src.superannotate import SAClient +from src.superannotate import AppException +from src.superannotate import __version__ + + +class TestDocumentUrls(TestCase): + CLIENT = SAClient() + TEAM_DATA = CLIENT.get_team_metadata() + BLANK_PAYLOAD = { + "SDK": True, + "Team": TEAM_DATA["name"], + "Team Owner": TEAM_DATA["creator_id"], + "Version": __version__, + "Success": True + } + PROJECT_NAME = "TEST_MIX" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PATH = "data_set" + + @classmethod + def setUpClass(cls) -> None: + cls.tearDownClass() + print(cls.PROJECT_NAME) + cls._project = cls.CLIENT.create_project( + cls.PROJECT_NAME, cls.PROJECT_DESCRIPTION, cls.PROJECT_TYPE + ) + + @classmethod + def tearDownClass(cls) -> None: + cls._safe_delete_project(cls.PROJECT_NAME) + + + @classmethod + def _safe_delete_project(cls, project_name): + projects = cls.CLIENT.search_projects(project_name, return_metadata=True) + for project in projects: + try: + cls.CLIENT.delete_project(project) + except Exception: + raise + + @property + def default_payload(self): + return copy.copy(self.BLANK_PAYLOAD) + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_get_team_metadata(self, track_method): + team = self.CLIENT.get_team_metadata() + team_owner = team["creator_id"] + result = list(track_method.call_args)[0] + payload = self.default_payload + assert result[0] == team_owner + assert result[1] == "get_team_metadata" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_search_team_contributors(self, track_method): + kwargs = { + "email": "user@supernnotate.com", + "first_name": "first_name", + "last_name": "last_name", + "return_metadata": False} + self.CLIENT.search_team_contributors(**kwargs) + result = list(track_method.call_args)[0] + payload = self.default_payload + payload.update(kwargs) + assert result[1] == "search_team_contributors" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_search_projects(self, track_method): + kwargs = { + "name": self.PROJECT_NAME, + "include_complete_image_count": True, + "status": "NotStarted", + "return_metadata": False} + self.CLIENT.search_projects(**kwargs) + result = list(track_method.call_args)[0] + payload = self.default_payload + payload.update(kwargs) + assert result[1] == "search_projects" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_create_project(self, track_method): + kwargs = { + "project_name": self.PROJECT_NAME, + "project_description": self.PROJECT_DESCRIPTION, + "project_type": self.PROJECT_TYPE, + "settings": {"a": 1, "b": 2} + } + try: + self.CLIENT.create_project(**kwargs) + except AppException: + pass + result = list(track_method.call_args)[0] + payload = self.default_payload + payload["Success"] = False + payload.update(kwargs) + payload["settings"] = list(kwargs["settings"].keys()) + assert result[1] == "create_project" + assert payload == list(track_method.call_args)[0][2] + + @patch("lib.app.interface.base_interface.Tracker._track") + def test_create_project_multi_thread(self, track_method): + project_1 = self.PROJECT_NAME + "_1" + project_2 = self.PROJECT_NAME + "_2" + try: + kwargs_1 = { + "project_name": project_1, + "project_description": self.PROJECT_DESCRIPTION, + "project_type": self.PROJECT_TYPE, + } + kwargs_2 = { + "project_name": project_2, + "project_description": self.PROJECT_DESCRIPTION, + "project_type": self.PROJECT_TYPE, + } + thread_1 = threading.Thread(target=self.CLIENT.create_project, kwargs=kwargs_1) + thread_2 = threading.Thread(target=self.CLIENT.create_project, kwargs=kwargs_2) + thread_1.start() + thread_2.start() + thread_1.join() + thread_2.join() + r1, r2 = track_method.call_args_list + r1_pr_name = r1[0][2].pop("project_name") + r2_pr_name = r2[0][2].pop("project_name") + assert r1_pr_name == project_1 + assert r2_pr_name == project_2 + assert r1[0][2] == r2[0][2] + finally: + self._safe_delete_project(project_1) + self._safe_delete_project(project_2) \ No newline at end of file diff --git a/tests/integration/test_attach_document_urls.py b/tests/integration/test_attach_document_urls.py deleted file mode 100644 index 64d434728..000000000 --- a/tests/integration/test_attach_document_urls.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -from os.path import dirname -from os.path import join - -from src.superannotate import SAClient -sa = SAClient() -from src.superannotate import AppException -import src.superannotate.lib.core as constances -from tests.integration.base import BaseTestCase - - -class TestDocumentUrls(BaseTestCase): - PROJECT_NAME = "document attach urls" - PATH_TO_URLS = "csv_files/text_urls.csv" - PATH_TO_50K_URLS = "501_urls.csv" - PROJECT_DESCRIPTION = "desc" - PROJECT_TYPE = "Document" diff --git a/tests/integration/test_attach_video_urls.py b/tests/integration/test_attach_video_urls.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/integration/test_assign_images.py b/tests/unit/test_client.py similarity index 100% rename from tests/integration/test_assign_images.py rename to tests/unit/test_client.py diff --git a/tests/unit/test_validators.py b/tests/unit/test_validators.py index eeed05e77..a10421564 100644 --- a/tests/unit/test_validators.py +++ b/tests/unit/test_validators.py @@ -3,8 +3,8 @@ import tempfile from os.path import dirname from unittest import TestCase -from unittest.mock import patch - +`from unittest.mock import patch +` from pydantic import ValidationError from src.superannotate import SAClient From e4e11ebd857f5fe48a07e5f491995dbba6aac6aa Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 2 Jun 2022 16:30:13 +0400 Subject: [PATCH 10/59] tod --- tests/unit/test_client.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/unit/test_client.py diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py deleted file mode 100644 index e69de29bb..000000000 From f34e46f53a704d254ba49ad12f796f7bdbc1e642 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Thu, 2 Jun 2022 18:29:14 +0400 Subject: [PATCH 11/59] no more Skippable exceptions, deprecation messages and more --- .../lib/app/interface/base_interface.py | 15 ++++++--------- .../lib/app/interface/sdk_interface.py | 19 +++++++++++++++++-- src/superannotate/lib/core/exceptions.py | 5 ----- src/superannotate/lib/core/reporter.py | 4 ++-- src/superannotate/lib/core/usecases/base.py | 5 +---- src/superannotate/lib/core/usecases/items.py | 10 ++-------- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 4dfd4f1fb..60c251af4 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -46,9 +46,12 @@ def __init__( f"CLI's superannotate init to generate default location config file." ) config_repo = ConfigRepository(config_path) + main_endpoint = config_repo.get_one("main_endpoint").value + if not main_endpoint: + main_endpoint = constants.BACKEND_URL token, host, ssl_verify = ( Controller.validate_token(config_repo.get_one("token").value), - config_repo.get_one("main_endpoint").value, + main_endpoint, config_repo.get_one("ssl_verify").value, ) self._host = host @@ -57,19 +60,13 @@ def __init__( BaseInterfaceFacade.REGISTRY.append(self) @property - @abstractmethod def host(self): - raise NotImplementedError + return self._host @property - @abstractmethod def token(self): - raise NotImplementedError + return self._token - @property - @abstractmethod - def logger(self): - raise NotImplementedError class Tracker: diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index cb74e5076..7dba7221c 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -693,10 +693,9 @@ def assign_items( if not response.errors: logger.info(f"Assign items to user {user}") - elif response.status != "Skip": + else: raise AppException(response.errors) - @validate_arguments def unassign_items( self, project: Union[NotEmptyStr, dict], item_names: List[NotEmptyStr] ): @@ -731,6 +730,14 @@ def assign_images( :param user: user email :type user: str """ + + warning_msg = ( + "We're deprecating the assign_images function. Please use assign_items instead." + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.assign_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) project = self.controller.get_project_metadata(project_name).data @@ -778,6 +785,14 @@ def unassign_images( :param image_names: list of images to unassign :type image_names: list of str """ + + warning_msg = ( + "We're deprecating the unassign_images function. Please use unassign_items instead." + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.unassign_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) response = self.controller.un_assign_items( diff --git a/src/superannotate/lib/core/exceptions.py b/src/superannotate/lib/core/exceptions.py index b14acac5d..4228e5cb2 100644 --- a/src/superannotate/lib/core/exceptions.py +++ b/src/superannotate/lib/core/exceptions.py @@ -25,11 +25,6 @@ class AppValidationException(AppException): """ -class SkippableAppValidationException(AppValidationException): - """ - App validation exception - """ - class ImageProcessingException(AppException): """ App validation exception diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index f93781170..47f83ed00 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -45,8 +45,8 @@ def __exit__(self, type, value, traceback): def __del__(self): globs = globals() - if "SESSIONS" in globs and globs["SESSIONS"].get(self.pk): - del globs["SESSIONS"][self.pk] + # if "SESSIONS" in globs and globs.get("SESSIONS", {}).get(self.pk): + # del globs["SESSIONS"][self.pk] @property def data(self): diff --git a/src/superannotate/lib/core/usecases/base.py b/src/superannotate/lib/core/usecases/base.py index a15325890..97ed22ee8 100644 --- a/src/superannotate/lib/core/usecases/base.py +++ b/src/superannotate/lib/core/usecases/base.py @@ -4,7 +4,7 @@ from typing import Iterable from typing import List -from lib.core.exceptions import AppValidationException, SkippableAppValidationException +from lib.core.exceptions import AppValidationException from lib.core.reporter import Reporter from lib.core.response import Response @@ -24,9 +24,6 @@ def _validate(self): if name.startswith("validate_"): method = getattr(self, name) method() - except SkippableAppValidationException as e: - self._response.errors = e - self._response.status = 'Skip' except AppValidationException as e: self._response.errors = e diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index b955880d3..dbd9906fa 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -12,7 +12,7 @@ from lib.core.entities import TmpImageEntity from lib.core.entities import VideoEntity from lib.core.exceptions import AppException -from lib.core.exceptions import AppValidationException, SkippableAppValidationException +from lib.core.exceptions import AppValidationException from lib.core.exceptions import BackendError from lib.core.reporter import Reporter from lib.core.repositories import BaseReadOnlyRepository @@ -215,11 +215,7 @@ def validate_user(self, ): if c["user_id"] == self._user: return True - logger.warning( - f"Skipping {self._user}. {self._user} is not a verified contributor for the {self._project.name}" - ) - - raise SkippableAppValidationException(f"{self._user} is not a verified contributor for the {self._project.name}") + raise AppValidationException(f"{self._user} is not a verified contributor for the {self._project.name}") def execute(self): if self.is_valid(): @@ -236,8 +232,6 @@ def execute(self): f"Cant assign {', '.join(self._item_names[i: i + self.CHUNK_SIZE])}" ) continue - else: - self._response.status = 'Ok' return self._response From 525f2515740b0599b43f94a97470711f18cc9a9b Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Fri, 3 Jun 2022 17:33:55 +0400 Subject: [PATCH 12/59] Session fix --- src/superannotate/__init__.py | 12 +- .../lib/app/analytics/aggregators.py | 2 +- .../lib/app/interface/base_interface.py | 9 +- .../lib/app/interface/sdk_interface.py | 14 +-- src/superannotate/lib/core/__init__.py | 5 +- src/superannotate/lib/core/config.py | 70 +++++++++++ src/superannotate/lib/core/reporter.py | 49 +------- src/superannotate/lib/core/response.py | 1 + .../lib/core/serviceproviders.py | 1 - .../lib/core/usecases/annotations.py | 4 +- src/superannotate/lib/core/usecases/items.py | 13 +- .../lib/infrastructure/controller.py | 4 +- .../lib/infrastructure/stream_data_handler.py | 5 +- src/superannotate/version.py | 2 +- .../mixpanel/test_mixpanel_decorator.py | 2 +- tests/unit/test_controller_init.py | 114 ------------------ tests/unit/test_init.py | 21 ++++ 17 files changed, 128 insertions(+), 200 deletions(-) create mode 100644 src/superannotate/lib/core/config.py delete mode 100644 tests/unit/test_controller_init.py create mode 100644 tests/unit/test_init.py diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index dfc7b0db8..a10fb8b8d 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -2,20 +2,15 @@ import sys sys.path.append(os.path.split(os.path.realpath(__file__))[0]) -from superannotate.lib.app.input_converters.conversion import ( - convert_json_version, -) # noqa -from superannotate.lib.app.input_converters.conversion import ( - convert_project_type, -) # noqa - import logging.config # noqa import requests # noqa from packaging.version import parse # noqa +from superannotate.lib.app.input_converters import convert_json_version # noqa +from superannotate.lib.app.input_converters import convert_project_type # noqa from superannotate.lib.app.analytics.class_analytics import class_distribution # noqa from superannotate.lib.app.exceptions import AppException # noqa -from superannotate.lib.app.input_converters import convert_json_version +from superannotate.lib.app.input_converters import convert_json_version # noqa from superannotate.lib.app.input_converters import convert_project_type # noqa from superannotate.lib.app.input_converters import export_annotation # noqa from superannotate.lib.app.input_converters import import_annotation # noqa @@ -26,7 +21,6 @@ from superannotate.logger import get_default_logger # noqa from superannotate.version import __version__ # noqa - SESSIONS = {} __all__ = [ diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index 9df04d7f0..9e92c4a00 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -1,6 +1,5 @@ import copy import json -from dataclasses import dataclass from pathlib import Path from typing import List from typing import Optional @@ -8,6 +7,7 @@ import lib.core as constances import pandas as pd +from dataclasses import dataclass from lib.app.exceptions import AppException from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from lib.core import PIXEL_ANNOTATION_POSTFIX diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 60c251af4..06ff5de47 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -1,7 +1,6 @@ import functools import os import sys -from abc import abstractmethod from inspect import signature from pathlib import Path from types import FunctionType @@ -11,8 +10,8 @@ import lib.core as constants from lib.app.helpers import extract_project_folder from lib.app.interface.types import validate_arguments +from lib.core import CONFIG from lib.core.exceptions import AppException -from lib.core.reporter import Session from lib.infrastructure.controller import Controller from lib.infrastructure.repositories import ConfigRepository from mixpanel import Mixpanel @@ -27,15 +26,14 @@ def __init__( token: str = None, config_path: str = constants.CONFIG_PATH, ): - host = constants.BACKEND_URL env_token = os.environ.get("SA_TOKEN") + host = os.environ.get("SA_URL", constants.BACKEND_URL) version = os.environ.get("SA_VERSION", "v1") ssl_verify = bool(os.environ.get("SA_SSL", True)) if token: token = Controller.validate_token(token=token) elif env_token: host = os.environ.get("SA_URL", constants.BACKEND_URL) - token = Controller.validate_token(env_token) else: config_path = os.path.expanduser(str(config_path)) @@ -68,7 +66,6 @@ def token(self): return self._token - class Tracker: def get_mp_instance(self) -> Mixpanel: if self.client: @@ -149,7 +146,7 @@ def _track_method(self, args, kwargs, success: bool): self._track( user_id, event_name, - {**default, **properties, **Session.get_current_session().data}, + {**default, **properties, **CONFIG.get_current_session().data}, ) def __get__(self, obj, owner=None): diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 7dba7221c..ac8d3ec2c 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -61,7 +61,6 @@ logger = get_default_logger() - class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta): def get_team_metadata(self): """Returns team metadata @@ -313,7 +312,6 @@ def rename_project(self, project: NotEmptyStr, new_name: NotEmptyStr): ) return ProjectSerializer(response.data).serialize() - def get_folder_metadata(self, project: NotEmptyStr, folder_name: NotEmptyStr): """Returns folder metadata @@ -687,9 +685,7 @@ def assign_items( project_name, folder_name = extract_project_folder(project) - response = self.controller.assign_items( - project, folder_name, item_names, user - ) + response = self.controller.assign_items(project, folder_name, item_names, user) if not response.errors: logger.info(f"Assign items to user {user}") @@ -733,8 +729,8 @@ def assign_images( warning_msg = ( "We're deprecating the assign_images function. Please use assign_items instead." - "Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.assign_items" + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.assign_items" ) logger.warning(warning_msg) warnings.warn(warning_msg, DeprecationWarning) @@ -788,8 +784,8 @@ def unassign_images( warning_msg = ( "We're deprecating the unassign_images function. Please use unassign_items instead." - "Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.unassign_items" + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.unassign_items" ) logger.warning(warning_msg) warnings.warn(warning_msg, DeprecationWarning) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index c30d67469..3956efc86 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -1,6 +1,6 @@ from os.path import expanduser -from pathlib import Path +from superannotate.lib.core.config import Config from superannotate.lib.core.enums import AnnotationStatus from superannotate.lib.core.enums import ImageQuality from superannotate.lib.core.enums import ProjectState @@ -13,6 +13,9 @@ from superannotate.lib.core.enums import UserRole +CONFIG = Config() + + CONFIG_PATH = "~/.superannotate/config.json" CONFIG_FILE_LOCATION = expanduser(CONFIG_PATH) LOG_FILE_LOCATION = expanduser("~/.superannotate/sa.log") diff --git a/src/superannotate/lib/core/config.py b/src/superannotate/lib/core/config.py new file mode 100644 index 000000000..e5d955e12 --- /dev/null +++ b/src/superannotate/lib/core/config.py @@ -0,0 +1,70 @@ +import threading +from typing import Dict + +from dataclasses import dataclass +from dataclasses import field + + +class Session: + def __init__(self): + self.pk = threading.get_ident() + self._data_dict = {} + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + if type is not None: + return False + + def __del__(self): + Config().delete_current_session() + + @property + def data(self): + return self._data_dict + + @staticmethod + def get_current_session(): + return Config().get_current_session() + + def __setitem__(self, key, item): + self._data_dict[key] = item + + def __getitem__(self, key): + return self._data_dict[key] + + def __repr__(self): + return repr(self._data_dict) + + def clear(self): + return self._data_dict.clear() + + +class Singleton(type): + _instances = {} + _lock = threading.Lock() + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + with cls._lock: + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + +@dataclass() +class Config(metaclass=Singleton): + SESSIONS: Dict[int, Session] = field(default_factory=dict) + + def get_current_session(self): + session = self.SESSIONS.get(threading.get_ident()) + if not session: + session = Session() + self.SESSIONS.update({session.pk: session}) + return session + + def delete_current_session(self): + ident = threading.get_ident() + if ident in self.SESSIONS: + del self.SESSIONS[ident] diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 47f83ed00..9e4ad2d69 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -6,6 +6,7 @@ from typing import Union import tqdm +from lib.core import CONFIG from superannotate.logger import get_default_logger @@ -31,51 +32,6 @@ def init_spin(self): sys.stdout.write("\b") -class Session: - def __init__(self): - self.pk = threading.get_ident() - self._data_dict = {} - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - if type is not None: - return False - - def __del__(self): - globs = globals() - # if "SESSIONS" in globs and globs.get("SESSIONS", {}).get(self.pk): - # del globs["SESSIONS"][self.pk] - - @property - def data(self): - return self._data_dict - - @staticmethod - def get_current_session(): - globs = globals() - if not globs.get("SESSIONS") or not globs["SESSIONS"].get( - threading.get_ident() - ): - session = Session() - globals().update({"SESSIONS": {session.pk: session}}) - return session - return globs["SESSIONS"][threading.get_ident()] - - def __setitem__(self, key, item): - self._data_dict[key] = item - - def __getitem__(self, key): - return self._data_dict[key] - - def __repr__(self): - return repr(self._data_dict) - - def clear(self): - return self._data_dict.clear() - - class Reporter: def __init__( self, @@ -83,7 +39,6 @@ def __init__( log_warning: bool = True, disable_progress_bar: bool = False, log_debug: bool = True, - session: Session = None, ): self.logger = get_default_logger() self._log_info = log_info @@ -95,7 +50,7 @@ def __init__( self.debug_messages = [] self.custom_messages = defaultdict(set) self.progress_bar = None - self.session = session + self.session = CONFIG.get_current_session() self._spinner = None def start_spinner(self): diff --git a/src/superannotate/lib/core/response.py b/src/superannotate/lib/core/response.py index 86aab3020..107bd28c3 100644 --- a/src/superannotate/lib/core/response.py +++ b/src/superannotate/lib/core/response.py @@ -10,6 +10,7 @@ def __init__(self, status: str = None, data: Union[dict, list] = None): def __str__(self): return f"Response object with status:{self.status}, data : {self.data}, errors: {self.errors} " + @property def data(self): return self._data diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 345a0f4db..7a9868912 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -218,7 +218,6 @@ def un_assign_items( project_id: int, folder_name: str, item_names: list, - ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index c553db8a5..b64bd9bf8 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -816,7 +816,9 @@ def execute(self): if self.is_valid(): export_path = str( self.destination - / Path(f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}") + / Path( + f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}" + ) ) self.reporter.log_info( f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index dbd9906fa..8d580e6fe 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -18,13 +18,14 @@ from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider +from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase from superannotate.logger import get_default_logger -from lib.core.usecases.base import BaseReportableUseCase logger = get_default_logger() + class GetItem(BaseReportableUseCase): def __init__( self, @@ -209,13 +210,17 @@ def __init__( self._user = user self._service = service - def validate_user(self, ): + def validate_user( + self, + ): for c in self._project.users: if c["user_id"] == self._user: return True - raise AppValidationException(f"{self._user} is not a verified contributor for the {self._project.name}") + raise AppValidationException( + f"{self._user} is not a verified contributor for the {self._project.name}" + ) def execute(self): if self.is_valid(): @@ -237,6 +242,7 @@ def execute(self): class UnAssignItemsUseCase(BaseUseCase): CHUNK_SIZE = 500 + def __init__( self, service: SuperannotateServiceProvider, @@ -267,7 +273,6 @@ def execute(self): return self._response - class AttachItems(BaseReportableUseCase): CHUNK_SIZE = 500 diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index accfddd70..7f4f97b06 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -838,7 +838,9 @@ def delete_images( def assign_items( self, project_name: str, folder_name: str, item_names: list, user: str ): - project_entity = self.get_project_metadata(project_name, include_contributors = True).data['project'] + project_entity = self.get_project_metadata( + project_name, include_contributors=True + ).data["project"] folder = self._get_folder(project_entity, folder_name) use_case = usecases.AssignItemsUseCase( project=project_entity, diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 844c5fe36..ad12f2175 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -131,8 +131,5 @@ async def download_data( method, session, url, self._process_data(data), params=params ): self._store_annotation( - download_path, - postfix, - annotation, - self._callback + download_path, postfix, annotation, self._callback ) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 6914911d9..cd72ef523 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev1" +__version__ = "4.3.5dev11" diff --git a/tests/integration/mixpanel/test_mixpanel_decorator.py b/tests/integration/mixpanel/test_mixpanel_decorator.py index f99b57768..86dd9811b 100644 --- a/tests/integration/mixpanel/test_mixpanel_decorator.py +++ b/tests/integration/mixpanel/test_mixpanel_decorator.py @@ -8,7 +8,7 @@ from src.superannotate import __version__ -class TestDocumentUrls(TestCase): +class TestMixpanel(TestCase): CLIENT = SAClient() TEAM_DATA = CLIENT.get_team_metadata() BLANK_PAYLOAD = { diff --git a/tests/unit/test_controller_init.py b/tests/unit/test_controller_init.py deleted file mode 100644 index 756163e82..000000000 --- a/tests/unit/test_controller_init.py +++ /dev/null @@ -1,114 +0,0 @@ -# import os -# from os.path import join -# import json -# import pkg_resources -# import tempfile -# import pytest -# from unittest import TestCase -# from unittest.mock import mock_open -# from unittest.mock import patch -# -# -# from src.superannotate.lib.app.interface.cli_interface import CLIFacade -# from tests.utils.helpers import catch_prints -# -# -# try: -# CLI_VERSION = pkg_resources.get_distribution("superannotate").version -# except Exception: -# CLI_VERSION = None -# -# -# class CLITest(TestCase): -# CONFIG_FILE_DATA = '{"main_endpoint": "https://amazonaws.com:3000","token": "c9c55ct=6085","ssl_verify": false}' -# -# @pytest.mark.skip(reason="Need to adjust") -# @patch('builtins.input') -# def test_init_update(self, input_mock): -# input_mock.side_effect = ["y", "token"] -# with patch('builtins.open', mock_open(read_data=self.CONFIG_FILE_DATA)) as config_file: -# try: -# with catch_prints() as out: -# cli = CLIFacade() -# cli.init() -# except SystemExit: -# input_mock.assert_called_with("Input the team SDK token from https://app.superannotate.com/team : ") -# config_file().write.assert_called_once_with( -# json.dumps( -# {"main_endpoint": "https://api.devsuperannotate.com", "ssl_verify": False, "token": "token"}, -# indent=4 -# ) -# ) -# self.assertEqual(out.getvalue().strip(), "Configuration file successfully updated.") -# -# @pytest.mark.skip(reason="Need to adjust") -# @patch('builtins.input') -# def test_init_create(self, input_mock): -# input_mock.side_effect = ["token"] -# with patch('builtins.open', mock_open(read_data="{}")) as config_file: -# try: -# with catch_prints() as out: -# cli = CLIFacade() -# cli.init() -# except SystemExit: -# input_mock.assert_called_with("Input the team SDK token from https://app.superannotate.com/team : ") -# config_file().write.assert_called_once_with( -# json.dumps( -# {"token": "token"}, -# indent=4 -# ) -# ) -# self.assertEqual(out.getvalue().strip(), "Configuration file successfully created.") -# -# -# class SKDInitTest(TestCase): -# TEST_TOKEN = "toke=123" -# -# VALID_JSON = { -# "token": "a"*28 + "=1234" -# } -# INVALID_JSON = { -# "token": "a" * 28 + "=1234asd" -# } -# FILE_NAME = "config.json" -# FILE_NAME_2 = "config.json" -# -# def test_env_flow(self): -# import superannotate as sa -# os.environ.update({"SA_TOKEN": self.TEST_TOKEN}) -# self.assertEqual(sa.controller.token, self.TEST_TOKEN) -# -# def test_init_via_config_file(self): -# with tempfile.TemporaryDirectory() as temp_dir: -# token_path = f"{temp_dir}/config.json" -# with open(token_path, "w") as temp_config: -# json.dump({"token": self.TEST_TOKEN}, temp_config) -# temp_config.close() -# from src.superannotate import SAClient -# sa = SAClient() -# sa.init(token_path) -# -# @patch("lib.infrastructure.controller.Controller.retrieve_configs") -# def test_init_default_configs_open(self, retrieve_configs): -# from src.superannotate import SAClient -# sa = SAClient() -# try: -# sa.init() -# except Exception: -# self.assertTrue(retrieve_configs.call_args[0], sa.constances.CONFIG_FILE_LOCATION) -# -# def test_init(self): -# with tempfile.TemporaryDirectory() as temp_dir: -# path = join(temp_dir, self.FILE_NAME) -# with open(path, "w") as config: -# json.dump(self.VALID_JSON, config) -# from src.superannotate import SAClient -# sa = SAClient() -# sa.init(path) -# self.assertEqual(sa.get_default_controller().team_id, 1234) -# -# def test_(self): -# from src.superannotate import SAClient -# sa = SAClient() -# sa.init("~/.superannotate/prod_config.json") -# sa.search_projects() \ No newline at end of file diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py new file mode 100644 index 000000000..9b7fcaee7 --- /dev/null +++ b/tests/unit/test_init.py @@ -0,0 +1,21 @@ +import os + +from src.superannotate import SAClient +from src.superannotate.lib.core import CONFIG_PATH +from src.superannotate.lib.core import CONFIG +from src.superannotate.lib.infrastructure.repositories import ConfigRepository + + +def test_init_from_token(): + config_repo = ConfigRepository(CONFIG_PATH) + main_endpoint = config_repo.get_one("main_endpoint").value + os.environ.update({"SA_URL": main_endpoint}) + token = config_repo.get_one("token").value + + sa_1 = SAClient(token=token) + sa_2 = SAClient(token=token) + sa_1.get_team_metadata() + sa_2.get_team_metadata() + + assert len(CONFIG.SESSIONS) == 1 + From 9a2e13bb13c5899866adc9e55102aaae8ad86229 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 6 Jun 2022 12:44:44 +0400 Subject: [PATCH 13/59] change assets provider url --- src/superannotate/lib/infrastructure/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index b66026bf5..95e3c896e 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -70,7 +70,7 @@ def __init__( @property def assets_provider_url(self): if self.api_url != constance.BACKEND_URL: - return "https://sa-assets-provider.us-west-2.elasticbeanstalk.com/api/v1/" + return "https://assets-provider.devsuperannotate.com/api/v1/" return "https://assets-provider.superannotate.com/api/v1/" @timed_lru_cache(seconds=360) @@ -234,7 +234,7 @@ class SuperannotateBackendService(BaseBackendService): URL_DELETE_ANNOTATIONS = "annotations/remove" URL_DELETE_ANNOTATIONS_PROGRESS = "annotations/getRemoveStatus" URL_GET_LIMITS = "project/{}/limitationDetails" - URL_GET_ANNOTATIONS = "annotations/stream" + URL_GET_ANNOTATIONS = "images/annotations/stream" URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" URL_GET_INTEGRATIONS = "integrations" URL_ATTACH_INTEGRATIONS = "image/integration/create" From 08b286a0ea6b097f85297e8c6d13b0dde0b5173a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 2 Jun 2022 16:09:58 +0400 Subject: [PATCH 14/59] Enums --- pytest.ini | 2 +- src/superannotate/__init__.py | 2 ++ src/superannotate/lib/app/interface/types.py | 13 +++++++++++++ src/superannotate/lib/core/enums.py | 8 ++++++++ tests/unit/test_enum_arguments_handeling.py | 20 ++++++++++++++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_enum_arguments_handeling.py diff --git a/pytest.ini b/pytest.ini index 86c2d4c63..260f700fe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.7 log_cli=true python_files = test_*.py -addopts = -n auto --dist=loadscope \ No newline at end of file +;addopts = -n auto --dist=loadscope \ No newline at end of file diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index f6bfa0eed..9b0769d02 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -18,6 +18,7 @@ from superannotate.lib.core import PACKAGE_VERSION_UPGRADE # noqa from superannotate.logger import get_default_logger # noqa from superannotate.version import __version__ # noqa +import superannotate.lib.core.enums as enums SESSIONS = {} @@ -26,6 +27,7 @@ "__version__", "SAClient", # Utils + "enums", "AppException", # analytics "class_distribution", diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index 5103468cd..b408c3f9a 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -22,10 +22,23 @@ from pydantic import validate_arguments as pydantic_validate_arguments from pydantic import ValidationError from pydantic.errors import StrRegexError +from pydantic import errors +from pydantic.errors import PydanticTypeError NotEmptyStr = constr(strict=True, min_length=1) +class EnumMemberError(PydanticTypeError): + code = 'enum' + + def __str__(self) -> str: + permitted = ', '.join(str(v.name) for v in self.enum_values) # type: ignore + return f'Available values are: {permitted}' + + +errors.EnumMemberError = EnumMemberError + + class EmailStr(StrictStr): @classmethod def validate(cls, value: Union[str]) -> Union[str]: diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index 62a5bcc58..0fc7cc3a6 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -8,8 +8,13 @@ def __new__(cls, title, value): obj._value_ = value obj.__doc__ = title obj._type = "titled_enum" + cls._value2member_map_[title] = obj return obj + @classmethod + def choices(cls): + return tuple(cls._value2member_map_.keys()) + @DynamicClassAttribute def name(self) -> str: return self.__doc__ @@ -41,6 +46,9 @@ def titles(cls): def equals(self, other: Enum): return self.__doc__.lower() == other.__doc__.lower() + def __eq__(self, other): + return super(BaseTitledEnum, self).__eq__(other) + class AnnotationTypes(str, Enum): BBOX = "bbox" diff --git a/tests/unit/test_enum_arguments_handeling.py b/tests/unit/test_enum_arguments_handeling.py new file mode 100644 index 000000000..7ff71fbb8 --- /dev/null +++ b/tests/unit/test_enum_arguments_handeling.py @@ -0,0 +1,20 @@ +# from typing import Literal +from pydantic.typing import Literal + +from superannotate import enums +from superannotate import SAClient +from superannotate.lib.app.interface.types import validate_arguments + + + +@validate_arguments +def foo(status: enums.ProjectStatus): + return status + + +def test_enum_arg(): + SAClient() + assert foo(1) == 1 + assert foo("NotStarted") == 1 + assert foo(enums.ProjectStatus.NotStarted.name) == 1 + assert foo(enums.ProjectStatus.NotStarted.value) == 1 From 3fe5760996b4321ea4913ff5b39d69219ec29554 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Mon, 6 Jun 2022 17:40:33 +0400 Subject: [PATCH 15/59] fixed naming and folder name / project name issue --- src/superannotate/lib/app/interface/sdk_interface.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index ac8d3ec2c..9ef763de2 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -668,7 +668,7 @@ def delete_images( ) def assign_items( - self, project: Union[NotEmptyStr, dict], item_names: List[str], user: str + self, project: Union[NotEmptyStr, dict], items: List[str], user: str ): """Assigns items to a user. The assignment role, QA or Annotator, will be deduced from the user's role in the project. The type of the objects` image, video or text @@ -685,7 +685,7 @@ def assign_items( project_name, folder_name = extract_project_folder(project) - response = self.controller.assign_items(project, folder_name, item_names, user) + response = self.controller.assign_items(project_name, folder_name, items, user) if not response.errors: logger.info(f"Assign items to user {user}") @@ -693,7 +693,7 @@ def assign_items( raise AppException(response.errors) def unassign_items( - self, project: Union[NotEmptyStr, dict], item_names: List[NotEmptyStr] + self, project: Union[NotEmptyStr, dict], items: List[NotEmptyStr] ): """Removes assignment of given items for all assignees. With SDK, the user can be assigned to a role in the project with the share_project @@ -707,7 +707,7 @@ def unassign_items( project_name, folder_name = extract_project_folder(project) response = self.controller.un_assign_items( - project_name=project_name, folder_name=folder_name, item_names=item_names + project_name=project_name, folder_name=folder_name, item_names=items ) if response.errors: raise AppException(response.errors) From 83d7e8931f5e9e845209f57a2df40e86b7200242 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Mon, 6 Jun 2022 17:47:05 +0400 Subject: [PATCH 16/59] fixed docstrings --- src/superannotate/lib/app/interface/sdk_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 9ef763de2..fbc1d0e03 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -677,7 +677,7 @@ def assign_items( :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param item_names: list of item names to assign + :param items: list of items to assign :type item_names: list of str :param user: user email :type user: str @@ -701,7 +701,7 @@ def unassign_items( :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param item_names: list of items to unassign + :param items: list of items to unassign :type item_names: list of str """ project_name, folder_name = extract_project_folder(project) From 29912ad56066f5fcb7772cde63d966eb955b1acf Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 7 Jun 2022 19:13:17 +0400 Subject: [PATCH 17/59] Update download annotations --- src/superannotate/__init__.py | 4 +- .../lib/app/interface/base_interface.py | 58 +++++++------ .../lib/app/interface/sdk_interface.py | 85 ++++++++++--------- src/superannotate/lib/app/interface/types.py | 10 +-- src/superannotate/lib/core/__init__.py | 1 - src/superannotate/lib/core/enums.py | 2 +- .../lib/core/serviceproviders.py | 5 +- .../lib/core/usecases/annotations.py | 17 ++-- .../lib/infrastructure/services.py | 2 +- .../lib/infrastructure/stream_data_handler.py | 14 +-- .../annotations/test_download_annotations.py | 6 +- .../test_single_annotation_download.py | 7 +- 12 files changed, 115 insertions(+), 96 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 9b0769d02..6dc5a463d 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -8,7 +8,7 @@ from packaging.version import parse # noqa from superannotate.lib.app.analytics.class_analytics import class_distribution # noqa from superannotate.lib.app.exceptions import AppException # noqa -from superannotate.lib.app.input_converters import convert_json_version +from superannotate.lib.app.input_converters import convert_json_version # noqa from superannotate.lib.app.input_converters import convert_project_type # noqa from superannotate.lib.app.input_converters import export_annotation # noqa from superannotate.lib.app.input_converters import import_annotation # noqa @@ -18,7 +18,7 @@ from superannotate.lib.core import PACKAGE_VERSION_UPGRADE # noqa from superannotate.logger import get_default_logger # noqa from superannotate.version import __version__ # noqa -import superannotate.lib.core.enums as enums +import superannotate.lib.core.enums as enums # noqa SESSIONS = {} diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 3904a97a6..e5e1135cb 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -7,6 +7,7 @@ from types import FunctionType from typing import Iterable from typing import Sized +from typing import Tuple import lib.core as constants from lib.app.helpers import extract_project_folder @@ -22,44 +23,45 @@ class BaseInterfaceFacade: REGISTRY = [] - def __init__( - self, - token: str = None, - config_path: str = constants.CONFIG_PATH, - ): - host = constants.BACKEND_URL - env_token = os.environ.get("SA_TOKEN") + def __init__(self, token: str = None, config_path: str = None): version = os.environ.get("SA_VERSION", "v1") - ssl_verify = bool(os.environ.get("SA_SSL", True)) + _token, _config_path = None, None + _host = os.environ.get("SA_URL", constants.BACKEND_URL) + _ssl_verify = bool(os.environ.get("SA_SSL", True)) if token: - token = Controller.validate_token(token=token) - elif env_token: - host = os.environ.get("SA_URL", constants.BACKEND_URL) - - token = Controller.validate_token(env_token) + _token = Controller.validate_token(token=token) + elif config_path: + _token, _host, _ssl_verify = self._retrieve_configs(config_path) else: - config_path = os.path.expanduser(str(config_path)) - if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): - raise AppException( - f"SuperAnnotate config file {str(config_path)} not found." - f" Please provide correct config file location to sa.init() or use " - f"CLI's superannotate init to generate default location config file." + _token = os.environ.get("SA_TOKEN") + if not _token: + _toke, _host, _ssl_verify = self._retrieve_configs( + constants.CONFIG_PATH ) - config_repo = ConfigRepository(config_path) - token, host, ssl_verify = ( - Controller.validate_token(config_repo.get_one("token").value), - config_repo.get_one("main_endpoint").value, - config_repo.get_one("ssl_verify").value, - ) - self._host = host - self._token = token - self.controller = Controller(token, host, ssl_verify, version) + self._token, self._host = _host, _token + self.controller = Controller(_token, _host, _ssl_verify, version) def __new__(cls, *args, **kwargs): obj = super().__new__(cls, *args, **kwargs) cls.REGISTRY.append(obj) return obj + @staticmethod + def _retrieve_configs(path) -> Tuple[str, str, str]: + config_path = os.path.expanduser(str(path)) + if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + raise AppException( + f"SuperAnnotate config file {str(config_path)} not found." + f" Please provide correct config file location to sa.init() or use " + f"CLI's superannotate init to generate default location config file." + ) + config_repo = ConfigRepository(config_path) + return ( + Controller.validate_token(config_repo.get_one("token").value), + config_repo.get_one("main_endpoint").value, + config_repo.get_one("ssl_verify").value, + ) + @property @abstractmethod def host(self): diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 59ef32313..187ef17a6 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -13,7 +13,7 @@ from typing import Union import boto3 -import lib.core as constances +import lib.core as constants from lib.app.annotation_helpers import add_annotation_bbox_to_json from lib.app.annotation_helpers import add_annotation_comment_to_json from lib.app.annotation_helpers import add_annotation_point_to_json @@ -62,6 +62,13 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta): + def __init__( + self, + token: str = None, + config_path: str = constants.CONFIG_PATH, + ): + super().__init__(token, config_path) + def get_team_metadata(self): """Returns team metadata @@ -415,11 +422,11 @@ def copy_image( ).data if destination_project_metadata["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ] or source_project_metadata["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException( LIMITED_FUNCTIONS[source_project_metadata["project"].type] @@ -685,8 +692,8 @@ def assign_images( project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) @@ -803,12 +810,12 @@ def upload_images_from_folder_to_project( folder_path: Union[NotEmptyStr, Path], extensions: Optional[ Union[List[NotEmptyStr], Tuple[NotEmptyStr]] - ] = constances.DEFAULT_IMAGE_EXTENSIONS, + ] = constants.DEFAULT_IMAGE_EXTENSIONS, annotation_status="NotStarted", from_s3_bucket=None, exclude_file_patterns: Optional[ Iterable[NotEmptyStr] - ] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS, + ] = constants.DEFAULT_FILE_EXCLUDE_PATTERNS, recursive_subfolders: Optional[StrictBool] = False, image_quality_in_editor: Optional[str] = None, ): @@ -865,7 +872,7 @@ def upload_images_from_folder_to_project( if exclude_file_patterns: exclude_file_patterns = list(exclude_file_patterns) + list( - constances.DEFAULT_FILE_EXCLUDE_PATTERNS + constants.DEFAULT_FILE_EXCLUDE_PATTERNS ) exclude_file_patterns = list(set(exclude_file_patterns)) @@ -1021,12 +1028,12 @@ def prepare_export( folders = folder_names if not annotation_statuses: annotation_statuses = [ - constances.AnnotationStatus.NOT_STARTED.name, - constances.AnnotationStatus.IN_PROGRESS.name, - constances.AnnotationStatus.QUALITY_CHECK.name, - constances.AnnotationStatus.RETURNED.name, - constances.AnnotationStatus.COMPLETED.name, - constances.AnnotationStatus.SKIPPED.name, + constants.AnnotationStatus.NOT_STARTED.name, + constants.AnnotationStatus.IN_PROGRESS.name, + constants.AnnotationStatus.QUALITY_CHECK.name, + constants.AnnotationStatus.RETURNED.name, + constants.AnnotationStatus.COMPLETED.name, + constants.AnnotationStatus.SKIPPED.name, ] response = self.controller.prepare_export( project_name=project_name, @@ -1045,7 +1052,7 @@ def upload_videos_from_folder_to_project( folder_path: Union[NotEmptyStr, Path], extensions: Optional[ Union[Tuple[NotEmptyStr], List[NotEmptyStr]] - ] = constances.DEFAULT_VIDEO_EXTENSIONS, + ] = constants.DEFAULT_VIDEO_EXTENSIONS, exclude_file_patterns: Optional[List[NotEmptyStr]] = (), recursive_subfolders: Optional[StrictBool] = False, target_fps: Optional[int] = None, @@ -1532,8 +1539,8 @@ def upload_preannotations_from_folder_to_project( project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) if recursive_subfolders: @@ -1588,8 +1595,8 @@ def upload_image_annotations( project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) @@ -1616,7 +1623,7 @@ def upload_image_annotations( mask=mask, verbose=verbose, ) - if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: + if response.errors and not response.errors == constants.INVALID_JSON_MESSAGE: raise AppException(response.errors) def download_model(self, model: MLModel, output_dir: Union[str, Path]): @@ -1674,8 +1681,8 @@ def benchmark( project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) @@ -1826,8 +1833,8 @@ def add_annotation_bbox_to_image( project_name, folder_name = extract_project_folder(project) project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) response = self.controller.get_annotations( @@ -1884,8 +1891,8 @@ def add_annotation_point_to_image( project_name, folder_name = extract_project_folder(project) project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) response = self.controller.get_annotations( @@ -1939,8 +1946,8 @@ def add_annotation_comment_to_image( project_name, folder_name = extract_project_folder(project) project = self.controller.get_project_metadata(project_name).data if project["project"].type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, + constants.ProjectType.VIDEO.value, + constants.ProjectType.DOCUMENT.value, ]: raise AppException(LIMITED_FUNCTIONS[project["project"].type]) response = self.controller.get_annotations( @@ -2126,8 +2133,8 @@ def aggregate_annotations_as_df( :rtype: pandas DataFrame """ if project_type in ( - constances.ProjectType.VECTOR.name, - constances.ProjectType.PIXEL.name, + constants.ProjectType.VECTOR.name, + constants.ProjectType.PIXEL.name, ): from superannotate.lib.app.analytics.common import ( aggregate_image_annotations_as_df, @@ -2141,8 +2148,8 @@ def aggregate_annotations_as_df( folder_names=folder_names, ) elif project_type in ( - constances.ProjectType.VIDEO.name, - constances.ProjectType.DOCUMENT.name, + constants.ProjectType.VIDEO.name, + constants.ProjectType.DOCUMENT.name, ): from superannotate.lib.app.analytics.aggregators import DataAggregator @@ -2153,21 +2160,21 @@ def aggregate_annotations_as_df( ).aggregate_annotations_as_df() def delete_annotations( - self, project: NotEmptyStr, image_names: Optional[List[NotEmptyStr]] = None + self, project: NotEmptyStr, item_names: Optional[List[NotEmptyStr]] = None ): """ - Delete image annotations from a given list of images. + Delete item annotations from a given list of items. :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param image_names: image names. If None, all image annotations from a given project/folder will be deleted. - :type image_names: list of strs + :param item_names: image names. If None, all image annotations from a given project/folder will be deleted. + :type item_names: list of strs """ project_name, folder_name = extract_project_folder(project) response = self.controller.delete_annotations( - project_name=project_name, folder_name=folder_name, item_names=image_names + project_name=project_name, folder_name=folder_name, item_names=item_names ) if response.errors: raise AppException(response.errors) diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index b408c3f9a..780c88a83 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -14,6 +14,7 @@ from pydantic import BaseModel from pydantic import conlist from pydantic import constr +from pydantic import errors from pydantic import Extra from pydantic import Field from pydantic import parse_obj_as @@ -21,19 +22,18 @@ from pydantic import StrictStr from pydantic import validate_arguments as pydantic_validate_arguments from pydantic import ValidationError -from pydantic.errors import StrRegexError -from pydantic import errors from pydantic.errors import PydanticTypeError +from pydantic.errors import StrRegexError NotEmptyStr = constr(strict=True, min_length=1) class EnumMemberError(PydanticTypeError): - code = 'enum' + code = "enum" def __str__(self) -> str: - permitted = ', '.join(str(v.name) for v in self.enum_values) # type: ignore - return f'Available values are: {permitted}' + permitted = ", ".join(str(v.name) for v in self.enum_values) # type: ignore + return f"Available values are: {permitted}" errors.EnumMemberError = EnumMemberError diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index c30d67469..cd1c0345c 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -1,5 +1,4 @@ from os.path import expanduser -from pathlib import Path from superannotate.lib.core.enums import AnnotationStatus from superannotate.lib.core.enums import ImageQuality diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index 0fc7cc3a6..edefb6027 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -47,7 +47,7 @@ def equals(self, other: Enum): return self.__doc__.lower() == other.__doc__.lower() def __eq__(self, other): - return super(BaseTitledEnum, self).__eq__(other) + return super().__eq__(other) class AnnotationTypes(str, Enum): diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 1861e8ea1..67ffe5063 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -318,7 +318,10 @@ async def download_annotations( postfix: str, items: List[str] = None, callback: Callable = None, - ) -> List[dict]: + ) -> int: + """ + Returns the number of items downloaded + """ raise NotImplementedError def upload_priority_scores( diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index c553db8a5..2a12bdb56 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -809,14 +809,17 @@ def get_items_count(path: str): def coroutine_wrapper(coroutine): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.run_until_complete(coroutine) + count = loop.run_until_complete(coroutine) loop.close() + return count def execute(self): if self.is_valid(): export_path = str( self.destination - / Path(f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}") + / Path( + f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}" + ) ) self.reporter.log_info( f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" @@ -840,7 +843,7 @@ def execute(self): if not folders: loop = asyncio.new_event_loop() - loop.run_until_complete( + count = loop.run_until_complete( self._backend_client.download_annotations( team_id=self._project.team_id, project_id=self._project.id, @@ -868,12 +871,12 @@ def execute(self): callback=self._callback, ) ) - _ = [_ for _ in executor.map(self.coroutine_wrapper, coroutines)] + count = sum( + [i for i in executor.map(self.coroutine_wrapper, coroutines)] + ) self.reporter.stop_spinner() - self.reporter.log_info( - f"Downloaded annotations for {self.get_items_count(export_path)} items." - ) + self.reporter.log_info(f"Downloaded annotations for {count} items.") self.download_annotation_classes(export_path) self._response.data = os.path.abspath(export_path) return self._response diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 4ddc12351..e52e33797 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -1044,7 +1044,7 @@ async def download_annotations( postfix: str, items: List[str] = None, callback: Callable = None, - ) -> List[dict]: + ) -> int: import aiohttp async with aiohttp.ClientSession( diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 844c5fe36..675701501 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -109,7 +109,11 @@ async def download_data( method: str = "post", params=None, chunk_size: int = 100, - ): + ) -> int: + """ + Returns the number of items downloaded + """ + items_downloaded: int = 0 if chunk_size and data: for i in range(0, len(data), chunk_size): data_to_process = data[i : i + chunk_size] @@ -126,13 +130,13 @@ async def download_data( annotation, self._callback, ) + items_downloaded += 1 else: async for annotation in self.fetch( method, session, url, self._process_data(data), params=params ): self._store_annotation( - download_path, - postfix, - annotation, - self._callback + download_path, postfix, annotation, self._callback ) + items_downloaded += 1 + return items_downloaded diff --git a/tests/integration/annotations/test_download_annotations.py b/tests/integration/annotations/test_download_annotations.py index 1083d2c76..b32709515 100644 --- a/tests/integration/annotations/test_download_annotations.py +++ b/tests/integration/annotations/test_download_annotations.py @@ -60,11 +60,11 @@ def test_download_annotations_from_folders(self): f"{self.PROJECT_NAME}{'/' + folder if folder else ''}", self.folder_path ) with tempfile.TemporaryDirectory() as temp_dir: - annotations_path = sa.download_annotations(f"{self.PROJECT_NAME}", temp_dir) - self.assertEqual(len(os.listdir(annotations_path)), 5) + annotations_path = sa.download_annotations(f"{self.PROJECT_NAME}", temp_dir, recursive=True) + self.assertEqual(len(os.listdir(annotations_path)), 7) @pytest.mark.flaky(reruns=3) - def test_download_annotations_from_folders(self): + def test_download_empty_annotations_from_folders(self): sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME_2) sa.create_annotation_classes_from_classes_json( diff --git a/tests/integration/test_single_annotation_download.py b/tests/integration/test_single_annotation_download.py index fb00b50c5..6e2f4a3ab 100644 --- a/tests/integration/test_single_annotation_download.py +++ b/tests/integration/test_single_annotation_download.py @@ -1,15 +1,16 @@ - import filecmp import json import os import tempfile from os.path import dirname + import pytest from src.superannotate import SAClient -sa = SAClient() from tests.integration.base import BaseTestCase +sa = SAClient() + class TestSingleAnnotationDownloadUpload(BaseTestCase): PROJECT_NAME = "test_single_annotation" @@ -68,7 +69,7 @@ def test_annotation_download_upload_vector(self): ) ) # TODO: - #assert downloaded_json == uploaded_json + # assert downloaded_json == uploaded_json class TestSingleAnnotationDownloadUploadPixel(BaseTestCase): From 40f5f58eba533796183cce88aa74fb5ac153f2b8 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 8 Jun 2022 12:36:05 +0400 Subject: [PATCH 18/59] class BaseInterfaceFacade: --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index cd72ef523..611cf222f 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev11" +__version__ = "4.3.5dev12" From 0eb724d581df1386ee91abc4d645fc500a1b15ef Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 8 Jun 2022 15:12:04 +0400 Subject: [PATCH 19/59] Docstring fix --- src/superannotate/lib/app/interface/sdk_interface.py | 6 +++--- tests/integration/items/test_set_annotation_statuses.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index c2767823b..b4f110d03 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2669,7 +2669,7 @@ def set_annotation_statuses( self, project: Union[NotEmptyStr, dict], annotation_status: AnnotationStatuses, - item_names: Optional[List[NotEmptyStr]] = None, + items: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of items @@ -2686,7 +2686,7 @@ def set_annotation_statuses( :type annotation_status: str :param items: item names to set the mentioned status for. If None, all the items in the project will be used. - :type items: str + :type items: list of strs """ project_name, folder_name = extract_project_folder(project) @@ -2694,7 +2694,7 @@ def set_annotation_statuses( project_name=project_name, folder_name=folder_name, annotation_status=annotation_status, - item_names=item_names, + item_names=items, ) if response.errors: raise AppException(response.errors) diff --git a/tests/integration/items/test_set_annotation_statuses.py b/tests/integration/items/test_set_annotation_statuses.py index 1ab0423ce..c9574b6a8 100644 --- a/tests/integration/items/test_set_annotation_statuses.py +++ b/tests/integration/items/test_set_annotation_statuses.py @@ -65,7 +65,7 @@ def test_image_annotation_status_via_names(self): def test_image_annotation_status_via_invalid_names(self): sa.attach_items( - self.PROJECT_NAME, self.ATTACHMENT_LIST, annotation_status="InProgress" + self.PROJECT_NAME, "InProgress", self.ATTACHMENT_LIST ) with self.assertRaisesRegexp(AppException, SetAnnotationStatues.ERROR_MESSAGE): sa.set_annotation_statuses( From a36b443f85abd0b2f60d4b66df0bb5a5e1b5c619 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 9 Jun 2022 10:58:40 +0400 Subject: [PATCH 20/59] Interface fix --- src/superannotate/lib/app/interface/base_interface.py | 2 +- src/superannotate/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index e7c38e87a..0c2773cc5 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -41,7 +41,7 @@ def __init__(self, token: str = None, config_path: str = None): self.controller = Controller(_token, _host, _ssl_verify, version) def __new__(cls, *args, **kwargs): - obj = super().__new__(cls, *args, **kwargs) + obj = super().__new__(cls) cls.REGISTRY.append(obj) return obj diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 611cf222f..68c065807 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev12" +__version__ = "4.3.5dev13" From f3c3686e57565c1e856b4e1ae3a830107dad8b8a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 9 Jun 2022 15:12:37 +0400 Subject: [PATCH 21/59] Mixpanel fix --- .../lib/app/interface/base_interface.py | 39 +- src/superannotate/lib/app/mixp/__init__.py | 0 .../lib/app/mixp/utils/__init__.py | 0 .../lib/app/mixp/utils/parsers.py | 988 ------------------ .../lib/infrastructure/services.py | 2 +- .../lib/infrastructure/stream_data_handler.py | 3 +- src/superannotate/version.py | 2 +- 7 files changed, 22 insertions(+), 1012 deletions(-) delete mode 100644 src/superannotate/lib/app/mixp/__init__.py delete mode 100644 src/superannotate/lib/app/mixp/utils/__init__.py delete mode 100644 src/superannotate/lib/app/mixp/utils/parsers.py diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 0c2773cc5..2013db2a6 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -37,13 +37,9 @@ def __init__(self, token: str = None, config_path: str = None): _toke, _host, _ssl_verify = self._retrieve_configs( constants.CONFIG_PATH ) - self._token, self._host = _host, _token + self._token, self._host = _token, _host self.controller = Controller(_token, _host, _ssl_verify, version) - - def __new__(cls, *args, **kwargs): - obj = super().__new__(cls) - cls.REGISTRY.append(obj) - return obj + BaseInterfaceFacade.REGISTRY.append(self) @staticmethod def _retrieve_configs(path) -> Tuple[str, str, str]: @@ -51,8 +47,6 @@ def _retrieve_configs(path) -> Tuple[str, str, str]: if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): raise AppException( f"SuperAnnotate config file {str(config_path)} not found." - f" Please provide correct config file location to sa.init() or use " - f"CLI's superannotate init to generate default location config file." ) config_repo = ConfigRepository(config_path) return ( @@ -72,11 +66,12 @@ def token(self): class Tracker: def get_mp_instance(self) -> Mixpanel: - if self.client: - if self.client.host == constants.BACKEND_URL: - return Mixpanel("ca95ed96f80e8ec3be791e2d3097cf51") - else: - return Mixpanel("e741d4863e7e05b1a45833d01865ef0d") + client = self.get_client() + mp_token = "ca95ed96f80e8ec3be791e2d3097cf51" + if client: + if client.host != constants.BACKEND_URL: + mp_token = "e741d4863e7e05b1a45833d01865ef0d" + return Mixpanel(mp_token) @staticmethod def get_default_payload(team_name, user_id): @@ -92,8 +87,7 @@ def __init__(self, function): self._client = None functools.update_wrapper(self, function) - @property - def client(self): + def get_client(self): if not self._client: if BaseInterfaceFacade.REGISTRY: self._client = BaseInterfaceFacade.REGISTRY[-1] @@ -101,7 +95,8 @@ def client(self): from lib.app.interface.sdk_interface import SAClient self._client = SAClient() - return self._client + elif hasattr(self._client, "controller"): + return self._client @staticmethod def extract_arguments(function, *args, **kwargs) -> dict: @@ -138,12 +133,14 @@ def _track(self, user_id: str, event_name: str, data: dict): self.get_mp_instance().track(user_id, event_name, data) def _track_method(self, args, kwargs, success: bool): + client = self.get_client() + if not client: + return function_name = self.function.__name__ if self.function else "" arguments = self.extract_arguments(self.function, *args, **kwargs) event_name, properties = self.default_parser(function_name, arguments) - - user_id = self.client.controller.team_data.creator_id - team_name = self.client.controller.team_data.name + user_id = client.controller.team_data.creator_id + team_name = client.controller.team_data.name properties["Success"] = success default = self.get_default_payload(team_name=team_name, user_id=user_id) @@ -177,7 +174,9 @@ def __call__(self, *args, **kwargs): class TrackableMeta(type): def __new__(mcs, name, bases, attrs): for attr_name, attr_value in attrs.items(): - if isinstance(attr_value, FunctionType): + if isinstance( + attr_value, FunctionType + ) and not attr_value.__name__.startswith("_"): attrs[attr_name] = Tracker(validate_arguments(attr_value)) tmp = super().__new__(mcs, name, bases, attrs) return tmp diff --git a/src/superannotate/lib/app/mixp/__init__.py b/src/superannotate/lib/app/mixp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/superannotate/lib/app/mixp/utils/__init__.py b/src/superannotate/lib/app/mixp/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py deleted file mode 100644 index a67a85b40..000000000 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ /dev/null @@ -1,988 +0,0 @@ -import os -from pathlib import Path - -import lib.core as constances -from lib.app.helpers import extract_project_folder -from lib.core.entities import IntegrationEntity -from lib.core.enums import ProjectType -from lib.infrastructure.controller import Controller - - -def get_project_name(project): - project_name = "" - if isinstance(project, dict): - project_name = project["name"] - if isinstance(project, str): - if "/" in project: - project_name = project.split("/")[0] - else: - project_name = project - return project_name - - -def get_team_metadata(**kwargs): - return {"event_name": "get_team_metadata", "properties": {}} - - -def invite_contributors_to_team(**kwargs): - admin = kwargs.get("admin") - if not admin: - admin_value = False - else: - admin_value = admin - return { - "event_name": "invite_contributors_to_team", - "properties": {"Admin": admin_value}, - } - - -def search_team_contributors(**kwargs): - return { - "event_name": "search_team_contributors", - "properties": { - "Email": bool(kwargs.get("email")), - "Name": bool(kwargs.get("first_name")), - "Surname": bool(kwargs.get("last_name")), - }, - } - - -def search_projects(**kwargs): - project = kwargs.get("name") - status = kwargs.get("status") - return { - "event_name": "search_projects", - "properties": { - "Metadata": bool(kwargs.get("return_metadata")), - "project_name": get_project_name(project[0]) if project else None, - "status": status, - }, - } - - -def create_project(**kwargs): - project = kwargs["project_name"] - project_type = kwargs["project_type"] - return { - "event_name": "create_project", - "properties": { - "Project Type": project_type, - "project_name": get_project_name(project), - }, - } - - -def create_project_from_metadata(**kwargs): - project = kwargs.get("project_metadata") - - return { - "event_name": "create_project_from_metadata", - "properties": {"project_name": get_project_name(project)}, - } - - -def clone_project(**kwargs): - project = kwargs.get("project_name") - - project_metadata = ( - Controller.get_default().get_project_metadata(project).data["project"] - ) - project_type = ProjectType.get_name(project_metadata.type) - - return { - "event_name": "clone_project", - "properties": { - "External": bool( - project_metadata.upload_state == constances.UploadState.EXTERNAL.value - ), - "Project Type": project_type, - "Copy Classes": bool(kwargs.get("copy_annotation_classes")), - "Copy Settings": bool(kwargs.get("copy_settings")), - "Copy Workflow": bool(kwargs.get("copy_workflow")), - "Copy Contributors": bool(kwargs.get("copy_contributors")), - "project_name": get_project_name(project), - }, - } - - -def upload_images_to_project(**kwargs): - project = kwargs["project"] - - img_paths = kwargs.get("img_paths") - return { - "event_name": "upload_images_to_project", - "properties": { - "Image Count": len(img_paths) if img_paths else None, - "Annotation Status": bool(kwargs.get("annotation_status")), - "From S3": bool(kwargs.get("from_s3")), - "project_name": get_project_name(project), - }, - } - - -def upload_image_to_project(**kwargs): - project = kwargs["project"] - - return { - "event_name": "upload_image_to_project", - "properties": { - "Image Name": bool(kwargs.get("image_name")), - "Annotation Status": bool(kwargs.get("annotation_status")), - "project_name": get_project_name(project), - }, - } - - -def upload_video_to_project(**kwargs): - project = kwargs["project"] - - return { - "event_name": "upload_video_to_project", - "properties": { - "project_name": get_project_name(project), - "FPS": bool(kwargs.get("target_fps")), - "Start": bool(kwargs.get("start_time")), - "End": bool(kwargs.get("end_time")), - }, - } - - -def set_images_annotation_statuses(**kwargs): - project = kwargs["project"] - annotation_status = kwargs.get("annotation_status") - image_names = kwargs["image_names"] - return { - "event_name": "set_images_annotation_statuses", - "properties": { - "project_name": get_project_name(project), - "Image Count": len(image_names) if image_names else None, - "Annotation Status": annotation_status, - }, - } - - -def download_image_annotations(**kwargs): - project = kwargs["project"] - - return { - "event_name": "download_image_annotations", - "properties": {"project_name": get_project_name(project)}, - } - - -def add_annotation_comment_to_image(**kwargs): - project = kwargs["project"] - - return { - "event_name": "add_annotation_comment_to_image", - "properties": {"project_name": get_project_name(project)}, - } - - -def delete_annotation_class(**kwargs): - project = kwargs["project"] - - return { - "event_name": "delete_annotation_class", - "properties": {"project_name": get_project_name(project)}, - } - - -def download_annotation_classes_json(**kwargs): - project = kwargs["project"] - - return { - "event_name": "download_annotation_classes_json", - "properties": {"project_name": get_project_name(project)}, - } - - -def search_annotation_classes(**kwargs): - project = kwargs["project"] - name_contains = kwargs.get("name_contains") - - return { - "event_name": "search_annotation_classes", - "properties": { - "project_name": get_project_name(project), - "name_contains": bool(name_contains), - }, - } - - -def get_project_image_count(**kwargs): - project = kwargs["project"] - - return { - "event_name": "get_project_image_count", - "properties": {"project_name": get_project_name(project)}, - } - - -def get_project_settings(**kwargs): - project = kwargs["project"] - - return { - "event_name": "get_project_settings", - "properties": {"project_name": get_project_name(project)}, - } - - -def get_project_metadata(**kwargs): - project = kwargs["project"] - - return { - "event_name": "get_project_metadata", - "properties": {"project_name": get_project_name(project)}, - } - - -def delete_project(**kwargs): - project = kwargs["project"] - - return { - "event_name": "delete_project", - "properties": {"project_name": get_project_name(project)}, - } - - -def rename_project(**kwargs): - project = kwargs["project"] - return { - "event_name": "rename_project", - "properties": {"project_name": get_project_name(project)}, - } - - -def get_project_workflow(**kwargs): - project = kwargs["project"] - - return { - "event_name": "get_project_workflow", - "properties": {"project_name": get_project_name(project)}, - } - - -def set_project_workflow(**kwargs): - project = kwargs["project"] - return { - "event_name": "set_project_workflow", - "properties": {"project_name": get_project_name(project)}, - } - - -def create_folder(**kwargs): - project = kwargs["project"] - return { - "event_name": "create_folder", - "properties": {"project_name": get_project_name(project)}, - } - - -def get_folder_metadata(**kwargs): - project = kwargs["project"] - return { - "event_name": "get_folder_metadata", - "properties": {"project_name": get_project_name(project)}, - } - - -def download_model(**kwargs): - model = kwargs["model"] - return { - "event_name": "download_model", - "properties": {"model": model}, - } - - -def convert_project_type(**kwargs): - return { - "event_name": "convert_project_type", - "properties": {}, - } - - -def convert_json_version(**kwargs): - return { - "event_name": "convert_json_version", - "properties": {}, - } - - -def upload_image_annotations(**kwargs): - project = kwargs["project"] - return { - "event_name": "upload_image_annotations", - "properties": { - "project_name": get_project_name(project), - "Pixel": bool("mask" in kwargs), - }, - } - - -def download_image(**kwargs): - project = kwargs["project"] - return { - "event_name": "download_image", - "properties": { - "project_name": get_project_name(project), - "Download Annotations": bool("include_annotations" in kwargs), - "Download Fuse": bool("include_fuse" in kwargs), - "Download Overlay": bool("include_overlay" in kwargs), - }, - } - - -def copy_image(**kwargs): - project = kwargs["source_project"] - return { - "event_name": "copy_image", - "properties": { - "project_name": get_project_name(project), - "Copy Annotations": bool("include_annotations" in kwargs), - "Copy Annotation Status": bool("copy_annotation_status" in kwargs), - "Copy Pin": bool("copy_pin" in kwargs), - }, - } - - -def run_prediction(**kwargs): - project = kwargs["project"] - project_name = get_project_name(project) - res = Controller.get_default().get_project_metadata(project_name) - project_metadata = res.data["project"] - project_type = ProjectType.get_name(project_metadata.typy) - image_list = kwargs["images_list"] - return { - "event_name": "run_prediction", - "properties": { - "Project Type": project_type, - "Image Count": len(image_list) if image_list else None, - }, - } - - -def upload_videos_from_folder_to_project(**kwargs): - folder_path = kwargs["folder_path"] - glob_iterator = Path(folder_path).glob("*") - return { - "event_name": "upload_videos_from_folder_to_project", - "properties": {"Video Count": sum(1 for _ in glob_iterator)}, - } - - -def export_annotation(**kwargs): - dataset_format = kwargs["dataset_format"] - project_type = kwargs["project_type"] - if not project_type: - project_type = "Vector" - - task = kwargs.get("task") - if not task: - task = "object_detection" - return { - "event_name": "export_annotation", - "properties": { - "Format": dataset_format, - "Project Type": project_type, - "Task": task, - }, - } - - -def import_annotation(**kwargs): - dataset_format = kwargs["dataset_format"] - project_type = kwargs["project_type"] - if not project_type: - project_type = "Vector" - task = kwargs.get("task") - if not task: - task = "object_detection" - return { - "event_name": "import_annotation", - "properties": { - "Format": dataset_format, - "Project Type": project_type, - "Task": task, - }, - } - - -def consensus(**kwargs): - folder_names = kwargs["folder_names"] - image_list = kwargs["image_list"] - annot_type = kwargs.get("annot_type") - if not annot_type: - annot_type = "bbox" - show_plots = kwargs.get("show_plots") - if not show_plots: - show_plots = False - return { - "event_name": "consensus", - "properties": { - "Folder Count": len(folder_names), - "Image Count": len(image_list) if image_list else None, - "Annotation Type": annot_type, - "Plot": show_plots, - }, - } - - -def benchmark(**kwargs): - folder_names = kwargs.get("folder_names") - image_list = kwargs.get("image_list") - annot_type = kwargs.get("annot_type") - if not annot_type: - annot_type = "bbox" - show_plots = kwargs.get("show_plots") - if not show_plots: - show_plots = False - - return { - "event_name": "benchmark", - "properties": { - "Folder Count": len(folder_names) if folder_names else None, - "Image Count": len(image_list) if image_list else None, - "Annotation Type": annot_type, - "Plot": show_plots, - }, - } - - -def upload_annotations_from_folder_to_project(**kwargs): - project = kwargs["project"] - project_name = get_project_name(project) - res = Controller.get_default().get_project_metadata(project_name) - project_metadata = res.data["project"] - project_type = ProjectType.get_name(project_metadata.type) - - folder_path = kwargs["folder_path"] - glob_iterator = Path(folder_path).glob("*.json") - return { - "event_name": "upload_annotations_from_folder_to_project", - "properties": { - "Annotation Count": sum(1 for _ in glob_iterator), - "Project Type": project_type, - "From S3": bool("from_s3_bucket" in kwargs), - }, - } - - -def upload_preannotations_from_folder_to_project(**kwargs): - project = kwargs["project"] - - project_name = get_project_name(project) - res = Controller.get_default().get_project_metadata(project_name) - project_metadata = res.data["project"] - project_type = ProjectType.get_name(project_metadata.type) - folder_path = kwargs["folder_path"] - glob_iterator = Path(folder_path).glob("*.json") - return { - "event_name": "upload_preannotations_from_folder_to_project", - "properties": { - "Annotation Count": sum(1 for _ in glob_iterator), - "Project Type": project_type, - "From S3": bool("from_s3_bucket" in kwargs), - }, - } - - -def upload_images_from_folder_to_project(**kwargs): - folder_path = kwargs["folder_path"] - recursive_subfolders = kwargs["recursive_subfolders"] - extensions = kwargs["extensions"] - if not extensions: - extensions = constances.DEFAULT_IMAGE_EXTENSIONS - exclude_file_patterns = kwargs["exclude_file_patterns"] - if not exclude_file_patterns: - exclude_file_patterns = constances.DEFAULT_FILE_EXCLUDE_PATTERNS - - paths = [] - for extension in extensions: - if not recursive_subfolders: - paths += list(Path(folder_path).glob(f"*.{extension.lower()}")) - if os.name != "nt": - paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) - else: - paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) - if os.name != "nt": - paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) - - filtered_paths = [] - for path in paths: - not_in_exclude_list = [x not in Path(path).name for x in exclude_file_patterns] - if all(not_in_exclude_list): - filtered_paths.append(path) - - return { - "event_name": "upload_images_from_folder_to_project", - "properties": { - "Image Count": len(filtered_paths), - "Custom Extentions": bool(kwargs["extensions"]), - "Annotation Status": bool(kwargs.get("annotation_status")), - "From S3": bool(kwargs.get("from_s3_bucket")), - "Custom Exclude Patters": bool(kwargs["exclude_file_patterns"]), - }, - } - - -def prepare_export(**kwargs): - project = kwargs["project"] - return { - "event_name": "prepare_export", - "properties": { - "project_name": get_project_name(project), - "Folder Count": bool(kwargs.get("folder_names")), - "Annotation Statuses": bool(kwargs.get("annotation_statuses")), - "Include Fuse": bool(kwargs.get("include_fuse")), - "Only Pinned": bool(kwargs.get("only_pinned")), - }, - } - - -def download_export(**kwargs): - project = kwargs["project"] - - return { - "event_name": "download_export", - "properties": { - "project_name": get_project_name(project), - "to_s3_bucket": bool(kwargs.get("to_s3_bucket")), - }, - } - - -def assign_images(**kwargs): - project = kwargs["project"] - project_name, folder_name = extract_project_folder(project) - image_names = kwargs.get("image_names") - user = kwargs.get("user") - - contributors = ( - Controller.get_default() - .get_project_metadata(project_name=project_name, include_contributors=True) - .data["contributors"] - ) - contributor = None - for c in contributors: - if c["user_id"] == user: - contributor = c - user_role = "ADMIN" - if contributor["user_role"] == 3: - user_role = "ANNOTATOR" - if contributor["user_role"] == 4: - user_role = "QA" - is_root = True - if folder_name: - is_root = False - - return { - "event_name": "assign_images", - "properties": { - "project_name": project_name, - "Assign Folder": is_root, - "Image Count": len(image_names) if image_names else None, - "User Role": user_role, - }, - } - - -def pin_image(**kwargs): - project = kwargs["project"] - - return { - "event_name": "pin_image", - "properties": { - "project_name": get_project_name(project), - "Pin": bool("pin" in kwargs), - }, - } - - -def set_image_annotation_status(**kwargs): - project = kwargs["project"] - - return { - "event_name": "set_image_annotation_status", - "properties": { - "project_name": get_project_name(project), - "Annotation Status": bool("annotation_status" in kwargs), - }, - } - - -def add_annotation_bbox_to_image(**kwargs): - project = kwargs["project"] - - return { - "event_name": "add_annotation_bbox_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool("annotation_class_attributes" in kwargs), - "Error": bool("error" in kwargs), - }, - } - - -def add_annotation_point_to_image(**kwargs): - project = kwargs["project"] - - return { - "event_name": "add_annotation_point_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool("annotation_class_attributes" in kwargs), - "Error": bool("error" in kwargs), - }, - } - - -def create_annotation_class(**kwargs): - project = kwargs["project"] - class_type = kwargs.get("class_type") - - return { - "event_name": "create_annotation_class", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool("attribute_groups" in kwargs), - "class_type": class_type if class_type else "object", - }, - } - - -def create_annotation_classes_from_classes_json(**kwargs): - project = kwargs["project"] - return { - "event_name": "create_annotation_classes_from_classes_json", - "properties": { - "project_name": get_project_name(project), - "From S3": bool("from_s3_bucket" in kwargs), - }, - } - - -def class_distribution(**kwargs): - return { - "event_name": "class_distribution", - "properties": {"Plot": bool("visualize" in kwargs)}, - } - - -def set_project_default_image_quality_in_editor(**kwargs): - project = kwargs["project"] - - image_quality_in_editor = kwargs.get("image_quality_in_editor") - return { - "event_name": "set_project_default_image_quality_in_editor", - "properties": { - "project_name": get_project_name(project), - "Image Quality": image_quality_in_editor, - }, - } - - -def get_exports(**kwargs): - project = kwargs["project"] - - return { - "event_name": "get_exports", - "properties": { - "project_name": get_project_name(project), - "Metadata": bool("return_metadata" in kwargs), - }, - } - - -def search_folders(**kwargs): - project = kwargs["project"] - - return { - "event_name": "search_folders", - "properties": { - "project_name": get_project_name(project), - "Metadata": bool("return_metadata" in kwargs), - }, - } - - -def aggregate_annotations_as_df(**kwargs): - folder_names = kwargs.get("folder_names") - if not folder_names: - folder_names = [] - - project_type = kwargs["project_type"] - - return { - "event_name": "aggregate_annotations_as_df", - "properties": {"Folder Count": len(folder_names), "Project Type": project_type}, - } - - -def delete_folders(**kwargs): - project = kwargs["project"] - folder_names = kwargs.get("folder_names") - if not folder_names: - folder_names = [] - return { - "event_name": "delete_folders", - "properties": { - "project_name": get_project_name(project), - "Folder Count": len(folder_names), - }, - } - - -def delete_images(**kwargs): - project = kwargs["project"] - project_name, folder_name = extract_project_folder(project) - - image_names = kwargs.get("image_names", False) - if not image_names: - res = Controller.get_default().search_images(project_name, folder_name) - image_names = res.data - return { - "event_name": "delete_images", - "properties": {"project_name": project_name, "Image Count": len(image_names)}, - } - - -def unassign_folder(**kwargs): - return {"event_name": "unassign_folder", "properties": {}} - - -def assign_folder(**kwargs): - users = kwargs.get("users") - return {"event_name": "assign_folder", "properties": {"User Count": len(users)}} - - -def unassign_images(**kwargs): - image_names = kwargs.get("image_names") - - project = kwargs["project"] - - _, folder_name = extract_project_folder(project) - is_root = True - if folder_name: - is_root = False - - return { - "event_name": "unassign_images", - "properties": {"Assign Folder": is_root, "Image Count": len(image_names)}, - } - - -def delete_annotations(**kwargs): - return {"event_name": "delete_annotations", "properties": {}} - - -def validate_annotations(**kwargs): - project_type = kwargs["project_type"] - return { - "event_name": "validate_annotations", - "properties": {"Project Type": project_type}, - } - - -def add_contributors_to_project(**kwargs): - user_role = kwargs.get("role") - - return { - "event_name": "add_contributors_to_project", - "properties": {"User Role": user_role}, - } - - -def get_annotations(**kwargs): - project = kwargs["project"] - items = kwargs["items"] - - return { - "event_name": "get_annotations", - "properties": { - "Project": project, - "items_count": len(items) if items else None, - }, - } - - -def get_annotations_per_frame(**kwargs): - project = kwargs["project"] - fps = kwargs["fps"] - if not fps: - fps = 1 - return { - "event_name": "get_annotations_per_frame", - "properties": {"Project": project, "fps": fps}, - } - - -def upload_priority_scores(**kwargs): - scores = kwargs["scores"] - return { - "event_name": "upload_priority_scores", - "properties": {"Score Count": len(scores) if scores else None}, - } - - -def get_integrations(**kwargs): - return { - "event_name": "get_integrations", - "properties": {}, - } - - -def attach_items_from_integrated_storage(**kwargs): - project = kwargs.get("project") - project_name, _ = extract_project_folder(project) - integration = kwargs.get("integration") - folder_path = kwargs.get("folder_path") - - if isinstance(integration, str): - integration = IntegrationEntity(name=integration) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - return { - "event_name": "attach_items_from_integrated_storage", - "properties": { - "project_type": ProjectType.get_name(project.type), - "integration_name": integration.name, - "folder_path": bool(folder_path), - }, - } - - -def query(**kwargs): - project = kwargs["project"] - query_str = kwargs["query"] - project_name, folder_name = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - return { - "event_name": "query", - "properties": { - "project_type": ProjectType.get_name(project.type), - "query": query_str, - }, - } - - -def get_item_metadata(**kwargs): - project = kwargs["project"] - project_name, _ = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - return { - "event_name": "get_item_metadata", - "properties": {"project_type": ProjectType.get_name(project.type)}, - } - - -def search_items(**kwargs): - project = kwargs["project"] - name_contains = kwargs["name_contains"] - annotation_status = kwargs["annotation_status"] - annotator_email = kwargs["annotator_email"] - qa_email = kwargs["qa_email"] - recursive = kwargs["recursive"] - project_name, folder_name = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - return { - "event_name": "search_items", - "properties": { - "project_type": ProjectType.get_name(project.type), - "query": query, - "name_contains": len(name_contains) if name_contains else False, - "annotation_status": annotation_status if annotation_status else False, - "annotator_email": bool(annotator_email), - "qa_email": bool(qa_email), - "recursive": bool(recursive), - }, - } - - -def move_items(**kwargs): - project = kwargs["source"] - project_name, _ = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - items = kwargs["items"] - return { - "event_name": "move_items", - "properties": { - "project_type": ProjectType.get_name(project.type), - "items_count": len(items) if items else None, - }, - } - - -def copy_items(**kwargs): - project = kwargs["source"] - project_name, _ = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - items = kwargs["items"] - return { - "event_name": "copy_items", - "properties": { - "project_type": ProjectType.get_name(project.type), - "items_count": len(items) if items else None, - "include_annotations": kwargs["include_annotations"], - }, - } - - -def attach_items(**kwargs): - project = kwargs["project"] - project_name, _ = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - attachments = kwargs["attachments"] - return { - "event_name": "attach_items", - "properties": { - "project_type": ProjectType.get_name(project.type), - "attachments": "csv" if isinstance(attachments, (str, Path)) else "dict", - "annotation_status": kwargs["annotation_status"], - }, - } - - -def set_annotation_statuses(**kwargs): - project = kwargs["project"] - project_name, folder_name = extract_project_folder(project) - return { - "event_name": "set_annotation_statuses", - "properties": { - "item_count": len(kwargs.get("item_names", [])), - "annotation_status": kwargs["annotation_status"], - "root": folder_name == "root", - }, - } - - -def download_annotations(**kwargs): - project = kwargs["project"] - project_name, folder_name = extract_project_folder(project) - project = ( - Controller.get_default().get_project_metadata(project_name).data["project"] - ) - return { - "event_name": "download_annotations", - "properties": { - "project_name": project_name, - "project_type": ProjectType.get_name(project.type), - "root": bool(folder_name), - "recursive": kwargs["recursive"], - "path": bool(kwargs["path"]), - "callback": bool(kwargs["callback"]), - }, - } diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 8297f9ada..3e6e7406e 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -187,7 +187,7 @@ class SuperannotateBackendService(BaseBackendService): Manage projects, images and team in the Superannotate """ - DEFAULT_CHUNK_SIZE = 1000 + DEFAULT_CHUNK_SIZE = 5000 URL_USERS = "users" URL_LIST_PROJECTS = "projects" diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 675701501..717adb257 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -61,7 +61,6 @@ async def get_data( verify_ssl: bool = False, ): async with aiohttp.ClientSession( - raise_for_status=True, headers=self._headers, connector=aiohttp.TCPConnector(ssl=verify_ssl), ) as session: @@ -108,7 +107,7 @@ async def download_data( session, method: str = "post", params=None, - chunk_size: int = 100, + chunk_size: int = 5000, ) -> int: """ Returns the number of items downloaded diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 68c065807..ab247526f 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev13" +__version__ = "4.3.5dev14" From 2ea55d06442d182d5c9f9df81b326b47198f6a15 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Thu, 16 Jun 2022 11:03:33 +0400 Subject: [PATCH 22/59] fix set_statuses --- src/superannotate/lib/core/usecases/items.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 8d580e6fe..10ff35a02 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -600,12 +600,17 @@ def validate_items(self): ) self._item_names = [item.name for item in self._items.get_all(condition)] return - existing_items = self._backend_service.get_bulk_images( - project_id=self._project.id, - team_id=self._project.team_id, - folder_id=self._folder.uuid, - images=self._item_names, - ) + existing_items = [] + for i in range(0, len(self._item_names), self.CHUNK_SIZE): + + search_names = self._item_names[i:i+self.CHUNK_SIZE] + existing_items += self._backend_service.get_bulk_images( + project_id=self._project.id, + team_id=self._project.team_id, + folder_id=self._folder.uuid, + images=search_names, + ) + if not existing_items: raise AppValidationException(self.ERROR_MESSAGE) if existing_items: From 1bd18ff433c175116c1d838a645dae00f180ed52 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Thu, 16 Jun 2022 17:36:58 +0400 Subject: [PATCH 23/59] fixed chunk size and search by chunks for copy_items and set_annotation_statuses + some formatting by black --- .../lib/app/analytics/aggregators.py | 2 +- .../lib/app/interface/sdk_interface.py | 4 +- src/superannotate/lib/core/config.py | 3 +- src/superannotate/lib/core/usecases/items.py | 47 +++++++++++++------ .../lib/infrastructure/services.py | 1 + 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index 9e92c4a00..9df04d7f0 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -1,5 +1,6 @@ import copy import json +from dataclasses import dataclass from pathlib import Path from typing import List from typing import Optional @@ -7,7 +8,6 @@ import lib.core as constances import pandas as pd -from dataclasses import dataclass from lib.app.exceptions import AppException from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from lib.core import PIXEL_ANNOTATION_POSTFIX diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index c2767823b..3a0a43d91 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -798,7 +798,7 @@ def unassign_images( warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) - response = self.controller.un_assign_items( + response = self.controller.un_assign_images( project_name=project_name, folder_name=folder_name, image_names=image_names ) if response.errors: @@ -2698,6 +2698,8 @@ def set_annotation_statuses( ) if response.errors: raise AppException(response.errors) + else: + logger.info("Annotation statuses of items changed") return response.data def download_annotations( diff --git a/src/superannotate/lib/core/config.py b/src/superannotate/lib/core/config.py index e5d955e12..a09c7aef8 100644 --- a/src/superannotate/lib/core/config.py +++ b/src/superannotate/lib/core/config.py @@ -1,8 +1,7 @@ import threading -from typing import Dict - from dataclasses import dataclass from dataclasses import field +from typing import Dict class Session: diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 10ff35a02..b6568094e 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -380,7 +380,7 @@ class CopyItems(BaseReportableUseCase): Return skipped item names. """ - CHUNK_SIZE = 1000 + CHUNK_SIZE = 500 def __init__( self, @@ -431,12 +431,18 @@ def execute(self): ) items = [item.name for item in self._items.get_all(condition)] - existing_items = self._backend_service.get_bulk_images( - project_id=self._project.id, - team_id=self._project.team_id, - folder_id=self._to_folder.uuid, - images=items, - ) + existing_items = [] + for i in range(0, len(items), self.CHUNK_SIZE): + cand_items = self._backend_service.get_bulk_images( + project_id=self._project.id, + team_id=self._project.team_id, + folder_id=self._to_folder.uuid, + images=items[i : i + self.CHUNK_SIZE], + ) + if isinstance(cand_items, dict): + continue + existing_items += cand_items + duplications = [item["name"] for item in existing_items] items_to_copy = list(set(items) - set(duplications)) skipped_items = duplications @@ -471,12 +477,19 @@ def execute(self): except BackendError as e: self._response.errors = AppException(e) return self._response - existing_items = self._backend_service.get_bulk_images( - project_id=self._project.id, - team_id=self._project.team_id, - folder_id=self._to_folder.uuid, - images=items, - ) + + existing_items = [] + for i in range(0, len(items), self.CHUNK_SIZE): + cand_items = self._backend_service.get_bulk_images( + project_id=self._project.id, + team_id=self._project.team_id, + folder_id=self._to_folder.uuid, + images=items[i : i + self.CHUNK_SIZE], + ) + if isinstance(cand_items, dict): + continue + existing_items += cand_items + existing_item_names_set = {item["name"] for item in existing_items} items_to_copy_names_set = set(items_to_copy) copied_items = existing_item_names_set.intersection( @@ -603,14 +616,18 @@ def validate_items(self): existing_items = [] for i in range(0, len(self._item_names), self.CHUNK_SIZE): - search_names = self._item_names[i:i+self.CHUNK_SIZE] - existing_items += self._backend_service.get_bulk_images( + search_names = self._item_names[i : i + self.CHUNK_SIZE] + cand_items = self._backend_service.get_bulk_images( project_id=self._project.id, team_id=self._project.team_id, folder_id=self._folder.uuid, images=search_names, ) + if isinstance(cand_items, dict): + continue + existing_items += cand_items + if not existing_items: raise AppValidationException(self.ERROR_MESSAGE) if existing_items: diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 3e6e7406e..45fcd2ea9 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -693,6 +693,7 @@ def set_images_statuses_bulk( def get_bulk_images( self, project_id: int, team_id: int, folder_id: int, images: List[str] ) -> List[dict]: + bulk_get_images_url = urljoin(self.api_url, self.URL_BULK_GET_IMAGES) res = self._request( bulk_get_images_url, From 7f7cab84aa8259e89b46093be593cb62174c9607 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Mon, 20 Jun 2022 13:01:33 +0400 Subject: [PATCH 24/59] log fixes --- .../lib/app/interface/sdk_interface.py | 4 +-- src/superannotate/lib/core/usecases/items.py | 29 ++++++++----------- .../lib/infrastructure/services.py | 10 +++---- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 3a0a43d91..210bba051 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -692,10 +692,10 @@ def assign_items( project_name, folder_name = extract_project_folder(project) - response = self.controller.assign_items(project_name, folder_name, items, user) + response, cnt_assigned = self.controller.assign_items(project_name, folder_name, items, user) if not response.errors: - logger.info(f"Assign items to user {user}") + logger.info(f"Assigned {cnt_assigned}/{len(items)} items to user {user}") else: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index b6568094e..e1e29e7fe 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -210,34 +210,29 @@ def __init__( self._user = user self._service = service - def validate_user( - self, - ): - for c in self._project.users: - if c["user_id"] == self._user: - return True - - raise AppValidationException( - f"{self._user} is not a verified contributor for the {self._project.name}" - ) + def validate_item_names(self, ): + self._item_names = list(set(self._item_names)) def execute(self): + cnt_assigned = 0 if self.is_valid(): for i in range(0, len(self._item_names), self.CHUNK_SIZE): - is_assigned = self._service.assign_items( + status, response = self._service.assign_items( team_id=self._project.team_id, project_id=self._project.id, folder_name=self._folder.name, user=self._user, item_names=self._item_names[i : i + self.CHUNK_SIZE], # noqa: E203 ) - if not is_assigned: - self._response.errors = AppException( - f"Cant assign {', '.join(self._item_names[i: i + self.CHUNK_SIZE])}" - ) - continue - return self._response + if status == 406 and 'error' in response: # User not found + self._response.errors+=response['error'] + return self._response, 0 + elif 'error' in response: + response['successCount'] = 0 + + cnt_assigned+=response['successCount'] + return self._response, cnt_assigned class UnAssignItemsUseCase(BaseUseCase): diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 45fcd2ea9..92d2ec459 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -134,10 +134,10 @@ def _request( if response.status_code == 404 and retried < 3: return self._request( url, - method="get", - data=None, - headers=None, - params=None, + method=method, + data=data, + headers=headers, + params=params, retried=retried + 1, content_type=content_type, ) @@ -777,7 +777,7 @@ def assign_items( "folder_name": folder_name, }, ) - return res.ok + return res.status_code, res.json() def un_assign_items( self, From f48a5d1666401d9f18662594d5218956dd789f3a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 8 Jun 2022 15:12:04 +0400 Subject: [PATCH 25/59] Docstring fix --- src/superannotate/lib/app/analytics/aggregators.py | 2 +- src/superannotate/lib/app/interface/base_interface.py | 9 ++++++--- src/superannotate/lib/app/interface/sdk_interface.py | 8 ++++---- src/superannotate/lib/core/config.py | 3 ++- tests/integration/items/test_set_annotation_statuses.py | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index 9df04d7f0..9e92c4a00 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -1,6 +1,5 @@ import copy import json -from dataclasses import dataclass from pathlib import Path from typing import List from typing import Optional @@ -8,6 +7,7 @@ import lib.core as constances import pandas as pd +from dataclasses import dataclass from lib.app.exceptions import AppException from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from lib.core import PIXEL_ANNOTATION_POSTFIX diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 2013db2a6..e29e6a97b 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -34,7 +34,7 @@ def __init__(self, token: str = None, config_path: str = None): else: _token = os.environ.get("SA_TOKEN") if not _token: - _toke, _host, _ssl_verify = self._retrieve_configs( + _token, _host, _ssl_verify = self._retrieve_configs( constants.CONFIG_PATH ) self._token, self._host = _token, _host @@ -90,11 +90,14 @@ def __init__(self, function): def get_client(self): if not self._client: if BaseInterfaceFacade.REGISTRY: - self._client = BaseInterfaceFacade.REGISTRY[-1] + return BaseInterfaceFacade.REGISTRY[-1] else: from lib.app.interface.sdk_interface import SAClient - self._client = SAClient() + try: + return SAClient() + except Exception: + pass elif hasattr(self._client, "controller"): return self._client diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 3a0a43d91..a614e478d 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -65,7 +65,7 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta): def __init__( self, token: str = None, - config_path: str = constants.CONFIG_PATH, + config_path: str = None, ): super().__init__(token, config_path) @@ -2669,7 +2669,7 @@ def set_annotation_statuses( self, project: Union[NotEmptyStr, dict], annotation_status: AnnotationStatuses, - item_names: Optional[List[NotEmptyStr]] = None, + items: Optional[List[NotEmptyStr]] = None, ): """Sets annotation statuses of items @@ -2686,7 +2686,7 @@ def set_annotation_statuses( :type annotation_status: str :param items: item names to set the mentioned status for. If None, all the items in the project will be used. - :type items: str + :type items: list of strs """ project_name, folder_name = extract_project_folder(project) @@ -2694,7 +2694,7 @@ def set_annotation_statuses( project_name=project_name, folder_name=folder_name, annotation_status=annotation_status, - item_names=item_names, + item_names=items, ) if response.errors: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/config.py b/src/superannotate/lib/core/config.py index a09c7aef8..e5d955e12 100644 --- a/src/superannotate/lib/core/config.py +++ b/src/superannotate/lib/core/config.py @@ -1,7 +1,8 @@ import threading +from typing import Dict + from dataclasses import dataclass from dataclasses import field -from typing import Dict class Session: diff --git a/tests/integration/items/test_set_annotation_statuses.py b/tests/integration/items/test_set_annotation_statuses.py index 1ab0423ce..6950c6012 100644 --- a/tests/integration/items/test_set_annotation_statuses.py +++ b/tests/integration/items/test_set_annotation_statuses.py @@ -65,7 +65,7 @@ def test_image_annotation_status_via_names(self): def test_image_annotation_status_via_invalid_names(self): sa.attach_items( - self.PROJECT_NAME, self.ATTACHMENT_LIST, annotation_status="InProgress" + self.PROJECT_NAME, self.ATTACHMENT_LIST, "InProgress" ) with self.assertRaisesRegexp(AppException, SetAnnotationStatues.ERROR_MESSAGE): sa.set_annotation_statuses( From 163b8055c9990fafe08feeef94b480ddcfec38be Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 20 Jun 2022 14:35:45 +0400 Subject: [PATCH 26/59] added ServiceResponse serialization --- .../lib/app/analytics/aggregators.py | 2 +- .../lib/app/interface/sdk_interface.py | 12 +++++------ src/superannotate/lib/core/config.py | 3 ++- src/superannotate/lib/core/service_types.py | 2 +- .../lib/core/serviceproviders.py | 2 +- src/superannotate/lib/core/usecases/items.py | 21 ++++++++++--------- .../lib/infrastructure/services.py | 6 +++--- 7 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index 9df04d7f0..9e92c4a00 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -1,6 +1,5 @@ import copy import json -from dataclasses import dataclass from pathlib import Path from typing import List from typing import Optional @@ -8,6 +7,7 @@ import lib.core as constances import pandas as pd +from dataclasses import dataclass from lib.app.exceptions import AppException from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from lib.core import PIXEL_ANNOTATION_POSTFIX diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 2198c15ea..8be2bf30b 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -684,19 +684,19 @@ def assign_items( :param project: project name or folder path (e.g., "project1/folder1") :type project: str + :param items: list of items to assign - :type item_names: list of str + :type items: list of str + :param user: user email :type user: str """ project_name, folder_name = extract_project_folder(project) - response, cnt_assigned = self.controller.assign_items(project_name, folder_name, items, user) + response = self.controller.assign_items(project_name, folder_name, items, user) - if not response.errors: - logger.info(f"Assigned {cnt_assigned}/{len(items)} items to user {user}") - else: + if response.errors: raise AppException(response.errors) def unassign_items( @@ -709,7 +709,7 @@ def unassign_items( :param project: project name or folder path (e.g., "project1/folder1") :type project: str :param items: list of items to unassign - :type item_names: list of str + :type items: list of str """ project_name, folder_name = extract_project_folder(project) diff --git a/src/superannotate/lib/core/config.py b/src/superannotate/lib/core/config.py index a09c7aef8..e5d955e12 100644 --- a/src/superannotate/lib/core/config.py +++ b/src/superannotate/lib/core/config.py @@ -1,7 +1,8 @@ import threading +from typing import Dict + from dataclasses import dataclass from dataclasses import field -from typing import Dict class Session: diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index f793234f3..1f4189201 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -80,7 +80,7 @@ def __init__(self, response, content_type=None): "content": response.content, } if response.ok: - if content_type: + if content_type and content_type != self.__class__: data["data"] = content_type(**response.json()) else: data["data"] = response.json() diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 7a70f2fe1..2276bf447 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -182,7 +182,7 @@ def assign_items( folder_name: str, user: str, item_names: list, - ): + ) -> ServiceResponse: raise NotImplementedError def get_bulk_images( diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index e1e29e7fe..a29ede6fd 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -22,7 +22,6 @@ from lib.core.usecases.base import BaseUseCase from superannotate.logger import get_default_logger - logger = get_default_logger() @@ -210,28 +209,30 @@ def __init__( self._user = user self._service = service - - def validate_item_names(self, ): + def validate_item_names( + self, + ): self._item_names = list(set(self._item_names)) def execute(self): cnt_assigned = 0 if self.is_valid(): for i in range(0, len(self._item_names), self.CHUNK_SIZE): - status, response = self._service.assign_items( + response = self._service.assign_items( team_id=self._project.team_id, project_id=self._project.id, folder_name=self._folder.name, user=self._user, item_names=self._item_names[i : i + self.CHUNK_SIZE], # noqa: E203 ) - if status == 406 and 'error' in response: # User not found - self._response.errors+=response['error'] - return self._response, 0 - elif 'error' in response: - response['successCount'] = 0 + if not response.ok and "error" in response: # User not found + self._response.errors += response.error + break - cnt_assigned+=response['successCount'] + cnt_assigned += response.data["successCount"] + logger.info( + f"Assigned {cnt_assigned}/{len(self._item_names)} items to user {self._user}" + ) return self._response, cnt_assigned diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 92d2ec459..278087c1f 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -765,9 +765,9 @@ def assign_items( folder_name: str, user: str, item_names: list, - ): + ) -> ServiceResponse: assign_items_url = urljoin(self.api_url, self.URL_ASSIGN_ITEMS) - res = self._request( + return self._request( assign_items_url, "put", params={"team_id": team_id, "project_id": project_id}, @@ -776,8 +776,8 @@ def assign_items( "assign_user_id": user, "folder_name": folder_name, }, + content_type=ServiceResponse, ) - return res.status_code, res.json() def un_assign_items( self, From 8a9ab3c46f9abc7c085a8085b3a77a6db60632af Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:53:17 +0400 Subject: [PATCH 27/59] Update service_types.py --- src/superannotate/lib/core/service_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index 1f4189201..2a8d890a1 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -80,7 +80,7 @@ def __init__(self, response, content_type=None): "content": response.content, } if response.ok: - if content_type and content_type != self.__class__: + if content_type and content_type not is self.__class__: data["data"] = content_type(**response.json()) else: data["data"] = response.json() From 1cf9bbf6b44fd63d82a1588c3171886444d7f282 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:53:39 +0400 Subject: [PATCH 28/59] Update service_types.py --- src/superannotate/lib/core/service_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index 2a8d890a1..664846217 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -80,7 +80,7 @@ def __init__(self, response, content_type=None): "content": response.content, } if response.ok: - if content_type and content_type not is self.__class__: + if content_type and not content_type is self.__class__: data["data"] = content_type(**response.json()) else: data["data"] = response.json() From 69b9fae44d516856d7e24967bb1e7511608fa5b1 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Tue, 21 Jun 2022 12:31:15 +0400 Subject: [PATCH 29/59] in progress changes --- .../lib/app/interface/sdk_interface.py | 46 ++++++++++++++++--- src/superannotate/lib/core/service_types.py | 5 +- .../lib/infrastructure/controller.py | 17 +++++++ .../lib/infrastructure/services.py | 12 +++++ 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index a614e478d..0d2b02e20 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -650,28 +650,62 @@ def set_images_annotation_statuses( logger.info("Annotations status of images changed") def delete_images( - self, project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None + self, project: Union[NotEmptyStr, dict], image_names: Optional[list[str]] = None ): - """Delete images in project. + """delete images in project. :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param image_names: to be deleted images' names. If None, all the images will be deleted + :param image_names: to be deleted images' names. if none, all the images will be deleted :type image_names: list of strs """ + + warning_msg = ( + "We're deprecating the delete_images function. Please use delete_items instead." + "Learn more. \n" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.assign_items" + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) - if not isinstance(image_names, list) and image_names is not None: - raise AppException("Image_names should be a list of str or None.") + if not isinstance(image_names, list) and image_names is not none: + raise appexception("image_names should be a list of str or none.") response = self.controller.delete_images( project_name=project_name, folder_name=folder_name, image_names=image_names ) + if response.errors: + raise appexception(response.errors) + + logger.info( + f"images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" + ) + + def delete_items( + self, project: str, items: Optional[List[str]] = None + ): + """Delete items in a given project. + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param items: to be deleted items' names. If None, all the items will be deleted + :type items: list of str + """ + project_name, folder_name = extract_project_folder(project) + + ## Type checking should be done in controller or by PyDantic? + # if not isinstance(image_names, list) and image_names is not None: + # raise AppException("Image_names should be a list of str or None.") + + response = self.controller.delete_items( + project_name=project_name, folder_name=folder_name, items=items + ) if response.errors: raise AppException(response.errors) logger.info( - f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" + f"Items deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) def assign_items( diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index f793234f3..c7eb5e851 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -80,7 +80,7 @@ def __init__(self, response, content_type=None): "content": response.content, } if response.ok: - if content_type: + if content_type and content_type is not self.__class__: data["data"] = content_type(**response.json()) else: data["data"] = response.json() @@ -92,4 +92,5 @@ def ok(self): @property def error(self): - return getattr(self.data, "error", "Unknown error.") + default_message = self.data["reason"] if self.data["reason"] else "Unknown error" + return getattr(self.data, "error", default_message) diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 7f4f97b06..6a7591d47 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -835,6 +835,23 @@ def delete_images( ) return use_case.execute() + def delete_items( + self, + project_name: str, + folder_name: str, + items: List[str] = None, + ): + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + + use_case = usecases.DeleteItemsUseCase( + project=project, + folder=folder, + items=items, + backend_service_provider=self._backend_client, + ) + return use_case.execute() + def assign_items( self, project_name: str, folder_name: str, item_names: list, user: str ): diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 45fcd2ea9..da52977fb 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -202,6 +202,7 @@ class SuperannotateBackendService(BaseBackendService): URL_GET_ITEMS = "items" URL_BULK_GET_IMAGES = "images/getBulk" URL_DELETE_FOLDERS = "image/delete/images" + URL_DELETE_ITEMS = "image/delete/images" URL_CREATE_IMAGE = "image/ext-create" URL_PROJECT_SETTINGS = "project/{}/settings" URL_PROJECT_WORKFLOW = "project/{}/workflow" @@ -717,6 +718,17 @@ def delete_images(self, project_id: int, team_id: int, image_ids: List[int]): ) return res.json() + def delete_items(self, project_id: int, team_id: int, item_ids: List[int]): + delete_items_url = urljoin(self.api_url, self.URL_DELETE_ITEMS) + res = self._request( + delete_items_url, + "put", + params={"team_id": team_id, "project_id": project_id}, + data={"image_ids": item_ids}, + ) + + return ServiceResponse(res, ServiceResponse) + def assign_images( self, team_id: int, From 952aeb31d71e0f145d09fdb5ae18dd1f1a02455f Mon Sep 17 00:00:00 2001 From: VavoTK Date: Tue, 21 Jun 2022 12:35:57 +0400 Subject: [PATCH 30/59] added 'not' and returning reason if it exists and error does not exist in data --- src/superannotate/lib/core/service_types.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index 664846217..c01bf18cd 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -80,7 +80,7 @@ def __init__(self, response, content_type=None): "content": response.content, } if response.ok: - if content_type and not content_type is self.__class__: + if content_type and not content_type is not self.__class__: data["data"] = content_type(**response.json()) else: data["data"] = response.json() @@ -92,4 +92,5 @@ def ok(self): @property def error(self): - return getattr(self.data, "error", "Unknown error.") + default_message = self.reason if self.reason else "Unknown Error" + return getattr(self.data, "error", default_message) From b453be37caa87cc5603d92ad4b5dd2d34d6d901f Mon Sep 17 00:00:00 2001 From: VavoTK Date: Tue, 21 Jun 2022 15:37:39 +0400 Subject: [PATCH 31/59] using ServiceType, fixing some stuff --- src/superannotate/lib/core/service_types.py | 11 ++++++++--- src/superannotate/lib/core/usecases/items.py | 10 ++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index c01bf18cd..1a4ad67ce 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -79,11 +79,13 @@ def __init__(self, response, content_type=None): "reason": response.reason, "content": response.content, } - if response.ok: - if content_type and not content_type is not self.__class__: + try: + if content_type and content_type is not self.__class__: data["data"] = content_type(**response.json()) else: data["data"] = response.json() + except Exception as e: + data["data"] = {} super().__init__(**data) @property @@ -93,4 +95,7 @@ def ok(self): @property def error(self): default_message = self.reason if self.reason else "Unknown Error" - return getattr(self.data, "error", default_message) + if isinstance(self.data, dict): + return self.data.get("error", default_message) + else: + return getattr(self.data, "error", default_message) diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index a29ede6fd..73f80fd81 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -216,6 +216,7 @@ def validate_item_names( def execute(self): cnt_assigned = 0 + total_count = len(self._item_names) if self.is_valid(): for i in range(0, len(self._item_names), self.CHUNK_SIZE): response = self._service.assign_items( @@ -225,15 +226,16 @@ def execute(self): user=self._user, item_names=self._item_names[i : i + self.CHUNK_SIZE], # noqa: E203 ) - if not response.ok and "error" in response: # User not found + if not response.ok and response.error: # User not found + print(response.error) self._response.errors += response.error - break + return self._response cnt_assigned += response.data["successCount"] logger.info( - f"Assigned {cnt_assigned}/{len(self._item_names)} items to user {self._user}" + f"Assigned {cnt_assigned}/{total_count} items to user {self._user}" ) - return self._response, cnt_assigned + return self._response class UnAssignItemsUseCase(BaseUseCase): From 8b0b4387153c95b1b00020d5ac6d3067b2379a3d Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:47:54 +0400 Subject: [PATCH 32/59] Update items.py --- src/superannotate/lib/core/usecases/items.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 73f80fd81..b30e73ef2 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -227,7 +227,6 @@ def execute(self): item_names=self._item_names[i : i + self.CHUNK_SIZE], # noqa: E203 ) if not response.ok and response.error: # User not found - print(response.error) self._response.errors += response.error return self._response From 3c85504b24c5966973305eb6496aa23853107f1d Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 21 Jun 2022 17:33:45 +0400 Subject: [PATCH 33/59] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index ab247526f..30c3fa299 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev14" +__version__ = "4.3.5dev15" From 1c74f5cce2df8eb1fbf51bda97ed26f480c5ef0b Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 22 Jun 2022 10:52:59 +0400 Subject: [PATCH 34/59] initial commit --- docs/source/superannotate.sdk.rst | 9 +++++ .../lib/app/interface/sdk_interface.py | 16 +++++++++ src/superannotate/lib/core/entities/base.py | 7 ++++ src/superannotate/lib/core/service_types.py | 3 +- .../lib/core/serviceproviders.py | 3 ++ .../lib/core/usecases/projects.py | 36 +++++++++++++++++++ .../lib/infrastructure/controller.py | 9 +++++ .../lib/infrastructure/services.py | 10 ++++++ tests/integration/test_benchmark.py | 2 ++ .../integration/test_create_from_full_info.py | 0 10 files changed, 94 insertions(+), 1 deletion(-) delete mode 100644 tests/integration/test_create_from_full_info.py diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 3d2888431..3f1474198 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -73,11 +73,20 @@ ______ .. automethod:: superannotate.SAClient.attach_items .. automethod:: superannotate.SAClient.copy_items .. automethod:: superannotate.SAClient.move_items +.. automethod:: superannotate.SAClient.assign_items +.. automethod:: superannotate.SAClient.unassign_items .. automethod:: superannotate.SAClient.get_item_metadata .. automethod:: superannotate.SAClient.set_annotation_statuses ---------- +Subsets +______ + +.. automethod:: superannotate.SAClient.get_subsets + +---------- + Images ______ diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 1aa2de52c..76b450c37 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2744,3 +2744,19 @@ def download_annotations( if response.errors: raise AppException(response.errors) return response.data + + def get_subsets(self, project: Union[NotEmptyStr, dict]): + """Get Subsets + + :param project: project name (e.g., “project1”) + :type project: str + + :return: subsets’ metadata + :rtype: list of dicts + """ + project_name, _ = extract_project_folder(project) + + response = self.controller.list_subsets(project_name) + if response.errors: + raise AppException(response.errors) + return BaseSerializer.serialize_iterable(response.data) diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 52bdc4c51..8455acd26 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -27,6 +27,13 @@ def validate(cls, v: datetime): return v.isoformat() +class SubSetEntity(BaseModel): + name: str + + class Config: + extra = Extra.ignore + + class TimedBaseModel(BaseModel): createdAt: StringDate = Field(None, alias="createdAt") updatedAt: StringDate = Field(None, alias="updatedAt") diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index 1a4ad67ce..c661c6101 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -6,6 +6,7 @@ from pydantic import BaseModel from pydantic import Extra +from pydantic import parse_obj_as class Limit(BaseModel): @@ -81,7 +82,7 @@ def __init__(self, response, content_type=None): } try: if content_type and content_type is not self.__class__: - data["data"] = content_type(**response.json()) + data["data"] = parse_obj_as(content_type, response.json()) else: data["data"] = response.json() except Exception as e: diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 2276bf447..3d834417a 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -368,3 +368,6 @@ def saqul_query( def validate_saqul_query(self, team_id: int, project_id: int, query: str) -> dict: raise NotImplementedError + + def list_sub_sets(self, team_id: int, project_id: int) -> ServiceResponse: + raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 8461619f9..2e7846450 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -19,6 +19,7 @@ from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository +from lib.core.response import Response from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase @@ -1079,3 +1080,38 @@ def execute(self): ) self._response.data = invited, list(to_skip) return self._response + + +class ListSubsetsUseCase(BaseReportableUseCase): + def __init__( + self, + reporter: Reporter, + project: ProjectEntity, + backend_client: SuperannotateServiceProvider, + ): + super().__init__(reporter) + self._project = project + self._backend_client = backend_client + + def validate_project(self): + if self._project.sync_status != constances.ProjectState.SYNCED.value: + raise AppException("Data is not synced.") + + response = self._backend_client.validate_saqul_query( + self._project.team_id, self._project.id, "_" + ) + error = response.get("error") + if error: + raise AppException(response["error"]) + + def execute(self) -> Response: + if self.is_valid(): + sub_sets_response = self._backend_client.list_sub_sets( + team_id=self._project.team_id, project_id=self._project.id + ) + if sub_sets_response.ok: + self._response.data = [] + else: + self._response.data = [] + + return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 7f4f97b06..ea169c919 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1689,3 +1689,12 @@ def download_annotations( callback=callback, ) return use_case.execute() + + def list_subsets(self, project_name: str): + project = self._get_project(project_name) + use_case = usecases.ListSubsetsUseCase( + reporter=self.get_default_reporter(), + project=project, + backend_client=self.backend_client, + ) + return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 278087c1f..7dcce270e 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -14,6 +14,7 @@ import lib.core as constance import requests.packages.urllib3 +from lib.core import entities from lib.core.exceptions import AppException from lib.core.exceptions import BackendError from lib.core.reporter import Reporter @@ -240,6 +241,7 @@ class SuperannotateBackendService(BaseBackendService): URL_ATTACH_INTEGRATIONS = "image/integration/create" URL_SAQUL_QUERY = "/images/search/advanced" URL_VALIDATE_SAQUL_QUERY = "/images/parse/query/advanced" + URL_LIST_SUBSETS = "/project/{project_id}/subset" def upload_priority_scores( self, team_id: int, project_id: int, folder_id: int, priorities: list @@ -1190,3 +1192,11 @@ def validate_saqul_query(self, team_id: int, project_id: int, query: str) -> dic return self._request( validate_query_url, "post", params=params, data=data ).json() + + def list_sub_sets(self, team_id: int, project_id: int) -> ServiceResponse: + return self._request( + urljoin(self.api_url, self.URL_LIST_SUBSETS.format(project_id=project_id)), + "get", + params=dict(team_id=team_id), + content_type=List[entities.SubSetEntity], + ) diff --git a/tests/integration/test_benchmark.py b/tests/integration/test_benchmark.py index 3c626acbe..3f487e67b 100644 --- a/tests/integration/test_benchmark.py +++ b/tests/integration/test_benchmark.py @@ -3,7 +3,9 @@ from os.path import dirname import pytest + from src.superannotate import SAClient + sa = SAClient() from tests.integration.base import BaseTestCase diff --git a/tests/integration/test_create_from_full_info.py b/tests/integration/test_create_from_full_info.py deleted file mode 100644 index e69de29bb..000000000 From 143d30f76bccb1ec851d0367eaf49e09e908a9df Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 22 Jun 2022 12:09:24 +0400 Subject: [PATCH 35/59] added subset to query function --- .../lib/app/interface/sdk_interface.py | 17 ++++- .../lib/core/entities/__init__.py | 2 + src/superannotate/lib/core/entities/base.py | 1 + .../lib/core/serviceproviders.py | 7 ++- src/superannotate/lib/core/usecases/items.py | 63 +++++++++++++++---- .../lib/core/usecases/projects.py | 2 +- .../lib/infrastructure/controller.py | 7 ++- .../lib/infrastructure/services.py | 20 ++++-- tests/integration/items/test_saqul_query.py | 13 +++- 9 files changed, 107 insertions(+), 25 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 76b450c37..e64454dd6 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2423,7 +2423,12 @@ def attach_items_from_integrated_storage( if response.errors: raise AppException(response.errors) - def query(self, project: NotEmptyStr, query: Optional[NotEmptyStr]): + def query( + self, + project: NotEmptyStr, + query: Optional[NotEmptyStr] = None, + subset: Optional[NotEmptyStr] = None, + ): """Return items that satisfy the given query. Query syntax should be in SuperAnnotate query language(https://doc.superannotate.com/docs/query-search-1). @@ -2433,11 +2438,17 @@ def query(self, project: NotEmptyStr, query: Optional[NotEmptyStr]): :param query: SAQuL query string. :type query: str + :param subset: subset name. Allows you to query items in a specific subset. + To return all the items in the specified subset, set the value of query param to None. + :type subset: str + :return: queried items’ metadata list :rtype: list of dicts """ project_name, folder_name = extract_project_folder(project) - response = self.controller.query_entities(project_name, folder_name, query) + response = self.controller.query_entities( + project_name, folder_name, query, subset + ) if response.errors: raise AppException(response.errors) return BaseSerializer.serialize_iterable(response.data) @@ -2759,4 +2770,4 @@ def get_subsets(self, project: Union[NotEmptyStr, dict]): response = self.controller.list_subsets(project_name) if response.errors: raise AppException(response.errors) - return BaseSerializer.serialize_iterable(response.data) + return BaseSerializer.serialize_iterable(response.data, ["name"]) diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index d05a2bec7..41605abf7 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -2,6 +2,7 @@ from lib.core.entities.base import BaseEntity as TmpBaseEntity from lib.core.entities.base import ProjectEntity from lib.core.entities.base import SettingEntity +from lib.core.entities.base import SubSetEntity from lib.core.entities.integrations import IntegrationEntity from lib.core.entities.items import DocumentEntity from lib.core.entities.items import Entity @@ -31,6 +32,7 @@ __all__ = [ # base "SettingEntity", + "SubSetEntity", # items "TmpImageEntity", "BaseEntity", diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 8455acd26..f3b6875f3 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -28,6 +28,7 @@ def validate(cls, v: datetime): class SubSetEntity(BaseModel): + id: Optional[int] name: str class Config: diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 3d834417a..1e43daba7 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -362,7 +362,12 @@ def attach_integrations( raise NotImplementedError def saqul_query( - self, team_id: int, project_id: int, query: str, folder_id: int + self, + team_id: int, + project_id: int, + folder_id: int, + query: str = None, + subset_id: int = None, ) -> ServiceResponse: raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index b30e73ef2..c8a2293cd 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -1,5 +1,6 @@ import copy from typing import List +from typing import Optional import superannotate.lib.core as constances from lib.core.conditions import Condition @@ -9,6 +10,7 @@ from lib.core.entities import Entity from lib.core.entities import FolderEntity from lib.core.entities import ProjectEntity +from lib.core.entities import SubSetEntity from lib.core.entities import TmpImageEntity from lib.core.entities import VideoEntity from lib.core.exceptions import AppException @@ -78,7 +80,7 @@ def execute(self) -> Response: return self._response -class QueryEntities(BaseReportableUseCase): +class QueryEntitiesUseCase(BaseReportableUseCase): def __init__( self, reporter: Reporter, @@ -86,33 +88,70 @@ def __init__( folder: FolderEntity, backend_service_provider: SuperannotateServiceProvider, query: str, + subset: str, ): super().__init__(reporter) self._project = project self._folder = folder self._backend_client = backend_service_provider self._query = query + self._subset = subset def validate_query(self): - response = self._backend_client.validate_saqul_query( - self._project.team_id, self._project.id, self._query - ) - if response.get("error"): - raise AppException(response["error"]) - if response["isValidQuery"]: - self._query = response["parsedQuery"] - else: - raise AppException("Incorrect query.") if self._project.sync_status != constances.ProjectState.SYNCED.value: raise AppException("Data is not synced.") + if self._query: + response = self._backend_client.validate_saqul_query( + self._project.team_id, self._project.id, self._query + ) + if response.get("error"): + raise AppException(response["error"]) + if response["isValidQuery"]: + self._query = response["parsedQuery"] + else: + raise AppException("Incorrect query.") + else: + response = self._backend_client.validate_saqul_query( + self._project.team_id, self._project.id, "-" + ) + if response.get("error"): + raise AppException(response["error"]) def execute(self) -> Response: + if not any([self._query, self._subset]): + self._response.errors = AppException( + "AppException: The query and subset params cannot have the value None at the same time." + ) + return self._response + if self.is_valid(): + query_kwargs = {} + if self._subset: + subset: Optional[SubSetEntity] = None + response = self._backend_client.list_sub_sets( + team_id=self._project.team_id, project_id=self._project.id + ) + if response.ok: + subset = next( + (_sub for _sub in response.data if _sub.name == self._subset), + None, + ) + if not subset: + self._response.errors = AppException( + "Subset not found. Use the superannotate." + "get_subsets() function to get a list of the available subsets." + ) + return self._response + query_kwargs["subset_id"] = subset.id + if self._query: + query_kwargs["query"] = self._query + query_kwargs["folder_id"] = ( + None if self._folder.name == "root" else self._folder.uuid + ) service_response = self._backend_client.saqul_query( self._project.team_id, self._project.id, - self._query, - folder_id=None if self._folder.name == "root" else self._folder.uuid, + **query_kwargs, ) if service_response.ok: data = [] diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 2e7846450..5f260c286 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -1110,7 +1110,7 @@ def execute(self) -> Response: team_id=self._project.team_id, project_id=self._project.id ) if sub_sets_response.ok: - self._response.data = [] + self._response.data = sub_sets_response.data else: self._response.data = [] diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index ea169c919..dbe85cd7b 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1509,15 +1509,18 @@ def attach_integrations( ) return use_case.execute() - def query_entities(self, project_name: str, folder_name: str, query: str = None): + def query_entities( + self, project_name: str, folder_name: str, query: str = None, subset: str = None + ): project = self._get_project(project_name) folder = self._get_folder(project, folder_name) - use_case = usecases.QueryEntities( + use_case = usecases.QueryEntitiesUseCase( reporter=self.get_default_reporter(), project=project, folder=folder, query=query, + subset=subset, backend_service_provider=self.backend_client, ) return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 7dcce270e..c9f1787db 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -189,6 +189,7 @@ class SuperannotateBackendService(BaseBackendService): """ DEFAULT_CHUNK_SIZE = 5000 + SAQUL_CHUNK_SIZE = 50 URL_USERS = "users" URL_LIST_PROJECTS = "projects" @@ -1155,9 +1156,14 @@ def attach_integrations( return response.ok def saqul_query( - self, team_id: int, project_id: int, query: str, folder_id: int + self, + team_id: int, + project_id: int, + folder_id: int, + query: str = None, + subset_id: int = None, ) -> ServiceResponse: - CHUNK_SIZE = 50 + query_url = urljoin(self.api_url, self.URL_SAQUL_QUERY) params = { "team_id": team_id, @@ -1166,18 +1172,22 @@ def saqul_query( } if folder_id: params["folder_id"] = folder_id - data = {"query": query, "image_index": 0} + if subset_id: + params["subset_id"] = subset_id + data = {"image_index": 0} + if query: + data["query"] = query items = [] for _ in range(self.MAX_ITEMS_COUNT): response = self._request(query_url, "post", params=params, data=data) if response.ok: response_items = response.json() items.extend(response_items) - if len(response_items) < CHUNK_SIZE: + if len(response_items) < self.SAQUL_CHUNK_SIZE: service_response = ServiceResponse(response) service_response.data = items return service_response - data["image_index"] += CHUNK_SIZE + data["image_index"] += self.SAQUL_CHUNK_SIZE return ServiceResponse(response) def validate_saqul_query(self, team_id: int, project_id: int, query: str) -> dict: diff --git a/tests/integration/items/test_saqul_query.py b/tests/integration/items/test_saqul_query.py index d7bd6523c..fdf2d0b84 100644 --- a/tests/integration/items/test_saqul_query.py +++ b/tests/integration/items/test_saqul_query.py @@ -2,9 +2,10 @@ from pathlib import Path from src.superannotate import SAClient -sa = SAClient() from tests.integration.base import BaseTestCase +sa = SAClient() + class TestEntitiesSearchVector(BaseTestCase): PROJECT_NAME = "TestEntitiesSearchVector" @@ -35,6 +36,16 @@ def test_query(self): self.assertEqual(len(entities), 1) assert all([entity["path"] == f"{self.PROJECT_NAME}/{self.FOLDER_NAME}" for entity in entities]) + try: + self.assertRaises( + Exception, sa.query(f"{self.PROJECT_NAME}/{self.FOLDER_NAME}", self.TEST_QUERY, subset="something") + ) + except Exception as e: + self.assertEqual( + str(e), + "Subset not found. Use the superannotate.get_subsets() function to get a list of the available subsets." + ) + def test_validate_saqul_query(self): try: self.assertRaises(Exception, sa.query(self.PROJECT_NAME, self.TEST_INVALID_QUERY)) From b77a503354437403f8aa12fb9f50b85cc5872676 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Wed, 22 Jun 2022 14:22:05 +0400 Subject: [PATCH 36/59] fixing wrong capitalization --- src/superannotate/lib/app/interface/sdk_interface.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 99917f8a8..f2e3f382e 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -652,11 +652,11 @@ def set_images_annotation_statuses( def delete_images( self, project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None ): - """delete images in project. + """Delete Images in project. :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param image_names: to be deleted images' names. if none, all the images will be deleted + :param image_names: to be deleted images' names. If None, all the images will be deleted :type image_names: list of strs """ @@ -670,13 +670,13 @@ def delete_images( project_name, folder_name = extract_project_folder(project) if not isinstance(image_names, list) and image_names is not none: - raise appexception("image_names should be a list of str or none.") + raise AppException("image_names should be a list of str or None.") response = self.controller.delete_images( project_name=project_name, folder_name=folder_name, image_names=image_names ) if response.errors: - raise appexception(response.errors) + raise AppException(response.errors) logger.info( f"images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" From 96a17f27546b61c89125821c1745b246654c8638 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Wed, 22 Jun 2022 14:23:02 +0400 Subject: [PATCH 37/59] fixing wrong capitalization --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index f2e3f382e..0a4f24e26 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -679,7 +679,7 @@ def delete_images( raise AppException(response.errors) logger.info( - f"images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" + f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) def delete_items( From 74febc88cd670720ab0b811282449d71a5fbfe5c Mon Sep 17 00:00:00 2001 From: VavoTK Date: Wed, 22 Jun 2022 14:23:36 +0400 Subject: [PATCH 38/59] fixing wrong capitalization --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 0a4f24e26..a6cffd058 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -669,7 +669,7 @@ def delete_images( warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) - if not isinstance(image_names, list) and image_names is not none: + if not isinstance(image_names, list) and image_names is not None: raise AppException("image_names should be a list of str or None.") response = self.controller.delete_images( From 4e25c50bd72907a3276b6e8f0cbea3e2ce5ca747 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Wed, 22 Jun 2022 14:24:46 +0400 Subject: [PATCH 39/59] fixing deprecation warning --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index a6cffd058..498307ac4 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -663,7 +663,7 @@ def delete_images( warning_msg = ( "We're deprecating the delete_images function. Please use delete_items instead." "Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.assign_items" + "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.delete_items" ) logger.warning(warning_msg) warnings.warn(warning_msg, DeprecationWarning) From fd1e8c471fb8794865204e61855dd4caa395fb4c Mon Sep 17 00:00:00 2001 From: VavoTK Date: Thu, 23 Jun 2022 17:13:06 +0400 Subject: [PATCH 40/59] classIds, and frame picking logic --- src/superannotate/lib/core/video_convertor.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 4203b7266..0e1536e1e 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -15,6 +15,7 @@ class Annotation(BaseModel): instanceId: int type: str className: Optional[str] + classId: Optional[int] x: Optional[Any] y: Optional[Any] points: Optional[Dict] @@ -57,6 +58,7 @@ def get_frame(self, frame_no: int): def _interpolate( self, class_name: str, + class_id: int, from_frame: int, to_frame: int, data: dict, @@ -87,6 +89,7 @@ def _interpolate( instanceId=instance_id, type=annotation_type, className=class_name, + classId=class_id, attributes=data["attributes"], keyframe=False, **tmp_data @@ -105,23 +108,21 @@ def pairwise(data: list): return zip(a, b) def get_median(self, annotations: List[dict]) -> dict: - if len(annotations) == 1: + if len(annotations) >= 1: return annotations[0] - first_annotations = annotations[:1][0] - median = ( - first_annotations["timestamp"] // self.ratio - ) * self.ratio + self.ratio / 2 - median_annotation = first_annotations - distance = abs(median - first_annotations["timestamp"]) - for annotation in annotations[1:]: - annotation_distance = abs(median - annotation["timestamp"]) - if annotation_distance < distance: - distance = annotation_distance - median_annotation = annotation - return median_annotation - - def calculate_sped(self, from_frame, to_frame): - pass + ## Let's just leave the code for reference. + # first_annotations = annotations[:1][0] + # median = ( + # first_annotations["timestamp"] // self.ratio + # ) * self.ratio + self.ratio / 2 + # median_annotation = first_annotations + # distance = abs(median - first_annotations["timestamp"]) + # for annotation in annotations[1:]: + # annotation_distance = abs(median - annotation["timestamp"]) + # if annotation_distance < distance: + # distance = annotation_distance + # median_annotation = annotation + # return median_annotation @staticmethod def merge_first_frame(frames_mapping): @@ -141,6 +142,7 @@ def _interpolate_frames( to_frame_no, annotation_type, class_name, + class_id, instance_id, ): steps = None @@ -171,6 +173,7 @@ def _interpolate_frames( ] return self._interpolate( class_name=class_name, + class_id=class_id, from_frame=from_frame_no, to_frame=to_frame_no, data=from_frame, @@ -184,6 +187,7 @@ def _process(self): instance_id = next(self.id_generator) annotation_type = instance["meta"]["type"] class_name = instance["meta"].get("className") + class_id = instance["meta"].get("classId", -1) for parameter in instance["parameters"]: frames_mapping = defaultdict(list) interpolated_frames = {} @@ -207,6 +211,7 @@ def _process(self): to_frame=to_frame, to_frame_no=to_frame_no, class_name=class_name, + class_id=class_id, annotation_type=annotation_type, instance_id=instance_id, ) @@ -222,6 +227,7 @@ def _process(self): instanceId=instance_id, type=annotation_type, className=class_name, + classId=class_id, x=frame.get("x"), y=frame.get("y"), points=frame.get("points"), @@ -234,6 +240,7 @@ def _process(self): instanceId=instance_id, type=annotation_type, className=class_name, + classId=class_id, x=median.get("x"), y=median.get("y"), points=median.get("points"), From f5a7187f9ec939f4fd4df828a00b23c0c59534da Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 23 Jun 2022 17:34:24 +0400 Subject: [PATCH 41/59] Fix typo --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/usecases/items.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index ef152d550..9c05b8e8b 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2199,8 +2199,8 @@ def upload_images_to_project( return uploaded, failed_images, duplications raise AppException(use_case.response.errors) + @staticmethod def aggregate_annotations_as_df( - self, project_root: Union[NotEmptyStr, Path], project_type: ProjectTypes, folder_names: Optional[List[Union[Path, NotEmptyStr]]] = None, diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index fb3a20279..e0b7632fc 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -132,7 +132,7 @@ def __init__( self._subset = subset def validate_query(self): - if self._project.sync_status != constances.ProjectState.SYNCED.value: + if self._project.sync_status != constants.ProjectState.SYNCED.value: raise AppException("Data is not synced.") if self._query: response = self._backend_client.validate_saqul_query( From 76f9dd8d0d5ba92a4b07ae906f0fe1237298106a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 23 Jun 2022 17:42:58 +0400 Subject: [PATCH 42/59] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 30c3fa299..a52dd2572 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev15" +__version__ = "4.3.5dev16" From e2d18817cf542e3e16747429f4039bd451c87c25 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 23 Jun 2022 16:17:56 +0400 Subject: [PATCH 43/59] Updated the docs --- docs/source/index.rst | 4 +- docs/source/superannotate.sdk.rst | 8 +- docs/source/tutorial.sdk.rst | 29 ++++--- requirements_dev.txt | 2 +- .../lib/app/interface/sdk_interface.py | 85 +++---------------- src/superannotate/lib/core/usecases/images.py | 55 ------------ src/superannotate/lib/core/usecases/items.py | 11 ++- .../lib/core/usecases/projects.py | 2 +- .../lib/infrastructure/controller.py | 22 ----- tests/integration/items/test_search_items.py | 5 +- .../items/test_set_annotation_statuses.py | 6 +- .../test_depricated_functions_document.py | 4 - .../test_depricated_functions_video.py | 4 - tests/integration/test_interface.py | 2 +- 14 files changed, 50 insertions(+), 189 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1d28b8ff6..6b52ceeea 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -29,7 +29,9 @@ SuperAnnotate Python SDK allows access to the platform without web browser: .. code-block:: python - import superannotate as sa + from superannotate import SAClient + + sa = SAClient() sa.create_project("Example Project 1", "example", "Vector") diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 3f1474198..665f722a7 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -8,14 +8,12 @@ API Reference Remote functions ---------------- -Initialization and authentication +Instantiation and authentication _________________________________ -.. automethod:: superannotate.SAClient.__init__ +.. autoclass:: superannotate.SAClient -.. _ref_projects: - Projects ________ @@ -93,8 +91,6 @@ ______ .. _ref_search_images: .. automethod:: superannotate.SAClient.download_image -.. automethod:: superannotate.SAClient.set_image_annotation_status -.. automethod:: superannotate.SAClient.set_images_annotation_statuses .. automethod:: superannotate.SAClient.download_image_annotations .. automethod:: superannotate.SAClient.upload_image_annotations .. automethod:: superannotate.SAClient.copy_image diff --git a/docs/source/tutorial.sdk.rst b/docs/source/tutorial.sdk.rst index ebb52279b..ee66d8880 100644 --- a/docs/source/tutorial.sdk.rst +++ b/docs/source/tutorial.sdk.rst @@ -77,14 +77,14 @@ Include the package in your Python code: .. code-block:: python - import superannotate as sa + from superannotate import SAClient SDK is ready to be used if default location config file was created using the :ref:`CLI init `. Otherwise to authenticate SDK with the :ref:`custom config file `: .. code-block:: python - sa.init("") + sa = SAClient(config_path="") Creating a project ____________________________ @@ -273,80 +273,81 @@ You can find more information annotation format conversion :ref:`here =v1.0.43dev5 +superannotate_schemas>=v1.0.45dev1 diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index ef152d550..92f5b68aa 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -62,6 +62,18 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta): + """Create SAClient instance to authorize SDK in a team scope. + In case of no argument has been provided, SA_TOKEN environmental variable + will be checked or $HOME/.superannotate/config.json will be used. + + :param token: team token + :type token: str + + :param config_path: path to config file + :type config_path: path-like (str or Path) + + """ + def __init__( self, token: str = None, @@ -618,37 +630,6 @@ def pin_image( is_pinned=int(pin), ) - def set_images_annotation_statuses( - self, - project: Union[NotEmptyStr, dict], - annotation_status: NotEmptyStr, - image_names: Optional[List[NotEmptyStr]] = None, - ): - """Sets annotation statuses of images - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_names: image names. If None, all the images in the project will be used - :type image_names: list of str - :param annotation_status: annotation status to set, - should be one of NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - """ - warning_msg = ( - "We're deprecating the set_images_annotation_statuses function. Please use set_annotation_statuses instead. " - "Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.set_annotation_statuses" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, folder_name = extract_project_folder(project) - response = self.controller.set_images_annotation_statuses( - project_name, folder_name, image_names, annotation_status - ) - if response.errors: - raise AppException(response.errors) - logger.info("Annotations status of images changed") - def delete_images( self, project: Union[NotEmptyStr, dict], image_names: Optional[List[str]] = None ): @@ -682,9 +663,7 @@ def delete_images( f"Images deleted in project {project_name}{'/' + folder_name if folder_name else ''}" ) - def delete_items( - self, project: str, items: Optional[List[str]] = None - ): + def delete_items(self, project: str, items: Optional[List[str]] = None): """Delete items in a given project. :param project: project name or folder path (e.g., "project1/folder1") @@ -700,7 +679,6 @@ def delete_items( if response.errors: raise AppException(response.errors) - def assign_items( self, project: Union[NotEmptyStr, dict], items: List[str], user: str ): @@ -1427,41 +1405,6 @@ def download_export( raise AppException(response.errors) logger.info(response.data) - def set_image_annotation_status( - self, - project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - annotation_status: NotEmptyStr, - ): - """Sets the image annotation status - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param image_name: image name - :type image_name: str - :param annotation_status: annotation status to set, - should be one of NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - - :return: metadata of the updated image - :rtype: dict - """ - warning_msg = ( - "We're deprecating the set_image_annotation_status function. Please use set_annotation_statuses instead. " - "Learn more. \n" - "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.set_annotation_statuses" - ) - logger.warning(warning_msg) - warnings.warn(warning_msg, DeprecationWarning) - project_name, folder_name = extract_project_folder(project) - response = self.controller.set_images_annotation_statuses( - project_name, folder_name, [image_name], annotation_status - ) - if response.errors: - raise AppException(response.errors) - image = self.controller.get_item(project_name, folder_name, image_name).data - return BaseSerializer(image).serialize() - def set_project_workflow( self, project: Union[NotEmptyStr, dict], new_workflow: List[dict] ): @@ -1720,7 +1663,7 @@ def download_model(self, model: MLModel, output_dir: Union[str, Path]): :param model: the model that needs to be downloaded :type model: dict - :param output_dir: the directiory in which the files will be saved + :param output_dir: the directory in which the files will be saved :type output_dir: str :return: the metadata of the model :rtype: dict diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 41611999e..0bf4461fe 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -2063,61 +2063,6 @@ def execute(self): return self._response -class SetImageAnnotationStatuses(BaseUseCase): - CHUNK_SIZE = 500 - - def __init__( - self, - service: SuperannotateServiceProvider, - projects: BaseReadOnlyRepository, - image_names: list, - team_id: int, - project_id: int, - folder_id: int, - images_repo: BaseManageableRepository, - annotation_status: int, - ): - super().__init__() - self._service = service - self._projects = projects - self._image_names = image_names - self._team_id = team_id - self._project_id = project_id - self._folder_id = folder_id - self._annotation_status = annotation_status - self._images_repo = images_repo - - def validate_project_type(self): - project = self._projects.get_one(uuid=self._project_id, team_id=self._team_id) - if project.type in constances.LIMITED_FUNCTIONS: - raise AppValidationException(constances.LIMITED_FUNCTIONS[project.type]) - - def execute(self): - if self.is_valid(): - if self._image_names is None: - condition = ( - Condition("team_id", self._team_id, EQ) - & Condition("project_id", self._project_id, EQ) - & Condition("folder_id", self._folder_id, EQ) - ) - self._image_names = [ - image.name for image in self._images_repo.get_all(condition) - ] - for i in range(0, len(self._image_names), self.CHUNK_SIZE): - status_changed = self._service.set_images_statuses_bulk( - image_names=self._image_names[ - i : i + self.CHUNK_SIZE # noqa: E203 - ], - team_id=self._team_id, - project_id=self._project_id, - folder_id=self._folder_id, - annotation_status=self._annotation_status, - ) - if not status_changed: - self._response.errors = AppException("Failed to change status.") - return self._response - - class CreateAnnotationClassUseCase(BaseUseCase): def __init__( self, diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index fb3a20279..4c38cd0f3 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -9,6 +9,7 @@ from lib.core.entities import DocumentEntity from lib.core.entities import Entity from lib.core.entities import FolderEntity +from lib.core.entities import ImageEntity from lib.core.entities import ProjectEntity from lib.core.entities import SubSetEntity from lib.core.entities import TmpImageEntity @@ -22,11 +23,11 @@ from lib.core.serviceproviders import SuperannotateServiceProvider from lib.core.usecases.base import BaseReportableUseCase from lib.core.usecases.base import BaseUseCase -from lib.core.entities import ImageEntity from superannotate.logger import get_default_logger logger = get_default_logger() + class GetBulkItems(BaseUseCase): def __init__( self, @@ -56,11 +57,12 @@ def execute(self): if not response.ok: raise AppException(response.error) - #TODO stop using Image Entity when it gets deprecated and from_dict gets implemented for items + # TODO stop using Image Entity when it gets deprecated and from_dict gets implemented for items res += [ImageEntity.from_dict(**item) for item in response.data] self._response.data = res return self._response + class GetItem(BaseReportableUseCase): def __init__( self, @@ -132,8 +134,8 @@ def __init__( self._subset = subset def validate_query(self): - if self._project.sync_status != constances.ProjectState.SYNCED.value: - raise AppException("Data is not synced.") + if self._project.sync_status != constants.ProjectState.SYNCED.value: + raise AppException("Project data is not synced.") if self._query: response = self._backend_client.validate_saqul_query( self._project.team_id, self._project.id, self._query @@ -722,6 +724,7 @@ def execute(self): break return self._response + class DeleteItemsUseCase(BaseUseCase): CHUNK_SIZE = 1000 diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 5f260c286..aa6fc129e 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -1095,7 +1095,7 @@ def __init__( def validate_project(self): if self._project.sync_status != constances.ProjectState.SYNCED.value: - raise AppException("Data is not synced.") + raise AppException("Project data is not synced.") response = self._backend_client.validate_saqul_query( self._project.team_id, self._project.id, "_" diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 15e255cd3..de18dbd0d 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -795,28 +795,6 @@ def set_project_settings(self, project_name: str, new_settings: List[dict]): ) return use_case.execute() - def set_images_annotation_statuses( - self, - project_name: str, - folder_name: str, - image_names: list, - annotation_status: str, - ): - project_entity = self._get_project(project_name) - folder_entity = self._get_folder(project_entity, folder_name) - images_repo = ImageRepository(service=self._backend_client) - use_case = usecases.SetImageAnnotationStatuses( - service=self._backend_client, - projects=self.projects, - image_names=image_names, - team_id=project_entity.team_id, - project_id=project_entity.id, - folder_id=folder_entity.uuid, - images_repo=images_repo, - annotation_status=constances.AnnotationStatus.get_value(annotation_status), - ) - return use_case.execute() - def delete_images( self, project_name: str, diff --git a/tests/integration/items/test_search_items.py b/tests/integration/items/test_search_items.py index 5854c29c4..0c1051472 100644 --- a/tests/integration/items/test_search_items.py +++ b/tests/integration/items/test_search_items.py @@ -32,8 +32,9 @@ def test_search_items_metadata(self): assert len(sa.search_items(self.PROJECT_NAME, name_contains="1.jp")) == 1 assert len(sa.search_items(self.PROJECT_NAME, name_contains=".jpg")) == 4 assert len(sa.search_items(self.PROJECT_NAME, recursive=True)) == 4 - sa.set_image_annotation_status(self.PROJECT_NAME, self.IMAGE1_NAME, constances.AnnotationStatus.COMPLETED.name) - sa.set_image_annotation_status(self.PROJECT_NAME, self.IMAGE2_NAME, constances.AnnotationStatus.COMPLETED.name) + sa.set_annotation_statuses( + self.PROJECT_NAME, constances.AnnotationStatus.COMPLETED.name, [self.IMAGE1_NAME, self.IMAGE2_NAME] + ) assert len( sa.search_items(self.PROJECT_NAME, annotation_status=constances.AnnotationStatus.COMPLETED.name) ) == 2 diff --git a/tests/integration/items/test_set_annotation_statuses.py b/tests/integration/items/test_set_annotation_statuses.py index 6950c6012..ddfb123ff 100644 --- a/tests/integration/items/test_set_annotation_statuses.py +++ b/tests/integration/items/test_set_annotation_statuses.py @@ -72,11 +72,11 @@ def test_image_annotation_status_via_invalid_names(self): self.PROJECT_NAME, "QualityCheck", ["self.EXAMPLE_IMAGE_1", "self.EXAMPLE_IMAGE_2"] ) - def test_set_image_annotation_status(self): + def test_set_annotation_statuses(self): sa.attach_items( self.PROJECT_NAME, [self.ATTACHMENT_LIST[0]] ) - data = sa.set_image_annotation_status( - self.PROJECT_NAME, self.ATTACHMENT_LIST[0]["name"], annotation_status="Completed" + data = sa.set_annotation_statuses( + self.PROJECT_NAME, annotation_status="Completed", items=[self.ATTACHMENT_LIST[0]["name"]] ) assert data["annotation_status"] == "Completed" \ No newline at end of file diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py index 58bc213a5..eee0d1ae5 100644 --- a/tests/integration/test_depricated_functions_document.py +++ b/tests/integration/test_depricated_functions_document.py @@ -83,10 +83,6 @@ def test_deprecated_functions(self): sa.download_image_annotations(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "./") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.set_image_annotation_status(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "Completed") - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.copy_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, self.PROJECT_NAME_2) except AppException as e: diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py index 8def00a64..2540a85bd 100644 --- a/tests/integration/test_depricated_functions_video.py +++ b/tests/integration/test_depricated_functions_video.py @@ -76,10 +76,6 @@ def test_deprecated_functions(self): sa.download_image_annotations(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "./") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.set_image_annotation_status(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "Completed") - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.copy_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, self.PROJECT_NAME_2) except AppException as e: diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 9739ddca8..e4b779cee 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -95,7 +95,7 @@ def test_download_image_annotations(self): def test_search_project(self): sa.upload_images_from_folder_to_project(self.PROJECT_NAME, self.folder_path) - sa.set_image_annotation_status(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1, "Completed") + sa.set_annotation_statuses(self.PROJECT_NAME, "Completed", [self.EXAMPLE_IMAGE_1]) data = sa.search_projects(self.PROJECT_NAME, return_metadata=True, include_complete_image_count=True) self.assertIsNotNone(data[0]['completed_images_count']) From e0a28b58367cf16d46350e9106422d6c260f2e58 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 28 Jun 2022 15:25:50 +0400 Subject: [PATCH 44/59] Subsets updates --- .../lib/app/interface/base_interface.py | 35 ++++++++++--------- src/superannotate/lib/core/usecases/items.py | 15 ++++---- .../lib/core/usecases/projects.py | 5 +-- src/superannotate/lib/core/video_convertor.py | 2 +- src/superannotate/version.py | 2 +- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index e29e6a97b..37ab6fa55 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -136,22 +136,25 @@ def _track(self, user_id: str, event_name: str, data: dict): self.get_mp_instance().track(user_id, event_name, data) def _track_method(self, args, kwargs, success: bool): - client = self.get_client() - if not client: - return - function_name = self.function.__name__ if self.function else "" - arguments = self.extract_arguments(self.function, *args, **kwargs) - event_name, properties = self.default_parser(function_name, arguments) - user_id = client.controller.team_data.creator_id - team_name = client.controller.team_data.name - - properties["Success"] = success - default = self.get_default_payload(team_name=team_name, user_id=user_id) - self._track( - user_id, - event_name, - {**default, **properties, **CONFIG.get_current_session().data}, - ) + try: + client = self.get_client() + if not client: + return + function_name = self.function.__name__ if self.function else "" + arguments = self.extract_arguments(self.function, *args, **kwargs) + event_name, properties = self.default_parser(function_name, arguments) + user_id = client.controller.team_data.creator_id + team_name = client.controller.team_data.name + + properties["Success"] = success + default = self.get_default_payload(team_name=team_name, user_id=user_id) + self._track( + user_id, + event_name, + {**default, **properties, **CONFIG.get_current_session().data}, + ) + except BaseException: + pass def __get__(self, obj, owner=None): if obj is not None: diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 4c38cd0f3..96a10f019 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -133,9 +133,7 @@ def __init__( self._query = query self._subset = subset - def validate_query(self): - if self._project.sync_status != constants.ProjectState.SYNCED.value: - raise AppException("Project data is not synced.") + def validate_arguments(self): if self._query: response = self._backend_client.validate_saqul_query( self._project.team_id, self._project.id, self._query @@ -153,13 +151,16 @@ def validate_query(self): if response.get("error"): raise AppException(response["error"]) - def execute(self) -> Response: if not any([self._query, self._subset]): - self._response.errors = AppException( - "AppException: The query and subset params cannot have the value None at the same time." + raise AppException( + "The query and subset params cannot have the value None at the same time." + ) + if all([self._query, self._subset]) and not self._folder.is_root: + raise AppException( + "The folder name should be specified in the query string." ) - return self._response + def execute(self) -> Response: if self.is_valid(): query_kwargs = {} if self._subset: diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index aa6fc129e..5362925b7 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -1093,10 +1093,7 @@ def __init__( self._project = project self._backend_client = backend_client - def validate_project(self): - if self._project.sync_status != constances.ProjectState.SYNCED.value: - raise AppException("Project data is not synced.") - + def validate_arguments(self): response = self._backend_client.validate_saqul_query( self._project.team_id, self._project.id, "_" ) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 0e1536e1e..420f0fc18 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -110,7 +110,7 @@ def pairwise(data: list): def get_median(self, annotations: List[dict]) -> dict: if len(annotations) >= 1: return annotations[0] - ## Let's just leave the code for reference. + # Let's just leave the code for reference. # first_annotations = annotations[:1][0] # median = ( # first_annotations["timestamp"] // self.ratio diff --git a/src/superannotate/version.py b/src/superannotate/version.py index a52dd2572..59f031f54 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev16" +__version__ = "4.3.5dev17" From dcb528839dd1171f6567160f23f81756b63dd70f Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 28 Jun 2022 16:12:35 +0400 Subject: [PATCH 45/59] Update version.py --- src/superannotate/lib/core/usecases/items.py | 2 +- src/superannotate/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/core/usecases/items.py b/src/superannotate/lib/core/usecases/items.py index 96a10f019..20a5fd8ef 100644 --- a/src/superannotate/lib/core/usecases/items.py +++ b/src/superannotate/lib/core/usecases/items.py @@ -155,7 +155,7 @@ def validate_arguments(self): raise AppException( "The query and subset params cannot have the value None at the same time." ) - if all([self._query, self._subset]) and not self._folder.is_root: + if self._subset and not self._folder.is_root: raise AppException( "The folder name should be specified in the query string." ) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index 59f031f54..b34a245c9 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev17" +__version__ = "4.3.5dev19" From c18c5e44a40baac09d83297d8f93e6a04bef928a Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Tue, 28 Jun 2022 16:50:22 +0400 Subject: [PATCH 46/59] Update version.py --- docs/source/conf.py | 7 +++++++ src/superannotate/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 49ebdb687..95c641434 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,3 +54,10 @@ html_static_path = [] autodoc_typehints = "description" +html_show_sourcelink = False + +html_context = { +"display_github": False, # Add 'Edit on Github' link instead of 'View page source' +"last_updated": True, +"commit": False, +} \ No newline at end of file diff --git a/src/superannotate/version.py b/src/superannotate/version.py index b34a245c9..bc9f93610 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev19" +__version__ = "4.3.5dev20" From 627c65dfa2b0953678d3cace4a03a0f08c3dcf62 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 29 Jun 2022 10:23:46 +0400 Subject: [PATCH 47/59] Doc string updates --- .../lib/app/interface/sdk_interface.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index af98a7dd2..6fee9a903 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2049,7 +2049,7 @@ def upload_image_to_project( def search_models( self, name: Optional[NotEmptyStr] = None, - type_: Optional[NotEmptyStr] = None, + type_: Optional[NotEmptyStr] = None, # noqa project_id: Optional[int] = None, task: Optional[NotEmptyStr] = None, include_global: Optional[StrictBool] = True, @@ -2058,17 +2058,21 @@ def search_models( :param name: search string :type name: str - :param type_: ml model type string - :type type_: str + + :param type\_: ml model type string + :type type\_: str + :param project_id: project id :type project_id: int + :param task: training task :type task: str + :param include_global: include global ml models :type include_global: bool - :return: ml model metadata - :rtype: list of dicts + :return: ml model metadata + :rtype: list of dicts """ res = self.controller.search_models( name=name, From 0a0efaf8d3e4e538471b1f0156a673cc89bab815 Mon Sep 17 00:00:00 2001 From: VavoTK Date: Tue, 28 Jun 2022 18:56:09 +0400 Subject: [PATCH 48/59] removing tests for duration, no interpolation for polygon and polyline --- src/superannotate/lib/core/video_convertor.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 420f0fc18..fc0dae47b 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -18,7 +18,7 @@ class Annotation(BaseModel): classId: Optional[int] x: Optional[Any] y: Optional[Any] - points: Optional[Dict] + points: Any attributes: Optional[List[Any]] = [] keyframe: bool = False @@ -30,7 +30,6 @@ class FrameAnnotation(BaseModel): class VideoFrameGenerator: def __init__(self, annotation_data: dict, fps: int): - self.validate_annotations(annotation_data) self.id_generator = iter(itertools.count(0)) self._annotation_data = annotation_data self.duration = annotation_data["metadata"]["duration"] / (1000 * 1000) @@ -42,12 +41,6 @@ def __init__(self, annotation_data: dict, fps: int): self._mapping = {} self._process() - @staticmethod - def validate_annotations(annotation_data: dict): - duration = annotation_data["metadata"].get("duration") - if duration is None: - raise AppException("Video not annotated yet") - def get_frame(self, frame_no: int): try: return self.annotations[frame_no] @@ -81,10 +74,9 @@ def _interpolate( "x": round(data["x"] + steps["x"] * idx, 2), "y": round(data["y"] + steps["y"] * idx, 2), } - elif annotation_type in (AnnotationTypes.POLYGON, AnnotationTypes.POLYLINE): - tmp_data["points"] = [ - point + steps[idx] * 2 for idx, point in enumerate(data["points"]) - ] + else: + tmp_data["points"] = data["points"] + annotations[frame_idx] = Annotation( instanceId=instance_id, type=annotation_type, From 8f12e39b18822cdb61e9596ca6f35b492a0e7ac5 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 29 Jun 2022 15:07:13 +0400 Subject: [PATCH 49/59] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index bc9f93610..c2e7c65a7 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev20" +__version__ = "4.3.5dev21" From 23e7684adee3bde626c606c93dcc9ee78612396e Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 29 Jun 2022 16:14:46 +0400 Subject: [PATCH 50/59] Schema changes --- requirements_dev.txt | 2 +- .../lib/infrastructure/controller.py | 2 +- .../test_get_annotations_per_frame.py | 1 + tests/unit/test_validators.py | 9 +- tests/unit/test_video_valitation.py | 770 ++++++++++++++++++ 5 files changed, 776 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_video_valitation.py diff --git a/requirements_dev.txt b/requirements_dev.txt index eb70f0774..a51762b8e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,2 +1,2 @@ -superannotate_schemas>=v1.0.45dev1 +superannotate_schemas>=v1.0.45dev4 diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index de18dbd0d..695de6002 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1355,7 +1355,7 @@ def delete_annotations( @staticmethod def validate_annotations( - project_type: str, annotation: dict, allow_extra: bool = False + project_type: str, annotation: dict, allow_extra: bool = True ): use_case = usecases.ValidateAnnotationUseCase( project_type, diff --git a/tests/integration/annotations/test_get_annotations_per_frame.py b/tests/integration/annotations/test_get_annotations_per_frame.py index 16bfaa96c..85560cec8 100644 --- a/tests/integration/annotations/test_get_annotations_per_frame.py +++ b/tests/integration/annotations/test_get_annotations_per_frame.py @@ -56,3 +56,4 @@ def test_video_annotation_upload(self): ) ) ) + diff --git a/tests/unit/test_validators.py b/tests/unit/test_validators.py index a10421564..e4dea2171 100644 --- a/tests/unit/test_validators.py +++ b/tests/unit/test_validators.py @@ -3,15 +3,13 @@ import tempfile from os.path import dirname from unittest import TestCase -`from unittest.mock import patch -` -from pydantic import ValidationError +from unittest.mock import patch -from src.superannotate import SAClient -sa = SAClient() from superannotate_schemas.validators import AnnotationValidators +from src.superannotate import SAClient +sa = SAClient() VECTOR_ANNOTATION_JSON_WITH_BBOX = """ { "metadata": { @@ -1668,7 +1666,6 @@ def test_validate_video_point_labels(self, mock_print): "instances[0].meta.pointLabels value is not a valid dict", ) - def test_validate_video_point_labels_bad_keys(self): with tempfile.TemporaryDirectory() as tmpdir_name: with open(f"{tmpdir_name}/test_validate_video_point_labels_bad_keys.json", diff --git a/tests/unit/test_video_valitation.py b/tests/unit/test_video_valitation.py new file mode 100644 index 000000000..0b4ae30df --- /dev/null +++ b/tests/unit/test_video_valitation.py @@ -0,0 +1,770 @@ +from unittest import TestCase + +from src.superannotate import SAClient + +sa = SAClient() +PAYLOAD = { + "metadata": { + "name": "video_file_example_1", + "duration": 30526667, + "width": 1920, + "height": 1080, + "lastAction": { + "timestamp": 1656499500695, + "email": "arturn@superannotate.com" + }, + "projectId": 202655, + "url": "https://sa-public-files.s3.us-west-2.amazonaws.com/Video+project/video_file_example_1.mp4", + "status": "InProgress", + "error": None, + "annotatorEmail": None, + "qaEmail": None + }, + "instances": [ + { + "meta": { + "type": "bbox", + "classId": 1379945, + "className": "tree", + "start": 0, + "end": 2515879, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-03-02T12:23:10.887Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-05-17T13:30:11.963Z", + "pointLabels": {} + }, + "parameters": [ + { + "start": 0, + "end": 2515879, + "timestamps": [ + { + "points": { + "x1": 496.13, + "y1": 132.02, + "x2": 898.05, + "y2": 515.25 + }, + "timestamp": 0, + "attributes": [ + { + "id": 2699916, + "groupId": 1096002, + "name": "standing", + "groupName": "state" + } + ] + }, + { + "points": { + "x1": 744.37, + "y1": 66.41, + "x2": 1146.29, + "y2": 449.64 + }, + "timestamp": 640917, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + }, + { + "points": { + "x1": 857.56, + "y1": 227.21, + "x2": 1259.48, + "y2": 610.44 + }, + "timestamp": 1215864, + "attributes": [ + { + "id": 2699916, + "groupId": 1096002, + "name": "standing", + "groupName": "state" + } + ] + }, + { + "points": { + "x1": 857.56, + "y1": 227.21, + "x2": 1259.48, + "y2": 610.44 + }, + "timestamp": 1573648, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + }, + { + "points": { + "x1": 1038.3, + "y1": 270.54, + "x2": 1440.22, + "y2": 653.77 + }, + "timestamp": 2255379, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + }, + { + "points": { + "x1": 1038.3, + "y1": 270.54, + "x2": 1440.22, + "y2": 653.77 + }, + "timestamp": 2515879, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "bbox", + "classId": 1379945, + "className": "tree", + "start": 2790828, + "end": 5068924, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-03-02T12:36:52.126Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-03-02T12:38:24.056Z", + "pointLabels": {} + }, + "parameters": [ + { + "start": 2790828, + "end": 5068924, + "timestamps": [ + { + "points": { + "x1": 507.3, + "y1": 569.55, + "x2": 979.58, + "y2": 965.84 + }, + "timestamp": 2790828, + "attributes": [] + }, + { + "points": { + "x1": 507.3, + "y1": 569.55, + "x2": 979.58, + "y2": 965.84 + }, + "timestamp": 2888183, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + }, + { + "points": { + "x1": 507.3, + "y1": 569.55, + "x2": 979.58, + "y2": 965.84 + }, + "timestamp": 5068924, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "bbox", + "classId": 1379946, + "className": "car", + "start": 4502645, + "end": 6723950, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-03-02T12:39:19.970Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-03-02T12:41:13.536Z", + "pointLabels": {} + }, + "parameters": [ + { + "start": 4502645, + "end": 6723950, + "timestamps": [ + { + "points": { + "x1": 792.08, + "y1": 671.23, + "x2": 1178.97, + "y2": 987.47 + }, + "timestamp": 4502645, + "attributes": [] + }, + { + "points": { + "x1": 792.08, + "y1": 671.23, + "x2": 1178.97, + "y2": 987.47 + }, + "timestamp": 5330158, + "attributes": [ + { + "id": 2699918, + "groupId": 1096003, + "name": "goes", + "groupName": "movement" + }, + { + "id": 2699920, + "groupId": 1096004, + "name": "lights_on", + "groupName": "lights" + } + ] + }, + { + "points": { + "x1": 792.08, + "y1": 671.23, + "x2": 1178.97, + "y2": 987.47 + }, + "timestamp": 5825043, + "attributes": [ + { + "id": 2699920, + "groupId": 1096004, + "name": "lights_on", + "groupName": "lights" + }, + { + "id": 2699919, + "groupId": 1096003, + "name": "stops", + "groupName": "movement" + } + ] + }, + { + "points": { + "x1": 792.08, + "y1": 671.23, + "x2": 1178.97, + "y2": 987.47 + }, + "timestamp": 6303703, + "attributes": [ + { + "id": 2699919, + "groupId": 1096003, + "name": "stops", + "groupName": "movement" + }, + { + "id": 2699921, + "groupId": 1096004, + "name": "lights_off", + "groupName": "lights" + } + ] + }, + { + "points": { + "x1": 792.08, + "y1": 671.23, + "x2": 1178.97, + "y2": 987.47 + }, + "timestamp": 6723950, + "attributes": [ + { + "id": 2699919, + "groupId": 1096003, + "name": "stops", + "groupName": "movement" + }, + { + "id": 2699921, + "groupId": 1096004, + "name": "lights_off", + "groupName": "lights" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "event", + "classId": 1379946, + "className": "car", + "start": 1655026, + "end": 3365220, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-03-02T12:41:39.135Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-03-02T12:41:39.135Z" + }, + "parameters": [ + { + "start": 1655026, + "end": 3365220, + "timestamps": [ + { + "timestamp": 1655026, + "attributes": [] + }, + { + "timestamp": 3365220, + "attributes": [] + } + ] + } + ] + }, + { + "meta": { + "type": "point", + "classId": 1379946, + "className": "car", + "start": 617519, + "end": 2342388, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-05-17T11:38:50.041Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-05-17T11:50:57.784Z" + }, + "parameters": [ + { + "start": 617519, + "end": 2342388, + "timestamps": [ + { + "x": 612.81, + "y": 606.79, + "timestamp": 617519, + "attributes": [ + { + "id": 2699918, + "groupId": 1096003, + "name": "goes", + "groupName": "movement" + }, + { + "id": 2699920, + "groupId": 1096004, + "name": "lights_on", + "groupName": "lights" + } + ] + }, + { + "x": 653.05, + "y": 648.81, + "timestamp": 1266439, + "attributes": [ + { + "id": 2699918, + "groupId": 1096003, + "name": "goes", + "groupName": "movement" + }, + { + "id": 2699920, + "groupId": 1096004, + "name": "lights_on", + "groupName": "lights" + } + ] + }, + { + "x": 691.9399999999999, + "y": 682.7099999999999, + "timestamp": 1569965, + "attributes": [ + { + "id": 2699918, + "groupId": 1096003, + "name": "goes", + "groupName": "movement" + }, + { + "id": 2699921, + "groupId": 1096004, + "name": "lights_off", + "groupName": "lights" + } + ] + }, + { + "x": 691.9399999999999, + "y": 682.7099999999999, + "timestamp": 2342388, + "attributes": [ + { + "id": 2699921, + "groupId": 1096004, + "name": "lights_off", + "groupName": "lights" + }, + { + "id": 2699919, + "groupId": 1096003, + "name": "stops", + "groupName": "movement" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "point", + "classId": 1379946, + "className": "car", + "start": 2878270, + "end": 5084595, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-05-17T11:40:28.691Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-05-17T12:02:43.429Z" + }, + "parameters": [ + { + "start": 2878270, + "end": 5084595, + "timestamps": [ + { + "x": 915.5, + "y": 461.21, + "timestamp": 2878270, + "attributes": [ + { + "id": 2699918, + "groupId": 1096003, + "name": "goes", + "groupName": "movement" + }, + { + "id": 2699920, + "groupId": 1096004, + "name": "lights_on", + "groupName": "lights" + } + ] + }, + { + "x": 915.5, + "y": 461.21, + "timestamp": 5084595, + "attributes": [ + { + "id": 2699918, + "groupId": 1096003, + "name": "goes", + "groupName": "movement" + }, + { + "id": 2699920, + "groupId": 1096004, + "name": "lights_on", + "groupName": "lights" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "polygon", + "classId": 1379945, + "className": "tree", + "start": 5664421, + "end": 8368555, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-06-29T10:43:15.995Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-06-29T10:43:57.540Z" + }, + "parameters": [ + { + "start": 5664421, + "end": 8368555, + "timestamps": [ + { + "points": [ + 651.25, + 365.32, + 855.92, + 240.26, + 1017.63, + 503.6 + ], + "timestamp": 5664421, + "attributes": [] + }, + { + "points": [ + 651.25, + 365.32, + 855.92, + 240.26, + 1017.63, + 503.6 + ], + "timestamp": 6496259, + "attributes": [ + { + "id": 2699916, + "groupId": 1096002, + "name": "standing", + "groupName": "state" + } + ] + }, + { + "points": [ + 839.98, + 358.45, + 1044.65, + 233.39, + 1206.36, + 496.73 + ], + "timestamp": 6826353, + "attributes": [ + { + "id": 2699916, + "groupId": 1096002, + "name": "standing", + "groupName": "state" + } + ] + }, + { + "points": [ + 839.98, + 358.45, + 1044.65, + 233.39, + 1206.36, + 496.73 + ], + "timestamp": 8368555, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + } + ] + } + ] + }, + { + "meta": { + "type": "polyline", + "classId": 1379945, + "className": "tree", + "start": 8754105, + "end": 11312997, + "createdBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "createdAt": "2022-06-29T10:44:15.979Z", + "updatedBy": { + "email": "arturn@superannotate.com", + "role": "Admin" + }, + "updatedAt": "2022-06-29T10:45:00.660Z" + }, + "parameters": [ + { + "start": 8754105, + "end": 11312997, + "timestamps": [ + { + "points": [ + 679.05, + 412.73, + 1050.61, + 484.06, + 885.75, + 737.4 + ], + "timestamp": 8754105, + "attributes": [ + { + "id": 2699916, + "groupId": 1096002, + "name": "standing", + "groupName": "state" + } + ] + }, + { + "points": [ + 679.05, + 412.73, + 1050.61, + 484.06, + 885.75, + 737.4 + ], + "timestamp": 9467109, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + }, + { + "points": [ + 790.96, + 292.26, + 1162.52, + 363.59, + 997.66, + 616.93 + ], + "timestamp": 9757592, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + }, + { + "points": [ + 790.96, + 292.26, + 1162.52, + 363.59, + 997.66, + 616.93 + ], + "timestamp": 11312997, + "attributes": [ + { + "id": 2699917, + "groupId": 1096002, + "name": "falling", + "groupName": "state" + } + ] + } + ] + } + ] + } + ], + "tags": [ + "tg2", + "tg1" + ] +} + + +class TestVideoValidators(TestCase): + + def test_polygon_polyline(self): + response = sa.controller.validate_annotations("video", PAYLOAD) + assert not response.report_messages From 59d66f3b218befae5a10e7b5d78cf386611f22d6 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 29 Jun 2022 16:15:29 +0400 Subject: [PATCH 51/59] Update version.py --- requirements_dev.txt | 2 +- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/video_convertor.py | 4 +++- src/superannotate/version.py | 2 +- .../integration/annotations/test_get_annotations_per_frame.py | 3 +-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index a51762b8e..9415ce9fc 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,2 +1,2 @@ -superannotate_schemas>=v1.0.45dev4 +superannotate_schemas>=v1.0.45dev5 diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 6fee9a903..b8acf7eaa 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2231,7 +2231,7 @@ def validate_annotations( with open(annotations_json) as file: annotation_data = json.loads(file.read()) response = Controller.validate_annotations( - project_type, annotation_data, allow_extra=False + project_type, annotation_data ) if response.errors: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index fc0dae47b..cf066c67c 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -32,7 +32,9 @@ class VideoFrameGenerator: def __init__(self, annotation_data: dict, fps: int): self.id_generator = iter(itertools.count(0)) self._annotation_data = annotation_data - self.duration = annotation_data["metadata"]["duration"] / (1000 * 1000) + duration = annotation_data["metadata"]["duration"] + duration = 0 if not duration else duration + self.duration = duration / (1000 * 1000) self.fps = fps self.ratio = 1000 * 1000 / fps self._frame_id = 1 diff --git a/src/superannotate/version.py b/src/superannotate/version.py index c2e7c65a7..b86859dad 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev21" +__version__ = "4.3.5dev22" diff --git a/tests/integration/annotations/test_get_annotations_per_frame.py b/tests/integration/annotations/test_get_annotations_per_frame.py index 85560cec8..18be86750 100644 --- a/tests/integration/annotations/test_get_annotations_per_frame.py +++ b/tests/integration/annotations/test_get_annotations_per_frame.py @@ -38,7 +38,7 @@ def annotations_path(self): return os.path.join(self.folder_path, self.ANNOTATIONS_PATH) def test_video_annotation_upload(self): - # sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) + sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, self.classes_path) _, _, _ = sa.attach_items( self.PROJECT_NAME, @@ -56,4 +56,3 @@ def test_video_annotation_upload(self): ) ) ) - From 050cb2aeec1cfebdb8b5d9b4d8f203789bf57052 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Wed, 29 Jun 2022 18:11:24 +0400 Subject: [PATCH 52/59] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index b86859dad..bf2fe805c 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev22" +__version__ = "4.3.5dev23" From 6d5a75b14a76d3c5de36fb8e14b3a6fdf7ec5b94 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:13:14 +0400 Subject: [PATCH 53/59] Update superannotate.sdk.rst --- docs/source/superannotate.sdk.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index 665f722a7..a15085c5e 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -71,6 +71,7 @@ ______ .. automethod:: superannotate.SAClient.attach_items .. automethod:: superannotate.SAClient.copy_items .. automethod:: superannotate.SAClient.move_items +.. automethod:: superannotate.SAClient.delete_items .. automethod:: superannotate.SAClient.assign_items .. automethod:: superannotate.SAClient.unassign_items .. automethod:: superannotate.SAClient.get_item_metadata @@ -404,4 +405,4 @@ Utility functions -------------------------------- .. autofunction:: superannotate.SAClient.consensus -.. autofunction:: superannotate.SAClient.benchmark \ No newline at end of file +.. autofunction:: superannotate.SAClient.benchmark From 81163e5051308cd06a0104a5a0bafc593b08b38c Mon Sep 17 00:00:00 2001 From: VavoTK Date: Thu, 30 Jun 2022 14:58:53 +0400 Subject: [PATCH 54/59] fixing interpolation for event type --- src/superannotate/lib/core/video_convertor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index cf066c67c..b8397cddb 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -76,7 +76,7 @@ def _interpolate( "x": round(data["x"] + steps["x"] * idx, 2), "y": round(data["y"] + steps["y"] * idx, 2), } - else: + elif annotation_type != AnnotationTypes.EVENT: tmp_data["points"] = data["points"] annotations[frame_idx] = Annotation( From 988d1ecac8c3ab46d86a8559bba40c53b7bf83e5 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 30 Jun 2022 17:16:06 +0400 Subject: [PATCH 55/59] Update sdk_interface.py --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index b8acf7eaa..86ea8618c 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2413,7 +2413,7 @@ def query( :type query: str :param subset: subset name. Allows you to query items in a specific subset. - To return all the items in the specified subset, set the value of query param to None. + To return all the items in the specified subset, set the value of query param to None. :type subset: str :return: queried items’ metadata list From 13947bcdb1957a1e5d7c5ec4428ad4960a8b3660 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 30 Jun 2022 17:17:29 +0400 Subject: [PATCH 56/59] Update version.py --- src/superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/version.py b/src/superannotate/version.py index bf2fe805c..9ebe20157 100644 --- a/src/superannotate/version.py +++ b/src/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.3.5dev23" +__version__ = "4.3.5b1" From 167b043dba9bdde045127ad7ee6c760fa3aa7b62 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan <84702976+VaghinakDev@users.noreply.github.com> Date: Thu, 30 Jun 2022 17:39:32 +0400 Subject: [PATCH 57/59] Update sdk_interface.py --- src/superannotate/lib/app/interface/sdk_interface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index b8acf7eaa..76650e39e 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2703,7 +2703,9 @@ def download_annotations( :param path: local directory path where the annotations will be downloaded. If none, the current directory is used. :type path: Path-like (str or Path) - :param items: project name (root) or folder path to move items to. + :param items: list of item names whose annotations will be downloaded + (e.g., ["Image_1.jpeg", "Image_2.jpeg"]). If the value is None, then all the annotations of the given directory will be downloaded. + :type items: list of str :param recursive: download annotations from the project’s root and all of its folders with the preserved structure. From eec0312c4f2d004e5913b32bb35464f29d6f9434 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Sun, 3 Jul 2022 11:57:25 +0400 Subject: [PATCH 58/59] Updated changelog --- CHANGELOG.md | 26 ++++++++++++++++++- .../lib/app/interface/sdk_interface.py | 9 ++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b265a96c7..c1f954747 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog -All release higlighths of this project will be documented in this file. +All release highlights of this project will be documented in this file. +## 4.4.0 - July 03, 2022 +### Added +- `superannotate.SAClient()` _class_ to instantiate team-level authentication and inheriting methods to access the back-end. +- `SAClient.download_annotations()` _method_ to download annotations without preparing an Export object. +- `SAClient.get_subsets()` _method_ to get the existing subsets for a given project. +- `SAClient.assign_items()` _method_ to assign items in a given project to annotators or quality specialists. +- `SAClient.unassign_items()` _method_ to remove assignments from items. +- `SAClient.delete_items()` _method_ to delete items in a given project. +### Updated +- `JSON Schema` for video annotations to version `1.0.45` to show **polygon** and **polyline** annotations. +- `SAClient.get_annotations_per_frame()` _method_ to show **polygon** and **polyline** annotations. +- `SAClient.get_annotations_per_frame()` _method_ to pick instances closer to a given **frame start** instead of the **median**. +- `SAClient.query()` _method_ to add the `subset` argument to support querying in a subset. +### Fixed +- `SAClient.set_annotation_statuses()` _method_ to address the issue occurring with more than 500 items. +- `SAClient.get_annotations()` _method_ to address the `PayloadError` occurring with more than 20000 items. +- `SAClient.get_annotations()` _method_ to address the missing `'duration'` and `'tags'` keys for newly uploaded and unannotated videos. +- `SAClient.get_annotations_per_frame()` _method_ to address missing `'duration'` and `'tags'` keys for newly uploaded and unannotated videos. +- `SAClient.get_annotations_per_frame()` _method_ to address the wrong `classId` value for unclassified instances. +### Removed +- `superannotate.init()` _function_. Please instantiate `superannotate.SAClient()` _class_ to authenticate. +- `superannotate.set_image_annotation_status()` _function_. Please use the `SAClient.set_annotation_statuses()` _method_ instead. +- `superannotate.set_images_annotations_statuses()` _function_. Please use the `SAClient.set_annotation_statuses()` _method_ instead. +### ## 4.3.2 - April 10 2022 ### Added - `query()` function to run SAQuL queries via SDK. diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index a4df98ce5..c441cb500 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -572,7 +572,7 @@ def search_annotation_classes( :type project: str :param name_contains: search string. Returns those classes, where the given string is found anywhere within its name. If None, all annotation classes will be returned. - :type name_prefix: str + :type name_contains: str :return: annotation classes of the project :rtype: list of dicts @@ -1614,12 +1614,19 @@ def upload_image_annotations( :param project: project name or folder path (e.g., "project1/folder1") :type project: str + :param image_name: image name :type image_name: str + :param annotation_json: annotations in SuperAnnotate format JSON dict or path to JSON file :type annotation_json: dict or Path-like (str or Path) + :param mask: BytesIO object or filepath to mask annotation for pixel projects in SuperAnnotate format :type mask: BytesIO or Path-like (str or Path) + + :param verbose: Turns on verbose output logging during the proces. + :type verbose: bool + """ project_name, folder_name = extract_project_folder(project) From fab318b0ab1689bca662ef4dfd2bd30916ec7641 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Sun, 3 Jul 2022 12:01:59 +0400 Subject: [PATCH 59/59] Update schemaes version for prod env --- requirements_prod.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_prod.txt b/requirements_prod.txt index e573cfbfa..61bea0eb7 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1 +1 @@ -superannotate_schemas==1.0.41 +superannotate_schemas==1.0.45