diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index f146b3494..5d4e141a5 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -34,7 +34,9 @@ def __init__(self, token: TokenStr = None, config_path: str = None): config = ConfigEntity(SA_TOKEN=token) elif config_path: config_path = Path(config_path) - if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + if not Path(config_path).is_file() or not os.access( + config_path, os.R_OK + ): raise AppException( f"SuperAnnotate config file {str(config_path)} not found." ) @@ -77,8 +79,12 @@ def _retrieve_configs_from_json(path: Path) -> typing.Union[ConfigEntity]: config = ConfigEntity(SA_TOKEN=token) except pydantic.ValidationError: raise pydantic.ValidationError( - [pydantic.error_wrappers.ErrorWrapper(ValueError("Invalid token."), loc='token')], - model=ConfigEntity + [ + pydantic.error_wrappers.ErrorWrapper( + ValueError("Invalid token."), loc="token" + ) + ], + model=ConfigEntity, ) host = json_data.get("main_endpoint") verify_ssl = json_data.get("ssl_verify") diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index 5bc92b438..0f8faf8ef 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -51,4 +51,5 @@ def wrapped(self, *args, **kwargs): return pydantic_validate_arguments(func)(self, *args, **kwargs) except ValidationError as e: raise AppException(wrap_error(e)) from e + return wrapped diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 48e328b09..7322412e4 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -4,19 +4,21 @@ from logging.handlers import RotatingFileHandler from os.path import expanduser -from superannotate.lib.core.config import Config -from superannotate.lib.core.enums import AnnotationStatus -from superannotate.lib.core.enums import ApprovalStatus -from superannotate.lib.core.enums import FolderStatus -from superannotate.lib.core.enums import ImageQuality -from superannotate.lib.core.enums import ProjectStatus -from superannotate.lib.core.enums import ProjectType -from superannotate.lib.core.enums import SegmentationStatus -from superannotate.lib.core.enums import TrainingStatus -from superannotate.lib.core.enums import UploadState -from superannotate.lib.core.enums import UserRole +from lib.core.config import Config +from lib.core.enums import AnnotationStatus +from lib.core.enums import ApprovalStatus +from lib.core.enums import FolderStatus +from lib.core.enums import ImageQuality +from lib.core.enums import ProjectStatus +from lib.core.enums import ProjectType +from lib.core.enums import SegmentationStatus +from lib.core.enums import TrainingStatus +from lib.core.enums import UploadState +from lib.core.enums import UserRole + CONFIG = Config() +BACKEND_URL = "https://api.superannotate.com" HOME_PATH = expanduser("~/.superannotate") CONFIG_JSON_PATH = f"{HOME_PATH}/config.json" @@ -32,38 +34,34 @@ def setup_logging(level=DEFAULT_LOGGING_LEVEL, file_path=LOG_FILE_LOCATION): - global _loggers - if not _loggers.get("sa"): - logger = logging.getLogger("sa") - logger.propagate = True - logger.setLevel(level) - stream_handler = logging.StreamHandler() - formatter = Formatter("SA-PYTHON-SDK - %(levelname)s - %(message)s") - stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) - try: - log_file_path = os.path.join(file_path, "sa.log") - open(log_file_path, "w").close() - if os.access(log_file_path, os.W_OK): - file_handler = RotatingFileHandler( - log_file_path, - maxBytes=5 * 1024 * 1024, - backupCount=5, - mode="a", - ) - file_formatter = Formatter( - "SA-PYTHON-SDK - %(levelname)s - %(asctime)s - %(message)s" - ) - file_handler.setFormatter(file_formatter) - logger.addHandler(file_handler) - except OSError: - pass - finally: - _loggers["sa"] = logger + logger = logging.getLogger("sa") + for handler in logger.handlers[:]: # remove all old handlers + logger.removeHandler(handler) + logger.propagate = True + logger.setLevel(level) + stream_handler = logging.StreamHandler() + formatter = Formatter("SA-PYTHON-SDK - %(levelname)s - %(message)s") + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + try: + log_file_path = os.path.join(file_path, "sa.log") + open(log_file_path, "w").close() + if os.access(log_file_path, os.W_OK): + file_handler = RotatingFileHandler( + log_file_path, + maxBytes=5 * 1024 * 1024, + backupCount=5, + mode="a", + ) + file_formatter = Formatter( + "SA-PYTHON-SDK - %(levelname)s - %(asctime)s - %(message)s" + ) + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + except OSError as e: + logging.error(e) -BACKEND_URL = "https://api.superannotate.com" - DEFAULT_IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "tif", "tiff", "webp", "bmp"] DEFAULT_FILE_EXCLUDE_PATTERNS = ["___save.png", "___fuse.png"] DEFAULT_VIDEO_EXTENSIONS = ["mp4", "avi", "mov", "webm", "flv", "mpg", "ogg"] diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 22f29feea..86e87a41e 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -15,8 +15,8 @@ from lib.core.enums import BaseTitledEnum from pydantic import BaseModel as PydanticBaseModel from pydantic import Extra -from pydantic import StrictStr from pydantic import Field +from pydantic import StrictStr from pydantic.datetime_parse import parse_datetime from pydantic.typing import is_namedtuple from pydantic.utils import ROOT_KEY @@ -295,7 +295,7 @@ def map_fields(entity: dict) -> dict: class TokenStr(StrictStr): - regex = r'^[-.@_A-Za-z0-9]+=\d+$' + regex = r"^[-.@_A-Za-z0-9]+=\d+$" @classmethod def validate(cls, value: Union[str]) -> Union[str]: diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 3c864e319..32cbea5c2 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -60,6 +60,10 @@ def __init__( self.session = CONFIG.get_current_session() self._spinner = None + @property + def log_enabled(self): + return self._log_info + @property def spinner(self): return Spinner() diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index fd24f699e..7eaaa7858 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -312,20 +312,23 @@ def delete_multiple( class BaseAnnotationService(SuperannotateServiceProvider): @abstractmethod - async def get_small_annotations( + async def get_big_annotation( self, project: entities.ProjectEntity, - folder: entities.FolderEntity, - items: List[str], + item: entities.BaseItemEntity, reporter: Reporter, - callback: Callable = None, - ) -> List[dict]: + ) -> dict: raise NotImplementedError @abstractmethod - async def get_big_annotation( - self, project: entities.ProjectEntity, item: dict, reporter: Reporter - ) -> dict: + async def list_small_annotations( + self, + project: entities.ProjectEntity, + folder: entities.FolderEntity, + item_ids: List[int], + reporter: Reporter, + callback: Callable = None, + ) -> List[dict]: raise NotImplementedError @abstractmethod @@ -333,7 +336,7 @@ def sort_items_by_size( self, project: entities.ProjectEntity, folder: entities.FolderEntity, - item_names: List[str], + item_ids: List[int], ) -> Dict[str, List]: raise NotImplementedError @@ -356,7 +359,7 @@ async def download_small_annotations( reporter: Reporter, download_path: str, postfix: str, - items: List[str] = None, + item_ids: List[int], callback: Callable = None, ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 986388e45..f6ae1cc56 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -1,7 +1,7 @@ import asyncio -import concurrent.futures import copy import io +import itertools import json import logging import os @@ -12,6 +12,7 @@ from dataclasses import dataclass from datetime import datetime from itertools import islice +from operator import itemgetter from pathlib import Path from typing import Any from typing import Callable @@ -31,6 +32,7 @@ from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import BaseItemEntity +from lib.core.entities import ConfigEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity from lib.core.entities import ProjectEntity @@ -40,6 +42,7 @@ from lib.core.response import Response from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import BaseServiceProvider +from lib.core.serviceproviders import ServiceResponse from lib.core.types import PriorityScoreEntity from lib.core.usecases.base import BaseReportableUseCase from lib.core.video_convertor import VideoFrameGenerator @@ -63,6 +66,18 @@ class Report: missing_attrs: list +def get_or_raise(response: ServiceResponse): + if response.ok: + return response.data + else: + raise AppException(response.error) + + +def divide_to_chunks(it, size): + it = iter(it) + return iter(lambda: tuple(islice(it, size)), ()) + + def log_report( report: Report, ): @@ -110,8 +125,9 @@ def set_annotation_statuses_in_progress( item_names=item_names[i : i + chunk_size], # noqa: E203 annotation_status=constants.AnnotationStatus.IN_PROGRESS.value, ) - if not status_changed: + if not status_changed.ok: failed_on_chunk = True + logger.debug(status_changed.error) return not failed_on_chunk @@ -201,8 +217,8 @@ async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]: if is_uploaded and callback: callback(item_data) return item_data.item.name, is_uploaded - except Exception: - logger.debug(traceback.format_exc()) + except Exception as e: + logger.debug(e) report.failed_annotations.append(item_data.item.name) finally: reporter.update_progress() @@ -657,8 +673,8 @@ async def distribute_queues(self, items_to_upload: List[ItemToUpload]): else: self._small_files_queue.put_nowait(item_to_upload) break - except Exception: - logger.debug(traceback.format_exc()) + except Exception as e: + logger.debug(e) self._report.failed_annotations.append(item_to_upload.item.name) self.reporter.update_progress() data[idx][1] = True @@ -723,8 +739,8 @@ def execute(self): try: nest_asyncio.apply() asyncio.run(self.run_workers(items_to_upload)) - except Exception: - logger.debug(traceback.format_exc()) + except Exception as e: + logger.debug(e) self._response.errors = AppException("Can't upload annotations.") self.reporter.finish_progress() self._log_report() @@ -993,146 +1009,10 @@ def execute(self): return self._response -class GetAnnotations(BaseReportableUseCase): - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - item_names: Optional[List[str]], - service_provider: BaseServiceProvider, - show_process: bool = True, - ): - super().__init__(reporter) - self._project = project - self._folder = folder - self._service_provider = service_provider - self._item_names = item_names - self._show_process = show_process - self._item_names_provided = True - self._big_annotations_queue = None - - def validate_project_type(self): - if self._project.type == constants.ProjectType.PIXEL.value: - raise AppException("The function is not supported for Pixel projects.") - - def validate_item_names(self): - if self._item_names: - item_names = list(dict.fromkeys(self._item_names)) - len_unique_items, len_items = len(item_names), len(self._item_names) - if len_unique_items < len_items: - self.reporter.log_info( - f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." - ) - self._item_names = item_names - elif self._item_names is None: - self._item_names_provided = False - condition = Condition("project_id", self._project.id, EQ) & Condition( - "folder_id", self._folder.id, EQ - ) - - self._item_names = [ - item.name for item in self._service_provider.items.list(condition).data - ] - else: - self._item_names = [] - - def _prettify_annotations(self, annotations: List[dict]): - re_struct = {} - - if self._item_names_provided: - for annotation in annotations: - re_struct[annotation["metadata"]["name"]] = annotation - try: - return [re_struct[x] for x in self._item_names if x in re_struct] - except KeyError: - raise AppException("Broken data.") - - return annotations - - async def get_big_annotation(self): - - large_annotations = [] - while True: - item = await self._big_annotations_queue.get() - if not item: - await self._big_annotations_queue.put(None) - break - large_annotation = ( - await self._service_provider.annotations.get_big_annotation( - project=self._project, - item=item, - reporter=self.reporter, - ) - ) - large_annotations.append(large_annotation) - return large_annotations - - async def get_small_annotations(self, item_names): - return await self._service_provider.annotations.get_small_annotations( - project=self._project, - folder=self._folder, - items=item_names, - reporter=self.reporter, - ) - - async def distribute_to_queue(self, big_annotations): - for item in big_annotations: - await self._big_annotations_queue.put(item) - await self._big_annotations_queue.put(None) - - async def run_workers(self, big_annotations, small_annotations): - self._big_annotations_queue = asyncio.Queue() - annotations = await asyncio.gather( - self.distribute_to_queue(big_annotations), - self.get_small_annotations(small_annotations), - self.get_big_annotation(), - self.get_big_annotation(), - self.get_big_annotation(), - ) - - annotations = [i for x in annotations[1:] for i in x if x] - return annotations - - def execute(self): - if self.is_valid(): - items_count = len(self._item_names) - if not items_count: - self.reporter.log_info("No annotations to download.") - self._response.data = [] - return self._response - self.reporter.log_info( - f"Getting {items_count} annotations from " - f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}." - ) - self.reporter.start_progress(items_count, disable=not self._show_process) - - items = self._service_provider.annotations.sort_items_by_size( - project=self._project, folder=self._folder, item_names=self._item_names - ) - small_annotations = [x["name"] for x in items["small"]] - try: - nest_asyncio.apply() - annotations = asyncio.run( - self.run_workers(items["large"], small_annotations) - ) - except Exception as e: - self.reporter.log_error(str(e)) - self._response.errors = AppException("Can't get annotations.") - return self._response - received_items_count = len(annotations) - self.reporter.finish_progress() - if items_count > received_items_count: - self.reporter.log_warning( - f"Could not find annotations for {items_count - received_items_count}/{items_count} items." - ) - self._response.data = self._prettify_annotations(annotations) - return self._response - - class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( self, + config: ConfigEntity, reporter: Reporter, project: ProjectEntity, folder: FolderEntity, @@ -1141,6 +1021,7 @@ def __init__( service_provider: BaseServiceProvider, ): super().__init__(reporter) + self._config = config self._project = project self._folder = folder self._video_name = video_name @@ -1158,12 +1039,12 @@ def execute(self): if self.is_valid(): self.reporter.disable_info() response = GetAnnotations( - reporter=self.reporter, + config=self._config, + reporter=Reporter(log_info=False), project=self._project, folder=self._folder, item_names=[self._video_name], service_provider=self._service_provider, - show_process=False, ).execute() self.reporter.enable_info() if response.data: @@ -1276,229 +1157,6 @@ def execute(self): return self._response -class DownloadAnnotations(BaseReportableUseCase): - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - destination: str, - recursive: bool, - item_names: List[str], - service_provider: BaseServiceProvider, - callback: Callable = None, - ): - super().__init__(reporter) - self._project = project - self._folder = folder - self._destination = destination - self._recursive = recursive - self._item_names = item_names - self._service_provider = service_provider - self._callback = callback - self._big_file_queues = [] - self._small_file_queues = [] - - def validate_item_names(self): - if self._item_names: - item_names = list(dict.fromkeys(self._item_names)) - len_unique_items, len_items = len(item_names), len(self._item_names) - if len_unique_items < len_items: - self.reporter.log_info( - f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." - ) - self._item_names = item_names - - def validate_destination(self): - if self._destination: - destination = str(self._destination) - if not os.path.exists(destination) or not os.access( - destination, os.X_OK | os.W_OK - ): - raise AppException( - f"Local path {destination} is not an existing directory or access denied." - ) - - @property - def destination(self) -> Path: - return Path(self._destination if self._destination else "") - - def get_postfix(self): - if self._project.type == constants.ProjectType.VECTOR: - return "___objects.json" - elif self._project.type == constants.ProjectType.PIXEL.value: - return "___pixel.json" - return ".json" - - def download_annotation_classes(self, path: str): - response = self._service_provider.annotation_classes.list( - Condition("project_id", self._project.id, EQ) - ) - if response.ok: - classes_path = Path(path) / "classes" - classes_path.mkdir(parents=True, exist_ok=True) - with open(classes_path / "classes.json", "w+", encoding="utf-8") as file: - json.dump( - [ - i.dict( - exclude_unset=True, - by_alias=True, - exclude={ - "attribute_groups": {"__all__": {"is_multiselect"}} - }, - ) - for i in response.data - ], - file, - indent=4, - ) - else: - self._response.errors = AppException("Cant download classes.") - - @staticmethod - def get_items_count(path: str): - return sum([len(files) for r, d, files in os.walk(path)]) - - async def download_big_annotations(self, queue_idx, export_path): - while True: - cur_queue = self._big_file_queues[queue_idx] - item = await cur_queue.get() - cur_queue.task_done() - if item: - postfix = self.get_postfix() - await self._service_provider.annotations.download_big_annotation( - project=self._project, - item=item, - download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", - postfix=postfix, - callback=self._callback, - ) - else: - cur_queue.put_nowait(None) - break - - async def download_small_annotations( - self, queue_idx, export_path, folder: FolderEntity - ): - cur_queue = self._small_file_queues[queue_idx] - items = [] - item = "" - postfix = self.get_postfix() - while item is not None: - item = await cur_queue.get() - if item: - items.append(item) - await self._service_provider.annotations.download_small_annotations( - project=self._project, - folder=folder, - items=items, - reporter=self.reporter, - download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", - postfix=postfix, - callback=self._callback, - ) - - async def distribute_to_queues( - self, item_names, sm_queue_id, l_queue_id, folder: FolderEntity - ): - try: - resp = self._service_provider.annotations.sort_items_by_size( - project=self._project, folder=folder, item_names=item_names - ) - - for item in resp["large"]: - await self._big_file_queues[l_queue_id].put(item) - - for item in resp["small"]: - await self._small_file_queues[sm_queue_id].put(item["name"]) - finally: - await self._big_file_queues[l_queue_id].put(None) - await self._small_file_queues[sm_queue_id].put(None) - - async def run_workers(self, item_names, folder: FolderEntity, export_path): - try: - self._big_file_queues.append(asyncio.Queue()) - self._small_file_queues.append(asyncio.Queue()) - small_file_queue_idx = len(self._small_file_queues) - 1 - big_file_queue_idx = len(self._big_file_queues) - 1 - res = await asyncio.gather( - self.distribute_to_queues( - item_names, small_file_queue_idx, big_file_queue_idx, folder - ), - self.download_big_annotations(big_file_queue_idx, export_path), - self.download_big_annotations(big_file_queue_idx, export_path), - self.download_big_annotations(big_file_queue_idx, export_path), - self.download_small_annotations( - small_file_queue_idx, export_path, folder - ), - return_exceptions=True, - ) - if any(res): - self.reporter.log_error(f"Error {str([i for i in res if i])}") - except Exception as e: - self.reporter.log_error(f"Error {str(e)}") - - def execute(self): - if self.is_valid(): - export_path = str( - self.destination - / Path( - f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}" - ) - ) - self.reporter.log_info( - f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" - ) - self.reporter.start_spinner() - - folders = [] - if self._folder.is_root and self._recursive: - folders = self._service_provider.folders.list( - Condition("project_id", self._project.id, EQ) - ).data - if not folders: - folders.append(self._folder) - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - nest_asyncio.apply() - futures = [] - for folder in folders: - if not self._item_names: - condition = Condition( - "project_id", self._project.id, EQ - ) & Condition("folder_id", folder.id, EQ) - item_names = [ - item.name - for item in self._service_provider.items.list( - condition - ).data - ] - else: - item_names = self._item_names - new_export_path = export_path - if not folder.is_root and self._folder.is_root: - new_export_path += f"/{folder.name}" - if not item_names: - continue - futures.append( - executor.submit( - asyncio.run, - self.run_workers(item_names, folder, new_export_path), - ) - ) - - for future in concurrent.futures.as_completed(futures): - exception = future.exception() - if exception: - self._response.errors = exception - - self.reporter.stop_spinner() - count = self.get_items_count(export_path) - self.reporter.log_info(f"Downloaded annotations for {count} items.") - self.download_annotation_classes(export_path) - self._response.data = os.path.abspath(export_path) - return self._response - - class ValidateAnnotationUseCase(BaseReportableUseCase): DEFAULT_VERSION = "V1.00" SCHEMAS: Dict[str, Draft7Validator] = {} @@ -1715,3 +1373,422 @@ def execute(self) -> Response: self._response.data = list(sorted(errors_report, key=lambda x: x[0])) return self._response + + +class GetAnnotations(BaseReportableUseCase): + def __init__( + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + item_names: Optional[List[str]], + service_provider: BaseServiceProvider, + ): + super().__init__(reporter) + self._config = config + self._project = project + self._folder = folder + self._service_provider = service_provider + self._item_names = item_names + self._item_names_provided = True + self._big_annotations_queue = None + self._small_annotations_queue = None + + def validate_project_type(self): + if self._project.type == constants.ProjectType.PIXEL.value: + raise AppException("The function is not supported for Pixel projects.") + + def validate_item_names(self): + if self._item_names: # if names provided + unique_item_names = list(set(self._item_names)) + len_unique_items, len_items = len(unique_item_names), len(self._item_names) + if len_unique_items < len_items: + self.reporter.log_info( + f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." + ) + # Keep order required + self._item_names = [i for i in self._item_names if i in unique_item_names] + elif self._item_names is None: + self._item_names_provided = False + condition = Condition("project_id", self._project.id, EQ) & Condition( + "folder_id", self._folder.id, EQ + ) + + self._item_names = [ + item.name for item in self._service_provider.items.list(condition).data + ] + else: + self._item_names = [] + + def _prettify_annotations(self, annotations: List[dict]): + re_struct = {} + if self._item_names_provided: + for annotation in annotations: + re_struct[annotation["metadata"]["name"]] = annotation + try: + return [re_struct[x] for x in self._item_names if x in re_struct] + except KeyError: + raise AppException("Broken data.") + + return annotations + + async def get_big_annotation(self): + large_annotations = [] + while True: + item: BaseItemEntity = await self._big_annotations_queue.get() + if item: + large_annotations.append( + await self._service_provider.annotations.get_big_annotation( + project=self._project, + item=item, + reporter=self.reporter, + ) + ) + else: + await self._big_annotations_queue.put(None) + break + return large_annotations + + async def get_small_annotations(self): + small_annotations = [] + while True: + items = await self._small_annotations_queue.get() + if items: + annotations = ( + await self._service_provider.annotations.list_small_annotations( + project=self._project, + folder=self._folder, + item_ids=[i.id for i in items], + reporter=self.reporter, + ) + ) + small_annotations.extend(annotations) + else: + await self._small_annotations_queue.put(None) + break + return small_annotations + + async def run_workers( + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[BaseItemEntity], + ): + annotations = [] + if big_annotations: + self._big_annotations_queue = asyncio.Queue() + for item in big_annotations: + self._big_annotations_queue.put_nowait(item) + self._big_annotations_queue.put_nowait(None) + annotations.extend( + list( + itertools.chain.from_iterable( + await asyncio.gather( + *[ + self.get_big_annotation() + for _ in range( + max(self._config.MAX_COROUTINE_COUNT // 2, 1) + ) + ] + ) + ) + ) + ) + if small_annotations: + self._small_annotations_queue = asyncio.Queue() + small_chunks = divide_to_chunks( + small_annotations, size=self._config.ANNOTATION_CHUNK_SIZE + ) + for chunk in small_chunks: + self._small_annotations_queue.put_nowait(chunk) + self._small_annotations_queue.put_nowait(None) + + annotations.extend( + list( + itertools.chain.from_iterable( + await asyncio.gather( + *[ + self.get_small_annotations() + for _ in range(self._config.MAX_COROUTINE_COUNT) + ] + ) + ) + ) + ) + return list(filter(None, annotations)) + + def execute(self): + if self.is_valid(): + if self._item_names: + items = get_or_raise( + self._service_provider.items.list_by_names( + self._project, self._folder, self._item_names + ) + ) + else: + condition = Condition("project_id", self._project.id, EQ) & Condition( + "folder_id", self._folder.id, EQ + ) + items = get_or_raise(self._service_provider.items.list(condition)) + id_item_map = {i.id: i for i in items} + + if not items: + logger.info("No annotations to download.") + self._response.data = [] + return self._response + items_count = len(items) + logger.info( + f"Getting {items_count} annotations from " + f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}." + ) + self.reporter.start_progress( + items_count, + disable=logger.level > logging.INFO or self.reporter.log_enabled, + ) + + sort_response = self._service_provider.annotations.sort_items_by_size( + project=self._project, folder=self._folder, item_ids=list(id_item_map) + ) + large_item_ids = set(map(itemgetter("id"), sort_response["large"])) + small_items_ids = set(map(itemgetter("id"), sort_response["small"])) + large_items = list(filter(lambda item: item.id in large_item_ids, items)) + small_items = list(filter(lambda item: item.id in small_items_ids, items)) + try: + nest_asyncio.apply() + annotations = asyncio.run(self.run_workers(large_items, small_items)) + except Exception as e: + logger.error(e) + self._response.errors = AppException("Can't get annotations.") + return self._response + received_items_count = len(annotations) + self.reporter.finish_progress() + if items_count > received_items_count: + self.reporter.log_warning( + f"Could not find annotations for {items_count - received_items_count}/{items_count} items." + ) + self._response.data = self._prettify_annotations(annotations) + return self._response + + +class DownloadAnnotations(BaseReportableUseCase): + def __init__( + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + destination: str, + recursive: bool, + item_names: List[str], + service_provider: BaseServiceProvider, + callback: Callable = None, + ): + super().__init__(reporter) + self._config = config + self._project = project + self._folder = folder + self._destination = destination + self._recursive = recursive + self._item_names = item_names + self._service_provider = service_provider + self._callback = callback + self._big_file_queue = None + self._small_file_queue = None + + def validate_item_names(self): + if self._item_names: + item_names = list(dict.fromkeys(self._item_names)) + len_unique_items, len_items = len(item_names), len(self._item_names) + if len_unique_items < len_items: + self.reporter.log_info( + f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." + ) + self._item_names = item_names + + def validate_destination(self): + if self._destination: + destination = str(self._destination) + if not os.path.exists(destination) or not os.access( + destination, os.X_OK | os.W_OK + ): + raise AppException( + f"Local path {destination} is not an existing directory or access denied." + ) + + @property + def destination(self) -> Path: + return Path(self._destination if self._destination else "") + + def get_postfix(self): + if self._project.type == constants.ProjectType.VECTOR: + return "___objects.json" + elif self._project.type == constants.ProjectType.PIXEL.value: + return "___pixel.json" + return ".json" + + def download_annotation_classes(self, path: str): + response = self._service_provider.annotation_classes.list( + Condition("project_id", self._project.id, EQ) + ) + if response.ok: + classes_path = Path(path) / "classes" + classes_path.mkdir(parents=True, exist_ok=True) + with open(classes_path / "classes.json", "w+", encoding="utf-8") as file: + json.dump( + [ + i.dict( + exclude_unset=True, + by_alias=True, + exclude={ + "attribute_groups": {"__all__": {"is_multiselect"}} + }, + ) + for i in response.data + ], + file, + indent=4, + ) + else: + self._response.errors = AppException("Cant download classes.") + + @staticmethod + def get_items_count(path: str): + return sum([len(files) for r, d, files in os.walk(path)]) + + async def download_big_annotations(self, export_path): + while True: + item = await self._big_file_queue.get() + self._big_file_queue.task_done() + if item: + postfix = self.get_postfix() + await self._service_provider.annotations.download_big_annotation( + project=self._project, + item=item, + download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", + postfix=postfix, + callback=self._callback, + ) + else: + self._big_file_queue.put_nowait(None) + break + + async def download_small_annotations(self, export_path, folder: FolderEntity): + postfix = self.get_postfix() + while True: + items = await self._small_file_queue.get() + if items: + await self._service_provider.annotations.download_small_annotations( + project=self._project, + folder=folder, + item_ids=[i.id for i in items], + reporter=self.reporter, + download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", + postfix=postfix, + callback=self._callback, + ) + else: + self._small_file_queue.put_nowait(None) + break + + async def run_workers( + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[BaseItemEntity], + folder: FolderEntity, + export_path, + ): + if big_annotations: + self._big_file_queue = asyncio.Queue() + for item in big_annotations: + self._big_file_queue.put_nowait(item) + self._big_file_queue.put_nowait(None) + await asyncio.gather( + *[ + self.download_big_annotations(export_path) + for _ in range(max(self._config.MAX_COROUTINE_COUNT // 2, 1)) + ] + ) + + if small_annotations: + self._small_file_queue = asyncio.Queue() + small_chunks = divide_to_chunks( + small_annotations, size=self._config.ANNOTATION_CHUNK_SIZE + ) + for chunk in small_chunks: + self._small_file_queue.put_nowait(chunk) + self._small_file_queue.put_nowait(None) + await asyncio.gather( + *[ + self.download_small_annotations(export_path, folder) + for _ in range(self._config.MAX_COROUTINE_COUNT) + ] + ) + + def execute(self): + if self.is_valid(): + export_path = str( + self.destination + / Path( + f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}" + ) + ) + logger.info( + f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" + ) + self.reporter.start_spinner() + folders = [] + if self._folder.is_root and self._recursive: + folders = self._service_provider.folders.list( + Condition("project_id", self._project.id, EQ) + ).data + if not folders: + folders.append(self._folder) + nest_asyncio.apply() + for folder in folders: + if self._item_names: + items = get_or_raise( + self._service_provider.items.list_by_names( + self._project, folder, self._item_names + ) + ) + else: + condition = Condition( + "project_id", self._project.id, EQ + ) & Condition("folder_id", folder.id, EQ) + items = get_or_raise(self._service_provider.items.list(condition)) + if not items: + continue + new_export_path = export_path + if not folder.is_root and self._folder.is_root: + new_export_path += f"/{folder.name}" + + id_item_map = {i.id: i for i in items} + sort_response = self._service_provider.annotations.sort_items_by_size( + project=self._project, + folder=self._folder, + item_ids=list(id_item_map), + ) + large_item_ids = set(map(itemgetter("id"), sort_response["large"])) + small_items_ids = set(map(itemgetter("id"), sort_response["small"])) + large_items = list( + filter(lambda item: item.id in large_item_ids, items) + ) + small_items = list( + filter(lambda item: item.id in small_items_ids, items) + ) + try: + asyncio.run( + self.run_workers( + large_items, small_items, folder, new_export_path + ) + ) + except Exception as e: + logger.error(e) + self._response.errors = AppException("Can't get annotations.") + return self._response + self.reporter.stop_spinner() + count = self.get_items_count(export_path) + self.reporter.log_info(f"Downloaded annotations for {count} items.") + self.download_annotation_classes(export_path) + self._response.data = os.path.abspath(export_path) + return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index a1d3efef2..40e2d1e62 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -503,6 +503,10 @@ def update(self, project: ProjectEntity, item: BaseItemEntity): class AnnotationManager(BaseManager): + def __init__(self, service_provider: ServiceProvider, config: ConfigEntity): + super().__init__(service_provider) + self._config = config + def list( self, project: ProjectEntity, @@ -511,6 +515,7 @@ def list( verbose=True, ): use_case = usecases.GetAnnotations( + config=self._config, reporter=Reporter(log_info=verbose, log_warning=verbose), project=project, folder=folder, @@ -529,6 +534,7 @@ def download( callback: Optional[Callable], ): use_case = usecases.DownloadAnnotations( + config=self._config, reporter=Reporter(), project=project, folder=folder, @@ -808,7 +814,7 @@ def __init__(self, config: ConfigEntity): self.projects = ProjectManager(self.service_provider) self.folders = FolderManager(self.service_provider) self.items = ItemManager(self.service_provider) - self.annotations = AnnotationManager(self.service_provider) + self.annotations = AnnotationManager(self.service_provider, config) self.custom_fields = CustomFieldManager(self.service_provider) self.subsets = SubsetManager(self.service_provider) self.models = ModelManager(self.service_provider) @@ -1226,6 +1232,7 @@ def get_annotations_per_frame( folder = self.get_folder(project, folder_name) use_case = usecases.GetVideoAnnotationsPerFrame( + config=self._config, reporter=self.get_default_reporter(), project=project, folder=folder, diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py index cdb6278e7..be87a312c 100644 --- a/src/superannotate/lib/infrastructure/services/annotation.py +++ b/src/superannotate/lib/infrastructure/services/annotation.py @@ -29,9 +29,6 @@ class AnnotationService(BaseAnnotationService): URL_GET_ANNOTATIONS = "items/annotations/download" URL_UPLOAD_ANNOTATIONS = "items/annotations/upload" - URL_LARGE_ANNOTATION = "items/{item_id}/annotations/download" - URL_SYNC_LARGE_ANNOTATION = "items/{item_id}/annotations/sync" - URL_SYNC_LARGE_ANNOTATION_STATUS = "items/{item_id}/annotations/sync/status" URL_CLASSIFY_ITEM_SIZE = "items/annotations/download/method" URL_DOWNLOAD_LARGE_ANNOTATION = "items/{item_id}/annotations/download" URL_START_FILE_UPLOAD_PROCESS = "items/{item_id}/annotations/upload/multipart/start" @@ -45,6 +42,7 @@ class AnnotationService(BaseAnnotationService): @property def assets_provider_url(self): + if self.client.api_url != constants.BACKEND_URL: return f"https://assets-provider.devsuperannotate.com/api/{self.ASSETS_PROVIDER_VERSION}/" return f"https://assets-provider.superannotate.com/api/{self.ASSETS_PROVIDER_VERSION}/" @@ -71,7 +69,7 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): } sync_url = urljoin( self.assets_provider_url, - self.URL_SYNC_LARGE_ANNOTATION.format(item_id=item_id), + self.URL_START_FILE_SYNC.format(item_id=item_id), ) async with aiohttp.ClientSession( connector=aiohttp.TCPConnector(ssl=False), @@ -86,7 +84,7 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): synced = False sync_status_url = urljoin( self.assets_provider_url, - self.URL_SYNC_LARGE_ANNOTATION_STATUS.format(item_id=item_id), + self.URL_START_FILE_SYNC_STATUS.format(item_id=item_id), ) while synced != "SUCCESS": synced = await session.get(sync_status_url, params=sync_params) @@ -96,11 +94,14 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): return synced async def get_big_annotation( - self, project: entities.ProjectEntity, item: dict, reporter: Reporter + self, + project: entities.ProjectEntity, + item: entities.BaseItemEntity, + reporter: Reporter, ) -> dict: url = urljoin( self.assets_provider_url, - self.URL_LARGE_ANNOTATION.format(item_id=item["id"]), + self.URL_DOWNLOAD_LARGE_ANNOTATION.format(item_id=item.id), ) query_params = { @@ -111,7 +112,7 @@ async def get_big_annotation( } await self._sync_large_annotation( - team_id=project.team_id, project_id=project.id, item_id=item["id"] + team_id=project.team_id, project_id=project.id, item_id=item.id ) async with aiohttp.ClientSession( @@ -125,11 +126,11 @@ async def get_big_annotation( reporter.update_progress() return large_annotation - async def get_small_annotations( + async def list_small_annotations( self, project: entities.ProjectEntity, folder: entities.FolderEntity, - items: List[str], + item_ids: List[int], reporter: Reporter, callback: Callable = None, ) -> List[dict]: @@ -142,26 +143,21 @@ async def get_small_annotations( handler = StreamedAnnotations( self.client.default_headers, reporter, - map_function=lambda x: {"image_names": x}, + map_function=lambda x: {"image_ids": x}, callback=callback, ) - - loop = asyncio.new_event_loop() - - return loop.run_until_complete( - handler.get_data( - url=urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS), - data=items, - params=query_params, - chunk_size=self.DEFAULT_CHUNK_SIZE, - ) + return await handler.list_annotations( + method="post", + url=urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS), + data=item_ids, + params=query_params, ) def sort_items_by_size( self, project: entities.ProjectEntity, folder: entities.FolderEntity, - item_names: List[str], + item_ids: List[int], ) -> Dict[str, List]: chunk_size = 2000 query_params = { @@ -170,10 +166,9 @@ def sort_items_by_size( } response_data = {"small": [], "large": []} - for i in range(0, len(item_names), chunk_size): + for i in range(0, len(item_ids), chunk_size): body = { - "item_names": item_names[i : i + chunk_size], # noqa - "folder_id": folder.id, + "item_ids": item_ids[i : i + chunk_size], # noqa } # noqa response = self.client.request( url=urljoin(self.assets_provider_url, self.URL_CLASSIFY_ITEM_SIZE), @@ -235,7 +230,7 @@ async def download_small_annotations( reporter: Reporter, download_path: str, postfix: str, - items: List[str] = None, + item_ids: List[int], callback: Callable = None, ): query_params = { @@ -246,15 +241,15 @@ async def download_small_annotations( handler = StreamedAnnotations( headers=self.client.default_headers, reporter=reporter, - map_function=lambda x: {"image_names": x}, + map_function=lambda x: {"image_ids": x}, callback=callback, ) - return await handler.download_data( + return await handler.download_annotations( + method="post", url=urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS), - data=items, + data=item_ids, params=query_params, - chunk_size=self.DEFAULT_CHUNK_SIZE, download_path=download_path, postfix=postfix, ) @@ -301,6 +296,7 @@ async def upload_small_annotations( if not _response.ok: logger.debug(await _response.text()) raise AppException("Can't upload annotations.") + logger.debug(_response.status) data_json = await _response.json() response = UploadAnnotationsResponse() response.status = _response.status diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py index cf092cb96..6c5746183 100644 --- a/src/superannotate/lib/infrastructure/services/http_client.py +++ b/src/superannotate/lib/infrastructure/services/http_client.py @@ -207,6 +207,6 @@ def serialize_response( data["data"] = data_json return content_type(**data) except json.decoder.JSONDecodeError: - data['_error'] = response.content + data["_error"] = response.content data["reason"] = response.reason return content_type(**data) diff --git a/src/superannotate/lib/infrastructure/services/item.py b/src/superannotate/lib/infrastructure/services/item.py index 5837701d1..1870c52bf 100644 --- a/src/superannotate/lib/infrastructure/services/item.py +++ b/src/superannotate/lib/infrastructure/services/item.py @@ -22,15 +22,15 @@ class ItemService(BaseItemService): URL_LIST = "items" URL_GET = "image/{}" - URL_LIST_BY_NAMES = "images/getBulk" URL_ATTACH = "image/ext-create" + URL_GET_BY_ID = "image/{image_id}" URL_MOVE_MULTIPLE = "image/move" + URL_SET_ANNOTATION_STATUSES = "image/updateAnnotationStatusBulk" + URL_LIST_BY_NAMES = "images/getBulk" URL_COPY_MULTIPLE = "images/copy-image-or-folders" URL_COPY_PROGRESS = "images/copy-image-progress" URL_DELETE_ITEMS = "image/delete/images" - URL_SET_ANNOTATION_STATUSES = "image/updateAnnotationStatusBulk" URL_SET_APPROVAL_STATUSES = "/items/bulk/change" - URL_GET_BY_ID = "image/{image_id}" PROJECT_TYPE_RESPONSE_MAP = { ProjectType.VECTOR: ImageResponse, diff --git a/src/superannotate/lib/infrastructure/services/project.py b/src/superannotate/lib/infrastructure/services/project.py index c251da22b..c304b0d48 100644 --- a/src/superannotate/lib/infrastructure/services/project.py +++ b/src/superannotate/lib/infrastructure/services/project.py @@ -13,8 +13,8 @@ class ProjectService(BaseProjectService): URL_LIST = "projects" URL_GET = "project/{}" URL_SETTINGS = "project/{}/settings" - URL_SHARE = "project/{}/share/bulk" URL_WORKFLOW = "project/{}/workflow" + URL_SHARE = "project/{}/share/bulk" URL_SHARE_PROJECT = "project/{}/share" URL_WORKFLOW_ATTRIBUTE = "project/{}/workflow_attribute" URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 3d4569bbc..345ada58c 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -1,7 +1,7 @@ -import asyncio import copy import json import os +import typing from typing import Callable import aiohttp @@ -30,6 +30,12 @@ def __init__( self._map_function = map_function self._items_downloaded = 0 + def get_json(self, data: bytes): + try: + return json.loads(data) + except json.decoder.JSONDecodeError as e: + self._reporter.log_error(f"Invalud chunk: {str(e)}") + async def fetch( self, method: str, @@ -41,35 +47,30 @@ async def fetch( kwargs = {"params": params, "json": {"folder_id": params.pop("folder_id")}} if data: kwargs["json"].update(data) - response = await session._request(method, url, **kwargs, timeout=TIMEOUT) + response = await session._request( # noqa + method, url, **kwargs, timeout=TIMEOUT + ) buffer = b"" async for line in response.content.iter_any(): - slices = line.split(self.DELIMITER) - if len(slices) == 2 and slices[0]: - self._reporter.update_progress() - buffer += slices[0] - yield json.loads(buffer) - buffer = b"" - elif len(slices) > 2: - for _slice in slices[:-1]: - if not _slice: - continue - self._reporter.update_progress() - yield json.loads(buffer + _slice) - buffer = b"" - buffer += slices[-1] + slices = (buffer + line).split(self.DELIMITER) + for _slice in slices[:-1]: + yield self.get_json(_slice) + buffer = slices[-1] if buffer: - yield json.loads(buffer) + yield self.get_json(buffer) self._reporter.update_progress() - async def process_chunk( + async def list_annotations( self, method: str, url: str, - data: dict = None, + data: typing.List[int] = None, params: dict = None, verify_ssl=False, ): + params = copy.copy(params) + params["limit"] = len(data) + annotations = [] async with aiohttp.ClientSession( headers=self._headers, timeout=TIMEOUT, @@ -81,21 +82,25 @@ async def process_chunk( session, url, self._process_data(data), - params=params, + params=copy.copy(params), ): - self._annotations.append( + annotations.append( self._callback(annotation) if self._callback else annotation ) - async def store_chunk( + return annotations + + async def download_annotations( self, method: str, url: str, download_path, postfix, - data: dict = None, + data: typing.List[int], params: dict = None, ): + params = copy.copy(params) + params["limit"] = len(data) async with aiohttp.ClientSession( headers=self._headers, timeout=TIMEOUT, @@ -120,31 +125,6 @@ async def store_chunk( ) self._items_downloaded += 1 - async def get_data( - self, - url: str, - data: list, - method: str = "post", - params=None, - chunk_size: int = 5000, - verify_ssl: bool = False, - ): - params["limit"] = chunk_size - await asyncio.gather( - *[ - self.process_chunk( - method=method, - url=url, - data=data[i : i + chunk_size], - params=copy.copy(params), - verify_ssl=verify_ssl, - ) - for i in range(0, len(data), chunk_size) - ] - ) - - return self._annotations - @staticmethod def _store_annotation(path, postfix, annotation: dict, callback: Callable = None): os.makedirs(path, exist_ok=True) @@ -156,32 +136,3 @@ def _process_data(self, data): if data and self._map_function: return self._map_function(data) return data - - async def download_data( - self, - url: str, - data: list, - download_path: str, - postfix: str, - method: str = "post", - params=None, - chunk_size: int = 5000, - ) -> int: - """ - Returns the number of items downloaded - """ - params["limit"] = chunk_size - await asyncio.gather( - *[ - self.store_chunk( - method=method, - url=url, - data=data[i : i + chunk_size], # noqa - params=copy.copy(params), - download_path=download_path, - postfix=postfix, - ) - for i in range(0, len(data), chunk_size) - ] - ) - return self._items_downloaded diff --git a/tests/integration/annotations/test_annotation_delete.py b/tests/integration/annotations/test_annotation_delete.py index 57b705af7..5d4943b71 100644 --- a/tests/integration/annotations/test_annotation_delete.py +++ b/tests/integration/annotations/test_annotation_delete.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from src.superannotate import AppException from src.superannotate import SAClient from tests.integration.base import BaseTestCase @@ -29,16 +30,13 @@ def classes_json(self): ) def test_delete_annotations(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=1) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, f"{self.folder_path}" ) - sa.delete_annotations(self.PROJECT_NAME) annotations = sa.get_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1]) del annotations[0]["metadata"]["projectId"] @@ -46,8 +44,8 @@ def test_delete_annotations(self): { "metadata": { "name": "example_image_1.jpg", - "height": 683, - "width": 1024, + "height": None, + "width": None, "isPredicted": False, "status": "NotStarted", "pinned": False, @@ -61,9 +59,7 @@ def test_delete_annotations(self): ] def test_delete_annotations_by_name(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=1) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) @@ -77,8 +73,8 @@ def test_delete_annotations_by_name(self): { "metadata": { "name": "example_image_1.jpg", - "height": 683, - "width": 1024, + "height": None, + "width": None, "isPredicted": False, "status": "NotStarted", "pinned": False, @@ -92,27 +88,22 @@ def test_delete_annotations_by_name(self): ] def test_delete_annotations_by_not_existing_name(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=1) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, f"{self.folder_path}" ) - self.assertRaises( - Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2] - ) + with self.assertRaisesRegexp( + AppException, "Invalid item names or empty folder." + ): + sa.delete_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2]) @pytest.mark.flaky(reruns=2) def test_delete_annotations_wrong_path(self): sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) + self._attach_items(count=1, folder=self.TEST_FOLDER_NAME) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) @@ -122,23 +113,3 @@ def test_delete_annotations_wrong_path(self): self.assertRaises( Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1] ) - - def test_delete_annotations_from_folder(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) - sa.create_annotation_classes_from_classes_json( - self.PROJECT_NAME, self.folder_path + "/classes/classes.json" - ) - sa.upload_annotations_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", f"{self.folder_path}" - ) - sa.delete_annotations( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", [self.EXAMPLE_IMAGE_1] - ) - annotations = sa.get_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1]) - assert len(annotations) == 0 diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index 450cacdec..088b60ebd 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -66,12 +66,7 @@ def test_get_annotations_order(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_from_folder(self): sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) + self._attach_items(count=4, folder=self.FOLDER_NAME) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) @@ -93,9 +88,7 @@ def test_get_annotations_from_folder(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_all(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=4) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) @@ -108,14 +101,9 @@ def test_get_annotations_all(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_all_plus_folder(self): sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) + self._attach_items(count=4) + self._attach_items(count=4, folder=self.FOLDER_NAME) + sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) @@ -138,6 +126,19 @@ def test_get_annotations10000(self): a = sa.get_annotations(self.PROJECT_NAME) assert len(a) == count + def test_get_annotation(self): # to_delete + count = 2 + sa.attach_items( + self.PROJECT_NAME, + [ + {"name": f"example_image_{i}.jpg", "url": f"url_{i}"} + for i in range(count) + ], # noqa + ) + assert len(sa.search_items(self.PROJECT_NAME)) == count + a = sa.get_annotations(self.PROJECT_NAME) + assert len(a) == count + class TestGetAnnotationsVideo(BaseTestCase): PROJECT_NAME = "test attach multiple video urls" diff --git a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py index 922581f53..5f8a5647a 100644 --- a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py +++ b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py @@ -8,8 +8,7 @@ sa = SAClient() -class \ - TestAnnotationUploadVector(BaseTestCase): +class TestAnnotationUploadVector(BaseTestCase): PROJECT_NAME = "Test-Upload_annotations_from_folder_to_project" PROJECT_DESCRIPTION = "Desc" PROJECT_TYPE = "Vector" diff --git a/tests/integration/base.py b/tests/integration/base.py index bc88c80a4..206938d63 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -18,7 +18,6 @@ def __init__(self, *args, **kwargs): def setUp(self, *args, **kwargs): self.tearDown() - print(self.PROJECT_NAME) self._project = sa.create_project( self.PROJECT_NAME, self.PROJECT_DESCRIPTION, self.PROJECT_TYPE ) diff --git a/tests/integration/classes/test_create_bed_handling.py b/tests/integration/classes/test_create_bed_handling.py index d13ff55a1..926eb872b 100644 --- a/tests/integration/classes/test_create_bed_handling.py +++ b/tests/integration/classes/test_create_bed_handling.py @@ -1,3 +1,4 @@ +import pytest from src.superannotate import SAClient from tests.integration.base import BaseTestCase @@ -11,6 +12,7 @@ class TestCreateAnnotationClass(BaseTestCase): TEST_LARGE_CLASSES_JSON = "large_classes_json.json" EXAMPLE_IMAGE_1 = "example_image_1.jpg" + @pytest.mark.flaky(reruns=2) def test_multi_select_to_checklist(self): sa.create_annotation_class( self.PROJECT_NAME, diff --git a/tests/integration/custom_fields/test_custom_schema.py b/tests/integration/custom_fields/test_custom_schema.py index 7837c5d3d..1f52059ee 100644 --- a/tests/integration/custom_fields/test_custom_schema.py +++ b/tests/integration/custom_fields/test_custom_schema.py @@ -12,6 +12,29 @@ class TestCustomSchema(BaseTestCase): PROJECT_DESCRIPTION = "desc" PROJECT_TYPE = "Vector" PAYLOAD = {"test": {"type": "number"}, "tester": {"type": "number"}} + INITIAL_ALL_VALID = { + "test_string_email": { + "type": "string", + "format": "email", + "enum": [ + "abc@gmail.com", + "afg@ahg.com", + "name+@gmail.com", + "name+surname@mail.ru", + ], + }, + "test_type_number": {"type": "number"}, + "number_and_range": {"type": "number", "minimum": -1, "maximum": 100.5}, + "string_and_enum": {"type": "string", "enum": ["one", "two", "tree"]}, + "string_and_date": {"type": "string", "format": "date"}, + "number_and_enum": {"type": "number", "enum": [1.2, 0, -345, 100, 1, 0.1]}, + "string_date_enum": { + "type": "string", + "format": "date", + "enum": ["2022-12-11", "2000-10-9"], + }, + "just_string": {"type": "string"}, + } def test_create_schema(self): data = sa.create_custom_fields(self.PROJECT_NAME, self.PAYLOAD) @@ -35,6 +58,12 @@ def test_get_schema(self): sa.create_custom_fields(self.PROJECT_NAME, self.PAYLOAD) self.assertEqual(sa.get_custom_fields(self.PROJECT_NAME), self.PAYLOAD) + def test_create_large_schema(self): + sa.create_custom_fields(self.PROJECT_NAME, self.INITIAL_ALL_VALID) + self.assertEqual( + sa.get_custom_fields(self.PROJECT_NAME), self.INITIAL_ALL_VALID + ) + def test_delete_schema(self): payload = copy.copy(self.PAYLOAD) sa.create_custom_fields(self.PROJECT_NAME, payload) diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 2a83e14ba..d7b9cdf36 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -21,8 +21,6 @@ class TestInterface(BaseTestCase): EXAMPLE_IMAGE_1 = "example_image_1.jpg" EXAMPLE_IMAGE_2 = "example_image_2.jpg" NEW_IMAGE_NAME = "new_name_yup" - IMAGE_PATH_IN_S3 = "MP.MB/img1.bmp" - TEST_S3_BUCKET_NAME = "test-openseadragon-1212" TEST_INVALID_ANNOTATION_FOLDER_PATH = "sample_project_vector_invalid" @property @@ -45,27 +43,6 @@ def folder_path_with_multiple_images(self): dirname(dirname(__file__)), self.TEST_FOLDER_PATH_WITH_MULTIPLE_IMAGERS ) - @pytest.mark.flaky(reruns=4) - def test_delete_items(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) - - path = f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}" - sa.upload_images_from_folder_to_project( - path, - self.folder_path, - annotation_status="InProgress", - ) - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 4) - sa.delete_items(path) - - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 0) - def test_delete_folder(self): with self.assertRaises(AppException): sa.delete_folders(self.PROJECT_NAME, ["non-existing folder"]) @@ -84,11 +61,7 @@ def test_get_project_metadata(self): self.assertIsNotNone(metadata_with_users.get("contributors")) def test_upload_annotations_from_folder_to_project(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, - self.folder_path, - annotation_status="Completed", - ) + self._attach_items(count=4) uploaded_annotations, _, _ = sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, self.folder_path ) @@ -161,18 +134,6 @@ def test_upload_images_to_project_image_quality_in_editor(self): image_quality_in_editor="random_string", ) - @pytest.mark.flaky(reruns=2) - def test_image_upload_with_set_name_on_platform(self): - sa.upload_image_to_project( - self.PROJECT_NAME, - self.IMAGE_PATH_IN_S3, - self.NEW_IMAGE_NAME, - from_s3_bucket=self.TEST_S3_BUCKET_NAME, - ) - assert self.NEW_IMAGE_NAME in [ - i["name"] for i in sa.search_items(self.PROJECT_NAME) - ] - def test_download_fuse_without_classes(self): sa.upload_image_to_project( self.PROJECT_NAME, f"{self.folder_path}/{self.EXAMPLE_IMAGE_1}" diff --git a/tests/integration/test_recursive_folder_pixel.py b/tests/integration/test_recursive_folder_pixel.py deleted file mode 100644 index c27586488..000000000 --- a/tests/integration/test_recursive_folder_pixel.py +++ /dev/null @@ -1,30 +0,0 @@ -from src.superannotate import SAClient -from tests.integration.base import BaseTestCase - -sa = SAClient() - - -class TestRecursiveFolderPixel(BaseTestCase): - PROJECT_NAME = "pixel_recursive_test" - PROJECT_DESCRIPTION = "Desc" - PROJECT_TYPE = "Pixel" - S3_FOLDER_PATH = "pixel_recursive_annotations" - JSON_POSTFIX = "*.json" - - def test_recursive_upload_pixel(self): - uploaded, _, duplicated = sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, - self.S3_FOLDER_PATH, - from_s3_bucket="test-openseadragon-1212", - recursive_subfolders=True, - ) - - uploaded, failed, missing = sa.upload_annotations_from_folder_to_project( - self.PROJECT_NAME, - self.S3_FOLDER_PATH, - from_s3_bucket="test-openseadragon-1212", - recursive_subfolders=True, - ) - self.assertEqual(4, len(uploaded)) - self.assertEqual(0, len(failed)) - self.assertEqual(1, len(missing)) diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index a304ea091..491146b7c 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -19,9 +19,7 @@ def folder_path(self): def test_upload_priority_scores(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=4) uploaded, skipped = sa.upload_priority_scores( self.PROJECT_NAME, scores=[{"name": "example_image_1.jpg", "priority": 1}] ) diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 741f6c28d..dcb353d70 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -6,7 +6,6 @@ from unittest.mock import patch import superannotate.lib.core as constants -from superannotate.lib.app.interface.types import validate_arguments from superannotate import AppException from superannotate import SAClient @@ -53,12 +52,14 @@ def test_init_via_config_json_invalid_json(self): config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path), patch( - "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path + "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path ): with open(f"{config_dir}/config.json", "w") as config_json: json.dump({"token": "INVALID_TOKEN"}, config_json) for kwargs in ({}, {"config_path": f"{config_dir}/config.json"}): - with self.assertRaisesRegexp(AppException, r"(\s+)token(\s+)Invalid token."): + with self.assertRaisesRegexp( + AppException, r"(\s+)token(\s+)Invalid token." + ): SAClient(**kwargs) @patch("lib.core.usecases.GetTeamUseCase") @@ -106,7 +107,7 @@ def test_init_via_config_ini_invalid_token(self): config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path), patch( - "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path + "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path ): with open(f"{config_dir}/config.ini", "w") as config_ini: config_parser = ConfigParser() @@ -118,7 +119,9 @@ def test_init_via_config_ini_invalid_token(self): config_parser.write(config_ini) for kwargs in ({}, {"config_path": f"{config_dir}/config.ini"}): - with self.assertRaisesRegexp(AppException, r"(\s+)SA_TOKEN(\s+)Invalid token."): + with self.assertRaisesRegexp( + AppException, r"(\s+)SA_TOKEN(\s+)Invalid token." + ): SAClient(**kwargs) def test_invalid_config_path(self):