From f21fb180752678851633d8a4acca9f1fdd34ab12 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 21 Feb 2023 18:42:46 +0400 Subject: [PATCH 1/2] updated get/download annotations --- .../lib/app/interface/base_interface.py | 12 +- src/superannotate/lib/app/interface/types.py | 1 + src/superannotate/lib/core/entities/base.py | 4 +- src/superannotate/lib/core/reporter.py | 4 + .../lib/core/serviceproviders.py | 20 +- .../lib/core/usecases/annotations.py | 1055 +++++++++-------- .../lib/infrastructure/controller.py | 9 +- .../lib/infrastructure/services/annotation.py | 53 +- .../infrastructure/services/http_client.py | 2 +- .../lib/infrastructure/stream_data_handler.py | 69 +- .../annotations/test_annotation_delete.py | 1 + .../annotations/test_get_annotations.py | 13 + ...load_annotations_from_folder_to_project.py | 3 +- tests/unit/test_init.py | 13 +- 14 files changed, 676 insertions(+), 583 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index f146b3494..5d4e141a5 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -34,7 +34,9 @@ def __init__(self, token: TokenStr = None, config_path: str = None): config = ConfigEntity(SA_TOKEN=token) elif config_path: config_path = Path(config_path) - if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): + if not Path(config_path).is_file() or not os.access( + config_path, os.R_OK + ): raise AppException( f"SuperAnnotate config file {str(config_path)} not found." ) @@ -77,8 +79,12 @@ def _retrieve_configs_from_json(path: Path) -> typing.Union[ConfigEntity]: config = ConfigEntity(SA_TOKEN=token) except pydantic.ValidationError: raise pydantic.ValidationError( - [pydantic.error_wrappers.ErrorWrapper(ValueError("Invalid token."), loc='token')], - model=ConfigEntity + [ + pydantic.error_wrappers.ErrorWrapper( + ValueError("Invalid token."), loc="token" + ) + ], + model=ConfigEntity, ) host = json_data.get("main_endpoint") verify_ssl = json_data.get("ssl_verify") diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index 5bc92b438..0f8faf8ef 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -51,4 +51,5 @@ def wrapped(self, *args, **kwargs): return pydantic_validate_arguments(func)(self, *args, **kwargs) except ValidationError as e: raise AppException(wrap_error(e)) from e + return wrapped diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 22f29feea..86e87a41e 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -15,8 +15,8 @@ from lib.core.enums import BaseTitledEnum from pydantic import BaseModel as PydanticBaseModel from pydantic import Extra -from pydantic import StrictStr from pydantic import Field +from pydantic import StrictStr from pydantic.datetime_parse import parse_datetime from pydantic.typing import is_namedtuple from pydantic.utils import ROOT_KEY @@ -295,7 +295,7 @@ def map_fields(entity: dict) -> dict: class TokenStr(StrictStr): - regex = r'^[-.@_A-Za-z0-9]+=\d+$' + regex = r"^[-.@_A-Za-z0-9]+=\d+$" @classmethod def validate(cls, value: Union[str]) -> Union[str]: diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 3c864e319..32cbea5c2 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -60,6 +60,10 @@ def __init__( self.session = CONFIG.get_current_session() self._spinner = None + @property + def log_enabled(self): + return self._log_info + @property def spinner(self): return Spinner() diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index fd24f699e..8726131ff 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -324,16 +324,30 @@ async def get_small_annotations( @abstractmethod async def get_big_annotation( - self, project: entities.ProjectEntity, item: dict, reporter: Reporter + self, + project: entities.ProjectEntity, + item: entities.BaseItemEntity, + reporter: Reporter, ) -> dict: raise NotImplementedError + @abstractmethod + async def list_small_annotations( + self, + project: entities.ProjectEntity, + folder: entities.FolderEntity, + item_ids: List[int], + reporter: Reporter, + callback: Callable = None, + ) -> List[dict]: + raise NotImplementedError + @abstractmethod def sort_items_by_size( self, project: entities.ProjectEntity, folder: entities.FolderEntity, - item_names: List[str], + item_ids: List[int], ) -> Dict[str, List]: raise NotImplementedError @@ -356,7 +370,7 @@ async def download_small_annotations( reporter: Reporter, download_path: str, postfix: str, - items: List[str] = None, + item_ids: List[int], callback: Callable = None, ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 986388e45..34ed01e38 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -1,7 +1,7 @@ import asyncio -import concurrent.futures import copy import io +import itertools import json import logging import os @@ -12,6 +12,7 @@ from dataclasses import dataclass from datetime import datetime from itertools import islice +from operator import itemgetter from pathlib import Path from typing import Any from typing import Callable @@ -24,13 +25,16 @@ import aiofiles import boto3 import jsonschema.validators -import lib.core as constants import nest_asyncio from jsonschema import Draft7Validator from jsonschema import ValidationError -from lib.core.conditions import Condition +from pydantic import BaseModel + +import lib.core as constants from lib.core.conditions import CONDITION_EQ as EQ +from lib.core.conditions import Condition from lib.core.entities import BaseItemEntity +from lib.core.entities import ConfigEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity from lib.core.entities import ProjectEntity @@ -40,10 +44,10 @@ from lib.core.response import Response from lib.core.service_types import UploadAnnotationAuthData from lib.core.serviceproviders import BaseServiceProvider +from lib.core.serviceproviders import ServiceResponse from lib.core.types import PriorityScoreEntity from lib.core.usecases.base import BaseReportableUseCase from lib.core.video_convertor import VideoFrameGenerator -from pydantic import BaseModel logger = logging.getLogger("sa") @@ -63,8 +67,20 @@ class Report: missing_attrs: list +def get_or_raise(response: ServiceResponse): + if response.ok: + return response.data + else: + raise AppException(response.error) + + +def divide_to_chunks(it, size): + it = iter(it) + return iter(lambda: tuple(islice(it, size)), ()) + + def log_report( - report: Report, + report: Report, ): if report.missing_classes: logger.warning( @@ -96,18 +112,18 @@ class Config: def set_annotation_statuses_in_progress( - service_provider: BaseServiceProvider, - project: ProjectEntity, - folder: FolderEntity, - item_names: List[str], - chunk_size=500, + service_provider: BaseServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + item_names: List[str], + chunk_size=500, ) -> bool: failed_on_chunk = False for i in range(0, len(item_names), chunk_size): status_changed = service_provider.items.set_statuses( project=project, folder=folder, - item_names=item_names[i : i + chunk_size], # noqa: E203 + item_names=item_names[i: i + chunk_size], # noqa: E203 annotation_status=constants.AnnotationStatus.IN_PROGRESS.value, ) if not status_changed: @@ -116,13 +132,13 @@ def set_annotation_statuses_in_progress( async def upload_small_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def upload(_chunk): failed_annotations, missing_classes, missing_attr_groups, missing_attrs = ( @@ -167,9 +183,9 @@ async def upload(_chunk): queue.put_nowait(None) break if ( - _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB - or sum([len(i.item.name) for i in chunk]) - >= URI_THRESHOLD - (len(chunk) + 1) * 14 + _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB + or sum([len(i.item.name) for i in chunk]) + >= URI_THRESHOLD - (len(chunk) + 1) * 14 ): await upload(chunk) chunk = [] @@ -181,13 +197,13 @@ async def upload(_chunk): async def upload_big_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]: try: @@ -223,13 +239,13 @@ class UploadAnnotationsUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - annotations: List[dict], - service_provider: BaseServiceProvider, - keep_status: bool = False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + annotations: List[dict], + service_provider: BaseServiceProvider, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -256,7 +272,7 @@ def _validate_json(self, json_data: dict) -> list: def list_existing_items(self, item_names: List[str]) -> List[BaseItemEntity]: existing_items = [] for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -417,17 +433,17 @@ class UploadAnnotationsFromFolderUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - team: TeamEntity, - annotation_paths: List[str], - service_provider: BaseServiceProvider, - pre_annotation: bool = False, - client_s3_bucket=None, - folder_path: str = None, - keep_status=False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + team: TeamEntity, + annotation_paths: List[str], + service_provider: BaseServiceProvider, + pre_annotation: bool = False, + client_s3_bucket=None, + folder_path: str = None, + keep_status=False, ): super().__init__(reporter) self._project = project @@ -468,7 +484,7 @@ def get_name_path_mappings(annotation_paths): return name_path_mappings def _log_report( - self, + self, ): if self._report.missing_classes: logger.warning( @@ -525,7 +541,7 @@ def prepare_annotation(self, annotation: dict, size) -> dict: return annotation async def get_annotation( - self, path: str + self, path: str ) -> (Optional[Tuple[io.StringIO]], Optional[io.BytesIO]): mask = None mask_path = path.replace( @@ -570,12 +586,12 @@ def extract_name(value: Path): return path def get_existing_name_item_mapping( - self, name_path_mappings: Dict[str, str] + self, name_path_mappings: Dict[str, str] ) -> dict: item_names = list(name_path_mappings.keys()) existing_name_item_mapping = {} for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -596,7 +612,7 @@ def annotation_upload_data(self) -> UploadAnnotationAuthData: tmp = self._service_provider.get_annotation_upload_data( project=self._project, folder=self._folder, - item_ids=self._item_ids[i : i + CHUNK_SIZE], + item_ids=self._item_ids[i: i + CHUNK_SIZE], ) if not tmp.ok: raise AppException(tmp.error) @@ -762,22 +778,22 @@ def execute(self): class UploadAnnotationUseCase(BaseReportableUseCase): def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - image: ImageEntity, - team: TeamEntity, - service_provider: BaseServiceProvider, - reporter: Reporter, - annotation_upload_data: UploadAnnotationAuthData = None, - annotations: dict = None, - s3_bucket=None, - client_s3_bucket=None, - mask=None, - verbose: bool = True, - annotation_path: str = None, - pass_validation: bool = False, - keep_status: bool = False, + self, + project: ProjectEntity, + folder: FolderEntity, + image: ImageEntity, + team: TeamEntity, + service_provider: BaseServiceProvider, + reporter: Reporter, + annotation_upload_data: UploadAnnotationAuthData = None, + annotations: dict = None, + s3_bucket=None, + client_s3_bucket=None, + mask=None, + verbose: bool = True, + annotation_path: str = None, + pass_validation: bool = False, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -960,8 +976,8 @@ def execute(self): ) if ( - self._project.type == constants.ProjectType.PIXEL.value - and mask + self._project.type == constants.ProjectType.PIXEL.value + and mask ): self.s3_bucket.put_object( Key=self.annotation_upload_data.images[self._image.id][ @@ -993,154 +1009,19 @@ def execute(self): return self._response -class GetAnnotations(BaseReportableUseCase): - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - item_names: Optional[List[str]], - service_provider: BaseServiceProvider, - show_process: bool = True, - ): - super().__init__(reporter) - self._project = project - self._folder = folder - self._service_provider = service_provider - self._item_names = item_names - self._show_process = show_process - self._item_names_provided = True - self._big_annotations_queue = None - - def validate_project_type(self): - if self._project.type == constants.ProjectType.PIXEL.value: - raise AppException("The function is not supported for Pixel projects.") - - def validate_item_names(self): - if self._item_names: - item_names = list(dict.fromkeys(self._item_names)) - len_unique_items, len_items = len(item_names), len(self._item_names) - if len_unique_items < len_items: - self.reporter.log_info( - f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." - ) - self._item_names = item_names - elif self._item_names is None: - self._item_names_provided = False - condition = Condition("project_id", self._project.id, EQ) & Condition( - "folder_id", self._folder.id, EQ - ) - - self._item_names = [ - item.name for item in self._service_provider.items.list(condition).data - ] - else: - self._item_names = [] - - def _prettify_annotations(self, annotations: List[dict]): - re_struct = {} - - if self._item_names_provided: - for annotation in annotations: - re_struct[annotation["metadata"]["name"]] = annotation - try: - return [re_struct[x] for x in self._item_names if x in re_struct] - except KeyError: - raise AppException("Broken data.") - - return annotations - - async def get_big_annotation(self): - - large_annotations = [] - while True: - item = await self._big_annotations_queue.get() - if not item: - await self._big_annotations_queue.put(None) - break - large_annotation = ( - await self._service_provider.annotations.get_big_annotation( - project=self._project, - item=item, - reporter=self.reporter, - ) - ) - large_annotations.append(large_annotation) - return large_annotations - - async def get_small_annotations(self, item_names): - return await self._service_provider.annotations.get_small_annotations( - project=self._project, - folder=self._folder, - items=item_names, - reporter=self.reporter, - ) - - async def distribute_to_queue(self, big_annotations): - for item in big_annotations: - await self._big_annotations_queue.put(item) - await self._big_annotations_queue.put(None) - - async def run_workers(self, big_annotations, small_annotations): - self._big_annotations_queue = asyncio.Queue() - annotations = await asyncio.gather( - self.distribute_to_queue(big_annotations), - self.get_small_annotations(small_annotations), - self.get_big_annotation(), - self.get_big_annotation(), - self.get_big_annotation(), - ) - - annotations = [i for x in annotations[1:] for i in x if x] - return annotations - - def execute(self): - if self.is_valid(): - items_count = len(self._item_names) - if not items_count: - self.reporter.log_info("No annotations to download.") - self._response.data = [] - return self._response - self.reporter.log_info( - f"Getting {items_count} annotations from " - f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}." - ) - self.reporter.start_progress(items_count, disable=not self._show_process) - - items = self._service_provider.annotations.sort_items_by_size( - project=self._project, folder=self._folder, item_names=self._item_names - ) - small_annotations = [x["name"] for x in items["small"]] - try: - nest_asyncio.apply() - annotations = asyncio.run( - self.run_workers(items["large"], small_annotations) - ) - except Exception as e: - self.reporter.log_error(str(e)) - self._response.errors = AppException("Can't get annotations.") - return self._response - received_items_count = len(annotations) - self.reporter.finish_progress() - if items_count > received_items_count: - self.reporter.log_warning( - f"Could not find annotations for {items_count - received_items_count}/{items_count} items." - ) - self._response.data = self._prettify_annotations(annotations) - return self._response - - class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - video_name: str, - fps: int, - service_provider: BaseServiceProvider, + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + video_name: str, + fps: int, + service_provider: BaseServiceProvider ): super().__init__(reporter) + self._config = config self._project = project self._folder = folder self._video_name = video_name @@ -1158,12 +1039,12 @@ def execute(self): if self.is_valid(): self.reporter.disable_info() response = GetAnnotations( - reporter=self.reporter, + config=self._config, + reporter=Reporter(log_info=False), project=self._project, folder=self._folder, item_names=[self._video_name], service_provider=self._service_provider, - show_process=False, ).execute() self.reporter.enable_info() if response.data: @@ -1192,13 +1073,13 @@ class UploadPriorityScoresUseCase(BaseReportableUseCase): CHUNK_SIZE = 100 def __init__( - self, - reporter, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScoreEntity], - project_folder_name: str, - service_provider: BaseServiceProvider, + self, + reporter, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScoreEntity], + project_folder_name: str, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._project = project @@ -1254,8 +1135,8 @@ def execute(self): if iterations: for i in iterations: priorities_to_upload = priorities[ - i : i + self.CHUNK_SIZE - ] # noqa: E203 + i: i + self.CHUNK_SIZE + ] # noqa: E203 res = self._service_provider.projects.upload_priority_scores( project=self._project, folder=self._folder, @@ -1276,280 +1157,57 @@ def execute(self): return self._response -class DownloadAnnotations(BaseReportableUseCase): +class ValidateAnnotationUseCase(BaseReportableUseCase): + DEFAULT_VERSION = "V1.00" + SCHEMAS: Dict[str, Draft7Validator] = {} + PATTERN_MAP = { + "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ", + "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email", + } + def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - destination: str, - recursive: bool, - item_names: List[str], - service_provider: BaseServiceProvider, - callback: Callable = None, + self, + reporter: Reporter, + team_id: int, + project_type: int, + annotation: dict, + service_provider: BaseServiceProvider, ): super().__init__(reporter) - self._project = project - self._folder = folder - self._destination = destination - self._recursive = recursive - self._item_names = item_names + self._team_id = team_id + self._project_type = project_type + self._annotation = annotation self._service_provider = service_provider - self._callback = callback - self._big_file_queues = [] - self._small_file_queues = [] - - def validate_item_names(self): - if self._item_names: - item_names = list(dict.fromkeys(self._item_names)) - len_unique_items, len_items = len(item_names), len(self._item_names) - if len_unique_items < len_items: - self.reporter.log_info( - f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." - ) - self._item_names = item_names - - def validate_destination(self): - if self._destination: - destination = str(self._destination) - if not os.path.exists(destination) or not os.access( - destination, os.X_OK | os.W_OK - ): - raise AppException( - f"Local path {destination} is not an existing directory or access denied." - ) - - @property - def destination(self) -> Path: - return Path(self._destination if self._destination else "") - def get_postfix(self): - if self._project.type == constants.ProjectType.VECTOR: - return "___objects.json" - elif self._project.type == constants.ProjectType.PIXEL.value: - return "___pixel.json" - return ".json" + @staticmethod + def _get_const(items, path=()): + properties = items.get("properties", {}) + _type, _meta = properties.get("type"), properties.get("meta") + if _meta and _meta.get("type"): + path = path + ("meta",) + path, _type = ValidateAnnotationUseCase._get_const(_meta, path) + if _type and properties.get("type", {}).get("const"): + path = path + ("type",) + path, _type = path, properties["type"]["const"] + return path, _type - def download_annotation_classes(self, path: str): - response = self._service_provider.annotation_classes.list( - Condition("project_id", self._project.id, EQ) - ) - if response.ok: - classes_path = Path(path) / "classes" - classes_path.mkdir(parents=True, exist_ok=True) - with open(classes_path / "classes.json", "w+", encoding="utf-8") as file: - json.dump( - [ - i.dict( - exclude_unset=True, - by_alias=True, - exclude={ - "attribute_groups": {"__all__": {"is_multiselect"}} - }, - ) - for i in response.data - ], - file, - indent=4, - ) - else: - self._response.errors = AppException("Cant download classes.") + @staticmethod + def _get_by_path(path: tuple, data: dict): + tmp = data + for i in path: + tmp = tmp.get(i, {}) + return tmp @staticmethod - def get_items_count(path: str): - return sum([len(files) for r, d, files in os.walk(path)]) + def oneOf(validator, oneOf, instance, schema): # noqa + sub_schemas = enumerate(oneOf) + const_key = None + for index, sub_schema in sub_schemas: - async def download_big_annotations(self, queue_idx, export_path): - while True: - cur_queue = self._big_file_queues[queue_idx] - item = await cur_queue.get() - cur_queue.task_done() - if item: - postfix = self.get_postfix() - await self._service_provider.annotations.download_big_annotation( - project=self._project, - item=item, - download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", - postfix=postfix, - callback=self._callback, - ) - else: - cur_queue.put_nowait(None) - break - - async def download_small_annotations( - self, queue_idx, export_path, folder: FolderEntity - ): - cur_queue = self._small_file_queues[queue_idx] - items = [] - item = "" - postfix = self.get_postfix() - while item is not None: - item = await cur_queue.get() - if item: - items.append(item) - await self._service_provider.annotations.download_small_annotations( - project=self._project, - folder=folder, - items=items, - reporter=self.reporter, - download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", - postfix=postfix, - callback=self._callback, - ) - - async def distribute_to_queues( - self, item_names, sm_queue_id, l_queue_id, folder: FolderEntity - ): - try: - resp = self._service_provider.annotations.sort_items_by_size( - project=self._project, folder=folder, item_names=item_names - ) - - for item in resp["large"]: - await self._big_file_queues[l_queue_id].put(item) - - for item in resp["small"]: - await self._small_file_queues[sm_queue_id].put(item["name"]) - finally: - await self._big_file_queues[l_queue_id].put(None) - await self._small_file_queues[sm_queue_id].put(None) - - async def run_workers(self, item_names, folder: FolderEntity, export_path): - try: - self._big_file_queues.append(asyncio.Queue()) - self._small_file_queues.append(asyncio.Queue()) - small_file_queue_idx = len(self._small_file_queues) - 1 - big_file_queue_idx = len(self._big_file_queues) - 1 - res = await asyncio.gather( - self.distribute_to_queues( - item_names, small_file_queue_idx, big_file_queue_idx, folder - ), - self.download_big_annotations(big_file_queue_idx, export_path), - self.download_big_annotations(big_file_queue_idx, export_path), - self.download_big_annotations(big_file_queue_idx, export_path), - self.download_small_annotations( - small_file_queue_idx, export_path, folder - ), - return_exceptions=True, - ) - if any(res): - self.reporter.log_error(f"Error {str([i for i in res if i])}") - except Exception as e: - self.reporter.log_error(f"Error {str(e)}") - - def execute(self): - if self.is_valid(): - export_path = str( - self.destination - / Path( - f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}" - ) - ) - self.reporter.log_info( - f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" - ) - self.reporter.start_spinner() - - folders = [] - if self._folder.is_root and self._recursive: - folders = self._service_provider.folders.list( - Condition("project_id", self._project.id, EQ) - ).data - if not folders: - folders.append(self._folder) - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - nest_asyncio.apply() - futures = [] - for folder in folders: - if not self._item_names: - condition = Condition( - "project_id", self._project.id, EQ - ) & Condition("folder_id", folder.id, EQ) - item_names = [ - item.name - for item in self._service_provider.items.list( - condition - ).data - ] - else: - item_names = self._item_names - new_export_path = export_path - if not folder.is_root and self._folder.is_root: - new_export_path += f"/{folder.name}" - if not item_names: - continue - futures.append( - executor.submit( - asyncio.run, - self.run_workers(item_names, folder, new_export_path), - ) - ) - - for future in concurrent.futures.as_completed(futures): - exception = future.exception() - if exception: - self._response.errors = exception - - self.reporter.stop_spinner() - count = self.get_items_count(export_path) - self.reporter.log_info(f"Downloaded annotations for {count} items.") - self.download_annotation_classes(export_path) - self._response.data = os.path.abspath(export_path) - return self._response - - -class ValidateAnnotationUseCase(BaseReportableUseCase): - DEFAULT_VERSION = "V1.00" - SCHEMAS: Dict[str, Draft7Validator] = {} - PATTERN_MAP = { - "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ", - "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email", - } - - def __init__( - self, - reporter: Reporter, - team_id: int, - project_type: int, - annotation: dict, - service_provider: BaseServiceProvider, - ): - super().__init__(reporter) - self._team_id = team_id - self._project_type = project_type - self._annotation = annotation - self._service_provider = service_provider - - @staticmethod - def _get_const(items, path=()): - properties = items.get("properties", {}) - _type, _meta = properties.get("type"), properties.get("meta") - if _meta and _meta.get("type"): - path = path + ("meta",) - path, _type = ValidateAnnotationUseCase._get_const(_meta, path) - if _type and properties.get("type", {}).get("const"): - path = path + ("type",) - path, _type = path, properties["type"]["const"] - return path, _type - - @staticmethod - def _get_by_path(path: tuple, data: dict): - tmp = data - for i in path: - tmp = tmp.get(i, {}) - return tmp - - @staticmethod - def oneOf(validator, oneOf, instance, schema): # noqa - sub_schemas = enumerate(oneOf) - const_key = None - for index, sub_schema in sub_schemas: - - const_key, _type = ValidateAnnotationUseCase._get_const(sub_schema) - if const_key: - instance_type = ValidateAnnotationUseCase._get_by_path( - const_key, instance + const_key, _type = ValidateAnnotationUseCase._get_const(sub_schema) + if const_key: + instance_type = ValidateAnnotationUseCase._get_by_path( + const_key, instance ) if not instance_type: yield ValidationError("type required") @@ -1681,9 +1339,9 @@ def extract_messages(self, path, error, report): for sub_error in sorted(error.context, key=lambda e: e.schema_path): tmp_path = sub_error.path # if sub_error.path else real_path _path = ( - f"{''.join(path)}" - + ("." if tmp_path else "") - + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) + f"{''.join(path)}" + + ("." if tmp_path else "") + + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) ) if sub_error.context: self.extract_messages(_path, sub_error, report) @@ -1715,3 +1373,414 @@ def execute(self) -> Response: self._response.data = list(sorted(errors_report, key=lambda x: x[0])) return self._response + + +class GetAnnotations(BaseReportableUseCase): + def __init__( + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + item_names: Optional[List[str]], + service_provider: BaseServiceProvider, + ): + super().__init__(reporter) + self._config = config + self._project = project + self._folder = folder + self._service_provider = service_provider + self._item_names = item_names + self._item_names_provided = True + self._big_annotations_queue = None + self._small_annotations_queue = None + + def validate_project_type(self): + if self._project.type == constants.ProjectType.PIXEL.value: + raise AppException("The function is not supported for Pixel projects.") + + def validate_item_names(self): + if self._item_names: # if names provided + unique_item_names = list(set(self._item_names)) + len_unique_items, len_items = len(unique_item_names), len(self._item_names) + if len_unique_items < len_items: + self.reporter.log_info( + f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." + ) + self._item_names = unique_item_names + elif self._item_names is None: + self._item_names_provided = False + condition = Condition("project_id", self._project.id, EQ) & Condition( + "folder_id", self._folder.id, EQ + ) + + self._item_names = [ + item.name for item in self._service_provider.items.list(condition).data + ] + else: + self._item_names = [] + + def _prettify_annotations(self, annotations: List[dict]): + re_struct = {} + if self._item_names_provided: + for annotation in annotations: + re_struct[annotation["metadata"]["name"]] = annotation + try: + return [re_struct[x] for x in self._item_names if x in re_struct] + except KeyError: + raise AppException("Broken data.") + + return annotations + + async def get_big_annotation(self): + large_annotations = [] + while True: + item: BaseItemEntity = await self._big_annotations_queue.get() + if item: + large_annotations.append( + await self._service_provider.annotations.get_big_annotation( + project=self._project, + item=item, + reporter=self.reporter, + ) + ) + else: + await self._big_annotations_queue.put(None) + break + return large_annotations + + async def get_small_annotations(self): + small_annotations = [] + while True: + items = await self._small_annotations_queue.get() + if items: + annotations = ( + await self._service_provider.annotations.list_small_annotations( + project=self._project, + folder=self._folder, + item_ids=[i.id for i in items], + reporter=self.reporter, + ) + ) + small_annotations.extend(annotations) + else: + await self._small_annotations_queue.put(None) + break + return small_annotations + + async def run_workers( + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[BaseItemEntity], + ): + annotations = [] + if big_annotations: + self._big_annotations_queue = asyncio.Queue() + for item in big_annotations: + self._big_annotations_queue.put_nowait(item) + self._big_annotations_queue.put_nowait(None) + annotations.extend( + asyncio.gather( + *[ + self.get_big_annotation() + for _ in range(max(self._config.MAX_COROUTINE_COUNT // 2, 1)) + ] + ) + ) + if small_annotations: + self._small_annotations_queue = asyncio.Queue() + small_chunks = divide_to_chunks( + small_annotations, size=self._config.ANNOTATION_CHUNK_SIZE + ) + for chunk in small_chunks: + self._small_annotations_queue.put_nowait(chunk) + self._small_annotations_queue.put_nowait(None) + + annotations.extend( + list( + itertools.chain.from_iterable( + await asyncio.gather( + *[ + self.get_small_annotations() + for _ in range(self._config.MAX_COROUTINE_COUNT) + ] + ) + ) + ) + ) + return list(filter(None, annotations)) + + def execute(self): + if self.is_valid(): + if self._item_names: + items = get_or_raise( + self._service_provider.items.list_by_names( + self._project, self._folder, self._item_names + ) + ) + else: + condition = Condition("project_id", self._project.id, EQ) & Condition( + "folder_id", self._folder.id, EQ + ) + items = get_or_raise(self._service_provider.items.list(condition)) + id_item_map = {i.id: i for i in items} + + if not items: + logger.info("No annotations to download.") + self._response.data = [] + return self._response + items_count = len(items) + logger.info( + f"Getting {items_count} annotations from " + f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}." + ) + self.reporter.start_progress( + items_count, disable=logger.level > logging.INFO or self.reporter.log_enabled + ) + + sort_response = self._service_provider.annotations.sort_items_by_size( + project=self._project, folder=self._folder, item_ids=list(id_item_map) + ) + large_item_ids = set(map(itemgetter("id"), sort_response["large"])) + small_items_ids = set(map(itemgetter("id"), sort_response["small"])) + large_items = list(filter(lambda item: item.id in large_item_ids, items)) + small_items = list(filter(lambda item: item.id in small_items_ids, items)) + try: + nest_asyncio.apply() + annotations = asyncio.run(self.run_workers(large_items, small_items)) + except Exception as e: + logger.error(e) + self._response.errors = AppException("Can't get annotations.") + return self._response + received_items_count = len(annotations) + self.reporter.finish_progress() + if items_count > received_items_count: + self.reporter.log_warning( + f"Could not find annotations for {items_count - received_items_count}/{items_count} items." + ) + self._response.data = self._prettify_annotations(annotations) + return self._response + + +class DownloadAnnotations(BaseReportableUseCase): + def __init__( + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + destination: str, + recursive: bool, + item_names: List[str], + service_provider: BaseServiceProvider, + callback: Callable = None, + ): + super().__init__(reporter) + self._config = config + self._project = project + self._folder = folder + self._destination = destination + self._recursive = recursive + self._item_names = item_names + self._service_provider = service_provider + self._callback = callback + self._big_file_queue = None + self._small_file_queue = None + + def validate_item_names(self): + if self._item_names: + item_names = list(dict.fromkeys(self._item_names)) + len_unique_items, len_items = len(item_names), len(self._item_names) + if len_unique_items < len_items: + self.reporter.log_info( + f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." + ) + self._item_names = item_names + + def validate_destination(self): + if self._destination: + destination = str(self._destination) + if not os.path.exists(destination) or not os.access( + destination, os.X_OK | os.W_OK + ): + raise AppException( + f"Local path {destination} is not an existing directory or access denied." + ) + + @property + def destination(self) -> Path: + return Path(self._destination if self._destination else "") + + def get_postfix(self): + if self._project.type == constants.ProjectType.VECTOR: + return "___objects.json" + elif self._project.type == constants.ProjectType.PIXEL.value: + return "___pixel.json" + return ".json" + + def download_annotation_classes(self, path: str): + response = self._service_provider.annotation_classes.list( + Condition("project_id", self._project.id, EQ) + ) + if response.ok: + classes_path = Path(path) / "classes" + classes_path.mkdir(parents=True, exist_ok=True) + with open(classes_path / "classes.json", "w+", encoding="utf-8") as file: + json.dump( + [ + i.dict( + exclude_unset=True, + by_alias=True, + exclude={ + "attribute_groups": {"__all__": {"is_multiselect"}} + }, + ) + for i in response.data + ], + file, + indent=4, + ) + else: + self._response.errors = AppException("Cant download classes.") + + @staticmethod + def get_items_count(path: str): + return sum([len(files) for r, d, files in os.walk(path)]) + + async def download_big_annotations(self, export_path): + while True: + item = await self._big_file_queue.get() + self._big_file_queue.task_done() + if item: + postfix = self.get_postfix() + await self._service_provider.annotations.download_big_annotation( + project=self._project, + item=item, + download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", + postfix=postfix, + callback=self._callback, + ) + else: + self._big_file_queue.put_nowait(None) + break + + async def download_small_annotations(self, export_path, folder: FolderEntity): + postfix = self.get_postfix() + while True: + items = await self._small_file_queue.get() + if items: + await self._service_provider.annotations.download_small_annotations( + project=self._project, + folder=folder, + item_ids=[i.id for i in items], + reporter=self.reporter, + download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", + postfix=postfix, + callback=self._callback, + ) + else: + self._small_file_queue.put_nowait(None) + break + + async def run_workers( + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[BaseItemEntity], + folder: FolderEntity, + export_path, + ): + if big_annotations: + self._big_file_queue = asyncio.Queue() + for item in big_annotations: + self._big_file_queue.put_nowait(item) + self._big_file_queue.put_nowait(None) + await asyncio.gather( + *[ + self.download_big_annotations(export_path) + for _ in range(max(self._config.MAX_COROUTINE_COUNT // 2, 1)) + ] + ) + + if small_annotations: + self._small_file_queue = asyncio.Queue() + small_chunks = divide_to_chunks( + small_annotations, size=self._config.ANNOTATION_CHUNK_SIZE + ) + for chunk in small_chunks: + self._small_file_queue.put_nowait(chunk) + self._small_file_queue.put_nowait(None) + await asyncio.gather( + *[ + self.download_small_annotations(export_path, folder) + for _ in range(self._config.MAX_COROUTINE_COUNT) + ] + ) + + def execute(self): + if self.is_valid(): + export_path = str( + self.destination + / Path( + f"{self._project.name} {datetime.now().strftime('%B %d %Y %H_%M')}" + ) + ) + logger.info( + f"Downloading the annotations of the requested items to {export_path}\nThis might take a while…" + ) + self.reporter.start_spinner() + folders = [] + if self._folder.is_root and self._recursive: + folders = self._service_provider.folders.list( + Condition("project_id", self._project.id, EQ) + ).data + if not folders: + folders.append(self._folder) + nest_asyncio.apply() + for folder in folders: + if self._item_names: + items = get_or_raise( + self._service_provider.items.list_by_names( + self._project, folder, self._item_names + ) + ) + else: + condition = Condition( + "project_id", self._project.id, EQ + ) & Condition("folder_id", folder.id, EQ) + items = get_or_raise(self._service_provider.items.list(condition)) + if not items: + continue + new_export_path = export_path + if not folder.is_root and self._folder.is_root: + new_export_path += f"/{folder.name}" + + id_item_map = {i.id: i for i in items} + sort_response = self._service_provider.annotations.sort_items_by_size( + project=self._project, + folder=self._folder, + item_ids=list(id_item_map), + ) + large_item_ids = set(map(itemgetter("id"), sort_response["large"])) + small_items_ids = set(map(itemgetter("id"), sort_response["small"])) + large_items = list( + filter(lambda item: item.id in large_item_ids, items) + ) + small_items = list( + filter(lambda item: item.id in small_items_ids, items) + ) + try: + asyncio.run( + self.run_workers( + large_items, small_items, folder, new_export_path + ) + ) + except Exception as e: + logger.error(e) + self._response.errors = AppException("Can't get annotations.") + return self._response + self.reporter.stop_spinner() + count = self.get_items_count(export_path) + self.reporter.log_info(f"Downloaded annotations for {count} items.") + self.download_annotation_classes(export_path) + self._response.data = os.path.abspath(export_path) + return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index a1d3efef2..40e2d1e62 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -503,6 +503,10 @@ def update(self, project: ProjectEntity, item: BaseItemEntity): class AnnotationManager(BaseManager): + def __init__(self, service_provider: ServiceProvider, config: ConfigEntity): + super().__init__(service_provider) + self._config = config + def list( self, project: ProjectEntity, @@ -511,6 +515,7 @@ def list( verbose=True, ): use_case = usecases.GetAnnotations( + config=self._config, reporter=Reporter(log_info=verbose, log_warning=verbose), project=project, folder=folder, @@ -529,6 +534,7 @@ def download( callback: Optional[Callable], ): use_case = usecases.DownloadAnnotations( + config=self._config, reporter=Reporter(), project=project, folder=folder, @@ -808,7 +814,7 @@ def __init__(self, config: ConfigEntity): self.projects = ProjectManager(self.service_provider) self.folders = FolderManager(self.service_provider) self.items = ItemManager(self.service_provider) - self.annotations = AnnotationManager(self.service_provider) + self.annotations = AnnotationManager(self.service_provider, config) self.custom_fields = CustomFieldManager(self.service_provider) self.subsets = SubsetManager(self.service_provider) self.models = ModelManager(self.service_provider) @@ -1226,6 +1232,7 @@ def get_annotations_per_frame( folder = self.get_folder(project, folder_name) use_case = usecases.GetVideoAnnotationsPerFrame( + config=self._config, reporter=self.get_default_reporter(), project=project, folder=folder, diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py index cdb6278e7..ac7adab0a 100644 --- a/src/superannotate/lib/infrastructure/services/annotation.py +++ b/src/superannotate/lib/infrastructure/services/annotation.py @@ -96,11 +96,14 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): return synced async def get_big_annotation( - self, project: entities.ProjectEntity, item: dict, reporter: Reporter + self, + project: entities.ProjectEntity, + item: entities.BaseItemEntity, + reporter: Reporter, ) -> dict: url = urljoin( self.assets_provider_url, - self.URL_LARGE_ANNOTATION.format(item_id=item["id"]), + self.URL_LARGE_ANNOTATION.format(item_id=item.id), ) query_params = { @@ -111,7 +114,7 @@ async def get_big_annotation( } await self._sync_large_annotation( - team_id=project.team_id, project_id=project.id, item_id=item["id"] + team_id=project.team_id, project_id=project.id, item_id=item.id ) async with aiohttp.ClientSession( @@ -125,6 +128,33 @@ async def get_big_annotation( reporter.update_progress() return large_annotation + async def list_small_annotations( + self, + project: entities.ProjectEntity, + folder: entities.FolderEntity, + item_ids: List[int], + reporter: Reporter, + callback: Callable = None, + ) -> List[dict]: + query_params = { + "team_id": project.team_id, + "project_id": project.id, + "folder_id": folder.id, + } + + handler = StreamedAnnotations( + self.client.default_headers, + reporter, + map_function=lambda x: {"image_ids": x}, + callback=callback, + ) + return await handler.list_annotations( + method="post", + url=urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS), + data=item_ids, + params=query_params, + ) + async def get_small_annotations( self, project: entities.ProjectEntity, @@ -161,7 +191,7 @@ def sort_items_by_size( self, project: entities.ProjectEntity, folder: entities.FolderEntity, - item_names: List[str], + item_ids: List[int], ) -> Dict[str, List]: chunk_size = 2000 query_params = { @@ -170,10 +200,9 @@ def sort_items_by_size( } response_data = {"small": [], "large": []} - for i in range(0, len(item_names), chunk_size): + for i in range(0, len(item_ids), chunk_size): body = { - "item_names": item_names[i : i + chunk_size], # noqa - "folder_id": folder.id, + "item_ids": item_ids[i : i + chunk_size], # noqa } # noqa response = self.client.request( url=urljoin(self.assets_provider_url, self.URL_CLASSIFY_ITEM_SIZE), @@ -235,7 +264,7 @@ async def download_small_annotations( reporter: Reporter, download_path: str, postfix: str, - items: List[str] = None, + item_ids: List[int], callback: Callable = None, ): query_params = { @@ -246,15 +275,15 @@ async def download_small_annotations( handler = StreamedAnnotations( headers=self.client.default_headers, reporter=reporter, - map_function=lambda x: {"image_names": x}, + map_function=lambda x: {"image_ids": x}, callback=callback, ) - return await handler.download_data( + return await handler.download_annotations( + method="post", url=urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS), - data=items, + data=item_ids, params=query_params, - chunk_size=self.DEFAULT_CHUNK_SIZE, download_path=download_path, postfix=postfix, ) diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py index cf092cb96..6c5746183 100644 --- a/src/superannotate/lib/infrastructure/services/http_client.py +++ b/src/superannotate/lib/infrastructure/services/http_client.py @@ -207,6 +207,6 @@ def serialize_response( data["data"] = data_json return content_type(**data) except json.decoder.JSONDecodeError: - data['_error'] = response.content + data["_error"] = response.content data["reason"] = response.reason return content_type(**data) diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 3d4569bbc..1fb6dc828 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -1,7 +1,7 @@ -import asyncio import copy import json import os +import typing from typing import Callable import aiohttp @@ -62,11 +62,11 @@ async def fetch( yield json.loads(buffer) self._reporter.update_progress() - async def process_chunk( + async def list_annotations( self, method: str, url: str, - data: dict = None, + data: typing.List[int] = None, params: dict = None, verify_ssl=False, ): @@ -81,19 +81,20 @@ async def process_chunk( session, url, self._process_data(data), - params=params, + params=copy.copy(params), ): self._annotations.append( self._callback(annotation) if self._callback else annotation ) + return self._annotations - async def store_chunk( + async def download_annotations( self, method: str, url: str, download_path, postfix, - data: dict = None, + data: typing.List[int], params: dict = None, ): async with aiohttp.ClientSession( @@ -107,7 +108,7 @@ async def store_chunk( session, url, self._process_data(data), - params=params, + params=copy.copy(params), ): self._annotations.append( self._callback(annotation) if self._callback else annotation @@ -120,31 +121,6 @@ async def store_chunk( ) self._items_downloaded += 1 - async def get_data( - self, - url: str, - data: list, - method: str = "post", - params=None, - chunk_size: int = 5000, - verify_ssl: bool = False, - ): - params["limit"] = chunk_size - await asyncio.gather( - *[ - self.process_chunk( - method=method, - url=url, - data=data[i : i + chunk_size], - params=copy.copy(params), - verify_ssl=verify_ssl, - ) - for i in range(0, len(data), chunk_size) - ] - ) - - return self._annotations - @staticmethod def _store_annotation(path, postfix, annotation: dict, callback: Callable = None): os.makedirs(path, exist_ok=True) @@ -156,32 +132,3 @@ def _process_data(self, data): if data and self._map_function: return self._map_function(data) return data - - async def download_data( - self, - url: str, - data: list, - download_path: str, - postfix: str, - method: str = "post", - params=None, - chunk_size: int = 5000, - ) -> int: - """ - Returns the number of items downloaded - """ - params["limit"] = chunk_size - await asyncio.gather( - *[ - self.store_chunk( - method=method, - url=url, - data=data[i : i + chunk_size], # noqa - params=copy.copy(params), - download_path=download_path, - postfix=postfix, - ) - for i in range(0, len(data), chunk_size) - ] - ) - return self._items_downloaded diff --git a/tests/integration/annotations/test_annotation_delete.py b/tests/integration/annotations/test_annotation_delete.py index 57b705af7..b9bd0af02 100644 --- a/tests/integration/annotations/test_annotation_delete.py +++ b/tests/integration/annotations/test_annotation_delete.py @@ -61,6 +61,7 @@ def test_delete_annotations(self): ] def test_delete_annotations_by_name(self): + # self._attach_items(4) sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" ) diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index 450cacdec..c1417eb20 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -138,6 +138,19 @@ def test_get_annotations10000(self): a = sa.get_annotations(self.PROJECT_NAME) assert len(a) == count + def test_get_annotation(self): # to_delete + count = 2 + sa.attach_items( + self.PROJECT_NAME, + [ + {"name": f"example_image_{i}.jpg", "url": f"url_{i}"} + for i in range(count) + ], # noqa + ) + assert len(sa.search_items(self.PROJECT_NAME)) == count + a = sa.get_annotations(self.PROJECT_NAME) + assert len(a) == count + class TestGetAnnotationsVideo(BaseTestCase): PROJECT_NAME = "test attach multiple video urls" diff --git a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py index 922581f53..5f8a5647a 100644 --- a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py +++ b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py @@ -8,8 +8,7 @@ sa = SAClient() -class \ - TestAnnotationUploadVector(BaseTestCase): +class TestAnnotationUploadVector(BaseTestCase): PROJECT_NAME = "Test-Upload_annotations_from_folder_to_project" PROJECT_DESCRIPTION = "Desc" PROJECT_TYPE = "Vector" diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 741f6c28d..dcb353d70 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -6,7 +6,6 @@ from unittest.mock import patch import superannotate.lib.core as constants -from superannotate.lib.app.interface.types import validate_arguments from superannotate import AppException from superannotate import SAClient @@ -53,12 +52,14 @@ def test_init_via_config_json_invalid_json(self): config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path), patch( - "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path + "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path ): with open(f"{config_dir}/config.json", "w") as config_json: json.dump({"token": "INVALID_TOKEN"}, config_json) for kwargs in ({}, {"config_path": f"{config_dir}/config.json"}): - with self.assertRaisesRegexp(AppException, r"(\s+)token(\s+)Invalid token."): + with self.assertRaisesRegexp( + AppException, r"(\s+)token(\s+)Invalid token." + ): SAClient(**kwargs) @patch("lib.core.usecases.GetTeamUseCase") @@ -106,7 +107,7 @@ def test_init_via_config_ini_invalid_token(self): config_ini_path = f"{config_dir}/config.ini" config_json_path = f"{config_dir}/config.json" with patch("lib.core.CONFIG_INI_FILE_LOCATION", config_ini_path), patch( - "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path + "lib.core.CONFIG_JSON_FILE_LOCATION", config_json_path ): with open(f"{config_dir}/config.ini", "w") as config_ini: config_parser = ConfigParser() @@ -118,7 +119,9 @@ def test_init_via_config_ini_invalid_token(self): config_parser.write(config_ini) for kwargs in ({}, {"config_path": f"{config_dir}/config.ini"}): - with self.assertRaisesRegexp(AppException, r"(\s+)SA_TOKEN(\s+)Invalid token."): + with self.assertRaisesRegexp( + AppException, r"(\s+)SA_TOKEN(\s+)Invalid token." + ): SAClient(**kwargs) def test_invalid_config_path(self): From 68de1dab9cb4c91a0ff935bd20efd48afdad2080 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 23 Feb 2023 12:17:43 +0400 Subject: [PATCH 2/2] Tests updates --- src/superannotate/lib/core/__init__.py | 80 +++-- .../lib/core/serviceproviders.py | 11 - .../lib/core/usecases/annotations.py | 308 +++++++++--------- .../lib/infrastructure/services/annotation.py | 43 +-- .../lib/infrastructure/services/item.py | 6 +- .../lib/infrastructure/services/project.py | 2 +- .../lib/infrastructure/stream_data_handler.py | 42 +-- .../annotations/test_annotation_delete.py | 56 +--- .../annotations/test_get_annotations.py | 22 +- tests/integration/base.py | 1 - .../classes/test_create_bed_handling.py | 2 + .../custom_fields/test_custom_schema.py | 29 ++ tests/integration/test_interface.py | 41 +-- .../test_recursive_folder_pixel.py | 30 -- .../test_upload_priority_scores.py | 4 +- 15 files changed, 280 insertions(+), 397 deletions(-) delete mode 100644 tests/integration/test_recursive_folder_pixel.py diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 48e328b09..7322412e4 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -4,19 +4,21 @@ from logging.handlers import RotatingFileHandler from os.path import expanduser -from superannotate.lib.core.config import Config -from superannotate.lib.core.enums import AnnotationStatus -from superannotate.lib.core.enums import ApprovalStatus -from superannotate.lib.core.enums import FolderStatus -from superannotate.lib.core.enums import ImageQuality -from superannotate.lib.core.enums import ProjectStatus -from superannotate.lib.core.enums import ProjectType -from superannotate.lib.core.enums import SegmentationStatus -from superannotate.lib.core.enums import TrainingStatus -from superannotate.lib.core.enums import UploadState -from superannotate.lib.core.enums import UserRole +from lib.core.config import Config +from lib.core.enums import AnnotationStatus +from lib.core.enums import ApprovalStatus +from lib.core.enums import FolderStatus +from lib.core.enums import ImageQuality +from lib.core.enums import ProjectStatus +from lib.core.enums import ProjectType +from lib.core.enums import SegmentationStatus +from lib.core.enums import TrainingStatus +from lib.core.enums import UploadState +from lib.core.enums import UserRole + CONFIG = Config() +BACKEND_URL = "https://api.superannotate.com" HOME_PATH = expanduser("~/.superannotate") CONFIG_JSON_PATH = f"{HOME_PATH}/config.json" @@ -32,38 +34,34 @@ def setup_logging(level=DEFAULT_LOGGING_LEVEL, file_path=LOG_FILE_LOCATION): - global _loggers - if not _loggers.get("sa"): - logger = logging.getLogger("sa") - logger.propagate = True - logger.setLevel(level) - stream_handler = logging.StreamHandler() - formatter = Formatter("SA-PYTHON-SDK - %(levelname)s - %(message)s") - stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) - try: - log_file_path = os.path.join(file_path, "sa.log") - open(log_file_path, "w").close() - if os.access(log_file_path, os.W_OK): - file_handler = RotatingFileHandler( - log_file_path, - maxBytes=5 * 1024 * 1024, - backupCount=5, - mode="a", - ) - file_formatter = Formatter( - "SA-PYTHON-SDK - %(levelname)s - %(asctime)s - %(message)s" - ) - file_handler.setFormatter(file_formatter) - logger.addHandler(file_handler) - except OSError: - pass - finally: - _loggers["sa"] = logger + logger = logging.getLogger("sa") + for handler in logger.handlers[:]: # remove all old handlers + logger.removeHandler(handler) + logger.propagate = True + logger.setLevel(level) + stream_handler = logging.StreamHandler() + formatter = Formatter("SA-PYTHON-SDK - %(levelname)s - %(message)s") + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + try: + log_file_path = os.path.join(file_path, "sa.log") + open(log_file_path, "w").close() + if os.access(log_file_path, os.W_OK): + file_handler = RotatingFileHandler( + log_file_path, + maxBytes=5 * 1024 * 1024, + backupCount=5, + mode="a", + ) + file_formatter = Formatter( + "SA-PYTHON-SDK - %(levelname)s - %(asctime)s - %(message)s" + ) + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + except OSError as e: + logging.error(e) -BACKEND_URL = "https://api.superannotate.com" - DEFAULT_IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "tif", "tiff", "webp", "bmp"] DEFAULT_FILE_EXCLUDE_PATTERNS = ["___save.png", "___fuse.png"] DEFAULT_VIDEO_EXTENSIONS = ["mp4", "avi", "mov", "webm", "flv", "mpg", "ogg"] diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 8726131ff..7eaaa7858 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -311,17 +311,6 @@ def delete_multiple( class BaseAnnotationService(SuperannotateServiceProvider): - @abstractmethod - async def get_small_annotations( - self, - project: entities.ProjectEntity, - folder: entities.FolderEntity, - items: List[str], - reporter: Reporter, - callback: Callable = None, - ) -> List[dict]: - raise NotImplementedError - @abstractmethod async def get_big_annotation( self, diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 34ed01e38..f6ae1cc56 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -25,14 +25,12 @@ import aiofiles import boto3 import jsonschema.validators +import lib.core as constants import nest_asyncio from jsonschema import Draft7Validator from jsonschema import ValidationError -from pydantic import BaseModel - -import lib.core as constants -from lib.core.conditions import CONDITION_EQ as EQ from lib.core.conditions import Condition +from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import BaseItemEntity from lib.core.entities import ConfigEntity from lib.core.entities import FolderEntity @@ -48,6 +46,7 @@ from lib.core.types import PriorityScoreEntity from lib.core.usecases.base import BaseReportableUseCase from lib.core.video_convertor import VideoFrameGenerator +from pydantic import BaseModel logger = logging.getLogger("sa") @@ -80,7 +79,7 @@ def divide_to_chunks(it, size): def log_report( - report: Report, + report: Report, ): if report.missing_classes: logger.warning( @@ -112,33 +111,34 @@ class Config: def set_annotation_statuses_in_progress( - service_provider: BaseServiceProvider, - project: ProjectEntity, - folder: FolderEntity, - item_names: List[str], - chunk_size=500, + service_provider: BaseServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + item_names: List[str], + chunk_size=500, ) -> bool: failed_on_chunk = False for i in range(0, len(item_names), chunk_size): status_changed = service_provider.items.set_statuses( project=project, folder=folder, - item_names=item_names[i: i + chunk_size], # noqa: E203 + item_names=item_names[i : i + chunk_size], # noqa: E203 annotation_status=constants.AnnotationStatus.IN_PROGRESS.value, ) - if not status_changed: + if not status_changed.ok: failed_on_chunk = True + logger.debug(status_changed.error) return not failed_on_chunk async def upload_small_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def upload(_chunk): failed_annotations, missing_classes, missing_attr_groups, missing_attrs = ( @@ -183,9 +183,9 @@ async def upload(_chunk): queue.put_nowait(None) break if ( - _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB - or sum([len(i.item.name) for i in chunk]) - >= URI_THRESHOLD - (len(chunk) + 1) * 14 + _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB + or sum([len(i.item.name) for i in chunk]) + >= URI_THRESHOLD - (len(chunk) + 1) * 14 ): await upload(chunk) chunk = [] @@ -197,13 +197,13 @@ async def upload(_chunk): async def upload_big_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]: try: @@ -217,8 +217,8 @@ async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]: if is_uploaded and callback: callback(item_data) return item_data.item.name, is_uploaded - except Exception: - logger.debug(traceback.format_exc()) + except Exception as e: + logger.debug(e) report.failed_annotations.append(item_data.item.name) finally: reporter.update_progress() @@ -239,13 +239,13 @@ class UploadAnnotationsUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - annotations: List[dict], - service_provider: BaseServiceProvider, - keep_status: bool = False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + annotations: List[dict], + service_provider: BaseServiceProvider, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -272,7 +272,7 @@ def _validate_json(self, json_data: dict) -> list: def list_existing_items(self, item_names: List[str]) -> List[BaseItemEntity]: existing_items = [] for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -433,17 +433,17 @@ class UploadAnnotationsFromFolderUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - team: TeamEntity, - annotation_paths: List[str], - service_provider: BaseServiceProvider, - pre_annotation: bool = False, - client_s3_bucket=None, - folder_path: str = None, - keep_status=False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + team: TeamEntity, + annotation_paths: List[str], + service_provider: BaseServiceProvider, + pre_annotation: bool = False, + client_s3_bucket=None, + folder_path: str = None, + keep_status=False, ): super().__init__(reporter) self._project = project @@ -484,7 +484,7 @@ def get_name_path_mappings(annotation_paths): return name_path_mappings def _log_report( - self, + self, ): if self._report.missing_classes: logger.warning( @@ -541,7 +541,7 @@ def prepare_annotation(self, annotation: dict, size) -> dict: return annotation async def get_annotation( - self, path: str + self, path: str ) -> (Optional[Tuple[io.StringIO]], Optional[io.BytesIO]): mask = None mask_path = path.replace( @@ -586,12 +586,12 @@ def extract_name(value: Path): return path def get_existing_name_item_mapping( - self, name_path_mappings: Dict[str, str] + self, name_path_mappings: Dict[str, str] ) -> dict: item_names = list(name_path_mappings.keys()) existing_name_item_mapping = {} for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -612,7 +612,7 @@ def annotation_upload_data(self) -> UploadAnnotationAuthData: tmp = self._service_provider.get_annotation_upload_data( project=self._project, folder=self._folder, - item_ids=self._item_ids[i: i + CHUNK_SIZE], + item_ids=self._item_ids[i : i + CHUNK_SIZE], ) if not tmp.ok: raise AppException(tmp.error) @@ -673,8 +673,8 @@ async def distribute_queues(self, items_to_upload: List[ItemToUpload]): else: self._small_files_queue.put_nowait(item_to_upload) break - except Exception: - logger.debug(traceback.format_exc()) + except Exception as e: + logger.debug(e) self._report.failed_annotations.append(item_to_upload.item.name) self.reporter.update_progress() data[idx][1] = True @@ -739,8 +739,8 @@ def execute(self): try: nest_asyncio.apply() asyncio.run(self.run_workers(items_to_upload)) - except Exception: - logger.debug(traceback.format_exc()) + except Exception as e: + logger.debug(e) self._response.errors = AppException("Can't upload annotations.") self.reporter.finish_progress() self._log_report() @@ -778,22 +778,22 @@ def execute(self): class UploadAnnotationUseCase(BaseReportableUseCase): def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - image: ImageEntity, - team: TeamEntity, - service_provider: BaseServiceProvider, - reporter: Reporter, - annotation_upload_data: UploadAnnotationAuthData = None, - annotations: dict = None, - s3_bucket=None, - client_s3_bucket=None, - mask=None, - verbose: bool = True, - annotation_path: str = None, - pass_validation: bool = False, - keep_status: bool = False, + self, + project: ProjectEntity, + folder: FolderEntity, + image: ImageEntity, + team: TeamEntity, + service_provider: BaseServiceProvider, + reporter: Reporter, + annotation_upload_data: UploadAnnotationAuthData = None, + annotations: dict = None, + s3_bucket=None, + client_s3_bucket=None, + mask=None, + verbose: bool = True, + annotation_path: str = None, + pass_validation: bool = False, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -976,8 +976,8 @@ def execute(self): ) if ( - self._project.type == constants.ProjectType.PIXEL.value - and mask + self._project.type == constants.ProjectType.PIXEL.value + and mask ): self.s3_bucket.put_object( Key=self.annotation_upload_data.images[self._image.id][ @@ -1011,14 +1011,14 @@ def execute(self): class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( - self, - config: ConfigEntity, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - video_name: str, - fps: int, - service_provider: BaseServiceProvider + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + video_name: str, + fps: int, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._config = config @@ -1073,13 +1073,13 @@ class UploadPriorityScoresUseCase(BaseReportableUseCase): CHUNK_SIZE = 100 def __init__( - self, - reporter, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScoreEntity], - project_folder_name: str, - service_provider: BaseServiceProvider, + self, + reporter, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScoreEntity], + project_folder_name: str, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._project = project @@ -1135,8 +1135,8 @@ def execute(self): if iterations: for i in iterations: priorities_to_upload = priorities[ - i: i + self.CHUNK_SIZE - ] # noqa: E203 + i : i + self.CHUNK_SIZE + ] # noqa: E203 res = self._service_provider.projects.upload_priority_scores( project=self._project, folder=self._folder, @@ -1166,12 +1166,12 @@ class ValidateAnnotationUseCase(BaseReportableUseCase): } def __init__( - self, - reporter: Reporter, - team_id: int, - project_type: int, - annotation: dict, - service_provider: BaseServiceProvider, + self, + reporter: Reporter, + team_id: int, + project_type: int, + annotation: dict, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._team_id = team_id @@ -1339,9 +1339,9 @@ def extract_messages(self, path, error, report): for sub_error in sorted(error.context, key=lambda e: e.schema_path): tmp_path = sub_error.path # if sub_error.path else real_path _path = ( - f"{''.join(path)}" - + ("." if tmp_path else "") - + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) + f"{''.join(path)}" + + ("." if tmp_path else "") + + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) ) if sub_error.context: self.extract_messages(_path, sub_error, report) @@ -1377,13 +1377,13 @@ def execute(self) -> Response: class GetAnnotations(BaseReportableUseCase): def __init__( - self, - config: ConfigEntity, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - item_names: Optional[List[str]], - service_provider: BaseServiceProvider, + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + item_names: Optional[List[str]], + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._config = config @@ -1407,7 +1407,8 @@ def validate_item_names(self): self.reporter.log_info( f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." ) - self._item_names = unique_item_names + # Keep order required + self._item_names = [i for i in self._item_names if i in unique_item_names] elif self._item_names is None: self._item_names_provided = False condition = Condition("project_id", self._project.id, EQ) & Condition( @@ -1469,9 +1470,9 @@ async def get_small_annotations(self): return small_annotations async def run_workers( - self, - big_annotations: List[BaseItemEntity], - small_annotations: List[BaseItemEntity], + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[BaseItemEntity], ): annotations = [] if big_annotations: @@ -1479,35 +1480,41 @@ async def run_workers( for item in big_annotations: self._big_annotations_queue.put_nowait(item) self._big_annotations_queue.put_nowait(None) - annotations.extend( - asyncio.gather( - *[ - self.get_big_annotation() - for _ in range(max(self._config.MAX_COROUTINE_COUNT // 2, 1)) - ] - ) - ) - if small_annotations: - self._small_annotations_queue = asyncio.Queue() - small_chunks = divide_to_chunks( - small_annotations, size=self._config.ANNOTATION_CHUNK_SIZE - ) - for chunk in small_chunks: - self._small_annotations_queue.put_nowait(chunk) - self._small_annotations_queue.put_nowait(None) - annotations.extend( list( itertools.chain.from_iterable( await asyncio.gather( *[ - self.get_small_annotations() - for _ in range(self._config.MAX_COROUTINE_COUNT) + self.get_big_annotation() + for _ in range( + max(self._config.MAX_COROUTINE_COUNT // 2, 1) + ) ] ) ) ) ) + if small_annotations: + self._small_annotations_queue = asyncio.Queue() + small_chunks = divide_to_chunks( + small_annotations, size=self._config.ANNOTATION_CHUNK_SIZE + ) + for chunk in small_chunks: + self._small_annotations_queue.put_nowait(chunk) + self._small_annotations_queue.put_nowait(None) + + annotations.extend( + list( + itertools.chain.from_iterable( + await asyncio.gather( + *[ + self.get_small_annotations() + for _ in range(self._config.MAX_COROUTINE_COUNT) + ] + ) + ) + ) + ) return list(filter(None, annotations)) def execute(self): @@ -1535,7 +1542,8 @@ def execute(self): f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}." ) self.reporter.start_progress( - items_count, disable=logger.level > logging.INFO or self.reporter.log_enabled + items_count, + disable=logger.level > logging.INFO or self.reporter.log_enabled, ) sort_response = self._service_provider.annotations.sort_items_by_size( @@ -1564,16 +1572,16 @@ def execute(self): class DownloadAnnotations(BaseReportableUseCase): def __init__( - self, - config: ConfigEntity, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - destination: str, - recursive: bool, - item_names: List[str], - service_provider: BaseServiceProvider, - callback: Callable = None, + self, + config: ConfigEntity, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + destination: str, + recursive: bool, + item_names: List[str], + service_provider: BaseServiceProvider, + callback: Callable = None, ): super().__init__(reporter) self._config = config @@ -1601,7 +1609,7 @@ def validate_destination(self): if self._destination: destination = str(self._destination) if not os.path.exists(destination) or not os.access( - destination, os.X_OK | os.W_OK + destination, os.X_OK | os.W_OK ): raise AppException( f"Local path {destination} is not an existing directory or access denied." @@ -1683,11 +1691,11 @@ async def download_small_annotations(self, export_path, folder: FolderEntity): break async def run_workers( - self, - big_annotations: List[BaseItemEntity], - small_annotations: List[BaseItemEntity], - folder: FolderEntity, - export_path, + self, + big_annotations: List[BaseItemEntity], + small_annotations: List[BaseItemEntity], + folder: FolderEntity, + export_path, ): if big_annotations: self._big_file_queue = asyncio.Queue() diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py index ac7adab0a..be87a312c 100644 --- a/src/superannotate/lib/infrastructure/services/annotation.py +++ b/src/superannotate/lib/infrastructure/services/annotation.py @@ -29,9 +29,6 @@ class AnnotationService(BaseAnnotationService): URL_GET_ANNOTATIONS = "items/annotations/download" URL_UPLOAD_ANNOTATIONS = "items/annotations/upload" - URL_LARGE_ANNOTATION = "items/{item_id}/annotations/download" - URL_SYNC_LARGE_ANNOTATION = "items/{item_id}/annotations/sync" - URL_SYNC_LARGE_ANNOTATION_STATUS = "items/{item_id}/annotations/sync/status" URL_CLASSIFY_ITEM_SIZE = "items/annotations/download/method" URL_DOWNLOAD_LARGE_ANNOTATION = "items/{item_id}/annotations/download" URL_START_FILE_UPLOAD_PROCESS = "items/{item_id}/annotations/upload/multipart/start" @@ -45,6 +42,7 @@ class AnnotationService(BaseAnnotationService): @property def assets_provider_url(self): + if self.client.api_url != constants.BACKEND_URL: return f"https://assets-provider.devsuperannotate.com/api/{self.ASSETS_PROVIDER_VERSION}/" return f"https://assets-provider.superannotate.com/api/{self.ASSETS_PROVIDER_VERSION}/" @@ -71,7 +69,7 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): } sync_url = urljoin( self.assets_provider_url, - self.URL_SYNC_LARGE_ANNOTATION.format(item_id=item_id), + self.URL_START_FILE_SYNC.format(item_id=item_id), ) async with aiohttp.ClientSession( connector=aiohttp.TCPConnector(ssl=False), @@ -86,7 +84,7 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): synced = False sync_status_url = urljoin( self.assets_provider_url, - self.URL_SYNC_LARGE_ANNOTATION_STATUS.format(item_id=item_id), + self.URL_START_FILE_SYNC_STATUS.format(item_id=item_id), ) while synced != "SUCCESS": synced = await session.get(sync_status_url, params=sync_params) @@ -103,7 +101,7 @@ async def get_big_annotation( ) -> dict: url = urljoin( self.assets_provider_url, - self.URL_LARGE_ANNOTATION.format(item_id=item.id), + self.URL_DOWNLOAD_LARGE_ANNOTATION.format(item_id=item.id), ) query_params = { @@ -155,38 +153,6 @@ async def list_small_annotations( params=query_params, ) - async def get_small_annotations( - self, - project: entities.ProjectEntity, - folder: entities.FolderEntity, - items: List[str], - reporter: Reporter, - callback: Callable = None, - ) -> List[dict]: - query_params = { - "team_id": project.team_id, - "project_id": project.id, - "folder_id": folder.id, - } - - handler = StreamedAnnotations( - self.client.default_headers, - reporter, - map_function=lambda x: {"image_names": x}, - callback=callback, - ) - - loop = asyncio.new_event_loop() - - return loop.run_until_complete( - handler.get_data( - url=urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS), - data=items, - params=query_params, - chunk_size=self.DEFAULT_CHUNK_SIZE, - ) - ) - def sort_items_by_size( self, project: entities.ProjectEntity, @@ -330,6 +296,7 @@ async def upload_small_annotations( if not _response.ok: logger.debug(await _response.text()) raise AppException("Can't upload annotations.") + logger.debug(_response.status) data_json = await _response.json() response = UploadAnnotationsResponse() response.status = _response.status diff --git a/src/superannotate/lib/infrastructure/services/item.py b/src/superannotate/lib/infrastructure/services/item.py index 5837701d1..1870c52bf 100644 --- a/src/superannotate/lib/infrastructure/services/item.py +++ b/src/superannotate/lib/infrastructure/services/item.py @@ -22,15 +22,15 @@ class ItemService(BaseItemService): URL_LIST = "items" URL_GET = "image/{}" - URL_LIST_BY_NAMES = "images/getBulk" URL_ATTACH = "image/ext-create" + URL_GET_BY_ID = "image/{image_id}" URL_MOVE_MULTIPLE = "image/move" + URL_SET_ANNOTATION_STATUSES = "image/updateAnnotationStatusBulk" + URL_LIST_BY_NAMES = "images/getBulk" URL_COPY_MULTIPLE = "images/copy-image-or-folders" URL_COPY_PROGRESS = "images/copy-image-progress" URL_DELETE_ITEMS = "image/delete/images" - URL_SET_ANNOTATION_STATUSES = "image/updateAnnotationStatusBulk" URL_SET_APPROVAL_STATUSES = "/items/bulk/change" - URL_GET_BY_ID = "image/{image_id}" PROJECT_TYPE_RESPONSE_MAP = { ProjectType.VECTOR: ImageResponse, diff --git a/src/superannotate/lib/infrastructure/services/project.py b/src/superannotate/lib/infrastructure/services/project.py index c251da22b..c304b0d48 100644 --- a/src/superannotate/lib/infrastructure/services/project.py +++ b/src/superannotate/lib/infrastructure/services/project.py @@ -13,8 +13,8 @@ class ProjectService(BaseProjectService): URL_LIST = "projects" URL_GET = "project/{}" URL_SETTINGS = "project/{}/settings" - URL_SHARE = "project/{}/share/bulk" URL_WORKFLOW = "project/{}/workflow" + URL_SHARE = "project/{}/share/bulk" URL_SHARE_PROJECT = "project/{}/share" URL_WORKFLOW_ATTRIBUTE = "project/{}/workflow_attribute" URL_UPLOAD_PRIORITY_SCORES = "images/updateEntropy" diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 1fb6dc828..345ada58c 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -30,6 +30,12 @@ def __init__( self._map_function = map_function self._items_downloaded = 0 + def get_json(self, data: bytes): + try: + return json.loads(data) + except json.decoder.JSONDecodeError as e: + self._reporter.log_error(f"Invalud chunk: {str(e)}") + async def fetch( self, method: str, @@ -41,25 +47,17 @@ async def fetch( kwargs = {"params": params, "json": {"folder_id": params.pop("folder_id")}} if data: kwargs["json"].update(data) - response = await session._request(method, url, **kwargs, timeout=TIMEOUT) + response = await session._request( # noqa + method, url, **kwargs, timeout=TIMEOUT + ) buffer = b"" async for line in response.content.iter_any(): - slices = line.split(self.DELIMITER) - if len(slices) == 2 and slices[0]: - self._reporter.update_progress() - buffer += slices[0] - yield json.loads(buffer) - buffer = b"" - elif len(slices) > 2: - for _slice in slices[:-1]: - if not _slice: - continue - self._reporter.update_progress() - yield json.loads(buffer + _slice) - buffer = b"" - buffer += slices[-1] + slices = (buffer + line).split(self.DELIMITER) + for _slice in slices[:-1]: + yield self.get_json(_slice) + buffer = slices[-1] if buffer: - yield json.loads(buffer) + yield self.get_json(buffer) self._reporter.update_progress() async def list_annotations( @@ -70,6 +68,9 @@ async def list_annotations( params: dict = None, verify_ssl=False, ): + params = copy.copy(params) + params["limit"] = len(data) + annotations = [] async with aiohttp.ClientSession( headers=self._headers, timeout=TIMEOUT, @@ -83,10 +84,11 @@ async def list_annotations( self._process_data(data), params=copy.copy(params), ): - self._annotations.append( + annotations.append( self._callback(annotation) if self._callback else annotation ) - return self._annotations + + return annotations async def download_annotations( self, @@ -97,6 +99,8 @@ async def download_annotations( data: typing.List[int], params: dict = None, ): + params = copy.copy(params) + params["limit"] = len(data) async with aiohttp.ClientSession( headers=self._headers, timeout=TIMEOUT, @@ -108,7 +112,7 @@ async def download_annotations( session, url, self._process_data(data), - params=copy.copy(params), + params=params, ): self._annotations.append( self._callback(annotation) if self._callback else annotation diff --git a/tests/integration/annotations/test_annotation_delete.py b/tests/integration/annotations/test_annotation_delete.py index b9bd0af02..5d4943b71 100644 --- a/tests/integration/annotations/test_annotation_delete.py +++ b/tests/integration/annotations/test_annotation_delete.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from src.superannotate import AppException from src.superannotate import SAClient from tests.integration.base import BaseTestCase @@ -29,16 +30,13 @@ def classes_json(self): ) def test_delete_annotations(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=1) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, f"{self.folder_path}" ) - sa.delete_annotations(self.PROJECT_NAME) annotations = sa.get_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1]) del annotations[0]["metadata"]["projectId"] @@ -46,8 +44,8 @@ def test_delete_annotations(self): { "metadata": { "name": "example_image_1.jpg", - "height": 683, - "width": 1024, + "height": None, + "width": None, "isPredicted": False, "status": "NotStarted", "pinned": False, @@ -61,10 +59,7 @@ def test_delete_annotations(self): ] def test_delete_annotations_by_name(self): - # self._attach_items(4) - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=1) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) @@ -78,8 +73,8 @@ def test_delete_annotations_by_name(self): { "metadata": { "name": "example_image_1.jpg", - "height": 683, - "width": 1024, + "height": None, + "width": None, "isPredicted": False, "status": "NotStarted", "pinned": False, @@ -93,27 +88,22 @@ def test_delete_annotations_by_name(self): ] def test_delete_annotations_by_not_existing_name(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=1) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, f"{self.folder_path}" ) - self.assertRaises( - Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2] - ) + with self.assertRaisesRegexp( + AppException, "Invalid item names or empty folder." + ): + sa.delete_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2]) @pytest.mark.flaky(reruns=2) def test_delete_annotations_wrong_path(self): sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) + self._attach_items(count=1, folder=self.TEST_FOLDER_NAME) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, self.folder_path + "/classes/classes.json" ) @@ -123,23 +113,3 @@ def test_delete_annotations_wrong_path(self): self.assertRaises( Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1] ) - - def test_delete_annotations_from_folder(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) - sa.create_annotation_classes_from_classes_json( - self.PROJECT_NAME, self.folder_path + "/classes/classes.json" - ) - sa.upload_annotations_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", f"{self.folder_path}" - ) - sa.delete_annotations( - f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", [self.EXAMPLE_IMAGE_1] - ) - annotations = sa.get_annotations(self.PROJECT_NAME, [self.EXAMPLE_IMAGE_1]) - assert len(annotations) == 0 diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index c1417eb20..088b60ebd 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -66,12 +66,7 @@ def test_get_annotations_order(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_from_folder(self): sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) - - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) + self._attach_items(count=4, folder=self.FOLDER_NAME) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) @@ -93,9 +88,7 @@ def test_get_annotations_from_folder(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_all(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=4) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) @@ -108,14 +101,9 @@ def test_get_annotations_all(self): @pytest.mark.flaky(reruns=3) def test_get_annotations_all_plus_folder(self): sa.create_folder(self.PROJECT_NAME, self.FOLDER_NAME) - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) - sa.upload_images_from_folder_to_project( - f"{self.PROJECT_NAME}/{self.FOLDER_NAME}", - self.folder_path, - annotation_status="InProgress", - ) + self._attach_items(count=4) + self._attach_items(count=4, folder=self.FOLDER_NAME) + sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) diff --git a/tests/integration/base.py b/tests/integration/base.py index bc88c80a4..206938d63 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -18,7 +18,6 @@ def __init__(self, *args, **kwargs): def setUp(self, *args, **kwargs): self.tearDown() - print(self.PROJECT_NAME) self._project = sa.create_project( self.PROJECT_NAME, self.PROJECT_DESCRIPTION, self.PROJECT_TYPE ) diff --git a/tests/integration/classes/test_create_bed_handling.py b/tests/integration/classes/test_create_bed_handling.py index d13ff55a1..926eb872b 100644 --- a/tests/integration/classes/test_create_bed_handling.py +++ b/tests/integration/classes/test_create_bed_handling.py @@ -1,3 +1,4 @@ +import pytest from src.superannotate import SAClient from tests.integration.base import BaseTestCase @@ -11,6 +12,7 @@ class TestCreateAnnotationClass(BaseTestCase): TEST_LARGE_CLASSES_JSON = "large_classes_json.json" EXAMPLE_IMAGE_1 = "example_image_1.jpg" + @pytest.mark.flaky(reruns=2) def test_multi_select_to_checklist(self): sa.create_annotation_class( self.PROJECT_NAME, diff --git a/tests/integration/custom_fields/test_custom_schema.py b/tests/integration/custom_fields/test_custom_schema.py index 7837c5d3d..1f52059ee 100644 --- a/tests/integration/custom_fields/test_custom_schema.py +++ b/tests/integration/custom_fields/test_custom_schema.py @@ -12,6 +12,29 @@ class TestCustomSchema(BaseTestCase): PROJECT_DESCRIPTION = "desc" PROJECT_TYPE = "Vector" PAYLOAD = {"test": {"type": "number"}, "tester": {"type": "number"}} + INITIAL_ALL_VALID = { + "test_string_email": { + "type": "string", + "format": "email", + "enum": [ + "abc@gmail.com", + "afg@ahg.com", + "name+@gmail.com", + "name+surname@mail.ru", + ], + }, + "test_type_number": {"type": "number"}, + "number_and_range": {"type": "number", "minimum": -1, "maximum": 100.5}, + "string_and_enum": {"type": "string", "enum": ["one", "two", "tree"]}, + "string_and_date": {"type": "string", "format": "date"}, + "number_and_enum": {"type": "number", "enum": [1.2, 0, -345, 100, 1, 0.1]}, + "string_date_enum": { + "type": "string", + "format": "date", + "enum": ["2022-12-11", "2000-10-9"], + }, + "just_string": {"type": "string"}, + } def test_create_schema(self): data = sa.create_custom_fields(self.PROJECT_NAME, self.PAYLOAD) @@ -35,6 +58,12 @@ def test_get_schema(self): sa.create_custom_fields(self.PROJECT_NAME, self.PAYLOAD) self.assertEqual(sa.get_custom_fields(self.PROJECT_NAME), self.PAYLOAD) + def test_create_large_schema(self): + sa.create_custom_fields(self.PROJECT_NAME, self.INITIAL_ALL_VALID) + self.assertEqual( + sa.get_custom_fields(self.PROJECT_NAME), self.INITIAL_ALL_VALID + ) + def test_delete_schema(self): payload = copy.copy(self.PAYLOAD) sa.create_custom_fields(self.PROJECT_NAME, payload) diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 2a83e14ba..d7b9cdf36 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -21,8 +21,6 @@ class TestInterface(BaseTestCase): EXAMPLE_IMAGE_1 = "example_image_1.jpg" EXAMPLE_IMAGE_2 = "example_image_2.jpg" NEW_IMAGE_NAME = "new_name_yup" - IMAGE_PATH_IN_S3 = "MP.MB/img1.bmp" - TEST_S3_BUCKET_NAME = "test-openseadragon-1212" TEST_INVALID_ANNOTATION_FOLDER_PATH = "sample_project_vector_invalid" @property @@ -45,27 +43,6 @@ def folder_path_with_multiple_images(self): dirname(dirname(__file__)), self.TEST_FOLDER_PATH_WITH_MULTIPLE_IMAGERS ) - @pytest.mark.flaky(reruns=4) - def test_delete_items(self): - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) - - path = f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}" - sa.upload_images_from_folder_to_project( - path, - self.folder_path, - annotation_status="InProgress", - ) - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 4) - sa.delete_items(path) - - num_images = sa.get_project_image_count( - self.PROJECT_NAME, with_all_subfolders=True - ) - self.assertEqual(num_images, 0) - def test_delete_folder(self): with self.assertRaises(AppException): sa.delete_folders(self.PROJECT_NAME, ["non-existing folder"]) @@ -84,11 +61,7 @@ def test_get_project_metadata(self): self.assertIsNotNone(metadata_with_users.get("contributors")) def test_upload_annotations_from_folder_to_project(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, - self.folder_path, - annotation_status="Completed", - ) + self._attach_items(count=4) uploaded_annotations, _, _ = sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, self.folder_path ) @@ -161,18 +134,6 @@ def test_upload_images_to_project_image_quality_in_editor(self): image_quality_in_editor="random_string", ) - @pytest.mark.flaky(reruns=2) - def test_image_upload_with_set_name_on_platform(self): - sa.upload_image_to_project( - self.PROJECT_NAME, - self.IMAGE_PATH_IN_S3, - self.NEW_IMAGE_NAME, - from_s3_bucket=self.TEST_S3_BUCKET_NAME, - ) - assert self.NEW_IMAGE_NAME in [ - i["name"] for i in sa.search_items(self.PROJECT_NAME) - ] - def test_download_fuse_without_classes(self): sa.upload_image_to_project( self.PROJECT_NAME, f"{self.folder_path}/{self.EXAMPLE_IMAGE_1}" diff --git a/tests/integration/test_recursive_folder_pixel.py b/tests/integration/test_recursive_folder_pixel.py deleted file mode 100644 index c27586488..000000000 --- a/tests/integration/test_recursive_folder_pixel.py +++ /dev/null @@ -1,30 +0,0 @@ -from src.superannotate import SAClient -from tests.integration.base import BaseTestCase - -sa = SAClient() - - -class TestRecursiveFolderPixel(BaseTestCase): - PROJECT_NAME = "pixel_recursive_test" - PROJECT_DESCRIPTION = "Desc" - PROJECT_TYPE = "Pixel" - S3_FOLDER_PATH = "pixel_recursive_annotations" - JSON_POSTFIX = "*.json" - - def test_recursive_upload_pixel(self): - uploaded, _, duplicated = sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, - self.S3_FOLDER_PATH, - from_s3_bucket="test-openseadragon-1212", - recursive_subfolders=True, - ) - - uploaded, failed, missing = sa.upload_annotations_from_folder_to_project( - self.PROJECT_NAME, - self.S3_FOLDER_PATH, - from_s3_bucket="test-openseadragon-1212", - recursive_subfolders=True, - ) - self.assertEqual(4, len(uploaded)) - self.assertEqual(0, len(failed)) - self.assertEqual(1, len(missing)) diff --git a/tests/integration/test_upload_priority_scores.py b/tests/integration/test_upload_priority_scores.py index a304ea091..491146b7c 100644 --- a/tests/integration/test_upload_priority_scores.py +++ b/tests/integration/test_upload_priority_scores.py @@ -19,9 +19,7 @@ def folder_path(self): def test_upload_priority_scores(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items(count=4) uploaded, skipped = sa.upload_priority_scores( self.PROJECT_NAME, scores=[{"name": "example_image_1.jpg", "priority": 1}] )