diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index d0263a781..a3caad3a0 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -2,7 +2,7 @@ import sys -__version__ = "4.4.5b2" +__version__ = "4.4.5b3" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index a0635a3d2..2704e1e72 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -23,12 +23,14 @@ import aiofiles import boto3 import jsonschema.validators -import lib.core as constants import nest_asyncio from jsonschema import Draft7Validator from jsonschema import ValidationError -from lib.core.conditions import Condition +from pydantic import BaseModel + +import lib.core as constants from lib.core.conditions import CONDITION_EQ as EQ +from lib.core.conditions import Condition from lib.core.entities import BaseItemEntity from lib.core.entities import FolderEntity from lib.core.entities import ImageEntity @@ -42,7 +44,6 @@ from lib.core.types import PriorityScore from lib.core.usecases.base import BaseReportableUseCase from lib.core.video_convertor import VideoFrameGenerator -from pydantic import BaseModel from superannotate.logger import get_default_logger logger = get_default_logger() @@ -65,7 +66,7 @@ class Report: def log_report( - report: Report, + report: Report, ): if report.missing_classes: logger.warning( @@ -97,18 +98,18 @@ class Config: def set_annotation_statuses_in_progress( - service_provider: BaseServiceProvider, - project: ProjectEntity, - folder: FolderEntity, - item_names: List[str], - chunk_size=500, + service_provider: BaseServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + item_names: List[str], + chunk_size=500, ) -> bool: failed_on_chunk = False for i in range(0, len(item_names), chunk_size): status_changed = service_provider.items.set_statuses( project=project, folder=folder, - item_names=item_names[i : i + chunk_size], # noqa: E203 + item_names=item_names[i: i + chunk_size], # noqa: E203 annotation_status=constants.AnnotationStatus.IN_PROGRESS.value, ) if not status_changed: @@ -117,13 +118,13 @@ def set_annotation_statuses_in_progress( async def upload_small_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def upload(_chunk): failed_annotations, missing_classes, missing_attr_groups, missing_attrs = ( @@ -168,9 +169,9 @@ async def upload(_chunk): queue.put_nowait(None) break if ( - _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB - or sum([len(i.item.name) for i in chunk]) - >= URI_THRESHOLD - (len(chunk) + 1) * 14 + _size + item_data.file_size >= ANNOTATION_CHUNK_SIZE_MB + or sum([len(i.item.name) for i in chunk]) + >= URI_THRESHOLD - (len(chunk) + 1) * 14 ): await upload(chunk) chunk = [] @@ -182,13 +183,13 @@ async def upload(_chunk): async def upload_big_annotations( - project: ProjectEntity, - folder: FolderEntity, - queue: asyncio.Queue, - service_provider: BaseServiceProvider, - reporter: Reporter, - report: Report, - callback: Callable = None, + project: ProjectEntity, + folder: FolderEntity, + queue: asyncio.Queue, + service_provider: BaseServiceProvider, + reporter: Reporter, + report: Report, + callback: Callable = None, ): async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]: try: @@ -224,13 +225,13 @@ class UploadAnnotationsUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - annotations: List[dict], - service_provider: BaseServiceProvider, - keep_status: bool = False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + annotations: List[dict], + service_provider: BaseServiceProvider, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -257,7 +258,7 @@ def _validate_json(self, json_data: dict) -> list: def list_existing_items(self, item_names: List[str]) -> List[BaseItemEntity]: existing_items = [] for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -294,7 +295,7 @@ async def distribute_queues(self, items_to_upload: List[ItemToUpload]): "name" ] ) - continue + break self._small_files_queue.put_nowait(item_to_upload) break except Exception: @@ -416,17 +417,17 @@ class UploadAnnotationsFromFolderUseCase(BaseReportableUseCase): URI_THRESHOLD = 4 * 1024 - 120 def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - team: TeamEntity, - annotation_paths: List[str], - service_provider: BaseServiceProvider, - pre_annotation: bool = False, - client_s3_bucket=None, - folder_path: str = None, - keep_status=False, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + team: TeamEntity, + annotation_paths: List[str], + service_provider: BaseServiceProvider, + pre_annotation: bool = False, + client_s3_bucket=None, + folder_path: str = None, + keep_status=False, ): super().__init__(reporter) self._project = project @@ -467,7 +468,7 @@ def get_name_path_mappings(annotation_paths): return name_path_mappings def _log_report( - self, + self, ): if self._report.missing_classes: logger.warning( @@ -521,7 +522,7 @@ def prepare_annotation(self, annotation: dict, size) -> dict: return annotation async def get_annotation( - self, path: str + self, path: str ) -> (Optional[Tuple[io.StringIO]], Optional[io.BytesIO]): mask = None mask_path = path.replace( @@ -559,17 +560,17 @@ def chunks(data, size: int = 10000): def extract_name(value: str): return os.path.basename( value.replace(constants.PIXEL_ANNOTATION_POSTFIX, "") - .replace(constants.VECTOR_ANNOTATION_POSTFIX, "") - .replace(constants.ATTACHED_VIDEO_ANNOTATION_POSTFIX, ""), + .replace(constants.VECTOR_ANNOTATION_POSTFIX, "") + .replace(constants.ATTACHED_VIDEO_ANNOTATION_POSTFIX, ""), ) def get_existing_name_item_mapping( - self, name_path_mappings: Dict[str, str] + self, name_path_mappings: Dict[str, str] ) -> dict: item_names = list(name_path_mappings.keys()) existing_name_item_mapping = {} for i in range(0, len(item_names), self.CHUNK_SIZE): - items_to_check = item_names[i : i + self.CHUNK_SIZE] # noqa: E203 + items_to_check = item_names[i: i + self.CHUNK_SIZE] # noqa: E203 response = self._service_provider.items.list_by_names( project=self._project, folder=self._folder, names=items_to_check ) @@ -742,22 +743,22 @@ def execute(self): class UploadAnnotationUseCase(BaseReportableUseCase): def __init__( - self, - project: ProjectEntity, - folder: FolderEntity, - image: ImageEntity, - team: TeamEntity, - service_provider: BaseServiceProvider, - reporter: Reporter, - annotation_upload_data: UploadAnnotationAuthData = None, - annotations: dict = None, - s3_bucket=None, - client_s3_bucket=None, - mask=None, - verbose: bool = True, - annotation_path: str = None, - pass_validation: bool = False, - keep_status: bool = False, + self, + project: ProjectEntity, + folder: FolderEntity, + image: ImageEntity, + team: TeamEntity, + service_provider: BaseServiceProvider, + reporter: Reporter, + annotation_upload_data: UploadAnnotationAuthData = None, + annotations: dict = None, + s3_bucket=None, + client_s3_bucket=None, + mask=None, + verbose: bool = True, + annotation_path: str = None, + pass_validation: bool = False, + keep_status: bool = False, ): super().__init__(reporter) self._project = project @@ -937,8 +938,8 @@ def execute(self): ) if ( - self._project.type == constants.ProjectType.PIXEL.value - and mask + self._project.type == constants.ProjectType.PIXEL.value + and mask ): self.s3_bucket.put_object( Key=self.annotation_upload_data.images[self._image.id][ @@ -972,13 +973,13 @@ def execute(self): class GetAnnotations(BaseReportableUseCase): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - item_names: Optional[List[str]], - service_provider: BaseServiceProvider, - show_process: bool = True, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + item_names: Optional[List[str]], + service_provider: BaseServiceProvider, + show_process: bool = True, ): super().__init__(reporter) self._project = project @@ -1108,13 +1109,13 @@ def execute(self): class GetVideoAnnotationsPerFrame(BaseReportableUseCase): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - video_name: str, - fps: int, - service_provider: BaseServiceProvider, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + video_name: str, + fps: int, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._project = project @@ -1168,13 +1169,13 @@ class UploadPriorityScoresUseCase(BaseReportableUseCase): CHUNK_SIZE = 100 def __init__( - self, - reporter, - project: ProjectEntity, - folder: FolderEntity, - scores: List[PriorityScore], - project_folder_name: str, - service_provider: BaseServiceProvider, + self, + reporter, + project: ProjectEntity, + folder: FolderEntity, + scores: List[PriorityScore], + project_folder_name: str, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._project = project @@ -1230,8 +1231,8 @@ def execute(self): if iterations: for i in iterations: priorities_to_upload = priorities[ - i : i + self.CHUNK_SIZE - ] # noqa: E203 + i: i + self.CHUNK_SIZE + ] # noqa: E203 res = self._service_provider.projects.upload_priority_scores( project=self._project, folder=self._folder, @@ -1254,15 +1255,15 @@ def execute(self): class DownloadAnnotations(BaseReportableUseCase): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - destination: str, - recursive: bool, - item_names: List[str], - service_provider: BaseServiceProvider, - callback: Callable = None, + self, + reporter: Reporter, + project: ProjectEntity, + folder: FolderEntity, + destination: str, + recursive: bool, + item_names: List[str], + service_provider: BaseServiceProvider, + callback: Callable = None, ): super().__init__(reporter) self._project = project @@ -1289,7 +1290,7 @@ def validate_destination(self): if self._destination: destination = str(self._destination) if not os.path.exists(destination) or not os.access( - destination, os.X_OK | os.W_OK + destination, os.X_OK | os.W_OK ): raise AppException( f"Local path {destination} is not an existing directory or access denied." @@ -1362,7 +1363,7 @@ async def download_big_annotations(self, queue_idx, export_path): break async def download_small_annotations( - self, queue_idx, export_path, folder: FolderEntity + self, queue_idx, export_path, folder: FolderEntity ): cur_queue = self._small_file_queues[queue_idx] items = [] @@ -1383,7 +1384,7 @@ async def download_small_annotations( ) async def distribute_to_queues( - self, item_names, sm_queue_id, l_queue_id, folder: FolderEntity + self, item_names, sm_queue_id, l_queue_id, folder: FolderEntity ): try: resp = self._service_provider.annotations.sort_items_by_size( @@ -1491,12 +1492,12 @@ class ValidateAnnotationUseCase(BaseReportableUseCase): } def __init__( - self, - reporter: Reporter, - team_id: int, - project_type: int, - annotation: dict, - service_provider: BaseServiceProvider, + self, + reporter: Reporter, + team_id: int, + project_type: int, + annotation: dict, + service_provider: BaseServiceProvider, ): super().__init__(reporter) self._team_id = team_id @@ -1661,9 +1662,9 @@ def extract_messages(self, path, error, report): for sub_error in sorted(error.context, key=lambda e: e.schema_path): tmp_path = sub_error.path # if sub_error.path else real_path _path = ( - f"{''.join(path)}" - + ("." if tmp_path else "") - + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) + f"{''.join(path)}" + + ("." if tmp_path else "") + + "".join(ValidateAnnotationUseCase.extract_path(tmp_path)) ) if sub_error.context: self.extract_messages(_path, sub_error, report)