From 6af08e31235f94dbf6a555ebf94782cbbcbc9613 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Mon, 21 Feb 2022 11:26:16 +0400 Subject: [PATCH 1/3] Add deprication message --- src/superannotate/lib/app/interface/sdk_interface.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index d33b84af2..4cba46e51 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -547,6 +547,12 @@ def upload_images_from_public_urls_to_project( and not-uploaded images' urls :rtype: tuple of list of strs """ + warning_msg = ( + "The upload_images_from_public_urls function is deprecated and will be removed with the coming release, " + "please use attach_image_urls_to_project instead." + ) + logger.warning(warning_msg) + warnings.warn(warning_msg, DeprecationWarning) project_name, folder_name = extract_project_folder(project) From 714ee8576ba4ad7781b558f0ed91e2e521aae789 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 21 Feb 2022 16:22:38 +0400 Subject: [PATCH 2/3] Fix logs --- .../lib/app/interface/cli_interface.py | 3 ++- .../lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/app/mixp/decorators.py | 4 ++-- .../lib/app/mixp/utils/parsers.py | 2 +- src/superannotate/lib/core/data_handlers.py | 7 +++---- src/superannotate/lib/core/reporter.py | 3 ++- src/superannotate/lib/core/serviceproviders.py | 10 +++++++++- .../lib/core/usecases/annotations.py | 18 ++++++++++++++++-- .../lib/core/usecases/projects.py | 1 - src/superannotate/lib/core/video_convertor.py | 4 ++-- .../lib/infrastructure/services.py | 12 ++++++++++-- .../lib/infrastructure/stream_data_handler.py | 6 +++++- 12 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index b1ec8f86e..9b28c89a2 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -23,7 +23,8 @@ from lib.infrastructure.repositories import ConfigRepository -controller = Controller.get_default() +controller = Controller() +controller.retrieve_configs(constances.CONFIG_FILE_LOCATION) class CLIFacade(BaseInterfaceFacade): diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index d33b84af2..2e61157c3 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2416,7 +2416,7 @@ def add_annotation_comment_to_image( image_name: NotEmptyStr, comment_text: NotEmptyStr, comment_coords: List[float], - comment_author: NotEmptyStr, + comment_author: EmailStr, resolved: Optional[StrictBool] = False, ): """Add a comment to SuperAnnotate format annotation JSON diff --git a/src/superannotate/lib/app/mixp/decorators.py b/src/superannotate/lib/app/mixp/decorators.py index ba5454162..92e779546 100644 --- a/src/superannotate/lib/app/mixp/decorators.py +++ b/src/superannotate/lib/app/mixp/decorators.py @@ -88,8 +88,8 @@ def __call__(self, *args, **kwargs): else: raise Exception( "SuperAnnotate config file not found." - f" Please provide correct config file location to sa.init() or use " - f"CLI's superannotate init to generate default location config file." + " Please provide correct config file location to sa.init() or use " + "CLI's superannotate init to generate default location config file." ) except Exception as e: self._success = False diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index 334a8dc60..e93011cc2 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -1226,4 +1226,4 @@ def get_annotations_per_frame(*args, **kwargs): return { "event_name": "get_annotations_per_frame", "properties": {"Project": project, "fps": fps}, - } \ No newline at end of file + } diff --git a/src/superannotate/lib/core/data_handlers.py b/src/superannotate/lib/core/data_handlers.py index a13d1edbd..493d1815c 100644 --- a/src/superannotate/lib/core/data_handlers.py +++ b/src/superannotate/lib/core/data_handlers.py @@ -8,13 +8,12 @@ from typing import Dict from typing import List -from superannotate_schemas.schemas.classes import AnnotationClass -from superannotate_schemas.schemas.classes import Attribute -from superannotate_schemas.schemas.classes import AttributeGroup - import lib.core as constances from lib.core.enums import ClassTypeEnum from lib.core.reporter import Reporter +from superannotate_schemas.schemas.classes import AnnotationClass +from superannotate_schemas.schemas.classes import Attribute +from superannotate_schemas.schemas.classes import AttributeGroup class BaseDataHandler(metaclass=ABCMeta): diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 1642ceb96..835f4d265 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -63,7 +63,8 @@ def finish_progress(self): self.progress_bar.close() def update_progress(self, value: int = 1): - self.progress_bar.update(value) + if self.progress_bar: + self.progress_bar.update(value) def generate_report(self) -> str: report = "" diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 8c7df3276..f85cb3979 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -5,6 +5,7 @@ from typing import List from typing import Tuple +from lib.core.reporter import Reporter from lib.core.service_types import ServiceResponse @@ -304,5 +305,12 @@ def get_limitations( raise NotImplementedError @abstractmethod - def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: List[str]) -> List[dict]: + def get_annotations( + self, + project_id: int, + team_id: int, + folder_id: int, + items: List[str], + reporter: Reporter + ) -> List[dict]: raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 8fc839fc4..f4ee6f812 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -514,12 +514,24 @@ def validate_item_names(self): self._item_names = item_names def execute(self): + items_count = len(self._item_names) + self.reporter.log_info( + f"Getting {items_count} annotations from " + f"{self._project.name}{f'/{self._folder.name}' if self._folder else ''}." + ) + self.reporter.start_progress(items_count) annotations = self._client.get_annotations( team_id=self._project.team_id, project_id=self._project.uuid, folder_id=self._folder.uuid, - items=self._item_names + items=self._item_names, + reporter=self.reporter ) + received_items_count = len(annotations) + if items_count > received_items_count: + self.reporter.log_warning( + f"Could not find annotations for {items_count - received_items_count}/{items_count} items." + ) self._response.data = annotations return self._response @@ -549,6 +561,8 @@ def execute(self): item_names=[self._video_name], backend_service_provider=self._client ).execute() + generator = VideoFrameGenerator(response.data[0], fps=self._fps) + self.reporter.log_info(f"Getting annotations for {generator.frames_count} frames from {self._video_name}.") if response.errors: self._response.errors = response.errors return self._response @@ -556,7 +570,7 @@ def execute(self): self._response.errors = AppException(f"Video {self._video_name} not found.") annotations = response.data if annotations: - self._response.data = list(VideoFrameGenerator(response.data[0], fps=self._fps)) + self._response.data = list(generator) else: self._response.data = [] return self._response diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 67559f50b..40663a291 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -1,6 +1,5 @@ import copy from collections import defaultdict -from typing import Iterable from typing import List from typing import Type diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 27203d42c..d21608aeb 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -41,7 +41,7 @@ def __init__(self, annotation_data: dict, fps: int): self.fps = fps self.ratio = 1000 * 1000 / fps self._frame_id = 1 - self._frames_count = self.duration * fps + self.frames_count = self.duration * fps self.annotations: dict = {} self._mapping = {} self._process() @@ -141,5 +141,5 @@ def _process(self): ) def __iter__(self): - for frame_no in range(1, int(self._frames_count) + 1): + for frame_no in range(1, int(self.frames_count) + 1): yield self.get_frame(frame_no).dict() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 7339d9434..1844ba3c5 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -13,6 +13,7 @@ import lib.core as constance import requests.packages.urllib3 from lib.core.exceptions import AppException +from lib.core.reporter import Reporter from lib.core.service_types import DownloadMLModelAuthData from lib.core.service_types import ServiceResponse from lib.core.service_types import UploadAnnotationAuthData @@ -1016,7 +1017,14 @@ def get_limitations( content_type=UserLimits, ) - def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: List[str]) -> List[dict]: + def get_annotations( + self, + project_id: int, + team_id: int, + folder_id: int, + items: List[str], + reporter: Reporter + ) -> List[dict]: get_limits_url = urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS) query_params = { "team_id": team_id, @@ -1025,7 +1033,7 @@ def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: if folder_id: query_params["folder_id"] = folder_id - handler = StreamedAnnotations(self.default_headers) + handler = StreamedAnnotations(self.default_headers, reporter) loop = asyncio.new_event_loop() return loop.run_until_complete(handler.get_data( diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 709507f12..5c4abf548 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -3,6 +3,7 @@ from typing import List import aiohttp +from lib.core.reporter import Reporter def map_image_names_to_fetch_streamed_data(data: List[str]): @@ -15,9 +16,10 @@ def map_image_names_to_fetch_streamed_data(data: List[str]): class StreamedAnnotations: DELIMITER = b"\\n;)\\n" - def __init__(self, headers: dict): + def __init__(self, headers: dict, reporter: Reporter): self._headers = headers self._annotations = [] + self._reporter = reporter async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, data: dict = None, params: dict = None): @@ -53,6 +55,8 @@ async def get_data( if chunk_size: for i in range(0, len(data), chunk_size): await self.fetch(method, session, url, map_function(data[i:i + chunk_size]), params=params) + self._reporter.update_progress(chunk_size) else: await self.fetch(method, session, url, map_function(data), params=params) + self._reporter.update_progress(len(data)) return self._annotations From 79400bc0c423ef07c728a6c8181c3dda1b090f76 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Tue, 22 Feb 2022 14:47:38 +0400 Subject: [PATCH 3/3] Fixed streamed data chinks handeling --- .../lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/reporter.py | 4 +- .../lib/core/usecases/annotations.py | 59 ++++++++++++------- src/superannotate/lib/core/usecases/images.py | 4 +- .../lib/infrastructure/controller.py | 2 + .../lib/infrastructure/services.py | 1 + .../lib/infrastructure/stream_data_handler.py | 10 ++-- .../annotations/test_get_annotations.py | 24 ++++++-- tests/profiling/profiling.py | 0 9 files changed, 73 insertions(+), 33 deletions(-) delete mode 100644 tests/profiling/profiling.py diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 2e61157c3..472a3b47a 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2850,7 +2850,7 @@ def invite_contributors_to_team( @Trackable @validate_arguments -def get_annotations(project: NotEmptyStr, items: Optional[List[NotEmptyStr]]): +def get_annotations(project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None): """Returns annotations for the given list of items. :param project: project name diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 835f4d265..b8e140e27 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -46,9 +46,9 @@ def log_debug(self, value: str): self.debug_messages.append(value) def start_progress( - self, iterations: Union[int, range], description: str = "Processing" + self, iterations: Union[int, range], description: str = "Processing", disable=False ): - self.progress_bar = self.get_progress_bar(iterations, description) + self.progress_bar = self.get_progress_bar(iterations, description, disable) @staticmethod def get_progress_bar( diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index f4ee6f812..3bcf1eb39 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -9,6 +9,8 @@ import boto3 import lib.core as constances +from lib.core.conditions import Condition +from lib.core.conditions import CONDITION_EQ as EQ from lib.core.data_handlers import ChainedAnnotationHandlers from lib.core.data_handlers import DocumentTagHandler from lib.core.data_handlers import LastActionHandler @@ -494,14 +496,18 @@ def __init__( reporter: Reporter, project: ProjectEntity, folder: FolderEntity, + images: BaseManageableRepository, item_names: Optional[List[str]], - backend_service_provider: SuerannotateServiceProvider + backend_service_provider: SuerannotateServiceProvider, + show_process: bool = True ): super().__init__(reporter) self._project = project self._folder = folder + self._images = images self._item_names = item_names self._client = backend_service_provider + self._show_process = show_process def validate_item_names(self): if self._item_names: @@ -512,27 +518,36 @@ def validate_item_names(self): f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items." ) self._item_names = item_names + else: + condition = ( + Condition("team_id", self._project.team_id, EQ) + & Condition("project_id", self._project.uuid, EQ) + & Condition("folder_id", self._folder.uuid, EQ) + ) + + self._item_names = [item.name for item in self._images.get_all(condition)] def execute(self): - items_count = len(self._item_names) - self.reporter.log_info( - f"Getting {items_count} annotations from " - f"{self._project.name}{f'/{self._folder.name}' if self._folder else ''}." - ) - self.reporter.start_progress(items_count) - annotations = self._client.get_annotations( - team_id=self._project.team_id, - project_id=self._project.uuid, - folder_id=self._folder.uuid, - items=self._item_names, - reporter=self.reporter - ) - received_items_count = len(annotations) - if items_count > received_items_count: - self.reporter.log_warning( - f"Could not find annotations for {items_count - received_items_count}/{items_count} items." + if self.is_valid(): + items_count = len(self._item_names) + self.reporter.log_info( + f"Getting {items_count} annotations from " + f"{self._project.name}{f'/{self._folder.name}' if self._folder else ''}." ) - self._response.data = annotations + self.reporter.start_progress(items_count, disable=not self._show_process) + annotations = self._client.get_annotations( + team_id=self._project.team_id, + project_id=self._project.uuid, + folder_id=self._folder.uuid, + items=self._item_names, + reporter=self.reporter + ) + received_items_count = len(annotations) + if items_count > received_items_count: + self.reporter.log_warning( + f"Could not find annotations for {items_count - received_items_count}/{items_count} items." + ) + self._response.data = annotations return self._response @@ -542,6 +557,7 @@ def __init__( reporter: Reporter, project: ProjectEntity, folder: FolderEntity, + images: BaseManageableRepository, video_name: str, fps: int, backend_service_provider: SuerannotateServiceProvider @@ -549,6 +565,7 @@ def __init__( super().__init__(reporter) self._project = project self._folder = folder + self._images = images self._video_name = video_name self._fps = fps self._client = backend_service_provider @@ -558,8 +575,10 @@ def execute(self): reporter=self.reporter, project=self._project, folder=self._folder, + images=self._images, item_names=[self._video_name], - backend_service_provider=self._client + backend_service_provider=self._client, + show_process=False ).execute() generator = VideoFrameGenerator(response.data[0], fps=self._fps) self.reporter.log_info(f"Getting annotations for {generator.frames_count} frames from {self._video_name}.") diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index eb04047ff..0e583e6c2 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -1928,8 +1928,8 @@ def execute(self): project=self._project, folder=self._folder, attachments=self._attachments[ - i : i + self.CHUNK_SIZE - ], # noqa: E203 + i : i + self.CHUNK_SIZE # noqa: E203 + ], backend_service_provider=self._backend_service, annotation_status=self._annotation_status, upload_state_code=self._upload_state_code, diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 7b6bde783..89c6307b6 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1626,6 +1626,7 @@ def get_annotations(self, project_name: str, folder_name: str, item_names: List[ reporter=self.default_reporter, project=project, folder=folder, + images=self.images, item_names=item_names, backend_service_provider=self.backend_client ) @@ -1639,6 +1640,7 @@ def get_annotations_per_frame(self, project_name: str, folder_name: str, video_n reporter=self.default_reporter, project=project, folder=folder, + images=self.images, video_name=video_name, fps=fps, backend_service_provider=self.backend_client diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 1844ba3c5..b1eecc2b7 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -176,6 +176,7 @@ class SuperannotateBackendService(BaseBackendService): DEFAULT_CHUNK_SIZE = 1000 URL_USERS = "users" + URL_LIST_ALL_IMAGES = "/images/getImagesWithAnnotationPaths" URL_LIST_PROJECTS = "projects" URL_FOLDERS_IMAGES = "images-folders" URL_CREATE_PROJECT = "project" diff --git a/src/superannotate/lib/infrastructure/stream_data_handler.py b/src/superannotate/lib/infrastructure/stream_data_handler.py index 5c4abf548..a28ae0679 100644 --- a/src/superannotate/lib/infrastructure/stream_data_handler.py +++ b/src/superannotate/lib/infrastructure/stream_data_handler.py @@ -25,18 +25,21 @@ async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, dat params: dict = None): response = await session._request(method, url, json=data, params=params) buffer = b"" - async for line in response.content: + async for line in response.content.iter_any(): slices = line.split(self.DELIMITER) if len(slices) == 1: buffer += slices[0] continue elif slices[0]: self._annotations.append(json.loads(buffer + slices[0])) + self._reporter.update_progress() for data in slices[1:-1]: self._annotations.append(json.loads(data)) + self._reporter.update_progress() buffer = slices[-1] if buffer: self._annotations.append(json.loads(buffer)) + self._reporter.update_progress() return self._annotations async def get_data( @@ -54,9 +57,8 @@ async def get_data( if chunk_size: for i in range(0, len(data), chunk_size): - await self.fetch(method, session, url, map_function(data[i:i + chunk_size]), params=params) - self._reporter.update_progress(chunk_size) + data_to_process = data[i:i + chunk_size] + await self.fetch(method, session, url, map_function(data_to_process), params=params) else: await self.fetch(method, session, url, map_function(data), params=params) - self._reporter.update_progress(len(data)) return self._annotations diff --git a/tests/integration/annotations/test_get_annotations.py b/tests/integration/annotations/test_get_annotations.py index 9eb87ad3f..8a73fb0b5 100644 --- a/tests/integration/annotations/test_get_annotations.py +++ b/tests/integration/annotations/test_get_annotations.py @@ -1,12 +1,13 @@ -from pathlib import Path +import json import os +from pathlib import Path from typing import List -import json -import pytest -import src.superannotate as sa +import pytest from pydantic import parse_obj_as from superannotate_schemas.schemas.internal import VectorAnnotation + +import src.superannotate as sa from tests.integration.base import BaseTestCase @@ -40,3 +41,18 @@ def test_get_annotations(self): annotation_data = json.load(annotation_file) self.assertEqual(len(annotation_data["instances"]), len(annotations[0]["instances"])) parse_obj_as(List[VectorAnnotation], annotations) + + @pytest.mark.flaky(reruns=3) + def test_get_annotations_all(self): + sa.init() + sa.upload_images_from_folder_to_project( + self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" + ) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" + ) + _, _, _ = sa.upload_annotations_from_folder_to_project( + self.PROJECT_NAME, self.folder_path + ) + annotations = sa.get_annotations(f"{self.PROJECT_NAME}") + self.assertEqual(len(annotations), 4) diff --git a/tests/profiling/profiling.py b/tests/profiling/profiling.py deleted file mode 100644 index e69de29bb..000000000