From 439695d059f4a228e77d31cc61dff23ee5ef4ece Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Fri, 21 Jan 2022 11:33:54 +0400 Subject: [PATCH 1/4] Add log --- .../lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/__init__.py | 8 ++++++++ .../lib/core/entities/__init__.py | 4 +++- src/superannotate/lib/core/reporter.py | 18 ++++++++---------- .../lib/core/usecases/annotations.py | 19 +++++++++++++++---- src/superannotate/lib/core/usecases/images.py | 6 ++++-- .../lib/infrastructure/controller.py | 5 +++-- tests/integration/test_interface.py | 17 +++++++++++++++++ 8 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 6e4eb26a3..5f70d80b0 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2112,7 +2112,7 @@ def upload_image_annotations( mask=mask, verbose=verbose, ) - if response.errors: + if response.errors and not response.errors == constances.INVALID_JSON_MESSSAGE: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 339cbc513..4c26eadad 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -117,6 +117,14 @@ " Run 'pip install --upgrade superannotate' to" " upgrade from your version {} to {}" ) + +USE_VALIDATE_MESSAGE = ( + "Use the validate_annotations function to discover the possible reason(s) for " + "which an annotation is invalid." +) + +INVALID_JSON_MESSSAGE = "Invalid json" + __alL__ = ( ProjectType, UserRole, diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index c36f9135b..596e28932 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -15,7 +15,9 @@ from superannotate_schemas.schemas.internal.pixel import PixelAnnotation from superannotate_schemas.schemas.internal.vector import VectorAnnotation from superannotate_schemas.schemas.internal.video import VideoAnnotation -from superannotate_schemas.schemas.internal.video import VideoAnnotation as VideoExportAnnotation +from superannotate_schemas.schemas.internal.video import ( + VideoAnnotation as VideoExportAnnotation, +) __all__ = [ "BaseEntity", diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 052ef3a1c..90dc6c758 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -46,16 +46,12 @@ def start_progress( @staticmethod def get_progress_bar( - iterations: Union[int, range], description: str = "Processing", disable=False + iterations: Union[int, range], description: str = "Processing", disable=False ): if isinstance(iterations, range): - return tqdm.tqdm( - iterations, desc=description, disable=disable - ) + return tqdm.tqdm(iterations, desc=description, disable=disable) else: - return tqdm.tqdm( - total=iterations, desc=description, disable=disable - ) + return tqdm.tqdm(total=iterations, desc=description, disable=disable) def finish_progress(self): self.progress_bar.close() @@ -80,7 +76,7 @@ def messages(self): yield f"{key} [{', '.join(values)}]" -class Progress(object): +class Progress: def __init__(self, iterations: Union[int, range], description: str = "Processing"): self._iterations = iterations self._description = description @@ -95,5 +91,7 @@ def __exit__(self, type, value, traceback): def update(self, value=1): if not self._progress_bar: - self._progress_bar = Reporter.get_progress_bar(self._iterations, self._description) - self._progress_bar.update(value) \ No newline at end of file + self._progress_bar = Reporter.get_progress_bar( + self._iterations, self._description + ) + self._progress_bar.update(value) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 5f22e8d5b..3fecb41d0 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -25,8 +25,8 @@ from lib.core.usecases.base import BaseReportableUseCae from lib.core.usecases.images import GetBulkImages from lib.core.usecases.images import ValidateAnnotationUseCase -from superannotate_schemas.validators import AnnotationValidators from superannotate.logger import get_default_logger +from superannotate_schemas.validators import AnnotationValidators logger = get_default_logger() @@ -137,6 +137,12 @@ def annotations_to_upload(self): self._annotations_to_upload = annotations_to_upload return self._annotations_to_upload + @property + def missing_annotations(self): + if not self._missing_annotations: + self._missing_annotations = [] + return self._missing_annotations + def get_annotation_upload_data( self, image_ids: List[int] ) -> UploadAnnotationAuthData: @@ -223,7 +229,9 @@ def _log_report(self): logger.warning(template.format("', '".join(values))) if self.reporter.custom_messages.get("invalid_jsons"): logger.warning( - f"Couldn't validate {len(self.reporter.custom_messages['invalid_jsons'])}/{len(self._annotations_to_upload + self._missing_annotations)} annotations from {self._folder_path}." + f"Couldn't validate {len(self.reporter.custom_messages['invalid_jsons'])}/" + f"{len(self.annotations_to_upload + self.missing_annotations)} annotations from {self._folder_path}. " + f"{constances.USE_VALIDATE_MESSAGE}" ) def execute(self): @@ -281,7 +289,7 @@ def execute(self): self._response.data = ( uploaded_annotations, failed_annotations, - [annotation.path for annotation in self._missing_annotations], + [annotation.path for annotation in self.missing_annotations], ) return self._response @@ -483,6 +491,9 @@ def execute(self): self._project.name, ) else: - self._response.errors = "Invalid json" + self._response.errors = constances.INVALID_JSON_MESSSAGE self.reporter.store_message("invalid_jsons", self._annotation_path) + self.reporter.log_warning( + f"Couldn't validate annotations. {constances.USE_VALIDATE_MESSAGE}" + ) return self._response diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 8994a4713..1477676f7 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -35,11 +35,11 @@ from lib.core.exceptions import ImageProcessingException from lib.core.plugin import ImagePlugin from lib.core.plugin import VideoPlugin +from lib.core.reporter import Progress from lib.core.reporter import Reporter from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response -from lib.core.reporter import Progress from lib.core.serviceproviders import SuerannotateServiceProvider from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.usecases.base import BaseReportableUseCae @@ -3230,7 +3230,9 @@ def execute(self) -> Response: if set(duplicate_images) == set(frame_names): continue uploaded_paths = [] - with Progress(total_frames_count, f"Uploading {Path(path).name}") as progress: + with Progress( + total_frames_count, f"Uploading {Path(path).name}" + ) as progress: for _ in frames_generator: use_case = UploadImagesFromFolderToProject( project=self._project, diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f433f7a5e..f4dd274f8 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -32,8 +32,9 @@ from lib.infrastructure.repositories import TeamRepository from lib.infrastructure.repositories import WorkflowRepository from lib.infrastructure.services import SuperannotateBackendService -from superannotate_schemas.validators import AnnotationValidators from superannotate.logger import get_default_logger +from superannotate_schemas.validators import AnnotationValidators + class SingleInstanceMetaClass(type): _instances = {} @@ -1312,7 +1313,7 @@ def upload_image_annotations( backend_service_provider=self._backend_client, mask=mask, verbose=verbose, - reporter=self.default_reporter, + reporter=Reporter(log_info=True, log_warning=True), validators=self.annotation_validators, ) return use_case.execute() diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 54c793565..5e73fb7b3 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -156,6 +156,23 @@ def test_download_fuse_without_classes(self): ) self.assertIsNotNone(result) + def test_validate_log_for_single_uplaod(self): + with self.assertLogs() as logs: + sa.upload_image_to_project(self.PROJECT_NAME, f"{self.folder_path}/{self.EXAMPLE_IMAGE_1}") + sa.upload_image_annotations( + self.PROJECT_NAME, self.EXAMPLE_IMAGE_1, { + "metadatas": { + "name": "example_image_1.jpg", + "width": 1024, + "height": 683, + "status": "Completed", + }, + "instance": [] + } + ) + self.assertEqual(len(logs[1][0]), 150) + + class TestPixelInterface(BaseTestCase): PROJECT_NAME = "Interface Pixel test" From 75fa3b95f7cd744777ef7ed6fd065171bdaf150a Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Fri, 21 Jan 2022 11:59:12 +0400 Subject: [PATCH 2/4] Fix video upload variable --- src/superannotate/lib/core/usecases/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 1477676f7..6a98d4227 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -3208,7 +3208,7 @@ def execute(self) -> Response: image_quality_in_editor=self._image_quality_in_editor, ) if not frames_generator_use_case.is_valid(): - self._response.errors = use_case.response.errors + self._response.errors = frames_generator_use_case.response.errors return self._response frames_generator = frames_generator_use_case.execute() From 4e4d7cc5e0f341ff29820074f33e1a80dc6ed2aa Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Fri, 21 Jan 2022 12:38:50 +0400 Subject: [PATCH 3/4] Change uplaod state on clone --- src/superannotate/lib/app/analytics/aggregators.py | 2 +- src/superannotate/lib/core/usecases/projects.py | 7 +++++++ tests/integration/projects/test_clone_project.py | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index e29f88c6e..b33368095 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -1,5 +1,6 @@ import copy import json +from dataclasses import dataclass from pathlib import Path from typing import List from typing import Optional @@ -7,7 +8,6 @@ import lib.core as constances import pandas as pd -from dataclasses import dataclass from lib.app.exceptions import AppException from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX from lib.core import PIXEL_ANNOTATION_POSTFIX diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index f170bd447..86159831b 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -515,6 +515,13 @@ def _copy_workflow( def execute(self): if self.is_valid(): + if self._project_to_create.project_type in ( + constances.ProjectType.PIXEL.value, + constances.ProjectType.VECTOR.value, + ): + self._project_to_create.upload_state = ( + constances.UploadState.INITIAL.value + ) project = self._projects.insert(self._project_to_create) self.reporter.log_info( f"Created project {self._project_to_create.name} with type" diff --git a/tests/integration/projects/test_clone_project.py b/tests/integration/projects/test_clone_project.py index a095fcb29..ea64ba7a7 100644 --- a/tests/integration/projects/test_clone_project.py +++ b/tests/integration/projects/test_clone_project.py @@ -4,6 +4,7 @@ import pytest import src.superannotate as sa from tests import DATA_SET_PATH +from src.superannotate import constances class TestCloneProject(TestCase): @@ -78,7 +79,7 @@ def test_create_like_project(self): self.PROJECT_NAME_2, self.PROJECT_NAME_1, copy_contributors=True ) source_project = sa.get_project_metadata(self.PROJECT_NAME_1) - self.assertEqual(new_project['upload_state'], source_project['upload_state']) + self.assertEqual(new_project['upload_state'], constances.UploadState.INITIAL.name) new_settings = sa.get_project_settings(self.PROJECT_NAME_2) image_quality = None From 082361a547c72ab8c827713670b1c3a03079824c Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 24 Jan 2022 10:33:27 +0400 Subject: [PATCH 4/4] tod --- src/superannotate/lib/app/interface/sdk_interface.py | 2 +- src/superannotate/lib/core/__init__.py | 2 +- src/superannotate/lib/core/reporter.py | 6 ++++++ src/superannotate/lib/core/usecases/annotations.py | 11 ++++++----- src/superannotate/lib/core/usecases/images.py | 1 - src/superannotate/lib/infrastructure/controller.py | 6 ++---- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 87df3ac5c..867fe057f 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2112,7 +2112,7 @@ def upload_image_annotations( mask=mask, verbose=verbose, ) - if response.errors and not response.errors == constances.INVALID_JSON_MESSSAGE: + if response.errors and not response.errors == constances.INVALID_JSON_MESSAGE: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 4c26eadad..c6bd226e1 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -123,7 +123,7 @@ "which an annotation is invalid." ) -INVALID_JSON_MESSSAGE = "Invalid json" +INVALID_JSON_MESSAGE = "Invalid json" __alL__ = ( ProjectType, diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index 90dc6c758..1642ceb96 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -24,6 +24,12 @@ def __init__( self.custom_messages = defaultdict(set) self.progress_bar = None + def disable_warnings(self): + self._log_warning = False + + def enable_warnings(self): + self._log_warning = True + def log_info(self, value: str): if self._log_info: self.logger.info(value) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 3fecb41d0..e62bdf08b 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -168,6 +168,7 @@ def _upload_annotation( bucket, ): try: + self.reporter.disable_warnings() response = UploadAnnotationUseCase( project=self._project, folder=self._folder, @@ -196,6 +197,8 @@ def _upload_annotation( except Exception as e: logger.debug(str(e), exc_info=True) return path, False + finally: + self.reporter.enable_warnings() def get_bucket_to_upload(self, ids: List[int]): upload_data = self.get_annotation_upload_data(ids) @@ -485,13 +488,11 @@ def execute(self): ) self._images.update(self._image) if self._verbose: - logger.info( - "Uploading annotations for image %s in project %s.", - str(self._image.name), - self._project.name, + self.reporter.log_info( + f"Uploading annotations for image {str(self._image.name)} in project {self._project.name}." ) else: - self._response.errors = constances.INVALID_JSON_MESSSAGE + self._response.errors = constances.INVALID_JSON_MESSAGE self.reporter.store_message("invalid_jsons", self._annotation_path) self.reporter.log_warning( f"Couldn't validate annotations. {constances.USE_VALIDATE_MESSAGE}" diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index e32161383..14b8af8d6 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -40,7 +40,6 @@ from lib.core.repositories import BaseManageableRepository from lib.core.repositories import BaseReadOnlyRepository from lib.core.response import Response -from lib.core.reporter import Progress from lib.core.serviceproviders import SuerannotateServiceProvider from lib.core.usecases.base import BaseInteractiveUseCase from lib.core.usecases.base import BaseReportableUseCae diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index f4dd274f8..f72451825 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -216,9 +216,7 @@ def team_id(self) -> int: @property def default_reporter(self): - if not self._reporter: - self._reporter = Reporter() - return self._reporter + return Reporter() @timed_lru_cache(seconds=3600) def get_auth_data(self, project_id: int, team_id: int, folder_id: int): @@ -1313,7 +1311,7 @@ def upload_image_annotations( backend_service_provider=self._backend_client, mask=mask, verbose=verbose, - reporter=Reporter(log_info=True, log_warning=True), + reporter=self.default_reporter, validators=self.annotation_validators, ) return use_case.execute()