From ac8930a1f92ca3242021cc29fbe47ea178f2164f Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 29 Sep 2021 16:08:59 +0400 Subject: [PATCH] Fixed backend error --- pytest.ini | 2 +- src/superannotate/lib/core/usecases/images.py | 115 +++++++++++------- .../lib/infrastructure/controller.py | 105 +--------------- tests/integration/test_annotation_delete.py | 2 + .../test_annotation_upload_pixel.py | 3 + tests/integration/test_fuse_gen.py | 3 +- tests/integration/test_image_quality.py | 2 +- tests/integration/test_interface.py | 9 +- tests/integration/test_limitations.py | 30 +++++ tests/integration/test_ml_funcs.py | 1 + 10 files changed, 121 insertions(+), 151 deletions(-) create mode 100644 tests/integration/test_limitations.py diff --git a/pytest.ini b/pytest.ini index 022233bc3..084beff5b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.0 log_cli=true python_files = test_*.py -addopts = -n32 --dist=loadscope \ No newline at end of file +addopts = -n auto --dist=loadscope \ No newline at end of file diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index bc42b2b48..284c9612b 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -281,20 +281,22 @@ def execute(self): except AppValidationException as e: self._response.errors = e return self._response - - backend_response = self._backend_service.attach_files( - project_id=self._project.uuid, - folder_id=self._folder.uuid, - team_id=self._project.team_id, - files=to_upload, - annotation_status_code=self.annotation_status_code, - upload_state_code=self.upload_state_code, - meta=meta, - ) - if isinstance(backend_response, dict) and "error" in backend_response: - self._response.errors = AppException(backend_response["error"]) + if to_upload: + backend_response = self._backend_service.attach_files( + project_id=self._project.uuid, + folder_id=self._folder.uuid, + team_id=self._project.team_id, + files=to_upload, + annotation_status_code=self.annotation_status_code, + upload_state_code=self.upload_state_code, + meta=meta, + ) + if isinstance(backend_response, dict) and "error" in backend_response: + self._response.errors = AppException(backend_response["error"]) + else: + self._response.data = backend_response, duplications else: - self._response.data = backend_response, duplications + self._response.data = [], duplications return self._response @@ -1089,7 +1091,7 @@ def __init__( self, project: ProjectEntity, folder: FolderEntity, - s3_repo: BaseManageableRepository, + s3_repo, settings: BaseManageableRepository, backend_client: SuerannotateServiceProvider, annotation_status: str, @@ -1113,15 +1115,19 @@ def __init__( self._annotation_status = annotation_status self._auth_data = None - def validate_auth_data(self): - response = self._backend_client.get_s3_upload_auth_token( - team_id=self._project.team_id, - folder_id=self._folder.uuid, - project_id=self._project.uuid, + @property + def s3_repo(self): + self._auth_data = self._backend_client.get_s3_upload_auth_token( + self._project.team_id, self._folder.uuid, self._project.uuid + ) + if "error" in self._auth_data: + raise AppException(self._auth_data.get("error")) + return self._s3_repo( + self._auth_data["accessKeyId"], + self._auth_data["secretAccessKey"], + self._auth_data["sessionToken"], + self._auth_data["bucket"], ) - if "error" in response: - raise AppException(response.get("error")) - self._auth_data = response def validate_project_type(self): if self._project.project_type in [ @@ -1195,7 +1201,7 @@ def execute(self) -> Response: else Path(self._image_path).name, project_settings=self._settings.get_all(), image=image_bytes, - s3_repo=self._s3_repo, + s3_repo=self.s3_repo, upload_path=self.auth_data["filePath"], image_quality_in_editor=self._image_quality_in_editor, ).execute() @@ -1459,7 +1465,7 @@ def execute(self): folder=self._folder, backend_service_provider=self._backend_client, attachments=[ - image.entity for image in uploaded_images[i : i + 100] + image.entity for image in uploaded_images[i : i + 100] # noqa: E203 ], annotation_status=self._annotation_status, upload_state_code=constances.UploadState.BASIC.value, @@ -1581,6 +1587,27 @@ def __init__( self._annotation_status = annotation_status self._image_quality_in_editor = image_quality_in_editor self._settings = settings + self._auth_data = None + + @property + def auth_data(self): + if not self._auth_data: + self._auth_data = self._backend_service.get_s3_upload_auth_token( + self._project.team_id, self._folder.uuid, self._project.uuid + ) + return self._auth_data + + @property + def s3_repo(self): + + if "error" in self.auth_data: + raise AppException(self._auth_data.get("error")) + return self._s3_repo( + self.auth_data["accessKeyId"], + self.auth_data["secretAccessKey"], + self.auth_data["sessionToken"], + self.auth_data["bucket"], + ) def validate_limitations(self): response = self._backend_service.get_limitations( @@ -1625,18 +1652,6 @@ def validate_annotation_status(self): else: self._annotation_status = constances.AnnotationStatus.NOT_STARTED - def get_auth_data(self, project_id: int, team_id: int, folder_id: int): - response = self._backend_service.get_s3_upload_auth_token( - team_id, folder_id, project_id - ) - if "error" in response: - raise AppException(response.get("error")) - return response - - @property - def s3_repo(self): - return self._s3_repo - def upload_image(self, image_url, image_name=None): download_response = DownloadImageFromPublicUrlUseCase( project=self._project, image_url=image_url, image_name=image_name @@ -1662,10 +1677,8 @@ def upload_image(self, image_url, image_name=None): project_settings=self._settings, image_path=image_name, image=content, - s3_repo=self._s3_repo, - upload_path=self.get_auth_data( - self._project.uuid, self._project.team_id, self._folder.uuid - )["filePath"], + s3_repo=self.s3_repo, + upload_path=self.auth_data["filePath"], image_quality_in_editor=self._image_quality_in_editor, ).execute() @@ -1912,7 +1925,7 @@ def execute(self): response = AttachFileUrlsUseCase( project=self._project, folder=self._folder, - attachments=self._attachments[i : i + 500], # noqa: E203 + attachments=self._attachments[i : i + self.CHUNK_SIZE], # noqa: E203 backend_service_provider=self._backend_service, annotation_status=self._annotation_status, upload_state_code=self._upload_state_code, @@ -1938,7 +1951,7 @@ def __init__( to_folder: FolderEntity, backend_service: SuerannotateServiceProvider, images: BaseManageableRepository, - to_upload_s3_repo: BaseManageableRepository, + s3_repo, project_settings: List[ProjectSettingEntity], include_annotations: Optional[bool] = True, copy_annotation_status: Optional[bool] = True, @@ -1951,7 +1964,7 @@ def __init__( self._image_name = image_name self._to_project = to_project self._to_folder = to_folder - self._to_upload_s3_repo = to_upload_s3_repo + self._s3_repo = s3_repo self._project_settings = project_settings self._include_annotations = include_annotations self._copy_annotation_status = copy_annotation_status @@ -2003,6 +2016,20 @@ def validate_limitations(self): if response.data.super_user_limit and response.data.super_user_limit.remaining_image_count < 1: raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE) + @property + def s3_repo(self): + self._auth_data = self._backend_service.get_s3_upload_auth_token( + self._to_project.team_id, self._to_folder.uuid, self._to_project.uuid + ) + if "error" in self._auth_data: + raise AppException(self._auth_data.get("error")) + return self._s3_repo( + self._auth_data["accessKeyId"], + self._auth_data["secretAccessKey"], + self._auth_data["sessionToken"], + self._auth_data["bucket"], + ) + def execute(self) -> Response: if self.is_valid(): image = ( @@ -2039,7 +2066,7 @@ def execute(self) -> Response: image=image_bytes, project_settings=self._project_settings, upload_path=auth_data["filePath"], - s3_repo=self._to_upload_s3_repo, + s3_repo=self.s3_repo, ).execute() if s3_response.errors: raise AppException(s3_response.errors) diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 1b9eb7a54..d558e8dd4 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -298,50 +298,6 @@ def update_project(self, name: str, project_data: dict) -> Response: use_case = usecases.UpdateProjectUseCase(project, project_data, self.projects) return use_case.execute() - def upload_images( - self, - project_name: str, - folder_name: str, - images: List[ImageEntity], - annotation_status: str = None, - ): - project = self._get_project(project_name) - folder = self._get_folder(project, folder_name) - use_case = usecases.AttachFileUrlsUseCase( - project=project, - folder=folder, - backend_service_provider=self._backend_client, - attachments=images, - annotation_status=annotation_status, - upload_state_code=constances.UploadState.BASIC.value, - ) - return use_case.execute() - - def upload_image_to_s3( - self, - project_name: str, - image_path: str, # image path to upload - image_bytes: io.BytesIO, - folder_name: str = None, # project folder path - image_quality_in_editor: str = None, - ): - project = self._get_project(project_name) - folder = self._get_folder(project, folder_name) - s3_repo = self.get_s3_repository(self.team_id, project.uuid, folder.uuid) - auth_data = self.get_auth_data(project.uuid, self.team_id, folder.uuid) - use_case = usecases.UploadImageS3UseCase( - project=project, - project_settings=ProjectSettingsRepository( - self._backend_client, project - ).get_all(), - image_path=image_path, - image=image_bytes, - s3_repo=s3_repo, - upload_path=auth_data["filePath"], - image_quality_in_editor=image_quality_in_editor, - ) - return use_case.execute() - def upload_image_to_project( self, project_name: str, @@ -367,7 +323,7 @@ def upload_image_to_project( settings=ProjectSettingsRepository( service=self._backend_client, project=project ), - s3_repo=self.get_s3_repository(self.team_id, project.uuid, folder.uuid), + s3_repo=self.s3_repo, backend_client=self._backend_client, image_path=image_path, image_bytes=image_bytes, @@ -455,7 +411,7 @@ def upload_images_from_public_urls_to_project( settings=ProjectSettingsRepository( service=self._backend_client, project=project ), - s3_repo=self.get_s3_repository(self.team_id, project.uuid, folder.uuid), + s3_repo=self.s3_repo, image_quality_in_editor=image_quality_in_editor, annotation_status=annotation_status, ) @@ -492,27 +448,6 @@ def clone_project( ) return use_case.execute() - def attach_urls( - self, - project_name: str, - files: List[ImageEntity], - folder_name: str = None, - annotation_status: str = None, - upload_state_code: int = None, - ): - project = self._get_project(project_name) - folder = self._get_folder(project, folder_name) - - use_case = usecases.AttachFileUrlsUseCase( - project=project, - folder=folder, - attachments=files, - backend_service_provider=self._backend_client, - annotation_status=annotation_status, - upload_state_code=upload_state_code, - ) - return use_case.execute() - def interactive_attach_urls( self, project_name: str, @@ -723,9 +658,7 @@ def copy_image( project_settings=ProjectSettingsRepository( self._backend_client, to_project ).get_all(), - to_upload_s3_repo=self.get_s3_repository( - self.team_id, to_project.uuid, to_folder.uuid - ), + s3_repo=self.s3_repo, copy_annotation_status=copy_annotation_status, move=move, ) @@ -1593,13 +1526,6 @@ def list_images( ) return use_case.execute() - @staticmethod - def upload_file_to_s3(to_s3_bucket, path, s3_key: str): - use_case = usecases.UploadFileToS3UseCase( - to_s3_bucket=to_s3_bucket, path=path, s3_key=s3_key - ) - return use_case.execute() - def search_models( self, name: str, @@ -1643,28 +1569,3 @@ def delete_annotations( image_names=image_names, ) return use_case.execute() - - def get_duplicated_images( - self, project_name: str, folder_name: str, images: List[str] - ): - project = self._get_project(project_name) - folder = self._get_folder(project, folder_name) - use_case = usecases.GetBulkImages( - service=self._backend_client, - project_id=project.uuid, - team_id=project.team_id, - folder_id=folder.uuid, - images=images, - ) - return use_case.execute().data - - def get_project_limitations(self, project_name: str, folder_name: str): - project = self._get_project(project_name) - folder = self._get_folder(project, folder_name) - - return usecases.GetUserLimitsUseCase( - service=self._backend_client, - project_id=project.uuid, - team_id=project.team_id, - folder_id=folder.uuid, - ).execute() diff --git a/tests/integration/test_annotation_delete.py b/tests/integration/test_annotation_delete.py index 42fdfa0a0..1e45a0966 100644 --- a/tests/integration/test_annotation_delete.py +++ b/tests/integration/test_annotation_delete.py @@ -1,5 +1,6 @@ import os from os.path import dirname +import pytest import src.superannotate as sa from tests.integration.base import BaseTestCase @@ -70,6 +71,7 @@ def test_delete_annotations_by_not_existing_name(self): ) self.assertRaises(Exception, sa.delete_annotations, self.PROJECT_NAME, [self.EXAMPLE_IMAGE_2]) + @pytest.mark.flaky(reruns=2) def test_delete_annotations_wrong_path(self): sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) sa.upload_images_from_folder_to_project( diff --git a/tests/integration/test_annotation_upload_pixel.py b/tests/integration/test_annotation_upload_pixel.py index 4fcdbf4ae..fda574756 100644 --- a/tests/integration/test_annotation_upload_pixel.py +++ b/tests/integration/test_annotation_upload_pixel.py @@ -1,9 +1,11 @@ import os from os.path import dirname +import pytest import src.superannotate as sa from tests.integration.base import BaseTestCase + class TestRecursiveFolderPixel(BaseTestCase): PROJECT_NAME = "test_recursive_pixel" PROJECT_DESCRIPTION = "Desc" @@ -15,6 +17,7 @@ class TestRecursiveFolderPixel(BaseTestCase): def folder_path(self): return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) + @pytest.mark.flaky(reruns=2) def test_recursive_annotation_upload_pixel(self): sa.upload_images_from_folder_to_project( self.PROJECT_NAME, self.folder_path, recursive_subfolders=False diff --git a/tests/integration/test_fuse_gen.py b/tests/integration/test_fuse_gen.py index 177300efc..1abb39011 100644 --- a/tests/integration/test_fuse_gen.py +++ b/tests/integration/test_fuse_gen.py @@ -3,7 +3,7 @@ import tempfile from os.path import dirname from unittest import TestCase - +import pytest import numpy as np import src.superannotate as sa from PIL import Image @@ -193,6 +193,7 @@ def test_fuse_image_create_pixel(self): self.assertEqual(im1_array.dtype, im2_array.dtype) self.assertTrue(np.array_equal(im1_array, im2_array)) + @pytest.mark.flaky(reruns=3) def test_fuse_image_create_pixel_with_no_classes(self): with tempfile.TemporaryDirectory() as temp_dir: temp_dir = pathlib.Path(temp_dir) diff --git a/tests/integration/test_image_quality.py b/tests/integration/test_image_quality.py index fc6d057cd..6f9e3f390 100644 --- a/tests/integration/test_image_quality.py +++ b/tests/integration/test_image_quality.py @@ -64,7 +64,7 @@ def test_image_quality_setting2(self): -class TestImageQuality(BaseTestCase): +class TestPixelImageQuality(BaseTestCase): PROJECT_NAME = "pixel image q" PROJECT_DESCRIPTION = "Desc" PROJECT_TYPE = "Pixel" diff --git a/tests/integration/test_interface.py b/tests/integration/test_interface.py index 880937d8e..b689d4932 100644 --- a/tests/integration/test_interface.py +++ b/tests/integration/test_interface.py @@ -165,9 +165,14 @@ def test_upload_images_to_project_image_quality_in_editor(self): image_quality_in_editor='random_string' ) + @pytest.mark.flaky(reruns=2) def test_image_upload_with_set_name_on_platform(self): - sa.upload_image_to_project(self.PROJECT_NAME, self.IMAGE_PATH_IN_S3 , self.NEW_IMAGE_NAME,from_s3_bucket=self.TEST_S3_BUCKET_NAME) - self.assertIn(sa.search_images(self.PROJECT_NAME)[0],self.NEW_IMAGE_NAME) + sa.upload_image_to_project( + self.PROJECT_NAME, + self.IMAGE_PATH_IN_S3, + self.NEW_IMAGE_NAME, from_s3_bucket=self.TEST_S3_BUCKET_NAME + ) + self.assertIn(sa.search_images(self.PROJECT_NAME)[0], self.NEW_IMAGE_NAME) def test_download_fuse_without_classes(self): sa.upload_image_to_project(self.PROJECT_NAME, f"{self.folder_path}/{self.EXAMPLE_IMAGE_1}") diff --git a/tests/integration/test_limitations.py b/tests/integration/test_limitations.py new file mode 100644 index 000000000..ff4e6e495 --- /dev/null +++ b/tests/integration/test_limitations.py @@ -0,0 +1,30 @@ +import filecmp +import os +import tempfile +from os.path import dirname +import pytest + +import src.superannotate as sa +from tests.integration.base import BaseTestCase +from src.superannotate import AppException + + +class TestImageQuality(BaseTestCase): + PROJECT_NAME = "Limitation Test" + PROJECT_DESCRIPTION = "Desc" + PROJECT_TYPE = "Vector" + TEST_FOLDER_PTH = "data_set" + TEST_FOLDER_PATH = "data_set/sample_project_vector" + EXAMPLE_IMAGE_1 = "example_image_1.jpg" + + @property + def folder_path(self): + return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) + + def test_image_quality_setting1(self): + uploaded, _, __ = sa.upload_images_from_folder_to_project( + project=self._project["name"], folder_path=self.folder_path + ) + uploaded, _, __ = sa.upload_images_from_folder_to_project( + project=self._project["name"], folder_path=os.path.join(dirname(dirname(__file__)), "data_set") + ) diff --git a/tests/integration/test_ml_funcs.py b/tests/integration/test_ml_funcs.py index 634e74d57..ddb5b077d 100644 --- a/tests/integration/test_ml_funcs.py +++ b/tests/integration/test_ml_funcs.py @@ -26,6 +26,7 @@ def test_run_prediction_with_non_exist_images(self): self.PROJECT_NAME, ["NonExistantImage.jpg"], self.MODEL_NAME ) + @pytest.mark.flaky(reruns=2) def test_run_prediction_for_all_images(self): sa.upload_images_from_folder_to_project( project=self.PROJECT_NAME, folder_path=self.folder_path