From 8293d43fc6544f878aeba03dc9e5184c4de99ee7 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Mon, 15 Aug 2022 16:22:20 +0400 Subject: [PATCH] upload changes --- .../lib/core/entities/classes.py | 2 + .../lib/core/usecases/annotations.py | 40 ++++++++++------- src/superannotate/lib/core/usecases/images.py | 1 + .../lib/core/usecases/projects.py | 44 +++++++++---------- .../lib/infrastructure/services.py | 21 ++++----- .../annotations/test_preannotation_upload.py | 9 ++-- .../test_vector_annotation_validation.py | 21 +++++++++ tests/integration/test_get_exports.py | 24 ---------- 8 files changed, 85 insertions(+), 77 deletions(-) diff --git a/src/superannotate/lib/core/entities/classes.py b/src/superannotate/lib/core/entities/classes.py index cb4050ac0..595f175d1 100644 --- a/src/superannotate/lib/core/entities/classes.py +++ b/src/superannotate/lib/core/entities/classes.py @@ -77,6 +77,8 @@ class AttributeGroup(BaseModel): class Config: use_enum_values = True + exclude_unset = True + exclude_none = True def __hash__(self): return hash(f"{self.id}{self.class_id}{self.name}") diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 177a8df33..e8764a34f 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -70,6 +70,7 @@ class UploadAnnotationsUseCase(BaseReportableUseCase): STATUS_CHANGE_CHUNK_SIZE = 100 AUTH_DATA_CHUNK_SIZE = 500 THREADS_COUNT = 4 + URI_THRESHOLD = 4 * 1024 - 120 @dataclass class AnnotationToUpload: @@ -317,6 +318,19 @@ async def _upload_small_annotations(self, chunk) -> Report: async def upload_small_annotations(self): chunk = [] + + async def upload(_chunk): + try: + report = await self._upload_small_annotations(chunk) + self._report.failed_annotations.extend(report.failed_annotations) + self._report.missing_classes.extend(report.missing_classes) + self._report.missing_attr_groups.extend(report.missing_attr_groups) + self._report.missing_attrs.extend(report.missing_attrs) + except Exception: + import traceback + traceback.print_exc() + self._report.failed_annotations.extend([i.name for i in chunk]) + while True: item = await self._small_files_queue.get() self._small_files_queue.task_done() @@ -324,19 +338,15 @@ async def upload_small_annotations(self): self._small_files_queue.put_nowait(None) break chunk.append(item) - if sys.getsizeof(chunk) >= self.CHUNK_SIZE_MB: - report = await self._upload_small_annotations(chunk) - self._report.failed_annotations.extend(report.failed_annotations) - self._report.missing_classes.extend(report.missing_classes) - self._report.missing_attr_groups.extend(report.missing_attr_groups) - self._report.missing_attrs.extend(report.missing_attrs) + if ( + sys.getsizeof(chunk) >= self.CHUNK_SIZE_MB or + sum([len(i.name) for i in chunk]) >= self.URI_THRESHOLD - len(chunk) * 14 + ): + await upload(chunk) chunk = [] + if chunk: - report = await self._upload_small_annotations(chunk) - self._report.failed_annotations.extend(report.failed_annotations) - self._report.missing_classes.extend(report.missing_classes) - self._report.missing_attr_groups.extend(report.missing_attr_groups) - self._report.missing_attrs.extend(report.missing_attrs) + await upload(chunk) async def _upload_big_annotation(self, item) -> Tuple[str, bool]: try: @@ -583,7 +593,7 @@ def set_defaults(team_id, annotation_data: dict, project_type: int): default_data = {} annotation_data["metadata"]["lastAction"] = { "email": team_id, - "timestamp": int(time.time()) + "timestamp": int(round(time.time() * 1000)) } instances = annotation_data.get("instances", []) if project_type in constants.ProjectType.images: @@ -594,14 +604,14 @@ def set_defaults(team_id, annotation_data: dict, project_type: int): instance["meta"] = { **default_data, **instance["meta"], - "creationType": "Preannotation", - } # noqa + "creationType": "Preannotation", # noqa + } else: for idx, instance in enumerate(instances): instances[idx] = { **default_data, **instance, - "creationType": "Preannotation", + "creationType": "Preannotation", # noqa } return annotation_data diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index 625a49293..583e8d1be 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -1795,6 +1795,7 @@ def execute(self): headers=annotation_json_creds["headers"], ) if not response.ok: + # TODO remove logger.warning("Couldn't load annotations.") self._response.data = (None, None) return self._response diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 347a0a566..84f67012c 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -574,31 +574,42 @@ def execute(self): ) annotation_classes_entity_mapping = defaultdict(AnnotationClassEntity) annotation_classes_created = False - if self._include_annotation_classes: + if self._include_settings: self.reporter.log_info( - f"Cloning annotation classes from {self._project.name} to {self._project_to_create.name}." + f"Cloning settings from {self._project.name} to {self._project_to_create.name}." ) try: - self._copy_annotation_classes( - annotation_classes_entity_mapping, project - ) - annotation_classes_created = True + self._copy_settings(project) except (AppException, RequestException) as e: self.reporter.log_warning( - f"Failed to clone annotation classes from {self._project.name} to {self._project_to_create.name}." + f"Failed to clone settings from {self._project.name} to {self._project_to_create.name}." ) self.reporter.log_debug(str(e), exc_info=True) + if self._include_contributors: + self.reporter.log_info( + f"Cloning contributors from {self._project.name} to {self._project_to_create.name}." + ) + try: + self._copy_include_contributors(project) + except (AppException, RequestException) as e: + self.reporter.log_warning( + f"Failed to clone contributors from {self._project.name} to {self._project_to_create.name}." + ) + self.reporter.log_debug(str(e), exc_info=True) - if self._include_settings: + if self._include_annotation_classes: self.reporter.log_info( - f"Cloning settings from {self._project.name} to {self._project_to_create.name}." + f"Cloning annotation classes from {self._project.name} to {self._project_to_create.name}." ) try: - self._copy_settings(project) + self._copy_annotation_classes( + annotation_classes_entity_mapping, project + ) + annotation_classes_created = True except (AppException, RequestException) as e: self.reporter.log_warning( - f"Failed to clone settings from {self._project.name} to {self._project_to_create.name}." + f"Failed to clone annotation classes from {self._project.name} to {self._project_to_create.name}." ) self.reporter.log_debug(str(e), exc_info=True) @@ -626,17 +637,6 @@ def execute(self): f"Failed to workflow from {self._project.name} to {self._project_to_create.name}." ) self.reporter.log_debug(str(e), exc_info=True) - if self._include_contributors: - self.reporter.log_info( - f"Cloning contributors from {self._project.name} to {self._project_to_create.name}." - ) - try: - self._copy_include_contributors(project) - except (AppException, RequestException) as e: - self.reporter.log_warning( - f"Failed to clone contributors from {self._project.name} to {self._project_to_create.name}." - ) - self.reporter.log_debug(str(e), exc_info=True) self._response.data = self._projects.get_one( uuid=project.id, team_id=project.team_id diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index 05e3f680f..3a7e7ea52 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -18,6 +18,7 @@ import aiohttp import requests.packages.urllib3 from pydantic import BaseModel +from pydantic import parse_obj_as from requests.exceptions import HTTPError import lib.core as constance @@ -1374,15 +1375,16 @@ async def upload_annotations( items_name_file_map: Dict[str, io.StringIO], ) -> UploadAnnotationsResponse: url = urljoin( - "self.assets_provider_url", - f"{self.URL_UPLOAD_ANNOTATIONS}?{'&'.join(f'image_names[]={item_name}' for item_name in items_name_file_map.keys())}", + self.assets_provider_url, + # "https://0ef1-178-160-196-42.ngrok.io/api/v1.01/", + (f"{self.URL_UPLOAD_ANNOTATIONS}?{'&'.join(f'image_names[]={item_name}' for item_name in items_name_file_map.keys())}"), ) headers = copy.copy(self.default_headers) del headers["Content-Type"] async with aiohttp.ClientSession( headers=headers, - connector=aiohttp.TCPConnector(ssl=self._verify_ssl), + connector=aiohttp.TCPConnector(ssl=self._verify_ssl) ) as session: data = aiohttp.FormData() @@ -1400,13 +1402,12 @@ async def upload_annotations( }, data=data ) - from pydantic import parse_obj_as - data_json = await _response.json() - response = ServiceResponse() - response.status = _response.status - response._content = await _response.text() - response.data = parse_obj_as(UploadAnnotationsResponse, data_json) - return response + data_json = await _response.json() + response = ServiceResponse() + response.status = _response.status + response._content = await _response.text() + response.data = parse_obj_as(UploadAnnotationsResponse, data_json) + return response async def upload_big_annotation( self, diff --git a/tests/integration/annotations/test_preannotation_upload.py b/tests/integration/annotations/test_preannotation_upload.py index d16c35b53..4c9ebc17b 100644 --- a/tests/integration/annotations/test_preannotation_upload.py +++ b/tests/integration/annotations/test_preannotation_upload.py @@ -18,22 +18,19 @@ def folder_path(self): return os.path.join(Path(__file__).parent.parent.parent, self.TEST_FOLDER_PATH) def test_pre_annotation_folder_upload_download(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) + self._attach_items() sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" ) - _, _, _ = sa.upload_annotations_from_folder_to_project( + uploaded, _, _ = sa.upload_annotations_from_folder_to_project( self.PROJECT_NAME, self.folder_path ) + assert len(uploaded) == 4 count_in = len(list(Path(self.folder_path).glob("*.json"))) images = sa.search_items(self.PROJECT_NAME) with tempfile.TemporaryDirectory() as tmp_dir: for image in images: image_name = image["name"] sa.download_image_annotations(self.PROJECT_NAME, image_name, tmp_dir) - count_out = len(list(Path(tmp_dir).glob("*.json"))) - self.assertEqual(count_in, count_out) diff --git a/tests/integration/annotations/validations/test_vector_annotation_validation.py b/tests/integration/annotations/validations/test_vector_annotation_validation.py index bce521462..afc6e0929 100644 --- a/tests/integration/annotations/validations/test_vector_annotation_validation.py +++ b/tests/integration/annotations/validations/test_vector_annotation_validation.py @@ -35,3 +35,24 @@ def test_validate_instances(self, mock_print): "instances[0] invalid type\n" "instances[1] 'points' is a required property" ) + + @patch('builtins.print') + def test_validate_create_dby(self, mock_print): + is_valid = sa.validate_annotations( + "vector", + { + "metadata": {"name": "12"}, + "instances": [ + { + "type": "bbox", + "created_by": {} + }, + {"type": "bbox"} + ] + } + ) + assert not is_valid + mock_print.assert_any_call( + "instances[0] invalid type\n" + "instances[1] 'points' is a required property" + ) diff --git a/tests/integration/test_get_exports.py b/tests/integration/test_get_exports.py index 66f9f7c4b..dd7d7c0ab 100644 --- a/tests/integration/test_get_exports.py +++ b/tests/integration/test_get_exports.py @@ -42,27 +42,3 @@ def test_get_exports(self): exports_new = sa.get_exports(self.PROJECT_NAME) assert len(exports_new) == len(exports_old) + 1 - - -class TestPixelExportConvert(BaseTestCase): - PROJECT_NAME = "Pixel_Export" - PROJECT_DESCRIPTION = "Desc" - PROJECT_TYPE = "Pixel" - TEST_FOLDER_PTH = "data_set" - TEST_FOLDER_PATH = "data_set/sample_project_pixel" - - @property - def folder_path(self): - return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) - - def test_convert_pixel_exported_data(self): - sa.upload_images_from_folder_to_project(self.PROJECT_NAME, self.folder_path) - sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path) - export = sa.prepare_export(self.PROJECT_NAME) - with tempfile.TemporaryDirectory() as tmp_dir: - sa.download_export(self.PROJECT_NAME, export["name"], tmp_dir) - with tempfile.TemporaryDirectory() as converted_data_tmp_dir: - export_annotation( - tmp_dir, converted_data_tmp_dir, "COCO", "export", "Pixel", "panoptic_segmentation" - ) - self.assertEqual(1, len(list(glob.glob(converted_data_tmp_dir + "/*.json"))))