diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 2dfe694b6..6b857dd54 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -3806,6 +3806,7 @@ def download_annotations( items: Optional[List[NotEmptyStr]] = None, recursive: bool = False, callback: Callable = None, + data_spec: Literal["default", "multimodal"] = "default", ): """Downloads annotation JSON files of the selected items to the local directory. @@ -3831,6 +3832,31 @@ def download_annotations( The function receives each annotation as an argument and the returned value will be applied to the download. :type callback: callable + :param data_spec: Specifies the format for processing and transforming annotations before upload. + + Options are: + - default: Retains the annotations in their original format. + - multimodal: Converts annotations for multimodal projects, optimizing for + compact and multimodal-specific data representation. + + :type data_spec: str, optional + + Example Usage of Multimodal Projects:: + + from superannotate import SAClient + + + sa = SAClient() + + # Call the get_annotations function + response = sa.download_annotations( + project="project1/folder1", + path="path/to/download", + items=["item_1", "item_2"], + data_spec='multimodal' + ) + + :return: local path of the downloaded annotations folder. :rtype: str """ @@ -3843,6 +3869,7 @@ def download_annotations( recursive=recursive, item_names=items, callback=callback, + transform_version="llmJsonV2" if data_spec == "multimodal" else None, ) if response.errors: raise AppException(response.errors) diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index d8fbab28e..008651cb0 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -501,6 +501,7 @@ async def download_big_annotation( download_path: str, item: entities.BaseItemEntity, callback: Callable = None, + transform_version: str = None, ): raise NotImplementedError @@ -513,6 +514,7 @@ async def download_small_annotations( download_path: str, item_ids: List[int], callback: Callable = None, + transform_version: str = None, ): raise NotImplementedError diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index 5409eba6e..281f87fae 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -1634,6 +1634,7 @@ def __init__( item_names: List[str], service_provider: BaseServiceProvider, callback: Callable = None, + transform_version=None, ): super().__init__(reporter) self._config = config @@ -1645,6 +1646,7 @@ def __init__( self._service_provider = service_provider self._callback = callback self._big_file_queue = None + self._transform_version = transform_version def validate_items(self): if self._item_names: @@ -1724,6 +1726,7 @@ async def download_small_annotations( reporter=self.reporter, download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}", callback=self._callback, + transform_version=self._transform_version, ) async def run_workers( diff --git a/src/superannotate/lib/infrastructure/annotation_adapter.py b/src/superannotate/lib/infrastructure/annotation_adapter.py index c333231cf..13436510a 100644 --- a/src/superannotate/lib/infrastructure/annotation_adapter.py +++ b/src/superannotate/lib/infrastructure/annotation_adapter.py @@ -44,7 +44,9 @@ def get_component_value(self, component_id: str): return None def set_component_value(self, component_id: str, value: Any): - self.annotation.setdefault("data", {}).setdefault(component_id, {})["value"] = value + self.annotation.setdefault("data", {}).setdefault(component_id, {})[ + "value" + ] = value return self diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 0e9f703b1..53e21a620 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -932,6 +932,7 @@ def download( recursive: bool, item_names: Optional[List[str]], callback: Optional[Callable], + transform_version: str, ): use_case = usecases.DownloadAnnotations( config=self._config, @@ -943,6 +944,7 @@ def download( item_names=item_names, service_provider=self.service_provider, callback=callback, + transform_version=transform_version, ) return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py index 27a27dd39..a1f5d9e46 100644 --- a/src/superannotate/lib/infrastructure/services/annotation.py +++ b/src/superannotate/lib/infrastructure/services/annotation.py @@ -67,7 +67,9 @@ def get_schema(self, project_type: int, version: str): }, ) - async def _sync_large_annotation(self, team_id, project_id, item_id): + async def _sync_large_annotation( + self, team_id, project_id, item_id, transform_version: str = None + ): sync_params = { "team_id": team_id, "project_id": project_id, @@ -77,6 +79,8 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): "current_source": "main", "desired_source": "secondary", } + if transform_version: + sync_params["transform_version"] = transform_version sync_url = urljoin( self.get_assets_provider_url(), self.URL_START_FILE_SYNC.format(item_id=item_id), @@ -120,11 +124,12 @@ async def get_big_annotation( "annotation_type": "MAIN", "version": "V1.00", } - if transform_version: - query_params["desired_transform_version"] = transform_version await self._sync_large_annotation( - team_id=project.team_id, project_id=project.id, item_id=item.id + team_id=project.team_id, + project_id=project.id, + item_id=item.id, + transform_version=transform_version, ) async with AIOHttpSession( @@ -202,6 +207,7 @@ async def download_big_annotation( download_path: str, item: entities.BaseItemEntity, callback: Callable = None, + transform_version: str = None, ): item_id = item.id item_name = item.name @@ -218,7 +224,10 @@ async def download_big_annotation( ) await self._sync_large_annotation( - team_id=project.team_id, project_id=project.id, item_id=item_id + team_id=project.team_id, + project_id=project.id, + item_id=item_id, + transform_version=transform_version, ) async with AIOHttpSession( @@ -252,12 +261,15 @@ async def download_small_annotations( download_path: str, item_ids: List[int], callback: Callable = None, + transform_version: str = None, ): query_params = { "team_id": project.team_id, "project_id": project.id, "folder_id": folder.id, } + if transform_version: + query_params["transform_version"] = transform_version handler = StreamedAnnotations( headers=self.client.default_headers, reporter=reporter, diff --git a/tests/data_set/editor_templates/from1_classes.json b/tests/data_set/editor_templates/form1_classes.json similarity index 100% rename from tests/data_set/editor_templates/from1_classes.json rename to tests/data_set/editor_templates/form1_classes.json diff --git a/tests/integration/annotations/test_upload_annotations.py b/tests/integration/annotations/test_upload_annotations.py index 0732b5749..181ace079 100644 --- a/tests/integration/annotations/test_upload_annotations.py +++ b/tests/integration/annotations/test_upload_annotations.py @@ -1,5 +1,6 @@ import json import os +import tempfile import time from pathlib import Path @@ -133,23 +134,19 @@ def test_upload_large_annotations(self): ) == 5 -class MultiModalUploadAnnotations(BaseTestCase): +class MultiModalUploadDownloadAnnotations(BaseTestCase): PROJECT_NAME = "TestMultimodalUploadAnnotations" PROJECT_TYPE = "Multimodal" PROJECT_DESCRIPTION = "DESCRIPTION" - EDITOR_TEMPLATE_PATH = os.path.join( - Path(__file__).parent.parent.parent, "data_set/editor_templates/form1.json" - ) - JSONL_ANNOTATIONS_PATH = os.path.join( - DATA_SET_PATH, "multimodal/annotations/jsonl/form1.jsonl" - ) - JSONL_ANNOTATIONS_WITH_CATEGORIES_PATH = os.path.join( - DATA_SET_PATH, "multimodal/annotations/jsonl/form1_with_categories.jsonl" - ) - CLASSES_TEMPLATE_PATH = os.path.join( - Path(__file__).parent.parent.parent, - "data_set/editor_templates/from1_classes.json", + BASE_PATH = Path(__file__).parent.parent.parent + DATA_SET_PATH = BASE_PATH / "data_set" + + EDITOR_TEMPLATE_PATH = DATA_SET_PATH / "editor_templates/form1.json" + JSONL_ANNOTATIONS_PATH = DATA_SET_PATH / "multimodal/annotations/jsonl/form1.jsonl" + JSONL_ANNOTATIONS_WITH_CATEGORIES_PATH = ( + DATA_SET_PATH / "multimodal/annotations/jsonl/form1_with_categories.jsonl" ) + CLASSES_TEMPLATE_PATH = DATA_SET_PATH / "editor_templates" / "form1_classes.json" def setUp(self, *args, **kwargs): self.tearDown() @@ -244,3 +241,32 @@ def test_upload_with_integer_names(self): sa.get_annotations( f"{self.PROJECT_NAME}/test_folder", data_spec="multimodal" ) + + def test_download_annotations(self): + with open(self.JSONL_ANNOTATIONS_PATH) as f: + data = [json.loads(line) for line in f] + sa.upload_annotations( + self.PROJECT_NAME, annotations=data, data_spec="multimodal" + ) + + annotations = sa.get_annotations( + f"{self.PROJECT_NAME}/test_folder", data_spec="multimodal" + ) + + with tempfile.TemporaryDirectory() as tmpdir: + sa.download_annotations( + f"{self.PROJECT_NAME}/test_folder", path=tmpdir, data_spec="multimodal" + ) + downloaded_files = list(Path(f"{tmpdir}/test_folder").glob("*.json")) + assert len(downloaded_files) > 0, "No annotations were downloaded" + downloaded_data = [] + for file_path in downloaded_files: + with open(file_path) as f: + downloaded_data.append(json.load(f)) + + assert len(downloaded_data) == len( + annotations + ), "Mismatch in annotation count" + assert ( + downloaded_data == annotations + ), "Downloaded annotations do not match uploaded annotations" diff --git a/tests/integration/items/test_item_context.py b/tests/integration/items/test_item_context.py index 63f7cf310..3a50e2d40 100644 --- a/tests/integration/items/test_item_context.py +++ b/tests/integration/items/test_item_context.py @@ -19,7 +19,7 @@ class TestMultimodalProjectBasic(BaseTestCase): ) CLASSES_TEMPLATE_PATH = os.path.join( Path(__file__).parent.parent.parent, - "data_set/editor_templates/from1_classes.json", + "data_set/editor_templates/form1_classes.json", ) def setUp(self, *args, **kwargs):