From 1462b10ca5b530b019736380c366c380391044e1 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 3 Jul 2024 17:52:24 +0400 Subject: [PATCH] Added sdk_core custom fields --- .../lib/app/interface/sdk_interface.py | 72 ++++--- .../lib/core/usecases/__init__.py | 1 - .../lib/core/usecases/custom_fields.py | 176 ------------------ .../lib/infrastructure/controller.py | 53 ------ .../lib/infrastructure/serviceprovider.py | 2 - .../infrastructure/services/custom_field.py | 58 ------ .../custom_fields/test_custom_schema.py | 3 +- 7 files changed, 37 insertions(+), 328 deletions(-) delete mode 100644 src/superannotate/lib/core/usecases/custom_fields.py delete mode 100644 src/superannotate/lib/infrastructure/services/custom_field.py diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 3713d3b12..6f8f1b434 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -2440,7 +2440,7 @@ def query( _items = folder.list_items(query=query, subset_id=subset_id) else: _items = project.list_items(query=query, subset_id=subset_id) - exclude = {"custom_metadata", "meta"} + exclude = {"meta"} return [serialize_item(i, project).dict(exclude=exclude) for i in _items] def get_item_metadata( @@ -2609,14 +2609,20 @@ def search_items( items.extend( [ serialize_item(i, project, folder).dict(exclude=exclude) - for i in folder.list_items(condition=search_condition) + for i in folder.list_items( + condition=search_condition, + include_custom_metadata=include_custom_metadata, + ) ] ) else: folder = project.get_folder(folder_name) items = [ serialize_item(i, project, folder).dict(exclude=exclude) - for i in folder.list_items(condition=search_condition) + for i in folder.list_items( + condition=search_condition, + include_custom_metadata=include_custom_metadata, + ) ] return items @@ -2994,13 +3000,8 @@ def create_custom_fields(self, project: NotEmptyStr, fields: dict): """ project_name, _ = extract_project_folder(project) - project = self.controller.projects.get_by_name(project_name).data - response = self.controller.custom_fields.create_schema( - project=project, schema=fields - ) - if response.errors: - raise AppException(response.errors) - return response.data + project = self.controller.get_project(project_name) + return project.create_custom_fields(fields) def get_custom_fields(self, project: NotEmptyStr): """Get the schema of the custom fields defined for the project @@ -3042,11 +3043,8 @@ def get_custom_fields(self, project: NotEmptyStr): } """ project_name, _ = extract_project_folder(project) - project = self.controller.projects.get_by_name(project_name).data - response = self.controller.custom_fields.get_schema(project=project) - if response.errors: - raise AppException(response.errors) - return response.data + project = self.controller.get_project(project_name) + return project.get_custom_fields() def delete_custom_fields( self, project: NotEmptyStr, fields: conlist(str, min_items=1) @@ -3096,13 +3094,9 @@ def delete_custom_fields( """ project_name, _ = extract_project_folder(project) - project = self.controller.projects.get_by_name(project_name).data - response = self.controller.custom_fields.delete_schema( - project=project, fields=fields - ) - if response.errors: - raise AppException(response.errors) - return response.data + project = self.controller.get_project(project_name) + project.delete_custom_field(fields) + return project.get_custom_fields() def upload_custom_values( self, project: NotEmptyStr, items: conlist(Dict[str, dict], min_items=1) @@ -3164,19 +3158,21 @@ def upload_custom_values( :: { - "successful_items_count": 2, - "failed_items_names": ["image_3.png"] + "succeeded": ["image_1.png", "image_2.png"], + "failed": ["image_3.png"] } """ project_name, folder_name = extract_project_folder(project) - project, folder = self.controller.get_project_folder(project_name, folder_name) - response = self.controller.custom_fields.upload_values( - project=project, folder=folder, items=items - ) - if response.errors: - raise AppException(response.errors) - return response.data + project = self.controller.get_project(project_name) + folder = project.get_folder(folder_name) + _item_fields_mapping = {} + for i in items: + _item_fields_mapping.update(i) + _items = folder.list_items(item_names=list(_item_fields_mapping.keys())) + item_fields_map = {i: _item_fields_mapping[i.name] for i in _items} + succeeded_items, failed_items = folder.set_custom_field_values(item_fields_map) + return {"succeeded": succeeded_items, "failed": failed_items} def delete_custom_values( self, project: NotEmptyStr, items: conlist(Dict[str, List[str]], min_items=1) @@ -3207,12 +3203,14 @@ def delete_custom_values( ) """ project_name, folder_name = extract_project_folder(project) - project, folder = self.controller.get_project_folder(project_name, folder_name) - response = self.controller.custom_fields.delete_values( - project=project, folder=folder, items=items - ) - if response.errors: - raise AppException(response.errors) + project = self.controller.get_project(project_name) + folder = project.get_folder(folder_name) + _item_fields_mapping = {} + for i in items: + _item_fields_mapping.update(i) + _items = folder.list_items(item_names=list(_item_fields_mapping.keys())) + item_fields_map = {i: _item_fields_mapping[i.name] for i in _items} + return folder.delete_custom_field_values(item_fields_map) def add_items_to_subset( self, project: NotEmptyStr, subset: NotEmptyStr, items: List[dict] diff --git a/src/superannotate/lib/core/usecases/__init__.py b/src/superannotate/lib/core/usecases/__init__.py index 81ba93a7f..7141f56b0 100644 --- a/src/superannotate/lib/core/usecases/__init__.py +++ b/src/superannotate/lib/core/usecases/__init__.py @@ -1,6 +1,5 @@ from lib.core.usecases.annotations import * # noqa: F403 F401 from lib.core.usecases.classes import * # noqa: F403 F401 -from lib.core.usecases.custom_fields import * # noqa: F403 F401 from lib.core.usecases.images import * # noqa: F403 F401 from lib.core.usecases.integrations import * # noqa: F403 F401 from lib.core.usecases.items import * # noqa: F403 F401 diff --git a/src/superannotate/lib/core/usecases/custom_fields.py b/src/superannotate/lib/core/usecases/custom_fields.py deleted file mode 100644 index 241b6e182..000000000 --- a/src/superannotate/lib/core/usecases/custom_fields.py +++ /dev/null @@ -1,176 +0,0 @@ -from typing import Dict -from typing import List - -from lib.core.entities import FolderEntity -from lib.core.entities import ProjectEntity -from lib.core.reporter import Reporter -from lib.core.response import Response -from lib.core.serviceproviders import BaseServiceProvider -from lib.core.usecases import BaseReportableUseCase - - -class CreateCustomSchemaUseCase(BaseReportableUseCase): - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - schema: dict, - service_provider: BaseServiceProvider, - ): - super().__init__(reporter) - self._project = project - self._schema = schema - self._service_provider = service_provider - - def execute(self) -> Response: - response = self._service_provider.custom_fields.create_schema( - project=self._project, - schema=self._schema, - ) - if response.ok: - self._response.data = response.data - else: - error = response.error - if isinstance(error, list): - error = "-" + "\n-".join(error) - self._response.errors = error - return self._response - - -class GetCustomSchemaUseCase(BaseReportableUseCase): - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - service_provider: BaseServiceProvider, - ): - super().__init__(reporter) - self._project = project - self._service_provider = service_provider - - def execute(self) -> Response: - response = self._service_provider.custom_fields.get_schema( - project=self._project - ) - if response.ok: - self._response.data = response.data - else: - self._response.errors = response.error - return self._response - - -class DeleteCustomSchemaUseCase(BaseReportableUseCase): - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - fields: List[str], - service_provider: BaseServiceProvider, - ): - super().__init__(reporter) - self._project = project - self._fields = fields - self._service_provider = service_provider - - def execute(self) -> Response: - if self._fields: - self.reporter.log_info("Matched fields deleted from schema.") - response = self._service_provider.custom_fields.delete_fields( - project=self._project, - fields=self._fields, - ) - if response.ok: - use_case_response = GetCustomSchemaUseCase( - reporter=self.reporter, - project=self._project, - service_provider=self._service_provider, - ).execute() - if use_case_response.errors: - self._response.errors = use_case_response.errors - else: - self._response.data = use_case_response.data - else: - self._response.errors = response.error - return self._response - - -class UploadCustomValuesUseCase(BaseReportableUseCase): - CHUNK_SIZE = 5000 - - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - items: List[Dict[str, str]], - service_provider: BaseServiceProvider, - ): - super().__init__(reporter) - self._project = project - self._folder = folder - self._items = items - self._service_provider = service_provider - - def execute(self) -> Response: - uploaded_items, failed_items = [], [] - self.reporter.log_info( - "Validating metadata against the schema of the custom fields. " - "Valid metadata will be attached to the specified item." - ) - with self.reporter.spinner: - for idx in range(0, len(self._items), self.CHUNK_SIZE): - response = self._service_provider.custom_fields.upload_fields( - project=self._project, - folder=self._folder, - items=self._items[idx : idx + self.CHUNK_SIZE], # noqa: E203 - ) - if not response.ok: - self._response.errors = response.error - return self._response - failed_items.extend(response.data.failed_items) - - if failed_items: - self.reporter.log_error( - f"The metadata dicts of {len(failed_items)} items are invalid because they don't match " - f'the schema of the custom fields defined for the "{self._project.name}" project.' - ) - self._response.data = { - "succeeded": list( - {list(item)[0] for item in self._items} ^ set(failed_items) - ), - "failed": failed_items, - } - return self._response - - -class DeleteCustomValuesUseCase(BaseReportableUseCase): - CHUNK_SIZE = 5000 - - def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - folder: FolderEntity, - items: List[Dict[str, List[str]]], - service_provider: BaseServiceProvider, - ): - super().__init__(reporter) - self._project = project - self._folder = folder - self._items = items - self._service_provider = service_provider - - def execute(self) -> Response: - for idx in range(0, len(self._items), self.CHUNK_SIZE): - response = self._service_provider.custom_fields.delete_values( - project=self._project, - folder=self._folder, - items=self._items[idx : idx + self.CHUNK_SIZE], # noqa: E203 - ) - if not response.ok: - self._response.errors = response.error - return self._response - self.reporter.log_info( - "Corresponding fields and their values removed from items." - ) - return self._response diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 62fa78b7d..7c75ba3d4 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -580,58 +580,6 @@ def upload_image_annotations( return use_case.execute() -class CustomFieldManager(BaseManager): - def create_schema(self, project: ProjectEntity, schema: dict): - use_case = usecases.CreateCustomSchemaUseCase( - reporter=Reporter(), - project=project, - schema=schema, - service_provider=self.service_provider, - ) - return use_case.execute() - - def get_schema(self, project: ProjectEntity): - use_case = usecases.GetCustomSchemaUseCase( - reporter=Reporter(), - project=project, - service_provider=self.service_provider, - ) - return use_case.execute() - - def delete_schema(self, project: ProjectEntity, fields: List[str]): - use_case = usecases.DeleteCustomSchemaUseCase( - reporter=Reporter(), - project=project, - fields=fields, - service_provider=self.service_provider, - ) - return use_case.execute() - - def upload_values( - self, project: ProjectEntity, folder: FolderEntity, items: List[dict] - ): - use_case = usecases.UploadCustomValuesUseCase( - reporter=Reporter(), - project=project, - folder=folder, - items=items, - service_provider=self.service_provider, - ) - return use_case.execute() - - def delete_values( - self, project: ProjectEntity, folder: FolderEntity, items: List[dict] - ): - use_case = usecases.DeleteCustomValuesUseCase( - reporter=Reporter(), - project=project, - folder=folder, - items=items, - service_provider=self.service_provider, - ) - return use_case.execute() - - class ModelManager(BaseManager): def list(self, condition: Condition): use_case = usecases.SearchMLModels( @@ -757,7 +705,6 @@ def __init__(self, config: ConfigEntity): self.annotations = AnnotationManager( self.service_provider, config, self._session ) - self.custom_fields = CustomFieldManager(self.service_provider, self._session) self.subsets = SubsetManager(self.service_provider, self._session) self.models = ModelManager(self.service_provider, self._session) self.integrations = IntegrationManager(self.service_provider, self._session) diff --git a/src/superannotate/lib/infrastructure/serviceprovider.py b/src/superannotate/lib/infrastructure/serviceprovider.py index 378a2a9ad..bf0d90e7a 100644 --- a/src/superannotate/lib/infrastructure/serviceprovider.py +++ b/src/superannotate/lib/infrastructure/serviceprovider.py @@ -13,7 +13,6 @@ from lib.core.serviceproviders import BaseServiceProvider from lib.infrastructure.services.annotation import AnnotationService from lib.infrastructure.services.annotation_class import AnnotationClassService -from lib.infrastructure.services.custom_field import CustomFieldService from lib.infrastructure.services.http_client import HttpClient from lib.infrastructure.services.integration import IntegrationService from lib.infrastructure.services.item import ItemService @@ -49,7 +48,6 @@ def __init__(self, client: HttpClient): self.items = ItemService(client) self.annotations = AnnotationService(client) self.annotation_classes = AnnotationClassService(client) - self.custom_fields = CustomFieldService(client) self.subsets = SubsetService(client) self.models = ModelsService(client) self.integrations = IntegrationService(client) diff --git a/src/superannotate/lib/infrastructure/services/custom_field.py b/src/superannotate/lib/infrastructure/services/custom_field.py deleted file mode 100644 index 1b8ccac7e..000000000 --- a/src/superannotate/lib/infrastructure/services/custom_field.py +++ /dev/null @@ -1,58 +0,0 @@ -from collections import ChainMap -from typing import Dict -from typing import List - -from lib.core import entities -from lib.core.service_types import UploadCustomFieldValuesResponse -from lib.core.serviceproviders import BaseCustomFieldService - - -class CustomFieldService(BaseCustomFieldService): - URL_CREATE_CUSTOM_SCHEMA = "/project/{project_id}/custom/metadata/schema" - URL_UPLOAD_CUSTOM_VALUE = "/project/{project_id}/custom/metadata/item" - - def create_schema(self, project: entities.ProjectEntity, schema: dict): - return self.client.request( - self.URL_CREATE_CUSTOM_SCHEMA.format(project_id=project.id), - "post", - data=dict(data=schema), - ) - - def get_schema(self, project: entities.ProjectEntity): - return self.client.request( - self.URL_CREATE_CUSTOM_SCHEMA.format(project_id=project.id), "get" - ) - - def delete_fields(self, project: entities.ProjectEntity, fields: List[str]): - return self.client.request( - self.URL_CREATE_CUSTOM_SCHEMA.format(project_id=project.id), - "delete", - data=dict(custom_fields=fields), - ) - - def upload_fields( - self, - project: entities.ProjectEntity, - folder: entities.FolderEntity, - items: List[dict], - ): - return self.client.request( - self.URL_UPLOAD_CUSTOM_VALUE.format(project_id=project.id), - "post", - params=dict(folder_id=folder.id), - data=dict(data=dict(ChainMap(*items))), - content_type=UploadCustomFieldValuesResponse, - ) - - def delete_values( - self, - project: entities.ProjectEntity, - folder: entities.FolderEntity, - items: List[Dict[str, List[str]]], - ): - return self.client.request( - self.URL_UPLOAD_CUSTOM_VALUE.format(project_id=project.id), - "delete", - params=dict(folder_id=folder.id), - data=dict(data=dict(ChainMap(*items))), - ) diff --git a/tests/integration/custom_fields/test_custom_schema.py b/tests/integration/custom_fields/test_custom_schema.py index 1f52059ee..1c621405f 100644 --- a/tests/integration/custom_fields/test_custom_schema.py +++ b/tests/integration/custom_fields/test_custom_schema.py @@ -2,6 +2,7 @@ from src.superannotate import AppException from src.superannotate import SAClient +from superannotate_core.core.exceptions import SAException from tests.integration.base import BaseTestCase sa = SAClient() @@ -180,5 +181,5 @@ def test_create_invalid(self): "-Minimum spec value of age_range can not be higher than maximum value.\n" "-Spec value type of age_enum is not valid." ) - with self.assertRaisesRegexp(AppException, error_msg): + with self.assertRaisesRegexp(SAException, error_msg): sa.create_custom_fields(self.PROJECT_NAME, INVALID_SCHEMA)