diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 263963c84..112238104 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,18 @@ History All release highlights of this project will be documented in this file. +4.4.18 - January 18, 2024 +__________________________ + + +**Updated** + + - Improved error handling. + +**Removed** + + - dependency from ``jsonschema``. + 4.4.17 - December 21, 2023 __________________________ diff --git a/pytest.ini b/pytest.ini index 3b63ad975..c33efcaae 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,5 +3,5 @@ minversion = 3.7 log_cli=true python_files = test_*.py ;pytest_plugins = ['pytest_profiling'] -;addopts = -n auto --dist=loadscope +addopts = -n auto --dist=loadscope diff --git a/requirements.txt b/requirements.txt index d3fae5493..7ff9c88c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,8 @@ pandas~=2.0 ffmpeg-python~=0.2 pillow>=9.5,~=10.0 tqdm~=4.66.1 -requests~=2.31.0 -aiofiles==23.1.0 +requests==2.* +aiofiles==23.* fire==0.4.0 mixpanel==4.8.3 -jsonschema==3.2.0 +superannotate-schemas==1.0.47b5 diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index aae84e184..6f9119918 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = "4.4.17" +__version__ = "4.4.18" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 162940830..846722f07 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -79,7 +79,7 @@ "Tiled", "Other", "PointCloud", - "CustomEditor", + "GenAI", ] ANNOTATION_STATUS = Literal[ diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index 2bad17386..5cd85f136 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -122,7 +122,10 @@ def serialize( if exclude: for field in exclude: to_exclude[field] = True - + if self._entity.classes: + self._entity.classes = [ + i.dict(by_alias=True, exclude_unset=True) for i in self._entity.classes + ] data = super().serialize(fields, by_alias, flat, to_exclude) if data.get("settings"): data["settings"] = [ diff --git a/src/superannotate/lib/core/entities/classes.py b/src/superannotate/lib/core/entities/classes.py index 24b204a9e..d2821d3d8 100644 --- a/src/superannotate/lib/core/entities/classes.py +++ b/src/superannotate/lib/core/entities/classes.py @@ -73,6 +73,7 @@ class AttributeGroup(TimedBaseModel): group_type: Optional[GroupTypeEnum] class_id: Optional[StrictInt] name: Optional[StrictStr] + isRequired: bool = Field(default=False) attributes: Optional[List[Attribute]] default_value: Any diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index 16aa26fa5..8c3d3380e 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -93,7 +93,7 @@ class ProjectType(BaseTitledEnum): TILED = "Tiled", 5 OTHER = "Other", 6 POINT_CLOUD = "PointCloud", 7 - CUSTOM_EDITOR = "CustomEditor", 8 + GEN_AI = "GenAI", 8 UNSUPPORTED_TYPE_1 = "UnsupportedType", 9 UNSUPPORTED_TYPE_2 = "UnsupportedType", 10 diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index f3b5d963a..f3ab4f860 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -25,10 +25,8 @@ import aiofiles import boto3 -import jsonschema.validators import lib.core as constants -from jsonschema import Draft7Validator -from jsonschema import ValidationError +import superannotate_schemas from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import BaseItemEntity @@ -309,7 +307,7 @@ def __init__( def validate_project_type(self): if self._project.type == constants.ProjectType.PIXEL.value: - raise ValidationError("Unsupported project type.") + raise AppException("Unsupported project type.") def _validate_json(self, json_data: dict) -> list: if self._project.type >= constants.ProjectType.PIXEL.value: @@ -1227,7 +1225,7 @@ def execute(self): class ValidateAnnotationUseCase(BaseReportableUseCase): DEFAULT_VERSION = "V1.00" - SCHEMAS: Dict[str, Draft7Validator] = {} + SCHEMAS: Dict[str, superannotate_schemas.Draft7Validator] = {} PATTERN_MAP = { "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ", "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email", @@ -1278,7 +1276,7 @@ def oneOf(validator, oneOf, instance, schema): # noqa const_key, instance ) if not instance_type: - yield ValidationError("type required") + yield superannotate_schemas.ValidationError("type required") return if const_key and instance_type == _type: errs = list( @@ -1286,7 +1284,9 @@ def oneOf(validator, oneOf, instance, schema): # noqa ) if not errs: return - yield ValidationError("invalid instance", context=errs) + yield superannotate_schemas.ValidationError( + "invalid instance", context=errs + ) return else: subschemas = enumerate(oneOf) @@ -1299,24 +1299,25 @@ def oneOf(validator, oneOf, instance, schema): # noqa break all_errors.extend(errs) else: - yield ValidationError( + yield superannotate_schemas.ValidationError( f"{instance!r} is not valid under any of the given schemas", context=all_errors[:1], ) - # yield from jsonschema._validators.oneOf( # noqa - # validator, oneOf, instance, schema - # ) if const_key: - yield ValidationError(f"invalid {'.'.join(const_key)}") + yield superannotate_schemas.ValidationError( + f"invalid {'.'.join(const_key)}" + ) @staticmethod def _pattern(validator, patrn, instance, schema): if validator.is_type(instance, "string") and not re.search(patrn, instance): _patrn = ValidateAnnotationUseCase.PATTERN_MAP.get(patrn) if _patrn: - yield ValidationError(f"{instance} {_patrn}") + yield superannotate_schemas.ValidationError(f"{instance} {_patrn}") else: - yield ValidationError(f"{instance} does not match {patrn}") + yield superannotate_schemas.ValidationError( + f"{instance} does not match {patrn}" + ) @staticmethod def iter_errors(self, instance, _schema=None): @@ -1325,7 +1326,7 @@ def iter_errors(self, instance, _schema=None): if _schema is True: return elif _schema is False: - yield jsonschema.exceptions.ValidationError( + yield superannotate_schemas.ValidationError( f"False schema does not allow {instance!r}", validator=None, validator_value=None, @@ -1334,7 +1335,7 @@ def iter_errors(self, instance, _schema=None): ) return - scope = jsonschema.validators._id_of(_schema) # noqa + scope = superannotate_schemas.validators._id_of(_schema) # noqa _schema = copy.copy(_schema) if scope: self.resolver.push_scope(scope) @@ -1344,7 +1345,7 @@ def iter_errors(self, instance, _schema=None): ref = _schema.pop("$ref") validators.append(("$ref", ref)) - validators.extend(jsonschema.validators.iteritems(_schema)) + validators.extend(superannotate_schemas.validators.iteritems(_schema)) for k, v in validators: validator = self.VALIDATORS.get(k) @@ -1381,7 +1382,7 @@ def extract_path(path): real_path.append(item) return real_path - def _get_validator(self, version: str) -> Draft7Validator: + def _get_validator(self, version: str) -> superannotate_schemas.Draft7Validator: key = f"{self._project_type}__{version}" validator = ValidateAnnotationUseCase.SCHEMAS.get(key) if not validator: @@ -1393,7 +1394,7 @@ def _get_validator(self, version: str) -> Draft7Validator: if not schema_response.data: ValidateAnnotationUseCase.SCHEMAS[key] = lambda x: x return ValidateAnnotationUseCase.SCHEMAS[key] - validator = jsonschema.Draft7Validator(schema_response.data) + validator = superannotate_schemas.Draft7Validator(schema_response.data) from functools import partial iter_errors = partial(self.iter_errors, validator) diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py index d16dd6661..9d478e6e9 100644 --- a/src/superannotate/lib/infrastructure/services/annotation.py +++ b/src/superannotate/lib/infrastructure/services/annotation.py @@ -80,9 +80,9 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): async with AIOHttpSession( connector=aiohttp.TCPConnector(ssl=False), headers=self.client.default_headers, + raise_for_status=True, ) as session: _response = await session.request("post", sync_url, params=sync_params) - _response.raise_for_status() sync_params.pop("current_source") sync_params.pop("desired_source") @@ -123,9 +123,9 @@ async def get_big_annotation( async with AIOHttpSession( connector=aiohttp.TCPConnector(ssl=False), headers=self.client.default_headers, + raise_for_status=True, ) as session: start_response = await session.request("post", url, params=query_params) - start_response.raise_for_status() large_annotation = await start_response.json() reporter.update_progress() @@ -206,9 +206,9 @@ async def download_big_annotation( async with AIOHttpSession( connector=aiohttp.TCPConnector(ssl=False), headers=self.client.default_headers, + raise_for_status=True, ) as session: start_response = await session.request("post", url, params=query_params) - start_response.raise_for_status() res = await start_response.json() Path(download_path).mkdir(exist_ok=True, parents=True) diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py index 071a907c5..78783a3e9 100644 --- a/src/superannotate/lib/infrastructure/services/http_client.py +++ b/src/superannotate/lib/infrastructure/services/http_client.py @@ -1,4 +1,5 @@ import asyncio +import io import json import logging import platform @@ -13,7 +14,6 @@ import aiohttp import requests -from aiohttp.client_exceptions import ClientError from lib.core.exceptions import AppException from lib.core.service_types import ServiceResponse from lib.core.serviceproviders import BaseClient @@ -229,19 +229,35 @@ class AIOHttpSession(aiohttp.ClientSession): RETRY_LIMIT = 3 BACKOFF_FACTOR = 0.3 + @staticmethod + def _copy_form_data(data: aiohttp.FormData) -> aiohttp.FormData: + form_data = aiohttp.FormData(quote_fields=False) + for field in data._fields: # noqa + if isinstance(field[2], io.IOBase): + field[2].seek(0) + form_data.add_field( + value=field[2], + content_type=field[1].get("Content-Type", ""), + **field[0], + ) + return form_data + async def request(self, *args, **kwargs) -> aiohttp.ClientResponse: attempts = self.RETRY_LIMIT delay = 0 for _ in range(attempts): delay += self.BACKOFF_FACTOR - attempts -= 1 try: response = await super()._request(*args, **kwargs) - except ClientError: - if not attempts: + if attempts <= 1 or response.status not in self.RETRY_STATUS_CODES: + return response + if isinstance(kwargs["data"], aiohttp.FormData): + raise RuntimeError(await response.text()) + except (aiohttp.ClientError, RuntimeError) as e: + if attempts <= 1: raise - await asyncio.sleep(delay) - continue - if response.status not in self.RETRY_STATUS_CODES or not attempts: - return response + data = kwargs["data"] + if isinstance(data, aiohttp.FormData): + kwargs["data"] = self._copy_form_data(data) + attempts -= 1 await asyncio.sleep(delay) diff --git a/src/superannotate/lib/infrastructure/services/item.py b/src/superannotate/lib/infrastructure/services/item.py index 59744700d..b8c68ff2b 100644 --- a/src/superannotate/lib/infrastructure/services/item.py +++ b/src/superannotate/lib/infrastructure/services/item.py @@ -40,6 +40,7 @@ class ItemService(BaseItemService): ProjectType.PIXEL: ImageResponse, ProjectType.DOCUMENT: DocumentResponse, ProjectType.POINT_CLOUD: PointCloudResponse, + ProjectType.GEN_AI: ImageResponse } def get_by_id(self, item_id, project_id, project_type): diff --git a/tests/integration/classes/test_create_annotation_class.py b/tests/integration/classes/test_create_annotation_class.py index f4e81bf82..2592f8984 100644 --- a/tests/integration/classes/test_create_annotation_class.py +++ b/tests/integration/classes/test_create_annotation_class.py @@ -44,6 +44,7 @@ def test_create_annotation_class_with_attr_and_default_value(self): attribute_groups=[ { "name": "test", + "isRequired:": False, "attributes": [{"name": "Car"}, {"name": "Track"}, {"name": "Bus"}], "default_value": "Bus", } diff --git a/tests/integration/export/test_export.py b/tests/integration/export/test_export.py index 073a3556f..04cb4c352 100644 --- a/tests/integration/export/test_export.py +++ b/tests/integration/export/test_export.py @@ -106,4 +106,4 @@ def test_upload_s3(self): Bucket=self.TEST_S3_BUCKET, Prefix=self.TMP_DIR ).get("Contents", []): files.append(object_data["Key"]) - self.assertEqual(33, len(files)) + self.assertEqual(25, len(files)) diff --git a/tests/integration/projects/test_basic_project.py b/tests/integration/projects/test_basic_project.py index 26e0e314a..8f5d899a3 100644 --- a/tests/integration/projects/test_basic_project.py +++ b/tests/integration/projects/test_basic_project.py @@ -11,6 +11,45 @@ sa = SAClient() +class TestGenAIProjectBasic(BaseTestCase): + PROJECT_NAME = "TestGenAICreate" + PROJECT_TYPE = "GenAI" + PROJECT_DESCRIPTION = "DESCRIPTION" + ANNOTATION_PATH = ( + "data_set/sample_project_vector/example_image_1.jpg___objects.json" + ) + + @property + def annotation_path(self): + return os.path.join(Path(__file__).parent.parent.parent, self.ANNOTATION_PATH) + + def test_search(self): + projects = sa.search_projects(self.PROJECT_NAME, return_metadata=True) + assert projects + + sa.create_annotation_class( + self.PROJECT_NAME, + "class1", + "#FFAAFF", + [ + { + "name": "Human", + "attributes": [{"name": "yes"}, {"name": "no"}], + }, + { + "name": "age", + "attributes": [{"name": "young"}, {"name": "old"}], + }, + ], + ) + sa.attach_items(self.PROJECT_NAME, attachments=[{"url": "", "name": "name"}]) + annotation = json.load(open(self.annotation_path)) + annotation["metadata"]["name"] = "name" + sa.upload_annotations(self.PROJECT_NAME, annotations=[annotation]) + data = sa.get_annotations(self.PROJECT_NAME) + assert data + + class TestProjectBasic(BaseTestCase): PROJECT_NAME = "TestWorkflowGet" PROJECT_TYPE = "Vector" diff --git a/tests/unit/test_classes_serialization.py b/tests/unit/test_classes_serialization.py index 3a081ce57..540f1d8e2 100644 --- a/tests/unit/test_classes_serialization.py +++ b/tests/unit/test_classes_serialization.py @@ -43,7 +43,7 @@ def test_empty_multiselect_excluded(self): "type": 1, "name": "asd", "color": "#0000FF", - "attribute_groups": [{"name": "sad"}], + "attribute_groups": [{"name": "sad", "isRequired": False}], } == serializer_data def test_empty_multiselect_bool_serializer(self): @@ -59,7 +59,7 @@ def test_empty_multiselect_bool_serializer(self): "type": 1, "name": "asd", "color": "#0000FF", - "attribute_groups": [{"name": "sad"}], + "attribute_groups": [{"name": "sad", "isRequired": False}], } == serializer_data def test_group_type_wrong_arg(self):