From 974b11b310bb494648005eef7175e8ae45fdcc4f Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Wed, 10 Jan 2024 18:12:27 +0400 Subject: [PATCH 1/3] Removed jsonschema tod --- pytest.ini | 2 +- requirements.txt | 6 +-- .../lib/core/usecases/annotations.py | 39 ++++++++++--------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/pytest.ini b/pytest.ini index 3b63ad975..c33efcaae 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,5 +3,5 @@ minversion = 3.7 log_cli=true python_files = test_*.py ;pytest_plugins = ['pytest_profiling'] -;addopts = -n auto --dist=loadscope +addopts = -n auto --dist=loadscope diff --git a/requirements.txt b/requirements.txt index d3fae5493..a660db6e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,8 @@ pandas~=2.0 ffmpeg-python~=0.2 pillow>=9.5,~=10.0 tqdm~=4.66.1 -requests~=2.31.0 -aiofiles==23.1.0 +requests==2.* +aiofiles==23.* fire==0.4.0 mixpanel==4.8.3 -jsonschema==3.2.0 +superannotate-schemas==1.0.46 diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index f3b5d963a..f3ab4f860 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -25,10 +25,8 @@ import aiofiles import boto3 -import jsonschema.validators import lib.core as constants -from jsonschema import Draft7Validator -from jsonschema import ValidationError +import superannotate_schemas from lib.core.conditions import Condition from lib.core.conditions import CONDITION_EQ as EQ from lib.core.entities import BaseItemEntity @@ -309,7 +307,7 @@ def __init__( def validate_project_type(self): if self._project.type == constants.ProjectType.PIXEL.value: - raise ValidationError("Unsupported project type.") + raise AppException("Unsupported project type.") def _validate_json(self, json_data: dict) -> list: if self._project.type >= constants.ProjectType.PIXEL.value: @@ -1227,7 +1225,7 @@ def execute(self): class ValidateAnnotationUseCase(BaseReportableUseCase): DEFAULT_VERSION = "V1.00" - SCHEMAS: Dict[str, Draft7Validator] = {} + SCHEMAS: Dict[str, superannotate_schemas.Draft7Validator] = {} PATTERN_MAP = { "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ", "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email", @@ -1278,7 +1276,7 @@ def oneOf(validator, oneOf, instance, schema): # noqa const_key, instance ) if not instance_type: - yield ValidationError("type required") + yield superannotate_schemas.ValidationError("type required") return if const_key and instance_type == _type: errs = list( @@ -1286,7 +1284,9 @@ def oneOf(validator, oneOf, instance, schema): # noqa ) if not errs: return - yield ValidationError("invalid instance", context=errs) + yield superannotate_schemas.ValidationError( + "invalid instance", context=errs + ) return else: subschemas = enumerate(oneOf) @@ -1299,24 +1299,25 @@ def oneOf(validator, oneOf, instance, schema): # noqa break all_errors.extend(errs) else: - yield ValidationError( + yield superannotate_schemas.ValidationError( f"{instance!r} is not valid under any of the given schemas", context=all_errors[:1], ) - # yield from jsonschema._validators.oneOf( # noqa - # validator, oneOf, instance, schema - # ) if const_key: - yield ValidationError(f"invalid {'.'.join(const_key)}") + yield superannotate_schemas.ValidationError( + f"invalid {'.'.join(const_key)}" + ) @staticmethod def _pattern(validator, patrn, instance, schema): if validator.is_type(instance, "string") and not re.search(patrn, instance): _patrn = ValidateAnnotationUseCase.PATTERN_MAP.get(patrn) if _patrn: - yield ValidationError(f"{instance} {_patrn}") + yield superannotate_schemas.ValidationError(f"{instance} {_patrn}") else: - yield ValidationError(f"{instance} does not match {patrn}") + yield superannotate_schemas.ValidationError( + f"{instance} does not match {patrn}" + ) @staticmethod def iter_errors(self, instance, _schema=None): @@ -1325,7 +1326,7 @@ def iter_errors(self, instance, _schema=None): if _schema is True: return elif _schema is False: - yield jsonschema.exceptions.ValidationError( + yield superannotate_schemas.ValidationError( f"False schema does not allow {instance!r}", validator=None, validator_value=None, @@ -1334,7 +1335,7 @@ def iter_errors(self, instance, _schema=None): ) return - scope = jsonschema.validators._id_of(_schema) # noqa + scope = superannotate_schemas.validators._id_of(_schema) # noqa _schema = copy.copy(_schema) if scope: self.resolver.push_scope(scope) @@ -1344,7 +1345,7 @@ def iter_errors(self, instance, _schema=None): ref = _schema.pop("$ref") validators.append(("$ref", ref)) - validators.extend(jsonschema.validators.iteritems(_schema)) + validators.extend(superannotate_schemas.validators.iteritems(_schema)) for k, v in validators: validator = self.VALIDATORS.get(k) @@ -1381,7 +1382,7 @@ def extract_path(path): real_path.append(item) return real_path - def _get_validator(self, version: str) -> Draft7Validator: + def _get_validator(self, version: str) -> superannotate_schemas.Draft7Validator: key = f"{self._project_type}__{version}" validator = ValidateAnnotationUseCase.SCHEMAS.get(key) if not validator: @@ -1393,7 +1394,7 @@ def _get_validator(self, version: str) -> Draft7Validator: if not schema_response.data: ValidateAnnotationUseCase.SCHEMAS[key] = lambda x: x return ValidateAnnotationUseCase.SCHEMAS[key] - validator = jsonschema.Draft7Validator(schema_response.data) + validator = superannotate_schemas.Draft7Validator(schema_response.data) from functools import partial iter_errors = partial(self.iter_errors, validator) From 96a5961ab812ef53707ffd688be3af29924313c6 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Thu, 11 Jan 2024 15:42:07 +0400 Subject: [PATCH 2/3] Added retries on form data fail and GenAI --- pytest.ini | 2 +- .../lib/app/interface/sdk_interface.py | 2 +- .../lib/core/entities/classes.py | 1 + src/superannotate/lib/core/enums.py | 2 +- .../infrastructure/services/http_client.py | 52 +++++++++++++------ .../classes/test_create_annotation_class.py | 1 + .../projects/test_basic_project.py | 10 ++++ 7 files changed, 51 insertions(+), 19 deletions(-) diff --git a/pytest.ini b/pytest.ini index c33efcaae..3b63ad975 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,5 +3,5 @@ minversion = 3.7 log_cli=true python_files = test_*.py ;pytest_plugins = ['pytest_profiling'] -addopts = -n auto --dist=loadscope +;addopts = -n auto --dist=loadscope diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 162940830..846722f07 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -79,7 +79,7 @@ "Tiled", "Other", "PointCloud", - "CustomEditor", + "GenAI", ] ANNOTATION_STATUS = Literal[ diff --git a/src/superannotate/lib/core/entities/classes.py b/src/superannotate/lib/core/entities/classes.py index 24b204a9e..48f93bd9a 100644 --- a/src/superannotate/lib/core/entities/classes.py +++ b/src/superannotate/lib/core/entities/classes.py @@ -73,6 +73,7 @@ class AttributeGroup(TimedBaseModel): group_type: Optional[GroupTypeEnum] class_id: Optional[StrictInt] name: Optional[StrictStr] + required: bool = Field(default=False) attributes: Optional[List[Attribute]] default_value: Any diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py index 16aa26fa5..8c3d3380e 100644 --- a/src/superannotate/lib/core/enums.py +++ b/src/superannotate/lib/core/enums.py @@ -93,7 +93,7 @@ class ProjectType(BaseTitledEnum): TILED = "Tiled", 5 OTHER = "Other", 6 POINT_CLOUD = "PointCloud", 7 - CUSTOM_EDITOR = "CustomEditor", 8 + GEN_AI = "GenAI", 8 UNSUPPORTED_TYPE_1 = "UnsupportedType", 9 UNSUPPORTED_TYPE_2 = "UnsupportedType", 10 diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py index 071a907c5..26b4485cc 100644 --- a/src/superannotate/lib/infrastructure/services/http_client.py +++ b/src/superannotate/lib/infrastructure/services/http_client.py @@ -1,4 +1,5 @@ import asyncio +import io import json import logging import platform @@ -13,7 +14,6 @@ import aiohttp import requests -from aiohttp.client_exceptions import ClientError from lib.core.exceptions import AppException from lib.core.service_types import ServiceResponse from lib.core.serviceproviders import BaseClient @@ -229,19 +229,39 @@ class AIOHttpSession(aiohttp.ClientSession): RETRY_LIMIT = 3 BACKOFF_FACTOR = 0.3 - async def request(self, *args, **kwargs) -> aiohttp.ClientResponse: - attempts = self.RETRY_LIMIT - delay = 0 - for _ in range(attempts): - delay += self.BACKOFF_FACTOR - attempts -= 1 - try: - response = await super()._request(*args, **kwargs) - except ClientError: - if not attempts: - raise + class AIOHttpSession(aiohttp.ClientSession): + RETRY_STATUS_CODES = [401, 403, 502, 503, 504] + RETRY_LIMIT = 3 + BACKOFF_FACTOR = 0.3 + + @staticmethod + def _copy_form_data(data: aiohttp.FormData) -> aiohttp.FormData: + form_data = aiohttp.FormData(quote_fields=False) + for field in data._fields: # noqa + if isinstance(field[2], io.IOBase): + field[2].seek(0) + form_data.add_field( + value=field[2], + content_type=field[1].get("Content-Type", ""), + **field[0], + ) + return form_data + + async def request(self, *args, **kwargs) -> aiohttp.ClientResponse: + attempts = self.RETRY_LIMIT + delay = 0 + for _ in range(attempts): + delay += self.BACKOFF_FACTOR + try: + response = await super()._request(*args, **kwargs) + if attempts <= 1 or response.status not in self.RETRY_STATUS_CODES: + return response + except (aiohttp.ClientError, RuntimeError) as e: + if attempts <= 1: + raise + if isinstance(e, RuntimeError): + data = kwargs["data"] + if isinstance(data, aiohttp.FormData): + kwargs["data"] = self._copy_form_data(data) + attempts -= 1 await asyncio.sleep(delay) - continue - if response.status not in self.RETRY_STATUS_CODES or not attempts: - return response - await asyncio.sleep(delay) diff --git a/tests/integration/classes/test_create_annotation_class.py b/tests/integration/classes/test_create_annotation_class.py index f4e81bf82..0b3f43e15 100644 --- a/tests/integration/classes/test_create_annotation_class.py +++ b/tests/integration/classes/test_create_annotation_class.py @@ -44,6 +44,7 @@ def test_create_annotation_class_with_attr_and_default_value(self): attribute_groups=[ { "name": "test", + "required": False, "attributes": [{"name": "Car"}, {"name": "Track"}, {"name": "Bus"}], "default_value": "Bus", } diff --git a/tests/integration/projects/test_basic_project.py b/tests/integration/projects/test_basic_project.py index 26e0e314a..2568efb95 100644 --- a/tests/integration/projects/test_basic_project.py +++ b/tests/integration/projects/test_basic_project.py @@ -11,6 +11,16 @@ sa = SAClient() +class TestGenAIProjectBasic(BaseTestCase): + PROJECT_NAME = "TestGenAICreate" + PROJECT_TYPE = "GenAI" + PROJECT_DESCRIPTION = "DESCRIPTION" + + def test_search(self): + projects = sa.search_projects(self.PROJECT_NAME, return_metadata=True) + assert projects + + class TestProjectBasic(BaseTestCase): PROJECT_NAME = "TestWorkflowGet" PROJECT_TYPE = "Vector" From 8d48fda9d4e1814a7d81127996e850ae7f02c5f3 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Fri, 12 Jan 2024 11:23:59 +0400 Subject: [PATCH 3/3] Added isRequired in the attribute groups --- src/superannotate/__init__.py | 2 +- src/superannotate/lib/core/entities/classes.py | 2 +- tests/integration/classes/test_create_annotation_class.py | 2 +- tests/integration/export/test_export.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index aae84e184..e0b73952d 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = "4.4.17" +__version__ = "4.4.18b1" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/core/entities/classes.py b/src/superannotate/lib/core/entities/classes.py index 48f93bd9a..d2821d3d8 100644 --- a/src/superannotate/lib/core/entities/classes.py +++ b/src/superannotate/lib/core/entities/classes.py @@ -73,7 +73,7 @@ class AttributeGroup(TimedBaseModel): group_type: Optional[GroupTypeEnum] class_id: Optional[StrictInt] name: Optional[StrictStr] - required: bool = Field(default=False) + isRequired: bool = Field(default=False) attributes: Optional[List[Attribute]] default_value: Any diff --git a/tests/integration/classes/test_create_annotation_class.py b/tests/integration/classes/test_create_annotation_class.py index 0b3f43e15..2592f8984 100644 --- a/tests/integration/classes/test_create_annotation_class.py +++ b/tests/integration/classes/test_create_annotation_class.py @@ -44,7 +44,7 @@ def test_create_annotation_class_with_attr_and_default_value(self): attribute_groups=[ { "name": "test", - "required": False, + "isRequired:": False, "attributes": [{"name": "Car"}, {"name": "Track"}, {"name": "Bus"}], "default_value": "Bus", } diff --git a/tests/integration/export/test_export.py b/tests/integration/export/test_export.py index 073a3556f..04cb4c352 100644 --- a/tests/integration/export/test_export.py +++ b/tests/integration/export/test_export.py @@ -106,4 +106,4 @@ def test_upload_s3(self): Bucket=self.TEST_S3_BUCKET, Prefix=self.TMP_DIR ).get("Contents", []): files.append(object_data["Key"]) - self.assertEqual(33, len(files)) + self.assertEqual(25, len(files))