diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3d1ef2641..e241f1b8e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ History All release highlights of this project will be documented in this file. 4.4.15 - August 20, 2023 -_______________________ +________________________ **Added** @@ -15,7 +15,7 @@ _______________________ 4.4.14 - August 20, 2023 -_______________________ +________________________ **Added** diff --git a/requirements.txt b/requirements.txt index 2a336fed6..88a3562b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -pydantic~=1.10 +pydantic>=1.10,!=2.0.* aiohttp~=3.8 boto3~=1.26 -opencv-python~=4.7 +opencv-python-headless~=4.7 packaging~=23.1 plotly~=5.14 email-validator~=2.0 pandas~=1.3 ffmpeg-python~=0.2 -pillow~=9.5 +pillow>=9.5,~=10.0 tqdm~=4.66.1 requests~=2.31.0 aiofiles==23.1.0 diff --git a/setup.py b/setup.py index 12417a176..0ba57b185 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def get_version(): description="Python SDK to SuperAnnotate platform", license="MIT", author="SuperAnnotate AI", - author_email="suppoort@superannotate.com", + author_email="support@superannotate.com", url="https://github.com/superannotateai/superannotate-python-sdk", long_description=open("README.rst").read(), long_description_content_type="text/x-rst", diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 46887581e..6a022d65e 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -3,7 +3,7 @@ import sys -__version__ = "4.4.15" +__version__ = "4.4.16dev1" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 5725a58a0..b36c3e979 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -11,13 +11,14 @@ from typing import Sized import lib.core as constants -import pydantic from lib.app.interface.types import validate_arguments from lib.core import CONFIG from lib.core import setup_logging from lib.core.entities.base import ConfigEntity from lib.core.entities.base import TokenStr from lib.core.exceptions import AppException +from lib.core.pydantic_v1 import ErrorWrapper +from lib.core.pydantic_v1 import ValidationError from lib.infrastructure.controller import Controller from lib.infrastructure.utils import extract_project_folder from lib.infrastructure.validators import wrap_error @@ -61,7 +62,7 @@ def __init__(self, token: TokenStr = None, config_path: str = None): raise AppException( f"SuperAnnotate config file {constants.CONFIG_INI_FILE_LOCATION} not found." ) - except pydantic.ValidationError as e: + except ValidationError as e: raise AppException(wrap_error(e)) except KeyError: raise @@ -78,13 +79,9 @@ def _retrieve_configs_from_json(path: Path) -> typing.Union[ConfigEntity]: token = json_data["token"] try: config = ConfigEntity(SA_TOKEN=token) - except pydantic.ValidationError: - raise pydantic.ValidationError( - [ - pydantic.error_wrappers.ErrorWrapper( - ValueError("Invalid token."), loc="token" - ) - ], + except ValidationError: + raise ValidationError( + [ErrorWrapper(ValueError("Invalid token."), loc="token")], model=ConfigEntity, ) host = json_data.get("main_endpoint") diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 84dc20d87..a467bbe8e 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -15,7 +15,6 @@ from typing import TypeVar from typing import Union -import pydantic from typing_extensions import Literal if sys.version_info < (3, 11): @@ -24,10 +23,7 @@ from typing import TypedDict, NotRequired, Required # noqa import boto3 -from pydantic import conlist -from pydantic import constr -from pydantic import parse_obj_as -from pydantic.error_wrappers import ValidationError + from tqdm import tqdm import lib.core as constants @@ -61,16 +57,18 @@ from lib.core.types import MLModel from lib.core.types import PriorityScoreEntity from lib.core.types import Project +from lib.core.pydantic_v1 import ValidationError +from lib.core.pydantic_v1 import constr +from lib.core.pydantic_v1 import conlist +from lib.core.pydantic_v1 import parse_obj_as from lib.infrastructure.utils import extract_project_folder from lib.infrastructure.validators import wrap_error logger = logging.getLogger("sa") - NotEmptyStr = TypeVar("NotEmptyStr", bound=constr(strict=True, min_length=1)) - PROJECT_STATUS = Literal["NotStarted", "InProgress", "Completed", "OnHold"] PROJECT_TYPE = Literal[ @@ -1131,6 +1129,7 @@ def prepare_export( annotation_statuses: Optional[List[ANNOTATION_STATUS]] = None, include_fuse: Optional[bool] = False, only_pinned=False, + **kwargs, ): """Prepare annotations and classes.json for export. Original and fused images for images with annotations can be included with include_fuse flag. @@ -1147,6 +1146,8 @@ def prepare_export( :type include_fuse: bool :param only_pinned: enable only pinned output in export. This option disables all other types of output. :type only_pinned: bool + :param kwargs: Arbitrary kwarg ``integration_name`` + can be provided which will be used as a storage to store export file :return: metadata object of the prepared export :rtype: dict @@ -1165,12 +1166,22 @@ def prepare_export( constants.AnnotationStatus.COMPLETED.name, constants.AnnotationStatus.SKIPPED.name, ] + integration_name = kwargs.get("integration_name") + integration_id = None + if integration_name: + for integration in self.controller.integrations.list().data: + if integration.name == integration_name: + integration_id = integration.id + break + else: + raise AppException("Integration not found.") response = self.controller.prepare_export( project_name=project_name, folder_names=folders, include_fuse=include_fuse, only_pinned=only_pinned, annotation_statuses=annotation_statuses, + integration_id=integration_id, ) if response.errors: raise AppException(response.errors) @@ -2095,7 +2106,7 @@ def delete_annotations( :param project: project name or folder path (e.g., "project1/folder1") :type project: str - :param item_names: item names. If None, all the items in the specified directory will be deleted. + :param item_names: item names. If None, all the annotations in the specified directory will be deleted. :type item_names: list of strs """ @@ -2538,6 +2549,15 @@ def attach_items( :return: uploaded, failed and duplicated item names :rtype: tuple of list of strs + + Example: + :: + client = SAClient() + client.attach_items( + project = "Medical Annotations", + attachments = [{"name": "item", "url": "https://..."}] + ) + """ project_name, folder_name = extract_project_folder(project) @@ -2549,7 +2569,7 @@ def attach_items( for item, count in collections.Counter(attachments).items() if count > 1 ] - except pydantic.ValidationError: + except ValidationError: ( unique_attachments, duplicate_attachments, diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index 0f8faf8ef..29a563c1e 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -3,14 +3,14 @@ from lib.core.enums import BaseTitledEnum from lib.core.exceptions import AppException +from lib.core.pydantic_v1 import constr +from lib.core.pydantic_v1 import errors +from lib.core.pydantic_v1 import pydantic_validate_arguments +from lib.core.pydantic_v1 import PydanticTypeError +from lib.core.pydantic_v1 import StrictStr +from lib.core.pydantic_v1 import StrRegexError +from lib.core.pydantic_v1 import ValidationError from lib.infrastructure.validators import wrap_error -from pydantic import constr -from pydantic import errors -from pydantic import StrictStr -from pydantic import validate_arguments as pydantic_validate_arguments -from pydantic import ValidationError -from pydantic.errors import PydanticTypeError -from pydantic.errors import StrRegexError class EnumMemberError(PydanticTypeError): diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py index c7d650088..2bad17386 100644 --- a/src/superannotate/lib/app/serializers.py +++ b/src/superannotate/lib/app/serializers.py @@ -5,8 +5,8 @@ from typing import Union import superannotate.lib.core as constance -from pydantic import BaseModel from superannotate.lib.core.entities import BaseEntity +from superannotate.lib.core.pydantic_v1 import BaseModel class BaseSerializer(ABC): diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 86e87a41e..5eadfbc29 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -13,16 +13,16 @@ from lib.core import LOG_FILE_LOCATION from lib.core.enums import AnnotationStatus from lib.core.enums import BaseTitledEnum -from pydantic import BaseModel as PydanticBaseModel -from pydantic import Extra -from pydantic import Field -from pydantic import StrictStr -from pydantic.datetime_parse import parse_datetime -from pydantic.typing import is_namedtuple -from pydantic.utils import ROOT_KEY -from pydantic.utils import sequence_like -from pydantic.utils import ValueItems -from typing_extensions import Literal +from lib.core.pydantic_v1 import BaseModel +from lib.core.pydantic_v1 import Extra +from lib.core.pydantic_v1 import Field +from lib.core.pydantic_v1 import is_namedtuple +from lib.core.pydantic_v1 import Literal +from lib.core.pydantic_v1 import parse_datetime +from lib.core.pydantic_v1 import ROOT_KEY +from lib.core.pydantic_v1 import sequence_like +from lib.core.pydantic_v1 import StrictStr +from lib.core.pydantic_v1 import ValueItems DATE_TIME_FORMAT_ERROR_MESSAGE = ( "does not match expected format YYYY-MM-DDTHH:MM:SS.fffZ" @@ -37,7 +37,7 @@ _missing = object() -class BaseModel(PydanticBaseModel): +class BaseModel(BaseModel): """ Added new extra keys - use_enum_names: that's for BaseTitledEnum to use names instead of enum objects @@ -121,7 +121,7 @@ def _get_value( exclude_none: bool, ) -> Any: - if isinstance(v, PydanticBaseModel): + if isinstance(v, BaseModel): v_dict = v.dict( by_alias=by_alias, exclude_unset=exclude_unset, diff --git a/src/superannotate/lib/core/entities/classes.py b/src/superannotate/lib/core/entities/classes.py index decdd2862..24b204a9e 100644 --- a/src/superannotate/lib/core/entities/classes.py +++ b/src/superannotate/lib/core/entities/classes.py @@ -5,17 +5,16 @@ from typing import Optional from lib.core.entities.base import BaseModel +from lib.core.entities.base import parse_datetime from lib.core.enums import BaseTitledEnum from lib.core.enums import ClassTypeEnum -from pydantic import BaseModel as BasePydanticModel -from pydantic import Extra -from pydantic import Field -from pydantic import StrictInt -from pydantic import StrictStr -from pydantic import validator -from pydantic.color import Color -from pydantic.color import ColorType -from pydantic.datetime_parse import parse_datetime +from lib.core.pydantic_v1 import Color +from lib.core.pydantic_v1 import ColorType +from lib.core.pydantic_v1 import Extra +from lib.core.pydantic_v1 import Field +from lib.core.pydantic_v1 import StrictInt +from lib.core.pydantic_v1 import StrictStr +from lib.core.pydantic_v1 import validator DATE_REGEX = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d{3})Z" DATE_TIME_FORMAT_ERROR_MESSAGE = ( @@ -23,7 +22,7 @@ ) -class HexColor(BasePydanticModel): +class HexColor(BaseModel): __root__: ColorType @validator("__root__", pre=True) diff --git a/src/superannotate/lib/core/entities/folder.py b/src/superannotate/lib/core/entities/folder.py index 8b25c66f1..0e27d63f8 100644 --- a/src/superannotate/lib/core/entities/folder.py +++ b/src/superannotate/lib/core/entities/folder.py @@ -3,7 +3,7 @@ from lib.core.entities.base import TimedBaseModel from lib.core.enums import FolderStatus -from pydantic import Extra +from lib.core.pydantic_v1 import Extra class FolderEntity(TimedBaseModel): diff --git a/src/superannotate/lib/core/entities/integrations.py b/src/superannotate/lib/core/entities/integrations.py index 3e3fd9009..731bc76af 100644 --- a/src/superannotate/lib/core/entities/integrations.py +++ b/src/superannotate/lib/core/entities/integrations.py @@ -1,7 +1,12 @@ from lib.core.entities.base import TimedBaseModel from lib.core.enums import IntegrationTypeEnum -from pydantic import Extra -from pydantic import Field + +try: + from pydantic.v1 import Extra + from pydantic.v1 import Field +except ImportError: + from pydantic import Extra + from pydantic import Field class IntegrationEntity(TimedBaseModel): diff --git a/src/superannotate/lib/core/entities/items.py b/src/superannotate/lib/core/entities/items.py index d62c2b226..a7781636c 100644 --- a/src/superannotate/lib/core/entities/items.py +++ b/src/superannotate/lib/core/entities/items.py @@ -3,8 +3,8 @@ from lib.core.entities.base import BaseItemEntity from lib.core.enums import ApprovalStatus from lib.core.enums import SegmentationStatus -from pydantic import Extra -from pydantic import Field +from lib.core.pydantic_v1 import Extra +from lib.core.pydantic_v1 import Field class ImageEntity(BaseItemEntity): diff --git a/src/superannotate/lib/core/entities/project.py b/src/superannotate/lib/core/entities/project.py index 86cc2f74b..ccb286dc6 100644 --- a/src/superannotate/lib/core/entities/project.py +++ b/src/superannotate/lib/core/entities/project.py @@ -11,13 +11,13 @@ from lib.core.enums import ProjectStatus from lib.core.enums import ProjectType from lib.core.enums import UserRole -from pydantic import Extra -from pydantic import Field -from pydantic import StrictBool -from pydantic import StrictFloat -from pydantic import StrictInt -from pydantic import StrictStr -from pydantic.datetime_parse import parse_datetime +from lib.core.pydantic_v1 import Extra +from lib.core.pydantic_v1 import Field +from lib.core.pydantic_v1 import parse_datetime +from lib.core.pydantic_v1 import StrictBool +from lib.core.pydantic_v1 import StrictFloat +from lib.core.pydantic_v1 import StrictInt +from lib.core.pydantic_v1 import StrictStr class StringDate(datetime.datetime): diff --git a/src/superannotate/lib/core/plugin.py b/src/superannotate/lib/core/plugin.py index bd5354b6b..662e78417 100644 --- a/src/superannotate/lib/core/plugin.py +++ b/src/superannotate/lib/core/plugin.py @@ -82,7 +82,7 @@ def generate_thumb(self): thumbnail_size = (128, 96) background = Image.new("RGB", thumbnail_size, "black") - image.thumbnail(thumbnail_size, Image.ANTIALIAS) + image.thumbnail(thumbnail_size, Image.LANCZOS) (w, h) = image.size background.paste( image, ((thumbnail_size[0] - w) // 2, (thumbnail_size[1] - h) // 2) @@ -99,7 +99,7 @@ def generate_huge(self, base_width: int = 600) -> Tuple[io.BytesIO, float, float width, height = im.size buffer = io.BytesIO() h_size = int(height * base_width / width) - im.resize((base_width, h_size), Image.ANTIALIAS).convert("RGB").save( + im.resize((base_width, h_size), Image.LANCZOS).convert("RGB").save( buffer, "JPEG" ) buffer.seek(0) diff --git a/src/superannotate/lib/core/pydantic_v1.py b/src/superannotate/lib/core/pydantic_v1.py new file mode 100644 index 000000000..6a00a4114 --- /dev/null +++ b/src/superannotate/lib/core/pydantic_v1.py @@ -0,0 +1,36 @@ +from packaging.version import parse as parse_version +from pydantic import VERSION + +if parse_version(VERSION).major < 2: + import pydantic +else: + import pydantic.v1 as pydantic # noqa + +BaseModel = pydantic.BaseModel +Field = pydantic.Field +Extra = pydantic.Extra +ValidationError = pydantic.ValidationError +StrictStr = pydantic.StrictStr +StrictInt = pydantic.StrictInt +StrictBool = pydantic.StrictBool +StrictFloat = pydantic.StrictFloat +ErrorWrapper = pydantic.error_wrappers.ErrorWrapper +parse_obj_as = pydantic.parse_obj_as +is_namedtuple = pydantic.typing.is_namedtuple # noqa +Literal = pydantic.typing.Literal # noqa +ValueItems = pydantic.utils.ValueItems # noqa +ROOT_KEY = pydantic.utils.ROOT_KEY # noqa +sequence_like = pydantic.utils.sequence_like # noqa +validator = pydantic.validator # noqa +constr = pydantic.constr # noqa +conlist = pydantic.conlist # noqa +parse_datetime = pydantic.datetime_parse.parse_datetime # noqa +Color = pydantic.color.Color # noqa +ColorType = pydantic.color.ColorType # noqa +validators = pydantic.validators # noqa +WrongConstantError = pydantic.errors.WrongConstantError +errors = pydantic.errors +PydanticTypeError = pydantic.errors.PydanticTypeError +pydantic_validate_arguments = pydantic.validate_arguments +StrRegexError = pydantic.errors.StrRegexError +create_model_from_typeddict = pydantic.annotated_types.create_model_from_typeddict diff --git a/src/superannotate/lib/core/service_types.py b/src/superannotate/lib/core/service_types.py index 2d4299c83..ef0a41101 100644 --- a/src/superannotate/lib/core/service_types.py +++ b/src/superannotate/lib/core/service_types.py @@ -6,9 +6,9 @@ from lib.core import entities from lib.core.exceptions import AppException -from pydantic import BaseModel -from pydantic import Extra -from pydantic import Field +from lib.core.pydantic_v1 import BaseModel +from lib.core.pydantic_v1 import Extra +from lib.core.pydantic_v1 import Field class Limit(BaseModel): diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index 0df645bd4..35040571e 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -562,6 +562,7 @@ def prepare_export( annotation_statuses: List[str], include_fuse: bool, only_pinned: bool, + integration_id: int, ) -> ServiceResponse: raise NotImplementedError diff --git a/src/superannotate/lib/core/types.py b/src/superannotate/lib/core/types.py index d49f630b0..297a551b1 100644 --- a/src/superannotate/lib/core/types.py +++ b/src/superannotate/lib/core/types.py @@ -1,8 +1,8 @@ from typing import Optional -from pydantic import BaseModel -from pydantic import constr -from pydantic import Extra +from lib.core.pydantic_v1 import BaseModel +from lib.core.pydantic_v1 import constr +from lib.core.pydantic_v1 import Extra NotEmptyStr = constr(strict=True, min_length=1) diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index d1e194b13..6fa39da63 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -48,7 +48,11 @@ from lib.core.usecases.base import BaseReportableUseCase from lib.core.video_convertor import VideoFrameGenerator from lib.infrastructure.utils import divide_to_chunks -from pydantic import BaseModel + +try: + from pydantic.v1 import BaseModel +except ImportError: + from pydantic import BaseModel logger = logging.getLogger("sa") @@ -231,6 +235,8 @@ async def upload(_chunk: List[ItemToUpload]): await upload(chunk) chunk = [] _size = 0 + if not chunk: + queue.put_nowait(None) chunk.append(item_data) _size += item_data.file_size if chunk: diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index d9e718ad1..1d3859246 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -45,6 +45,7 @@ def __init__( include_fuse: bool, only_pinned: bool, annotation_statuses: List[str] = None, + integration_id: int = None, ): super().__init__(), self._project = project @@ -53,6 +54,7 @@ def __init__( self._annotation_statuses = annotation_statuses self._include_fuse = include_fuse self._only_pinned = only_pinned + self._integration_id = integration_id def validate_only_pinned(self): if ( @@ -107,6 +109,7 @@ def execute(self): annotation_statuses=self._annotation_statuses, include_fuse=self._include_fuse, only_pinned=self._only_pinned, + integration_id=self._integration_id, ) if not response.ok: raise AppException(response.error) diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index 650e289ab..cc99e9772 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -6,7 +6,11 @@ from typing import Optional from lib.core.enums import AnnotationTypes -from pydantic import BaseModel + +try: + from pydantic.v1 import BaseModel +except ImportError: + from pydantic import BaseModel class Annotation(BaseModel): diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 2374e64c0..82e139d36 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1011,6 +1011,7 @@ def prepare_export( include_fuse: bool, only_pinned: bool, annotation_statuses: List[str] = None, + integration_id: int = None, ): project = self.get_project(project_name) use_case = usecases.PrepareExportUseCase( @@ -1020,6 +1021,7 @@ def prepare_export( include_fuse=include_fuse, only_pinned=only_pinned, annotation_statuses=annotation_statuses, + integration_id=integration_id, ) return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/serviceprovider.py b/src/superannotate/lib/infrastructure/serviceprovider.py index 1a9ebd3b1..756efad48 100644 --- a/src/superannotate/lib/infrastructure/serviceprovider.py +++ b/src/superannotate/lib/infrastructure/serviceprovider.py @@ -151,6 +151,7 @@ def prepare_export( annotation_statuses: List[str], include_fuse: bool, only_pinned: bool, + integration_id: int = None, ): annotation_statuses = ",".join( [str(constants.AnnotationStatus.get_value(i)) for i in annotation_statuses] @@ -165,7 +166,8 @@ def prepare_export( } if folders: data["folder_names"] = folders - + if integration_id is not None: + data["integration_id"] = integration_id return self.client.request( self.URL_PREPARE_EXPORT, "post", diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py index 5e1d2d4bd..d16dd6661 100644 --- a/src/superannotate/lib/infrastructure/services/annotation.py +++ b/src/superannotate/lib/infrastructure/services/annotation.py @@ -18,7 +18,12 @@ from lib.core.service_types import UploadAnnotationsResponse from lib.core.serviceproviders import BaseAnnotationService from lib.infrastructure.stream_data_handler import StreamedAnnotations -from pydantic import parse_obj_as + +try: + from pydantic.v1 import parse_obj_as +except ImportError: + from pydantic import parse_obj_as + from superannotate.lib.infrastructure.services.http_client import AIOHttpSession logger = logging.getLogger("sa") @@ -90,7 +95,7 @@ async def _sync_large_annotation(self, team_id, project_id, item_id): synced = await session.get(sync_status_url, params=sync_params) synced = await synced.json() synced = synced["status"] - await asyncio.sleep(1) + await asyncio.sleep(5) return synced async def get_big_annotation( @@ -248,13 +253,13 @@ async def upload_small_annotations( folder: entities.FolderEntity, items_name_data_map: Dict[str, dict], ) -> UploadAnnotationsResponse: - url = urljoin( - self.assets_provider_url, - ( - f"{self.URL_UPLOAD_ANNOTATIONS}?{'&'.join(f'image_names[]={item_name}' for item_name in items_name_data_map.keys())}" - ), - ) - + params = [ + ("team_id", project.team_id), + ("project_id", project.id), + ("folder_id", folder.id), + *[("image_names[]", item_name) for item_name in items_name_data_map.keys()], + ] + url = urljoin(self.assets_provider_url, f"{self.URL_UPLOAD_ANNOTATIONS}") headers = copy.copy(self.client.default_headers) del headers["Content-Type"] async with AIOHttpSession( @@ -277,12 +282,6 @@ async def upload_small_annotations( filename=key, content_type="application/json", ) - - params = { - "team_id": project.team_id, - "project_id": project.id, - "folder_id": folder.id, - } _response = await session.request( "post", url, params=params, data=form_data ) diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py index 87fce653a..5935b4662 100644 --- a/src/superannotate/lib/infrastructure/services/http_client.py +++ b/src/superannotate/lib/infrastructure/services/http_client.py @@ -12,7 +12,6 @@ from typing import List import aiohttp -import pydantic import requests from lib.core.exceptions import AppException from lib.core.service_types import ServiceResponse @@ -21,12 +20,19 @@ from requests.adapters import Retry from superannotate import __version__ +try: + from pydantic.v1 import BaseModel + from pydantic.v1 import parse_obj_as +except ImportError: + from pydantic import BaseModel + from pydantic import parse_obj_as + logger = logging.getLogger("sa") class PydanticEncoder(json.JSONEncoder): def default(self, obj): - if isinstance(obj, pydantic.BaseModel): + if isinstance(obj, BaseModel): return json.loads(obj.json(exclude_none=True)) return json.JSONEncoder.default(self, obj) @@ -168,7 +174,7 @@ def paginate( if item_type: response = ServiceResponse( status=_response.status, - res_data=pydantic.parse_obj_as(List[item_type], total), + res_data=parse_obj_as(List[item_type], total), ) else: response = ServiceResponse( diff --git a/src/superannotate/lib/infrastructure/validators.py b/src/superannotate/lib/infrastructure/validators.py index 351ea0367..44cfbea9a 100644 --- a/src/superannotate/lib/infrastructure/validators.py +++ b/src/superannotate/lib/infrastructure/validators.py @@ -2,9 +2,9 @@ import typing from collections import defaultdict -from pydantic import ValidationError -from pydantic import validators -from pydantic.errors import WrongConstantError +from lib.core.pydantic_v1 import ValidationError +from lib.core.pydantic_v1 import validators +from lib.core.pydantic_v1 import WrongConstantError def wrong_constant_error(self): @@ -39,12 +39,14 @@ def make_typeddict_validator( """ Wrapping to ignore extra keys """ - from pydantic.annotated_types import create_model_from_typeddict - from pydantic import Extra + from lib.core.pydantic_v1 import Extra + from lib.core.pydantic_v1 import create_model_from_typeddict + + create_model_from_typeddict = create_model_from_typeddict config.extra = Extra.ignore - TypedDictModel = create_model_from_typeddict( + TypedDictModel = create_model_from_typeddict( # noqa typeddict_cls, __config__=config, __module__=typeddict_cls.__module__, diff --git a/tests/integration/annotations/test_large_annotations.py b/tests/integration/annotations/test_large_annotations.py index d1d5902d9..b7a7ea4c1 100644 --- a/tests/integration/annotations/test_large_annotations.py +++ b/tests/integration/annotations/test_large_annotations.py @@ -18,6 +18,10 @@ class TestAnnotationUploadVector(BaseTestCase): TEST_FOLDER_PATH = "data_set/sample_vector_annotations_with_tag_classes" TEST_4_FOLDER_PATH = "data_set/sample_project_vector" TEST_BIG_FOLDER_PATH = "sample_big_json_vector" + TEST_BIG_ANNOTATION_NAME = "aearth_mov_001.jpg" + TEST_BIG_ANNOTATION_PATH = os.path.join( + DATA_SET_PATH, f"sample_big_json_vector/{TEST_BIG_ANNOTATION_NAME}.json" + ) IMAGE_NAME = "example_image_1.jpg" @@ -37,6 +41,25 @@ def _get_annotations_from_folder(path): annotations.append(json.load(open(i))) return annotations + def test_large_annotation_upload(self): + sa.attach_items( + self.PROJECT_NAME, [{"name": self.TEST_BIG_ANNOTATION_NAME, "url": "url_"}] + ) + sa.create_annotation_classes_from_classes_json( + self.PROJECT_NAME, + f"{self.big_annotations_folder_path}/classes/classes.json", + ) + annotation = json.load(open(self.TEST_BIG_ANNOTATION_PATH)) + with self.assertLogs("sa", level="INFO") as cm: + uploaded, _, _ = sa.upload_annotations( + self.PROJECT_NAME, [annotation] + ).values() + assert ( + "INFO:sa:Uploading 1/1 annotations to the project Test-upload_annotations." + == cm.output[0] + ) + assert len(uploaded) == 1 + def test_large_annotations_upload_get_download(self): items_to_attach = [ {"name": f"aearth_mov_00{i}.jpg", "url": f"url_{i}"} for i in range(1, 6) diff --git a/tests/integration/projects/test_get_project_metadata.py b/tests/integration/projects/test_get_project_metadata.py index f53774dfd..1ac00d9b0 100644 --- a/tests/integration/projects/test_get_project_metadata.py +++ b/tests/integration/projects/test_get_project_metadata.py @@ -1,5 +1,4 @@ from src.superannotate import SAClient -from tests import compare_result from tests.integration.base import BaseTestCase sa = SAClient() @@ -25,24 +24,6 @@ class TestGetProjectMetadata(BaseTestCase): "root_folder_completed_items_count": None, } - def test_metadata_payload(self): - """ - checking item_count for project retrieve - and exclude in list - """ - project = sa.get_project_metadata(self.PROJECT_NAME) - assert project["item_count"] == 0 - self._attach_items(count=5) - sa.create_folder(self.PROJECT_NAME, "tmp") - self._attach_items(count=5, folder="tmp") - project = sa.get_project_metadata(self.PROJECT_NAME) - assert project["item_count"] == 10 - projects = sa.search_projects(name=self.PROJECT_NAME, return_metadata=True) - assert "item_count" not in projects[0] - assert compare_result( - projects[0], self.EXPECTED_PROJECT_METADATA, self.IGNORE_KEYS - ) - def test_get_project_by_id(self): project_metadata = sa.get_project_metadata(self.PROJECT_NAME) project_by_id = sa.get_project_by_id(project_metadata["id"]) diff --git a/tests/integration/test_video.py b/tests/integration/test_video.py index 8e950186e..f4a4826d5 100644 --- a/tests/integration/test_video.py +++ b/tests/integration/test_video.py @@ -48,7 +48,7 @@ def test_video_upload_from_folder(self): self.PROJECT_NAME, self.folder_path, target_fps=1 ) - sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NAME) + sa.create_folder(self.PROJECT_NAME, self.TEST_FOLDER_NeAME) sa.upload_videos_from_folder_to_project( f"{self.PROJECT_NAME}/{self.TEST_FOLDER_NAME}", self.folder_path, diff --git a/tests/unit/test_classes_serialization.py b/tests/unit/test_classes_serialization.py index f5e5ca3b1..86ee7437f 100644 --- a/tests/unit/test_classes_serialization.py +++ b/tests/unit/test_classes_serialization.py @@ -3,8 +3,8 @@ from typing import List from unittest import TestCase -from pydantic import parse_obj_as -from pydantic import ValidationError +from pydantic.v1 import parse_obj_as +from pydantic.v1 import ValidationError from superannotate.lib.app.serializers import BaseSerializer from superannotate.lib.core.entities.classes import AnnotationClassEntity from superannotate.lib.core.entities.classes import AttributeGroup