diff --git a/CHANGELOG.md b/CHANGELOG.md index 25103e5a1..72901cf20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog All release highlights of this project will be documented in this file. + +## 4.4.6 - November 23, 2022 +### Updated +- `SAClient.aggregate_annotations_as_df` method to aggregate "comment" type instances. +- `SAClient.add_annotation_bbox_to_image`, `SAClient.add_annotation_point_to_image`, `SAClient.add_annotation_comment_to_image` _methods_ to add deprecation warnings. +### Fixed +- Special characters are being encoded after annotation upload (Windows) +- `SAClient.assign_folder` _method_ to address the invalid argument name. +- `SAClient.upload_images_from_folder_to_project` _method_ to address uploading of more than 500 items. +- `SAClient.upload_annotations_from_folder_to_project` _method_ to address the issue of a folder size being more than 25,5 MB. +- `SAClient.download_image` _method_ to address the KeyError 'id' when `include_annotations` is set to `True`. +### Removed +`SAClient.upload_preannotations_from_folder_to_project` _method_ +`SAClient.copy_image` _method_ +### ## 4.4.5 - October 23, 2022 ### Added - `SAClient.add_items_to_subset` _method_ to associate given items with a Subset. diff --git a/README.md b/README.md deleted file mode 100644 index a548dee90..000000000 --- a/README.md +++ /dev/null @@ -1,62 +0,0 @@ - - -# SuperAnnotate Python SDK - -SuperAnnotate Python SDK allows access to the platform without - web browser: - -```python -import superannotate as sa - -sa.create_project("Example Project 1", "example", "Vector") - -sa.upload_images_from_folder_to_project("Example Project 1", "") -``` - -## Installation - -SDK is available on PyPI: - -```console -pip install superannotate -``` - -The package officially supports Python 3.6+ and was tested under Linux and -Windows ([Anaconda](https://www.anaconda.com/products/individual#windows)) platforms. - -For more detailed installation steps and package usage please have a look at the -[tutorial](https://superannotate.readthedocs.io/en/stable/tutorial.sdk.html). - -## Supported Features - -- Search projects -- Create/delete a project -- Upload images to a project from a local or AWS S3 folder -- Upload videos to a project from a local folder -- Upload annotations/pre-annotations to a project from local or AWS S3 folder -- Set the annotation status of the images being uploaded -- Export annotations from a project to a local or AWS S3 folder -- Share and unshare a project with a team contributor -- Invite a team contributor -- Search images in a project -- Download a single image -- Copy/move image between projects -- Get image bytes (e.g., for numpy array creation) -- Set image annotation status -- Download image annotations/pre-annotations -- Create/download project annotation classes -- Convert annotation format from/to COCO -- Convert annotation format from VOC, SuperVisely, LabelBox, DataLoop, VGG, VoTT, SageMaker, GoogleCloud, YOLO -- Add annotations to images on platform -- Add annotations to local SuperAnnotate format JSONs -- CLI commands for simple tasks - -## Full SDK reference, tutorial available on [Read the Docs](https://superannotate.readthedocs.io) - -## License - -This SDK is distributed under the MIT License, see [LICENSE](./LICENSE). - -## Questions and Issues - -For questions and issues please use this repo's issue tracker on GitHub. diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..d95540dba --- /dev/null +++ b/README.rst @@ -0,0 +1,128 @@ +SuperAnnotate Python SDK +=============================== +|Python| |License| |Changelog| + + +Welcome to the SuperAnnotate Python Software Development Kit (SDK), which enables Python programmers to create software that incorporates services of the platform and effortlessly integrates SuperAnnotate into their AI process. + +.. |Python| image:: https://img.shields.io/static/v1?label=python&message=3.7/3.8/3.9/3.10/3.11&color=blue&style=flat-square + :target: https://pypi.org/project/superannotate/ + :alt: Python Versions +.. |License| image:: https://img.shields.io/static/v1?label=license&message=MIT&color=green&style=flat-square + :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/LICENSE/ + :alt: License +.. |Changelog| image:: https://img.shields.io/static/v1?label=change&message=log&color=yellow&style=flat-square + :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/CHANGELOG.md + :alt: Changelog + +Resources +--------------- + +- API Reference and User Guide available on `Read the Docs `__ +- `Platform documentation `__ + + +Authentication +--------------- + +.. code-block:: python + + >>> from superannotate import SAClient + # by environment variable SA_TOKEN + >>> sa_client = SAClient() + # by token + >>> sa_client = SAClient(token='') + # by config file + # default path is ~/.superannotate/config.json + >>> sa_client = SAClient(config_path='~/.superannotate/dev_config.json') + +Using superannotate +------------------- + +.. code-block:: python + + >>> from superannotate import SAClient + >>> sa_client =SAClient() + >>> project = 'Dogs' + >>> sa_client.create_project( + project_name=project, + project_description='Test project generated via SDK', + project_type='Vector' + ) + >>> sa_client.create_annotation_class( + project=project, + name='dog', + color='#F9E0FA', + class_type='tag' + ) + >>> sa_client.attach_items( + project=project, + attachments=[ + { + 'url': 'https://drive.google.com/uc?export=download&id=1ipOrZNSTlPUkI_hnrW9aUD5yULqqq5Vl', + 'name': 'dog.jpeg' + } + ] + ) + >>> sa_client.upload_annotations( + project=project, + annotations=[ + { + 'metadata': {'name': 'dog.jpeg'}, + 'instances': [ + {'type': 'tag', 'className': 'dog'} + ] + } + ] + ) + >>> sa_client.get_annotations(project=project, items=['dog.jpeg']) + +Installation +------------ + +SuperAnnotate python SDK is available on PyPI: + +.. code-block:: bash + + pip install superannotate + + +The package officially supports Python 3.7+ and was tested under Linux and +Windows (`Anaconda `__ +) platforms. + +For more detailed installation steps and package usage please have a look at the `tutorial `__ + + +Supported Features +------------------ + +- search/get/create/clone/update/delete projects +- search/get/create/delete folders +- assign folders to project contributors +- upload items to a project from a local or AWS S3 folder +- attach items by URL or from an integrated storage, meanwhile keeping them secure in your cloud provider +- get integrated cloud storages +- upload annotations (also from local or AWS S3 folder) +- delete annotations +- set items annotations statuses +- get/download/export annotations from a project (also to a local or AWS S3 folder) +- invite/search team contributors or add contributors to a specific project +- search/get/copy/move items in a project +- query items using SA Query Language +- define custom metadata for items and upload custom values (query based on your custom metadata) +- upload priority scores +- get available subsets (sets of segregated items), query items in a subset or add items to a subset +- assign or anassign items to project contributors +- download an image that has been uploaded to project +- search/create/download/delete project annotation classes +- search/download models +- run predictions +- convert annotations from/to COCO format +- convert annotation from VOC, SuperVisely, LabelBox, DataLoop, VGG, VoTT, SageMaker, GoogleCloud, YOLO formats +- CLI commands for simple tasks + +Questions and Issues +-------------------- + +For questions and issues please use this repo’s issue tracker on GitHub or contact support@superannotate.com. diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst index fd4e7fac6..53b626f4d 100644 --- a/docs/source/superannotate.sdk.rst +++ b/docs/source/superannotate.sdk.rst @@ -42,7 +42,6 @@ ________ .. automethod:: superannotate.SAClient.upload_videos_from_folder_to_project .. _ref_upload_annotations_from_folder_to_project: .. automethod:: superannotate.SAClient.upload_annotations_from_folder_to_project -.. automethod:: superannotate.SAClient.upload_preannotations_from_folder_to_project .. automethod:: superannotate.SAClient.add_contributors_to_project .. automethod:: superannotate.SAClient.get_project_settings .. automethod:: superannotate.SAClient.set_project_default_image_quality_in_editor @@ -107,7 +106,6 @@ ______ .. automethod:: superannotate.SAClient.download_image .. automethod:: superannotate.SAClient.download_image_annotations .. automethod:: superannotate.SAClient.upload_image_annotations -.. automethod:: superannotate.SAClient.copy_image .. automethod:: superannotate.SAClient.pin_image .. automethod:: superannotate.SAClient.add_annotation_bbox_to_image .. automethod:: superannotate.SAClient.add_annotation_point_to_image diff --git a/setup.py b/setup.py index 7094269e1..f7da21cc2 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,6 @@ def get_version(): with open("requirements.txt") as f: requirements.extend(f.read().splitlines()) -with open('README.md') as f: - readme = f.read() - -readme = "\n".join(readme.split('\n')[2:]) setup( name='superannotate', @@ -31,12 +27,12 @@ def get_version(): description='Python SDK to SuperAnnotate platform', license='MIT', author='SuperAnnotate AI', + author_email='suppoort@superannotate.com', url='https://github.com/superannotateai/superannotate-python-sdk', - long_description=readme, - long_description_content_type='text/markdown', + long_description=open('README.rst').read(), + long_description_content_type='text/x-rst', install_requires=requirements, setup_requires=['wheel'], - description_file="README.md", entry_points={ 'console_scripts': ['superannotatecli = superannotate.lib.app.bin.superannotate:main'] }, diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 6dc56ced2..e441f04f4 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -2,7 +2,7 @@ import sys -__version__ = "4.4.5" +__version__ = "4.4.6b2" sys.path.append(os.path.split(os.path.realpath(__file__))[0]) diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py index 4e241bf97..ec9ccdad5 100644 --- a/src/superannotate/lib/app/analytics/aggregators.py +++ b/src/superannotate/lib/app/analytics/aggregators.py @@ -91,6 +91,7 @@ class DataAggregator: "polygon": lambda annotation: annotation["points"], "polyline": lambda annotation: annotation["points"], "cuboid": lambda annotation: annotation["points"], + "comment": lambda annotation: annotation["points"], "point": lambda annotation: {"x": annotation["x"], "y": annotation["y"]}, "annotation_type": lambda annotation: dict( cx=annotation["cx"], @@ -123,7 +124,7 @@ def annotation_suffix(self): self._annotation_suffix = PIXEL_ANNOTATION_POSTFIX else: self._annotation_suffix = ATTACHED_VIDEO_ANNOTATION_POSTFIX - return self._annotation_suffix + return ATTACHED_VIDEO_ANNOTATION_POSTFIX def get_annotation_paths(self): annotations_paths = [] @@ -378,7 +379,7 @@ def __append_annotation(annotation_dict): for annotation_path in annotations_paths: annotation_json = json.load(open(annotation_path)) - parts = annotation_path.name.split(self.annotation_suffix) + parts = Path(annotation_path).name.split(self.annotation_suffix) if len(parts) != 2: continue image_name = parts[0] @@ -449,8 +450,8 @@ def __append_annotation(annotation_dict): attributes = annotation.get("attributes") user_metadata = self.__get_user_metadata(annotation) folder_name = None - if annotation_path.parent != Path(self.project_root): - folder_name = annotation_path.parent.name + if Path(annotation_path).parent != Path(self.project_root): + folder_name = Path(annotation_path).parent.name num_added = 0 if not attributes: annotation_dict = { diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index 7fe0ffec0..410e3c506 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -129,27 +129,6 @@ def export_project( ) sys.exit(0) - def upload_preannotations( - self, project, folder, dataset_name=None, task=None, format=None - ): - """ - To upload preannotations from folder to project use - Optional argument format accepts input annotation format. It can have COCO or SuperAnnotate values. - If the argument is not given then SuperAnnotate (the native annotation format) is assumed. - Only when COCO format is specified dataset-name and task arguments are required. - dataset-name specifies JSON filename (without extension) in . - task specifies the COCO task for conversion. Please see import_annotation_format for more details. - """ - self._upload_annotations( - project=project, - folder=folder, - format=format, - dataset_name=dataset_name, - task=task, - pre=True, - ) - sys.exit(0) - def upload_annotations( self, project, folder, dataset_name=None, task=None, format=None ): @@ -167,13 +146,10 @@ def upload_annotations( format=format, dataset_name=dataset_name, task=task, - pre=False, ) sys.exit(0) - def _upload_annotations( - self, project, folder, format, dataset_name, task, pre=True - ): + def _upload_annotations(self, project, folder, format, dataset_name, task): project_folder_name = project project_name, folder_name = split_project_path(project) project = SAClient().controller.get_project(project_name) @@ -197,14 +173,10 @@ def _upload_annotations( task=task, ) annotations_path = temp_dir - if pre: - SAClient().upload_preannotations_from_folder_to_project( - project_folder_name, annotations_path - ) - else: - SAClient().upload_annotations_from_folder_to_project( - project_folder_name, annotations_path - ) + + SAClient().upload_annotations_from_folder_to_project( + project_folder_name, annotations_path + ) sys.exit(0) def attach_image_urls( diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index e8d0a620f..57b9f2dd5 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -437,112 +437,6 @@ def search_folders( ] return [folder.name for folder in data if not folder.is_root] - def copy_image( - self, - source_project: Union[NotEmptyStr, dict], - image_name: NotEmptyStr, - destination_project: Union[NotEmptyStr, dict], - include_annotations: Optional[StrictBool] = False, - copy_annotation_status: Optional[StrictBool] = False, - copy_pin: Optional[StrictBool] = False, - ): - """Copy image to a project. The image's project is the same as destination - project then the name will be changed to _()., - where is the next available number deducted from project image list. - - :param source_project: project name plus optional subfolder in the project (e.g., "project1/folder1") or - metadata of the project of source project - :type source_project: str or dict - :param image_name: image name - :type image_name: str - :param destination_project: project name or metadata of the project of destination project - :type destination_project: str or dict - :param include_annotations: enables annotations copy - :type include_annotations: bool - :param copy_annotation_status: enables annotations status copy - :type copy_annotation_status: bool - :param copy_pin: enables image pin status copy - :type copy_pin: bool - """ - warning_msg = "The SAClient.copy_image method will be deprecated with the Superannotate Python SDK 4.4.6 release" - warnings.warn(warning_msg, DeprecationWarning) - logger.warning(warning_msg) - source_project_name, source_folder_name = extract_project_folder(source_project) - destination_project_name, destination_folder_name = extract_project_folder( - destination_project - ) - source_project_metadata = self.controller.projects.get_by_name( - source_project_name - ).data - destination_project_metadata = self.controller.projects.get_by_name( - destination_project_name - ).data - - if destination_project_metadata.type.value in [ - constants.ProjectType.VIDEO.value, - constants.ProjectType.DOCUMENT.value, - ] or source_project_metadata.type.value in [ - constants.ProjectType.VIDEO.value, - constants.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[source_project_metadata.type]) - - response = self.controller.copy_image( - from_project_name=source_project_name, - from_folder_name=source_folder_name, - to_project_name=destination_project_name, - to_folder_name=destination_folder_name, - image_name=image_name, - copy_annotation_status=copy_annotation_status, - ) - if response.errors: - raise AppException(response.errors) - if copy_pin: - destination_project = self.controller.get_project( - destination_project_metadata - ) - _folder = self.controller.get_folder( - destination_project, destination_folder_name - ) - item = self.controller.items.get_by_name( - destination_project_metadata, _folder, image_name - ).data - item.is_pinned = 1 - self.controller.items.update( - project=destination_project_metadata, - folder=_folder, - image_name=image_name, - is_pinned=1, - ) - if include_annotations: - source_project = self.controller.get_project(source_project_name) - source_folder = self.controller.get_folder( - source_project, source_folder_name - ) - source_image = self.controller.items.get_by_name( - source_project, source_folder, image_name - ).data - destination_project = self.controller.get_project(destination_project) - destination_folder = self.controller.get_folder( - destination_project, destination_folder_name - ) - destination_image = self.controller.items.get_by_name( - destination_project, destination_folder, image_name - ).data - self.controller.annotation_classes.copy_multiple( - source_project=source_project, - source_folder=source_folder, - source_item=source_image, - destination_project=destination_project, - destination_folder=destination_folder, - destination_item=destination_image, - ) - - logger.info( - f"Copied image {source_project}/{image_name}" - f" to {destination_project_name}/{destination_folder_name}." - ) - def get_project_metadata( self, project: Union[NotEmptyStr, dict], @@ -795,13 +689,18 @@ def assign_folder( :type users: list of str """ - contributors = ( - self.controller.projects.get_by_name( - project_name=project_name, include_contributors=True - ) - .data["project"] - .users + response = self.controller.projects.get_by_name(name=project_name) + if response.errors: + raise AppException(response.errors) + project = response.data + response = self.controller.projects.get_metadata( + project=project, include_contributors=True ) + + if response.errors: + raise AppException(response.errors) + + contributors = response.data.users verified_users = [i["user_id"] for i in contributors] verified_users = set(users).intersection(set(verified_users)) unverified_contributor = set(users) - verified_users @@ -1383,7 +1282,7 @@ def create_annotation_classes_from_classes_json( file.seek(0) data = file else: - data = open(classes_json) + data = open(classes_json, encoding="utf-8") classes_json = json.load(data) try: annotation_classes = parse_obj_as(List[AnnotationClassEntity], classes_json) @@ -1616,79 +1515,6 @@ def upload_annotations_from_folder_to_project( raise AppException(response.errors) return response.data - def upload_preannotations_from_folder_to_project( - self, - project: Union[NotEmptyStr, dict], - folder_path: Union[str, Path], - from_s3_bucket=None, - recursive_subfolders: Optional[StrictBool] = False, - ): - """Finds and uploads all JSON files in the folder_path as pre-annotations to the project. - - The JSON files should follow specific naming convention. For Vector - projects they should be named "___objects.json" (e.g., if - image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects - JSON file should be named "___pixel.json" and also second mask - image file should be present with the name "___save.png". In both cases - image with should be already present on the platform. - - Existing pre-annotations will be overwritten. - - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param folder_path: from which folder to upload the pre-annotations - :type folder_path: Path-like (str or Path) - :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem - :type from_s3_bucket: str - :param recursive_subfolders: enable recursive subfolder parsing - :type recursive_subfolders: bool - - :return: paths to pre-annotations uploaded and could-not-upload - :rtype: tuple of list of strs - """ - warning_msg = ( - "The SAClient.upload_preannotations_from_folder_to_project" - " method will be deprecated with the Superannotate Python SDK 4.4.6 release" - ) - warnings.warn(warning_msg, DeprecationWarning) - logger.warning(warning_msg) - project_name, folder_name = extract_project_folder(project) - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - project = self.controller.get_project(project_name) - if project.type in [ - constants.ProjectType.VIDEO, - constants.ProjectType.DOCUMENT, - ]: - raise AppException(LIMITED_FUNCTIONS[project.type]) - if recursive_subfolders: - logger.info( - "When using recursive subfolder parsing same name annotations in different " - "subfolders will overwrite each other.", - ) - logger.info( - "The JSON files should follow a specific naming convention, matching file names already present " - "on the platform. Existing annotations will be overwritten" - ) - annotation_paths = get_annotation_paths( - folder_path, from_s3_bucket, recursive_subfolders - ) - logger.info( - f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}." - ) - project, folder = self.controller.get_project_folder(project_name, folder_name) - response = self.controller.annotations.upload_from_folder( - project=project, - folder=folder, - team=self.controller.team, - annotation_paths=annotation_paths, # noqa: E203 - client_s3_bucket=from_s3_bucket, - folder_path=folder_path, - is_pre_annotations=True, - ) - if response.errors: - raise AppException(response.errors) - return response.data - def upload_image_annotations( self, project: Union[NotEmptyStr, dict], @@ -1971,6 +1797,12 @@ def add_annotation_bbox_to_image( :param error: if not None, marks annotation as error (True) or no-error (False) :type error: bool """ + warning_msg = ( + "The SAClient.add_annotation_bbox_to_image method will " + "be deprecated with the Superannotate Python SDK 4.4.7 release" + ) + warnings.warn(warning_msg, DeprecationWarning) + logger.warning(warning_msg) project_name, folder_name = extract_project_folder(project) project = self.controller.get_project(project_name) @@ -2035,6 +1867,12 @@ def add_annotation_point_to_image( :param error: if not None, marks annotation as error (True) or no-error (False) :type error: bool """ + warning_msg = ( + "The SAClient.add_annotation_point_to_image method will " + "be deprecated with the Superannotate Python SDK 4.4.7 release" + ) + warnings.warn(warning_msg, DeprecationWarning) + logger.warning(warning_msg) project, folder = self.controller.get_project_folder_by_path(project) if project.type in [ constants.ProjectType.VIDEO, @@ -2091,6 +1929,12 @@ def add_annotation_comment_to_image( :param resolved: comment resolve status :type resolved: bool """ + warning_msg = ( + "The SAClient.add_annotation_comment_to_image method will " + "be deprecated with the Superannotate Python SDK 4.4.7 release" + ) + warnings.warn(warning_msg, DeprecationWarning) + logger.warning(warning_msg) project_name, folder_name = extract_project_folder(project) project = self.controller.projects.get_by_name(project_name).data if project.type in [ diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py index d0290e6dc..1b46a188f 100644 --- a/src/superannotate/lib/core/usecases/annotations.py +++ b/src/superannotate/lib/core/usecases/annotations.py @@ -409,6 +409,7 @@ def execute(self): class UploadAnnotationsFromFolderUseCase(BaseReportableUseCase): MAX_WORKERS = 16 CHUNK_SIZE = 100 + CHUNK_SIZE_PATHS = 500 CHUNK_SIZE_MB = 10 * 1024 * 1024 STATUS_CHANGE_CHUNK_SIZE = 100 AUTH_DATA_CHUNK_SIZE = 500 @@ -532,7 +533,7 @@ async def get_annotation( if self._project.type == constants.ProjectType.PIXEL.value: mask = self.get_annotation_from_s3(self._client_s3_bucket, mask_path) else: - async with aiofiles.open(path) as file: + async with aiofiles.open(path, encoding="utf-8") as file: content = await file.read() if self._project.type == constants.ProjectType.PIXEL.value: async with aiofiles.open(mask_path, "rb") as mask: @@ -579,14 +580,26 @@ def get_existing_name_item_mapping( @property def annotation_upload_data(self) -> UploadAnnotationAuthData: - if not self._annotation_upload_data: - response = self._service_provider.get_annotation_upload_data( + CHUNK_SIZE = UploadAnnotationsFromFolderUseCase.CHUNK_SIZE_PATHS + + if self._annotation_upload_data: + return self._annotation_upload_data + + images = {} + for i in range(0, len(self._item_ids), CHUNK_SIZE): + tmp = self._service_provider.get_annotation_upload_data( project=self._project, folder=self._folder, - item_ids=self._item_ids, + item_ids=self._item_ids[i : i + CHUNK_SIZE], ) - if response.ok: - self._annotation_upload_data = response.data + if not tmp.ok: + raise AppException(tmp.errors) + else: + images.update(tmp.data.images) + + self._annotation_upload_data = tmp.data + self._annotation_upload_data.images = images + return self._annotation_upload_data @property @@ -835,7 +848,9 @@ def _get_annotation_json(self) -> tuple: ), ) else: - annotation_json = json.load(open(self._annotation_path)) + annotation_json = json.load( + open(self._annotation_path, encoding="utf-8") + ) if self._project.type == constants.ProjectType.PIXEL.value: mask = open( self._annotation_path.replace( @@ -1313,7 +1328,7 @@ def download_annotation_classes(self, path: str): if response.ok: classes_path = Path(path) / "classes" classes_path.mkdir(parents=True, exist_ok=True) - with open(classes_path / "classes.json", "w+") as file: + with open(classes_path / "classes.json", "w+", encoding="utf-8") as file: json.dump( [ i.dict( diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index d7d978e6e..ee76fb3c3 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -849,6 +849,7 @@ def execute(self) -> Response: class UploadImagesToProject(BaseInteractiveUseCase): MAX_WORKERS = 10 + LIST_NAME_CHUNK_SIZE = 500 def __init__( self, @@ -1023,6 +1024,7 @@ def filter_paths(self, paths: List[str]): for path in paths: name_path_map[Path(path).name].append(path) + CHUNK_SIZE = UploadImagesToProject.LIST_NAME_CHUNK_SIZE filtered_paths = [] duplicated_paths = [] for file_name in name_path_map: @@ -1030,17 +1032,22 @@ def filter_paths(self, paths: List[str]): duplicated_paths.append(name_path_map[file_name][1:]) filtered_paths.append(name_path_map[file_name][0]) - response = self._service_provider.items.list_by_names( - project=self._project, - folder=self._folder, - names=[image.split("/")[-1] for image in filtered_paths], - ) - if not response.ok: - self._response.errors = AppException(response.error) - return self._response + image_list = [] + for i in range(0, len(filtered_paths), CHUNK_SIZE): + response = self._service_provider.items.list_by_names( + project=self._project, + folder=self._folder, + names=[ + image.split("/")[-1] for image in filtered_paths[i : i + CHUNK_SIZE] + ], + ) + + if not response.ok: + raise AppException(response.error) + image_list.extend([image.name for image in response.data]) + image_list = set(image_list) images_to_upload = [] - image_list = [image.name for image in response.data] for path in filtered_paths: if Path(path).name not in image_list: @@ -1287,149 +1294,6 @@ def execute(self): return self._response -class CopyImageUseCase(BaseUseCase): - def __init__( - self, - from_project: ProjectEntity, - from_folder: FolderEntity, - image_name: str, - to_project: ProjectEntity, - to_folder: FolderEntity, - service_provider: BaseServiceProvider, - s3_repo, - include_annotations: Optional[bool] = True, - copy_annotation_status: Optional[bool] = True, - copy_pin: Optional[bool] = True, - move=False, - ): - super().__init__() - self._from_project = from_project - self._from_folder = from_folder - self._image_name = image_name - self._to_project = to_project - self._to_folder = to_folder - self._s3_repo = s3_repo - self._include_annotations = include_annotations - self._copy_annotation_status = copy_annotation_status - self._copy_pin = copy_pin - self._service_provider = service_provider - self._move = move - - def validate_copy_path(self): - if ( - self._from_project.name == self._to_project.name - and self._from_folder.name == self._to_folder.name - ): - raise AppValidationException( - "Cannot move image if source_project == destination_project." - ) - - def validate_project_type(self): - if self._from_project.type in ( - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ): - raise AppValidationException( - constances.LIMITED_FUNCTIONS[self._from_project.type] - ) - - def validate_limitations(self): - response = self._service_provider.get_limitations( - project=self._to_project, - folder=self._to_folder, - ) - if not response.ok: - raise AppValidationException(response.error) - - if self._move and self._from_project.id == self._to_project.id: - if self._from_folder.id == self._to_folder.id: - raise AppValidationException( - "Cannot move image if source_project == destination_project." - ) - - if response.data.folder_limit.remaining_image_count < 1: - raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE) - if response.data.project_limit.remaining_image_count < 1: - raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE) - if ( - response.data.user_limit - and response.data.user_limit.remaining_image_count < 1 - ): - raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE) - - @property - def s3_repo(self): - self._auth_data = self._service_provider.get_s3_upload_auth_token( - project=self._to_project, folder=self._to_folder - ).data - if "error" in self._auth_data: - raise AppException(self._auth_data.get("error")) - return self._s3_repo( - self._auth_data["accessKeyId"], - self._auth_data["secretAccessKey"], - self._auth_data["sessionToken"], - self._auth_data["bucket"], - ) - - def execute(self) -> Response: - if self.is_valid(): - image = ( - GetImageUseCase( - project=self._from_project, - folder=self._from_folder, - image_name=self._image_name, - service_provider=self._service_provider, - ) - .execute() - .data - ) - - image_bytes = ( - GetImageBytesUseCase( - project=self._from_project, - folder=self._from_folder, - image=image, - service_provider=self._service_provider, - ) - .execute() - .data - ) - image_path = f"{self._to_folder}/{self._image_name}" - - auth_data = self._service_provider.get_s3_upload_auth_token( - project=self._to_project, folder=self._to_folder - ) - if not auth_data.ok: - raise AppException(auth_data.error) - s3_response = UploadImageS3UseCase( - project=self._to_project, - image_path=image_path, - image=image_bytes, - service_provider=self._service_provider, - upload_path=auth_data.data["filePath"], - s3_repo=self.s3_repo, - ).execute() - if s3_response.errors: - raise AppException(s3_response.errors) - image_entity = s3_response.data - del image_bytes - - attach_response = AttachFileUrlsUseCase( - project=self._to_project, - folder=self._to_folder, - attachments=[image_entity], - service_provider=self._service_provider, - annotation_status=image.annotation_status_code - if self._copy_annotation_status - else None, - upload_state_code=constances.UploadState.BASIC.value, - ).execute() - if attach_response.errors: - raise AppException(attach_response.errors) - self._response.data = image_entity - return self._response - - class DeleteAnnotations(BaseUseCase): POLL_AWAIT_TIME = 2 CHUNK_SIZE = 2000 @@ -1593,15 +1457,14 @@ def fill_classes_data(self, annotations: dict): attribute["groupName"] = annotation_class["attribute_groups"][ attribute["groupId"] ]["name"] - if attribute["id"] not in list( + if attribute.get("id") in list( annotation_class["attribute_groups"][attribute["groupId"]][ "attributes" ].keys() ): - continue - attribute["name"] = annotation_class["attribute_groups"][ - attribute["groupId"] - ]["attributes"][attribute["id"]] + attribute["name"] = annotation_class["attribute_groups"][ + attribute["groupId"] + ]["attributes"][attribute["id"]] def execute(self): if self.is_valid(): diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py index ca179a2a3..758478f7e 100644 --- a/src/superannotate/lib/core/video_convertor.py +++ b/src/superannotate/lib/core/video_convertor.py @@ -178,6 +178,8 @@ def _process(self): for instance in self._annotation_data["instances"]: instance_id = next(self.id_generator) annotation_type = instance["meta"]["type"] + if annotation_type == "comment": + continue class_name = instance["meta"].get("className") class_id = instance["meta"].get("classId", -1) for parameter in instance.get("parameters", []): diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 2a89894db..61eb814ae 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -761,7 +761,7 @@ def __init__(self, token: str, host: str, ssl_verify: bool, version: str): self._user_id = None self._reporter = None - http_client = HttpClient(api_url=host, token=token) + http_client = HttpClient(api_url=host, token=token, verify_ssl=ssl_verify) self.service_provider = ServiceProvider(http_client) self._team = self.get_team().data @@ -997,32 +997,6 @@ def update(self, project: ProjectEntity, folder: FolderEntity): ) return use_case.execute() - def copy_image( - self, - from_project_name: str, - from_folder_name: str, - to_project_name: str, - to_folder_name: str, - image_name: str, - copy_annotation_status: bool = False, - move: bool = False, - ): - from_project = self.get_project(from_project_name) - to_project = self.get_project(to_project_name) - to_folder = self.get_folder(to_project, to_folder_name) - use_case = usecases.CopyImageUseCase( - from_project=from_project, - from_folder=self.get_folder(from_project, from_folder_name), - to_project=to_project, - to_folder=to_folder, - service_provider=self.service_provider, - image_name=image_name, - s3_repo=self.s3_repo, - copy_annotation_status=copy_annotation_status, - move=move, - ) - return use_case.execute() - def un_assign_folder(self, project_name: str, folder_name: str): project_entity = self.get_project(project_name) folder = self.get_folder(project_entity, folder_name) diff --git a/tests/integration/annotations/test_annotations_upload_status_change.py b/tests/integration/annotations/test_annotations_upload_status_change.py index 3be566e4d..0637beab3 100644 --- a/tests/integration/annotations/test_annotations_upload_status_change.py +++ b/tests/integration/annotations/test_annotations_upload_status_change.py @@ -38,18 +38,6 @@ def test_upload_annotations_from_folder_to_project__upload_status(self, reporter sa.get_item_metadata(self.PROJECT_NAME, self.IMAGE_NAME)["annotation_status"] ) - @pytest.mark.flaky(reruns=2) - @patch("lib.infrastructure.controller.Reporter") - def test_upload_preannotations_from_folder_to_project__upload_status(self, reporter): - reporter_mock = MagicMock() - reporter.return_value = reporter_mock - sa.upload_image_to_project(self.PROJECT_NAME, join(self.folder_path, self.IMAGE_NAME)) - sa.upload_preannotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path) - self.assertEqual( - constances.AnnotationStatus.IN_PROGRESS.name, - sa.get_item_metadata(self.PROJECT_NAME, self.IMAGE_NAME)["annotation_status"] - ) - @pytest.mark.flaky(reruns=2) @patch("lib.infrastructure.controller.Reporter") def test_upload_image_annotations__upload_status(self, reporter): diff --git a/tests/integration/annotations/test_missing_annotation_upload.py b/tests/integration/annotations/test_missing_annotation_upload.py index 098282f86..51ff0d9d9 100644 --- a/tests/integration/annotations/test_missing_annotation_upload.py +++ b/tests/integration/annotations/test_missing_annotation_upload.py @@ -45,23 +45,3 @@ def test_missing_annotation_upload(self): ] ) ) - - def test_missing_pre_annotation_upload(self): - - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="NotStarted" - ) - sa.create_annotation_classes_from_classes_json( - self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json" - ) - ( - uploaded, - could_not_upload, - missing_images, - ) = sa.upload_preannotations_from_folder_to_project( - self.PROJECT_NAME, self.folder_path - ) - self.assertEqual(len(uploaded), 3) - self.assertEqual(len(could_not_upload), 0) - self.assertEqual(len(missing_images), 1) - diff --git a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py index 252d46804..59c99d566 100644 --- a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py +++ b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py @@ -95,7 +95,7 @@ def test_upload_large_annotations(self): def test_upload_big_annotations(self): sa.attach_items( self.PROJECT_NAME, - [{"name": f"aearth_mov_00{i}.jpg", "url": f"url_{i}"} for i in range(1, 6)] # noqa + [{"name": f" rst-lintaearth_mov_00{i}.jpg", "url": f"url_{i}"} for i in range(1, 6)] # noqa ) sa.create_annotation_classes_from_classes_json( self.PROJECT_NAME, f"{self.big_annotations_folder_path}/classes/classes.json" diff --git a/tests/integration/test_basic_images.py b/tests/integration/test_basic_images.py index b6177c43a..25f835449 100644 --- a/tests/integration/test_basic_images.py +++ b/tests/integration/test_basic_images.py @@ -94,18 +94,6 @@ def folder_path(self): def classes_json_path(self): return f"{self.folder_path}/classes/classes.json" - def test_vector_annotations_with_tag_folder_upload_preannotation(self): - sa.upload_images_from_folder_to_project( - self.PROJECT_NAME, self.folder_path, annotation_status="InProgress" - ) - sa.create_annotation_classes_from_classes_json( - self.PROJECT_NAME, self.classes_json_path - ) - uploaded_annotations, _, _ = sa.upload_preannotations_from_folder_to_project( - self.PROJECT_NAME, self.folder_path - ) - self.assertEqual(len(uploaded_annotations), 1) - class TestPixelImages(BaseTestCase): PROJECT_NAME = "sample_project_pixel" diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index cd333ee31..f43f27000 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -127,19 +127,6 @@ def test_upload_export(self): self.assertEqual(len(list(test_dir.glob("*.jpg"))), 0) self.assertEqual(len(list(test_dir.glob("*.png"))), 0) - def test_vector_pre_annotation_folder_upload_download_cli(self): - self._create_project() - - sa.create_annotation_classes_from_classes_json( - self.PROJECT_NAME, f"{self.vector_folder_path}/classes/classes.json" - ) - self.safe_run(self._cli.upload_images, project=self.PROJECT_NAME, folder=str(self.convertor_data_path), - extensions="jpg", - set_annotation_status="QualityCheck") - self.safe_run(self._cli.upload_preannotations, project=self.PROJECT_NAME, folder=str(self.convertor_data_path), - format="COCO", - dataset_name="instances_test") - def test_vector_annotation_folder_upload_download_cli(self): self._create_project() sa.create_annotation_classes_from_classes_json( diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py index f021e67fc..0663322c9 100644 --- a/tests/integration/test_depricated_functions_document.py +++ b/tests/integration/test_depricated_functions_document.py @@ -83,10 +83,6 @@ def test_deprecated_functions(self): sa.download_image_annotations(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "./") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.copy_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, self.PROJECT_NAME_2) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.upload_images_to_project(self.PROJECT_NAME, [self.image_path, ]) except AppException as e: @@ -103,10 +99,6 @@ def test_deprecated_functions(self): sa.set_project_workflow(self.PROJECT_NAME, [{}]) except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.upload_preannotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.add_annotation_point_to_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, [1, 2], "some class") except AppException as e: diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py index 3ad847abb..1160c6472 100644 --- a/tests/integration/test_depricated_functions_video.py +++ b/tests/integration/test_depricated_functions_video.py @@ -76,10 +76,6 @@ def test_deprecated_functions(self): sa.download_image_annotations(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "./") except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.copy_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, self.PROJECT_NAME_2) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.upload_images_to_project(self.PROJECT_NAME, [self.image_path, ]) except AppException as e: @@ -96,10 +92,6 @@ def test_deprecated_functions(self): sa.set_project_workflow(self.PROJECT_NAME, [{}]) except AppException as e: self.assertIn(self.EXCEPTION_MESSAGE, str(e)) - try: - sa.upload_preannotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path) - except AppException as e: - self.assertIn(self.EXCEPTION_MESSAGE, str(e)) try: sa.get_project_workflow(self.PROJECT_NAME) except AppException as e: diff --git a/tests/integration/test_limitations.py b/tests/integration/test_limitations.py index cac030348..a45d575c1 100644 --- a/tests/integration/test_limitations.py +++ b/tests/integration/test_limitations.py @@ -1,21 +1,19 @@ import os -from unittest.mock import patch from os.path import dirname +from unittest.mock import patch -from src.superannotate import SAClient -sa = SAClient() from src.superannotate import AppException +from src.superannotate import SAClient from src.superannotate.lib.core import UPLOAD_FOLDER_LIMIT_ERROR_MESSAGE from src.superannotate.lib.core import UPLOAD_PROJECT_LIMIT_ERROR_MESSAGE from src.superannotate.lib.core import UPLOAD_USER_LIMIT_ERROR_MESSAGE -from src.superannotate.lib.core import COPY_FOLDER_LIMIT_ERROR_MESSAGE -from src.superannotate.lib.core import COPY_PROJECT_LIMIT_ERROR_MESSAGE -from src.superannotate.lib.core import COPY_SUPER_LIMIT_ERROR_MESSAGE from tests.integration.base import BaseTestCase from tests.moks.limitatoins import folder_limit_response from tests.moks.limitatoins import project_limit_response from tests.moks.limitatoins import user_limit_response +sa = SAClient() + class TestLimitsUploadImagesFromFolderToProject(BaseTestCase): PROJECT_NAME = "TestLimitsUploadImagesFromFolderToProject" @@ -49,43 +47,3 @@ def test_user_limitations(self, *_): _, _, __ = sa.upload_images_from_folder_to_project( project=self._project["name"], folder_path=self.folder_path ) - - -class TestLimitsCopyImage(BaseTestCase): - PROJECT_NAME = "TestLimitsCopyImage" - PROJECT_DESCRIPTION = "Desc" - PROJECT_TYPE = "Vector" - TEST_FOLDER_PTH = "data_set" - TEST_FOLDER_PATH = "data_set/sample_project_vector" - EXAMPLE_IMAGE_1 = "example_image_1.jpg" - - @property - def folder_path(self): - return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH) - - def test_folder_limitations(self): - sa.upload_image_to_project(self._project["name"], os.path.join(self.folder_path, self.EXAMPLE_IMAGE_1)) - sa.create_folder(self._project["name"], self._project["name"]) - with patch("lib.infrastructure.serviceprovider.ServiceProvider.get_limitations") as limit_response: - limit_response.return_value = folder_limit_response - with self.assertRaisesRegexp(AppException, COPY_FOLDER_LIMIT_ERROR_MESSAGE): - _, _, __ = sa.copy_image( - self._project["name"], self.folder_path, f"{self.PROJECT_NAME}/{self.PROJECT_NAME}") - - def test_project_limitations(self, ): - sa.upload_image_to_project(self._project["name"], os.path.join(self.folder_path, self.EXAMPLE_IMAGE_1)) - sa.create_folder(self._project["name"], self._project["name"]) - with patch("lib.infrastructure.serviceprovider.ServiceProvider.get_limitations") as limit_response: - limit_response.return_value = project_limit_response - with self.assertRaisesRegexp(AppException, COPY_PROJECT_LIMIT_ERROR_MESSAGE): - _, _, __ = sa.copy_image( - self._project["name"], self.folder_path, f"{self.PROJECT_NAME}/{self.PROJECT_NAME}") - - def test_user_limitations(self, ): - sa.upload_image_to_project(self._project["name"], os.path.join(self.folder_path, self.EXAMPLE_IMAGE_1)) - sa.create_folder(self._project["name"], self._project["name"]) - with patch("lib.infrastructure.serviceprovider.ServiceProvider.get_limitations") as limit_response: - limit_response.return_value = user_limit_response - with self.assertRaisesRegexp(AppException, COPY_SUPER_LIMIT_ERROR_MESSAGE): - _, _, __ = sa.copy_image( - self._project["name"], self.folder_path, f"{self.PROJECT_NAME}/{self.PROJECT_NAME}")