diff --git a/README.md b/README.md
deleted file mode 100644
index a548dee90..000000000
--- a/README.md
+++ /dev/null
@@ -1,62 +0,0 @@
-
-
-# SuperAnnotate Python SDK
-
-SuperAnnotate Python SDK allows access to the platform without
- web browser:
-
-```python
-import superannotate as sa
-
-sa.create_project("Example Project 1", "example", "Vector")
-
-sa.upload_images_from_folder_to_project("Example Project 1", "")
-```
-
-## Installation
-
-SDK is available on PyPI:
-
-```console
-pip install superannotate
-```
-
-The package officially supports Python 3.6+ and was tested under Linux and
-Windows ([Anaconda](https://www.anaconda.com/products/individual#windows)) platforms.
-
-For more detailed installation steps and package usage please have a look at the
-[tutorial](https://superannotate.readthedocs.io/en/stable/tutorial.sdk.html).
-
-## Supported Features
-
-- Search projects
-- Create/delete a project
-- Upload images to a project from a local or AWS S3 folder
-- Upload videos to a project from a local folder
-- Upload annotations/pre-annotations to a project from local or AWS S3 folder
-- Set the annotation status of the images being uploaded
-- Export annotations from a project to a local or AWS S3 folder
-- Share and unshare a project with a team contributor
-- Invite a team contributor
-- Search images in a project
-- Download a single image
-- Copy/move image between projects
-- Get image bytes (e.g., for numpy array creation)
-- Set image annotation status
-- Download image annotations/pre-annotations
-- Create/download project annotation classes
-- Convert annotation format from/to COCO
-- Convert annotation format from VOC, SuperVisely, LabelBox, DataLoop, VGG, VoTT, SageMaker, GoogleCloud, YOLO
-- Add annotations to images on platform
-- Add annotations to local SuperAnnotate format JSONs
-- CLI commands for simple tasks
-
-## Full SDK reference, tutorial available on [Read the Docs](https://superannotate.readthedocs.io)
-
-## License
-
-This SDK is distributed under the MIT License, see [LICENSE](./LICENSE).
-
-## Questions and Issues
-
-For questions and issues please use this repo's issue tracker on GitHub.
diff --git a/README.rst b/README.rst
new file mode 100644
index 000000000..d95540dba
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,128 @@
+SuperAnnotate Python SDK
+===============================
+|Python| |License| |Changelog|
+
+
+Welcome to the SuperAnnotate Python Software Development Kit (SDK), which enables Python programmers to create software that incorporates services of the platform and effortlessly integrates SuperAnnotate into their AI process.
+
+.. |Python| image:: https://img.shields.io/static/v1?label=python&message=3.7/3.8/3.9/3.10/3.11&color=blue&style=flat-square
+ :target: https://pypi.org/project/superannotate/
+ :alt: Python Versions
+.. |License| image:: https://img.shields.io/static/v1?label=license&message=MIT&color=green&style=flat-square
+ :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/LICENSE/
+ :alt: License
+.. |Changelog| image:: https://img.shields.io/static/v1?label=change&message=log&color=yellow&style=flat-square
+ :target: https://github.com/superannotateai/superannotate-python-sdk/blob/master/CHANGELOG.md
+ :alt: Changelog
+
+Resources
+---------------
+
+- API Reference and User Guide available on `Read the Docs `__
+- `Platform documentation `__
+
+
+Authentication
+---------------
+
+.. code-block:: python
+
+ >>> from superannotate import SAClient
+ # by environment variable SA_TOKEN
+ >>> sa_client = SAClient()
+ # by token
+ >>> sa_client = SAClient(token='')
+ # by config file
+ # default path is ~/.superannotate/config.json
+ >>> sa_client = SAClient(config_path='~/.superannotate/dev_config.json')
+
+Using superannotate
+-------------------
+
+.. code-block:: python
+
+ >>> from superannotate import SAClient
+ >>> sa_client =SAClient()
+ >>> project = 'Dogs'
+ >>> sa_client.create_project(
+ project_name=project,
+ project_description='Test project generated via SDK',
+ project_type='Vector'
+ )
+ >>> sa_client.create_annotation_class(
+ project=project,
+ name='dog',
+ color='#F9E0FA',
+ class_type='tag'
+ )
+ >>> sa_client.attach_items(
+ project=project,
+ attachments=[
+ {
+ 'url': 'https://drive.google.com/uc?export=download&id=1ipOrZNSTlPUkI_hnrW9aUD5yULqqq5Vl',
+ 'name': 'dog.jpeg'
+ }
+ ]
+ )
+ >>> sa_client.upload_annotations(
+ project=project,
+ annotations=[
+ {
+ 'metadata': {'name': 'dog.jpeg'},
+ 'instances': [
+ {'type': 'tag', 'className': 'dog'}
+ ]
+ }
+ ]
+ )
+ >>> sa_client.get_annotations(project=project, items=['dog.jpeg'])
+
+Installation
+------------
+
+SuperAnnotate python SDK is available on PyPI:
+
+.. code-block:: bash
+
+ pip install superannotate
+
+
+The package officially supports Python 3.7+ and was tested under Linux and
+Windows (`Anaconda `__
+) platforms.
+
+For more detailed installation steps and package usage please have a look at the `tutorial `__
+
+
+Supported Features
+------------------
+
+- search/get/create/clone/update/delete projects
+- search/get/create/delete folders
+- assign folders to project contributors
+- upload items to a project from a local or AWS S3 folder
+- attach items by URL or from an integrated storage, meanwhile keeping them secure in your cloud provider
+- get integrated cloud storages
+- upload annotations (also from local or AWS S3 folder)
+- delete annotations
+- set items annotations statuses
+- get/download/export annotations from a project (also to a local or AWS S3 folder)
+- invite/search team contributors or add contributors to a specific project
+- search/get/copy/move items in a project
+- query items using SA Query Language
+- define custom metadata for items and upload custom values (query based on your custom metadata)
+- upload priority scores
+- get available subsets (sets of segregated items), query items in a subset or add items to a subset
+- assign or anassign items to project contributors
+- download an image that has been uploaded to project
+- search/create/download/delete project annotation classes
+- search/download models
+- run predictions
+- convert annotations from/to COCO format
+- convert annotation from VOC, SuperVisely, LabelBox, DataLoop, VGG, VoTT, SageMaker, GoogleCloud, YOLO formats
+- CLI commands for simple tasks
+
+Questions and Issues
+--------------------
+
+For questions and issues please use this repo’s issue tracker on GitHub or contact support@superannotate.com.
diff --git a/docs/source/superannotate.sdk.rst b/docs/source/superannotate.sdk.rst
index fd4e7fac6..53b626f4d 100644
--- a/docs/source/superannotate.sdk.rst
+++ b/docs/source/superannotate.sdk.rst
@@ -42,7 +42,6 @@ ________
.. automethod:: superannotate.SAClient.upload_videos_from_folder_to_project
.. _ref_upload_annotations_from_folder_to_project:
.. automethod:: superannotate.SAClient.upload_annotations_from_folder_to_project
-.. automethod:: superannotate.SAClient.upload_preannotations_from_folder_to_project
.. automethod:: superannotate.SAClient.add_contributors_to_project
.. automethod:: superannotate.SAClient.get_project_settings
.. automethod:: superannotate.SAClient.set_project_default_image_quality_in_editor
@@ -107,7 +106,6 @@ ______
.. automethod:: superannotate.SAClient.download_image
.. automethod:: superannotate.SAClient.download_image_annotations
.. automethod:: superannotate.SAClient.upload_image_annotations
-.. automethod:: superannotate.SAClient.copy_image
.. automethod:: superannotate.SAClient.pin_image
.. automethod:: superannotate.SAClient.add_annotation_bbox_to_image
.. automethod:: superannotate.SAClient.add_annotation_point_to_image
diff --git a/setup.py b/setup.py
index 7094269e1..f7da21cc2 100644
--- a/setup.py
+++ b/setup.py
@@ -17,10 +17,6 @@ def get_version():
with open("requirements.txt") as f:
requirements.extend(f.read().splitlines())
-with open('README.md') as f:
- readme = f.read()
-
-readme = "\n".join(readme.split('\n')[2:])
setup(
name='superannotate',
@@ -31,12 +27,12 @@ def get_version():
description='Python SDK to SuperAnnotate platform',
license='MIT',
author='SuperAnnotate AI',
+ author_email='suppoort@superannotate.com',
url='https://github.com/superannotateai/superannotate-python-sdk',
- long_description=readme,
- long_description_content_type='text/markdown',
+ long_description=open('README.rst').read(),
+ long_description_content_type='text/x-rst',
install_requires=requirements,
setup_requires=['wheel'],
- description_file="README.md",
entry_points={
'console_scripts': ['superannotatecli = superannotate.lib.app.bin.superannotate:main']
},
diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py
index 6dc56ced2..994ec5292 100644
--- a/src/superannotate/__init__.py
+++ b/src/superannotate/__init__.py
@@ -2,7 +2,7 @@
import sys
-__version__ = "4.4.5"
+__version__ = "4.4.6dev4"
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
diff --git a/src/superannotate/lib/app/analytics/aggregators.py b/src/superannotate/lib/app/analytics/aggregators.py
index 4e241bf97..ec9ccdad5 100644
--- a/src/superannotate/lib/app/analytics/aggregators.py
+++ b/src/superannotate/lib/app/analytics/aggregators.py
@@ -91,6 +91,7 @@ class DataAggregator:
"polygon": lambda annotation: annotation["points"],
"polyline": lambda annotation: annotation["points"],
"cuboid": lambda annotation: annotation["points"],
+ "comment": lambda annotation: annotation["points"],
"point": lambda annotation: {"x": annotation["x"], "y": annotation["y"]},
"annotation_type": lambda annotation: dict(
cx=annotation["cx"],
@@ -123,7 +124,7 @@ def annotation_suffix(self):
self._annotation_suffix = PIXEL_ANNOTATION_POSTFIX
else:
self._annotation_suffix = ATTACHED_VIDEO_ANNOTATION_POSTFIX
- return self._annotation_suffix
+ return ATTACHED_VIDEO_ANNOTATION_POSTFIX
def get_annotation_paths(self):
annotations_paths = []
@@ -378,7 +379,7 @@ def __append_annotation(annotation_dict):
for annotation_path in annotations_paths:
annotation_json = json.load(open(annotation_path))
- parts = annotation_path.name.split(self.annotation_suffix)
+ parts = Path(annotation_path).name.split(self.annotation_suffix)
if len(parts) != 2:
continue
image_name = parts[0]
@@ -449,8 +450,8 @@ def __append_annotation(annotation_dict):
attributes = annotation.get("attributes")
user_metadata = self.__get_user_metadata(annotation)
folder_name = None
- if annotation_path.parent != Path(self.project_root):
- folder_name = annotation_path.parent.name
+ if Path(annotation_path).parent != Path(self.project_root):
+ folder_name = Path(annotation_path).parent.name
num_added = 0
if not attributes:
annotation_dict = {
diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py
index 7fe0ffec0..410e3c506 100644
--- a/src/superannotate/lib/app/interface/cli_interface.py
+++ b/src/superannotate/lib/app/interface/cli_interface.py
@@ -129,27 +129,6 @@ def export_project(
)
sys.exit(0)
- def upload_preannotations(
- self, project, folder, dataset_name=None, task=None, format=None
- ):
- """
- To upload preannotations from folder to project use
- Optional argument format accepts input annotation format. It can have COCO or SuperAnnotate values.
- If the argument is not given then SuperAnnotate (the native annotation format) is assumed.
- Only when COCO format is specified dataset-name and task arguments are required.
- dataset-name specifies JSON filename (without extension) in .
- task specifies the COCO task for conversion. Please see import_annotation_format for more details.
- """
- self._upload_annotations(
- project=project,
- folder=folder,
- format=format,
- dataset_name=dataset_name,
- task=task,
- pre=True,
- )
- sys.exit(0)
-
def upload_annotations(
self, project, folder, dataset_name=None, task=None, format=None
):
@@ -167,13 +146,10 @@ def upload_annotations(
format=format,
dataset_name=dataset_name,
task=task,
- pre=False,
)
sys.exit(0)
- def _upload_annotations(
- self, project, folder, format, dataset_name, task, pre=True
- ):
+ def _upload_annotations(self, project, folder, format, dataset_name, task):
project_folder_name = project
project_name, folder_name = split_project_path(project)
project = SAClient().controller.get_project(project_name)
@@ -197,14 +173,10 @@ def _upload_annotations(
task=task,
)
annotations_path = temp_dir
- if pre:
- SAClient().upload_preannotations_from_folder_to_project(
- project_folder_name, annotations_path
- )
- else:
- SAClient().upload_annotations_from_folder_to_project(
- project_folder_name, annotations_path
- )
+
+ SAClient().upload_annotations_from_folder_to_project(
+ project_folder_name, annotations_path
+ )
sys.exit(0)
def attach_image_urls(
diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py
index e8d0a620f..57b9f2dd5 100644
--- a/src/superannotate/lib/app/interface/sdk_interface.py
+++ b/src/superannotate/lib/app/interface/sdk_interface.py
@@ -437,112 +437,6 @@ def search_folders(
]
return [folder.name for folder in data if not folder.is_root]
- def copy_image(
- self,
- source_project: Union[NotEmptyStr, dict],
- image_name: NotEmptyStr,
- destination_project: Union[NotEmptyStr, dict],
- include_annotations: Optional[StrictBool] = False,
- copy_annotation_status: Optional[StrictBool] = False,
- copy_pin: Optional[StrictBool] = False,
- ):
- """Copy image to a project. The image's project is the same as destination
- project then the name will be changed to _().,
- where is the next available number deducted from project image list.
-
- :param source_project: project name plus optional subfolder in the project (e.g., "project1/folder1") or
- metadata of the project of source project
- :type source_project: str or dict
- :param image_name: image name
- :type image_name: str
- :param destination_project: project name or metadata of the project of destination project
- :type destination_project: str or dict
- :param include_annotations: enables annotations copy
- :type include_annotations: bool
- :param copy_annotation_status: enables annotations status copy
- :type copy_annotation_status: bool
- :param copy_pin: enables image pin status copy
- :type copy_pin: bool
- """
- warning_msg = "The SAClient.copy_image method will be deprecated with the Superannotate Python SDK 4.4.6 release"
- warnings.warn(warning_msg, DeprecationWarning)
- logger.warning(warning_msg)
- source_project_name, source_folder_name = extract_project_folder(source_project)
- destination_project_name, destination_folder_name = extract_project_folder(
- destination_project
- )
- source_project_metadata = self.controller.projects.get_by_name(
- source_project_name
- ).data
- destination_project_metadata = self.controller.projects.get_by_name(
- destination_project_name
- ).data
-
- if destination_project_metadata.type.value in [
- constants.ProjectType.VIDEO.value,
- constants.ProjectType.DOCUMENT.value,
- ] or source_project_metadata.type.value in [
- constants.ProjectType.VIDEO.value,
- constants.ProjectType.DOCUMENT.value,
- ]:
- raise AppException(LIMITED_FUNCTIONS[source_project_metadata.type])
-
- response = self.controller.copy_image(
- from_project_name=source_project_name,
- from_folder_name=source_folder_name,
- to_project_name=destination_project_name,
- to_folder_name=destination_folder_name,
- image_name=image_name,
- copy_annotation_status=copy_annotation_status,
- )
- if response.errors:
- raise AppException(response.errors)
- if copy_pin:
- destination_project = self.controller.get_project(
- destination_project_metadata
- )
- _folder = self.controller.get_folder(
- destination_project, destination_folder_name
- )
- item = self.controller.items.get_by_name(
- destination_project_metadata, _folder, image_name
- ).data
- item.is_pinned = 1
- self.controller.items.update(
- project=destination_project_metadata,
- folder=_folder,
- image_name=image_name,
- is_pinned=1,
- )
- if include_annotations:
- source_project = self.controller.get_project(source_project_name)
- source_folder = self.controller.get_folder(
- source_project, source_folder_name
- )
- source_image = self.controller.items.get_by_name(
- source_project, source_folder, image_name
- ).data
- destination_project = self.controller.get_project(destination_project)
- destination_folder = self.controller.get_folder(
- destination_project, destination_folder_name
- )
- destination_image = self.controller.items.get_by_name(
- destination_project, destination_folder, image_name
- ).data
- self.controller.annotation_classes.copy_multiple(
- source_project=source_project,
- source_folder=source_folder,
- source_item=source_image,
- destination_project=destination_project,
- destination_folder=destination_folder,
- destination_item=destination_image,
- )
-
- logger.info(
- f"Copied image {source_project}/{image_name}"
- f" to {destination_project_name}/{destination_folder_name}."
- )
-
def get_project_metadata(
self,
project: Union[NotEmptyStr, dict],
@@ -795,13 +689,18 @@ def assign_folder(
:type users: list of str
"""
- contributors = (
- self.controller.projects.get_by_name(
- project_name=project_name, include_contributors=True
- )
- .data["project"]
- .users
+ response = self.controller.projects.get_by_name(name=project_name)
+ if response.errors:
+ raise AppException(response.errors)
+ project = response.data
+ response = self.controller.projects.get_metadata(
+ project=project, include_contributors=True
)
+
+ if response.errors:
+ raise AppException(response.errors)
+
+ contributors = response.data.users
verified_users = [i["user_id"] for i in contributors]
verified_users = set(users).intersection(set(verified_users))
unverified_contributor = set(users) - verified_users
@@ -1383,7 +1282,7 @@ def create_annotation_classes_from_classes_json(
file.seek(0)
data = file
else:
- data = open(classes_json)
+ data = open(classes_json, encoding="utf-8")
classes_json = json.load(data)
try:
annotation_classes = parse_obj_as(List[AnnotationClassEntity], classes_json)
@@ -1616,79 +1515,6 @@ def upload_annotations_from_folder_to_project(
raise AppException(response.errors)
return response.data
- def upload_preannotations_from_folder_to_project(
- self,
- project: Union[NotEmptyStr, dict],
- folder_path: Union[str, Path],
- from_s3_bucket=None,
- recursive_subfolders: Optional[StrictBool] = False,
- ):
- """Finds and uploads all JSON files in the folder_path as pre-annotations to the project.
-
- The JSON files should follow specific naming convention. For Vector
- projects they should be named "___objects.json" (e.g., if
- image is cats.jpg the annotation filename should be cats.jpg___objects.json), for Pixel projects
- JSON file should be named "___pixel.json" and also second mask
- image file should be present with the name "___save.png". In both cases
- image with should be already present on the platform.
-
- Existing pre-annotations will be overwritten.
-
- :param project: project name or folder path (e.g., "project1/folder1")
- :type project: str
- :param folder_path: from which folder to upload the pre-annotations
- :type folder_path: Path-like (str or Path)
- :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem
- :type from_s3_bucket: str
- :param recursive_subfolders: enable recursive subfolder parsing
- :type recursive_subfolders: bool
-
- :return: paths to pre-annotations uploaded and could-not-upload
- :rtype: tuple of list of strs
- """
- warning_msg = (
- "The SAClient.upload_preannotations_from_folder_to_project"
- " method will be deprecated with the Superannotate Python SDK 4.4.6 release"
- )
- warnings.warn(warning_msg, DeprecationWarning)
- logger.warning(warning_msg)
- project_name, folder_name = extract_project_folder(project)
- project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
- project = self.controller.get_project(project_name)
- if project.type in [
- constants.ProjectType.VIDEO,
- constants.ProjectType.DOCUMENT,
- ]:
- raise AppException(LIMITED_FUNCTIONS[project.type])
- if recursive_subfolders:
- logger.info(
- "When using recursive subfolder parsing same name annotations in different "
- "subfolders will overwrite each other.",
- )
- logger.info(
- "The JSON files should follow a specific naming convention, matching file names already present "
- "on the platform. Existing annotations will be overwritten"
- )
- annotation_paths = get_annotation_paths(
- folder_path, from_s3_bucket, recursive_subfolders
- )
- logger.info(
- f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}."
- )
- project, folder = self.controller.get_project_folder(project_name, folder_name)
- response = self.controller.annotations.upload_from_folder(
- project=project,
- folder=folder,
- team=self.controller.team,
- annotation_paths=annotation_paths, # noqa: E203
- client_s3_bucket=from_s3_bucket,
- folder_path=folder_path,
- is_pre_annotations=True,
- )
- if response.errors:
- raise AppException(response.errors)
- return response.data
-
def upload_image_annotations(
self,
project: Union[NotEmptyStr, dict],
@@ -1971,6 +1797,12 @@ def add_annotation_bbox_to_image(
:param error: if not None, marks annotation as error (True) or no-error (False)
:type error: bool
"""
+ warning_msg = (
+ "The SAClient.add_annotation_bbox_to_image method will "
+ "be deprecated with the Superannotate Python SDK 4.4.7 release"
+ )
+ warnings.warn(warning_msg, DeprecationWarning)
+ logger.warning(warning_msg)
project_name, folder_name = extract_project_folder(project)
project = self.controller.get_project(project_name)
@@ -2035,6 +1867,12 @@ def add_annotation_point_to_image(
:param error: if not None, marks annotation as error (True) or no-error (False)
:type error: bool
"""
+ warning_msg = (
+ "The SAClient.add_annotation_point_to_image method will "
+ "be deprecated with the Superannotate Python SDK 4.4.7 release"
+ )
+ warnings.warn(warning_msg, DeprecationWarning)
+ logger.warning(warning_msg)
project, folder = self.controller.get_project_folder_by_path(project)
if project.type in [
constants.ProjectType.VIDEO,
@@ -2091,6 +1929,12 @@ def add_annotation_comment_to_image(
:param resolved: comment resolve status
:type resolved: bool
"""
+ warning_msg = (
+ "The SAClient.add_annotation_comment_to_image method will "
+ "be deprecated with the Superannotate Python SDK 4.4.7 release"
+ )
+ warnings.warn(warning_msg, DeprecationWarning)
+ logger.warning(warning_msg)
project_name, folder_name = extract_project_folder(project)
project = self.controller.projects.get_by_name(project_name).data
if project.type in [
diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py
index d0290e6dc..1b46a188f 100644
--- a/src/superannotate/lib/core/usecases/annotations.py
+++ b/src/superannotate/lib/core/usecases/annotations.py
@@ -409,6 +409,7 @@ def execute(self):
class UploadAnnotationsFromFolderUseCase(BaseReportableUseCase):
MAX_WORKERS = 16
CHUNK_SIZE = 100
+ CHUNK_SIZE_PATHS = 500
CHUNK_SIZE_MB = 10 * 1024 * 1024
STATUS_CHANGE_CHUNK_SIZE = 100
AUTH_DATA_CHUNK_SIZE = 500
@@ -532,7 +533,7 @@ async def get_annotation(
if self._project.type == constants.ProjectType.PIXEL.value:
mask = self.get_annotation_from_s3(self._client_s3_bucket, mask_path)
else:
- async with aiofiles.open(path) as file:
+ async with aiofiles.open(path, encoding="utf-8") as file:
content = await file.read()
if self._project.type == constants.ProjectType.PIXEL.value:
async with aiofiles.open(mask_path, "rb") as mask:
@@ -579,14 +580,26 @@ def get_existing_name_item_mapping(
@property
def annotation_upload_data(self) -> UploadAnnotationAuthData:
- if not self._annotation_upload_data:
- response = self._service_provider.get_annotation_upload_data(
+ CHUNK_SIZE = UploadAnnotationsFromFolderUseCase.CHUNK_SIZE_PATHS
+
+ if self._annotation_upload_data:
+ return self._annotation_upload_data
+
+ images = {}
+ for i in range(0, len(self._item_ids), CHUNK_SIZE):
+ tmp = self._service_provider.get_annotation_upload_data(
project=self._project,
folder=self._folder,
- item_ids=self._item_ids,
+ item_ids=self._item_ids[i : i + CHUNK_SIZE],
)
- if response.ok:
- self._annotation_upload_data = response.data
+ if not tmp.ok:
+ raise AppException(tmp.errors)
+ else:
+ images.update(tmp.data.images)
+
+ self._annotation_upload_data = tmp.data
+ self._annotation_upload_data.images = images
+
return self._annotation_upload_data
@property
@@ -835,7 +848,9 @@ def _get_annotation_json(self) -> tuple:
),
)
else:
- annotation_json = json.load(open(self._annotation_path))
+ annotation_json = json.load(
+ open(self._annotation_path, encoding="utf-8")
+ )
if self._project.type == constants.ProjectType.PIXEL.value:
mask = open(
self._annotation_path.replace(
@@ -1313,7 +1328,7 @@ def download_annotation_classes(self, path: str):
if response.ok:
classes_path = Path(path) / "classes"
classes_path.mkdir(parents=True, exist_ok=True)
- with open(classes_path / "classes.json", "w+") as file:
+ with open(classes_path / "classes.json", "w+", encoding="utf-8") as file:
json.dump(
[
i.dict(
diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py
index d7d978e6e..ee76fb3c3 100644
--- a/src/superannotate/lib/core/usecases/images.py
+++ b/src/superannotate/lib/core/usecases/images.py
@@ -849,6 +849,7 @@ def execute(self) -> Response:
class UploadImagesToProject(BaseInteractiveUseCase):
MAX_WORKERS = 10
+ LIST_NAME_CHUNK_SIZE = 500
def __init__(
self,
@@ -1023,6 +1024,7 @@ def filter_paths(self, paths: List[str]):
for path in paths:
name_path_map[Path(path).name].append(path)
+ CHUNK_SIZE = UploadImagesToProject.LIST_NAME_CHUNK_SIZE
filtered_paths = []
duplicated_paths = []
for file_name in name_path_map:
@@ -1030,17 +1032,22 @@ def filter_paths(self, paths: List[str]):
duplicated_paths.append(name_path_map[file_name][1:])
filtered_paths.append(name_path_map[file_name][0])
- response = self._service_provider.items.list_by_names(
- project=self._project,
- folder=self._folder,
- names=[image.split("/")[-1] for image in filtered_paths],
- )
- if not response.ok:
- self._response.errors = AppException(response.error)
- return self._response
+ image_list = []
+ for i in range(0, len(filtered_paths), CHUNK_SIZE):
+ response = self._service_provider.items.list_by_names(
+ project=self._project,
+ folder=self._folder,
+ names=[
+ image.split("/")[-1] for image in filtered_paths[i : i + CHUNK_SIZE]
+ ],
+ )
+
+ if not response.ok:
+ raise AppException(response.error)
+ image_list.extend([image.name for image in response.data])
+ image_list = set(image_list)
images_to_upload = []
- image_list = [image.name for image in response.data]
for path in filtered_paths:
if Path(path).name not in image_list:
@@ -1287,149 +1294,6 @@ def execute(self):
return self._response
-class CopyImageUseCase(BaseUseCase):
- def __init__(
- self,
- from_project: ProjectEntity,
- from_folder: FolderEntity,
- image_name: str,
- to_project: ProjectEntity,
- to_folder: FolderEntity,
- service_provider: BaseServiceProvider,
- s3_repo,
- include_annotations: Optional[bool] = True,
- copy_annotation_status: Optional[bool] = True,
- copy_pin: Optional[bool] = True,
- move=False,
- ):
- super().__init__()
- self._from_project = from_project
- self._from_folder = from_folder
- self._image_name = image_name
- self._to_project = to_project
- self._to_folder = to_folder
- self._s3_repo = s3_repo
- self._include_annotations = include_annotations
- self._copy_annotation_status = copy_annotation_status
- self._copy_pin = copy_pin
- self._service_provider = service_provider
- self._move = move
-
- def validate_copy_path(self):
- if (
- self._from_project.name == self._to_project.name
- and self._from_folder.name == self._to_folder.name
- ):
- raise AppValidationException(
- "Cannot move image if source_project == destination_project."
- )
-
- def validate_project_type(self):
- if self._from_project.type in (
- constances.ProjectType.VIDEO.value,
- constances.ProjectType.DOCUMENT.value,
- ):
- raise AppValidationException(
- constances.LIMITED_FUNCTIONS[self._from_project.type]
- )
-
- def validate_limitations(self):
- response = self._service_provider.get_limitations(
- project=self._to_project,
- folder=self._to_folder,
- )
- if not response.ok:
- raise AppValidationException(response.error)
-
- if self._move and self._from_project.id == self._to_project.id:
- if self._from_folder.id == self._to_folder.id:
- raise AppValidationException(
- "Cannot move image if source_project == destination_project."
- )
-
- if response.data.folder_limit.remaining_image_count < 1:
- raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE)
- if response.data.project_limit.remaining_image_count < 1:
- raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE)
- if (
- response.data.user_limit
- and response.data.user_limit.remaining_image_count < 1
- ):
- raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE)
-
- @property
- def s3_repo(self):
- self._auth_data = self._service_provider.get_s3_upload_auth_token(
- project=self._to_project, folder=self._to_folder
- ).data
- if "error" in self._auth_data:
- raise AppException(self._auth_data.get("error"))
- return self._s3_repo(
- self._auth_data["accessKeyId"],
- self._auth_data["secretAccessKey"],
- self._auth_data["sessionToken"],
- self._auth_data["bucket"],
- )
-
- def execute(self) -> Response:
- if self.is_valid():
- image = (
- GetImageUseCase(
- project=self._from_project,
- folder=self._from_folder,
- image_name=self._image_name,
- service_provider=self._service_provider,
- )
- .execute()
- .data
- )
-
- image_bytes = (
- GetImageBytesUseCase(
- project=self._from_project,
- folder=self._from_folder,
- image=image,
- service_provider=self._service_provider,
- )
- .execute()
- .data
- )
- image_path = f"{self._to_folder}/{self._image_name}"
-
- auth_data = self._service_provider.get_s3_upload_auth_token(
- project=self._to_project, folder=self._to_folder
- )
- if not auth_data.ok:
- raise AppException(auth_data.error)
- s3_response = UploadImageS3UseCase(
- project=self._to_project,
- image_path=image_path,
- image=image_bytes,
- service_provider=self._service_provider,
- upload_path=auth_data.data["filePath"],
- s3_repo=self.s3_repo,
- ).execute()
- if s3_response.errors:
- raise AppException(s3_response.errors)
- image_entity = s3_response.data
- del image_bytes
-
- attach_response = AttachFileUrlsUseCase(
- project=self._to_project,
- folder=self._to_folder,
- attachments=[image_entity],
- service_provider=self._service_provider,
- annotation_status=image.annotation_status_code
- if self._copy_annotation_status
- else None,
- upload_state_code=constances.UploadState.BASIC.value,
- ).execute()
- if attach_response.errors:
- raise AppException(attach_response.errors)
- self._response.data = image_entity
- return self._response
-
-
class DeleteAnnotations(BaseUseCase):
POLL_AWAIT_TIME = 2
CHUNK_SIZE = 2000
@@ -1593,15 +1457,14 @@ def fill_classes_data(self, annotations: dict):
attribute["groupName"] = annotation_class["attribute_groups"][
attribute["groupId"]
]["name"]
- if attribute["id"] not in list(
+ if attribute.get("id") in list(
annotation_class["attribute_groups"][attribute["groupId"]][
"attributes"
].keys()
):
- continue
- attribute["name"] = annotation_class["attribute_groups"][
- attribute["groupId"]
- ]["attributes"][attribute["id"]]
+ attribute["name"] = annotation_class["attribute_groups"][
+ attribute["groupId"]
+ ]["attributes"][attribute["id"]]
def execute(self):
if self.is_valid():
diff --git a/src/superannotate/lib/core/video_convertor.py b/src/superannotate/lib/core/video_convertor.py
index ca179a2a3..f951df256 100644
--- a/src/superannotate/lib/core/video_convertor.py
+++ b/src/superannotate/lib/core/video_convertor.py
@@ -178,6 +178,8 @@ def _process(self):
for instance in self._annotation_data["instances"]:
instance_id = next(self.id_generator)
annotation_type = instance["meta"]["type"]
+ if annotation_type == "comment":
+ continue
class_name = instance["meta"].get("className")
class_id = instance["meta"].get("classId", -1)
for parameter in instance.get("parameters", []):
@@ -185,7 +187,7 @@ def _process(self):
interpolated_frames = {}
for timestamp in parameter["timestamps"]:
frames_mapping[
- int(math.ceil(timestamp["timestamp"] / self.ratio))
+ int(math.ceil(timesstamp["timestamp"] / self.ratio))
].append(timestamp)
frames_mapping = self.merge_first_frame(frames_mapping)
diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py
index 2a89894db..61eb814ae 100644
--- a/src/superannotate/lib/infrastructure/controller.py
+++ b/src/superannotate/lib/infrastructure/controller.py
@@ -761,7 +761,7 @@ def __init__(self, token: str, host: str, ssl_verify: bool, version: str):
self._user_id = None
self._reporter = None
- http_client = HttpClient(api_url=host, token=token)
+ http_client = HttpClient(api_url=host, token=token, verify_ssl=ssl_verify)
self.service_provider = ServiceProvider(http_client)
self._team = self.get_team().data
@@ -997,32 +997,6 @@ def update(self, project: ProjectEntity, folder: FolderEntity):
)
return use_case.execute()
- def copy_image(
- self,
- from_project_name: str,
- from_folder_name: str,
- to_project_name: str,
- to_folder_name: str,
- image_name: str,
- copy_annotation_status: bool = False,
- move: bool = False,
- ):
- from_project = self.get_project(from_project_name)
- to_project = self.get_project(to_project_name)
- to_folder = self.get_folder(to_project, to_folder_name)
- use_case = usecases.CopyImageUseCase(
- from_project=from_project,
- from_folder=self.get_folder(from_project, from_folder_name),
- to_project=to_project,
- to_folder=to_folder,
- service_provider=self.service_provider,
- image_name=image_name,
- s3_repo=self.s3_repo,
- copy_annotation_status=copy_annotation_status,
- move=move,
- )
- return use_case.execute()
-
def un_assign_folder(self, project_name: str, folder_name: str):
project_entity = self.get_project(project_name)
folder = self.get_folder(project_entity, folder_name)
diff --git a/tests/integration/annotations/test_annotations_upload_status_change.py b/tests/integration/annotations/test_annotations_upload_status_change.py
index 3be566e4d..0637beab3 100644
--- a/tests/integration/annotations/test_annotations_upload_status_change.py
+++ b/tests/integration/annotations/test_annotations_upload_status_change.py
@@ -38,18 +38,6 @@ def test_upload_annotations_from_folder_to_project__upload_status(self, reporter
sa.get_item_metadata(self.PROJECT_NAME, self.IMAGE_NAME)["annotation_status"]
)
- @pytest.mark.flaky(reruns=2)
- @patch("lib.infrastructure.controller.Reporter")
- def test_upload_preannotations_from_folder_to_project__upload_status(self, reporter):
- reporter_mock = MagicMock()
- reporter.return_value = reporter_mock
- sa.upload_image_to_project(self.PROJECT_NAME, join(self.folder_path, self.IMAGE_NAME))
- sa.upload_preannotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path)
- self.assertEqual(
- constances.AnnotationStatus.IN_PROGRESS.name,
- sa.get_item_metadata(self.PROJECT_NAME, self.IMAGE_NAME)["annotation_status"]
- )
-
@pytest.mark.flaky(reruns=2)
@patch("lib.infrastructure.controller.Reporter")
def test_upload_image_annotations__upload_status(self, reporter):
diff --git a/tests/integration/annotations/test_missing_annotation_upload.py b/tests/integration/annotations/test_missing_annotation_upload.py
index 098282f86..51ff0d9d9 100644
--- a/tests/integration/annotations/test_missing_annotation_upload.py
+++ b/tests/integration/annotations/test_missing_annotation_upload.py
@@ -45,23 +45,3 @@ def test_missing_annotation_upload(self):
]
)
)
-
- def test_missing_pre_annotation_upload(self):
-
- sa.upload_images_from_folder_to_project(
- self.PROJECT_NAME, self.folder_path, annotation_status="NotStarted"
- )
- sa.create_annotation_classes_from_classes_json(
- self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json"
- )
- (
- uploaded,
- could_not_upload,
- missing_images,
- ) = sa.upload_preannotations_from_folder_to_project(
- self.PROJECT_NAME, self.folder_path
- )
- self.assertEqual(len(uploaded), 3)
- self.assertEqual(len(could_not_upload), 0)
- self.assertEqual(len(missing_images), 1)
-
diff --git a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py
index 252d46804..59c99d566 100644
--- a/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py
+++ b/tests/integration/annotations/test_upload_annotations_from_folder_to_project.py
@@ -95,7 +95,7 @@ def test_upload_large_annotations(self):
def test_upload_big_annotations(self):
sa.attach_items(
self.PROJECT_NAME,
- [{"name": f"aearth_mov_00{i}.jpg", "url": f"url_{i}"} for i in range(1, 6)] # noqa
+ [{"name": f" rst-lintaearth_mov_00{i}.jpg", "url": f"url_{i}"} for i in range(1, 6)] # noqa
)
sa.create_annotation_classes_from_classes_json(
self.PROJECT_NAME, f"{self.big_annotations_folder_path}/classes/classes.json"
diff --git a/tests/integration/test_basic_images.py b/tests/integration/test_basic_images.py
index b6177c43a..25f835449 100644
--- a/tests/integration/test_basic_images.py
+++ b/tests/integration/test_basic_images.py
@@ -94,18 +94,6 @@ def folder_path(self):
def classes_json_path(self):
return f"{self.folder_path}/classes/classes.json"
- def test_vector_annotations_with_tag_folder_upload_preannotation(self):
- sa.upload_images_from_folder_to_project(
- self.PROJECT_NAME, self.folder_path, annotation_status="InProgress"
- )
- sa.create_annotation_classes_from_classes_json(
- self.PROJECT_NAME, self.classes_json_path
- )
- uploaded_annotations, _, _ = sa.upload_preannotations_from_folder_to_project(
- self.PROJECT_NAME, self.folder_path
- )
- self.assertEqual(len(uploaded_annotations), 1)
-
class TestPixelImages(BaseTestCase):
PROJECT_NAME = "sample_project_pixel"
diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py
index cd333ee31..f43f27000 100644
--- a/tests/integration/test_cli.py
+++ b/tests/integration/test_cli.py
@@ -127,19 +127,6 @@ def test_upload_export(self):
self.assertEqual(len(list(test_dir.glob("*.jpg"))), 0)
self.assertEqual(len(list(test_dir.glob("*.png"))), 0)
- def test_vector_pre_annotation_folder_upload_download_cli(self):
- self._create_project()
-
- sa.create_annotation_classes_from_classes_json(
- self.PROJECT_NAME, f"{self.vector_folder_path}/classes/classes.json"
- )
- self.safe_run(self._cli.upload_images, project=self.PROJECT_NAME, folder=str(self.convertor_data_path),
- extensions="jpg",
- set_annotation_status="QualityCheck")
- self.safe_run(self._cli.upload_preannotations, project=self.PROJECT_NAME, folder=str(self.convertor_data_path),
- format="COCO",
- dataset_name="instances_test")
-
def test_vector_annotation_folder_upload_download_cli(self):
self._create_project()
sa.create_annotation_classes_from_classes_json(
diff --git a/tests/integration/test_depricated_functions_document.py b/tests/integration/test_depricated_functions_document.py
index f021e67fc..0663322c9 100644
--- a/tests/integration/test_depricated_functions_document.py
+++ b/tests/integration/test_depricated_functions_document.py
@@ -83,10 +83,6 @@ def test_deprecated_functions(self):
sa.download_image_annotations(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "./")
except AppException as e:
self.assertIn(self.EXCEPTION_MESSAGE, str(e))
- try:
- sa.copy_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, self.PROJECT_NAME_2)
- except AppException as e:
- self.assertIn(self.EXCEPTION_MESSAGE, str(e))
try:
sa.upload_images_to_project(self.PROJECT_NAME, [self.image_path, ])
except AppException as e:
@@ -103,10 +99,6 @@ def test_deprecated_functions(self):
sa.set_project_workflow(self.PROJECT_NAME, [{}])
except AppException as e:
self.assertIn(self.EXCEPTION_MESSAGE, str(e))
- try:
- sa.upload_preannotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path)
- except AppException as e:
- self.assertIn(self.EXCEPTION_MESSAGE, str(e))
try:
sa.add_annotation_point_to_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, [1, 2], "some class")
except AppException as e:
diff --git a/tests/integration/test_depricated_functions_video.py b/tests/integration/test_depricated_functions_video.py
index 3ad847abb..1160c6472 100644
--- a/tests/integration/test_depricated_functions_video.py
+++ b/tests/integration/test_depricated_functions_video.py
@@ -76,10 +76,6 @@ def test_deprecated_functions(self):
sa.download_image_annotations(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, "./")
except AppException as e:
self.assertIn(self.EXCEPTION_MESSAGE, str(e))
- try:
- sa.copy_image(self.PROJECT_NAME, self.UPLOAD_IMAGE_NAME, self.PROJECT_NAME_2)
- except AppException as e:
- self.assertIn(self.EXCEPTION_MESSAGE, str(e))
try:
sa.upload_images_to_project(self.PROJECT_NAME, [self.image_path, ])
except AppException as e:
@@ -96,10 +92,6 @@ def test_deprecated_functions(self):
sa.set_project_workflow(self.PROJECT_NAME, [{}])
except AppException as e:
self.assertIn(self.EXCEPTION_MESSAGE, str(e))
- try:
- sa.upload_preannotations_from_folder_to_project(self.PROJECT_NAME, self.folder_path)
- except AppException as e:
- self.assertIn(self.EXCEPTION_MESSAGE, str(e))
try:
sa.get_project_workflow(self.PROJECT_NAME)
except AppException as e:
diff --git a/tests/integration/test_limitations.py b/tests/integration/test_limitations.py
index cac030348..a45d575c1 100644
--- a/tests/integration/test_limitations.py
+++ b/tests/integration/test_limitations.py
@@ -1,21 +1,19 @@
import os
-from unittest.mock import patch
from os.path import dirname
+from unittest.mock import patch
-from src.superannotate import SAClient
-sa = SAClient()
from src.superannotate import AppException
+from src.superannotate import SAClient
from src.superannotate.lib.core import UPLOAD_FOLDER_LIMIT_ERROR_MESSAGE
from src.superannotate.lib.core import UPLOAD_PROJECT_LIMIT_ERROR_MESSAGE
from src.superannotate.lib.core import UPLOAD_USER_LIMIT_ERROR_MESSAGE
-from src.superannotate.lib.core import COPY_FOLDER_LIMIT_ERROR_MESSAGE
-from src.superannotate.lib.core import COPY_PROJECT_LIMIT_ERROR_MESSAGE
-from src.superannotate.lib.core import COPY_SUPER_LIMIT_ERROR_MESSAGE
from tests.integration.base import BaseTestCase
from tests.moks.limitatoins import folder_limit_response
from tests.moks.limitatoins import project_limit_response
from tests.moks.limitatoins import user_limit_response
+sa = SAClient()
+
class TestLimitsUploadImagesFromFolderToProject(BaseTestCase):
PROJECT_NAME = "TestLimitsUploadImagesFromFolderToProject"
@@ -49,43 +47,3 @@ def test_user_limitations(self, *_):
_, _, __ = sa.upload_images_from_folder_to_project(
project=self._project["name"], folder_path=self.folder_path
)
-
-
-class TestLimitsCopyImage(BaseTestCase):
- PROJECT_NAME = "TestLimitsCopyImage"
- PROJECT_DESCRIPTION = "Desc"
- PROJECT_TYPE = "Vector"
- TEST_FOLDER_PTH = "data_set"
- TEST_FOLDER_PATH = "data_set/sample_project_vector"
- EXAMPLE_IMAGE_1 = "example_image_1.jpg"
-
- @property
- def folder_path(self):
- return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH)
-
- def test_folder_limitations(self):
- sa.upload_image_to_project(self._project["name"], os.path.join(self.folder_path, self.EXAMPLE_IMAGE_1))
- sa.create_folder(self._project["name"], self._project["name"])
- with patch("lib.infrastructure.serviceprovider.ServiceProvider.get_limitations") as limit_response:
- limit_response.return_value = folder_limit_response
- with self.assertRaisesRegexp(AppException, COPY_FOLDER_LIMIT_ERROR_MESSAGE):
- _, _, __ = sa.copy_image(
- self._project["name"], self.folder_path, f"{self.PROJECT_NAME}/{self.PROJECT_NAME}")
-
- def test_project_limitations(self, ):
- sa.upload_image_to_project(self._project["name"], os.path.join(self.folder_path, self.EXAMPLE_IMAGE_1))
- sa.create_folder(self._project["name"], self._project["name"])
- with patch("lib.infrastructure.serviceprovider.ServiceProvider.get_limitations") as limit_response:
- limit_response.return_value = project_limit_response
- with self.assertRaisesRegexp(AppException, COPY_PROJECT_LIMIT_ERROR_MESSAGE):
- _, _, __ = sa.copy_image(
- self._project["name"], self.folder_path, f"{self.PROJECT_NAME}/{self.PROJECT_NAME}")
-
- def test_user_limitations(self, ):
- sa.upload_image_to_project(self._project["name"], os.path.join(self.folder_path, self.EXAMPLE_IMAGE_1))
- sa.create_folder(self._project["name"], self._project["name"])
- with patch("lib.infrastructure.serviceprovider.ServiceProvider.get_limitations") as limit_response:
- limit_response.return_value = user_limit_response
- with self.assertRaisesRegexp(AppException, COPY_SUPER_LIMIT_ERROR_MESSAGE):
- _, _, __ = sa.copy_image(
- self._project["name"], self.folder_path, f"{self.PROJECT_NAME}/{self.PROJECT_NAME}")