Skip to content

F 193 #226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dshabin merged 3 commits into from
Sep 29, 2021
Merged

F 193 #226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c68a04a
Fix pixel recursive
dshabin Sep 29, 2021
e8244cc
validate
dshabin Sep 29, 2021
dbd9732
Add log
dshabin Sep 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions src/superannotate/lib/app/interface/sdk_interface.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2249,6 +2249,8 @@ def attach_image_urls_to_project(
logger.warning(
constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images))
)
logger.info(constances.ATTACHING_FILES_MESSAGE.format(len(images_to_upload)))

use_case = controller.interactive_attach_urls(
project_name=project_name,
folder_name=folder_name,
Expand Down Expand Up @@ -2392,11 +2394,14 @@ def upload_annotations_from_folder_to_project(
annotation_paths=annotation_paths, # noqa: E203
client_s3_bucket=from_s3_bucket,
)
with tqdm(
total=len(annotation_paths), desc="Uploading annotations"
) as progress_bar:
for _ in use_case.execute():
progress_bar.update(1)
if use_case.is_valid():
with tqdm(
total=len(use_case.annotations_to_upload), desc="Uploading annotations"
) as progress_bar:
for _ in use_case.execute():
progress_bar.update(1)
else:
raise AppException(use_case.response.errors)
return use_case.data


Expand Down Expand Up @@ -3513,6 +3518,7 @@ def attach_document_urls_to_project(
logger.warning(
constances.ALREADY_EXISTING_FILES_WARNING.format(len(duplicate_images))
)
logger.info(constances.ATTACHING_FILES_MESSAGE.format(len(images_to_upload)))
use_case = controller.interactive_attach_urls(
project_name=project_name,
folder_name=folder_name,
Expand Down
12 changes: 9 additions & 3 deletions src/superannotate/lib/core/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,18 @@
ATTACH_USER_LIMIT_ERROR_MESSAGE = "The number of items you want to attach exceeds the limit of your subscription plan."


COPY_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of 50 000 items per folder."
COPY_FOLDER_LIMIT_ERROR_MESSAGE = (
"The number of items you want to copy exceeds the limit of 50 000 items per folder."
)
COPY_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of 500 000 items per project."
COPY_SUPER_LIMIT_ERROR_MESSAGE = "The number of items you want to copy exceeds the limit of your subscription plan."
COPY_SUPER_LIMIT_ERROR_MESSAGE = (
"The number of items you want to copy exceeds the limit of your subscription plan."
)


MOVE_FOLDER_LIMIT_ERROR_MESSAGE = "The number of items you want to move exceeds the limit of 50 000 items per folder."
MOVE_FOLDER_LIMIT_ERROR_MESSAGE = (
"The number of items you want to move exceeds the limit of 50 000 items per folder."
)
MOVE_PROJECT_LIMIT_ERROR_MESSAGE = "The number of items you want to move exceeds the limit of 500 000 items per project."

__alL__ = (
Expand Down
92 changes: 54 additions & 38 deletions src/superannotate/lib/core/usecases/images.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -833,14 +833,17 @@ def execute(self):

outline_color = 4 * (255,)
for instance in self.annotations["instances"]:
if (not instance.get("className")) or (not class_color_map.get(instance["className"])):
if (not instance.get("className")) or (
not class_color_map.get(instance["className"])
):
continue
color = class_color_map.get(instance["className"])
if not color:
class_color_map[instance["className"]] = self.generate_color()
for image in images:
fill_color = (
*class_color_map[instance["className"]], 255 if image.type == "fuse" else self.TRANSPARENCY
*class_color_map[instance["className"]],
255 if image.type == "fuse" else self.TRANSPARENCY,
)
if instance["type"] == "bbox":
image.content.draw_bbox(
Expand Down Expand Up @@ -909,7 +912,9 @@ def execute(self):
weight, height = image.get_size()
empty_image_arr = np.full((height, weight, 4), [0, 0, 0, 255], np.uint8)
for annotation in self.annotations["instances"]:
if (not annotation.get("className")) or (not class_color_map.get(annotation["className"])):
if (not annotation.get("className")) or (
not class_color_map.get(annotation["className"])
):
continue
fill_color = *class_color_map[annotation["className"]], 255
for part in annotation["parts"]:
Expand Down Expand Up @@ -1989,17 +1994,28 @@ def validate_limitations(self):
if self._move:
if self._from_project.uuid == self._to_project.uuid:
if self._from_folder.uuid == self._to_folder.uuid:
raise AppValidationException("Cannot move image if source_project == destination_project.")
raise AppValidationException(
"Cannot move image if source_project == destination_project."
)
elif response.data.folder_limit.remaining_image_count < 1:
raise AppValidationException(constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE)
raise AppValidationException(
constances.MOVE_FOLDER_LIMIT_ERROR_MESSAGE
)
elif response.data.project_limit.remaining_image_count < 1:
raise AppValidationException(constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE)
raise AppValidationException(
constances.MOVE_PROJECT_LIMIT_ERROR_MESSAGE
)
else:
if response.data.folder_limit.remaining_image_count < 1:
raise AppValidationException(constances.COPY_FOLDER_LIMIT_ERROR_MESSAGE)
if response.data.project_limit.remaining_image_count < 1:
raise AppValidationException(constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE)
if response.data.super_user_limit and response.data.super_user_limit.remaining_image_count < 1:
raise AppValidationException(
constances.COPY_PROJECT_LIMIT_ERROR_MESSAGE
)
if (
response.data.super_user_limit
and response.data.super_user_limit.remaining_image_count < 1
):
raise AppValidationException(constances.COPY_SUPER_LIMIT_ERROR_MESSAGE)

def execute(self) -> Response:
Expand Down Expand Up @@ -2662,6 +2678,8 @@ def execute(self):
else:
from_s3 = None

for _ in range(len(annotations_to_upload) - len(response.data.images)):
yield
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.MAX_WORKERS
) as executor:
Expand Down Expand Up @@ -2693,7 +2711,6 @@ def execute(self):
failed_annotations = [
annotation.path for annotation in failed_annotations
]

self._response.data = (
uploaded_annotations,
failed_annotations,
Expand All @@ -2705,42 +2722,41 @@ def execute(self):
def upload_to_s3(
self, image_id: int, image_info, bucket, from_s3, image_id_name_map
):
if from_s3:
file = io.BytesIO()
s3_object = from_s3.Object(
self._client_s3_bucket, image_id_name_map[image_id].path
)
s3_object.download_fileobj(file)
file.seek(0)
annotation_json = json.load(file)
else:
annotation_json = json.load(open(image_id_name_map[image_id].path))

self.fill_classes_data(annotation_json)

if not self._is_valid_json(annotation_json):
logger.warning(f"Invalid json {image_id_name_map[image_id].path}")
return image_id_name_map[image_id], False
bucket.put_object(
Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json),
)
if self._project.project_type == constances.ProjectType.PIXEL.value:
mask_filename = (
image_id_name_map[image_id].name + constances.ANNOTATION_MASK_POSTFIX
)
try:
if from_s3:
file = io.BytesIO()
s3_object = from_s3.Object(
self._client_s3_bucket, f"{self._folder_path}/{mask_filename}"
self._client_s3_bucket, image_id_name_map[image_id].path
)
s3_object.download_fileobj(file)
file.seek(0)
annotation_json = json.load(file)
else:
with open(f"{self._folder_path}/{mask_filename}", "rb") as mask_file:
file = io.BytesIO(mask_file.read())

bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
return image_id_name_map[image_id], True
annotation_json = json.load(open(image_id_name_map[image_id].path))
self.fill_classes_data(annotation_json)
if not self._is_valid_json(annotation_json):
logger.warning(f"Invalid json {image_id_name_map[image_id].path}")
return image_id_name_map[image_id], False
bucket.put_object(
Key=image_info["annotation_json_path"],
Body=json.dumps(annotation_json),
)
if self._project.project_type == constances.ProjectType.PIXEL.value:
mask_path = image_id_name_map[image_id].path.replace(
"___pixel.json", constances.ANNOTATION_MASK_POSTFIX
)
if from_s3:
file = io.BytesIO()
s3_object = from_s3.Object(self._client_s3_bucket, mask_path)
s3_object.download_fileobj(file)
file.seek(0)
else:
with open(mask_path, "rb") as mask_file:
file = io.BytesIO(mask_file.read())
bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
return image_id_name_map[image_id], True
except Exception as _:
return image_id_name_map[image_id], False

def report_missing_data(self):
if self.missing_classes:
Expand Down
32 changes: 32 additions & 0 deletions tests/integration/test_recursive_folder_pixel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import os
from os.path import dirname

import src.superannotate as sa
from tests.integration.base import BaseTestCase


class TestRecursiveFolderPixel(BaseTestCase):
PROJECT_NAME = "pixel_recursive_test"
PROJECT_DESCRIPTION = "Desc"
PROJECT_TYPE = "Pixel"
S3_FOLDER_PATH = "pixel_all_fuse"
JSON_POSTFIX = "*.json"


def test_recursive_upload_pixel(self):
uploaded, _, duplicated = sa.upload_images_from_folder_to_project(self.PROJECT_NAME,
self.S3_FOLDER_PATH,
from_s3_bucket="test-openseadragon-1212",
recursive_subfolders=True
)

uploaded, failed, missing = sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME,
self.S3_FOLDER_PATH,
from_s3_bucket="test-openseadragon-1212",
recursive_subfolders=True
)
self.assertEqual(115, len(uploaded))
self.assertEqual(0, len(failed))
self.assertEqual(0, len(missing))