From 9ef9f48a1346985559ef55cf11f1d11c907a0dc0 Mon Sep 17 00:00:00 2001 From: shab Date: Tue, 8 Jun 2021 15:44:59 +0400 Subject: [PATCH 1/2] assign images by chunk --- superannotate/db/project_images.py | 100 +++++++++++++---------------- superannotate/db/utils.py | 80 ++++++++++++++--------- 2 files changed, 94 insertions(+), 86 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 6592b6748..65665d0b8 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -21,7 +21,8 @@ ) from .teams import get_team_metadata from ..mixp.decorators import Trackable -from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, get_image_array_to_upload, __create_image, __copy_images, __move_images, get_project_folder_string +from .utils import _assign_images, _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, \ + get_image_array_to_upload, __create_image, __copy_images, __move_images, get_project_folder_string logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -29,12 +30,12 @@ @Trackable def upload_image_to_project( - project, - img, - image_name=None, - annotation_status="NotStarted", - from_s3_bucket=None, - image_quality_in_editor=None + project, + img, + image_name=None, + annotation_status="NotStarted", + from_s3_bucket=None, + image_quality_in_editor=None ): """Uploads image (io.BytesIO() or filepath to image) to project. Sets status of the uploaded image to set_status if it is not None. @@ -136,8 +137,8 @@ def upload_image_to_project( def _copy_images( - source_project, destination_project, image_names, include_annotations, - copy_annotation_status, copy_pin + source_project, destination_project, image_names, include_annotations, + copy_annotation_status, copy_pin ): NUM_TO_SEND = 500 source_project, source_project_folder = source_project @@ -166,7 +167,7 @@ def _copy_images( res['completed'] = [] for start_index in range(0, len(image_names), NUM_TO_SEND): json_req["image_names"] = image_names[start_index:start_index + - NUM_TO_SEND] + NUM_TO_SEND] response = _api.send_request( req_type='POST', path='/image/copy', @@ -192,12 +193,12 @@ def _copy_images( @Trackable def copy_images( - source_project, - image_names, - destination_project, - include_annotations=True, - copy_annotation_status=True, - copy_pin=True + source_project, + image_names, + destination_project, + include_annotations=True, + copy_annotation_status=True, + copy_pin=True ): """Copy images in bulk between folders in a project @@ -307,12 +308,12 @@ def delete_images(project, image_names): @Trackable def move_images( - source_project, - image_names, - destination_project, - include_annotations=True, - copy_annotation_status=True, - copy_pin=True, + source_project, + image_names, + destination_project, + include_annotations=True, + copy_annotation_status=True, + copy_pin=True, ): """Move images in bulk between folders in a project @@ -372,12 +373,12 @@ def move_images( @Trackable def copy_image( - source_project, - image_name, - destination_project, - include_annotations=False, - copy_annotation_status=False, - copy_pin=False + source_project, + image_name, + destination_project, + include_annotations=False, + copy_annotation_status=False, + copy_pin=False ): """Copy image to a project. The image's project is the same as destination project then the name will be changed to _()., @@ -417,10 +418,10 @@ def copy_image( else: for m in p.finditer(new_name): if m.start() + len(m.group() - ) + len(extension) - 1 == len(new_name): + ) + len(extension) - 1 == len(new_name): num = int(m.group()[2:-2]) new_name = new_name[:m.start() + - 2] + str(num + 1) + ")" + extension + 2] + str(num + 1) + ")" + extension break else: new_name = Path(new_name).stem + "_(1)" + extension @@ -440,9 +441,9 @@ def copy_image( def _copy_annotations_and_metadata( - source_project, source_project_folder, image_name, destination_project, - destination_project_folder, new_name, include_annotations, - copy_annotation_status, copy_pin + source_project, source_project_folder, image_name, destination_project, + destination_project_folder, new_name, include_annotations, + copy_annotation_status, copy_pin ): if include_annotations: annotations = get_image_annotations( @@ -479,12 +480,12 @@ def _copy_annotations_and_metadata( @Trackable def move_image( - source_project, - image_name, - destination_project, - include_annotations=True, - copy_annotation_status=True, - copy_pin=True + source_project, + image_name, + destination_project, + include_annotations=True, + copy_annotation_status=True, + copy_pin=True ): """Move image from source_project to destination_project. source_project and destination_project cannot be the same. @@ -582,23 +583,10 @@ def assign_images(project, image_names, user): if folder: folder_name = folder['name'] - params = {"project_id": project['id'], "team_id": project["team_id"]} - json_req = { - "image_names": image_names, - "assign_user_id": user, - "folder_name": folder_name, - } - response = _api.send_request( - req_type='PUT', - path='/images/editAssignment', - params=params, - json_req=json_req - ) - - if not response.ok: - raise SABaseException( - response.status_code, "Couldn't assign images " + response.text - ) + logs = _assign_images(folder_name=folder_name, image_names=image_names, user=user, project_id=project['id'], + team_id=project['team_id']) + for log in logs: + logger.warn(log) def assign_folder(project, folder_name, users): diff --git a/superannotate/db/utils.py b/superannotate/db/utils.py index 005901450..464ad9684 100644 --- a/superannotate/db/utils.py +++ b/superannotate/db/utils.py @@ -22,7 +22,6 @@ # Yield successive n-sized # chunks from l. def divide_chunks(l, n): - # looping till length l for i in range(0, len(l), n): yield l[i:i + n] @@ -42,7 +41,7 @@ def get_project_folder_string(project): def __move_images( - source_project, source_folder_id, destination_folder_id, image_names + source_project, source_folder_id, destination_folder_id, image_names ): """Move images in bulk between folders in a project @@ -90,8 +89,8 @@ def __move_images( def _copy_images_request( - team_id, project_id, image_names, destination_folder_id, source_folder_id, - include_annotations, copy_pin + team_id, project_id, image_names, destination_folder_id, source_folder_id, + include_annotations, copy_pin ): response = _api.send_request( req_type='POST', @@ -161,8 +160,8 @@ def copy_polling(image_names, source_project, poll_id): def __copy_images( - source_project, source_folder_id, destination_folder_id, image_names, - include_annotations, copy_pin + source_project, source_folder_id, destination_folder_id, image_names, + include_annotations, copy_pin ): """Copy images in bulk between folders in a project @@ -233,8 +232,8 @@ def create_empty_annotation(size, image_name): def upload_image_array_to_s3( - bucket, img_name, img_name_hash, size, orig_image, lores_image, huge_image, - thumbnail_image, prefix + bucket, img_name, img_name_hash, size, orig_image, lores_image, huge_image, + thumbnail_image, prefix ): key = prefix + img_name_hash bucket.put_object(Body=orig_image, Key=key) @@ -256,7 +255,7 @@ def upload_image_array_to_s3( def get_image_array_to_upload( - img_name, byte_io_orig, image_quality_in_editor, project_type + img_name, byte_io_orig, image_quality_in_editor, project_type ): if image_quality_in_editor not in ["original", "compressed"]: raise SABaseException(0, "NA ImageQuality in get_image_array_to_upload") @@ -272,8 +271,8 @@ def get_image_array_to_upload( if resolution > common.MAX_IMAGE_RESOLUTION[project_type]: raise SABaseException( 0, "Image resolution " + str(resolution) + - " too large. Max supported for " + project_type + " projects is " + - str(common.MAX_IMAGE_RESOLUTION[project_type]) + " too large. Max supported for " + project_type + " projects is " + + str(common.MAX_IMAGE_RESOLUTION[project_type]) ) if image_quality_in_editor == "original" and im_format in ['JPEG', 'JPG']: @@ -317,9 +316,9 @@ def get_image_array_to_upload( def __upload_images_to_aws_thread( - res, img_paths, project, annotation_status, prefix, thread_id, chunksize, - couldnt_upload, uploaded, tried_upload, image_quality_in_editor, - from_s3_bucket, project_folder_id + res, img_paths, project, annotation_status, prefix, thread_id, chunksize, + couldnt_upload, uploaded, tried_upload, image_quality_in_editor, + from_s3_bucket, project_folder_id ): len_img_paths = len(img_paths) start_index = thread_id * chunksize @@ -405,14 +404,14 @@ def __upload_images_to_aws_thread( def __create_image( - img_names, - img_paths, - project, - annotation_status, - remote_dir, - sizes, - project_folder_id, - upload_state="Initial" + img_names, + img_paths, + project, + annotation_status, + remote_dir, + sizes, + project_folder_id, + upload_state="Initial" ): if len(img_paths) == 0: return @@ -531,10 +530,9 @@ def get_duplicate_image_names(project_id, team_id, folder_id, image_paths): def _upload_images( - img_paths, team_id, folder_id, project_id, annotation_status, - from_s3_bucket, image_quality_in_editor, project, folder_name + img_paths, team_id, folder_id, project_id, annotation_status, + from_s3_bucket, image_quality_in_editor, project, folder_name ): - _NUM_THREADS = 10 uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] @@ -601,8 +599,8 @@ def _upload_images( def _attach_urls( - img_names_urls, team_id, folder_id, project_id, annotation_status, project, - folder_name + img_names_urls, team_id, folder_id, project_id, annotation_status, project, + folder_name ): _NUM_THREADS = 10 params = {'team_id': team_id, 'folder_id': folder_id} @@ -677,8 +675,8 @@ def _attach_urls( def __attach_image_urls_to_project_thread( - res, img_names_urls, project, annotation_status, prefix, thread_id, - chunksize, couldnt_upload, uploaded, tried_upload, project_folder_id + res, img_names_urls, project, annotation_status, prefix, thread_id, + chunksize, couldnt_upload, uploaded, tried_upload, project_folder_id ): len_img_paths = len(img_names_urls) start_index = thread_id * chunksize @@ -762,4 +760,26 @@ def get_templates_mapping(): templates_map = {} for template in templates: templates_map[template['name']] = template['id'] - return templates_map \ No newline at end of file + return templates_map + + +def _assign_images(folder_name, image_names, user, project_id, team_id): + image_names_lists = divide_chunks(image_names, 500) + params = {"project_id": project_id, "team_id": team_id} + messages = [] + for image_name_list in image_names_lists: + json_req = { + "image_names": image_name_list, + "assign_user_id": user, + "folder_name": folder_name, + } + response = _api.send_request( + req_type='PUT', + path='/images/editAssignment', + params=params, + json_req=json_req + ) + if not response.ok: + message = "Couldn't assign images " + response.text + messages.append(message) + return messages From 29bdb34ec5d78c83bb327c483b1aee580c152584 Mon Sep 17 00:00:00 2001 From: shab Date: Tue, 8 Jun 2021 16:17:55 +0400 Subject: [PATCH 2/2] Fix folder-image assign-unassign --- superannotate/db/project_images.py | 45 +++++++++++------------------- superannotate/db/utils.py | 25 +++++++++++++++++ 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 65665d0b8..08f3d7c39 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -21,7 +21,7 @@ ) from .teams import get_team_metadata from ..mixp.decorators import Trackable -from .utils import _assign_images, _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, \ +from .utils import _unassign_images, _assign_images, _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, \ get_image_array_to_upload, __create_image, __copy_images, __move_images, get_project_folder_string logger = logging.getLogger("superannotate-python-sdk") @@ -566,18 +566,18 @@ def assign_images(project, image_names, user): :param user: user email :type user: str """ - logger.info("Assign %s images to user %s", len(image_names), user) if len(image_names) == 0: return project, folder = get_project_and_folder_metadata(project) - - verified_users = get_team_metadata()["users"] - verified_users = [i['id'] for i in verified_users] + project_meta = get_project_metadata(project, include_contributors=True) + verified_users = project_meta["contributors"] + verified_users = [i['user_id'] for i in verified_users] if user not in verified_users: logging.warn( f'Skipping {user}. {user} is not a verified contributor for the {project["name"]}' ) + return folder_name = 'root' if folder: @@ -587,6 +587,7 @@ def assign_images(project, image_names, user): team_id=project['team_id']) for log in logs: logger.warn(log) + logger.info("Assign images to user %s", user) def assign_folder(project, folder_name, users): @@ -602,9 +603,9 @@ def assign_folder(project, folder_name, users): """ project_meta = get_project_metadata(project, include_contributors=True) - verified_users = get_team_metadata()["users"] + verified_users = project_meta["contributors"] + verified_users = [i['user_id'] for i in verified_users] project_name = project_meta['name'] - verified_users = [i['id'] for i in verified_users] verified_users = set(users).intersection(set(verified_users)) unverified_contributor = set(users) - verified_users @@ -614,6 +615,9 @@ def assign_folder(project, folder_name, users): ) continue + if not verified_users: + return + params = { "project_id": project_meta['id'], "team_id": project_meta["team_id"] @@ -677,27 +681,12 @@ def unassign_images(project, image_names): :type image_names: list of str """ project, folder = get_project_and_folder_metadata(project) + folder_name = 'root' if folder: folder_name = folder['name'] - params = { - "project_id": project['id'], - "team_id": project["team_id"] - } - json_req = { - "image_names": image_names, - "remove_user_ids": ["all"], - "folder_name": folder_name - } - - response = _api.send_request( - req_type='PUT', - path='/images/editAssignment', - params=params, - json_req=json_req - ) - - if not response.ok: - raise SABaseException( - response.status_code, "Couldn't unassign images " + response.text - ) + if not image_names: + return + logs = _unassign_images(folder_name=folder_name,image_names=image_names,project_id=project['id'],team_id=project['team_id']) + for log in logs: + logger.warn(log) diff --git a/superannotate/db/utils.py b/superannotate/db/utils.py index 464ad9684..4cbcf762c 100644 --- a/superannotate/db/utils.py +++ b/superannotate/db/utils.py @@ -783,3 +783,28 @@ def _assign_images(folder_name, image_names, user, project_id, team_id): message = "Couldn't assign images " + response.text messages.append(message) return messages + + +def _unassign_images(folder_name, image_names, project_id, team_id): + image_names_lists = divide_chunks(image_names, 500) + params = {"project_id": project_id, "team_id": team_id} + messages = [] + for image_name_list in image_names_lists: + json_req = { + "image_names": image_name_list, + "remove_user_ids": ["all"], + "folder_name": folder_name, + } + response = _api.send_request( + req_type='PUT', + path='/images/editAssignment', + params=params, + json_req=json_req + ) + if not response.ok: + message = "Couldn't assign images " + response.text + messages.append(message) + return messages + + +