From 251731216de63f7c67889caa31188e9c7281b10d Mon Sep 17 00:00:00 2001 From: Hovnatan Karapetyan Date: Fri, 2 Apr 2021 10:38:50 +0400 Subject: [PATCH 01/18] Code cleanup --- .pylintrc | 3 ++- superannotate/db/project_images.py | 2 +- superannotate/db/projects.py | 16 ++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3f8379cec..c0bc1b5e9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -23,7 +23,8 @@ disable= no-else-return, too-many-lines, too-many-arguments, - too-many-instance-attributes + too-many-instance-attributes, + broad-except [SPELLING] diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index d528a0015..156791b46 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -120,7 +120,7 @@ def upload_image_to_project( ) key = upload_image_array_to_s3(bucket, *images_info_and_array, prefix) except Exception as e: - raise SABaseException(0, "Couldn't upload to data server. " + str(e)) + raise SABaseException(0, "Couldn't upload to data server.") from e if project_folder is not None: project_folder_id = project_folder["id"] diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index a9334627f..077175bd6 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -265,7 +265,8 @@ def upload_video_to_project( if "ffprobe" in str(e): warning_str = "This could be because ffmpeg package is not installed. To install it, run: sudo apt install ffmpeg" logger.warning( - "Couldn't read video metadata to determine rotation. " + warning_str + "Couldn't read video metadata to determine rotation. %s", + warning_str ) video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) @@ -1082,7 +1083,7 @@ def __attach_image_urls_to_project_thread( for i in range(start_index, end_index): if i >= len_img_paths: break - name, url = img_names_urls[i] + name, _ = img_names_urls[i] tried_upload[thread_id].append(name) img_name_hash = str(uuid.uuid4()) + Path(name).suffix key = prefix + img_name_hash @@ -1186,7 +1187,7 @@ def upload_images_from_public_urls_to_project( ), daemon=True ) - logger.info('Downloading %s images' % len(img_urls)) + logger.info('Downloading %s images', len(img_urls)) tqdm_thread.start() with tempfile.TemporaryDirectory() as save_dir_name: save_dir = Path(save_dir_name) @@ -2225,11 +2226,10 @@ def get_project_default_image_quality_in_editor(project): for setting in get_project_settings(project): if "attribute" in setting and setting["attribute"] == "ImageQuality": return setting["value"] - else: - raise SABaseException( - 0, - "Image quality in editor should be 'compressed', 'original' or None for project settings value" - ) + raise SABaseException( + 0, + "Image quality in editor should be 'compressed', 'original' or None for project settings value" + ) def get_project_metadata( From 32fba219ac93ff8c2de400bc71a7ec3b4c5b7420 Mon Sep 17 00:00:00 2001 From: rcmangasaryan Date: Fri, 2 Apr 2021 15:19:30 +0400 Subject: [PATCH 02/18] Update logging for attach images --- superannotate/db/projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 077175bd6..7719a9f0b 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -1000,7 +1000,7 @@ def attach_image_urls_to_project( img_names_urls = image_data.values.tolist() len_img_names_urls = len(img_names_urls) logger.info( - "Uploading %s images to project %s.", len_img_names_urls, + "Attaching %s images to project %s.", len_img_names_urls, project_folder_name ) if len_img_names_urls == 0: From 664641e80e3dc92a2822f8ee5a07e2e91d7d0c5b Mon Sep 17 00:00:00 2001 From: shab Date: Mon, 5 Apr 2021 15:16:26 +0400 Subject: [PATCH 03/18] Add limitations --- superannotate/db/images.py | 34 +++++----- superannotate/db/project_images.py | 32 ++++----- superannotate/db/projects.py | 103 +++++++++++++++-------------- superannotate/db/utils.py | 19 ++++++ 4 files changed, 102 insertions(+), 86 deletions(-) create mode 100644 superannotate/db/utils.py diff --git a/superannotate/db/images.py b/superannotate/db/images.py index 6a268068a..49e66a382 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -231,30 +231,32 @@ def get_image_metadata(project, image_names, return_dict_on_single_output=True): else: project_folder_id = None + chunk_size = 500 + chunks = [image_names[i:i + chunk_size] for i in range(0, len(image_names), chunk_size)] + json_req = { 'project_id': project['id'], 'team_id': _api.team_id, - 'names': image_names, } + if project_folder_id is not None: json_req["folder_id"] = project_folder_id - response = _api.send_request( - req_type='POST', - path='/images/getBulk', - json_req=json_req, - ) - if not response.ok: - raise SABaseException( - response.status_code, - "Couldn't get image metadata. " + response.text - ) - metadata_raw = response.json() + metadata_raw = [] + for chunk in chunks: + json_req['names'] = chunk + response = _api.send_request( + req_type='POST', + path='/images/getBulk', + json_req=json_req, + ) + if not response.ok: + raise SABaseException(response.status_code,"Couldn't get image metadata. " + response.text) + metadata_raw += response.json() + metadata_without_deleted = [] - for im_metadata in metadata_raw: - if 'delete' in im_metadata and im_metadata['delete'] == 1: - continue - metadata_without_deleted.append(im_metadata) + metadata_without_deleted = [ i for i in metadata_raw if i['delete'] != 1 ] + if len(metadata_without_deleted) == 0: raise SABaseException( 0, diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index c952489cf..d1443606e 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -19,6 +19,7 @@ __create_image, get_image_array_to_upload, get_project_default_image_quality_in_editor, upload_image_array_to_s3 ) +from .utils import _get_upload_auth_token logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -50,7 +51,7 @@ def upload_image_to_project( Can be either "compressed" or "original". If None then the default value in project settings will be used. :type image_quality_in_editor: str """ - project, project_folder = get_project_and_folder_metadata(project) + project, folder = get_project_and_folder_metadata(project) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": raise SABaseException( @@ -92,21 +93,18 @@ def upload_image_to_project( 0, "Image name img_name should be set if img is not Pathlike" ) + if folder: + folder_id = folder["id"] + else: + folder_id = get_project_root_folder_id(project) + team_id, project_id = project["team_id"], project["id"] params = { 'team_id': team_id, + 'folder_id' : folder_id } - response = _api.send_request( - req_type='GET', - path=f'/project/{project_id}/sdkImageUploadToken', - params=params - ) - if not response.ok: - raise SABaseException( - response.status_code, "Couldn't get upload token " + response.text - ) - res = response.json() - prefix = res['filePath'] + res = _get_upload_auth_token(params=params,project_id=project_id) + prefix = res['filePath'] s3_session = boto3.Session( aws_access_key_id=res['accessKeyId'], aws_secret_access_key=res['secretAccessKey'], @@ -122,16 +120,12 @@ def upload_image_to_project( except Exception as e: raise SABaseException(0, "Couldn't upload to data server. " + str(e)) - if project_folder is not None: - project_folder_id = project_folder["id"] - else: - project_folder_id = None __create_image( [img_name], [key], project, annotation_status, prefix, [images_info_and_array[2]], - project_folder_id, + folder_id, upload_state="Basic" ) @@ -171,7 +165,7 @@ def _copy_images( destination_folder_id = get_project_root_folder_id(destination_project) json_req["destination_folder_id"] = destination_folder_id res = {} - res['skipped'] = 0 + res['skipped'] = [] for start_index in range(0, len(image_names), NUM_TO_SEND): json_req["image_names"] = image_names[start_index:start_index + NUM_TO_SEND] @@ -239,7 +233,7 @@ def copy_images( source_project["name"] + "" if source_project_folder is None else "/" + source_project_folder["name"], destination_project["name"] + "" if destination_project_folder is None else "/" + - destination_project_folder["name"], res["skipped"] + destination_project_folder["name"], len(res["skipped"]) ) return res["skipped"] diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 5713414df..7fc33a58a 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -34,12 +34,13 @@ fill_class_and_attribute_ids, get_annotation_classes_name_to_id, search_annotation_classes ) -from .images import get_image_metadata, search_images, search_images_all_folders +from .images import get_image_metadata, search_images, search_images_all_folders, get_project_root_folder_id from .project_api import ( get_project_and_folder_metadata, get_project_metadata_bare, get_project_metadata_with_users ) from .users import get_team_contributor_metadata +from .utils import _get_upload_auth_token logger = logging.getLogger("superannotate-python-sdk") @@ -827,9 +828,9 @@ def upload_images_to_project( :return: uploaded, could-not-upload, existing-images filepaths :rtype: tuple (3 members) of list of strs """ - project, project_folder = get_project_and_folder_metadata(project) - project_folder_name = project["name"] + ( - f'/{project_folder["name"]}' if project_folder else "" + project, folder = get_project_and_folder_metadata(project) + folder_name = project["name"] + ( + f'/{folder["name"]}' if folder else "" ) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": @@ -847,7 +848,7 @@ def upload_images_to_project( project ) team_id, project_id = project["team_id"], project["id"] - existing_images = search_images((project, project_folder)) + existing_images = search_images((project, folder)) duplicate_images = [] for existing_image in existing_images: i = -1 @@ -865,31 +866,31 @@ def upload_images_to_project( ) len_img_paths = len(img_paths) logger.info( - "Uploading %s images to project %s.", len_img_paths, project_folder_name + "Uploading %s images to project %s.", len_img_paths, folder_name ) if len_img_paths == 0: return ([], [], duplicate_images) - params = {'team_id': team_id} + + + if folder: + folder_id = folder["id"] + else: + folder_id = get_project_root_folder_id(project) + + params = {'team_id': team_id , 'folder_id' : folder_id } uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] couldnt_upload = [[] for _ in range(_NUM_THREADS)] finish_event = threading.Event() - chunksize = int(math.ceil(len(img_paths) / _NUM_THREADS)) - response = _api.send_request( - req_type='GET', - path=f'/project/{project_id}/sdkImageUploadToken', - params=params - ) - if not response.ok: - raise SABaseException( - response.status_code, "Couldn't get upload token " + response.text - ) - if project_folder is not None: - project_folder_id = project_folder["id"] - else: - project_folder_id = None - res = response.json() + + res = _get_upload_auth_token(params=params,project_id=project_id) + prefix = res['filePath'] + limit = res['availableImageCount'] + images_to_upload = img_paths[:limit] + images_to_skip = img_paths[limit:] + chunksize = int(math.ceil(len(images_to_upload) / _NUM_THREADS)) + tqdm_thread = threading.Thread( target=__tqdm_thread_image_upload, args=(len_img_paths, tried_upload, finish_event), @@ -902,9 +903,9 @@ def upload_images_to_project( t = threading.Thread( target=__upload_images_to_aws_thread, args=( - res, img_paths, project, annotation_status, prefix, thread_id, + res, images_to_upload, project, annotation_status, prefix, thread_id, chunksize, couldnt_upload, uploaded, tried_upload, - image_quality_in_editor, from_s3_bucket, project_folder_id + image_quality_in_editor, from_s3_bucket, folder_id ), daemon=True ) @@ -923,6 +924,7 @@ def upload_images_to_project( for f in upload_thread: list_of_uploaded.append(str(f)) + list_of_not_uploaded += images_to_skip return (list_of_uploaded, list_of_not_uploaded, duplicate_images) @@ -960,9 +962,9 @@ def attach_image_urls_to_project( :rtype: tuple """ - project, project_folder = get_project_and_folder_metadata(project) - project_folder_name = project["name"] + ( - f'/{project_folder["name"]}' if project_folder else "" + project, folder = get_project_and_folder_metadata(project) + folder_name = project["name"] + ( + f'/{folder["name"]}' if folder else "" ) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "Basic": @@ -978,7 +980,7 @@ def attach_image_urls_to_project( duplicate_idx_csv = existing_names.duplicated(subset="name", keep="first") duplicate_images = existing_names[duplicate_idx_csv]["name"].tolist() existing_names = existing_names[~duplicate_idx_csv] - existing_images = search_images((project, project_folder)) + existing_images = search_images((project, folder)) duplicate_idx = [] for ind, _ in image_data[image_data["name"].isnull()].iterrows(): while True: @@ -1000,37 +1002,35 @@ def attach_image_urls_to_project( ) image_data = pd.DataFrame(image_data, columns=["name", "url"]) img_names_urls = image_data.values.tolist() - len_img_names_urls = len(img_names_urls) logger.info( - "Uploading %s images to project %s.", len_img_names_urls, - project_folder_name + "Uploading %s images to project %s.", len(img_names_urls), + folder_name ) - if len_img_names_urls == 0: + if len(img_names_urls) == 0: return ([], [], duplicate_images) - params = {'team_id': team_id} + + if folder: + folder_id = folder["id"] + else: + folder_id = get_project_root_folder_id(project) + + params = {'team_id': team_id , 'folder_id' : folder_id } uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] couldnt_upload = [[] for _ in range(_NUM_THREADS)] finish_event = threading.Event() - chunksize = int(math.ceil(len_img_names_urls / _NUM_THREADS)) - response = _api.send_request( - req_type='GET', - path=f'/project/{project_id}/sdkImageUploadToken', - params=params - ) - if not response.ok: - raise SABaseException( - response.status_code, "Couldn't get upload token " + response.text - ) - if project_folder is not None: - project_folder_id = project_folder["id"] - else: - project_folder_id = None - res = response.json() + + res = _get_upload_auth_token(params=params,project_id=project_id) + prefix = res['filePath'] + limit = res['availableImageCount'] + images_to_upload = img_names_urls[:limit] + images_to_skip = img_names_urls[limit:] + chunksize = int(math.ceil(len(images_to_upload) / _NUM_THREADS)) + tqdm_thread = threading.Thread( target=__tqdm_thread_image_upload, - args=(len_img_names_urls, tried_upload, finish_event), + args=(len(images_to_upload), tried_upload, finish_event), daemon=True ) tqdm_thread.start() @@ -1039,9 +1039,9 @@ def attach_image_urls_to_project( t = threading.Thread( target=__attach_image_urls_to_project_thread, args=( - res, img_names_urls, project, annotation_status, prefix, + res, images_to_upload, project, annotation_status, prefix, thread_id, chunksize, couldnt_upload, uploaded, tried_upload, - project_folder_id + folder_id ), daemon=True ) @@ -1060,6 +1060,7 @@ def attach_image_urls_to_project( for f in upload_thread: list_of_uploaded.append(str(f)) + list_of_not_uploaded += [i[0] for i in images_to_skip ] return (list_of_uploaded, list_of_not_uploaded, duplicate_images) diff --git a/superannotate/db/utils.py b/superannotate/db/utils.py new file mode 100644 index 000000000..41078b942 --- /dev/null +++ b/superannotate/db/utils.py @@ -0,0 +1,19 @@ +from ..api import API +from ..exceptions import SABaseException, SAImageSizeTooLarge +_api = API.get_instance() + + +def _get_upload_auth_token(params,project_id): + response = _api.send_request( + req_type='GET', + path=f'/project/{project_id}/sdkImageUploadToken', + params=params + ) + if not response.ok: + raise SABaseException( + response.status_code, "Couldn't get upload token " + response.text + ) + + res = response.json() + return res + From 3eaa8f5abe1ea64f7e28baf6bc9ffc578c3b9283 Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Mon, 5 Apr 2021 18:12:50 +0400 Subject: [PATCH 04/18] Update version.py Update verison --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index 13ffcf429..879379eeb 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.2" +__version__ = "4.1.3b1" From 3a3d644005745072764f4eec70d8bf4639764ae1 Mon Sep 17 00:00:00 2001 From: shab Date: Tue, 6 Apr 2021 12:47:24 +0400 Subject: [PATCH 05/18] Fix training --- superannotate/common.py | 8 ++++++++ superannotate/db/search_projects.py | 6 ++++-- superannotate/ml/ml_funcs.py | 6 +++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/superannotate/common.py b/superannotate/common.py index aa591d74e..5568a19e3 100644 --- a/superannotate/common.py +++ b/superannotate/common.py @@ -51,6 +51,14 @@ "Failed": 4 } +_MODEL_TRAINING_TASKS = { + "Instance Segmentation for Pixel Projects" : "instance_segmentation_pixel", + "Instance Segmentation for Vector Projects" : "instance_segmentation_vector", + "Keypoint Detection for Vector Projects" : "keypoint_detection_vector", + "Object Detection for Vector Projects" : "object_detection_vector", + "Semantic Segmentation for Pixel Projects" : "semantic_segmentation_pixel" +} + def prediction_segmentation_status_from_str_to_int(status): return _PREDICTION_SEGMENTATION_STATUSES[status] diff --git a/superannotate/db/search_projects.py b/superannotate/db/search_projects.py index 6e20a3226..20b9cc25c 100644 --- a/superannotate/db/search_projects.py +++ b/superannotate/db/search_projects.py @@ -23,9 +23,11 @@ def search_projects( :rtype: list of strs or dicts """ result_list = [] + limit = 1000 params = { 'team_id': str(_api.team_id), 'offset': 0, + 'limit' : limit, 'completeImagesCount': include_complete_image_count } if name is not None: @@ -37,9 +39,9 @@ def search_projects( if response.ok: new_results = response.json() result_list += new_results["data"] - if new_results["count"] <= len(result_list): + params["offset"] += limit + if params["offset"] >= new_results["count"]: break - params["offset"] = len(result_list) else: raise SABaseException( response.status_code, diff --git a/superannotate/ml/ml_funcs.py b/superannotate/ml/ml_funcs.py index 75b34159f..e1550ef2b 100644 --- a/superannotate/ml/ml_funcs.py +++ b/superannotate/ml/ml_funcs.py @@ -17,7 +17,7 @@ from ..api import API from ..common import ( _AVAILABLE_SEGMENTATION_MODELS, model_training_status_int_to_str, - project_type_str_to_int, upload_state_int_to_str + project_type_str_to_int, upload_state_int_to_str,_MODEL_TRAINING_TASKS ) from ..db.images import get_image_metadata, search_images from ..exceptions import SABaseException @@ -253,11 +253,11 @@ def run_training( hyperparameters[item] = DEFAULT_HYPERPARAMETERS[item] complete_image_count = 0 for proj in project: - complete_image_count += proj['completedImagesCount'] + complete_image_count += proj['rootFolderCompletedImagesCount'] hyperparameters["name"] = model_name hyperparameters["description"] = model_description - hyperparameters["task"] = task + hyperparameters["task"] = _MODEL_TRAINING_TASKS[task] hyperparameters["base_model_id"] = base_model["id"] hyperparameters["project_ids"] = project_ids hyperparameters["image_count"] = complete_image_count From 8f2451098cbe872e941b4bb56c1b03f664540d9d Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Tue, 6 Apr 2021 18:12:56 +0400 Subject: [PATCH 06/18] Update version.py --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index 879379eeb..c1c69ac1c 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.3b1" +__version__ = "4.1.3b2" From 82f3fa239ab796cba22d8a5a949a2f0ec041c234 Mon Sep 17 00:00:00 2001 From: shab Date: Thu, 8 Apr 2021 12:31:46 +0400 Subject: [PATCH 07/18] Video upload cleanup --- superannotate/__init__.py | 2 +- superannotate/db/project_images.py | 4 +- superannotate/db/projects.py | 214 ++++++++++++++++------------- 3 files changed, 125 insertions(+), 95 deletions(-) diff --git a/superannotate/__init__.py b/superannotate/__init__.py index d5483db84..ecefca8f5 100644 --- a/superannotate/__init__.py +++ b/superannotate/__init__.py @@ -50,7 +50,7 @@ def consensus(*args, **kwargs): download_image_preannotations, get_image_annotations, get_image_bytes, get_image_metadata, get_image_preannotations, search_images, search_images_all_folders, set_image_annotation_status, - set_images_annotation_statuses, upload_image_annotations + set_images_annotation_statuses, upload_image_annotations, get_project_root_folder_id ) from .db.project_api import ( create_folder, delete_folders, get_folder_metadata, diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 437856820..c6273e7b1 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -213,6 +213,8 @@ def copy_images( :type copy_annotation_status: bool :param copy_pin: enables image pin status copy :type copy_pin: bool + :return: list of skipped image names + :rtype: list of strs """ source_project, source_project_folder = get_project_and_folder_metadata( source_project @@ -232,7 +234,7 @@ def copy_images( image_names, source_project["name"] + "" if source_project_folder is None else "/" + source_project_folder["name"], destination_project["name"] + - "" if destination_project_folder is None else "/" + + "" if destination_project_folder is None else destination_project["name"] + "/" + destination_project_folder["name"], len(res["skipped"]) ) return res["skipped"] diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 1c664440f..5178ca09e 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -205,57 +205,63 @@ def get_project_image_count(project, with_all_subfolders=False): return len(search_images_all_folders(project)) -def upload_video_to_project( - project, - video_path, - target_fps=None, - start_time=0.0, - end_time=None, - annotation_status="NotStarted", - image_quality_in_editor=None -): - """Uploads image frames from video to platform. Uploaded images will have - names "_.jpg". +def _get_video_frames_count(video_path): + """ + Get video frames count + """ + video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) + total_num_of_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + if total_num_of_frames < 0: + total_num_of_frames = 0 + flag = True + while flag: + flag, _ = video.read() + if flag: + total_num_of_frames += 1 + else: + break + return total_num_of_frames - :param project: project name or folder path (e.g., "project1/folder1") - :type project: str - :param video_path: video to upload - :type video_path: Pathlike (str or Path) - :param target_fps: how many frames per second need to extract from the video (approximate). - If None, all frames will be uploaded - :type target_fps: float - :param start_time: Time (in seconds) from which to start extracting frames - :type start_time: float - :param end_time: Time (in seconds) up to which to extract frames. If None up to end - :type end_time: float - :param annotation_status: value to set the annotation statuses of the uploaded - video frames NotStarted InProgress QualityCheck Returned Completed Skipped - :type annotation_status: str - :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. - Can be either "compressed" or "original". If None then the default value in project settings will be used. - :type image_quality_in_editor: str - :return: filenames of uploaded images - :rtype: list of strs +def _get_video_fps_ration(target_fps,video,ratio): """ - project, project_folder = get_project_and_folder_metadata(project) - upload_state = common.upload_state_int_to_str(project.get("upload_state")) - if upload_state == "External": - raise SABaseException( - 0, - "The function does not support projects containing images attached with URLs" + Get video fps / target fps ratio + """ + video_fps = float(video.get(cv2.CAP_PROP_FPS)) + if target_fps >= video_fps: + logger.warning( + "Video frame rate %s smaller than target frame rate %s. Cannot change frame rate.", + video_fps, target_fps + ) + else: + logger.info( + "Changing video frame rate from %s to target frame rate %s.", + video_fps, target_fps ) - logger.info("Uploading from video %s.", str(video_path)) + ratio = video_fps / target_fps + return ratio + +def _get_available_image_counts(project,folder): + if folder: + folder_id = folder["id"] + else: + folder_id = get_project_root_folder_id(project) + params = {'team_id': project['team_id'] , 'folder_id' : folder_id } + res = _get_upload_auth_token(params=params,project_id=project['id']) + return res['availableImageCount'] + +def _get_video_rotate_code(video_path): rotate_code = None try: + cv2_rotations = { + 90 : cv2.ROTATE_90_CLOCKWISE, + 180 : cv2.ROTATE_180, + 270 :cv2.ROTATE_90_COUNTERCLOCKWISE, + } + meta_dict = ffmpeg.probe(str(video_path)) rot = int(meta_dict['streams'][0]['tags']['rotate']) - if rot == 90: - rotate_code = cv2.ROTATE_90_CLOCKWISE - elif rot == 180: - rotate_code = cv2.ROTATE_180 - elif rot == 270: - rotate_code = cv2.ROTATE_90_COUNTERCLOCKWISE + rotate_code = cv2_rotations[rot] if rot != 0: logger.info( "Frame rotation of %s found. Output images will be rotated accordingly.", @@ -269,61 +275,30 @@ def upload_video_to_project( "Couldn't read video metadata to determine rotation. %s", warning_str ) + return rotate_code - video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) - if not video.isOpened(): - raise SABaseException(0, "Couldn't open video file " + str(video_path)) - - total_num_of_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) - if total_num_of_frames < 0: - total_num_of_frames = 0 - flag = True - while flag: - flag, frame = video.read() - if flag: - total_num_of_frames += 1 - else: - break - video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) - logger.info("Video frame count is %s.", total_num_of_frames) - - r = 1.0 - if target_fps is not None: - video_fps = float(video.get(cv2.CAP_PROP_FPS)) - if target_fps >= video_fps: - logger.warning( - "Video frame rate %s smaller than target frame rate %s. Cannot change frame rate.", - video_fps, target_fps - ) - else: - logger.info( - "Changing video frame rate from %s to target frame rate %s.", - video_fps, target_fps - ) - r = video_fps / target_fps - - zero_fill_count = len(str(total_num_of_frames)) - tempdir = tempfile.TemporaryDirectory() +def _extract_frames_from_video(start_time,end_time,ratio,video,video_path,tempdir,limit,rotate_code,total_num_of_frames): video_name = Path(video_path).stem frame_no = 0 frame_no_with_change = 1.0 extracted_frame_no = 1 logger.info("Extracting frames from video to %s.", tempdir.name) - while True: + zero_fill_count = len(str(total_num_of_frames)) + while extracted_frame_no < (limit + 1) : success, frame = video.read() if not success: break frame_no += 1 if round(frame_no_with_change) != frame_no: continue - frame_no_with_change += r + frame_no_with_change += ratio frame_time = video.get(cv2.CAP_PROP_POS_MSEC) / 1000.0 - if end_time is not None and frame_time > end_time: + if end_time and frame_time > end_time: break if frame_time < start_time: continue - if rotate_code is not None: + if rotate_code: frame = cv2.rotate(frame, rotate_code) cv2.imwrite( str( @@ -334,26 +309,79 @@ def upload_video_to_project( ), frame ) extracted_frame_no += 1 + return extracted_frame_no - 1 + + +def upload_video_to_project( + project, + video_path, + target_fps=None, + start_time=0.0, + end_time=None, + annotation_status="NotStarted", + image_quality_in_editor=None +): + """Uploads image frames from video to platform. Uploaded images will have + names "_.jpg". + + :param project: project name or folder path (e.g., "project1/folder1") + :type project: str + :param video_path: video to upload + :type video_path: Pathlike (str or Path) + :param target_fps: how many frames per second need to extract from the video (approximate). + If None, all frames will be uploaded + :type target_fps: float + :param start_time: Time (in seconds) from which to start extracting frames + :type start_time: float + :param end_time: Time (in seconds) up to which to extract frames. If None up to end + :type end_time: float + :param annotation_status: value to set the annotation statuses of the uploaded + video frames NotStarted InProgress QualityCheck Returned Completed Skipped + :type annotation_status: str + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. + Can be either "compressed" or "original". If None then the default value in project settings will be used. + :type image_quality_in_editor: str + + :return: filenames of uploaded images + :rtype: list of strs + """ + + project, folder = get_project_and_folder_metadata(project) + limit = _get_available_image_counts(project,folder) + + upload_state = common.upload_state_int_to_str(project.get("upload_state")) + if upload_state == "External": + raise SABaseException( + 0, + "The function does not support projects containing images attached with URLs" + ) + logger.info("Uploading from video %s.", str(video_path)) + rotate_code = _get_video_rotate_code(video_path) + video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) + if not video.isOpened(): + raise SABaseException(0, "Couldn't open video file " + str(video_path)) + total_num_of_frames = _get_video_frames_count(video_path) + logger.info("Video frame count is %s.", total_num_of_frames) + ratio = 1.0 + if target_fps: + ratio = _get_video_fps_ration(target_fps,video,ratio) + tempdir = tempfile.TemporaryDirectory() + extracted_frame_no = _extract_frames_from_video(start_time,end_time,ratio, + video,video_path,tempdir, + limit,rotate_code,total_num_of_frames) logger.info( "Extracted %s frames from video. Now uploading to platform.", - extracted_frame_no - 1 + extracted_frame_no ) - filenames = upload_images_from_folder_to_project( - (project, project_folder), + (project, folder), tempdir.name, extensions=["jpg"], annotation_status=annotation_status, image_quality_in_editor=image_quality_in_editor ) - - assert len(filenames[1]) == 0 - - filenames_base = [] - for file in filenames[0]: - filenames_base.append(Path(file).name) - + filenames_base = [Path(f).name for f in filenames[0]] return filenames_base @@ -398,7 +426,7 @@ def upload_videos_from_folder_to_project( :return: uploaded and not-uploaded video frame images' filenames :rtype: tuple of list of strs """ - project, project_folder = get_project_and_folder_metadata(project) + project, folder = get_project_and_folder_metadata(project) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": raise SABaseException( @@ -440,7 +468,7 @@ def upload_videos_from_folder_to_project( filenames = [] for path in filtered_paths: filenames += upload_video_to_project( - (project, project_folder), + (project, folder), path, target_fps=target_fps, start_time=start_time, From 7fc7034b249b549476c46a7cc1e498bbc068a66e Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Thu, 8 Apr 2021 12:34:55 +0400 Subject: [PATCH 08/18] Update version.py --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index c1c69ac1c..dc4b8a9d6 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.3b2" +__version__ = "4.1.3b3" From fdf0c2b2fd4d913e1186b289e4cf8230606c1961 Mon Sep 17 00:00:00 2001 From: shab Date: Thu, 8 Apr 2021 17:32:23 +0400 Subject: [PATCH 09/18] Fix video frames count --- superannotate/db/projects.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 5178ca09e..6ebe26158 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -210,16 +210,14 @@ def _get_video_frames_count(video_path): Get video frames count """ video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) - total_num_of_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) - if total_num_of_frames < 0: - total_num_of_frames = 0 - flag = True - while flag: - flag, _ = video.read() - if flag: - total_num_of_frames += 1 - else: - break + total_num_of_frames = 0 + flag = True + while flag: + flag, _ = video.read() + if flag: + total_num_of_frames += 1 + else: + break return total_num_of_frames From e464097a35e18ddf6093842d3a4f4f0b2268e162 Mon Sep 17 00:00:00 2001 From: Erik Harutyunyan Date: Fri, 9 Apr 2021 15:40:04 +0400 Subject: [PATCH 10/18] public links name column dtype fix --- superannotate/db/projects.py | 66 ++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 5178ca09e..1f2629e7c 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -223,7 +223,7 @@ def _get_video_frames_count(video_path): return total_num_of_frames -def _get_video_fps_ration(target_fps,video,ratio): +def _get_video_fps_ration(target_fps, video, ratio): """ Get video fps / target fps ratio """ @@ -232,7 +232,7 @@ def _get_video_fps_ration(target_fps,video,ratio): logger.warning( "Video frame rate %s smaller than target frame rate %s. Cannot change frame rate.", video_fps, target_fps - ) + ) else: logger.info( "Changing video frame rate from %s to target frame rate %s.", @@ -241,22 +241,24 @@ def _get_video_fps_ration(target_fps,video,ratio): ratio = video_fps / target_fps return ratio -def _get_available_image_counts(project,folder): + +def _get_available_image_counts(project, folder): if folder: folder_id = folder["id"] else: folder_id = get_project_root_folder_id(project) - params = {'team_id': project['team_id'] , 'folder_id' : folder_id } - res = _get_upload_auth_token(params=params,project_id=project['id']) + params = {'team_id': project['team_id'], 'folder_id': folder_id} + res = _get_upload_auth_token(params=params, project_id=project['id']) return res['availableImageCount'] + def _get_video_rotate_code(video_path): rotate_code = None try: cv2_rotations = { - 90 : cv2.ROTATE_90_CLOCKWISE, - 180 : cv2.ROTATE_180, - 270 :cv2.ROTATE_90_COUNTERCLOCKWISE, + 90: cv2.ROTATE_90_CLOCKWISE, + 180: cv2.ROTATE_180, + 270: cv2.ROTATE_90_COUNTERCLOCKWISE, } meta_dict = ffmpeg.probe(str(video_path)) @@ -278,14 +280,17 @@ def _get_video_rotate_code(video_path): return rotate_code -def _extract_frames_from_video(start_time,end_time,ratio,video,video_path,tempdir,limit,rotate_code,total_num_of_frames): +def _extract_frames_from_video( + start_time, end_time, ratio, video, video_path, tempdir, limit, rotate_code, + total_num_of_frames +): video_name = Path(video_path).stem frame_no = 0 frame_no_with_change = 1.0 extracted_frame_no = 1 logger.info("Extracting frames from video to %s.", tempdir.name) zero_fill_count = len(str(total_num_of_frames)) - while extracted_frame_no < (limit + 1) : + while extracted_frame_no < (limit + 1): success, frame = video.read() if not success: break @@ -347,7 +352,7 @@ def upload_video_to_project( """ project, folder = get_project_and_folder_metadata(project) - limit = _get_available_image_counts(project,folder) + limit = _get_available_image_counts(project, folder) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": @@ -365,11 +370,12 @@ def upload_video_to_project( logger.info("Video frame count is %s.", total_num_of_frames) ratio = 1.0 if target_fps: - ratio = _get_video_fps_ration(target_fps,video,ratio) + ratio = _get_video_fps_ration(target_fps, video, ratio) tempdir = tempfile.TemporaryDirectory() - extracted_frame_no = _extract_frames_from_video(start_time,end_time,ratio, - video,video_path,tempdir, - limit,rotate_code,total_num_of_frames) + extracted_frame_no = _extract_frames_from_video( + start_time, end_time, ratio, video, video_path, tempdir, limit, + rotate_code, total_num_of_frames + ) logger.info( "Extracted %s frames from video. Now uploading to platform.", extracted_frame_no @@ -855,9 +861,7 @@ def upload_images_to_project( :rtype: tuple (3 members) of list of strs """ project, folder = get_project_and_folder_metadata(project) - folder_name = project["name"] + ( - f'/{folder["name"]}' if folder else "" - ) + folder_name = project["name"] + (f'/{folder["name"]}' if folder else "") upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": raise SABaseException( @@ -897,19 +901,18 @@ def upload_images_to_project( if len_img_paths == 0: return ([], [], duplicate_images) - if folder: folder_id = folder["id"] else: folder_id = get_project_root_folder_id(project) - params = {'team_id': team_id , 'folder_id' : folder_id } + params = {'team_id': team_id, 'folder_id': folder_id} uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] couldnt_upload = [[] for _ in range(_NUM_THREADS)] finish_event = threading.Event() - res = _get_upload_auth_token(params=params,project_id=project_id) + res = _get_upload_auth_token(params=params, project_id=project_id) prefix = res['filePath'] limit = res['availableImageCount'] @@ -929,8 +932,8 @@ def upload_images_to_project( t = threading.Thread( target=__upload_images_to_aws_thread, args=( - res, images_to_upload, project, annotation_status, prefix, thread_id, - chunksize, couldnt_upload, uploaded, tried_upload, + res, images_to_upload, project, annotation_status, prefix, + thread_id, chunksize, couldnt_upload, uploaded, tried_upload, image_quality_in_editor, from_s3_bucket, folder_id ), daemon=True @@ -989,9 +992,7 @@ def attach_image_urls_to_project( """ project, folder = get_project_and_folder_metadata(project) - folder_name = project["name"] + ( - f'/{folder["name"]}' if folder else "" - ) + folder_name = project["name"] + (f'/{folder["name"]}' if folder else "") upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "Basic": raise SABaseException( @@ -1000,7 +1001,7 @@ def attach_image_urls_to_project( ) annotation_status = common.annotation_status_str_to_int(annotation_status) team_id, project_id = project["team_id"], project["id"] - image_data = pd.read_csv(attachments) + image_data = pd.read_csv(attachments, dtype=str) image_data = image_data[~image_data["url"].isnull()] existing_names = image_data[~image_data["name"].isnull()] duplicate_idx_csv = existing_names.duplicated(subset="name", keep="first") @@ -1029,8 +1030,7 @@ def attach_image_urls_to_project( image_data = pd.DataFrame(image_data, columns=["name", "url"]) img_names_urls = image_data.values.tolist() logger.info( - "Uploading %s images to project %s.", len(img_names_urls), - folder_name + "Uploading %s images to project %s.", len(img_names_urls), folder_name ) if len(img_names_urls) == 0: return ([], [], duplicate_images) @@ -1040,14 +1040,14 @@ def attach_image_urls_to_project( else: folder_id = get_project_root_folder_id(project) - params = {'team_id': team_id , 'folder_id' : folder_id } + params = {'team_id': team_id, 'folder_id': folder_id} uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] couldnt_upload = [[] for _ in range(_NUM_THREADS)] finish_event = threading.Event() - res = _get_upload_auth_token(params=params,project_id=project_id) - + res = _get_upload_auth_token(params=params, project_id=project_id) + prefix = res['filePath'] limit = res['availableImageCount'] images_to_upload = img_names_urls[:limit] @@ -1086,7 +1086,7 @@ def attach_image_urls_to_project( for f in upload_thread: list_of_uploaded.append(str(f)) - list_of_not_uploaded += [i[0] for i in images_to_skip ] + list_of_not_uploaded += [i[0] for i in images_to_skip] return (list_of_uploaded, list_of_not_uploaded, duplicate_images) From 84272142919f1d50c5de3533a5726295657c5ee6 Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 9 Apr 2021 16:43:29 +0400 Subject: [PATCH 11/18] Fix copy_images - video --- superannotate/db/project_images.py | 27 ++++---- superannotate/db/projects.py | 100 ++++++++++++++--------------- 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index c6273e7b1..96d1c1575 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -17,7 +17,8 @@ from .project_api import get_project_and_folder_metadata from .projects import ( __create_image, get_image_array_to_upload, - get_project_default_image_quality_in_editor, upload_image_array_to_s3 + get_project_default_image_quality_in_editor, upload_image_array_to_s3, + _get_available_image_counts ) from .utils import _get_upload_auth_token @@ -99,12 +100,9 @@ def upload_image_to_project( folder_id = get_project_root_folder_id(project) team_id, project_id = project["team_id"], project["id"] - params = { - 'team_id': team_id, - 'folder_id' : folder_id - } - res = _get_upload_auth_token(params=params,project_id=project_id) - prefix = res['filePath'] + params = {'team_id': team_id, 'folder_id': folder_id} + res = _get_upload_auth_token(params=params, project_id=project_id) + prefix = res['filePath'] s3_session = boto3.Session( aws_access_key_id=res['accessKeyId'], aws_secret_access_key=res['secretAccessKey'], @@ -224,6 +222,11 @@ def copy_images( ) if image_names is None: image_names = search_images((source_project, source_project_folder)) + + limit = _get_available_image_counts( + destination_project, destination_project_folder + ) + image_names = image_names[:limit] res = _copy_images( (source_project, source_project_folder), (destination_project, destination_project_folder), image_names, @@ -231,11 +234,11 @@ def copy_images( ) logger.info( "Copied images %s from %s to %s. Number of skipped images %s", - image_names, - source_project["name"] + "" if source_project_folder is None else "/" + - source_project_folder["name"], destination_project["name"] + - "" if destination_project_folder is None else destination_project["name"] + "/" + - destination_project_folder["name"], len(res["skipped"]) + image_names, source_project["name"] + + "" if source_project_folder is None else source_project["name"] + "/" + + source_project_folder["name"], destination_project["name"] + "" + if destination_project_folder is None else destination_project["name"] + + "/" + destination_project_folder["name"], len(res["skipped"]) ) return res["skipped"] diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 6ebe26158..f023ec65e 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -221,7 +221,7 @@ def _get_video_frames_count(video_path): return total_num_of_frames -def _get_video_fps_ration(target_fps,video,ratio): +def _get_video_fps_ration(target_fps, video, ratio): """ Get video fps / target fps ratio """ @@ -230,7 +230,7 @@ def _get_video_fps_ration(target_fps,video,ratio): logger.warning( "Video frame rate %s smaller than target frame rate %s. Cannot change frame rate.", video_fps, target_fps - ) + ) else: logger.info( "Changing video frame rate from %s to target frame rate %s.", @@ -239,22 +239,24 @@ def _get_video_fps_ration(target_fps,video,ratio): ratio = video_fps / target_fps return ratio -def _get_available_image_counts(project,folder): + +def _get_available_image_counts(project, folder): if folder: folder_id = folder["id"] else: folder_id = get_project_root_folder_id(project) - params = {'team_id': project['team_id'] , 'folder_id' : folder_id } - res = _get_upload_auth_token(params=params,project_id=project['id']) + params = {'team_id': project['team_id'], 'folder_id': folder_id} + res = _get_upload_auth_token(params=params, project_id=project['id']) return res['availableImageCount'] + def _get_video_rotate_code(video_path): rotate_code = None try: cv2_rotations = { - 90 : cv2.ROTATE_90_CLOCKWISE, - 180 : cv2.ROTATE_180, - 270 :cv2.ROTATE_90_COUNTERCLOCKWISE, + 90: cv2.ROTATE_90_CLOCKWISE, + 180: cv2.ROTATE_180, + 270: cv2.ROTATE_90_COUNTERCLOCKWISE, } meta_dict = ffmpeg.probe(str(video_path)) @@ -276,14 +278,26 @@ def _get_video_rotate_code(video_path): return rotate_code -def _extract_frames_from_video(start_time,end_time,ratio,video,video_path,tempdir,limit,rotate_code,total_num_of_frames): +def _extract_frames_from_video( + start_time, end_time, video_path, tempdir, limit, target_fps +): + video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) + if not video.isOpened(): + raise SABaseException(0, "Couldn't open video file " + str(video_path)) + total_num_of_frames = _get_video_frames_count(video_path) + logger.info("Video frame count is %s.", total_num_of_frames) + ratio = 1.0 + if target_fps: + ratio = _get_video_fps_ration(target_fps, video, ratio) + rotate_code = _get_video_rotate_code(video_path) video_name = Path(video_path).stem frame_no = 0 frame_no_with_change = 1.0 extracted_frame_no = 1 logger.info("Extracting frames from video to %s.", tempdir.name) zero_fill_count = len(str(total_num_of_frames)) - while extracted_frame_no < (limit + 1) : + extracted_frames_paths = [] + while len(extracted_frames_paths) < limit: success, frame = video.read() if not success: break @@ -298,16 +312,16 @@ def _extract_frames_from_video(start_time,end_time,ratio,video,video_path,tempdi continue if rotate_code: frame = cv2.rotate(frame, rotate_code) - cv2.imwrite( - str( - Path(tempdir.name) / ( - video_name + "_" + - str(extracted_frame_no).zfill(zero_fill_count) + ".jpg" - ) - ), frame + path = str( + Path(tempdir.name) / ( + video_name + "_" + + str(extracted_frame_no).zfill(zero_fill_count) + ".jpg" + ) ) + cv2.imwrite(path, frame) + extracted_frames_paths.append(path) extracted_frame_no += 1 - return extracted_frame_no - 1 + return extracted_frames_paths def upload_video_to_project( @@ -345,7 +359,7 @@ def upload_video_to_project( """ project, folder = get_project_and_folder_metadata(project) - limit = _get_available_image_counts(project,folder) + limit = _get_available_image_counts(project, folder) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": @@ -354,23 +368,13 @@ def upload_video_to_project( "The function does not support projects containing images attached with URLs" ) logger.info("Uploading from video %s.", str(video_path)) - rotate_code = _get_video_rotate_code(video_path) - video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) - if not video.isOpened(): - raise SABaseException(0, "Couldn't open video file " + str(video_path)) - - total_num_of_frames = _get_video_frames_count(video_path) - logger.info("Video frame count is %s.", total_num_of_frames) - ratio = 1.0 - if target_fps: - ratio = _get_video_fps_ration(target_fps,video,ratio) tempdir = tempfile.TemporaryDirectory() - extracted_frame_no = _extract_frames_from_video(start_time,end_time,ratio, - video,video_path,tempdir, - limit,rotate_code,total_num_of_frames) + extracted_frames = _extract_frames_from_video( + start_time, end_time, video_path, tempdir, limit, target_fps + ) logger.info( "Extracted %s frames from video. Now uploading to platform.", - extracted_frame_no + len(extracted_frames) ) filenames = upload_images_from_folder_to_project( (project, folder), @@ -853,9 +857,7 @@ def upload_images_to_project( :rtype: tuple (3 members) of list of strs """ project, folder = get_project_and_folder_metadata(project) - folder_name = project["name"] + ( - f'/{folder["name"]}' if folder else "" - ) + folder_name = project["name"] + (f'/{folder["name"]}' if folder else "") upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": raise SABaseException( @@ -895,24 +897,23 @@ def upload_images_to_project( if len_img_paths == 0: return ([], [], duplicate_images) - if folder: folder_id = folder["id"] else: folder_id = get_project_root_folder_id(project) - params = {'team_id': team_id , 'folder_id' : folder_id } + params = {'team_id': team_id, 'folder_id': folder_id} uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] couldnt_upload = [[] for _ in range(_NUM_THREADS)] finish_event = threading.Event() - res = _get_upload_auth_token(params=params,project_id=project_id) + res = _get_upload_auth_token(params=params, project_id=project_id) prefix = res['filePath'] limit = res['availableImageCount'] images_to_upload = img_paths[:limit] - images_to_skip = img_paths[limit:] + images_to_skip = [str(path) for path in img_paths[limit:]] chunksize = int(math.ceil(len(images_to_upload) / _NUM_THREADS)) tqdm_thread = threading.Thread( @@ -927,8 +928,8 @@ def upload_images_to_project( t = threading.Thread( target=__upload_images_to_aws_thread, args=( - res, images_to_upload, project, annotation_status, prefix, thread_id, - chunksize, couldnt_upload, uploaded, tried_upload, + res, images_to_upload, project, annotation_status, prefix, + thread_id, chunksize, couldnt_upload, uploaded, tried_upload, image_quality_in_editor, from_s3_bucket, folder_id ), daemon=True @@ -987,9 +988,7 @@ def attach_image_urls_to_project( """ project, folder = get_project_and_folder_metadata(project) - folder_name = project["name"] + ( - f'/{folder["name"]}' if folder else "" - ) + folder_name = project["name"] + (f'/{folder["name"]}' if folder else "") upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "Basic": raise SABaseException( @@ -1027,8 +1026,7 @@ def attach_image_urls_to_project( image_data = pd.DataFrame(image_data, columns=["name", "url"]) img_names_urls = image_data.values.tolist() logger.info( - "Uploading %s images to project %s.", len(img_names_urls), - folder_name + "Uploading %s images to project %s.", len(img_names_urls), folder_name ) if len(img_names_urls) == 0: return ([], [], duplicate_images) @@ -1038,14 +1036,14 @@ def attach_image_urls_to_project( else: folder_id = get_project_root_folder_id(project) - params = {'team_id': team_id , 'folder_id' : folder_id } + params = {'team_id': team_id, 'folder_id': folder_id} uploaded = [[] for _ in range(_NUM_THREADS)] tried_upload = [[] for _ in range(_NUM_THREADS)] couldnt_upload = [[] for _ in range(_NUM_THREADS)] finish_event = threading.Event() - res = _get_upload_auth_token(params=params,project_id=project_id) - + res = _get_upload_auth_token(params=params, project_id=project_id) + prefix = res['filePath'] limit = res['availableImageCount'] images_to_upload = img_names_urls[:limit] @@ -1084,7 +1082,7 @@ def attach_image_urls_to_project( for f in upload_thread: list_of_uploaded.append(str(f)) - list_of_not_uploaded += [i[0] for i in images_to_skip ] + list_of_not_uploaded += [i[0] for i in images_to_skip] return (list_of_uploaded, list_of_not_uploaded, duplicate_images) From b2bbf1b8625db8b2c47540306b4c016107e2f47f Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Fri, 9 Apr 2021 16:54:31 +0400 Subject: [PATCH 12/18] Update version.py --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index dc4b8a9d6..6de9fddb2 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.3b3" +__version__ = "4.1.3b4" From 3863ac2a7062d3730d19a3a4f8465093b05d0ca8 Mon Sep 17 00:00:00 2001 From: shab Date: Mon, 12 Apr 2021 15:11:44 +0400 Subject: [PATCH 13/18] Fix copy images --- superannotate/db/project_images.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 96d1c1575..4aa44ace7 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -164,6 +164,7 @@ def _copy_images( json_req["destination_folder_id"] = destination_folder_id res = {} res['skipped'] = [] + res['completed'] = [] for start_index in range(0, len(image_names), NUM_TO_SEND): json_req["image_names"] = image_names[start_index:start_index + NUM_TO_SEND] @@ -179,6 +180,7 @@ def _copy_images( response.status_code, "Couldn't copy images " + response.text ) res['skipped'] += response.json()['skipped'] + res['completed'] += response.json()['completed'] for image_name in image_names: _copy_annotations_and_metadata( @@ -226,21 +228,26 @@ def copy_images( limit = _get_available_image_counts( destination_project, destination_project_folder ) - image_names = image_names[:limit] + imgs_to_upload = image_names[:limit] res = _copy_images( (source_project, source_project_folder), - (destination_project, destination_project_folder), image_names, + (destination_project, destination_project_folder), imgs_to_upload, include_annotations, copy_annotation_status, copy_pin ) + uploaded_imgs = res['completed'] + skipped_imgs = [i for i in imgs_to_upload if i not in uploaded_imgs] + + skipped_images_count = len(skipped_imgs) + len(image_names[limit:]) + logger.info( "Copied images %s from %s to %s. Number of skipped images %s", - image_names, source_project["name"] + + uploaded_imgs, source_project["name"] + "" if source_project_folder is None else source_project["name"] + "/" + source_project_folder["name"], destination_project["name"] + "" if destination_project_folder is None else destination_project["name"] + - "/" + destination_project_folder["name"], len(res["skipped"]) + "/" + destination_project_folder["name"], skipped_images_count ) - return res["skipped"] + return skipped_imgs def delete_images(project, image_names): From f79e89494954c406a3d2e6d2c65fb74dd1cf9bbf Mon Sep 17 00:00:00 2001 From: Shabin Dilanchian Date: Mon, 12 Apr 2021 15:45:38 +0400 Subject: [PATCH 14/18] Update version.py --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index 6de9fddb2..0a39acda3 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.3b4" +__version__ = "4.1.3b5" From 9972f133132682fbd2d4cb2d0cb65871a0b7a6fe Mon Sep 17 00:00:00 2001 From: Davit Avagyan Date: Tue, 13 Apr 2021 11:36:48 +0400 Subject: [PATCH 15/18] Fixed hanging issue --- superannotate/db/project_images.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 4aa44ace7..8872057ad 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -52,6 +52,7 @@ def upload_image_to_project( Can be either "compressed" or "original". If None then the default value in project settings will be used. :type image_quality_in_editor: str """ + initial_project_inp = project project, folder = get_project_and_folder_metadata(project) upload_state = common.upload_state_int_to_str(project.get("upload_state")) if upload_state == "External": @@ -129,7 +130,7 @@ def upload_image_to_project( while True: try: - get_image_metadata(project, img_name) + get_image_metadata(initial_project_inp, img_name) except SABaseException: time.sleep(0.2) else: From c31e04c7138b739833fcf774f3dd5a8ee3c1fbea Mon Sep 17 00:00:00 2001 From: davitavagyan <68903643+davitavagyan@users.noreply.github.com> Date: Tue, 13 Apr 2021 11:39:23 +0400 Subject: [PATCH 16/18] Update version.py --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index 0a39acda3..806f4be03 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.3b5" +__version__ = "4.1.3b6" From 43f140b8de0bec1d611f2bcf42d5fa7af60eb18a Mon Sep 17 00:00:00 2001 From: Davit Avagyan Date: Tue, 13 Apr 2021 17:30:27 +0400 Subject: [PATCH 17/18] Move function fix, test fix --- superannotate/db/project_images.py | 8 +++----- tests/test_folders.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 8872057ad..ab4c45a6b 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -329,11 +329,9 @@ def move_images( ) if image_names is None: image_names = search_images((source_project, source_project_folder)) - _copy_images( - (source_project, source_project_folder), - (destination_project, destination_project_folder), image_names, - include_annotations, copy_annotation_status, copy_pin - ) + copy_images((source_project, source_project_folder), image_names, + (destination_project, destination_project_folder), + include_annotations, copy_annotation_status, copy_pin) delete_images((source_project, source_project_folder), image_names) logger.info( "Moved images %s from project %s to project %s", image_names, diff --git a/tests/test_folders.py b/tests/test_folders.py index 6fb07e17c..0570c137b 100644 --- a/tests/test_folders.py +++ b/tests/test_folders.py @@ -348,7 +348,7 @@ def test_copy_images(tmpdir): num_images = sa.get_project_image_count(project2) assert num_images == 4 - assert res == 2 + assert len(res) == 2 sa.copy_images( project, ["example_image_2.jpg", "example_image_3.jpg"], From 15eaa3007aeaa6b950cbd80a25d3cd2815c06a8a Mon Sep 17 00:00:00 2001 From: davitavagyan <68903643+davitavagyan@users.noreply.github.com> Date: Tue, 13 Apr 2021 17:31:21 +0400 Subject: [PATCH 18/18] Update version.py --- superannotate/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superannotate/version.py b/superannotate/version.py index 806f4be03..dfade6840 100644 --- a/superannotate/version.py +++ b/superannotate/version.py @@ -1 +1 @@ -__version__ = "4.1.3b6" +__version__ = "4.1.3b7"