diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index 9ad1a3ed1..24ef41562 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -1,14 +1,9 @@ -import concurrent.futures import json import logging import os import sys import tempfile import uuid -from collections import Counter -from collections import namedtuple -from io import BytesIO -from pathlib import Path from typing import Any from typing import Optional @@ -19,12 +14,21 @@ from lib.app.helpers import split_project_path from lib.app.input_converters.conversion import import_annotation from lib.app.interface.base_interface import BaseInterfaceFacade +from lib.app.interface.sdk_interface import attach_image_urls_to_project +from lib.app.interface.sdk_interface import attach_video_urls_to_project +from lib.app.interface.sdk_interface import create_folder +from lib.app.interface.sdk_interface import create_project +from lib.app.interface.sdk_interface import upload_images_from_folder_to_project +from lib.app.interface.sdk_interface import upload_videos_from_folder_to_project from lib.app.serializers import ImageSerializer from lib.core.entities import ConfigEntity +from lib.infrastructure.controller import Controller from lib.infrastructure.repositories import ConfigRepository from tqdm import tqdm + logger = logging.getLogger() +controller = Controller(logger) class CLIFacade(BaseInterfaceFacade): @@ -72,18 +76,14 @@ def create_project(self, name: str, description: str, type: str): """ To create a new project """ - response = self.controller.create_project(name, description, type) - if response.errors: - return response.errors - return response.data + create_project(name, description, type) + sys.exit(0) def create_folder(self, project: str, name: str): """ To create a new folder """ - response = self.controller.create_folder(project=project, folder_name=name) - if response.errors: - logger.critical(response.errors) + create_folder(project, name) sys.exit(0) def upload_images( @@ -104,79 +104,19 @@ def upload_images( Optional argument extensions accepts comma separated list of image extensions to look for. If the argument is not given then value jpg,jpeg,png,tif,tiff,webp,bmp is assumed. """ - uploaded_image_entities = [] - failed_images = [] - project_name, folder_name = split_project_path(project) - ProcessedImage = namedtuple("ProcessedImage", ["uploaded", "path", "entity"]) - - def upload_image(image_path: str): - with open(image_path, "rb") as image: - image_bytes = BytesIO(image.read()) - upload_response = self.controller.upload_image_to_s3( - project_name=project_name, - image_path=image_path, - image_bytes=image_bytes, - folder_name=folder_name, - image_quality_in_editor=image_quality_in_editor, - ) - - if not upload_response.errors and upload_response.data: - entity = upload_response.data - return ProcessedImage( - uploaded=True, path=entity.path, entity=entity - ) - else: - return ProcessedImage(uploaded=False, path=image_path, entity=None) - - paths = [] - - if isinstance(extensions, str): - extensions = extensions.strip().split(",") + if not isinstance(extensions, list): + extensions = extensions.split(",") - for extension in extensions: - if recursive_subfolders: - paths += list(Path(folder).rglob(f"*.{extension.lower()}")) - if os.name != "nt": - paths += list(Path(folder).rglob(f"*.{extension.upper()}")) - else: - paths += list(Path(folder).glob(f"*.{extension.lower()}")) - if os.name != "nt": - paths += list(Path(folder).glob(f"*.{extension.upper()}")) - - filtered_paths = [] - for path in paths: - not_in_exclude_list = [ - x not in Path(path).name for x in exclude_file_patterns - ] - if all(not_in_exclude_list): - filtered_paths.append(path) - - duplication_counter = Counter(filtered_paths) - images_to_upload, duplicated_images = ( - set(filtered_paths), - [item for item in duplication_counter if duplication_counter[item] > 1], + upload_images_from_folder_to_project( + project, + folder_path=folder, + extensions=extensions, + annotation_status=set_annotation_status, + from_s3_bucket=None, + exclude_file_patterns=exclude_file_patterns, + recursive_subfolders=recursive_subfolders, + image_quality_in_editor=image_quality_in_editor, ) - with tqdm(total=len(images_to_upload)) as progress_bar: - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - results = [ - executor.submit(upload_image, image_path) - for image_path in images_to_upload - ] - for future in concurrent.futures.as_completed(results): - processed_image = future.result() - if processed_image.uploaded and processed_image.entity: - uploaded_image_entities.append(processed_image.entity) - else: - failed_images.append(processed_image.path) - progress_bar.update(1) - - for i in range(0, len(uploaded_image_entities), 500): - self.controller.upload_images( - project_name=project_name, - folder_name=folder_name, - images=uploaded_image_entities[i : i + 500], # noqa: E203 - annotation_status=set_annotation_status, - ) sys.exit(0) def export_project( @@ -305,13 +245,22 @@ def attach_image_urls( """ To attach image URLs to project use: """ - self._attach_urls(project, attachments, annotation_status) + + attach_image_urls_to_project( + project=project, + attachments=attachments, + annotation_status=annotation_status, + ) sys.exit(0) def attach_video_urls( self, project: str, attachments: str, annotation_status: Optional[Any] = None ): - self._attach_urls(project, attachments, annotation_status) + attach_video_urls_to_project( + project=project, + attachments=attachments, + annotation_status=annotation_status, + ) sys.exit(0) def _attach_urls( @@ -372,58 +321,17 @@ def upload_videos( start-time specifies time (in seconds) from which to start extracting frames, default is 0.0. end-time specifies time (in seconds) up to which to extract frames. If it is not specified, then up to end is assumed. """ - project_name, folder_name = split_project_path(project) - - uploaded_image_entities = [] - failed_images = [] - - def _upload_image(image_path: str) -> str: - with open(image_path, "rb") as image: - image_bytes = BytesIO(image.read()) - upload_response = self.controller.upload_image_to_s3( - project_name=project_name, - image_path=image_path, - image_bytes=image_bytes, - folder_name=folder_name, - ) - if not upload_response.errors: - uploaded_image_entities.append(upload_response.data) - else: - return image_path - video_paths = [] - for extension in extensions: - if not recursive: - video_paths += list(Path(folder).glob(f"*.{extension.lower()}")) - if os.name != "nt": - video_paths += list(Path(folder).glob(f"*.{extension.upper()}")) - else: - video_paths += list(Path(folder).rglob(f"*.{extension.lower()}")) - if os.name != "nt": - video_paths += list(Path(folder).rglob(f"*.{extension.upper()}")) - video_paths = [str(path) for path in video_paths] - - for path in video_paths: - with tempfile.TemporaryDirectory() as temp_path: - res = self.controller.extract_video_frames( - project_name=project_name, - folder_name=folder_name, - video_path=path, - extract_path=temp_path, - target_fps=int(target_fps), - start_time=float(start_time), - end_time=end_time if not end_time else float(end_time), - annotation_status=set_annotation_status, - ) - if not res.errors: - extracted_frame_paths = res.data - for image_path in extracted_frame_paths: - failed_images.append(_upload_image(image_path)) - for i in range(0, len(uploaded_image_entities), 500): - self.controller.upload_images( - project_name=project_name, - folder_name=folder_name, - images=uploaded_image_entities[i : i + 500], # noqa: E203 - annotation_status=set_annotation_status, - ) + upload_videos_from_folder_to_project( + project=project, + folder_path=folder, + extensions=extensions, + exclude_file_patterns=(), + recursive_subfolders=recursive, + target_fps=target_fps, + start_time=start_time, + end_time=end_time, + annotation_status=set_annotation_status, + image_quality_in_editor=None, + ) sys.exit(0) diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index 7362fb5eb..e48afaa34 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -1150,7 +1150,9 @@ def assign_images(project: Union[str, dict], image_names: List[str], user: str): ) return - controller.assign_images(project_name, folder_name, image_names, user) + response = controller.assign_images(project_name, folder_name, image_names, user) + if not response.errors: + logger.info(f"Assign images to user {user}") @Trackable @@ -1558,9 +1560,7 @@ def upload_images_from_folder_to_project( "extensions should be a list or a tuple in upload_images_from_folder_to_project" ) - project_folder_name = project_name + ( - f"/{folder_name}" if folder_name != "root" else "" - ) + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") logger.info( "Uploading all images with extensions %s from %s to project %s. Excluded file patterns are: %s.", @@ -1886,6 +1886,8 @@ def upload_videos_from_folder_to_project( if all(not_in_exclude_list): filtered_paths.append(path) + project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") + logger.info( "Uploading all videos with extensions %s from %s to project %s. Excluded file patterns are: %s.", extensions, @@ -1894,8 +1896,7 @@ def upload_videos_from_folder_to_project( exclude_file_patterns, ) - uploaded_images, failed_images = [], [] - for path in tqdm(video_paths, desc="Uploading videos"): + for path in video_paths: with tempfile.TemporaryDirectory() as temp_path: res = controller.extract_video_frames( project_name=project_name, @@ -1917,17 +1918,31 @@ def upload_videos_from_folder_to_project( annotation_status=annotation_status, image_quality_in_editor=image_quality_in_editor, ) - images_to_upload, _ = use_case.images_to_upload + images_to_upload, duplicates = use_case.images_to_upload + logger.info( + "Extracted %s frames from video. Now uploading to platform.", + len(res.data), + ) + logger.info( + "Uploading %s images to project %s.", + len(images_to_upload), + str(project_folder_name), + ) + if len(duplicates): + logger.warning( + "%s already existing images found that won't be uploaded.", + len(duplicates), + ) if use_case.is_valid(): - for _ in use_case.execute(): - pass - uploaded, failed_images, _ = use_case.data - uploaded_images.append(uploaded) - failed_images.append(failed_images) + with tqdm( + total=len(images_to_upload), desc="Uploading images" + ) as progress_bar: + for _ in use_case.execute(): + progress_bar.update(1) else: raise AppValidationException(use_case.response.errors) - return uploaded_images, failed_images + return @Trackable diff --git a/src/superannotate/lib/core/plugin.py b/src/superannotate/lib/core/plugin.py index b2671f5e5..e5f599adc 100644 --- a/src/superannotate/lib/core/plugin.py +++ b/src/superannotate/lib/core/plugin.py @@ -1,5 +1,6 @@ import io import logging +import os from pathlib import Path from typing import List from typing import Tuple @@ -234,7 +235,6 @@ def extract_frames( ) ratio = fps / target_fps - extracted_frames_paths = [] zero_fill_count = len(str(frames_count)) rotate_code = VideoPlugin.get_video_rotate_code(video_path) @@ -242,9 +242,10 @@ def extract_frames( frame_number = 0 extracted_frame_number = 0 extracted_frame_ratio = ratio - logger.info("Extracting frames from video to %s.", extracted_frames_paths) + logger.info("Extracting frames from video to %s.", extract_path) + extracted_frames_paths = [] - while len(extracted_frames_paths) < limit: + while len(os.listdir(extract_path)) < limit: success, frame = video.read() if success: frame_time = video.get(cv2.CAP_PROP_POS_MSEC) / 1000.0 diff --git a/src/superannotate/lib/core/usecases.py b/src/superannotate/lib/core/usecases.py index 0fa5dacc6..42a025a20 100644 --- a/src/superannotate/lib/core/usecases.py +++ b/src/superannotate/lib/core/usecases.py @@ -1705,8 +1705,6 @@ def execute(self): f"Cant assign {', '.join(self._image_names[i: i + self.CHUNK_SIZE])}" ) continue - logger.info(f"Assign images to user {self._user}") - return self._response