From 439ef26f684dcb3317bdfd57f346452cdc9cfc63 Mon Sep 17 00:00:00 2001 From: Vaghinak Basentsyan Date: Fri, 7 Jan 2022 11:17:43 +0400 Subject: [PATCH] Friday-386 --- .devcontainer/devcontainer.json | 12 - .pre-commit-config.yaml | 30 +- pytest.ini | 2 +- requirements_dev.txt | 3 +- src/superannotate/__init__.py | 3 +- src/superannotate/lib/app/analytics/common.py | 5 +- .../lib/app/interface/cli_interface.py | 2 - .../lib/app/interface/sdk_interface.py | 227 ++++------------ src/superannotate/lib/app/interface/types.py | 1 - .../lib/app/mixp/utils/parsers.py | 2 +- src/superannotate/lib/core/__init__.py | 1 - src/superannotate/lib/core/conditions.py | 1 - .../lib/core/entities/__init__.py | 1 - src/superannotate/lib/core/entities/video.py | 4 +- src/superannotate/lib/core/plugin.py | 3 +- src/superannotate/lib/core/reporter.py | 9 +- .../lib/core/serviceproviders.py | 4 +- src/superannotate/lib/core/usecases/base.py | 5 +- src/superannotate/lib/core/usecases/images.py | 174 ++++++++++++ src/superannotate/lib/core/usecases/models.py | 1 - .../lib/core/usecases/projects.py | 256 +++++++++--------- .../lib/infrastructure/controller.py | 36 +++ .../lib/infrastructure/services.py | 6 +- 23 files changed, 434 insertions(+), 354 deletions(-) delete mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 18ad9719a..000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "pythonsdk", - "image": "superannotate/pythonsdk", - "runArgs": [], - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "foxundermoon.shell-format" - ], - "mounts": [], - "remoteEnv": {} -} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50dd35004..3982e77cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,18 @@ repos: + - repo: 'https://github.com/asottile/reorder_python_imports' + rev: v2.3.0 + hooks: + - id: reorder-python-imports + exclude: src/lib/app/analytics | src/lib/app/converters | src/lib/app/input_converters + name: 'Reorder Python imports (src, tests)' + args: + - '--application-directories' + - app + - repo: 'https://github.com/python/black' + rev: 19.10b0 + hooks: + - id: black + name: Code Formatter (black) - repo: 'https://gitlab.com/pycqa/flake8' rev: 3.8.2 hooks: @@ -7,7 +21,7 @@ repos: name: Style Guide Enforcement (flake8) args: - '--max-line-length=120' - - --ignore=D100,D203,D405,W503,E203,E501,F841,E126,E712 + - --ignore=D100,D203,D405,W503,E203,E501,F841,E126,E712,E123,E131,F821,E121 - repo: 'https://github.com/asottile/pyupgrade' rev: v2.4.3 hooks: @@ -16,26 +30,12 @@ repos: name: Upgrade syntax for newer versions of the language (pyupgrade) args: - '--py37-plus' - - repo: 'https://github.com/asottile/reorder_python_imports' - rev: v2.3.0 - hooks: - - id: reorder-python-imports - exclude: src/lib/app/analytics | src/lib/app/converters | src/lib/app/input_converters - name: 'Reorder Python imports (src, tests)' - args: - - '--application-directories' - - app - repo: 'https://github.com/pre-commit/pre-commit-hooks' rev: v3.1.0 hooks: - id: check-byte-order-marker - id: trailing-whitespace - id: end-of-file-fixer - - repo: 'https://github.com/python/black' - rev: 19.10b0 - hooks: - - id: black - name: Uncompromising Code Formatter (black) # - repo: 'https://github.com/asottile/dead' # rev: v1.3.0 # hooks: diff --git a/pytest.ini b/pytest.ini index 5e4682b38..d9ab3b434 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ minversion = 3.0 log_cli=true python_files = test_*.py -;addopts = -n auto --dist=lo adscope \ No newline at end of file +;addopts = -n auto --dist=loadscope \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 9e3d802fb..023e3a50e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -4,4 +4,5 @@ pytest==6.2.4 pytest-xdist==2.3.0 pytest-parallel==0.1.0 pytest-rerunfailures==10.2 -sphinx_rtd_theme==1.0.0 \ No newline at end of file +sphinx_rtd_theme==1.0.0 + diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py index 302f4d492..7e4c48806 100644 --- a/src/superannotate/__init__.py +++ b/src/superannotate/__init__.py @@ -238,7 +238,8 @@ "format": "SA-PYTHON-SDK - %(levelname)s - %(asctime)s - %(message)s" }, }, - "root": { # root logger + "root": { + # "level": "INFO", "level": "DEBUG", "handlers": ["console", "fileHandler"], }, diff --git a/src/superannotate/lib/app/analytics/common.py b/src/superannotate/lib/app/analytics/common.py index 99ac8c6b4..b23e1313a 100644 --- a/src/superannotate/lib/app/analytics/common.py +++ b/src/superannotate/lib/app/analytics/common.py @@ -7,7 +7,6 @@ from lib.app.exceptions import AppException from lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE - logger = logging.getLogger("root") @@ -412,9 +411,7 @@ def image_consensus(df, image_name, annot_type): """ try: - from shapely.geometry import box - from shapely.geometry import Point - from shapely.geometry import Polygon + from shapely.geometry import Point, Polygon, box except ImportError: raise ImportError( "To use superannotate.benchmark or superannotate.consensus functions please install " diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index a6004e323..ebcb91c79 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -1,4 +1,3 @@ -import json import os import sys import tempfile @@ -23,7 +22,6 @@ from lib.infrastructure.controller import Controller from lib.infrastructure.repositories import ConfigRepository - controller = Controller.get_instance() diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py index d77a3e78c..ce8105d35 100644 --- a/src/superannotate/lib/app/interface/sdk_interface.py +++ b/src/superannotate/lib/app/interface/sdk_interface.py @@ -35,7 +35,6 @@ from lib.core import LIMITED_FUNCTIONS from lib.core.enums import ImageQuality from lib.core.exceptions import AppException -from lib.core.plugin import VideoPlugin from lib.core.types import AttributeGroup from lib.core.types import ClassesJson from lib.core.types import MLModel @@ -46,7 +45,6 @@ from pydantic import StrictBool from tqdm import tqdm - controller = Controller.get_instance() logger = logging.getLogger("root") @@ -727,6 +725,10 @@ def get_project_metadata( the key "contributors" :type include_contributors: bool + :param include_complete_image_count: enables project complete image count output under + the key "completed_images_count" + :type include_complete_image_count: bool + :return: metadata of project :rtype: dict """ @@ -1155,19 +1157,27 @@ def upload_images_from_folder_to_project( :param project: project name or folder path (e.g., "project1/folder1") :type project: str or dict + :param folder_path: from which folder to upload the images :type folder_path: Path-like (str or Path) + :param extensions: tuple or list of filename extensions to include from folder :type extensions: tuple or list of strs - :param annotation_status: value to set the annotation statuses of the uploaded images NotStarted InProgress QualityCheck Returned Completed Skipped + + :param annotation_status: value to set the annotation statuses of the uploaded images + NotStarted InProgress QualityCheck Returned Completed Skipped :type annotation_status: str + :param from_s3_bucket: AWS S3 bucket to use. If None then folder_path is in local filesystem :type from_s3_bucket: str + :param exclude_file_patterns: filename patterns to exclude from uploading, default value is to exclude SuperAnnotate export related ["___save.png", "___fuse.png"] :type exclude_file_patterns: list or tuple of strs + :param recursive_subfolders: enable recursive subfolder parsing :type recursive_subfolders: bool + :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. Can be either "compressed" or "original". If None then the default value in project settings will be used. :type image_quality_in_editor: str @@ -1329,7 +1339,8 @@ def prepare_export( :type project: str :param folder_names: names of folders to include in the export. If None, whole project will be exported :type folder_names: list of str - :param annotation_statuses: images with which status to include, if None, ["NotStarted", "InProgress", "QualityCheck", "Returned", "Completed", "Skipped"] will be chose + :param annotation_statuses: images with which status to include, if None, + ["NotStarted", "InProgress", "QualityCheck", "Returned", "Completed", "Skipped"] will be chose list elements should be one of NotStarted InProgress QualityCheck Returned Completed Skipped :type annotation_statuses: list of strs :param include_fuse: enables fuse images in the export @@ -1374,7 +1385,7 @@ def upload_videos_from_folder_to_project( extensions: Optional[ Union[Tuple[NotEmptyStr], List[NotEmptyStr]] ] = constances.DEFAULT_VIDEO_EXTENSIONS, - exclude_file_patterns: Optional[List[NotEmptyStr]] = [], + exclude_file_patterns: Optional[List[NotEmptyStr]] = (), recursive_subfolders: Optional[StrictBool] = False, target_fps: Optional[int] = None, start_time: Optional[float] = 0.0, @@ -1402,7 +1413,8 @@ def upload_videos_from_folder_to_project( :type start_time: float :param end_time: Time (in seconds) up to which to extract frames. If None up to end :type end_time: float - :param annotation_status: value to set the annotation statuses of the uploaded images NotStarted InProgress QualityCheck Returned Completed Skipped + :param annotation_status: value to set the annotation statuses of the uploaded images + NotStarted InProgress QualityCheck Returned Completed Skipped :type annotation_status: str :param image_quality_in_editor: image quality be seen in SuperAnnotate web annotation editor. Can be either "compressed" or "original". If None then the default value in project settings will be used. @@ -1413,12 +1425,6 @@ def upload_videos_from_folder_to_project( """ project_name, folder_name = extract_project_folder(project) - project = controller.get_project_metadata(project_name).data - if project["project"].project_type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].project_type]) video_paths = [] for extension in extensions: @@ -1428,97 +1434,28 @@ def upload_videos_from_folder_to_project( video_paths += list(Path(folder_path).glob(f"*.{extension.upper()}")) else: logger.warning( - "When using recursive subfolder parsing same name videos in different subfolders will overwrite each other." + "When using recursive subfolder parsing same name videos " + "in different subfolders will overwrite each other." ) video_paths += list(Path(folder_path).rglob(f"*.{extension.lower()}")) if os.name != "nt": video_paths += list(Path(folder_path).rglob(f"*.{extension.upper()}")) - filtered_paths = [] - video_paths = [str(path) for path in video_paths] - for path in video_paths: - - not_in_exclude_list = [x not in Path(path).name for x in exclude_file_patterns] - if all(not_in_exclude_list): - filtered_paths.append(path) - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - logger.info( - f"Uploading all videos with extensions {extensions} from {str(folder_path)} to project {project_name}. Excluded file patterns are: {exclude_file_patterns}." - ) - uploaded_paths = [] - for path in filtered_paths: - progress_bar = None - with tempfile.TemporaryDirectory() as temp_path: - frame_names = VideoPlugin.get_extractable_frames( - path, start_time, end_time, target_fps - ) - duplicate_images = ( - controller.get_duplicate_images( - project_name=project_name, - folder_name=folder_name, - images=frame_names, - ) - .execute() - .data - ) - duplicate_images = [image.name for image in duplicate_images] - frames_generator = controller.extract_video_frames( - project_name=project_name, - folder_name=folder_name, - video_path=path, - extract_path=temp_path, - target_fps=target_fps, - start_time=start_time, - end_time=end_time, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - ) - total_frames_count = len(frame_names) - logger.info(f"Video frame count is {total_frames_count}.") - logger.info( - f"Extracted {total_frames_count} frames from video. Now uploading to platform.", - ) - logger.info( - f"Uploading {total_frames_count} images to project {str(project_folder_name)}." - ) - if len(duplicate_images): - logger.warning( - f"{len(duplicate_images)} already existing images found that won't be uploaded." - ) - if set(duplicate_images) == set(frame_names): - continue - - for _ in frames_generator: - use_case = controller.upload_images_from_folder_to_project( - project_name=project_name, - folder_name=folder_name, - folder_path=temp_path, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - ) - - images_to_upload, duplicates = use_case.images_to_upload - if not len(images_to_upload): - continue - if not progress_bar: - progress_bar = tqdm( - total=total_frames_count, desc="Uploading images" - ) - if use_case.is_valid(): - for _ in use_case.execute(): - progress_bar.update() - uploaded, failed_images, _ = use_case.response.data - uploaded_paths.extend(uploaded) - if failed_images: - logger.warning(f"Failed {len(failed_images)}.") - files = os.listdir(temp_path) - image_paths = [f"{temp_path}/{f}" for f in files] - for path in image_paths: - os.remove(path) - else: - raise AppException(use_case.response.errors) - - return uploaded_paths + video_paths = [str(path) for path in video_paths] + response = controller.upload_videos( + project_name=project_name, + folder_name=folder_name, + paths=video_paths, + target_fps=target_fps, + start_time=start_time, + exclude_file_patterns=exclude_file_patterns, + end_time=end_time, + annotation_status=annotation_status, + image_quality_in_editor=image_quality_in_editor, + ) + if response.errors: + raise AppException(response.errors) + return response.data.values @Trackable @@ -1559,83 +1496,19 @@ def upload_video_to_project( project_name, folder_name = extract_project_folder(project) - project = controller.get_project_metadata(project_name).data - if project["project"].project_type in [ - constances.ProjectType.VIDEO.value, - constances.ProjectType.DOCUMENT.value, - ]: - raise AppException(LIMITED_FUNCTIONS[project["project"].project_type]) - project_folder_name = project_name + (f"/{folder_name}" if folder_name else "") - - uploaded_paths = [] - path = video_path - progress_bar = None - with tempfile.TemporaryDirectory() as temp_path: - frame_names = VideoPlugin.get_extractable_frames( - video_path, start_time, end_time, target_fps - ) - duplicate_images = ( - controller.get_duplicate_images( - project_name=project_name, folder_name=folder_name, images=frame_names, - ) - .execute() - .data - ) - duplicate_images = [image.name for image in duplicate_images] - frames_generator = controller.extract_video_frames( - project_name=project_name, - folder_name=folder_name, - video_path=path, - extract_path=temp_path, - target_fps=target_fps, - start_time=start_time, - end_time=end_time, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - ) - total_frames_count = len(frame_names) - logger.info( - f"Extracted {total_frames_count} frames from video. Now uploading to platform.", - ) - logger.info( - f"Uploading {total_frames_count} images to project {str(project_folder_name)}." - ) - if len(duplicate_images): - logger.warning( - f"{len(duplicate_images)} already existing images found that won't be uploaded." - ) - if set(duplicate_images) == set(frame_names): - return [] - - for _ in frames_generator: - use_case = controller.upload_images_from_folder_to_project( - project_name=project_name, - folder_name=folder_name, - folder_path=temp_path, - annotation_status=annotation_status, - image_quality_in_editor=image_quality_in_editor, - ) - - images_to_upload, duplicates = use_case.images_to_upload - if not len(images_to_upload): - continue - if not progress_bar: - progress_bar = tqdm(total=total_frames_count, desc="Uploading images") - if use_case.is_valid(): - for _ in use_case.execute(): - progress_bar.update() - uploaded, failed_images, _ = use_case.response.data - uploaded_paths.extend(uploaded) - if failed_images: - logger.warning(f"Failed {len(failed_images)}.") - files = os.listdir(temp_path) - image_paths = [f"{temp_path}/{f}" for f in files] - for path in image_paths: - os.remove(path) - else: - raise AppException(use_case.response.errors) - - return uploaded_paths + response = controller.upload_videos( + project_name=project_name, + folder_name=folder_name, + paths=[video_path], + target_fps=target_fps, + start_time=start_time, + end_time=end_time, + annotation_status=annotation_status, + image_quality_in_editor=image_quality_in_editor, + ) + if response.errors: + raise AppException(response.errors) + return response.data.get(str(video_path), []) @Trackable @@ -2931,7 +2804,9 @@ def add_contributors_to_project( @Trackable @validate_arguments -def invite_contributors_to_team(emails: conlist(EmailStr, min_items=1), admin: StrictBool = False) -> Tuple[List[str], List[str]]: +def invite_contributors_to_team( + emails: conlist(EmailStr, min_items=1), admin: StrictBool = False +) -> Tuple[List[str], List[str]]: """Invites contributors to the team. :param emails: list of contributor emails diff --git a/src/superannotate/lib/app/interface/types.py b/src/superannotate/lib/app/interface/types.py index c3f979f03..36c5f522e 100644 --- a/src/superannotate/lib/app/interface/types.py +++ b/src/superannotate/lib/app/interface/types.py @@ -12,7 +12,6 @@ from pydantic import ValidationError from pydantic.errors import StrRegexError - NotEmptyStr = constr(strict=True, min_length=1) diff --git a/src/superannotate/lib/app/mixp/utils/parsers.py b/src/superannotate/lib/app/mixp/utils/parsers.py index 876aacf95..96b4bb17d 100644 --- a/src/superannotate/lib/app/mixp/utils/parsers.py +++ b/src/superannotate/lib/app/mixp/utils/parsers.py @@ -809,8 +809,8 @@ def upload_images_from_folder_to_project(*args, **kwargs): else: exclude_file_patterns = constances.DEFAULT_FILE_EXCLUDE_PATTERNS - from pathlib import Path import os + from pathlib import Path paths = [] for extension in extensions: diff --git a/src/superannotate/lib/core/__init__.py b/src/superannotate/lib/core/__init__.py index 9d2c3bed5..339cbc513 100644 --- a/src/superannotate/lib/core/__init__.py +++ b/src/superannotate/lib/core/__init__.py @@ -9,7 +9,6 @@ from superannotate.lib.core.enums import UploadState from superannotate.lib.core.enums import UserRole - CONFIG_FILE_LOCATION = str(Path.home() / ".superannotate" / "config.json") LOG_FILE_LOCATION = str(Path.home() / ".superannotate" / "sa.log") BACKEND_URL = "https://api.annotate.online" diff --git a/src/superannotate/lib/core/conditions.py b/src/superannotate/lib/core/conditions.py index 642da0373..c5697802a 100644 --- a/src/superannotate/lib/core/conditions.py +++ b/src/superannotate/lib/core/conditions.py @@ -3,7 +3,6 @@ from typing import List from typing import NamedTuple - CONDITION_OR = "|" CONDITION_AND = "&" CONDITION_EQ = "=" diff --git a/src/superannotate/lib/core/entities/__init__.py b/src/superannotate/lib/core/entities/__init__.py index 82b10c8e1..a7652f5b6 100644 --- a/src/superannotate/lib/core/entities/__init__.py +++ b/src/superannotate/lib/core/entities/__init__.py @@ -17,7 +17,6 @@ from lib.core.entities.video import VideoAnnotation from lib.core.entities.video_export import VideoAnnotation as VideoExportAnnotation - __all__ = [ "BaseEntity", "ProjectEntity", diff --git a/src/superannotate/lib/core/entities/video.py b/src/superannotate/lib/core/entities/video.py index 7b4e3537d..694a37a92 100644 --- a/src/superannotate/lib/core/entities/video.py +++ b/src/superannotate/lib/core/entities/video.py @@ -49,9 +49,7 @@ class BaseVideoInstance(BaseInstance): class BboxInstance(BaseVideoInstance): point_labels: Optional[ Dict[constr(regex=r"^[0-9]+$"), NotEmptyStr] # noqa F722 - ] = Field( - None, alias="pointLabels" - ) + ] = Field(None, alias="pointLabels") timeline: Dict[float, BboxTimeStamp] diff --git a/src/superannotate/lib/core/plugin.py b/src/superannotate/lib/core/plugin.py index 8012e4854..276823739 100644 --- a/src/superannotate/lib/core/plugin.py +++ b/src/superannotate/lib/core/plugin.py @@ -2,6 +2,7 @@ import logging from pathlib import Path from typing import List +from typing import Optional from typing import Tuple from typing import Union @@ -218,7 +219,7 @@ def get_video_rotate_code(video_path, log): @staticmethod def frames_generator( - video_path: str, start_time, end_time, target_fps: float, log=True + video_path: str, start_time, end_time, target_fps: Optional[float], log=True ): video = cv2.VideoCapture(str(video_path), cv2.CAP_FFMPEG) if not video.isOpened(): diff --git a/src/superannotate/lib/core/reporter.py b/src/superannotate/lib/core/reporter.py index cb956fe9c..c7e0ed48e 100644 --- a/src/superannotate/lib/core/reporter.py +++ b/src/superannotate/lib/core/reporter.py @@ -41,13 +41,18 @@ def log_debug(self, value: str): def start_progress( self, iterations: Union[int, range], description: str = "Processing" + ): + self.progress_bar = self.get_progress_bar(iterations, description) + + def get_progress_bar( + self, iterations: Union[int, range], description: str = "Processing" ): if isinstance(iterations, range): - self.progress_bar = tqdm.tqdm( + return tqdm.tqdm( iterations, desc=description, disable=self._disable_progress_bar ) else: - self.progress_bar = tqdm.tqdm( + return tqdm.tqdm( total=iterations, desc=description, disable=self._disable_progress_bar ) diff --git a/src/superannotate/lib/core/serviceproviders.py b/src/superannotate/lib/core/serviceproviders.py index a4d5beecb..e8ba14b4a 100644 --- a/src/superannotate/lib/core/serviceproviders.py +++ b/src/superannotate/lib/core/serviceproviders.py @@ -46,7 +46,9 @@ def share_project_bulk(self, project_id: int, team_id: int, users: Iterable): raise NotImplementedError @abstractmethod - def invite_contributors(self, team_id: int, team_role: int, emails: Iterable) -> Tuple[List[str], List[str]]: + def invite_contributors( + self, team_id: int, team_role: int, emails: Iterable + ) -> Tuple[List[str], List[str]]: raise NotImplementedError @abstractmethod diff --git a/src/superannotate/lib/core/usecases/base.py b/src/superannotate/lib/core/usecases/base.py index c3a4625d2..3f3e3750f 100644 --- a/src/superannotate/lib/core/usecases/base.py +++ b/src/superannotate/lib/core/usecases/base.py @@ -68,6 +68,7 @@ class BaseUserBasedUseCase(BaseReportableUseCae, metaclass=ABCMeta): """ class contain validation of unique emails """ + def __init__(self, reporter: Reporter, emails: List[str]): super().__init__(reporter) self._emails = emails @@ -75,7 +76,9 @@ def __init__(self, reporter: Reporter, emails: List[str]): def validate_emails(self): emails_to_add = set() unique_emails = [ - email for email in self._emails if email not in emails_to_add and not emails_to_add.add(email) + email + for email in self._emails + if email not in emails_to_add and not emails_to_add.add(email) ] if unique_emails: self.reporter.log_info( diff --git a/src/superannotate/lib/core/usecases/images.py b/src/superannotate/lib/core/usecases/images.py index b1640b355..a0f44c972 100644 --- a/src/superannotate/lib/core/usecases/images.py +++ b/src/superannotate/lib/core/usecases/images.py @@ -5,6 +5,7 @@ import logging import os.path import random +import tempfile import time import uuid from collections import defaultdict @@ -3100,3 +3101,176 @@ def execute(self) -> Response: f"There is not validator for type {self._project_type}." ) return self._response + + +class UploadVideosAsImages(BaseReportableUseCae): + def __init__( + self, + reporter: Reporter, + service: SuerannotateServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + settings: BaseManageableRepository, + s3_repo, + paths: List[str], + target_fps: int, + extensions: List[str] = constances.DEFAULT_VIDEO_EXTENSIONS, + exclude_file_patterns: List[str] = (), + start_time: Optional[float] = 0.0, + end_time: Optional[float] = None, + annotation_status: str = constances.AnnotationStatus.NOT_STARTED, + image_quality_in_editor=None, + ): + super().__init__(reporter) + self._service = service + self._project = project + self._folder = folder + self._settings = settings + self._s3_repo = s3_repo + self._paths = paths + self._target_fps = target_fps + self._extensions = extensions + self._exclude_file_patterns = exclude_file_patterns + self._start_time = start_time + self._end_time = end_time + self._annotation_status = annotation_status + self._image_quality_in_editor = image_quality_in_editor + + @property + def annotation_status(self): + if not self._annotation_status: + return constances.AnnotationStatus.NOT_STARTED.name + return self._annotation_status + + @property + def upload_path(self): + if self._folder.name != "root": + return f"{self._project.name}/{self._folder.name}" + return self._project.name + + @property + def exclude_file_patterns(self): + if not self._exclude_file_patterns: + return [] + return self._exclude_file_patterns + + @property + def extensions(self): + if not self._extensions: + return constances.DEFAULT_VIDEO_EXTENSIONS + return self._extensions + + def validate_project_type(self): + if self._project.project_type in constances.LIMITED_FUNCTIONS: + raise AppValidationException( + constances.LIMITED_FUNCTIONS[self._project.project_type] + ) + + def validate_paths(self): + validated_paths = set() + for path in self._paths: + path = Path(path) + if ( + path.exists() + and path.name not in self.exclude_file_patterns + and path.suffix.split(".")[-1] in self.extensions + ): + validated_paths.add(path) + if not validated_paths: + raise AppValidationException("There is no valid path.") + + self._paths = list(validated_paths) + + def execute(self) -> Response: + if self.is_valid(): + data = {} + for path in self._paths: + with tempfile.TemporaryDirectory() as temp_path: + frame_names = VideoPlugin.get_extractable_frames( + path, self._start_time, self._end_time, self._target_fps + ) + duplicate_images = ( + GetBulkImages( + service=self._service, + project_id=self._project.uuid, + team_id=self._project.team_id, + folder_id=self._folder.uuid, + images=frame_names, + ) + .execute() + .data + ) + duplicate_images = [image.name for image in duplicate_images] + frames_generator_use_case = ExtractFramesUseCase( + backend_service_provider=self._service, + project=self._project, + folder=self._folder, + video_path=path, + extract_path=temp_path, + start_time=self._start_time, + end_time=self._end_time, + target_fps=self._target_fps, + annotation_status_code=self.annotation_status, + image_quality_in_editor=self._image_quality_in_editor, + ) + if not frames_generator_use_case.is_valid(): + self._response.errors = use_case.response.errors + return self._response + + frames_generator = frames_generator_use_case.execute() + + total_frames_count = len(frame_names) + self.reporter.log_info( + f"Video frame count is {total_frames_count}." + ) + self.reporter.log_info( + f"Extracted {total_frames_count} frames from video. Now uploading to platform.", + ) + self.reporter.log_info( + f"Uploading {total_frames_count} images to project {str(self.upload_path)}." + ) + if len(duplicate_images): + self.reporter.log_warning( + f"{len(duplicate_images)} already existing images found that won't be uploaded." + ) + if set(duplicate_images) == set(frame_names): + continue + uploaded_paths = [] + for _ in frames_generator: + use_case = UploadImagesFromFolderToProject( + project=self._project, + folder=self._folder, + backend_client=self._service, + folder_path=temp_path, + settings=self._settings, + s3_repo=self._s3_repo, + annotation_status=self.annotation_status, + image_quality_in_editor=self._image_quality_in_editor, + ) + + images_to_upload, duplicates = use_case.images_to_upload + if not len(images_to_upload): + continue + progress = self.reporter.get_progress_bar( + total_frames_count, f"Uploading {Path(path).name}" + ) + if use_case.is_valid(): + for _ in use_case.execute(): + progress.update() + + uploaded, failed_images, _ = use_case.response.data + uploaded_paths.extend(uploaded) + if failed_images: + self.reporter.log_warning( + f"Failed {len(failed_images)}." + ) + files = os.listdir(temp_path) + image_paths = [f"{temp_path}/{f}" for f in files] + for image_path in image_paths: + os.remove(image_path) + else: + raise AppException(use_case.response.errors) + progress.close() + data[str(path)] = uploaded_paths + self._response.data = data + return self._response diff --git a/src/superannotate/lib/core/usecases/models.py b/src/superannotate/lib/core/usecases/models.py index 2ec04d59e..353820201 100644 --- a/src/superannotate/lib/core/usecases/models.py +++ b/src/superannotate/lib/core/usecases/models.py @@ -31,7 +31,6 @@ from lib.core.usecases.base import BaseUseCase from lib.core.usecases.images import GetBulkImages - logger = logging.getLogger("root") diff --git a/src/superannotate/lib/core/usecases/projects.py b/src/superannotate/lib/core/usecases/projects.py index 8a4bda5c0..ea47fed6b 100644 --- a/src/superannotate/lib/core/usecases/projects.py +++ b/src/superannotate/lib/core/usecases/projects.py @@ -30,7 +30,7 @@ class GetProjectsUseCase(BaseUseCase): def __init__( - self, condition: Condition, team_id: int, projects: BaseManageableRepository, + self, condition: Condition, team_id: int, projects: BaseManageableRepository, ): super().__init__() self._condition = condition @@ -46,7 +46,7 @@ def execute(self): class GetProjectByNameUseCase(BaseUseCase): def __init__( - self, name: str, team_id: int, projects: BaseManageableRepository, + self, name: str, team_id: int, projects: BaseManageableRepository, ): super().__init__() self._name = name @@ -74,18 +74,18 @@ def execute(self): class GetProjectMetaDataUseCase(BaseUseCase): def __init__( - self, - project: ProjectEntity, - service: SuerannotateServiceProvider, - annotation_classes: BaseManageableRepository, - settings: BaseManageableRepository, - workflows: BaseManageableRepository, - projects: BaseManageableRepository, - include_annotation_classes: bool, - include_settings: bool, - include_workflow: bool, - include_contributors: bool, - include_complete_image_count: bool, + self, + project: ProjectEntity, + service: SuerannotateServiceProvider, + annotation_classes: BaseManageableRepository, + settings: BaseManageableRepository, + workflows: BaseManageableRepository, + projects: BaseManageableRepository, + include_annotation_classes: bool, + include_settings: bool, + include_workflow: bool, + include_contributors: bool, + include_complete_image_count: bool, ): super().__init__() self._project = project @@ -127,9 +127,9 @@ def execute(self): if self._include_complete_image_count: projects = self._projects.get_all( condition=( - Condition("completeImagesCount", "true", EQ) - & Condition("name", self._project.name, EQ) - & Condition("team_id", self._project.team_id, EQ) + Condition("completeImagesCount", "true", EQ) + & Condition("name", self._project.name, EQ) + & Condition("team_id", self._project.team_id, EQ) ) ) if projects: @@ -158,17 +158,17 @@ def execute(self): class CreateProjectUseCase(BaseUseCase): def __init__( - self, - project: ProjectEntity, - projects: BaseManageableRepository, - backend_service_provider: SuerannotateServiceProvider, - settings_repo: Type[BaseManageableRepository], - annotation_classes_repo: BaseManageableRepository, - workflows_repo: BaseManageableRepository, - settings: List[ProjectSettingEntity] = None, - workflows: List[WorkflowEntity] = None, - annotation_classes: List[AnnotationClassEntity] = None, - contributors: Iterable[dict] = None, + self, + project: ProjectEntity, + projects: BaseManageableRepository, + backend_service_provider: SuerannotateServiceProvider, + settings_repo: Type[BaseManageableRepository], + annotation_classes_repo: BaseManageableRepository, + workflows_repo: BaseManageableRepository, + settings: List[ProjectSettingEntity] = None, + workflows: List[WorkflowEntity] = None, + annotation_classes: List[AnnotationClassEntity] = None, + contributors: Iterable[dict] = None, ): super().__init__() @@ -185,12 +185,12 @@ def __init__( def validate_project_name(self): if ( - len( - set(self._project.name).intersection( - constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES - ) + len( + set(self._project.name).intersection( + constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES ) - > 0 + ) + > 0 ): self._project.name = "".join( "_" @@ -274,7 +274,7 @@ def execute(self): class DeleteProjectUseCase(BaseUseCase): def __init__( - self, project_name: str, team_id: int, projects: BaseManageableRepository, + self, project_name: str, team_id: int, projects: BaseManageableRepository, ): super().__init__() @@ -297,10 +297,10 @@ def execute(self): class UpdateProjectUseCase(BaseUseCase): def __init__( - self, - project: ProjectEntity, - project_data: dict, - projects: BaseManageableRepository, + self, + project: ProjectEntity, + project_data: dict, + projects: BaseManageableRepository, ): super().__init__() @@ -311,12 +311,12 @@ def __init__( def validate_project_name(self): if self._project_data.get("name"): if ( - len( - set(self._project_data["name"]).intersection( - constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES - ) + len( + set(self._project_data["name"]).intersection( + constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES ) - > 0 + ) + > 0 ): self._project_data["name"] = "".join( "_" @@ -349,19 +349,19 @@ def execute(self): class CloneProjectUseCase(BaseReportableUseCae): def __init__( - self, - reporter: Reporter, - project: ProjectEntity, - project_to_create: ProjectEntity, - projects: BaseManageableRepository, - settings_repo: Type[BaseManageableRepository], - workflows_repo: Type[BaseManageableRepository], - annotation_classes_repo: Type[BaseManageableRepository], - backend_service_provider: SuerannotateServiceProvider, - include_annotation_classes: bool = True, - include_settings: bool = True, - include_workflow: bool = True, - include_contributors: bool = False, + self, + reporter: Reporter, + project: ProjectEntity, + project_to_create: ProjectEntity, + projects: BaseManageableRepository, + settings_repo: Type[BaseManageableRepository], + workflows_repo: Type[BaseManageableRepository], + annotation_classes_repo: Type[BaseManageableRepository], + backend_service_provider: SuerannotateServiceProvider, + include_annotation_classes: bool = True, + include_settings: bool = True, + include_workflow: bool = True, + include_contributors: bool = False, ): super().__init__(reporter) self._project = project @@ -391,12 +391,12 @@ def workflows(self): def validate_project_name(self): if self._project_to_create.name: if ( - len( - set(self._project_to_create.name).intersection( - constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES - ) + len( + set(self._project_to_create.name).intersection( + constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES ) - > 0 + ) + > 0 ): self._project_to_create.name = "".join( "_" @@ -422,7 +422,7 @@ def get_annotation_classes_repo(self, project: ProjectEntity): return self._annotation_classes_repo(self._backend_service, project) def _copy_annotation_classes( - self, annotation_classes_entity_mapping: dict, project: ProjectEntity + self, annotation_classes_entity_mapping: dict, project: ProjectEntity ): annotation_classes = self.annotation_classes.get_all() for annotation_class in annotation_classes: @@ -461,7 +461,7 @@ def _copy_settings(self, to_project: ProjectEntity): new_settings.update(setting_copy) def _copy_workflow( - self, annotation_classes_entity_mapping: dict, to_project: ProjectEntity + self, annotation_classes_entity_mapping: dict, to_project: ProjectEntity ): new_workflows = self._workflows_repo(self._backend_service, to_project) for workflow in self.workflows.get_all(): @@ -487,15 +487,15 @@ def _copy_workflow( workflow.class_id ].attribute_groups: if ( - attribute["attribute"]["attribute_group"]["name"] - == annotation_attribute["name"] + attribute["attribute"]["attribute_group"]["name"] + == annotation_attribute["name"] ): for annotation_attribute_value in annotation_attribute[ "attributes" ]: if ( - annotation_attribute_value["name"] - == attribute["attribute"]["name"] + annotation_attribute_value["name"] + == attribute["attribute"]["name"] ): workflow_attributes.append( { @@ -551,8 +551,8 @@ def execute(self): if self._include_workflow: if self._project.project_type in ( - constances.ProjectType.DOCUMENT.value, - constances.ProjectType.VIDEO.value, + constances.ProjectType.DOCUMENT.value, + constances.ProjectType.VIDEO.value, ): self.reporter.log_warning( "Workflow copy is deprecated for " @@ -593,11 +593,11 @@ def execute(self): class ShareProjectUseCase(BaseUseCase): def __init__( - self, - service: SuerannotateServiceProvider, - project_entity: ProjectEntity, - user_id: str, - user_role: str, + self, + service: SuerannotateServiceProvider, + project_entity: ProjectEntity, + user_id: str, + user_role: str, ): super().__init__() self._service = service @@ -624,10 +624,10 @@ def execute(self): class UnShareProjectUseCase(BaseUseCase): def __init__( - self, - service: SuerannotateServiceProvider, - project_entity: ProjectEntity, - user_id: str, + self, + service: SuerannotateServiceProvider, + project_entity: ProjectEntity, + user_id: str, ): super().__init__() self._service = service @@ -658,11 +658,11 @@ def execute(self): class GetWorkflowsUseCase(BaseUseCase): def __init__( - self, - project: ProjectEntity, - annotation_classes: BaseReadOnlyRepository, - workflows: BaseManageableRepository, - fill_classes=True, + self, + project: ProjectEntity, + annotation_classes: BaseReadOnlyRepository, + workflows: BaseManageableRepository, + fill_classes=True, ): super().__init__() self._project = project @@ -694,7 +694,7 @@ def execute(self): class GetAnnotationClassesUseCase(BaseUseCase): def __init__( - self, classes: BaseManageableRepository, condition: Condition = None, + self, classes: BaseManageableRepository, condition: Condition = None, ): super().__init__() self._classes = classes @@ -707,13 +707,13 @@ def execute(self): class UpdateSettingsUseCase(BaseUseCase): def __init__( - self, - projects: BaseReadOnlyRepository, - settings: BaseManageableRepository, - to_update: List, - backend_service_provider: SuerannotateServiceProvider, - project_id: int, - team_id: int, + self, + projects: BaseReadOnlyRepository, + settings: BaseManageableRepository, + to_update: List, + backend_service_provider: SuerannotateServiceProvider, + project_id: int, + team_id: int, ): super().__init__() self._projects = projects @@ -726,7 +726,7 @@ def __init__( def validate_image_quality(self): for setting in self._to_update: if setting["attribute"].lower() == "imagequality" and isinstance( - setting["value"], str + setting["value"], str ): setting["value"] = constances.ImageQuality.get_value(setting["value"]) return @@ -735,7 +735,7 @@ def validate_project_type(self): project = self._projects.get_one(uuid=self._project_id, team_id=self._team_id) for attribute in self._to_update: if attribute.get( - "attribute", "" + "attribute", "" ) == "ImageQuality" and project.project_type in [ constances.ProjectType.VIDEO.value, constances.ProjectType.DOCUMENT.value, @@ -771,11 +771,11 @@ def execute(self): class GetProjectImageCountUseCase(BaseUseCase): def __init__( - self, - service: SuerannotateServiceProvider, - project: ProjectEntity, - folder: FolderEntity, - with_all_sub_folders: bool = False, + self, + service: SuerannotateServiceProvider, + project: ProjectEntity, + folder: FolderEntity, + with_all_sub_folders: bool = False, ): super().__init__() self._service = service @@ -815,12 +815,12 @@ def execute(self): class SetWorkflowUseCase(BaseUseCase): def __init__( - self, - service: SuerannotateServiceProvider, - annotation_classes_repo: BaseManageableRepository, - workflow_repo: BaseManageableRepository, - steps: list, - project: ProjectEntity, + self, + service: SuerannotateServiceProvider, + annotation_classes_repo: BaseManageableRepository, + workflow_repo: BaseManageableRepository, + steps: list, + project: ProjectEntity, ): super().__init__() self._service = service @@ -876,8 +876,8 @@ def execute(self): "name" ] if not annotations_classes_attributes_map.get( - f"{annotation_class_name}__{attribute_group_name}__{attribute_name}", - None, + f"{annotation_class_name}__{attribute_group_name}__{attribute_name}", + None, ): raise AppException( "Attribute group name or attribute name not found in set_project_workflow." @@ -919,10 +919,10 @@ def execute(self): class SearchContributorsUseCase(BaseUseCase): def __init__( - self, - backend_service_provider: SuerannotateServiceProvider, - team_id: int, - condition: Condition = None, + self, + backend_service_provider: SuerannotateServiceProvider, + team_id: int, + condition: Condition = None, ): super().__init__() self._backend_service = backend_service_provider @@ -948,13 +948,13 @@ class AddContributorsToProject(BaseUserBasedUseCase): """ def __init__( - self, - reporter: Reporter, - team: TeamEntity, - project: ProjectEntity, - emails: list, - role: str, - service: SuerannotateServiceProvider, + self, + reporter: Reporter, + team: TeamEntity, + project: ProjectEntity, + emails: list, + role: str, + service: SuerannotateServiceProvider, ): super().__init__(reporter, emails) self._team = team @@ -1014,12 +1014,12 @@ class InviteContributorsToTeam(BaseUserBasedUseCase): """ def __init__( - self, - reporter: Reporter, - team: TeamEntity, - emails: list, - set_admin: bool, - service: SuerannotateServiceProvider, + self, + reporter: Reporter, + team: TeamEntity, + emails: list, + set_admin: bool, + service: SuerannotateServiceProvider, ): super().__init__(reporter, emails) self._team = team @@ -1030,7 +1030,9 @@ def execute(self): if self.is_valid(): team_users = {user.email for user in self._team.users} # collecting pending team users - team_users.update({user["email"] for user in self._team.pending_invitations}) + team_users.update( + {user["email"] for user in self._team.pending_invitations} + ) emails = set(self._emails) @@ -1045,8 +1047,10 @@ def execute(self): invited, failed = self._service.invite_contributors( team_id=self._team.uuid, # REMINDER UserRole.VIEWER is the contributor for the teams - team_role=constances.UserRole.ADMIN.value if self._set_admin else constances.UserRole.VIEWER.value, - emails=to_add + team_role=constances.UserRole.ADMIN.value + if self._set_admin + else constances.UserRole.VIEWER.value, + emails=to_add, ) if invited: self.reporter.log_info( diff --git a/src/superannotate/lib/infrastructure/controller.py b/src/superannotate/lib/infrastructure/controller.py index 4db0d802b..1f044b6e1 100644 --- a/src/superannotate/lib/infrastructure/controller.py +++ b/src/superannotate/lib/infrastructure/controller.py @@ -1587,3 +1587,39 @@ def invite_contributors_to_team(self, emails: list, set_admin: bool): service=self.backend_client, ) return use_case.execute() + + def upload_videos( + self, + project_name: str, + folder_name: str, + paths: List[str], + start_time: float, + extensions: List[str] = None, + exclude_file_patterns: List[str] = None, + end_time: Optional[float] = None, + target_fps: Optional[int] = None, + annotation_status: Optional[str] = None, + image_quality_in_editor: Optional[str] = None, + ): + project = self._get_project(project_name) + folder = self._get_folder(project, folder_name) + + use_case = usecases.UploadVideosAsImages( + reporter=self.default_reporter, + service=self.backend_client, + project=project, + folder=folder, + settings=ProjectSettingsRepository( + service=self._backend_client, project=project + ), + s3_repo=self.s3_repo, + paths=paths, + target_fps=target_fps, + extensions=extensions, + exclude_file_patterns=exclude_file_patterns, + start_time=start_time, + end_time=end_time, + annotation_status=annotation_status, + image_quality_in_editor=image_quality_in_editor, + ) + return use_case.execute() diff --git a/src/superannotate/lib/infrastructure/services.py b/src/superannotate/lib/infrastructure/services.py index d840f8ed5..60a9dbd30 100644 --- a/src/superannotate/lib/infrastructure/services.py +++ b/src/superannotate/lib/infrastructure/services.py @@ -511,14 +511,16 @@ def delete_team_invitation(self, team_id: int, token: str, email: str) -> bool: ) return res.ok - def invite_contributors(self, team_id: int, team_role: int, emails: list) -> Tuple[List[str], List[str]]: + def invite_contributors( + self, team_id: int, team_role: int, emails: list + ) -> Tuple[List[str], List[str]]: invite_contributors_url = urljoin( self.api_url, self.URL_INVITE_CONTRIBUTORS.format(team_id) ) res = self._request( invite_contributors_url, "post", - data=dict(emails=emails, team_role=team_role) + data=dict(emails=emails, team_role=team_role), ).json() return res["success"]["emails"], res["failed"]["emails"]