Fix logs

src/superannotate/lib/app/interface/cli_interface.py

@@ -23,7 +23,8 @@
 from lib.infrastructure.repositories import ConfigRepository
 
 
-controller = Controller.get_default()
+controller = Controller()
+controller.retrieve_configs(constances.CONFIG_FILE_LOCATION)
 
 
 class CLIFacade(BaseInterfaceFacade):

src/superannotate/lib/app/interface/sdk_interface.py

@@ -547,6 +547,12 @@ def upload_images_from_public_urls_to_project(
      and not-uploaded images' urls
     :rtype: tuple of list of strs
     """
+    warning_msg = (
+        "The upload_images_from_public_urls function is deprecated and will be removed with the coming release, "
+        "please use attach_image_urls_to_project instead."
+    )
+    logger.warning(warning_msg)
+    warnings.warn(warning_msg, DeprecationWarning)
 
     project_name, folder_name = extract_project_folder(project)
 
@@ -2416,7 +2422,7 @@ def add_annotation_comment_to_image(
     image_name: NotEmptyStr,
     comment_text: NotEmptyStr,
     comment_coords: List[float],
-    comment_author: NotEmptyStr,
+    comment_author: EmailStr,
     resolved: Optional[StrictBool] = False,
 ):
     """Add a comment to SuperAnnotate format annotation JSON
@@ -2850,7 +2856,7 @@ def invite_contributors_to_team(
 
 @Trackable
 @validate_arguments
-def get_annotations(project: NotEmptyStr, items: Optional[List[NotEmptyStr]]):
+def get_annotations(project: NotEmptyStr, items: Optional[List[NotEmptyStr]] = None):
     """Returns annotations for the given list of items.
 
     :param project: project name

src/superannotate/lib/app/mixp/decorators.py

@@ -88,8 +88,8 @@ def __call__(self, *args, **kwargs):
             else:
                 raise Exception(
                     "SuperAnnotate config file not found."
-                    f" Please provide correct config file location to sa.init(<path>) or use "
-                    f"CLI's superannotate init to generate default location config file."
+                    " Please provide correct config file location to sa.init(<path>) or use "
+                    "CLI's superannotate init to generate default location config file."
                 )
         except Exception as e:
             self._success = False

src/superannotate/lib/app/mixp/utils/parsers.py

@@ -1226,4 +1226,4 @@ def get_annotations_per_frame(*args, **kwargs):
     return {
         "event_name": "get_annotations_per_frame",
         "properties": {"Project": project, "fps": fps},
-    }
+    }

src/superannotate/lib/core/data_handlers.py

@@ -8,13 +8,12 @@
 from typing import Dict
 from typing import List
 
-from superannotate_schemas.schemas.classes import AnnotationClass
-from superannotate_schemas.schemas.classes import Attribute
-from superannotate_schemas.schemas.classes import AttributeGroup
-
 import lib.core as constances
 from lib.core.enums import ClassTypeEnum
 from lib.core.reporter import Reporter
+from superannotate_schemas.schemas.classes import AnnotationClass
+from superannotate_schemas.schemas.classes import Attribute
+from superannotate_schemas.schemas.classes import AttributeGroup
 
 
 class BaseDataHandler(metaclass=ABCMeta):

src/superannotate/lib/core/reporter.py

@@ -46,9 +46,9 @@ def log_debug(self, value: str):
         self.debug_messages.append(value)
 
     def start_progress(
-        self, iterations: Union[int, range], description: str = "Processing"
+        self, iterations: Union[int, range], description: str = "Processing", disable=False
     ):
-        self.progress_bar = self.get_progress_bar(iterations, description)
+        self.progress_bar = self.get_progress_bar(iterations, description, disable)
 
     @staticmethod
     def get_progress_bar(
@@ -63,7 +63,8 @@ def finish_progress(self):
         self.progress_bar.close()
 
     def update_progress(self, value: int = 1):
-        self.progress_bar.update(value)
+        if self.progress_bar:
+            self.progress_bar.update(value)
 
     def generate_report(self) -> str:
         report = ""

src/superannotate/lib/core/serviceproviders.py

@@ -5,6 +5,7 @@
 from typing import List
 from typing import Tuple
 
+from lib.core.reporter import Reporter
 from lib.core.service_types import ServiceResponse
 
 
@@ -304,5 +305,12 @@ def get_limitations(
         raise NotImplementedError
 
     @abstractmethod
-    def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: List[str]) -> List[dict]:
+    def get_annotations(
+            self,
+            project_id: int,
+            team_id: int,
+            folder_id: int,
+            items: List[str],
+            reporter: Reporter
+    ) -> List[dict]:
         raise NotImplementedError

src/superannotate/lib/core/usecases/annotations.py

@@ -9,6 +9,8 @@
 
 import boto3
 import lib.core as constances
+from lib.core.conditions import Condition
+from lib.core.conditions import CONDITION_EQ as EQ
 from lib.core.data_handlers import ChainedAnnotationHandlers
 from lib.core.data_handlers import DocumentTagHandler
 from lib.core.data_handlers import LastActionHandler
@@ -494,14 +496,18 @@ def __init__(
             reporter: Reporter,
             project: ProjectEntity,
             folder: FolderEntity,
+            images: BaseManageableRepository,
             item_names: Optional[List[str]],
-            backend_service_provider: SuerannotateServiceProvider
+            backend_service_provider: SuerannotateServiceProvider,
+            show_process: bool = True
     ):
         super().__init__(reporter)
         self._project = project
         self._folder = folder
+        self._images = images
         self._item_names = item_names
         self._client = backend_service_provider
+        self._show_process = show_process
 
     def validate_item_names(self):
         if self._item_names:
@@ -512,15 +518,36 @@ def validate_item_names(self):
                     f"Dropping duplicates. Found {len_unique_items}/{len_items} unique items."
                 )
                 self._item_names = item_names
+        else:
+            condition = (
+                    Condition("team_id", self._project.team_id, EQ)
+                    & Condition("project_id", self._project.uuid, EQ)
+                    & Condition("folder_id", self._folder.uuid, EQ)
+            )
+
+            self._item_names = [item.name for item in self._images.get_all(condition)]
 
     def execute(self):
-        annotations = self._client.get_annotations(
-            team_id=self._project.team_id,
-            project_id=self._project.uuid,
-            folder_id=self._folder.uuid,
-            items=self._item_names
-        )
-        self._response.data = annotations
+        if self.is_valid():
+            items_count = len(self._item_names)
+            self.reporter.log_info(
+                f"Getting {items_count} annotations from "
+                f"{self._project.name}{f'/{self._folder.name}' if self._folder else ''}."
+            )
+            self.reporter.start_progress(items_count, disable=not self._show_process)
+            annotations = self._client.get_annotations(
+                team_id=self._project.team_id,
+                project_id=self._project.uuid,
+                folder_id=self._folder.uuid,
+                items=self._item_names,
+                reporter=self.reporter
+            )
+            received_items_count = len(annotations)
+            if items_count > received_items_count:
+                self.reporter.log_warning(
+                    f"Could not find annotations for {items_count - received_items_count}/{items_count} items."
+                )
+            self._response.data = annotations
         return self._response
 
 
@@ -530,13 +557,15 @@ def __init__(
             reporter: Reporter,
             project: ProjectEntity,
             folder: FolderEntity,
+            images: BaseManageableRepository,
             video_name: str,
             fps: int,
             backend_service_provider: SuerannotateServiceProvider
     ):
         super().__init__(reporter)
         self._project = project
         self._folder = folder
+        self._images = images
         self._video_name = video_name
         self._fps = fps
         self._client = backend_service_provider
@@ -546,17 +575,21 @@ def execute(self):
             reporter=self.reporter,
             project=self._project,
             folder=self._folder,
+            images=self._images,
             item_names=[self._video_name],
-            backend_service_provider=self._client
+            backend_service_provider=self._client,
+            show_process=False
         ).execute()
+        generator = VideoFrameGenerator(response.data[0], fps=self._fps)
+        self.reporter.log_info(f"Getting annotations for {generator.frames_count} frames from {self._video_name}.")
         if response.errors:
             self._response.errors = response.errors
             return self._response
         if not response.data:
             self._response.errors = AppException(f"Video {self._video_name} not found.")
         annotations = response.data
         if annotations:
-            self._response.data = list(VideoFrameGenerator(response.data[0], fps=self._fps))
+            self._response.data = list(generator)
         else:
             self._response.data = []
         return self._response

src/superannotate/lib/core/usecases/images.py

@@ -1928,8 +1928,8 @@ def execute(self):
                     project=self._project,
                     folder=self._folder,
                     attachments=self._attachments[
-                        i : i + self.CHUNK_SIZE
-                    ],  # noqa: E203
+                        i : i + self.CHUNK_SIZE  # noqa: E203
+                    ],
                     backend_service_provider=self._backend_service,
                     annotation_status=self._annotation_status,
                     upload_state_code=self._upload_state_code,

src/superannotate/lib/core/usecases/projects.py

@@ -1,6 +1,5 @@
 import copy
 from collections import defaultdict
-from typing import Iterable
 from typing import List
 from typing import Type
 

src/superannotate/lib/core/video_convertor.py

@@ -41,7 +41,7 @@ def __init__(self, annotation_data: dict, fps: int):
         self.fps = fps
         self.ratio = 1000 * 1000 / fps
         self._frame_id = 1
-        self._frames_count = self.duration * fps
+        self.frames_count = self.duration * fps
         self.annotations: dict = {}
         self._mapping = {}
         self._process()
@@ -141,5 +141,5 @@ def _process(self):
                         )
 
     def __iter__(self):
-        for frame_no in range(1, int(self._frames_count) + 1):
+        for frame_no in range(1, int(self.frames_count) + 1):
             yield self.get_frame(frame_no).dict()

src/superannotate/lib/infrastructure/controller.py

@@ -1626,6 +1626,7 @@ def get_annotations(self, project_name: str, folder_name: str, item_names: List[
             reporter=self.default_reporter,
             project=project,
             folder=folder,
+            images=self.images,
             item_names=item_names,
             backend_service_provider=self.backend_client
         )
@@ -1639,6 +1640,7 @@ def get_annotations_per_frame(self, project_name: str, folder_name: str, video_n
             reporter=self.default_reporter,
             project=project,
             folder=folder,
+            images=self.images,
             video_name=video_name,
             fps=fps,
             backend_service_provider=self.backend_client

src/superannotate/lib/infrastructure/services.py

@@ -13,6 +13,7 @@
 import lib.core as constance
 import requests.packages.urllib3
 from lib.core.exceptions import AppException
+from lib.core.reporter import Reporter
 from lib.core.service_types import DownloadMLModelAuthData
 from lib.core.service_types import ServiceResponse
 from lib.core.service_types import UploadAnnotationAuthData
@@ -175,6 +176,7 @@ class SuperannotateBackendService(BaseBackendService):
     DEFAULT_CHUNK_SIZE = 1000
 
     URL_USERS = "users"
+    URL_LIST_ALL_IMAGES = "/images/getImagesWithAnnotationPaths"
     URL_LIST_PROJECTS = "projects"
     URL_FOLDERS_IMAGES = "images-folders"
     URL_CREATE_PROJECT = "project"
@@ -1016,7 +1018,14 @@ def get_limitations(
             content_type=UserLimits,
         )
 
-    def get_annotations(self, project_id: int, team_id: int, folder_id: int, items: List[str]) -> List[dict]:
+    def get_annotations(
+            self,
+            project_id: int,
+            team_id: int,
+            folder_id: int,
+            items: List[str],
+            reporter: Reporter
+    ) -> List[dict]:
         get_limits_url = urljoin(self.assets_provider_url, self.URL_GET_ANNOTATIONS)
         query_params = {
             "team_id": team_id,
@@ -1025,7 +1034,7 @@ def get_annotations(self, project_id: int, team_id: int, folder_id: int, items:
         if folder_id:
             query_params["folder_id"] = folder_id
 
-        handler = StreamedAnnotations(self.default_headers)
+        handler = StreamedAnnotations(self.default_headers, reporter)
         loop = asyncio.new_event_loop()
 
         return loop.run_until_complete(handler.get_data(

src/superannotate/lib/infrastructure/stream_data_handler.py

@@ -3,6 +3,7 @@
 from typing import List
 
 import aiohttp
+from lib.core.reporter import Reporter
 
 
 def map_image_names_to_fetch_streamed_data(data: List[str]):
@@ -15,26 +16,30 @@ def map_image_names_to_fetch_streamed_data(data: List[str]):
 class StreamedAnnotations:
     DELIMITER = b"\\n;)\\n"
 
-    def __init__(self, headers: dict):
+    def __init__(self, headers: dict, reporter: Reporter):
         self._headers = headers
         self._annotations = []
+        self._reporter = reporter
 
     async def fetch(self, method: str, session: aiohttp.ClientSession, url: str, data: dict = None,
                     params: dict = None):
         response = await session._request(method, url, json=data, params=params)
         buffer = b""
-        async for line in response.content:
+        async for line in response.content.iter_any():
             slices = line.split(self.DELIMITER)
             if len(slices) == 1:
                 buffer += slices[0]
                 continue
             elif slices[0]:
                 self._annotations.append(json.loads(buffer + slices[0]))
+                self._reporter.update_progress()
             for data in slices[1:-1]:
                 self._annotations.append(json.loads(data))
+                self._reporter.update_progress()
             buffer = slices[-1]
         if buffer:
             self._annotations.append(json.loads(buffer))
+            self._reporter.update_progress()
         return self._annotations
 
     async def get_data(
@@ -52,7 +57,8 @@ async def get_data(
 
             if chunk_size:
                 for i in range(0, len(data), chunk_size):
-                    await self.fetch(method, session, url, map_function(data[i:i + chunk_size]), params=params)
+                    data_to_process = data[i:i + chunk_size]
+                    await self.fetch(method, session, url, map_function(data_to_process), params=params)
             else:
                 await self.fetch(method, session, url, map_function(data), params=params)
         return self._annotations