From 6e64887e4b8c89ceaf4cba985899aca741f95301 Mon Sep 17 00:00:00 2001 From: shab Date: Tue, 27 Apr 2021 12:08:05 +0400 Subject: [PATCH 01/22] Mixpanel initial setup --- mixp/__init__.py | 0 mixp/app.py | 4 ++++ mixp/config.py | 1 + mixp/decorators.py | 34 ++++++++++++++++++++++++++++++++++ mixp/parsers.py | 8 ++++++++ superannotate/api.py | 15 +++++++++------ superannotate/db/images.py | 2 ++ 7 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 mixp/__init__.py create mode 100644 mixp/app.py create mode 100644 mixp/config.py create mode 100644 mixp/decorators.py create mode 100644 mixp/parsers.py diff --git a/mixp/__init__.py b/mixp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mixp/app.py b/mixp/app.py new file mode 100644 index 000000000..0306a73f1 --- /dev/null +++ b/mixp/app.py @@ -0,0 +1,4 @@ +from mixpanel import Mixpanel +from .config import TOKEN + +mp = Mixpanel(TOKEN) diff --git a/mixp/config.py b/mixp/config.py new file mode 100644 index 000000000..fd134f496 --- /dev/null +++ b/mixp/config.py @@ -0,0 +1 @@ +TOKEN = "e741d4863e7e05b1a45833d01865ef0d" \ No newline at end of file diff --git a/mixp/decorators.py b/mixp/decorators.py new file mode 100644 index 000000000..85e6cb7fe --- /dev/null +++ b/mixp/decorators.py @@ -0,0 +1,34 @@ +import sys +from .parsers import parsers +from .app import mp +from superannotate.api import API + +_api = API.get_instance() + +func_names_to_ignore = [] + + +def trackable(func): + def wrapper(*args, **kwargs): + try: + caller_function_name = sys._getframe().f_back.f_code.co_name + if caller_function_name not in func_names_to_ignore: + func_name_to_track = func.__name__ + data = parsers[func_name_to_track](*args, **kwargs) + user_id = _api.user_id + event_name = data['event_name'] + properties = data['properties'] + properties['SDK'] = True + properties['Paid'] = True + properties['Team'] = _api.team_name + properties['Team Owner'] = _api.user_id + properties['Project Name'] = properties.get( + 'project_name', None + ) + properties['Project Role'] = "Admin" + mp.track(user_id, event_name, properties) + except Exception as e: + print("--- ---- --- MIX PANEL EXCEPTION") + return func(*args, **kwargs) + + return wrapper diff --git a/mixp/parsers.py b/mixp/parsers.py new file mode 100644 index 000000000..3cb3a71a1 --- /dev/null +++ b/mixp/parsers.py @@ -0,0 +1,8 @@ +parsers = {} + + +def get_project_root_folder_id_parser(*args, **kwargs): + return {"event_name": "some_name", "properties": {"prop1": "value1"}} + + +parsers['get_project_root_folder_id'] = get_project_root_folder_id_parser diff --git a/superannotate/api.py b/superannotate/api.py index f6c78e75b..38cc412a7 100644 --- a/superannotate/api.py +++ b/superannotate/api.py @@ -24,10 +24,13 @@ def __init__(self): self._default_headers = None self._main_endpoint = None self.team_id = None + self.user_id = None + self.team_name = None if API.__instance is not None: raise SABaseException(0, "API class is a singleton!") API.__instance = self self._authenticated = False + self.init() def init(self, config_location=None): if config_location is None: @@ -78,15 +81,15 @@ def init(self, config_location=None): self._verify = self._api_config["ssl_verify"] self._session = None self._authenticated = True + response = self.send_request( req_type='GET', - path='/projects', - params={ - 'team_id': str(self.team_id), - 'offset': 0, - 'limit': 1 - } + path=f'/team/{self.team_id}', ) + + self.user_id = response.json()['creator_id'] + self.team_name = response.json()['name'] + if not self._verify: urllib3.disable_warnings( urllib3.exceptions.InsecureRequestWarning diff --git a/superannotate/db/images.py b/superannotate/db/images.py index befd774f7..6faf80be0 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -26,12 +26,14 @@ ) from .project_api import get_project_and_folder_metadata, get_project_metadata_bare from .utils import _get_boto_session_by_credentials +from mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() +@trackable def get_project_root_folder_id(project): """Get root folder ID Returns From 45a06f16b6fb41ca16aa3077c0796e19b2301b1a Mon Sep 17 00:00:00 2001 From: shab Date: Tue, 27 Apr 2021 18:21:42 +0400 Subject: [PATCH 02/22] Add event --- mixp/parsers.py | 8 -------- superannotate/db/images.py | 2 -- superannotate/db/teams.py | 3 +++ {mixp => superannotate/mixp}/__init__.py | 0 {mixp => superannotate/mixp}/app.py | 0 {mixp => superannotate/mixp}/config.py | 0 {mixp => superannotate/mixp}/decorators.py | 6 +++--- superannotate/mixp/parsers.py | 8 ++++++++ 8 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 mixp/parsers.py rename {mixp => superannotate/mixp}/__init__.py (100%) rename {mixp => superannotate/mixp}/app.py (100%) rename {mixp => superannotate/mixp}/config.py (100%) rename {mixp => superannotate/mixp}/decorators.py (88%) create mode 100644 superannotate/mixp/parsers.py diff --git a/mixp/parsers.py b/mixp/parsers.py deleted file mode 100644 index 3cb3a71a1..000000000 --- a/mixp/parsers.py +++ /dev/null @@ -1,8 +0,0 @@ -parsers = {} - - -def get_project_root_folder_id_parser(*args, **kwargs): - return {"event_name": "some_name", "properties": {"prop1": "value1"}} - - -parsers['get_project_root_folder_id'] = get_project_root_folder_id_parser diff --git a/superannotate/db/images.py b/superannotate/db/images.py index 6faf80be0..befd774f7 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -26,14 +26,12 @@ ) from .project_api import get_project_and_folder_metadata, get_project_metadata_bare from .utils import _get_boto_session_by_credentials -from mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() -@trackable def get_project_root_folder_id(project): """Get root folder ID Returns diff --git a/superannotate/db/teams.py b/superannotate/db/teams.py index 2cd6db03a..43a0e9e16 100644 --- a/superannotate/db/teams.py +++ b/superannotate/db/teams.py @@ -8,6 +8,8 @@ _api = API.get_instance() +from ..mixp.decorators import trackable + def invite_contributor_to_team(email, admin=False): """Invites a contributor to team @@ -30,6 +32,7 @@ def invite_contributor_to_team(email, admin=False): return response.json() +@trackable def get_team_metadata(): """Returns team metadata diff --git a/mixp/__init__.py b/superannotate/mixp/__init__.py similarity index 100% rename from mixp/__init__.py rename to superannotate/mixp/__init__.py diff --git a/mixp/app.py b/superannotate/mixp/app.py similarity index 100% rename from mixp/app.py rename to superannotate/mixp/app.py diff --git a/mixp/config.py b/superannotate/mixp/config.py similarity index 100% rename from mixp/config.py rename to superannotate/mixp/config.py diff --git a/mixp/decorators.py b/superannotate/mixp/decorators.py similarity index 88% rename from mixp/decorators.py rename to superannotate/mixp/decorators.py index 85e6cb7fe..e41845700 100644 --- a/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -5,14 +5,14 @@ _api = API.get_instance() -func_names_to_ignore = [] - def trackable(func): def wrapper(*args, **kwargs): try: + import superannotate + callers_to_ignore = dir(superannotate) caller_function_name = sys._getframe().f_back.f_code.co_name - if caller_function_name not in func_names_to_ignore: + if caller_function_name not in callers_to_ignore: func_name_to_track = func.__name__ data = parsers[func_name_to_track](*args, **kwargs) user_id = _api.user_id diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py new file mode 100644 index 000000000..edcc251bb --- /dev/null +++ b/superannotate/mixp/parsers.py @@ -0,0 +1,8 @@ +parsers = {} + + +def get_team_metadata(*args, **kwargs): + return {"event_name": "get_team_metadata", "properties": {}} + + +parsers['get_team_metadata'] = get_team_metadata From 97ebd7a16bbb2bc6bd972451ef4a840b44b37e25 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 28 Apr 2021 12:43:45 +0400 Subject: [PATCH 03/22] Add mixpanel events --- superannotate/api.py | 5 ++ superannotate/db/images.py | 2 + superannotate/db/projects.py | 6 +- superannotate/db/search_projects.py | 2 + superannotate/db/teams.py | 2 + superannotate/db/users.py | 2 + superannotate/mixp/app.py | 11 +++ superannotate/mixp/decorators.py | 25 +++--- superannotate/mixp/parsers.py | 120 ++++++++++++++++++++++++++++ 9 files changed, 161 insertions(+), 14 deletions(-) diff --git a/superannotate/api.py b/superannotate/api.py index 38cc412a7..5c618688d 100644 --- a/superannotate/api.py +++ b/superannotate/api.py @@ -9,6 +9,7 @@ from .exceptions import SABaseException from .version import __version__ +from .mixp.app import mp, get_default logger = logging.getLogger("superannotate-python-sdk") @@ -105,6 +106,10 @@ def init(self, config_location=None): raise SABaseException( 0, "Couldn't reach superannotate " + response.text ) + mp.track( + self.user_id, "SDK init", + get_default(self.team_name, self.user_id) + ) except SABaseException: self._authenticated = False self._session = None diff --git a/superannotate/db/images.py b/superannotate/db/images.py index befd774f7..d84a66de5 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -26,6 +26,7 @@ ) from .project_api import get_project_and_folder_metadata, get_project_metadata_bare from .utils import _get_boto_session_by_credentials +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") @@ -52,6 +53,7 @@ def get_project_root_folder_id(project): return response['data'][0]['id'] +@trackable def search_images( project, image_name_prefix=None, diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index c97888a0d..3164c3ba4 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -40,6 +40,7 @@ from .users import get_team_contributor_metadata from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, _upload_images, _attach_urls from tqdm import tqdm +from ..mixp.decorators import trackable _NUM_THREADS = 10 _TIME_TO_UPDATE_IN_TQDM = 1 @@ -48,6 +49,7 @@ _api = API.get_instance() +@trackable def create_project(project_name, project_description, project_type): """Create a new project in the team. @@ -102,6 +104,7 @@ def create_project(project_name, project_description, project_type): return res +@trackable def create_project_from_metadata(project_metadata): """Create a new project in the team using project metadata object dict. Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) @@ -592,7 +595,7 @@ def upload_images_from_folder_to_project( from_s3_bucket, image_quality_in_editor ) - +@trackable def upload_images_to_project( project, img_paths, @@ -1851,6 +1854,7 @@ def get_project_metadata( return result +@trackable def clone_project( project_name, from_project, diff --git a/superannotate/db/search_projects.py b/superannotate/db/search_projects.py index 1791990a5..644bb303e 100644 --- a/superannotate/db/search_projects.py +++ b/superannotate/db/search_projects.py @@ -6,8 +6,10 @@ logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() +from ..mixp.decorators import trackable +@trackable def search_projects( name=None, return_metadata=False, include_complete_image_count=False ): diff --git a/superannotate/db/teams.py b/superannotate/db/teams.py index 43a0e9e16..3ed70171e 100644 --- a/superannotate/db/teams.py +++ b/superannotate/db/teams.py @@ -11,6 +11,7 @@ from ..mixp.decorators import trackable +@trackable def invite_contributor_to_team(email, admin=False): """Invites a contributor to team @@ -57,6 +58,7 @@ def get_team_metadata(): return res +@trackable def delete_contributor_to_team_invitation(email): """Deletes team contributor invitation diff --git a/superannotate/db/users.py b/superannotate/db/users.py index dd122bb44..bf8d0b20d 100644 --- a/superannotate/db/users.py +++ b/superannotate/db/users.py @@ -4,10 +4,12 @@ from ..exceptions import SABaseException logger = logging.getLogger("superannotate-python-sdk") +from ..mixp.decorators import trackable _api = API.get_instance() +@trackable def search_team_contributors( email=None, first_name=None, last_name=None, return_metadata=False ): diff --git a/superannotate/mixp/app.py b/superannotate/mixp/app.py index 0306a73f1..45bf0e6b5 100644 --- a/superannotate/mixp/app.py +++ b/superannotate/mixp/app.py @@ -2,3 +2,14 @@ from .config import TOKEN mp = Mixpanel(TOKEN) + + +def get_default(team_name, user_id, project_name=None): + return { + "SDK": True, + "Paid": True, + "Team": team_name, + "Team Owner": user_id, + "Project Name": project_name, + "Project Role": "Admin", + } diff --git a/superannotate/mixp/decorators.py b/superannotate/mixp/decorators.py index e41845700..475f3c1d6 100644 --- a/superannotate/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -1,16 +1,17 @@ import sys from .parsers import parsers -from .app import mp +from .app import mp, get_default from superannotate.api import API _api = API.get_instance() +callers_to_ignore = [] def trackable(func): + callers_to_ignore.append(func.__name__) + def wrapper(*args, **kwargs): - try: - import superannotate - callers_to_ignore = dir(superannotate) + if 1: caller_function_name = sys._getframe().f_back.f_code.co_name if caller_function_name not in callers_to_ignore: func_name_to_track = func.__name__ @@ -18,17 +19,15 @@ def wrapper(*args, **kwargs): user_id = _api.user_id event_name = data['event_name'] properties = data['properties'] - properties['SDK'] = True - properties['Paid'] = True - properties['Team'] = _api.team_name - properties['Team Owner'] = _api.user_id - properties['Project Name'] = properties.get( - 'project_name', None + default = get_default( + _api.team_name, + _api.user_id, + project_name=properties.get('project_name', None) ) - properties['Project Role'] = "Admin" + properties = {**default, **properties} mp.track(user_id, event_name, properties) - except Exception as e: - print("--- ---- --- MIX PANEL EXCEPTION") + # except Exception as e: + # print("--- ---- --- MIX PANEL EXCEPTION") return func(*args, **kwargs) return wrapper diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index edcc251bb..4b9fdfbcb 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -6,3 +6,123 @@ def get_team_metadata(*args, **kwargs): parsers['get_team_metadata'] = get_team_metadata + + +def invite_contributor_to_team(*args, **kwargs): + admin = args[1:2] + if admin: + admin = "CUSTOM" + else: + admin = "DEFAULT" + + return { + "event_name": "invite_contributor_to_team", + "properties": { + "Admin": admin + } + } + + +parsers['invite_contributor_to_team'] = invite_contributor_to_team + + +def delete_contributor_to_team_invitation(*args, **kwargs): + return { + "event_name": "delete_contributor_to_team_invitation", + "properties": {} + } + + +parsers['delete_contributor_to_team_invitation' + ] = delete_contributor_to_team_invitation + + +def search_team_contributors(*args, **kwargs): + return { + "event_name": "search_team_contributors", + "properties": + { + "Email": bool(args[0:1]), + "Name": bool(args[1:2]), + "Surname": bool(args[2:3]) + } + } + + +parsers['search_team_contributors'] = search_team_contributors + + +def search_projects(*args, **kwargs): + return { + "event_name": "search_projects", + "properties": { + "Metadata": bool(args[2:3]) + } + } + + +parsers['search_projects'] = search_projects + + +def create_project(*args, **kwargs): + return { + "event_name": "create_project", + "properties": { + "Project Type": args[2:3][0] + } + } + + +parsers['create_project'] = create_project + + +def create_project_from_metadata(*args, **kwargs): + return {"event_name": "create_project_from_metadata", "properties": {}} + + +parsers['create_project_from_metadata'] = create_project_from_metadata + + +def clone_project(*args, **kwargs): + return { + "event_name": "clone_project", + "properties": + { + "Copy Classes": bool(args[3:4]), + "Copy Settings": bool(args[4:5]), + "Copy Workflow": bool(args[5:6]), + "Copy Contributors": bool(args[6:7]) + } + } + + +parsers['clone_project'] = clone_project + + +def search_images(*args, **kwargs): + return { + "event_name": "search_images", + "properties": + { + "Annotation Status": bool(args[2:3]), + "Metadata": bool(args[3:4]), + } + } + + +parsers['search_images'] = search_images + + +def upload_images_to_project(*args, **kwargs): + return { + "event_name": "upload_images_to_project", + "properties": + { + "Image Count": len(args[1]), + "Annotation Status": bool(args[2:3]), + "From S3": bool(args[3:4]) + } + } + + +parsers['upload_images_to_project'] = upload_images_to_project From 794839fe6a6024f0d0513153b603ef076e6dad76 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 28 Apr 2021 15:34:56 +0400 Subject: [PATCH 04/22] Fix parsers --- superannotate/db/project_images.py | 2 + superannotate/mixp/parsers.py | 71 +++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 276fe602f..a081a7112 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -19,11 +19,13 @@ get_project_default_image_quality_in_editor, _get_available_image_counts ) from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, get_image_array_to_upload, __create_image +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() +@trackable def upload_image_to_project( project, img, diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index 4b9fdfbcb..c8d8db8cb 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -9,7 +9,10 @@ def get_team_metadata(*args, **kwargs): def invite_contributor_to_team(*args, **kwargs): - admin = args[1:2] + admin = kwargs.get("admin", None) + if not admin: + admin = args[1:2] + if admin: admin = "CUSTOM" else: @@ -42,9 +45,9 @@ def search_team_contributors(*args, **kwargs): "event_name": "search_team_contributors", "properties": { - "Email": bool(args[0:1]), - "Name": bool(args[1:2]), - "Surname": bool(args[2:3]) + "Email": bool(args[0:1] or kwargs.get("email", None)), + "Name": bool(args[1:2] or kwargs.get("first_name", None)), + "Surname": bool(args[2:3] or kwargs.get("last_name", None)) } } @@ -55,9 +58,11 @@ def search_team_contributors(*args, **kwargs): def search_projects(*args, **kwargs): return { "event_name": "search_projects", - "properties": { - "Metadata": bool(args[2:3]) - } + "properties": + { + "Metadata": + bool(args[2:3] or kwargs.get("return_metadata", None)) + } } @@ -65,10 +70,14 @@ def search_projects(*args, **kwargs): def create_project(*args, **kwargs): + project_type = kwargs.get("project_type", None) + if not project_type: + project_type = args[2:3][0] + return { "event_name": "create_project", "properties": { - "Project Type": args[2:3][0] + "Project Type": project_type } } @@ -88,10 +97,17 @@ def clone_project(*args, **kwargs): "event_name": "clone_project", "properties": { - "Copy Classes": bool(args[3:4]), - "Copy Settings": bool(args[4:5]), - "Copy Workflow": bool(args[5:6]), - "Copy Contributors": bool(args[6:7]) + "Copy Classes": + bool( + args[3:4] or + kwargs.get("copy_annotation_classes", None) + ), + "Copy Settings": + bool(args[4:5] or kwargs.get("copy_settings", None)), + "Copy Workflow": + bool(args[5:6] or kwargs.get("copy_workflow", None)), + "Copy Contributors": + bool(args[6:7] or kwargs.get("copy_contributors", None)) } } @@ -104,8 +120,10 @@ def search_images(*args, **kwargs): "event_name": "search_images", "properties": { - "Annotation Status": bool(args[2:3]), - "Metadata": bool(args[3:4]), + "Annotation Status": + bool(args[2:3] or kwargs.get("annotation_status", None)), + "Metadata": + bool(args[3:4] or kwargs.get("return_metadata", None)), } } @@ -118,11 +136,30 @@ def upload_images_to_project(*args, **kwargs): "event_name": "upload_images_to_project", "properties": { - "Image Count": len(args[1]), - "Annotation Status": bool(args[2:3]), - "From S3": bool(args[3:4]) + "Image Count": + len(args[1]), + "Annotation Status": + bool(args[2:3] or kwargs.get("annotation_status", None)), + "From S3": + bool(args[3:4] or kwargs.get("from_s3", None)) } } parsers['upload_images_to_project'] = upload_images_to_project + + +def upload_image_to_project(*args, **kwargs): + return { + "event_name": "upload_image_to_project", + "properties": + { + "Image Name": + bool(args[2:3] or kwargs.get("image_name", None)), + "Annotation Status": + bool(args[3:4] or kwargs.get("annotation_status", None)) + } + } + + +parsers['upload_image_to_project'] = upload_image_to_project From 23516e986a6423994862b9eff051a3219544df35 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 28 Apr 2021 17:36:01 +0400 Subject: [PATCH 05/22] Fix mixpanel events --- superannotate/db/projects.py | 3 + superannotate/mixp/decorators.py | 1 + superannotate/mixp/parsers.py | 116 ++++++++++++++++++++++++++++--- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 3164c3ba4..700e92f79 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -595,6 +595,7 @@ def upload_images_from_folder_to_project( from_s3_bucket, image_quality_in_editor ) + @trackable def upload_images_to_project( project, @@ -733,6 +734,7 @@ def attach_image_urls_to_project( return (list_of_uploaded, list_of_not_uploaded, duplicate_images) +@trackable def upload_images_from_public_urls_to_project( project, img_urls, @@ -842,6 +844,7 @@ def upload_images_from_public_urls_to_project( ) +@trackable def upload_images_from_google_cloud_to_project( project, google_project, diff --git a/superannotate/mixp/decorators.py b/superannotate/mixp/decorators.py index 475f3c1d6..2c0e9730e 100644 --- a/superannotate/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -24,6 +24,7 @@ def wrapper(*args, **kwargs): _api.user_id, project_name=properties.get('project_name', None) ) + properties.pop("project_name", None) properties = {**default, **properties} mp.track(user_id, event_name, properties) # except Exception as e: diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index c8d8db8cb..bd8d74ae0 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -1,6 +1,18 @@ parsers = {} +def get_project_name(project): + project_name = "" + if isinstance(project, dict): + project_name = project['name'] + if isinstance(project, str): + if '/' in project: + project_name = project.split('/')[0] + else: + project_name = project + return project_name + + def get_team_metadata(*args, **kwargs): return {"event_name": "get_team_metadata", "properties": {}} @@ -56,12 +68,17 @@ def search_team_contributors(*args, **kwargs): def search_projects(*args, **kwargs): + project = kwargs.get("name", None) + if not project: + project = args[0:1][0] return { "event_name": "search_projects", "properties": { "Metadata": - bool(args[2:3] or kwargs.get("return_metadata", None)) + bool(args[2:3] or kwargs.get("return_metadata", None)), + "project_name": + get_project_name(project) } } @@ -70,15 +87,20 @@ def search_projects(*args, **kwargs): def create_project(*args, **kwargs): + project = kwargs.get("project_name", None) + if not project: + project = args[0:1][0] project_type = kwargs.get("project_type", None) if not project_type: project_type = args[2:3][0] return { "event_name": "create_project", - "properties": { - "Project Type": project_type - } + "properties": + { + "Project Type": project_type, + "project_name": get_project_name(project) + } } @@ -86,13 +108,24 @@ def create_project(*args, **kwargs): def create_project_from_metadata(*args, **kwargs): - return {"event_name": "create_project_from_metadata", "properties": {}} + project = kwargs.get("project_metadata", None) + if not project: + project = args[0:1][0] + return { + "event_name": "create_project_from_metadata", + "properties": { + "project_name": get_project_name(project) + } + } parsers['create_project_from_metadata'] = create_project_from_metadata def clone_project(*args, **kwargs): + project = kwargs.get("project_name", None) + if not project: + project = args[0:1][0] return { "event_name": "clone_project", "properties": @@ -107,7 +140,9 @@ def clone_project(*args, **kwargs): "Copy Workflow": bool(args[5:6] or kwargs.get("copy_workflow", None)), "Copy Contributors": - bool(args[6:7] or kwargs.get("copy_contributors", None)) + bool(args[6:7] or kwargs.get("copy_contributors", None)), + "project_name": + get_project_name(project) } } @@ -116,6 +151,9 @@ def clone_project(*args, **kwargs): def search_images(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] return { "event_name": "search_images", "properties": @@ -124,6 +162,8 @@ def search_images(*args, **kwargs): bool(args[2:3] or kwargs.get("annotation_status", None)), "Metadata": bool(args[3:4] or kwargs.get("return_metadata", None)), + "project_name": + get_project_name(project) } } @@ -132,16 +172,25 @@ def search_images(*args, **kwargs): def upload_images_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + img_paths = kwargs.get("img_paths", []) + if not img_paths: + img_paths += args[1:2][0] return { "event_name": "upload_images_to_project", "properties": { "Image Count": - len(args[1]), + len(img_paths), "Annotation Status": bool(args[2:3] or kwargs.get("annotation_status", None)), "From S3": - bool(args[3:4] or kwargs.get("from_s3", None)) + bool(args[3:4] or kwargs.get("from_s3", None)), + "project_name": + get_project_name(project) } } @@ -150,6 +199,9 @@ def upload_images_to_project(*args, **kwargs): def upload_image_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] return { "event_name": "upload_image_to_project", "properties": @@ -157,9 +209,55 @@ def upload_image_to_project(*args, **kwargs): "Image Name": bool(args[2:3] or kwargs.get("image_name", None)), "Annotation Status": - bool(args[3:4] or kwargs.get("annotation_status", None)) + bool(args[3:4] or kwargs.get("annotation_status", None)), + "project_name": + get_project_name(project) } } parsers['upload_image_to_project'] = upload_image_to_project + + +def upload_images_from_public_urls_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + img_urls = kwargs.get("img_urls", []) + if not img_urls: + img_urls += args[1:2][0] + return { + "event_name": "upload_images_from_public_urls_to_project", + "properties": + { + "Image Count": + len(img_urls), + "Image Name": + bool(args[2:3] or kwargs.get("img_names", None)), + "Annotation Status": + bool(args[3:4] or kwargs.get("annotation_status", None)), + "project_name": + get_project_name(project) + } + } + + +parsers['upload_images_from_public_urls_to_project' + ] = upload_images_from_public_urls_to_project + + +def upload_images_from_google_cloud_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + return { + "event_name": "upload_images_from_google_cloud_to_project", + "properties": { + "project_name": get_project_name(project) + } + } + + +parsers['upload_images_from_google_cloud_to_project' + ] = upload_images_from_google_cloud_to_project From 619fb74ee7808dce84c22b3348b282f654fc97fe Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 28 Apr 2021 19:10:48 +0400 Subject: [PATCH 06/22] Add mixpanel events --- superannotate/db/images.py | 1 + superannotate/db/projects.py | 3 ++ superannotate/mixp/parsers.py | 85 +++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) diff --git a/superannotate/db/images.py b/superannotate/db/images.py index d84a66de5..0fa7b8324 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -1201,6 +1201,7 @@ def create_fuse_image( return (fuse_path, ) +@trackable def set_images_annotation_statuses(project, image_names, annotation_status): """Sets annotation statuses of images diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 700e92f79..d4746f469 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -326,6 +326,7 @@ def _extract_frames_from_video( return extracted_frames_paths +@trackable def upload_video_to_project( project, video_path, @@ -683,6 +684,7 @@ def _tqdm_download( break +@trackable def attach_image_urls_to_project( project, attachments, annotation_status="NotStarted" ): @@ -931,6 +933,7 @@ def upload_images_from_google_cloud_to_project( ) +@trackable def upload_images_from_azure_blob_to_project( project, container_name, diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index bd8d74ae0..561c9cf0b 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -261,3 +261,88 @@ def upload_images_from_google_cloud_to_project(*args, **kwargs): parsers['upload_images_from_google_cloud_to_project' ] = upload_images_from_google_cloud_to_project + + +def upload_images_from_azure_blob_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + return { + "event_name": "upload_images_from_azure_blob_to_project", + "properties": { + "project_name": get_project_name(project) + } + } + + +parsers['upload_images_from_azure_blob_to_project' + ] = upload_images_from_azure_blob_to_project + + +def upload_video_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "upload_video_to_project", + "properties": + { + "project_name": get_project_name(project), + "FPS": bool(args[2:3] or kwargs.get("target_fps", None)), + "Start": bool(args[3:4] or kwargs.get("start_time", None)), + "End": bool(args[4:5] or kwargs.get("end_time", None)) + } + } + + +parsers['upload_video_to_project'] = upload_video_to_project + + +def attach_image_urls_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "attach_image_urls_to_project", + "properties": + { + "project_name": + get_project_name(project), + "Image Names": + bool(args[1:2] or kwargs.get("attachments", None)), + "Annotation Status": + bool(args[2:3] or kwargs.get("annotation_status", None)) + } + } + + +parsers['attach_image_urls_to_project'] = attach_image_urls_to_project + + +def set_images_annotation_statuses(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + annotation_status = kwargs.get("annotation_status", None) + if not annotation_status: + annotation_status = args[2:3][0] + + image_names = kwargs.get("image_names", []) + if not image_names: + image_names = args[1:2][0] + + return { + "event_name": "set_images_annotation_statuses", + "properties": + { + "project_name": get_project_name(project), + "Image Count": len(image_names), + "Annotation Status": annotation_status + } + } + + +parsers['set_images_annotation_statuses'] = set_images_annotation_statuses From d61fd1c116b7e54495d25ba63d7b75264a13069f Mon Sep 17 00:00:00 2001 From: shab Date: Thu, 29 Apr 2021 10:57:00 +0400 Subject: [PATCH 07/22] Add events --- superannotate/db/images.py | 4 +++ superannotate/mixp/parsers.py | 64 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/superannotate/db/images.py b/superannotate/db/images.py index 0fa7b8324..5c12efec3 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -762,6 +762,7 @@ def get_image_bytes(project, image_name, variant='original'): return img +@trackable def get_image_preannotations(project, image_name): """Get pre-annotations of the image. Only works for "vector" projects. @@ -780,6 +781,7 @@ def get_image_preannotations(project, image_name): return _get_image_pre_or_annotations(project, image_name, "pre") +@trackable def get_image_annotations(project, image_name): """Get annotations of the image. @@ -879,6 +881,7 @@ def _get_image_pre_or_annotations(project, image_name, pre): return result +@trackable def download_image_annotations(project, image_name, local_dir_path): """Downloads annotations of the image (JSON and mask if pixel type project) to local_dir_path. @@ -935,6 +938,7 @@ def _download_image_pre_or_annotations( return tuple(return_filepaths) +@trackable def download_image_preannotations(project, image_name, local_dir_path): """Downloads pre-annotations of the image to local_dir_path. Only works for "vector" projects. diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index 561c9cf0b..a7522ed2b 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -346,3 +346,67 @@ def set_images_annotation_statuses(*args, **kwargs): parsers['set_images_annotation_statuses'] = set_images_annotation_statuses + + +def get_image_annotations(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_image_annotations", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_image_annotations'] = get_image_annotations + + +def get_image_preannotations(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_image_preannotations", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_image_preannotations'] = get_image_preannotations + + +def download_image_annotations(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "download_image_annotations", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['download_image_annotations'] = download_image_annotations + + +def download_image_preannotations(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "download_image_preannotations", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['download_image_preannotations'] = download_image_preannotations From 567f679dcd8eeae94caead40e67acb0897832fda Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 30 Apr 2021 14:08:05 +0400 Subject: [PATCH 08/22] Add move --- .vscode/settings.json | 19 ++++----- superannotate/db/project_images.py | 63 ++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b8cddb9f4..6759c6307 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,12 +1,13 @@ { + "editor.formatOnSave": true, "python.formatting.provider": "yapf", - "python.linting.pylintPath": "venv_sa_conv/bin/pylint", - "python.linting.pylintEnabled": true, - "python.testing.pytestArgs": [ - "tests" - ], - "python.testing.unittestEnabled": false, - "python.testing.nosetestsEnabled": false, - "python.testing.pytestEnabled": true, - "python.pythonPath": "venv_sa_conv/bin/python" + // "python.linting.pylintPath": "venv_sa_conv/bin/pylint", + // "python.linting.pylintEnabled": true, + // "python.testing.pytestArgs": [ + // "tests" + // ], + // "python.testing.unittestEnabled": false, + // "python.testing.nosetestsEnabled": false, + // "python.testing.pytestEnabled": true, + // "python.pythonPath": "venv_sa_conv/bin/python" } \ No newline at end of file diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 276fe602f..feafc870f 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -315,28 +315,59 @@ def move_images( :param copy_pin: enables image pin status copy :type copy_pin: bool """ - source_project, source_project_folder = get_project_and_folder_metadata( + source_project_inp = source_project + destination_project_inp = destination_project + + source_project, source_folder = get_project_and_folder_metadata( source_project ) - destination_project, destination_project_folder = get_project_and_folder_metadata( + destination_project, destination_folder = get_project_and_folder_metadata( destination_project ) - if image_names is None: - image_names = search_images((source_project, source_project_folder)) - copy_images( - (source_project, source_project_folder), image_names, - (destination_project, destination_project_folder), include_annotations, - copy_annotation_status, copy_pin - ) - delete_images((source_project, source_project_folder), image_names) - logger.info( - "Moved images %s from project %s to project %s", image_names, - source_project["name"] + "" if source_project_folder is None else "/" + - source_project_folder["name"], destination_project["name"] + - "" if destination_project_folder is None else "/" + - destination_project_folder["name"] + root_folder_id = get_project_root_folder_id(source_project) + + destination_folder_id = root_folder_id + source_folder_id = root_folder_id + + if destination_folder: + destination_folder_id = destination_folder['id'] + if source_folder: + source_folder_id = source_folder['id'] + + response = _api.send_request( + req_type='POST', + path='/image/move', + params={ + "team_id": source_project["team_id"], + "project_id": source_project["id"] + }, + json_req={ + "image_names": image_names, + "destination_folder_id": destination_folder_id, + "source_folder_id": source_folder_id + } ) + res = response.json() + if res.get('error', None): + logger.error(res['error']) + return [] + + moved = res['moved'] + skipped = res['skipped'] + [ + i for i in image_names if i not in [*moved, *res['skipped']] + ] + + if len(moved) > 1: + message = f"Moved {len(moved)}/{len(image_names)} images from {source_project_inp} to {destination_project_inp}." + logger.info(message) + + elif len(moved) == 1: + message = f"Moved an image from {source_project_inp} to {destination_project_inp}." + logger.info(message) + + return skipped + def copy_image( source_project, From f5820be5bb2144d1b295203b237be8acf015c437 Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 30 Apr 2021 16:27:27 +0400 Subject: [PATCH 09/22] Add mixpanel events --- superannotate/analytics/common.py | 4 +- superannotate/db/annotation_classes.py | 10 +- superannotate/db/images.py | 7 +- superannotate/db/project_api.py | 9 +- superannotate/db/projects.py | 20 +- superannotate/input_converters/conversion.py | 6 +- superannotate/mixp/parsers.py | 503 +++++++++++++++++++ superannotate/ml/ml_funcs.py | 10 +- 8 files changed, 538 insertions(+), 31 deletions(-) diff --git a/superannotate/analytics/common.py b/superannotate/analytics/common.py index 501ee29d0..5256310af 100644 --- a/superannotate/analytics/common.py +++ b/superannotate/analytics/common.py @@ -6,10 +6,10 @@ import pandas as pd from ..exceptions import SABaseException - +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") - +@trackable def df_to_annotations(df, output_dir): """Converts and saves pandas DataFrame annotation info (see aggregate_annotations_as_df) in output_dir. diff --git a/superannotate/db/annotation_classes.py b/superannotate/db/annotation_classes.py index 467d5daaf..efbe06ecc 100644 --- a/superannotate/db/annotation_classes.py +++ b/superannotate/db/annotation_classes.py @@ -11,7 +11,7 @@ SANonExistingAnnotationClassNameException ) from .project_api import get_project_metadata_bare - +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -78,7 +78,7 @@ def create_annotation_class(project, name, color, attribute_groups=None): new_class = res[0] return new_class - +@trackable def delete_annotation_class(project, annotation_class): """Deletes annotation class from project @@ -199,7 +199,7 @@ def del_unn(d): assert len(res) == len(new_classes) return res - +@trackable def search_annotation_classes(project, name_prefix=None): """Searches annotation classes by name_prefix (case-insensitive) @@ -238,7 +238,7 @@ def search_annotation_classes(project, name_prefix=None): return result_list - +@trackable def get_annotation_class_metadata(project, annotation_class_name): """Returns annotation class metadata @@ -271,7 +271,7 @@ def get_annotation_class_metadata(project, annotation_class_name): " doesn't exist." ) - +@trackable def download_annotation_classes_json(project, folder): """Downloads project classes.json to folder diff --git a/superannotate/db/images.py b/superannotate/db/images.py index 5c12efec3..1806b2850 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -215,6 +215,7 @@ def process_result(x): return result_list +@trackable def get_image_metadata(project, image_names, return_dict_on_single_output=True): """Returns image metadata @@ -325,7 +326,7 @@ def set_image_annotation_status(project, image_name, annotation_status): return response - +@trackable def add_annotation_comment_to_image( project, image_name, @@ -683,7 +684,7 @@ def download_image( return (str(filepath_save), annotations_filepaths, fuse_path) - +@trackable def delete_image(project, image_name): """Deletes image @@ -705,7 +706,7 @@ def delete_image(project, image_name): ) logger.info("Successfully deleted image %s.", image_name) - +@trackable def get_image_bytes(project, image_name, variant='original'): """Returns an io.BytesIO() object of the image. Suitable for creating PIL.Image out of it. diff --git a/superannotate/db/project_api.py b/superannotate/db/project_api.py index dc2fa6afb..36e04b0e0 100644 --- a/superannotate/db/project_api.py +++ b/superannotate/db/project_api.py @@ -7,6 +7,7 @@ SANonExistingProjectNameException ) from .search_projects import search_projects +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -64,7 +65,7 @@ def get_project_metadata_with_users(project_metadata): ) return res - +@trackable def get_folder_metadata(project, folder_name): """Returns folder metadata @@ -91,7 +92,7 @@ def get_folder_metadata(project, folder_name): res = response.json() return res - +@trackable def get_project_and_folder_metadata(project): """Returns project and folder metadata tuple. If folder part is empty, than returned folder part is set to None. @@ -180,7 +181,7 @@ def search_folders(project, folder_name=None, return_metadata=False): return result_list - +@trackable def create_folder(project, folder_name): """Create a new folder in the project. @@ -259,7 +260,7 @@ def delete_folders(project, folder_names): "Folders %s deleted in project %s", folder_names, project["name"] ) - +@trackable def rename_folder(project, new_folder_name): """Renames folder in project. diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index d4746f469..14a77924b 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -132,7 +132,7 @@ def create_project_from_metadata(project_metadata): set_project_workflow(new_project_metadata, project_metadata["workflow"]) return new_project_metadata - +@trackable def delete_project(project): """Deletes the project @@ -152,7 +152,7 @@ def delete_project(project): ) logger.info("Successfully deleted project %s.", project["name"]) - +@trackable def rename_project(project, new_name): """Renames the project @@ -189,7 +189,7 @@ def rename_project(project, new_name): "Successfully renamed project %s to %s.", project["name"], new_name ) - +@trackable def get_project_image_count(project, with_all_subfolders=False): """Returns number of images in the project. @@ -1416,7 +1416,7 @@ def share_project(project, user, user_role): user["email"], common.user_role_int_to_str(user_role) ) - +@trackable def unshare_project(project, user): """Unshare (remove) user from project. @@ -1534,7 +1534,7 @@ def _get_upload_from_s3_bucket_to_project_status(project, project_folder): ) return response.json() - +@trackable def get_project_workflow(project): """Gets project's workflow. @@ -1576,7 +1576,7 @@ def get_project_workflow(project): raise SABaseException(0, "Couldn't find class_id in workflow") return res - +@trackable def set_project_workflow(project, new_workflow): """Sets project's workflow. @@ -1676,7 +1676,7 @@ def set_project_workflow(project, new_workflow): "Couldn't set project workflow " + response.text ) - +@trackable def get_project_settings(project): """Gets project's settings. @@ -1713,7 +1713,7 @@ def get_project_settings(project): raise SABaseException(0, "NA ImageQuality value") return res - +@trackable def set_project_settings(project, new_settings): """Sets project's settings. @@ -1796,7 +1796,7 @@ def set_project_default_image_quality_in_editor( }] ) - +@trackable def get_project_default_image_quality_in_editor(project): """Gets project's default image quality in editor setting. @@ -1814,7 +1814,7 @@ def get_project_default_image_quality_in_editor(project): "Image quality in editor should be 'compressed', 'original' or None for project settings value" ) - +@trackable def get_project_metadata( project, include_annotation_classes=False, diff --git a/superannotate/input_converters/conversion.py b/superannotate/input_converters/conversion.py index 98621c0dd..0c7846524 100644 --- a/superannotate/input_converters/conversion.py +++ b/superannotate/input_converters/conversion.py @@ -11,6 +11,8 @@ degrade_json, sa_convert_project_type, split_coco, upgrade_json ) +from ..mixp.decorators import trackable + ALLOWED_TASK_TYPES = [ 'panoptic_segmentation', 'instance_segmentation', 'keypoint_detection', 'object_detection', 'vector_annotation' @@ -389,7 +391,7 @@ def import_annotation( import_to_sa(args) - +@trackable def convert_project_type(input_dir, output_dir): """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. @@ -460,7 +462,7 @@ def coco_split_dataset( coco_json_path, image_dir, output_dir, dataset_list_name, ratio_list ) - +@trackable def convert_json_version(input_dir, output_dir, version=2): """ Converts SuperAnnotate JSON versions. Newest JSON version is 2. diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index a7522ed2b..d9922e56a 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -410,3 +410,506 @@ def download_image_preannotations(*args, **kwargs): parsers['download_image_preannotations'] = download_image_preannotations + + + + +def get_image_metadata(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_image_metadata", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_image_metadata'] = get_image_metadata + + + + +def get_image_bytes(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_image_bytes", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_image_bytes'] = get_image_bytes + + + +def delete_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "delete_image", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['delete_image'] = delete_image + + + +def add_annotation_comment_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_comment_to_image", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['add_annotation_comment_to_image'] = add_annotation_comment_to_image + + + +def delete_annotation_class(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "delete_annotation_class", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['delete_annotation_class'] = delete_annotation_class + + + + +def get_annotation_class_metadata(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_annotation_class_metadata", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_annotation_class_metadata'] = get_annotation_class_metadata + + + + +def download_annotation_classes_json(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "download_annotation_classes_json", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['download_annotation_classes_json'] = download_annotation_classes_json + + + +def search_annotation_classes(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "search_annotation_classes", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['search_annotation_classes'] = search_annotation_classes + + +def unshare_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "unshare_project", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['unshare_project'] = unshare_project + + + +def get_project_image_count(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_project_image_count", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_project_image_count'] = get_project_image_count + + + +def get_project_settings(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_project_settings", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_project_settings'] = get_project_settings + + + + +def set_project_settings(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "set_project_settings", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['set_project_settings'] = set_project_settings + + + +def get_project_default_image_quality_in_editor(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_project_default_image_quality_in_editor", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_project_default_image_quality_in_editor'] = get_project_default_image_quality_in_editor + + +def get_project_metadata(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_project_metadata", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_project_metadata'] = get_project_metadata + + +def delete_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "delete_project", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['delete_project'] = delete_project + + + + +def rename_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "rename_project", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['rename_project'] = rename_project + + + + +def get_project_workflow(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_project_workflow", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_project_workflow'] = get_project_workflow + + + +def set_project_workflow(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "set_project_workflow", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['set_project_workflow'] = set_project_workflow + + + + +def create_folder(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "create_folder", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['create_folder'] = create_folder + + + + +def get_folder_metadata(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_folder_metadata", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_folder_metadata'] = get_folder_metadata + + + + +def get_project_and_folder_metadata(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_project_and_folder_metadata", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['get_project_and_folder_metadata'] = get_project_and_folder_metadata + + + + + +def rename_folder(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "rename_folder", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['rename_folder'] = rename_folder + + + +def stop_model_training(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "stop_model_training", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['stop_model_training'] = stop_model_training + + + +def download_model(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "download_model", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['download_model'] = download_model + + + +def plot_model_metrics(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "plot_model_metrics", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['plot_model_metrics'] = plot_model_metrics + + +def delete_model(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "delete_model", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['delete_model'] = delete_model + + + +def convert_project_type(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "convert_project_type", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['convert_project_type'] = convert_project_type + + + +def convert_json_version(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "convert_json_version", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['convert_json_version'] = convert_json_version + + + +def df_to_annotations(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "df_to_annotations", + "properties": { + "project_name": get_project_name(project), + } + } + + +parsers['df_to_annotations'] = df_to_annotations + + diff --git a/superannotate/ml/ml_funcs.py b/superannotate/ml/ml_funcs.py index 544d6900e..eb4d5d67b 100644 --- a/superannotate/ml/ml_funcs.py +++ b/superannotate/ml/ml_funcs.py @@ -22,7 +22,7 @@ from .defaults import DEFAULT_HYPERPARAMETERS, NON_PLOTABLE_KEYS from .utils import log_process, make_plotly_specs, reformat_metrics_json from ..db.utils import _get_boto_session_by_credentials - +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -342,7 +342,7 @@ def run_training( time.sleep(5) return new_model - +@trackable @model_metadata def stop_model_training(model): '''This function will stop training model provided by either name or metadata, and return the ID @@ -364,7 +364,7 @@ def stop_model_training(model): logger.info("Failed to stop model training please try again") return model - +@trackable def plot_model_metrics(metric_json_list): """plots the metrics generated by neural network using plotly :param metric_json_list: list of .json files @@ -433,7 +433,7 @@ def get_plottable_cols(df): ) figure.show() - +@trackable @model_metadata def download_model(model, output_dir): """Downloads the neural network and related files @@ -495,7 +495,7 @@ def download_model(model, output_dir): logger.info("Downloaded model related files") return model - +@trackable @model_metadata def delete_model(model): '''This function deletes the provided model From 60da5c01bf361cca60f12ee39d277d83915b61bd Mon Sep 17 00:00:00 2001 From: shab Date: Mon, 3 May 2021 12:34:08 +0400 Subject: [PATCH 10/22] Add events --- superannotate/analytics/class_analytics.py | 5 +- superannotate/db/annotation_classes.py | 4 +- superannotate/db/images.py | 22 +- superannotate/db/project_images.py | 6 +- superannotate/db/projects.py | 4 +- superannotate/mixp/parsers.py | 378 +++++++++++++++++++++ 6 files changed, 399 insertions(+), 20 deletions(-) diff --git a/superannotate/analytics/class_analytics.py b/superannotate/analytics/class_analytics.py index 9c9c8f96b..7da230bb0 100644 --- a/superannotate/analytics/class_analytics.py +++ b/superannotate/analytics/class_analytics.py @@ -8,10 +8,10 @@ from plotly.subplots import make_subplots from .common import aggregate_annotations_as_df - +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") - +@trackable def class_distribution(export_root, project_names, visualize=False): """Aggregate distribution of classes across multiple projects. @@ -63,6 +63,7 @@ def class_distribution(export_root, project_names, visualize=False): return df +@trackable def attribute_distribution(export_root, project_names, visualize=False): """Aggregate distribution of attributes across multiple projects. diff --git a/superannotate/db/annotation_classes.py b/superannotate/db/annotation_classes.py index efbe06ecc..1aeaa65c9 100644 --- a/superannotate/db/annotation_classes.py +++ b/superannotate/db/annotation_classes.py @@ -16,7 +16,7 @@ _api = API.get_instance() - +@trackable def create_annotation_class(project, name, color, attribute_groups=None): """Create annotation class in project @@ -112,7 +112,7 @@ def delete_annotation_class(project, annotation_class): "Couldn't delete annotation class " + response.text ) - +@trackable def create_annotation_classes_from_classes_json( project, classes_json, from_s3_bucket=None ): diff --git a/superannotate/db/images.py b/superannotate/db/images.py index 1806b2850..02c16b04d 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -290,7 +290,7 @@ def get_image_metadata(project, image_names, return_dict_on_single_output=True): return metadata_without_deleted[0] return metadata_without_deleted - +@trackable def set_image_annotation_status(project, image_name, annotation_status): """Sets the image annotation status @@ -360,7 +360,7 @@ def add_annotation_comment_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_bbox_to_image( project, image_name, @@ -396,7 +396,7 @@ def add_annotation_bbox_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_polygon_to_image( project, image_name, @@ -430,7 +430,7 @@ def add_annotation_polygon_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_polyline_to_image( project, image_name, @@ -463,7 +463,7 @@ def add_annotation_polyline_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_point_to_image( project, image_name, @@ -496,7 +496,7 @@ def add_annotation_point_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_ellipse_to_image( project, image_name, @@ -529,7 +529,7 @@ def add_annotation_ellipse_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_template_to_image( project, image_name, @@ -569,7 +569,7 @@ def add_annotation_template_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def add_annotation_cuboid_to_image( project, image_name, @@ -605,7 +605,7 @@ def add_annotation_cuboid_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) - +@trackable def download_image( project, image_name, @@ -958,7 +958,7 @@ def download_image_preannotations(project, image_name, local_dir_path): project, image_name, local_dir_path, "pre" ) - +@trackable def upload_image_annotations( project, image_name, annotation_json, mask=None, verbose=True ): @@ -1033,7 +1033,7 @@ def upload_image_annotations( bucket = s3_resource.Bucket(res_mask["bucket"]) bucket.put_object(Key=res_mask['filePath'], Body=mask) - +@trackable def create_fuse_image( image, classes_json, project_type, in_memory=False, output_overlay=False ): diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index a081a7112..3892d7fd2 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -339,7 +339,7 @@ def move_images( destination_project_folder["name"] ) - +@trackable def copy_image( source_project, image_name, @@ -445,7 +445,7 @@ def _copy_annotations_and_metadata( img_metadata["is_pinned"] ) - +@trackable def move_image( source_project, image_name, @@ -488,7 +488,7 @@ def move_image( delete_image((source_project, source_project_folder), image_name) logger.info("Deleted image %s/%s.", source_project["name"], image_name) - +@trackable def pin_image(project, image_name, pin=True): """Pins (or unpins) image diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 14a77924b..89e802e5b 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -1383,7 +1383,7 @@ def upload_preannotations_from_folder_to_project( project, folder_path, "pre", from_s3_bucket, recursive_subfolders ) - +@trackable def share_project(project, user, user_role): """Share project with user. @@ -1777,7 +1777,7 @@ def set_project_settings(project, new_settings): ) return response.json() - +@trackable def set_project_default_image_quality_in_editor( project, image_quality_in_editor ): diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index d9922e56a..be3db8496 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -913,3 +913,381 @@ def df_to_annotations(*args, **kwargs): parsers['df_to_annotations'] = df_to_annotations + +def upload_image_annotations(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "upload_image_annotations", + "properties": { + "project_name": get_project_name(project), + "Pixel" : bool(args[3:4] or ("mask" in kwargs) ), + } + } + +parsers['upload_image_annotations'] = upload_image_annotations + + + + + + + +def download_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "download_image", + "properties": { + "project_name": get_project_name(project), + "Download Annotations": bool(args[3:4] or ("include_annotations" in kwargs) ), + "Download Fuse": bool(args[4:5] or ("include_fuse" in kwargs) ), + "Download Overlay": bool(args[5:6] or ("include_overlay" in kwargs) ), + } + } + +parsers['download_image'] = download_image + + + + + + +def copy_image(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "copy_image", + "properties": { + "project_name": get_project_name(project), + "Copy Annotations": bool(args[3:4] or ("include_annotations" in kwargs) ), + "Copy Annotation Status": bool(args[4:5] or ("copy_annotation_status" in kwargs) ), + "Copy Pin": bool(args[5:6] or ("copy_pin" in kwargs) ), + } + } + +parsers['copy_image'] = copy_image + + + + +def move_image(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "move_image", + "properties": { + "project_name": get_project_name(project), + "Move Annotations": bool(args[3:4] or ("include_annotations" in kwargs) ), + "Move Annotation Status": bool(args[4:5] or ("copy_annotation_status" in kwargs) ), + "Move Pin": bool(args[5:6] or ("copy_pin" in kwargs) ), + } + } + +parsers['move_image'] = move_image + + +def pin_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "pin_image", + "properties": { + "project_name": get_project_name(project), + "Pin": bool(args[2:3] or ("pin" in kwargs) ), + } + } + +parsers['pin_image'] = pin_image + + + + +def create_fuse_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + project_type = kwargs.get("project_type", None) + if not project_type: + project_type = args[2:3][0] + + return { + "event_name": "create_fuse_image", + "properties": { + "project_name": get_project_name(project), + "Project Type": project_type, + "Overlay": bool(args[4:5] or ("output_overlay" in kwargs) ), + } + } + +parsers['create_fuse_image'] = create_fuse_image + + + + + +def set_image_annotation_status(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "set_image_annotation_status", + "properties": { + "project_name": get_project_name(project), + "Annotation Status": bool(args[2:3] or ("annotation_status" in kwargs) ), + } + } + +parsers['set_image_annotation_status'] = set_image_annotation_status + + + + +def add_annotation_bbox_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_bbox_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[5:6] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_bbox_to_image'] = add_annotation_bbox_to_image + + + +def add_annotation_polygon_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_polygon_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[5:6] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_polygon_to_image'] = add_annotation_polygon_to_image + + + +def add_annotation_polyline_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_polyline_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[5:6] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_polyline_to_image'] = add_annotation_polyline_to_image + + + +def add_annotation_point_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_point_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[5:6] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_point_to_image'] = add_annotation_point_to_image + + +def add_annotation_ellipse_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_ellipse_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[5:6] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_ellipse_to_image'] = add_annotation_ellipse_to_image + + + + +def add_annotation_template_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_template_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[5:6] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[6:7] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_template_to_image'] = add_annotation_template_to_image + + + + + +def add_annotation_cuboid_to_image(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "add_annotation_cuboid_to_image", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), + "Error": bool(args[5:6] or ("error" in kwargs) ), + + } + } + +parsers['add_annotation_cuboid_to_image'] = add_annotation_cuboid_to_image + + + + +def create_annotation_class(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "create_annotation_class", + "properties": { + "project_name": get_project_name(project), + "Attributes": bool(args[3:4] or ("attribute_groups" in kwargs) ), + } + } + +parsers['create_annotation_class'] = create_annotation_class + + + +def create_annotation_classes_from_classes_json(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "create_annotation_classes_from_classes_json", + "properties": { + "project_name": get_project_name(project), + "From S3": bool(args[2:3] or ("from_s3_bucket" in kwargs) ), + } + } + +parsers['create_annotation_classes_from_classes_json'] = create_annotation_classes_from_classes_json + + + + +def class_distribution(*args, **kwargs): + return { + "event_name": "class_distribution", + "properties": { + "Plot": bool(args[2:3] or ("visualize" in kwargs) ), + } + } + +parsers['class_distribution'] = class_distribution + + + +def attribute_distribution(*args, **kwargs): + return { + "event_name": "attribute_distribution", + "properties": { + "Plot": bool(args[2:3] or ("visualize" in kwargs) ), + } + } + +parsers['attribute_distribution'] = attribute_distribution + + +def share_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + user_role = kwargs.get("user_role", None) + if not user_role: + user_role = args[2:3][0] + + + return { + "event_name": "share_project", + "properties": { + "project_name": get_project_name(project), + "User Role":user_role + } + } + +parsers['share_project'] = share_project + + + +def set_project_default_image_quality_in_editor(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + image_quality_in_editor = kwargs.get("image_quality_in_editor", None) + if not image_quality_in_editor: + image_quality_in_editor = args[1:2][0] + + + return { + "event_name": "set_project_default_image_quality_in_editor", + "properties": { + "project_name": get_project_name(project), + "Image Quality":image_quality_in_editor + } + } + +parsers['set_project_default_image_quality_in_editor'] = set_project_default_image_quality_in_editor + From a74030d6831e9baf796617b6901ce5022a366d35 Mon Sep 17 00:00:00 2001 From: shab Date: Mon, 3 May 2021 16:22:25 +0400 Subject: [PATCH 11/22] Add mixpanel events --- superannotate/analytics/common.py | 2 +- superannotate/dataframe_filtering.py | 7 +- superannotate/db/exports.py | 3 +- superannotate/db/project_api.py | 2 +- superannotate/mixp/parsers.py | 99 ++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/superannotate/analytics/common.py b/superannotate/analytics/common.py index 5256310af..509ae50a1 100644 --- a/superannotate/analytics/common.py +++ b/superannotate/analytics/common.py @@ -147,7 +147,7 @@ def df_to_annotations(df, output_dir): indent=4 ) - +@trackable def aggregate_annotations_as_df( project_root, include_classes_wo_annotations=False, diff --git a/superannotate/dataframe_filtering.py b/superannotate/dataframe_filtering.py index a5e50ecb2..ff5b21e69 100644 --- a/superannotate/dataframe_filtering.py +++ b/superannotate/dataframe_filtering.py @@ -1,6 +1,7 @@ import pandas as pd +from .mixp.decorators import trackable - +@trackable def filter_images_by_comments( annotations_df, include_unresolved_comments=True, @@ -39,7 +40,7 @@ def filter_images_by_comments( return list(images) - +@trackable def filter_images_by_tags(annotations_df, include=None, exclude=None): """Filter images on tags @@ -73,7 +74,7 @@ def filter_images_by_tags(annotations_df, include=None, exclude=None): return list(images) - +@trackable def filter_annotation_instances(annotations_df, include=None, exclude=None): """Filter annotation instances from project annotations pandas DataFrame. diff --git a/superannotate/db/exports.py b/superannotate/db/exports.py index 2282e94fd..cdd486682 100644 --- a/superannotate/db/exports.py +++ b/superannotate/db/exports.py @@ -19,6 +19,7 @@ SANonExistingExportNameException ) from .project_api import get_project_metadata_bare +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") @@ -55,7 +56,7 @@ def get_export_metadata(project, export_name): " is not unique. To use SDK please use unique export names." ) - +@trackable def get_exports(project, return_metadata=False): """Get all prepared exports of the project. diff --git a/superannotate/db/project_api.py b/superannotate/db/project_api.py index 36e04b0e0..74e43db18 100644 --- a/superannotate/db/project_api.py +++ b/superannotate/db/project_api.py @@ -131,7 +131,7 @@ def get_project_and_folder_metadata(project): raise SAIncorrectProjectArgument(project) return project, folder - +@trackable def search_folders(project, folder_name=None, return_metadata=False): """Folder name based case-insensitive search for folders in project. diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index be3db8496..e783658f6 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -1291,3 +1291,102 @@ def set_project_default_image_quality_in_editor(*args, **kwargs): parsers['set_project_default_image_quality_in_editor'] = set_project_default_image_quality_in_editor + + +def get_exports(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "get_exports", + "properties": { + "project_name": get_project_name(project), + "Metadata": bool(args[1:2] or ("return_metadata" in kwargs) ), + } + } + +parsers['get_exports'] = get_exports + + + + +def search_folders(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "search_folders", + "properties": { + "project_name": get_project_name(project), + "Metadata": bool(args[2:3] or ("return_metadata" in kwargs) ), + } + } + +parsers['search_folders'] = search_folders + + + +def filter_images_by_tags(*args, **kwargs): + return { + "event_name": "filter_images_by_tags", + "properties": { + "Include": bool(args[1:2] or ("include" in kwargs) ), + "Exclude" : bool(args[2:3] or ("exclude" in kwargs) ) + } + } + +parsers['filter_images_by_tags'] = filter_images_by_tags + + + +def filter_images_by_comments(*args, **kwargs): + return { + "event_name": "filter_images_by_comments", + "properties": { + "Include Unresolved Comments" : bool(args[1:2] or ("include_unresolved_comments" in kwargs) ), + "Include Resolved Comments": bool(args[2:3] or ("include_resolved_comments" in kwargs) ) + } + } + +parsers['filter_images_by_comments'] = filter_images_by_comments + + + +def filter_annotation_instances(*args, **kwargs): + return { + "event_name": "filter_annotation_instances", + "properties": { + "Include" : bool(args[1:2] or ("include" in kwargs) ), + "Exclude": bool(args[2:3] or ("exclude" in kwargs) ) + } + } + +parsers['filter_annotation_instances'] = filter_annotation_instances + + + + +def aggregate_annotations_as_df(*args, **kwargs): + folder_names = kwargs.get("folder_names", None) + if not folder_names: + folder_names = args[5:6] + if folder_names: + folder_names = args[5:6][0] + else: + folder_names = [] + + + return { + "event_name": "aggregate_annotations_as_df", + "properties": { + "Fodler Count" : len(folder_names), + } + } + +parsers['aggregate_annotations_as_df'] = aggregate_annotations_as_df + + + + From 54d9d495b1cd4fbb6f8123d8fb5e0ac25bce57a2 Mon Sep 17 00:00:00 2001 From: shab Date: Mon, 3 May 2021 18:21:53 +0400 Subject: [PATCH 12/22] Add mixpanel events --- superannotate/db/project_api.py | 2 +- superannotate/db/project_images.py | 2 +- superannotate/mixp/parsers.py | 39 ++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/superannotate/db/project_api.py b/superannotate/db/project_api.py index 74e43db18..a0edb0df7 100644 --- a/superannotate/db/project_api.py +++ b/superannotate/db/project_api.py @@ -225,7 +225,7 @@ def create_folder(project, folder_name): logger.info("Folder %s created in project %s", res["name"], project["name"]) return res - +@trackable def delete_folders(project, folder_names): """Delete folder in project. diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 3892d7fd2..1fdc9c8ed 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -246,7 +246,7 @@ def copy_images( ) return skipped_imgs - +@trackable def delete_images(project, image_names): """Delete images in project. diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/parsers.py index e783658f6..66535bcfc 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/parsers.py @@ -1388,5 +1388,44 @@ def aggregate_annotations_as_df(*args, **kwargs): parsers['aggregate_annotations_as_df'] = aggregate_annotations_as_df +def delete_folders(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + folder_names = kwargs.get("folder_names", None) + if not folder_names: + folder_names = args[1:2][0] + + + return { + "event_name": "delete_folders", + "properties": { + "project_name": get_project_name(project), + "Folder Count": len(folder_names), + } + } + +parsers['delete_folders'] = delete_folders + +def delete_images(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + image_names = kwargs.get("image_names", None) + if not image_names: + image_names = args[1:2][0] + + + return { + "event_name": "delete_images", + "properties": { + "project_name": get_project_name(project), + "Image Count": len(image_names), + } + } + +parsers['delete_images'] = delete_images From 49cf20a1a9701bcbb5ef7e9067e7661890a1c8f3 Mon Sep 17 00:00:00 2001 From: shab Date: Mon, 3 May 2021 18:23:50 +0400 Subject: [PATCH 13/22] Fix decorator --- superannotate/mixp/decorators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/superannotate/mixp/decorators.py b/superannotate/mixp/decorators.py index 2c0e9730e..2a761a96f 100644 --- a/superannotate/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -11,7 +11,7 @@ def trackable(func): callers_to_ignore.append(func.__name__) def wrapper(*args, **kwargs): - if 1: + try: caller_function_name = sys._getframe().f_back.f_code.co_name if caller_function_name not in callers_to_ignore: func_name_to_track = func.__name__ @@ -27,8 +27,8 @@ def wrapper(*args, **kwargs): properties.pop("project_name", None) properties = {**default, **properties} mp.track(user_id, event_name, properties) - # except Exception as e: - # print("--- ---- --- MIX PANEL EXCEPTION") + except Exception as e: + print("--- ---- --- MIX PANEL EXCEPTION") return func(*args, **kwargs) return wrapper From 59c9dc06d212491b874f2e56ecb4656b576cb2f0 Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 7 May 2021 12:15:32 +0400 Subject: [PATCH 14/22] Fix Copy-move --- superannotate/db/project_images.py | 88 +++++++-------- superannotate/db/utils.py | 165 ++++++++++++++++++++++++++++- 2 files changed, 203 insertions(+), 50 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index feafc870f..889a8d96f 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -18,7 +18,7 @@ from .projects import ( get_project_default_image_quality_in_editor, _get_available_image_counts ) -from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, get_image_array_to_upload, __create_image +from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, get_image_array_to_upload, __create_image, __copy_images, __move_images, get_project_folder_string logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -211,38 +211,43 @@ def copy_images( :return: list of skipped image names :rtype: list of strs """ - source_project, source_project_folder = get_project_and_folder_metadata( + + source_project_inp = source_project + destination_project_inp = destination_project + + source_project, source_folder = get_project_and_folder_metadata( source_project ) - destination_project, destination_project_folder = get_project_and_folder_metadata( + destination_project, destination_folder = get_project_and_folder_metadata( destination_project ) - if image_names is None: - image_names = search_images((source_project, source_project_folder)) + root_folder_id = get_project_root_folder_id(source_project) - limit = _get_available_image_counts( - destination_project, destination_project_folder - ) - imgs_to_upload = image_names[:limit] - res = _copy_images( - (source_project, source_project_folder), - (destination_project, destination_project_folder), imgs_to_upload, - include_annotations, copy_annotation_status, copy_pin - ) - uploaded_imgs = res['completed'] - skipped_imgs = [i for i in imgs_to_upload if i not in uploaded_imgs] + destination_folder_id = root_folder_id + source_folder_id = root_folder_id - skipped_images_count = len(skipped_imgs) + len(image_names[limit:]) + if destination_folder: + destination_folder_id = destination_folder['id'] + if source_folder: + source_folder_id = source_folder['id'] - logger.info( - "Copied images %s from %s to %s. Number of skipped images %s", - uploaded_imgs, source_project["name"] + - "" if source_project_folder is None else source_project["name"] + "/" + - source_project_folder["name"], destination_project["name"] + "" - if destination_project_folder is None else destination_project["name"] + - "/" + destination_project_folder["name"], skipped_images_count + if image_names == None: + image_names = search_images(source_project_inp) + + done_count, skipped_count, total_skipped_list = __copy_images( + source_project, source_folder_id, destination_folder_id, image_names, + include_annotations, copy_pin ) - return skipped_imgs + + if done_count > 1: + message = f"Copied {done_count}/{len(image_names)} images from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." + logger.info(message) + + elif done_count == 1: + message = f"Copied an image from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." + logger.info(message) + + return total_skipped_list def delete_images(project, image_names): @@ -314,6 +319,8 @@ def move_images( :type copy_annotation_status: bool :param copy_pin: enables image pin status copy :type copy_pin: bool + :return: list of skipped image names + :rtype: list of strs """ source_project_inp = source_project destination_project_inp = destination_project @@ -334,36 +341,19 @@ def move_images( if source_folder: source_folder_id = source_folder['id'] - response = _api.send_request( - req_type='POST', - path='/image/move', - params={ - "team_id": source_project["team_id"], - "project_id": source_project["id"] - }, - json_req={ - "image_names": image_names, - "destination_folder_id": destination_folder_id, - "source_folder_id": source_folder_id - } - ) - - res = response.json() - if res.get('error', None): - logger.error(res['error']) - return [] + if image_names == None: + image_names = search_images(source_project_inp) - moved = res['moved'] - skipped = res['skipped'] + [ - i for i in image_names if i not in [*moved, *res['skipped']] - ] + moved, skipped = __move_images( + source_project, source_folder_id, destination_folder_id, image_names + ) if len(moved) > 1: - message = f"Moved {len(moved)}/{len(image_names)} images from {source_project_inp} to {destination_project_inp}." + message = f"Moved {len(moved)}/{len(image_names)} images from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." logger.info(message) elif len(moved) == 1: - message = f"Moved an image from {source_project_inp} to {destination_project_inp}." + message = f"Moved an image from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." logger.info(message) return skipped diff --git a/superannotate/db/utils.py b/superannotate/db/utils.py index ee731c3c0..773de0419 100644 --- a/superannotate/db/utils.py +++ b/superannotate/db/utils.py @@ -2,6 +2,7 @@ from tqdm import tqdm import threading import io +import time from pathlib import Path from .. import common import logging @@ -10,7 +11,7 @@ import json from ..api import API from ..exceptions import SABaseException, SAImageSizeTooLarge, SANonExistingProjectNameException - +import datetime import boto3 _api = API.get_instance() @@ -26,6 +27,168 @@ def divide_chunks(l, n): yield l[i:i + n] +def get_project_folder_string(project): + if isinstance(project, dict): + return project['name'] + elif isinstance(project, tuple): + project, folder = project + project_name = project['name'] + if folder: + return project_name + '/' + folder['name'] + return project_name + elif isinstance(project, str): + return project + + +def __move_images( + source_project, source_folder_id, destination_folder_id, image_names +): + """Move images in bulk between folders in a project + + :param source_project: source project + :type source_project: dict + :param source_folder_id: source folder id + :type source_folder_id: int + :param image_names: image names. If None, all images from source project will be moved + :type image: list of str + :param destination_folder_id: destination folder id + :type destination_folder_id: int + :return: tuple of moved images list and skipped images list + :rtype: tuple of lists + """ + + image_names_lists = divide_chunks(image_names, 1000) + total_skipped = [] + total_moved = [] + + for image_names in image_names_lists: + response = _api.send_request( + req_type='POST', + path='/image/move', + params={ + "team_id": source_project["team_id"], + "project_id": source_project["id"] + }, + json_req={ + "image_names": image_names, + "destination_folder_id": destination_folder_id, + "source_folder_id": source_folder_id + } + ) + + if not response.ok: + raise SABaseException( + response.status_code, "Couldn't move images " + response.text + ) + res = response.json() + total_moved += res['done'] + + total_skipped = list(set(image_names) - set(total_moved)) + return (total_moved, total_skipped) + + +def __copy_images( + source_project, source_folder_id, destination_folder_id, image_names, + include_annotations, copy_pin +): + """Copy images in bulk between folders in a project + + :param source_project: source project + :type source_project: dict + :param source_folder_id: source folder id + :type source_folder_id: int + :param image_names: image names. If None, all images from source project will be moved + :type image: list of str + :param destination_folder_id: destination folder id + :type destination_folder_id: int + :param include_annotations: enables annotations copy + :type include_annotations: bool + :param copy_pin: enables image pin status copy + :type copy_pin: bool + """ + + team_id = source_project["team_id"] + project_id = source_project["id"] + + image_names_lists = divide_chunks(image_names, 1000) + total_skipped_count = 0 + total_done_count = 0 + total_skipped_list = [] + + for image_names in image_names_lists: + + duplicates = get_duplicate_image_names( + project_id=project_id, + team_id=team_id, + folder_id=destination_folder_id, + image_paths=image_names + ) + + total_skipped_list += duplicates + + image_names = list(set(image_names) - set(duplicates)) + + if not image_names: + total_skipped_count = len(total_skipped_list) + return (total_done_count, total_skipped_count, total_skipped_list) + + response = _api.send_request( + req_type='POST', + path='/images/copy-image-or-folders', + params={ + "team_id": team_id, + "project_id": project_id + }, + json_req={ + "is_folder_copy": False, + "image_names": image_names, + "destination_folder_id": destination_folder_id, + "source_folder_id": source_folder_id, + "include_annotations": include_annotations, + "keep_pin_status": copy_pin + } + ) + + if not response.ok: + raise SABaseException( + response.status_code, "Couldn't copy images " + response.text + ) + res = response.json() + poll_id = res['poll_id'] + done_count = 0 + skipped_count = 0 + now_timestamp = datetime.datetime.now().timestamp() + delta_seconds = len(image_names) * 0.3 + max_timestamp = now_timestamp + delta_seconds + while True: + time.sleep(4) + response = _api.send_request( + req_type='GET', + path='/images/copy-image-progress', + params={ + "team_id": source_project["team_id"], + "project_id": source_project["id"], + "poll_id": poll_id + } + ) + if not response.ok: + raise SABaseException( + response.status_code, + "Couldn't copy images " + response.text + ) + res = response.json() + done_count = int(res['done']) + skipped_count = int(res['skipped']) + total_count = int(res['total_count']) + now_timestamp = datetime.datetime.now().timestamp() + if (skipped_count + done_count + == total_count) or (now_timestamp > max_timestamp): + break + total_skipped_count += skipped_count + total_done_count += done_count + return (total_done_count, total_skipped_count, total_skipped_list) + + def create_empty_annotation(size, image_name): return { "metadata": { From d57361fedaf654f89b4f1d8b8fd9927fa09fa98e Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 7 May 2021 16:06:00 +0400 Subject: [PATCH 15/22] Fix parsers struct --- superannotate/mixp/decorators.py | 4 +- superannotate/mixp/utils/__init__.py | 0 superannotate/mixp/{ => utils}/parsers.py | 625 +++++++--------------- 3 files changed, 201 insertions(+), 428 deletions(-) create mode 100644 superannotate/mixp/utils/__init__.py rename superannotate/mixp/{ => utils}/parsers.py (70%) diff --git a/superannotate/mixp/decorators.py b/superannotate/mixp/decorators.py index 2a761a96f..20550b71d 100644 --- a/superannotate/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -1,7 +1,7 @@ import sys -from .parsers import parsers from .app import mp, get_default from superannotate.api import API +from .utils import parsers _api = API.get_instance() callers_to_ignore = [] @@ -15,7 +15,7 @@ def wrapper(*args, **kwargs): caller_function_name = sys._getframe().f_back.f_code.co_name if caller_function_name not in callers_to_ignore: func_name_to_track = func.__name__ - data = parsers[func_name_to_track](*args, **kwargs) + data = getattr(parsers, func_name_to_track)(*args, **kwargs) user_id = _api.user_id event_name = data['event_name'] properties = data['properties'] diff --git a/superannotate/mixp/utils/__init__.py b/superannotate/mixp/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/superannotate/mixp/parsers.py b/superannotate/mixp/utils/parsers.py similarity index 70% rename from superannotate/mixp/parsers.py rename to superannotate/mixp/utils/parsers.py index 66535bcfc..496656cca 100644 --- a/superannotate/mixp/parsers.py +++ b/superannotate/mixp/utils/parsers.py @@ -1,6 +1,3 @@ -parsers = {} - - def get_project_name(project): project_name = "" if isinstance(project, dict): @@ -17,9 +14,6 @@ def get_team_metadata(*args, **kwargs): return {"event_name": "get_team_metadata", "properties": {}} -parsers['get_team_metadata'] = get_team_metadata - - def invite_contributor_to_team(*args, **kwargs): admin = kwargs.get("admin", None) if not admin: @@ -38,9 +32,6 @@ def invite_contributor_to_team(*args, **kwargs): } -parsers['invite_contributor_to_team'] = invite_contributor_to_team - - def delete_contributor_to_team_invitation(*args, **kwargs): return { "event_name": "delete_contributor_to_team_invitation", @@ -48,10 +39,6 @@ def delete_contributor_to_team_invitation(*args, **kwargs): } -parsers['delete_contributor_to_team_invitation' - ] = delete_contributor_to_team_invitation - - def search_team_contributors(*args, **kwargs): return { "event_name": "search_team_contributors", @@ -64,9 +51,6 @@ def search_team_contributors(*args, **kwargs): } -parsers['search_team_contributors'] = search_team_contributors - - def search_projects(*args, **kwargs): project = kwargs.get("name", None) if not project: @@ -83,9 +67,6 @@ def search_projects(*args, **kwargs): } -parsers['search_projects'] = search_projects - - def create_project(*args, **kwargs): project = kwargs.get("project_name", None) if not project: @@ -104,9 +85,6 @@ def create_project(*args, **kwargs): } -parsers['create_project'] = create_project - - def create_project_from_metadata(*args, **kwargs): project = kwargs.get("project_metadata", None) if not project: @@ -119,9 +97,6 @@ def create_project_from_metadata(*args, **kwargs): } -parsers['create_project_from_metadata'] = create_project_from_metadata - - def clone_project(*args, **kwargs): project = kwargs.get("project_name", None) if not project: @@ -147,9 +122,6 @@ def clone_project(*args, **kwargs): } -parsers['clone_project'] = clone_project - - def search_images(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -168,9 +140,6 @@ def search_images(*args, **kwargs): } -parsers['search_images'] = search_images - - def upload_images_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -195,9 +164,6 @@ def upload_images_to_project(*args, **kwargs): } -parsers['upload_images_to_project'] = upload_images_to_project - - def upload_image_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -216,9 +182,6 @@ def upload_image_to_project(*args, **kwargs): } -parsers['upload_image_to_project'] = upload_image_to_project - - def upload_images_from_public_urls_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -243,10 +206,6 @@ def upload_images_from_public_urls_to_project(*args, **kwargs): } -parsers['upload_images_from_public_urls_to_project' - ] = upload_images_from_public_urls_to_project - - def upload_images_from_google_cloud_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -259,10 +218,6 @@ def upload_images_from_google_cloud_to_project(*args, **kwargs): } -parsers['upload_images_from_google_cloud_to_project' - ] = upload_images_from_google_cloud_to_project - - def upload_images_from_azure_blob_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -275,10 +230,6 @@ def upload_images_from_azure_blob_to_project(*args, **kwargs): } -parsers['upload_images_from_azure_blob_to_project' - ] = upload_images_from_azure_blob_to_project - - def upload_video_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -296,9 +247,6 @@ def upload_video_to_project(*args, **kwargs): } -parsers['upload_video_to_project'] = upload_video_to_project - - def attach_image_urls_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -318,9 +266,6 @@ def attach_image_urls_to_project(*args, **kwargs): } -parsers['attach_image_urls_to_project'] = attach_image_urls_to_project - - def set_images_annotation_statuses(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -345,9 +290,6 @@ def set_images_annotation_statuses(*args, **kwargs): } -parsers['set_images_annotation_statuses'] = set_images_annotation_statuses - - def get_image_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -361,9 +303,6 @@ def get_image_annotations(*args, **kwargs): } -parsers['get_image_annotations'] = get_image_annotations - - def get_image_preannotations(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -377,9 +316,6 @@ def get_image_preannotations(*args, **kwargs): } -parsers['get_image_preannotations'] = get_image_preannotations - - def download_image_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -393,9 +329,6 @@ def download_image_annotations(*args, **kwargs): } -parsers['download_image_annotations'] = download_image_annotations - - def download_image_preannotations(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -409,11 +342,6 @@ def download_image_preannotations(*args, **kwargs): } -parsers['download_image_preannotations'] = download_image_preannotations - - - - def get_image_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -427,11 +355,6 @@ def get_image_metadata(*args, **kwargs): } -parsers['get_image_metadata'] = get_image_metadata - - - - def get_image_bytes(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -445,10 +368,6 @@ def get_image_bytes(*args, **kwargs): } -parsers['get_image_bytes'] = get_image_bytes - - - def delete_image(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -462,10 +381,6 @@ def delete_image(*args, **kwargs): } -parsers['delete_image'] = delete_image - - - def add_annotation_comment_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -479,10 +394,6 @@ def add_annotation_comment_to_image(*args, **kwargs): } -parsers['add_annotation_comment_to_image'] = add_annotation_comment_to_image - - - def delete_annotation_class(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -496,11 +407,6 @@ def delete_annotation_class(*args, **kwargs): } -parsers['delete_annotation_class'] = delete_annotation_class - - - - def get_annotation_class_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -514,11 +420,6 @@ def get_annotation_class_metadata(*args, **kwargs): } -parsers['get_annotation_class_metadata'] = get_annotation_class_metadata - - - - def download_annotation_classes_json(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -532,10 +433,6 @@ def download_annotation_classes_json(*args, **kwargs): } -parsers['download_annotation_classes_json'] = download_annotation_classes_json - - - def search_annotation_classes(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -549,9 +446,6 @@ def search_annotation_classes(*args, **kwargs): } -parsers['search_annotation_classes'] = search_annotation_classes - - def unshare_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -565,10 +459,6 @@ def unshare_project(*args, **kwargs): } -parsers['unshare_project'] = unshare_project - - - def get_project_image_count(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -582,10 +472,6 @@ def get_project_image_count(*args, **kwargs): } -parsers['get_project_image_count'] = get_project_image_count - - - def get_project_settings(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -599,11 +485,6 @@ def get_project_settings(*args, **kwargs): } -parsers['get_project_settings'] = get_project_settings - - - - def set_project_settings(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -617,10 +498,6 @@ def set_project_settings(*args, **kwargs): } -parsers['set_project_settings'] = set_project_settings - - - def get_project_default_image_quality_in_editor(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -634,9 +511,6 @@ def get_project_default_image_quality_in_editor(*args, **kwargs): } -parsers['get_project_default_image_quality_in_editor'] = get_project_default_image_quality_in_editor - - def get_project_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -650,9 +524,6 @@ def get_project_metadata(*args, **kwargs): } -parsers['get_project_metadata'] = get_project_metadata - - def delete_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -666,11 +537,6 @@ def delete_project(*args, **kwargs): } -parsers['delete_project'] = delete_project - - - - def rename_project(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -684,11 +550,6 @@ def rename_project(*args, **kwargs): } -parsers['rename_project'] = rename_project - - - - def get_project_workflow(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -702,10 +563,6 @@ def get_project_workflow(*args, **kwargs): } -parsers['get_project_workflow'] = get_project_workflow - - - def set_project_workflow(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -719,11 +576,6 @@ def set_project_workflow(*args, **kwargs): } -parsers['set_project_workflow'] = set_project_workflow - - - - def create_folder(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -737,11 +589,6 @@ def create_folder(*args, **kwargs): } -parsers['create_folder'] = create_folder - - - - def get_folder_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -755,11 +602,6 @@ def get_folder_metadata(*args, **kwargs): } -parsers['get_folder_metadata'] = get_folder_metadata - - - - def get_project_and_folder_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -773,12 +615,6 @@ def get_project_and_folder_metadata(*args, **kwargs): } -parsers['get_project_and_folder_metadata'] = get_project_and_folder_metadata - - - - - def rename_folder(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -792,10 +628,6 @@ def rename_folder(*args, **kwargs): } -parsers['rename_folder'] = rename_folder - - - def stop_model_training(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -809,10 +641,6 @@ def stop_model_training(*args, **kwargs): } -parsers['stop_model_training'] = stop_model_training - - - def download_model(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -826,10 +654,6 @@ def download_model(*args, **kwargs): } -parsers['download_model'] = download_model - - - def plot_model_metrics(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -843,9 +667,6 @@ def plot_model_metrics(*args, **kwargs): } -parsers['plot_model_metrics'] = plot_model_metrics - - def delete_model(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -859,10 +680,6 @@ def delete_model(*args, **kwargs): } -parsers['delete_model'] = delete_model - - - def convert_project_type(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -876,10 +693,6 @@ def convert_project_type(*args, **kwargs): } -parsers['convert_project_type'] = convert_project_type - - - def convert_json_version(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -893,10 +706,6 @@ def convert_json_version(*args, **kwargs): } -parsers['convert_json_version'] = convert_json_version - - - def df_to_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -910,10 +719,6 @@ def df_to_annotations(*args, **kwargs): } -parsers['df_to_annotations'] = df_to_annotations - - - def upload_image_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: @@ -921,19 +726,13 @@ def upload_image_annotations(*args, **kwargs): return { "event_name": "upload_image_annotations", - "properties": { - "project_name": get_project_name(project), - "Pixel" : bool(args[3:4] or ("mask" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "Pixel": bool(args[3:4] or ("mask" in kwargs)), + } } -parsers['upload_image_annotations'] = upload_image_annotations - - - - - - def download_image(*args, **kwargs): project = kwargs.get("project", None) @@ -942,20 +741,19 @@ def download_image(*args, **kwargs): return { "event_name": "download_image", - "properties": { - "project_name": get_project_name(project), - "Download Annotations": bool(args[3:4] or ("include_annotations" in kwargs) ), - "Download Fuse": bool(args[4:5] or ("include_fuse" in kwargs) ), - "Download Overlay": bool(args[5:6] or ("include_overlay" in kwargs) ), - } + "properties": + { + "project_name": + get_project_name(project), + "Download Annotations": + bool(args[3:4] or ("include_annotations" in kwargs)), + "Download Fuse": + bool(args[4:5] or ("include_fuse" in kwargs)), + "Download Overlay": + bool(args[5:6] or ("include_overlay" in kwargs)), + } } -parsers['download_image'] = download_image - - - - - def copy_image(*args, **kwargs): project = kwargs.get("source_project", None) @@ -964,18 +762,19 @@ def copy_image(*args, **kwargs): return { "event_name": "copy_image", - "properties": { - "project_name": get_project_name(project), - "Copy Annotations": bool(args[3:4] or ("include_annotations" in kwargs) ), - "Copy Annotation Status": bool(args[4:5] or ("copy_annotation_status" in kwargs) ), - "Copy Pin": bool(args[5:6] or ("copy_pin" in kwargs) ), - } + "properties": + { + "project_name": + get_project_name(project), + "Copy Annotations": + bool(args[3:4] or ("include_annotations" in kwargs)), + "Copy Annotation Status": + bool(args[4:5] or ("copy_annotation_status" in kwargs)), + "Copy Pin": + bool(args[5:6] or ("copy_pin" in kwargs)), + } } -parsers['copy_image'] = copy_image - - - def move_image(*args, **kwargs): project = kwargs.get("source_project", None) @@ -984,16 +783,19 @@ def move_image(*args, **kwargs): return { "event_name": "move_image", - "properties": { - "project_name": get_project_name(project), - "Move Annotations": bool(args[3:4] or ("include_annotations" in kwargs) ), - "Move Annotation Status": bool(args[4:5] or ("copy_annotation_status" in kwargs) ), - "Move Pin": bool(args[5:6] or ("copy_pin" in kwargs) ), - } + "properties": + { + "project_name": + get_project_name(project), + "Move Annotations": + bool(args[3:4] or ("include_annotations" in kwargs)), + "Move Annotation Status": + bool(args[4:5] or ("copy_annotation_status" in kwargs)), + "Move Pin": + bool(args[5:6] or ("copy_pin" in kwargs)), + } } -parsers['move_image'] = move_image - def pin_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1002,16 +804,13 @@ def pin_image(*args, **kwargs): return { "event_name": "pin_image", - "properties": { - "project_name": get_project_name(project), - "Pin": bool(args[2:3] or ("pin" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "Pin": bool(args[2:3] or ("pin" in kwargs)), + } } -parsers['pin_image'] = pin_image - - - def create_fuse_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1024,18 +823,14 @@ def create_fuse_image(*args, **kwargs): return { "event_name": "create_fuse_image", - "properties": { - "project_name": get_project_name(project), - "Project Type": project_type, - "Overlay": bool(args[4:5] or ("output_overlay" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "Project Type": project_type, + "Overlay": bool(args[4:5] or ("output_overlay" in kwargs)), + } } -parsers['create_fuse_image'] = create_fuse_image - - - - def set_image_annotation_status(*args, **kwargs): project = kwargs.get("project", None) @@ -1044,16 +839,15 @@ def set_image_annotation_status(*args, **kwargs): return { "event_name": "set_image_annotation_status", - "properties": { - "project_name": get_project_name(project), - "Annotation Status": bool(args[2:3] or ("annotation_status" in kwargs) ), - } + "properties": + { + "project_name": + get_project_name(project), + "Annotation Status": + bool(args[2:3] or ("annotation_status" in kwargs)), + } } -parsers['set_image_annotation_status'] = set_image_annotation_status - - - def add_annotation_bbox_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1062,17 +856,19 @@ def add_annotation_bbox_to_image(*args, **kwargs): return { "event_name": "add_annotation_bbox_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[5:6] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[4:5] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[5:6] or ("error" in kwargs)), + } } -parsers['add_annotation_bbox_to_image'] = add_annotation_bbox_to_image - - def add_annotation_polygon_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1081,17 +877,19 @@ def add_annotation_polygon_to_image(*args, **kwargs): return { "event_name": "add_annotation_polygon_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[5:6] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[4:5] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[5:6] or ("error" in kwargs)), + } } -parsers['add_annotation_polygon_to_image'] = add_annotation_polygon_to_image - - def add_annotation_polyline_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1100,17 +898,19 @@ def add_annotation_polyline_to_image(*args, **kwargs): return { "event_name": "add_annotation_polyline_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[5:6] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[4:5] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[5:6] or ("error" in kwargs)), + } } -parsers['add_annotation_polyline_to_image'] = add_annotation_polyline_to_image - - def add_annotation_point_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1119,16 +919,19 @@ def add_annotation_point_to_image(*args, **kwargs): return { "event_name": "add_annotation_point_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[5:6] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[4:5] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[5:6] or ("error" in kwargs)), + } } -parsers['add_annotation_point_to_image'] = add_annotation_point_to_image - def add_annotation_ellipse_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1137,18 +940,19 @@ def add_annotation_ellipse_to_image(*args, **kwargs): return { "event_name": "add_annotation_ellipse_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[5:6] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[4:5] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[5:6] or ("error" in kwargs)), + } } -parsers['add_annotation_ellipse_to_image'] = add_annotation_ellipse_to_image - - - def add_annotation_template_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1157,19 +961,19 @@ def add_annotation_template_to_image(*args, **kwargs): return { "event_name": "add_annotation_template_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[5:6] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[6:7] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[5:6] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[6:7] or ("error" in kwargs)), + } } -parsers['add_annotation_template_to_image'] = add_annotation_template_to_image - - - - def add_annotation_cuboid_to_image(*args, **kwargs): project = kwargs.get("project", None) @@ -1178,18 +982,19 @@ def add_annotation_cuboid_to_image(*args, **kwargs): return { "event_name": "add_annotation_cuboid_to_image", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[4:5] or ("annotation_class_attributes" in kwargs) ), - "Error": bool(args[5:6] or ("error" in kwargs) ), - - } + "properties": + { + "project_name": + get_project_name(project), + "Attributes": + bool( + args[4:5] or ("annotation_class_attributes" in kwargs) + ), + "Error": + bool(args[5:6] or ("error" in kwargs)), + } } -parsers['add_annotation_cuboid_to_image'] = add_annotation_cuboid_to_image - - - def create_annotation_class(*args, **kwargs): project = kwargs.get("project", None) @@ -1198,15 +1003,13 @@ def create_annotation_class(*args, **kwargs): return { "event_name": "create_annotation_class", - "properties": { - "project_name": get_project_name(project), - "Attributes": bool(args[3:4] or ("attribute_groups" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "Attributes": bool(args[3:4] or ("attribute_groups" in kwargs)), + } } -parsers['create_annotation_class'] = create_annotation_class - - def create_annotation_classes_from_classes_json(*args, **kwargs): project = kwargs.get("project", None) @@ -1215,39 +1018,31 @@ def create_annotation_classes_from_classes_json(*args, **kwargs): return { "event_name": "create_annotation_classes_from_classes_json", - "properties": { - "project_name": get_project_name(project), - "From S3": bool(args[2:3] or ("from_s3_bucket" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "From S3": bool(args[2:3] or ("from_s3_bucket" in kwargs)), + } } -parsers['create_annotation_classes_from_classes_json'] = create_annotation_classes_from_classes_json - - - def class_distribution(*args, **kwargs): return { "event_name": "class_distribution", "properties": { - "Plot": bool(args[2:3] or ("visualize" in kwargs) ), + "Plot": bool(args[2:3] or ("visualize" in kwargs)), } } -parsers['class_distribution'] = class_distribution - - def attribute_distribution(*args, **kwargs): return { "event_name": "attribute_distribution", "properties": { - "Plot": bool(args[2:3] or ("visualize" in kwargs) ), + "Plot": bool(args[2:3] or ("visualize" in kwargs)), } } -parsers['attribute_distribution'] = attribute_distribution - def share_project(*args, **kwargs): project = kwargs.get("project", None) @@ -1258,18 +1053,15 @@ def share_project(*args, **kwargs): if not user_role: user_role = args[2:3][0] - return { "event_name": "share_project", - "properties": { - "project_name": get_project_name(project), - "User Role":user_role - } + "properties": + { + "project_name": get_project_name(project), + "User Role": user_role + } } -parsers['share_project'] = share_project - - def set_project_default_image_quality_in_editor(*args, **kwargs): project = kwargs.get("project", None) @@ -1280,93 +1072,82 @@ def set_project_default_image_quality_in_editor(*args, **kwargs): if not image_quality_in_editor: image_quality_in_editor = args[1:2][0] - return { "event_name": "set_project_default_image_quality_in_editor", - "properties": { - "project_name": get_project_name(project), - "Image Quality":image_quality_in_editor - } + "properties": + { + "project_name": get_project_name(project), + "Image Quality": image_quality_in_editor + } } -parsers['set_project_default_image_quality_in_editor'] = set_project_default_image_quality_in_editor - - def get_exports(*args, **kwargs): project = kwargs.get("project", None) if not project: project = args[0:1][0] - + return { "event_name": "get_exports", - "properties": { - "project_name": get_project_name(project), - "Metadata": bool(args[1:2] or ("return_metadata" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "Metadata": bool(args[1:2] or ("return_metadata" in kwargs)), + } } -parsers['get_exports'] = get_exports - - - def search_folders(*args, **kwargs): project = kwargs.get("project", None) if not project: project = args[0:1][0] - + return { "event_name": "search_folders", - "properties": { - "project_name": get_project_name(project), - "Metadata": bool(args[2:3] or ("return_metadata" in kwargs) ), - } + "properties": + { + "project_name": get_project_name(project), + "Metadata": bool(args[2:3] or ("return_metadata" in kwargs)), + } } -parsers['search_folders'] = search_folders - - def filter_images_by_tags(*args, **kwargs): return { "event_name": "filter_images_by_tags", - "properties": { - "Include": bool(args[1:2] or ("include" in kwargs) ), - "Exclude" : bool(args[2:3] or ("exclude" in kwargs) ) - } + "properties": + { + "Include": bool(args[1:2] or ("include" in kwargs)), + "Exclude": bool(args[2:3] or ("exclude" in kwargs)) + } } -parsers['filter_images_by_tags'] = filter_images_by_tags - - def filter_images_by_comments(*args, **kwargs): return { "event_name": "filter_images_by_comments", - "properties": { - "Include Unresolved Comments" : bool(args[1:2] or ("include_unresolved_comments" in kwargs) ), - "Include Resolved Comments": bool(args[2:3] or ("include_resolved_comments" in kwargs) ) - } + "properties": + { + "Include Unresolved Comments": + bool( + args[1:2] or ("include_unresolved_comments" in kwargs) + ), + "Include Resolved Comments": + bool(args[2:3] or ("include_resolved_comments" in kwargs)) + } } -parsers['filter_images_by_comments'] = filter_images_by_comments - - def filter_annotation_instances(*args, **kwargs): return { "event_name": "filter_annotation_instances", - "properties": { - "Include" : bool(args[1:2] or ("include" in kwargs) ), - "Exclude": bool(args[2:3] or ("exclude" in kwargs) ) - } + "properties": + { + "Include": bool(args[1:2] or ("include" in kwargs)), + "Exclude": bool(args[2:3] or ("exclude" in kwargs)) + } } -parsers['filter_annotation_instances'] = filter_annotation_instances - - - def aggregate_annotations_as_df(*args, **kwargs): folder_names = kwargs.get("folder_names", None) @@ -1377,16 +1158,13 @@ def aggregate_annotations_as_df(*args, **kwargs): else: folder_names = [] - return { "event_name": "aggregate_annotations_as_df", "properties": { - "Fodler Count" : len(folder_names), + "Fodler Count": len(folder_names), } } -parsers['aggregate_annotations_as_df'] = aggregate_annotations_as_df - def delete_folders(*args, **kwargs): project = kwargs.get("project", None) @@ -1396,19 +1174,16 @@ def delete_folders(*args, **kwargs): folder_names = kwargs.get("folder_names", None) if not folder_names: folder_names = args[1:2][0] - - + return { "event_name": "delete_folders", - "properties": { - "project_name": get_project_name(project), - "Folder Count": len(folder_names), - } + "properties": + { + "project_name": get_project_name(project), + "Folder Count": len(folder_names), + } } -parsers['delete_folders'] = delete_folders - - def delete_images(*args, **kwargs): project = kwargs.get("project", None) @@ -1418,14 +1193,12 @@ def delete_images(*args, **kwargs): image_names = kwargs.get("image_names", None) if not image_names: image_names = args[1:2][0] - - + return { "event_name": "delete_images", - "properties": { - "project_name": get_project_name(project), - "Image Count": len(image_names), - } + "properties": + { + "project_name": get_project_name(project), + "Image Count": len(image_names), + } } - -parsers['delete_images'] = delete_images From 9011b36dddd71c51f83661a359d7fdac6cd457ff Mon Sep 17 00:00:00 2001 From: Erik Harutyunyan Date: Mon, 10 May 2021 13:19:29 +0400 Subject: [PATCH 16/22] fixed consensus bug of skipping instances --- superannotate/consensus_benchmark/helpers.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/superannotate/consensus_benchmark/helpers.py b/superannotate/consensus_benchmark/helpers.py index 4de1508b5..f1d8d4959 100644 --- a/superannotate/consensus_benchmark/helpers.py +++ b/superannotate/consensus_benchmark/helpers.py @@ -80,32 +80,43 @@ def image_consensus(df, image_name, annot_type): "Invalid %s instance occured, skipping to the next one.", annot_type ) + visited_instances = {} + for proj, instances in projects_shaply_objs.items(): + visited_instances[proj] = [False] * len(instances) # match instances for curr_proj, curr_proj_instances in projects_shaply_objs.items(): - for curr_inst_data in curr_proj_instances: + for curr_id, curr_inst_data in enumerate(curr_proj_instances): curr_inst, curr_class, _, _ = curr_inst_data + if visited_instances[curr_proj][curr_id] == True: + continue max_instances = [] for other_proj, other_proj_instances in projects_shaply_objs.items( ): if curr_proj == other_proj: max_instances.append((curr_proj, *curr_inst_data)) - projects_shaply_objs[curr_proj].remove(curr_inst_data) + visited_instances[curr_proj][curr_id] = True else: if annot_type in ['polygon', 'bbox']: max_score = 0 else: max_score = float('-inf') max_inst_data = None - for other_inst_data in other_proj_instances: + max_inst_id = -1 + for other_id, other_inst_data in enumerate( + other_proj_instances + ): other_inst, other_class, _, _ = other_inst_data + if visited_instances[other_proj][other_id] == True: + continue score = instance_consensus(curr_inst, other_inst) if score > max_score and other_class == curr_class: max_score = score max_inst_data = other_inst_data + max_inst_id = other_id if max_inst_data is not None: max_instances.append((other_proj, *max_inst_data)) - projects_shaply_objs[other_proj].remove(max_inst_data) + visited_instances[other_proj][max_inst_id] = True if len(max_instances) == 1: image_data["creatorEmail"].append(max_instances[0][3]) image_data["attributes"].append(max_instances[0][4]) From bc5f287c4ce72a70d313b4781d4d2a81071efc44 Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 12 May 2021 15:17:03 +0400 Subject: [PATCH 17/22] Add mix panel events --- .../consensus_benchmark/benchmark.py | 2 + .../consensus_benchmark/consensus.py | 2 + superannotate/db/exports.py | 3 + superannotate/db/project_images.py | 3 + superannotate/db/projects.py | 18 +- superannotate/dicom_converter.py | 2 + superannotate/input_converters/conversion.py | 5 + superannotate/mixp/decorators.py | 3 +- superannotate/mixp/utils/parsers.py | 545 ++++++++++++++++++ superannotate/ml/ml_funcs.py | 8 + 10 files changed, 589 insertions(+), 2 deletions(-) diff --git a/superannotate/consensus_benchmark/benchmark.py b/superannotate/consensus_benchmark/benchmark.py index bfaf42d16..2526a647a 100644 --- a/superannotate/consensus_benchmark/benchmark.py +++ b/superannotate/consensus_benchmark/benchmark.py @@ -9,10 +9,12 @@ from .helpers import image_consensus, consensus_plot from ..db.exports import prepare_export, download_export from ..analytics.common import aggregate_annotations_as_df +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") +@trackable def benchmark( project, gt_folder, diff --git a/superannotate/consensus_benchmark/consensus.py b/superannotate/consensus_benchmark/consensus.py index 4fcf39be1..ec958e46e 100644 --- a/superannotate/consensus_benchmark/consensus.py +++ b/superannotate/consensus_benchmark/consensus.py @@ -9,10 +9,12 @@ from .helpers import image_consensus, consensus_plot from ..db.exports import prepare_export, download_export from ..analytics.common import aggregate_annotations_as_df +from ..mixp.decorators import trackable logger = logging.getLogger("superannotate-python-sdk") +@trackable def consensus( project, folder_names, diff --git a/superannotate/db/exports.py b/superannotate/db/exports.py index cdd486682..cfc1864b9 100644 --- a/superannotate/db/exports.py +++ b/superannotate/db/exports.py @@ -56,6 +56,7 @@ def get_export_metadata(project, export_name): " is not unique. To use SDK please use unique export names." ) + @trackable def get_exports(project, return_metadata=False): """Get all prepared exports of the project. @@ -98,6 +99,7 @@ def _get_export(export): return response.json() +@trackable def prepare_export( project, folder_names=None, @@ -220,6 +222,7 @@ def _download_file(url, local_filename): return local_filename +@trackable def download_export( project, export, folder_path, extract_zip_contents=True, to_s3_bucket=None ): diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 0a4d2757b..599843fc6 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -188,6 +188,7 @@ def _copy_images( return res +@trackable def copy_images( source_project, image_names, @@ -300,6 +301,7 @@ def delete_images(project, image_names): ) +@trackable def move_images( source_project, image_names, @@ -544,6 +546,7 @@ def pin_image(project, image_name, pin=True): ) +@trackable def assign_images(project, image_names, user): """Assigns images to a user. The assignment role, QA or Annotator, will be deduced from the user's role in the project. With SDK, the user can be diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 89e802e5b..4a8fbb770 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -132,6 +132,7 @@ def create_project_from_metadata(project_metadata): set_project_workflow(new_project_metadata, project_metadata["workflow"]) return new_project_metadata + @trackable def delete_project(project): """Deletes the project @@ -152,6 +153,7 @@ def delete_project(project): ) logger.info("Successfully deleted project %s.", project["name"]) + @trackable def rename_project(project, new_name): """Renames the project @@ -189,6 +191,7 @@ def rename_project(project, new_name): "Successfully renamed project %s to %s.", project["name"], new_name ) + @trackable def get_project_image_count(project, with_all_subfolders=False): """Returns number of images in the project. @@ -391,6 +394,7 @@ def upload_video_to_project( return filenames_base +@trackable def upload_videos_from_folder_to_project( project, folder_path, @@ -486,6 +490,7 @@ def upload_videos_from_folder_to_project( return filenames +@trackable def upload_images_from_folder_to_project( project, folder_path, @@ -1145,6 +1150,7 @@ def __upload_annotations_thread( uploaded[thread_id].append(full_path) +@trackable def upload_annotations_from_folder_to_project( project, folder_path, from_s3_bucket=None, recursive_subfolders=False ): @@ -1353,6 +1359,7 @@ def __tqdm_thread_upload_annotations( break +@trackable def upload_preannotations_from_folder_to_project( project, folder_path, from_s3_bucket=None, recursive_subfolders=False ): @@ -1383,6 +1390,7 @@ def upload_preannotations_from_folder_to_project( project, folder_path, "pre", from_s3_bucket, recursive_subfolders ) + @trackable def share_project(project, user, user_role): """Share project with user. @@ -1416,6 +1424,7 @@ def share_project(project, user, user_role): user["email"], common.user_role_int_to_str(user_role) ) + @trackable def unshare_project(project, user): """Unshare (remove) user from project. @@ -1443,7 +1452,7 @@ def unshare_project(project, user): raise SABaseException(response.status_code, response.text) logger.info("Unshared project %s from user ID %s", project["name"], user_id) - +@trackable def upload_images_from_s3_bucket_to_project( project, accessKeyId, @@ -1534,6 +1543,7 @@ def _get_upload_from_s3_bucket_to_project_status(project, project_folder): ) return response.json() + @trackable def get_project_workflow(project): """Gets project's workflow. @@ -1576,6 +1586,7 @@ def get_project_workflow(project): raise SABaseException(0, "Couldn't find class_id in workflow") return res + @trackable def set_project_workflow(project, new_workflow): """Sets project's workflow. @@ -1676,6 +1687,7 @@ def set_project_workflow(project, new_workflow): "Couldn't set project workflow " + response.text ) + @trackable def get_project_settings(project): """Gets project's settings. @@ -1713,6 +1725,7 @@ def get_project_settings(project): raise SABaseException(0, "NA ImageQuality value") return res + @trackable def set_project_settings(project, new_settings): """Sets project's settings. @@ -1777,6 +1790,7 @@ def set_project_settings(project, new_settings): ) return response.json() + @trackable def set_project_default_image_quality_in_editor( project, image_quality_in_editor @@ -1796,6 +1810,7 @@ def set_project_default_image_quality_in_editor( }] ) + @trackable def get_project_default_image_quality_in_editor(project): """Gets project's default image quality in editor setting. @@ -1814,6 +1829,7 @@ def get_project_default_image_quality_in_editor(project): "Image quality in editor should be 'compressed', 'original' or None for project settings value" ) + @trackable def get_project_metadata( project, diff --git a/superannotate/dicom_converter.py b/superannotate/dicom_converter.py index ce2f8e392..245debac7 100644 --- a/superannotate/dicom_converter.py +++ b/superannotate/dicom_converter.py @@ -3,8 +3,10 @@ import numpy as np import pydicom from PIL import Image +from .mixp.decorators import trackable +@trackable def dicom_to_rgb_sequence( input_dicom_file, output_dir, output_image_quality="original" ): diff --git a/superannotate/input_converters/conversion.py b/superannotate/input_converters/conversion.py index 0c7846524..d8cc5b92d 100644 --- a/superannotate/input_converters/conversion.py +++ b/superannotate/input_converters/conversion.py @@ -132,6 +132,7 @@ def _passes_converter_sanity(args, direction): ) +@trackable def export_annotation( input_dir, output_dir, @@ -211,6 +212,7 @@ def export_annotation( export_from_sa(args) +@trackable def import_annotation( input_dir, output_dir, @@ -391,6 +393,7 @@ def import_annotation( import_to_sa(args) + @trackable def convert_project_type(input_dir, output_dir): """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. @@ -411,6 +414,7 @@ def convert_project_type(input_dir, output_dir): sa_convert_project_type(input_dir, output_dir) +@trackable def coco_split_dataset( coco_json_path, image_dir, output_dir, dataset_list_name, ratio_list ): @@ -462,6 +466,7 @@ def coco_split_dataset( coco_json_path, image_dir, output_dir, dataset_list_name, ratio_list ) + @trackable def convert_json_version(input_dir, output_dir, version=2): """ diff --git a/superannotate/mixp/decorators.py b/superannotate/mixp/decorators.py index 20550b71d..317f2fcc7 100644 --- a/superannotate/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -5,6 +5,7 @@ _api = API.get_instance() callers_to_ignore = [] +always_trackable_func_names = ["upload_images_from_folder_to_project"] def trackable(func): @@ -13,7 +14,7 @@ def trackable(func): def wrapper(*args, **kwargs): try: caller_function_name = sys._getframe().f_back.f_code.co_name - if caller_function_name not in callers_to_ignore: + if caller_function_name not in callers_to_ignore or caller_function_name in always_trackable_func_names: func_name_to_track = func.__name__ data = getattr(parsers, func_name_to_track)(*args, **kwargs) user_id = _api.user_id diff --git a/superannotate/mixp/utils/parsers.py b/superannotate/mixp/utils/parsers.py index 496656cca..c3ab87289 100644 --- a/superannotate/mixp/utils/parsers.py +++ b/superannotate/mixp/utils/parsers.py @@ -776,6 +776,551 @@ def copy_image(*args, **kwargs): } +def run_prediction(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + from superannotate.db.projects import get_project_metadata as sa_get_project_metadata + project_name = get_project_name(project) + project_metadata = sa_get_project_metadata(project_name) + + image_list = kwargs.get("images_list", None) + if not image_list: + image_list = args[1:2][0] + + return { + "event_name": "run_prediction", + "properties": + { + "Project Type": project_metadata['type'], + "Image Count": len(image_list) + } + } + + +def run_segmentation(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + from superannotate.db.projects import get_project_metadata as sa_get_project_metadata + project_name = get_project_name(project) + project_metadata = sa_get_project_metadata(project_name) + + image_list = kwargs.get("images_list", None) + if not image_list: + image_list = args[1:2][0] + + model = kwargs.get("model", None) + if not model: + model = args[2:3][0] + + return { + "event_name": "run_segmentation", + "properties": + { + "Project Type": project_metadata['type'], + "Image Count": len(image_list), + "Model": model + } + } + + +def upload_videos_from_folder_to_project(*args, **kwargs): + folder_path = kwargs.get("folder_path", None) + if not folder_path: + folder_path = args[1:2][0] + + from pathlib import Path + glob_iterator = Path(folder_path).glob('*') + + return { + "event_name": "upload_videos_from_folder_to_project", + "properties": { + "Video Count": sum(1 for _ in glob_iterator), + } + } + + +def export_annotation(*args, **kwargs): + + dataset_format = kwargs.get("dataset_format", None) + if not dataset_format: + dataset_format = args[2:3][0] + + project_type = kwargs.get("project_type", None) + if not project_type: + project_type = args[4:5] + if not project_type: + project_type = 'Vector' + else: + project_type = args[4:5][0] + + task = kwargs.get("task", None) + if not task: + task = args[5:6] + if not task: + task = "object_detection" + else: + task = args[5:6][0] + + return { + "event_name": "export_annotation", + "properties": + { + "Format": dataset_format, + "Project Type": project_type, + "Task": task + } + } + + +def import_annotation(*args, **kwargs): + dataset_format = kwargs.get("dataset_format", None) + if not dataset_format: + dataset_format = args[2:3][0] + + project_type = kwargs.get("project_type", None) + if not project_type: + project_type = args[4:5] + if not project_type: + project_type = 'Vector' + else: + project_type = args[4:5][0] + + task = kwargs.get("task", None) + if not task: + task = args[5:6] + if not task: + task = "object_detection" + else: + task = args[5:6][0] + + return { + "event_name": "import_annotation", + "properties": + { + "Format": dataset_format, + "Project Type": project_type, + "Task": task + } + } + + +def move_images(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + image_names = kwargs.get("image_names", False) + if image_names == False: + image_names = args[1:2][0] + if image_names == None: + from superannotate.db.images import search_images as sa_search_images + image_names = sa_search_images(project) + + return { + "event_name": "move_images", + "properties": + { + "project_name": + get_project_name(project), + "Image Count": + len(image_names), + "Copy Annotations": + bool(args[3:4] or ("include_annotations" in kwargs)), + "Copy Annotation Status": + bool(args[4:5] or ("copy_annotation_status" in kwargs)), + "Copy Pin": + bool(args[5:6] or ("copy_pin" in kwargs)), + } + } + + +def copy_images(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + image_names = kwargs.get("image_names", False) + if image_names == False: + image_names = args[1:2][0] + if image_names == None: + from superannotate.db.images import search_images as sa_search_images + image_names = sa_search_images(project) + + return { + "event_name": "copy_images", + "properties": + { + "project_name": + get_project_name(project), + "Image Count": + len(image_names), + "Copy Annotations": + bool(args[3:4] or ("include_annotations" in kwargs)), + "Copy Annotation Status": + bool(args[4:5] or ("copy_annotation_status" in kwargs)), + "Copy Pin": + bool(args[5:6] or ("copy_pin" in kwargs)), + } + } + + +def consensus(*args, **kwargs): + + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + folder_names = kwargs.get("folder_names", None) + if not folder_names: + folder_names = args[1:2][0] + + image_list = kwargs.get("image_list", "empty") + if image_list == "empty": + image_list = args[3:4] + if image_list == None or len(image_list) == 0: + from superannotate.db.images import search_images as sa_search_images + image_list = sa_search_images(project) + else: + image_list = args[3:4][0] + + annot_type = kwargs.get("annot_type", "empty") + if annot_type == 'empty': + annot_type = args[4:5] + if not annot_type: + annot_type = "bbox" + else: + annot_type = args[4:5][0] + + show_plots = kwargs.get("show_plots", "empty") + if show_plots == "empty": + show_plots = args[5:6] + if not show_plots: + show_plots = False + else: + show_plots = args[5:6][0] + + return { + "event_name": "consensus", + "properties": + { + "Folder Count": len(folder_names), + "Image Count": len(image_list), + "Annotation Type": annot_type, + "Plot": show_plots + } + } + + +def benchmark(*args, **kwargs): + + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + folder_names = kwargs.get("folder_names", None) + if not folder_names: + folder_names = args[2:3][0] + + image_list = kwargs.get("image_list", "empty") + if image_list == "empty": + image_list = args[4:5] + if image_list == None or len(image_list) == 0: + from superannotate.db.images import search_images as sa_search_images + image_list = sa_search_images(project) + else: + image_list = args[4:5][0] + + annot_type = kwargs.get("annot_type", "empty") + if annot_type == 'empty': + annot_type = args[5:6] + if not annot_type: + annot_type = "bbox" + else: + annot_type = args[5:6][0] + + show_plots = kwargs.get("show_plots", "empty") + if show_plots == "empty": + show_plots = args[6:7] + if not show_plots: + show_plots = False + else: + show_plots = args[6:7][0] + + return { + "event_name": "benchmark", + "properties": + { + "Folder Count": len(folder_names), + "Image Count": len(image_list), + "Annotation Type": annot_type, + "Plot": show_plots + } + } + + +def upload_annotations_from_folder_to_project(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + from superannotate.db.projects import get_project_metadata as sa_get_project_metadata + project_name = get_project_name(project) + project_metadata = sa_get_project_metadata(project_name) + + folder_path = kwargs.get("folder_path", None) + if not folder_path: + folder_path = args[1:2][0] + + from pathlib import Path + glob_iterator = Path(folder_path).glob('*.json') + + return { + "event_name": "upload_annotations_from_folder_to_project", + "properties": + { + "Annotation Count": sum(1 for _ in glob_iterator), + "Project Type": project_metadata['type'], + "From S3": bool(args[2:3] or ("from_s3_bucket" in kwargs)) + } + } + + +def upload_preannotations_from_folder_to_project(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + from superannotate.db.projects import get_project_metadata as sa_get_project_metadata + project_name = get_project_name(project) + project_metadata = sa_get_project_metadata(project_name) + + folder_path = kwargs.get("folder_path", None) + if not folder_path: + folder_path = args[1:2][0] + + from pathlib import Path + glob_iterator = Path(folder_path).glob('*.json') + + return { + "event_name": "upload_preannotations_from_folder_to_project", + "properties": + { + "Annotation Count": sum(1 for _ in glob_iterator), + "Project Type": project_metadata['type'], + "From S3": bool(args[2:3] or ("from_s3_bucket" in kwargs)) + } + } + + +def upload_images_from_folder_to_project(*args, **kwargs): + project = kwargs.get("source_project", None) + if not project: + project = args[0:1][0] + + folder_path = kwargs.get("folder_path", None) + if not folder_path: + folder_path = args[1:2][0] + + from ... import common + extension = common.DEFAULT_IMAGE_EXTENSIONS + + from pathlib import Path + glob_iterator = Path(folder_path).glob(f'*.{extension}') + + return { + "event_name": "upload_images_from_folder_to_project", + "properties": + { + "Image Count": + sum(1 for _ in glob_iterator), + "Custom Extentions": + bool(args[2:3] or kwargs.get("extensions", None)), + "Annotation Status": + bool(args[3:4] or kwargs.get("annotation_status", None)), + "From S3": + bool(args[4:5] or kwargs.get("from_s3_bucket", None)), + "Custom Exclude Patters": + bool( + args[5:6] or kwargs.get("exclude_file_patterns", None) + ) + } + } + + +def upload_images_from_s3_bucket_to_project(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "upload_images_from_s3_bucket_to_project", + "properties": { + "project_name": get_project_name(project) + } + } + + +# def prepare_export( +# project, +# folder_names=None, +# annotation_statuses=None, +# include_fuse=False, +# only_pinned=False +# ): + +# Folder Count: len(folder_names), +# Annotation Statuses: True/False, +# Include Fuse: True/False, +# Only Pinned: True/False + + +def prepare_export(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "prepare_export", + "properties": + { + "project_name": + get_project_name(project), + "Folder Count": + bool(args[1:2] or kwargs.get("folder_names", None)), + "Annotation Statuses": + bool(args[2:3] or kwargs.get("annotation_statuses", None)), + "Include Fuse": + bool(args[3:4] or kwargs.get("include_fuse", None)), + "Only Pinned": + bool(args[4:5] or kwargs.get("only_pinned", None)), + } + } + + +def download_export(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + return { + "event_name": "download_export", + "properties": + { + "project_name": + get_project_name(project), + "to_s3_bucket": + bool(args[4:5] or kwargs.get("to_s3_bucket", None)), + } + } + + +def dicom_to_rgb_sequence(*args, **kwargs): + + return {"event_name": "dicom_to_rgb_sequence", "properties": {}} + + +def coco_split_dataset(*args, **kwargs): + ratio_list = kwargs.get("ratio_list", None) + if not ratio_list: + ratio_list = args[4:5][0] + + return { + "event_name": "coco_split_dataset", + "properties": { + "ratio_list": str(ratio_list) + } + } + + +def run_training(*args, **kwargs): + + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + project = project[0] + + task = kwargs.get("task", None) + if not task: + task = args[4:5][0] + + hyperparameters = kwargs.get("hyperparameters", None) + if not hyperparameters: + hyperparameters = args[4:5][0] + + from superannotate.db.projects import get_project_metadata as sa_get_project_metadata + project_name = get_project_name(project) + project_metadata = sa_get_project_metadata(project_name) + + log = kwargs.get("log", "empty") + if log == "empty": + log = args[6:7] + if not log: + log = False + else: + log = args[6:7][0] + + return { + "event_name": "run_training", + "properties": + { + "Project Type": project_metadata['type'], + "Task": task, + "Learning Rate": hyperparameters['base_lr'], + "Batch Size": hyperparameters['images_per_batch'], + "Log": log + } + } + + +# def assign_images(project, image_names, user): +# Assign Folder: IsRoot(project) , +# Image Count: len(image_names), +# User Role: Annotator/QA/... + + +def assign_images(*args, **kwargs): + project = kwargs.get("project", None) + if not project: + project = args[0:1][0] + + image_names = kwargs.get("image_names", None) + if not image_names: + image_names = args[1:2][0] + + user = kwargs.get("user", None) + if not user: + user = args[2:3][0] + + from superannotate.db.users import get_team_contributor_metadata + res = get_team_contributor_metadata(user) + user_role = "ADMIN" + if res['user_role'] == 3: + user_role = 'ANNOTATOR' + if res['user_role'] == 4: + user_role = 'QA' + + from superannotate.db.project_api import get_project_and_folder_metadata + project, folder = get_project_and_folder_metadata(project) + is_root = True + if folder: + is_root = False + + return { + "event_name": "assign_images", + "properties": + { + "project_name": get_project_name(project), + "Assign Folder": is_root, + "Image Count": len(image_names), + "User Role": user_role, + } + } + + def move_image(*args, **kwargs): project = kwargs.get("source_project", None) if not project: diff --git a/superannotate/ml/ml_funcs.py b/superannotate/ml/ml_funcs.py index eb4d5d67b..db1bbdf44 100644 --- a/superannotate/ml/ml_funcs.py +++ b/superannotate/ml/ml_funcs.py @@ -23,10 +23,12 @@ from .utils import log_process, make_plotly_specs, reformat_metrics_json from ..db.utils import _get_boto_session_by_credentials from ..mixp.decorators import trackable + logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() +@trackable @project_metadata @model_metadata def run_prediction(project, images_list, model): @@ -103,6 +105,7 @@ def run_prediction(project, images_list, model): return succeded_imgs, failed_imgs +@trackable @project_metadata def run_segmentation(project, images_list, model): """Starts smart segmentation on a list of images using the specified model @@ -178,6 +181,7 @@ def run_segmentation(project, images_list, model): return (succeded_imgs, failed_imgs) +@trackable @project_metadata @model_metadata def run_training( @@ -342,6 +346,7 @@ def run_training( time.sleep(5) return new_model + @trackable @model_metadata def stop_model_training(model): @@ -364,6 +369,7 @@ def stop_model_training(model): logger.info("Failed to stop model training please try again") return model + @trackable def plot_model_metrics(metric_json_list): """plots the metrics generated by neural network using plotly @@ -433,6 +439,7 @@ def get_plottable_cols(df): ) figure.show() + @trackable @model_metadata def download_model(model, output_dir): @@ -495,6 +502,7 @@ def download_model(model, output_dir): logger.info("Downloaded model related files") return model + @trackable @model_metadata def delete_model(model): From 80cb69d15a4fab5a2abd2dd503eace1837eb099e Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 12 May 2021 15:55:14 +0400 Subject: [PATCH 18/22] Change decorator --- superannotate/analytics/class_analytics.py | 8 ++- superannotate/analytics/common.py | 9 ++- .../consensus_benchmark/benchmark.py | 4 +- .../consensus_benchmark/consensus.py | 4 +- superannotate/dataframe_filtering.py | 11 ++-- superannotate/db/annotation_classes.py | 21 ++++--- superannotate/db/exports.py | 8 +-- superannotate/db/images.py | 58 ++++++++++++------- superannotate/db/project_api.py | 20 ++++--- superannotate/db/project_images.py | 18 +++--- superannotate/db/projects.py | 55 +++++++++--------- superannotate/db/search_projects.py | 4 +- superannotate/db/teams.py | 8 +-- superannotate/db/users.py | 4 +- superannotate/dicom_converter.py | 4 +- superannotate/input_converters/conversion.py | 12 ++-- superannotate/mixp/decorators.py | 29 ++++++---- superannotate/ml/ml_funcs.py | 16 ++--- 18 files changed, 167 insertions(+), 126 deletions(-) diff --git a/superannotate/analytics/class_analytics.py b/superannotate/analytics/class_analytics.py index 7da230bb0..ee47326ce 100644 --- a/superannotate/analytics/class_analytics.py +++ b/superannotate/analytics/class_analytics.py @@ -8,10 +8,12 @@ from plotly.subplots import make_subplots from .common import aggregate_annotations_as_df -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable + logger = logging.getLogger("superannotate-python-sdk") -@trackable + +@Trackable def class_distribution(export_root, project_names, visualize=False): """Aggregate distribution of classes across multiple projects. @@ -63,7 +65,7 @@ def class_distribution(export_root, project_names, visualize=False): return df -@trackable +@Trackable def attribute_distribution(export_root, project_names, visualize=False): """Aggregate distribution of attributes across multiple projects. diff --git a/superannotate/analytics/common.py b/superannotate/analytics/common.py index 509ae50a1..fb77ea04b 100644 --- a/superannotate/analytics/common.py +++ b/superannotate/analytics/common.py @@ -6,10 +6,12 @@ import pandas as pd from ..exceptions import SABaseException -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable + logger = logging.getLogger("superannotate-python-sdk") -@trackable + +@Trackable def df_to_annotations(df, output_dir): """Converts and saves pandas DataFrame annotation info (see aggregate_annotations_as_df) in output_dir. @@ -147,7 +149,8 @@ def df_to_annotations(df, output_dir): indent=4 ) -@trackable + +@Trackable def aggregate_annotations_as_df( project_root, include_classes_wo_annotations=False, diff --git a/superannotate/consensus_benchmark/benchmark.py b/superannotate/consensus_benchmark/benchmark.py index 2526a647a..4d93d2528 100644 --- a/superannotate/consensus_benchmark/benchmark.py +++ b/superannotate/consensus_benchmark/benchmark.py @@ -9,12 +9,12 @@ from .helpers import image_consensus, consensus_plot from ..db.exports import prepare_export, download_export from ..analytics.common import aggregate_annotations_as_df -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable logger = logging.getLogger("superannotate-python-sdk") -@trackable +@Trackable def benchmark( project, gt_folder, diff --git a/superannotate/consensus_benchmark/consensus.py b/superannotate/consensus_benchmark/consensus.py index ec958e46e..dfb4881d4 100644 --- a/superannotate/consensus_benchmark/consensus.py +++ b/superannotate/consensus_benchmark/consensus.py @@ -9,12 +9,12 @@ from .helpers import image_consensus, consensus_plot from ..db.exports import prepare_export, download_export from ..analytics.common import aggregate_annotations_as_df -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable logger = logging.getLogger("superannotate-python-sdk") -@trackable +@Trackable def consensus( project, folder_names, diff --git a/superannotate/dataframe_filtering.py b/superannotate/dataframe_filtering.py index ff5b21e69..4fc344531 100644 --- a/superannotate/dataframe_filtering.py +++ b/superannotate/dataframe_filtering.py @@ -1,7 +1,8 @@ import pandas as pd -from .mixp.decorators import trackable +from .mixp.decorators import Trackable -@trackable + +@Trackable def filter_images_by_comments( annotations_df, include_unresolved_comments=True, @@ -40,7 +41,8 @@ def filter_images_by_comments( return list(images) -@trackable + +@Trackable def filter_images_by_tags(annotations_df, include=None, exclude=None): """Filter images on tags @@ -74,7 +76,8 @@ def filter_images_by_tags(annotations_df, include=None, exclude=None): return list(images) -@trackable + +@Trackable def filter_annotation_instances(annotations_df, include=None, exclude=None): """Filter annotation instances from project annotations pandas DataFrame. diff --git a/superannotate/db/annotation_classes.py b/superannotate/db/annotation_classes.py index 1aeaa65c9..b34c2465b 100644 --- a/superannotate/db/annotation_classes.py +++ b/superannotate/db/annotation_classes.py @@ -11,12 +11,14 @@ SANonExistingAnnotationClassNameException ) from .project_api import get_project_metadata_bare -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable + logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() -@trackable + +@Trackable def create_annotation_class(project, name, color, attribute_groups=None): """Create annotation class in project @@ -78,7 +80,8 @@ def create_annotation_class(project, name, color, attribute_groups=None): new_class = res[0] return new_class -@trackable + +@Trackable def delete_annotation_class(project, annotation_class): """Deletes annotation class from project @@ -112,7 +115,8 @@ def delete_annotation_class(project, annotation_class): "Couldn't delete annotation class " + response.text ) -@trackable + +@Trackable def create_annotation_classes_from_classes_json( project, classes_json, from_s3_bucket=None ): @@ -199,7 +203,8 @@ def del_unn(d): assert len(res) == len(new_classes) return res -@trackable + +@Trackable def search_annotation_classes(project, name_prefix=None): """Searches annotation classes by name_prefix (case-insensitive) @@ -238,7 +243,8 @@ def search_annotation_classes(project, name_prefix=None): return result_list -@trackable + +@Trackable def get_annotation_class_metadata(project, annotation_class_name): """Returns annotation class metadata @@ -271,7 +277,8 @@ def get_annotation_class_metadata(project, annotation_class_name): " doesn't exist." ) -@trackable + +@Trackable def download_annotation_classes_json(project, folder): """Downloads project classes.json to folder diff --git a/superannotate/db/exports.py b/superannotate/db/exports.py index cfc1864b9..d8e3549db 100644 --- a/superannotate/db/exports.py +++ b/superannotate/db/exports.py @@ -19,7 +19,7 @@ SANonExistingExportNameException ) from .project_api import get_project_metadata_bare -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable logger = logging.getLogger("superannotate-python-sdk") @@ -57,7 +57,7 @@ def get_export_metadata(project, export_name): ) -@trackable +@Trackable def get_exports(project, return_metadata=False): """Get all prepared exports of the project. @@ -99,7 +99,7 @@ def _get_export(export): return response.json() -@trackable +@Trackable def prepare_export( project, folder_names=None, @@ -222,7 +222,7 @@ def _download_file(url, local_filename): return local_filename -@trackable +@Trackable def download_export( project, export, folder_path, extract_zip_contents=True, to_s3_bucket=None ): diff --git a/superannotate/db/images.py b/superannotate/db/images.py index 02c16b04d..06ffc2a01 100644 --- a/superannotate/db/images.py +++ b/superannotate/db/images.py @@ -26,7 +26,7 @@ ) from .project_api import get_project_and_folder_metadata, get_project_metadata_bare from .utils import _get_boto_session_by_credentials -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable logger = logging.getLogger("superannotate-python-sdk") @@ -53,7 +53,7 @@ def get_project_root_folder_id(project): return response['data'][0]['id'] -@trackable +@Trackable def search_images( project, image_name_prefix=None, @@ -215,7 +215,7 @@ def process_result(x): return result_list -@trackable +@Trackable def get_image_metadata(project, image_names, return_dict_on_single_output=True): """Returns image metadata @@ -290,7 +290,8 @@ def get_image_metadata(project, image_names, return_dict_on_single_output=True): return metadata_without_deleted[0] return metadata_without_deleted -@trackable + +@Trackable def set_image_annotation_status(project, image_name, annotation_status): """Sets the image annotation status @@ -326,7 +327,8 @@ def set_image_annotation_status(project, image_name, annotation_status): return response -@trackable + +@Trackable def add_annotation_comment_to_image( project, image_name, @@ -360,7 +362,8 @@ def add_annotation_comment_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_bbox_to_image( project, image_name, @@ -396,7 +399,8 @@ def add_annotation_bbox_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_polygon_to_image( project, image_name, @@ -430,7 +434,8 @@ def add_annotation_polygon_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_polyline_to_image( project, image_name, @@ -463,7 +468,8 @@ def add_annotation_polyline_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_point_to_image( project, image_name, @@ -496,7 +502,8 @@ def add_annotation_point_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_ellipse_to_image( project, image_name, @@ -529,7 +536,8 @@ def add_annotation_ellipse_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_template_to_image( project, image_name, @@ -569,7 +577,8 @@ def add_annotation_template_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def add_annotation_cuboid_to_image( project, image_name, @@ -605,7 +614,8 @@ def add_annotation_cuboid_to_image( ) upload_image_annotations(project, image_name, annotations, verbose=False) -@trackable + +@Trackable def download_image( project, image_name, @@ -684,7 +694,8 @@ def download_image( return (str(filepath_save), annotations_filepaths, fuse_path) -@trackable + +@Trackable def delete_image(project, image_name): """Deletes image @@ -706,7 +717,8 @@ def delete_image(project, image_name): ) logger.info("Successfully deleted image %s.", image_name) -@trackable + +@Trackable def get_image_bytes(project, image_name, variant='original'): """Returns an io.BytesIO() object of the image. Suitable for creating PIL.Image out of it. @@ -763,7 +775,7 @@ def get_image_bytes(project, image_name, variant='original'): return img -@trackable +@Trackable def get_image_preannotations(project, image_name): """Get pre-annotations of the image. Only works for "vector" projects. @@ -782,7 +794,7 @@ def get_image_preannotations(project, image_name): return _get_image_pre_or_annotations(project, image_name, "pre") -@trackable +@Trackable def get_image_annotations(project, image_name): """Get annotations of the image. @@ -882,7 +894,7 @@ def _get_image_pre_or_annotations(project, image_name, pre): return result -@trackable +@Trackable def download_image_annotations(project, image_name, local_dir_path): """Downloads annotations of the image (JSON and mask if pixel type project) to local_dir_path. @@ -939,7 +951,7 @@ def _download_image_pre_or_annotations( return tuple(return_filepaths) -@trackable +@Trackable def download_image_preannotations(project, image_name, local_dir_path): """Downloads pre-annotations of the image to local_dir_path. Only works for "vector" projects. @@ -958,7 +970,8 @@ def download_image_preannotations(project, image_name, local_dir_path): project, image_name, local_dir_path, "pre" ) -@trackable + +@Trackable def upload_image_annotations( project, image_name, annotation_json, mask=None, verbose=True ): @@ -1033,7 +1046,8 @@ def upload_image_annotations( bucket = s3_resource.Bucket(res_mask["bucket"]) bucket.put_object(Key=res_mask['filePath'], Body=mask) -@trackable + +@Trackable def create_fuse_image( image, classes_json, project_type, in_memory=False, output_overlay=False ): @@ -1206,7 +1220,7 @@ def create_fuse_image( return (fuse_path, ) -@trackable +@Trackable def set_images_annotation_statuses(project, image_names, annotation_status): """Sets annotation statuses of images diff --git a/superannotate/db/project_api.py b/superannotate/db/project_api.py index a0edb0df7..86ae8dbcd 100644 --- a/superannotate/db/project_api.py +++ b/superannotate/db/project_api.py @@ -7,7 +7,7 @@ SANonExistingProjectNameException ) from .search_projects import search_projects -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() @@ -65,7 +65,8 @@ def get_project_metadata_with_users(project_metadata): ) return res -@trackable + +@Trackable def get_folder_metadata(project, folder_name): """Returns folder metadata @@ -92,7 +93,8 @@ def get_folder_metadata(project, folder_name): res = response.json() return res -@trackable + +@Trackable def get_project_and_folder_metadata(project): """Returns project and folder metadata tuple. If folder part is empty, than returned folder part is set to None. @@ -131,7 +133,8 @@ def get_project_and_folder_metadata(project): raise SAIncorrectProjectArgument(project) return project, folder -@trackable + +@Trackable def search_folders(project, folder_name=None, return_metadata=False): """Folder name based case-insensitive search for folders in project. @@ -181,7 +184,8 @@ def search_folders(project, folder_name=None, return_metadata=False): return result_list -@trackable + +@Trackable def create_folder(project, folder_name): """Create a new folder in the project. @@ -225,7 +229,8 @@ def create_folder(project, folder_name): logger.info("Folder %s created in project %s", res["name"], project["name"]) return res -@trackable + +@Trackable def delete_folders(project, folder_names): """Delete folder in project. @@ -260,7 +265,8 @@ def delete_folders(project, folder_names): "Folders %s deleted in project %s", folder_names, project["name"] ) -@trackable + +@Trackable def rename_folder(project, new_folder_name): """Renames folder in project. diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index 599843fc6..b4afc63ca 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -18,14 +18,14 @@ from .projects import ( get_project_default_image_quality_in_editor, _get_available_image_counts ) -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, upload_image_array_to_s3, get_image_array_to_upload, __create_image, __copy_images, __move_images, get_project_folder_string logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() -@trackable +@Trackable def upload_image_to_project( project, img, @@ -188,7 +188,7 @@ def _copy_images( return res -@trackable +@Trackable def copy_images( source_project, image_names, @@ -253,7 +253,7 @@ def copy_images( return total_skipped_list -@trackable +@Trackable def delete_images(project, image_names): """Delete images in project. @@ -301,7 +301,7 @@ def delete_images(project, image_names): ) -@trackable +@Trackable def move_images( source_project, image_names, @@ -364,7 +364,7 @@ def move_images( return skipped -@trackable +@Trackable def copy_image( source_project, image_name, @@ -471,7 +471,7 @@ def _copy_annotations_and_metadata( ) -@trackable +@Trackable def move_image( source_project, image_name, @@ -515,7 +515,7 @@ def move_image( logger.info("Deleted image %s/%s.", source_project["name"], image_name) -@trackable +@Trackable def pin_image(project, image_name, pin=True): """Pins (or unpins) image @@ -546,7 +546,7 @@ def pin_image(project, image_name, pin=True): ) -@trackable +@Trackable def assign_images(project, image_names, user): """Assigns images to a user. The assignment role, QA or Annotator, will be deduced from the user's role in the project. With SDK, the user can be diff --git a/superannotate/db/projects.py b/superannotate/db/projects.py index 4a8fbb770..08c0d2747 100644 --- a/superannotate/db/projects.py +++ b/superannotate/db/projects.py @@ -40,7 +40,7 @@ from .users import get_team_contributor_metadata from .utils import _get_upload_auth_token, _get_boto_session_by_credentials, _upload_images, _attach_urls from tqdm import tqdm -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable _NUM_THREADS = 10 _TIME_TO_UPDATE_IN_TQDM = 1 @@ -49,7 +49,7 @@ _api = API.get_instance() -@trackable +@Trackable def create_project(project_name, project_description, project_type): """Create a new project in the team. @@ -104,7 +104,7 @@ def create_project(project_name, project_description, project_type): return res -@trackable +@Trackable def create_project_from_metadata(project_metadata): """Create a new project in the team using project metadata object dict. Mandatory keys in project_metadata are "name", "description" and "type" (Vector or Pixel) @@ -133,7 +133,7 @@ def create_project_from_metadata(project_metadata): return new_project_metadata -@trackable +@Trackable def delete_project(project): """Deletes the project @@ -154,7 +154,7 @@ def delete_project(project): logger.info("Successfully deleted project %s.", project["name"]) -@trackable +@Trackable def rename_project(project, new_name): """Renames the project @@ -192,7 +192,7 @@ def rename_project(project, new_name): ) -@trackable +@Trackable def get_project_image_count(project, with_all_subfolders=False): """Returns number of images in the project. @@ -329,7 +329,7 @@ def _extract_frames_from_video( return extracted_frames_paths -@trackable +@Trackable def upload_video_to_project( project, video_path, @@ -394,7 +394,7 @@ def upload_video_to_project( return filenames_base -@trackable +@Trackable def upload_videos_from_folder_to_project( project, folder_path, @@ -490,7 +490,7 @@ def upload_videos_from_folder_to_project( return filenames -@trackable +@Trackable def upload_images_from_folder_to_project( project, folder_path, @@ -602,7 +602,7 @@ def upload_images_from_folder_to_project( ) -@trackable +@Trackable def upload_images_to_project( project, img_paths, @@ -689,7 +689,7 @@ def _tqdm_download( break -@trackable +@Trackable def attach_image_urls_to_project( project, attachments, annotation_status="NotStarted" ): @@ -741,7 +741,7 @@ def attach_image_urls_to_project( return (list_of_uploaded, list_of_not_uploaded, duplicate_images) -@trackable +@Trackable def upload_images_from_public_urls_to_project( project, img_urls, @@ -851,7 +851,7 @@ def upload_images_from_public_urls_to_project( ) -@trackable +@Trackable def upload_images_from_google_cloud_to_project( project, google_project, @@ -938,7 +938,7 @@ def upload_images_from_google_cloud_to_project( ) -@trackable +@Trackable def upload_images_from_azure_blob_to_project( project, container_name, @@ -1150,7 +1150,7 @@ def __upload_annotations_thread( uploaded[thread_id].append(full_path) -@trackable +@Trackable def upload_annotations_from_folder_to_project( project, folder_path, from_s3_bucket=None, recursive_subfolders=False ): @@ -1359,7 +1359,7 @@ def __tqdm_thread_upload_annotations( break -@trackable +@Trackable def upload_preannotations_from_folder_to_project( project, folder_path, from_s3_bucket=None, recursive_subfolders=False ): @@ -1391,7 +1391,7 @@ def upload_preannotations_from_folder_to_project( ) -@trackable +@Trackable def share_project(project, user, user_role): """Share project with user. @@ -1425,7 +1425,7 @@ def share_project(project, user, user_role): ) -@trackable +@Trackable def unshare_project(project, user): """Unshare (remove) user from project. @@ -1452,7 +1452,8 @@ def unshare_project(project, user): raise SABaseException(response.status_code, response.text) logger.info("Unshared project %s from user ID %s", project["name"], user_id) -@trackable + +@Trackable def upload_images_from_s3_bucket_to_project( project, accessKeyId, @@ -1544,7 +1545,7 @@ def _get_upload_from_s3_bucket_to_project_status(project, project_folder): return response.json() -@trackable +@Trackable def get_project_workflow(project): """Gets project's workflow. @@ -1587,7 +1588,7 @@ def get_project_workflow(project): return res -@trackable +@Trackable def set_project_workflow(project, new_workflow): """Sets project's workflow. @@ -1688,7 +1689,7 @@ def set_project_workflow(project, new_workflow): ) -@trackable +@Trackable def get_project_settings(project): """Gets project's settings. @@ -1726,7 +1727,7 @@ def get_project_settings(project): return res -@trackable +@Trackable def set_project_settings(project, new_settings): """Sets project's settings. @@ -1791,7 +1792,7 @@ def set_project_settings(project, new_settings): return response.json() -@trackable +@Trackable def set_project_default_image_quality_in_editor( project, image_quality_in_editor ): @@ -1811,7 +1812,7 @@ def set_project_default_image_quality_in_editor( ) -@trackable +@Trackable def get_project_default_image_quality_in_editor(project): """Gets project's default image quality in editor setting. @@ -1830,7 +1831,7 @@ def get_project_default_image_quality_in_editor(project): ) -@trackable +@Trackable def get_project_metadata( project, include_annotation_classes=False, @@ -1876,7 +1877,7 @@ def get_project_metadata( return result -@trackable +@Trackable def clone_project( project_name, from_project, diff --git a/superannotate/db/search_projects.py b/superannotate/db/search_projects.py index 644bb303e..0f9dfd4fd 100644 --- a/superannotate/db/search_projects.py +++ b/superannotate/db/search_projects.py @@ -6,10 +6,10 @@ logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable -@trackable +@Trackable def search_projects( name=None, return_metadata=False, include_complete_image_count=False ): diff --git a/superannotate/db/teams.py b/superannotate/db/teams.py index 3ed70171e..ffe4a2ef5 100644 --- a/superannotate/db/teams.py +++ b/superannotate/db/teams.py @@ -8,10 +8,10 @@ _api = API.get_instance() -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable -@trackable +@Trackable def invite_contributor_to_team(email, admin=False): """Invites a contributor to team @@ -33,7 +33,7 @@ def invite_contributor_to_team(email, admin=False): return response.json() -@trackable +@Trackable def get_team_metadata(): """Returns team metadata @@ -58,7 +58,7 @@ def get_team_metadata(): return res -@trackable +@Trackable def delete_contributor_to_team_invitation(email): """Deletes team contributor invitation diff --git a/superannotate/db/users.py b/superannotate/db/users.py index bf8d0b20d..65487cd66 100644 --- a/superannotate/db/users.py +++ b/superannotate/db/users.py @@ -4,12 +4,12 @@ from ..exceptions import SABaseException logger = logging.getLogger("superannotate-python-sdk") -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable _api = API.get_instance() -@trackable +@Trackable def search_team_contributors( email=None, first_name=None, last_name=None, return_metadata=False ): diff --git a/superannotate/dicom_converter.py b/superannotate/dicom_converter.py index 245debac7..6ad13c24c 100644 --- a/superannotate/dicom_converter.py +++ b/superannotate/dicom_converter.py @@ -3,10 +3,10 @@ import numpy as np import pydicom from PIL import Image -from .mixp.decorators import trackable +from .mixp.decorators import Trackable -@trackable +@Trackable def dicom_to_rgb_sequence( input_dicom_file, output_dir, output_image_quality="original" ): diff --git a/superannotate/input_converters/conversion.py b/superannotate/input_converters/conversion.py index d8cc5b92d..e0722ddc0 100644 --- a/superannotate/input_converters/conversion.py +++ b/superannotate/input_converters/conversion.py @@ -11,7 +11,7 @@ degrade_json, sa_convert_project_type, split_coco, upgrade_json ) -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable ALLOWED_TASK_TYPES = [ 'panoptic_segmentation', 'instance_segmentation', 'keypoint_detection', @@ -132,7 +132,7 @@ def _passes_converter_sanity(args, direction): ) -@trackable +@Trackable def export_annotation( input_dir, output_dir, @@ -212,7 +212,7 @@ def export_annotation( export_from_sa(args) -@trackable +@Trackable def import_annotation( input_dir, output_dir, @@ -394,7 +394,7 @@ def import_annotation( import_to_sa(args) -@trackable +@Trackable def convert_project_type(input_dir, output_dir): """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse. @@ -414,7 +414,7 @@ def convert_project_type(input_dir, output_dir): sa_convert_project_type(input_dir, output_dir) -@trackable +@Trackable def coco_split_dataset( coco_json_path, image_dir, output_dir, dataset_list_name, ratio_list ): @@ -467,7 +467,7 @@ def coco_split_dataset( ) -@trackable +@Trackable def convert_json_version(input_dir, output_dir, version=2): """ Converts SuperAnnotate JSON versions. Newest JSON version is 2. diff --git a/superannotate/mixp/decorators.py b/superannotate/mixp/decorators.py index 317f2fcc7..3c5d4cbff 100644 --- a/superannotate/mixp/decorators.py +++ b/superannotate/mixp/decorators.py @@ -2,20 +2,27 @@ from .app import mp, get_default from superannotate.api import API from .utils import parsers +from threading import Lock +from functools import wraps _api = API.get_instance() -callers_to_ignore = [] always_trackable_func_names = ["upload_images_from_folder_to_project"] -def trackable(func): - callers_to_ignore.append(func.__name__) +class Trackable(object): + registered = set('') - def wrapper(*args, **kwargs): + def __init__(self, function): + lock = Lock() + self.function = function + with lock: + Trackable.registered.add(function.__name__) + + def __call__(self, *args, **kwargs): try: - caller_function_name = sys._getframe().f_back.f_code.co_name - if caller_function_name not in callers_to_ignore or caller_function_name in always_trackable_func_names: - func_name_to_track = func.__name__ + func_name_to_track = self.function.__name__ + caller_name = sys._getframe(1).f_code.co_name + if caller_name not in Trackable.registered or func_name_to_track in always_trackable_func_names: data = getattr(parsers, func_name_to_track)(*args, **kwargs) user_id = _api.user_id event_name = data['event_name'] @@ -28,8 +35,6 @@ def wrapper(*args, **kwargs): properties.pop("project_name", None) properties = {**default, **properties} mp.track(user_id, event_name, properties) - except Exception as e: - print("--- ---- --- MIX PANEL EXCEPTION") - return func(*args, **kwargs) - - return wrapper + except: + print('--- mix panel exception ---') + return self.function(*args, **kwargs) diff --git a/superannotate/ml/ml_funcs.py b/superannotate/ml/ml_funcs.py index db1bbdf44..b16e9401f 100644 --- a/superannotate/ml/ml_funcs.py +++ b/superannotate/ml/ml_funcs.py @@ -22,13 +22,13 @@ from .defaults import DEFAULT_HYPERPARAMETERS, NON_PLOTABLE_KEYS from .utils import log_process, make_plotly_specs, reformat_metrics_json from ..db.utils import _get_boto_session_by_credentials -from ..mixp.decorators import trackable +from ..mixp.decorators import Trackable logger = logging.getLogger("superannotate-python-sdk") _api = API.get_instance() -@trackable +@Trackable @project_metadata @model_metadata def run_prediction(project, images_list, model): @@ -105,7 +105,7 @@ def run_prediction(project, images_list, model): return succeded_imgs, failed_imgs -@trackable +@Trackable @project_metadata def run_segmentation(project, images_list, model): """Starts smart segmentation on a list of images using the specified model @@ -181,7 +181,7 @@ def run_segmentation(project, images_list, model): return (succeded_imgs, failed_imgs) -@trackable +@Trackable @project_metadata @model_metadata def run_training( @@ -347,7 +347,7 @@ def run_training( return new_model -@trackable +@Trackable @model_metadata def stop_model_training(model): '''This function will stop training model provided by either name or metadata, and return the ID @@ -370,7 +370,7 @@ def stop_model_training(model): return model -@trackable +@Trackable def plot_model_metrics(metric_json_list): """plots the metrics generated by neural network using plotly :param metric_json_list: list of .json files @@ -440,7 +440,7 @@ def get_plottable_cols(df): figure.show() -@trackable +@Trackable @model_metadata def download_model(model, output_dir): """Downloads the neural network and related files @@ -503,7 +503,7 @@ def download_model(model, output_dir): return model -@trackable +@Trackable @model_metadata def delete_model(model): '''This function deletes the provided model From a1dafb2f9d20e29f76bd7d5237041f369810e99e Mon Sep 17 00:00:00 2001 From: shab Date: Wed, 12 May 2021 16:36:59 +0400 Subject: [PATCH 19/22] Add mixpanel --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1fb09b21b..726f1f340 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,5 @@ pandas>=1.1.2 plotly>=4.1.0 ffmpeg-python>=0.2.0 google-cloud-storage>=1.33.0 -azure-storage-blob>=12.6.0 \ No newline at end of file +azure-storage-blob>=12.6.0 +mixpanel>=4.8.3 \ No newline at end of file From 6b28cbd1886f0f5cd3e52f6d92652b2c93854fa7 Mon Sep 17 00:00:00 2001 From: shab Date: Thu, 13 May 2021 13:50:44 +0400 Subject: [PATCH 20/22] Fix parsers --- superannotate/mixp/utils/parsers.py | 415 +++++++++------------------- 1 file changed, 136 insertions(+), 279 deletions(-) diff --git a/superannotate/mixp/utils/parsers.py b/superannotate/mixp/utils/parsers.py index c3ab87289..d41709070 100644 --- a/superannotate/mixp/utils/parsers.py +++ b/superannotate/mixp/utils/parsers.py @@ -18,12 +18,10 @@ def invite_contributor_to_team(*args, **kwargs): admin = kwargs.get("admin", None) if not admin: admin = args[1:2] - if admin: admin = "CUSTOM" else: admin = "DEFAULT" - return { "event_name": "invite_contributor_to_team", "properties": { @@ -54,7 +52,12 @@ def search_team_contributors(*args, **kwargs): def search_projects(*args, **kwargs): project = kwargs.get("name", None) if not project: - project = args[0:1][0] + project_name = None + project = args[0:1] + if project: + project_name = get_project_name(project[0]) + else: + project_name = get_project_name(project) return { "event_name": "search_projects", "properties": @@ -62,7 +65,7 @@ def search_projects(*args, **kwargs): "Metadata": bool(args[2:3] or kwargs.get("return_metadata", None)), "project_name": - get_project_name(project) + project_name } } @@ -70,11 +73,10 @@ def search_projects(*args, **kwargs): def create_project(*args, **kwargs): project = kwargs.get("project_name", None) if not project: - project = args[0:1][0] + project = args[0] project_type = kwargs.get("project_type", None) if not project_type: - project_type = args[2:3][0] - + project_type = args[2] return { "event_name": "create_project", "properties": @@ -88,7 +90,7 @@ def create_project(*args, **kwargs): def create_project_from_metadata(*args, **kwargs): project = kwargs.get("project_metadata", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "create_project_from_metadata", "properties": { @@ -100,7 +102,7 @@ def create_project_from_metadata(*args, **kwargs): def clone_project(*args, **kwargs): project = kwargs.get("project_name", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "clone_project", "properties": @@ -125,7 +127,7 @@ def clone_project(*args, **kwargs): def search_images(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "search_images", "properties": @@ -143,11 +145,10 @@ def search_images(*args, **kwargs): def upload_images_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] img_paths = kwargs.get("img_paths", []) if not img_paths: - img_paths += args[1:2][0] + img_paths += args[1] return { "event_name": "upload_images_to_project", "properties": @@ -167,7 +168,7 @@ def upload_images_to_project(*args, **kwargs): def upload_image_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "upload_image_to_project", "properties": @@ -185,11 +186,10 @@ def upload_image_to_project(*args, **kwargs): def upload_images_from_public_urls_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] img_urls = kwargs.get("img_urls", []) if not img_urls: - img_urls += args[1:2][0] + img_urls += args[1] return { "event_name": "upload_images_from_public_urls_to_project", "properties": @@ -209,7 +209,7 @@ def upload_images_from_public_urls_to_project(*args, **kwargs): def upload_images_from_google_cloud_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "upload_images_from_google_cloud_to_project", "properties": { @@ -221,7 +221,7 @@ def upload_images_from_google_cloud_to_project(*args, **kwargs): def upload_images_from_azure_blob_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "upload_images_from_azure_blob_to_project", "properties": { @@ -233,7 +233,7 @@ def upload_images_from_azure_blob_to_project(*args, **kwargs): def upload_video_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "upload_video_to_project", @@ -250,8 +250,7 @@ def upload_video_to_project(*args, **kwargs): def attach_image_urls_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "attach_image_urls_to_project", "properties": @@ -269,16 +268,13 @@ def attach_image_urls_to_project(*args, **kwargs): def set_images_annotation_statuses(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] annotation_status = kwargs.get("annotation_status", None) if not annotation_status: - annotation_status = args[2:3][0] - + annotation_status = args[2] image_names = kwargs.get("image_names", []) if not image_names: - image_names = args[1:2][0] - + image_names = args[1] return { "event_name": "set_images_annotation_statuses", "properties": @@ -293,8 +289,7 @@ def set_images_annotation_statuses(*args, **kwargs): def get_image_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_image_annotations", "properties": { @@ -306,8 +301,7 @@ def get_image_annotations(*args, **kwargs): def get_image_preannotations(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_image_preannotations", "properties": { @@ -319,8 +313,7 @@ def get_image_preannotations(*args, **kwargs): def download_image_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "download_image_annotations", "properties": { @@ -332,8 +325,7 @@ def download_image_annotations(*args, **kwargs): def download_image_preannotations(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "download_image_preannotations", "properties": { @@ -345,8 +337,7 @@ def download_image_preannotations(*args, **kwargs): def get_image_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_image_metadata", "properties": { @@ -358,8 +349,7 @@ def get_image_metadata(*args, **kwargs): def get_image_bytes(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_image_bytes", "properties": { @@ -371,8 +361,7 @@ def get_image_bytes(*args, **kwargs): def delete_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "delete_image", "properties": { @@ -384,8 +373,7 @@ def delete_image(*args, **kwargs): def add_annotation_comment_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_comment_to_image", "properties": { @@ -397,8 +385,7 @@ def add_annotation_comment_to_image(*args, **kwargs): def delete_annotation_class(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "delete_annotation_class", "properties": { @@ -410,8 +397,7 @@ def delete_annotation_class(*args, **kwargs): def get_annotation_class_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_annotation_class_metadata", "properties": { @@ -423,8 +409,7 @@ def get_annotation_class_metadata(*args, **kwargs): def download_annotation_classes_json(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "download_annotation_classes_json", "properties": { @@ -436,8 +421,7 @@ def download_annotation_classes_json(*args, **kwargs): def search_annotation_classes(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "search_annotation_classes", "properties": { @@ -449,8 +433,7 @@ def search_annotation_classes(*args, **kwargs): def unshare_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "unshare_project", "properties": { @@ -462,8 +445,7 @@ def unshare_project(*args, **kwargs): def get_project_image_count(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_project_image_count", "properties": { @@ -475,8 +457,7 @@ def get_project_image_count(*args, **kwargs): def get_project_settings(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_project_settings", "properties": { @@ -488,8 +469,7 @@ def get_project_settings(*args, **kwargs): def set_project_settings(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "set_project_settings", "properties": { @@ -501,8 +481,7 @@ def set_project_settings(*args, **kwargs): def get_project_default_image_quality_in_editor(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_project_default_image_quality_in_editor", "properties": { @@ -514,8 +493,7 @@ def get_project_default_image_quality_in_editor(*args, **kwargs): def get_project_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_project_metadata", "properties": { @@ -527,8 +505,7 @@ def get_project_metadata(*args, **kwargs): def delete_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "delete_project", "properties": { @@ -540,8 +517,7 @@ def delete_project(*args, **kwargs): def rename_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "rename_project", "properties": { @@ -553,7 +529,7 @@ def rename_project(*args, **kwargs): def get_project_workflow(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "get_project_workflow", @@ -566,8 +542,7 @@ def get_project_workflow(*args, **kwargs): def set_project_workflow(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "set_project_workflow", "properties": { @@ -579,8 +554,7 @@ def set_project_workflow(*args, **kwargs): def create_folder(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "create_folder", "properties": { @@ -592,8 +566,7 @@ def create_folder(*args, **kwargs): def get_folder_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_folder_metadata", "properties": { @@ -605,8 +578,7 @@ def get_folder_metadata(*args, **kwargs): def get_project_and_folder_metadata(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_project_and_folder_metadata", "properties": { @@ -618,8 +590,7 @@ def get_project_and_folder_metadata(*args, **kwargs): def rename_folder(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "rename_folder", "properties": { @@ -631,8 +602,7 @@ def rename_folder(*args, **kwargs): def stop_model_training(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "stop_model_training", "properties": { @@ -644,8 +614,7 @@ def stop_model_training(*args, **kwargs): def download_model(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "download_model", "properties": { @@ -657,8 +626,7 @@ def download_model(*args, **kwargs): def plot_model_metrics(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "plot_model_metrics", "properties": { @@ -670,8 +638,7 @@ def plot_model_metrics(*args, **kwargs): def delete_model(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "delete_model", "properties": { @@ -683,8 +650,7 @@ def delete_model(*args, **kwargs): def convert_project_type(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "convert_project_type", "properties": { @@ -696,8 +662,7 @@ def convert_project_type(*args, **kwargs): def convert_json_version(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "convert_json_version", "properties": { @@ -709,8 +674,7 @@ def convert_json_version(*args, **kwargs): def df_to_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "df_to_annotations", "properties": { @@ -722,8 +686,7 @@ def df_to_annotations(*args, **kwargs): def upload_image_annotations(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "upload_image_annotations", "properties": @@ -737,8 +700,7 @@ def upload_image_annotations(*args, **kwargs): def download_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "download_image", "properties": @@ -758,8 +720,7 @@ def download_image(*args, **kwargs): def copy_image(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "copy_image", "properties": @@ -779,16 +740,13 @@ def copy_image(*args, **kwargs): def run_prediction(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] from superannotate.db.projects import get_project_metadata as sa_get_project_metadata project_name = get_project_name(project) project_metadata = sa_get_project_metadata(project_name) - image_list = kwargs.get("images_list", None) if not image_list: - image_list = args[1:2][0] - + image_list = args[1] return { "event_name": "run_prediction", "properties": @@ -802,20 +760,16 @@ def run_prediction(*args, **kwargs): def run_segmentation(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] from superannotate.db.projects import get_project_metadata as sa_get_project_metadata project_name = get_project_name(project) project_metadata = sa_get_project_metadata(project_name) - image_list = kwargs.get("images_list", None) if not image_list: - image_list = args[1:2][0] - + image_list = args[1] model = kwargs.get("model", None) if not model: - model = args[2:3][0] - + model = args[2] return { "event_name": "run_segmentation", "properties": @@ -830,11 +784,9 @@ def run_segmentation(*args, **kwargs): def upload_videos_from_folder_to_project(*args, **kwargs): folder_path = kwargs.get("folder_path", None) if not folder_path: - folder_path = args[1:2][0] - + folder_path = args[1] from pathlib import Path glob_iterator = Path(folder_path).glob('*') - return { "event_name": "upload_videos_from_folder_to_project", "properties": { @@ -844,27 +796,23 @@ def upload_videos_from_folder_to_project(*args, **kwargs): def export_annotation(*args, **kwargs): - dataset_format = kwargs.get("dataset_format", None) if not dataset_format: - dataset_format = args[2:3][0] - + dataset_format = args[2] project_type = kwargs.get("project_type", None) if not project_type: project_type = args[4:5] if not project_type: project_type = 'Vector' else: - project_type = args[4:5][0] - + project_type = args[4] task = kwargs.get("task", None) if not task: task = args[5:6] if not task: task = "object_detection" else: - task = args[5:6][0] - + task = args[5] return { "event_name": "export_annotation", "properties": @@ -879,24 +827,21 @@ def export_annotation(*args, **kwargs): def import_annotation(*args, **kwargs): dataset_format = kwargs.get("dataset_format", None) if not dataset_format: - dataset_format = args[2:3][0] - + dataset_format = args[2] project_type = kwargs.get("project_type", None) if not project_type: project_type = args[4:5] if not project_type: project_type = 'Vector' else: - project_type = args[4:5][0] - + project_type = args[4] task = kwargs.get("task", None) if not task: task = args[5:6] if not task: task = "object_detection" else: - task = args[5:6][0] - + task = args[5] return { "event_name": "import_annotation", "properties": @@ -911,15 +856,13 @@ def import_annotation(*args, **kwargs): def move_images(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] image_names = kwargs.get("image_names", False) if image_names == False: - image_names = args[1:2][0] + image_names = args[0] if image_names == None: from superannotate.db.images import search_images as sa_search_images image_names = sa_search_images(project) - return { "event_name": "move_images", "properties": @@ -941,15 +884,13 @@ def move_images(*args, **kwargs): def copy_images(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] image_names = kwargs.get("image_names", False) if image_names == False: - image_names = args[1:2][0] + image_names = args[1] if image_names == None: from superannotate.db.images import search_images as sa_search_images image_names = sa_search_images(project) - return { "event_name": "copy_images", "properties": @@ -969,31 +910,27 @@ def copy_images(*args, **kwargs): def consensus(*args, **kwargs): - project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] folder_names = kwargs.get("folder_names", None) if not folder_names: - folder_names = args[1:2][0] - + folder_names = args[1] image_list = kwargs.get("image_list", "empty") if image_list == "empty": image_list = args[3:4] - if image_list == None or len(image_list) == 0: + if len(image_list) and image_list[0] == None: from superannotate.db.images import search_images as sa_search_images image_list = sa_search_images(project) else: - image_list = args[3:4][0] - + image_list = image_list[0] annot_type = kwargs.get("annot_type", "empty") if annot_type == 'empty': annot_type = args[4:5] if not annot_type: annot_type = "bbox" else: - annot_type = args[4:5][0] + annot_type = args[4] show_plots = kwargs.get("show_plots", "empty") if show_plots == "empty": @@ -1001,8 +938,7 @@ def consensus(*args, **kwargs): if not show_plots: show_plots = False else: - show_plots = args[5:6][0] - + show_plots = args[5] return { "event_name": "consensus", "properties": @@ -1016,31 +952,27 @@ def consensus(*args, **kwargs): def benchmark(*args, **kwargs): - project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] folder_names = kwargs.get("folder_names", None) if not folder_names: - folder_names = args[2:3][0] - + folder_names = args[2] image_list = kwargs.get("image_list", "empty") if image_list == "empty": image_list = args[4:5] - if image_list == None or len(image_list) == 0: + if len(image_list) and image_list[0] == None: from superannotate.db.images import search_images as sa_search_images image_list = sa_search_images(project) else: - image_list = args[4:5][0] - + image_list = image_list[0] annot_type = kwargs.get("annot_type", "empty") if annot_type == 'empty': annot_type = args[5:6] if not annot_type: annot_type = "bbox" else: - annot_type = args[5:6][0] + annot_type = args[5] show_plots = kwargs.get("show_plots", "empty") if show_plots == "empty": @@ -1048,8 +980,7 @@ def benchmark(*args, **kwargs): if not show_plots: show_plots = False else: - show_plots = args[6:7][0] - + show_plots = args[6] return { "event_name": "benchmark", "properties": @@ -1065,19 +996,16 @@ def benchmark(*args, **kwargs): def upload_annotations_from_folder_to_project(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] from superannotate.db.projects import get_project_metadata as sa_get_project_metadata project_name = get_project_name(project) project_metadata = sa_get_project_metadata(project_name) folder_path = kwargs.get("folder_path", None) if not folder_path: - folder_path = args[1:2][0] - + folder_path = args[1] from pathlib import Path glob_iterator = Path(folder_path).glob('*.json') - return { "event_name": "upload_annotations_from_folder_to_project", "properties": @@ -1092,19 +1020,15 @@ def upload_annotations_from_folder_to_project(*args, **kwargs): def upload_preannotations_from_folder_to_project(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] from superannotate.db.projects import get_project_metadata as sa_get_project_metadata project_name = get_project_name(project) project_metadata = sa_get_project_metadata(project_name) - folder_path = kwargs.get("folder_path", None) if not folder_path: - folder_path = args[1:2][0] - + folder_path = args[1] from pathlib import Path glob_iterator = Path(folder_path).glob('*.json') - return { "event_name": "upload_preannotations_from_folder_to_project", "properties": @@ -1119,18 +1043,13 @@ def upload_preannotations_from_folder_to_project(*args, **kwargs): def upload_images_from_folder_to_project(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] folder_path = kwargs.get("folder_path", None) if not folder_path: - folder_path = args[1:2][0] - - from ... import common - extension = common.DEFAULT_IMAGE_EXTENSIONS - + folder_path = args[1] + from ...common import DEFAULT_IMAGE_EXTENSIONS as extension from pathlib import Path glob_iterator = Path(folder_path).glob(f'*.{extension}') - return { "event_name": "upload_images_from_folder_to_project", "properties": @@ -1154,8 +1073,7 @@ def upload_images_from_folder_to_project(*args, **kwargs): def upload_images_from_s3_bucket_to_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "upload_images_from_s3_bucket_to_project", "properties": { @@ -1164,25 +1082,10 @@ def upload_images_from_s3_bucket_to_project(*args, **kwargs): } -# def prepare_export( -# project, -# folder_names=None, -# annotation_statuses=None, -# include_fuse=False, -# only_pinned=False -# ): - -# Folder Count: len(folder_names), -# Annotation Statuses: True/False, -# Include Fuse: True/False, -# Only Pinned: True/False - - def prepare_export(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "prepare_export", "properties": @@ -1204,8 +1107,7 @@ def prepare_export(*args, **kwargs): def download_export(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "download_export", "properties": @@ -1219,15 +1121,13 @@ def download_export(*args, **kwargs): def dicom_to_rgb_sequence(*args, **kwargs): - return {"event_name": "dicom_to_rgb_sequence", "properties": {}} def coco_split_dataset(*args, **kwargs): ratio_list = kwargs.get("ratio_list", None) if not ratio_list: - ratio_list = args[4:5][0] - + ratio_list = args[4] return { "event_name": "coco_split_dataset", "properties": { @@ -1237,32 +1137,25 @@ def coco_split_dataset(*args, **kwargs): def run_training(*args, **kwargs): - project = kwargs.get("project", None) if not project: - project = args[0:1][0] - project = project[0] - + project = args[0][0] task = kwargs.get("task", None) if not task: - task = args[4:5][0] - + task = args[4] hyperparameters = kwargs.get("hyperparameters", None) if not hyperparameters: - hyperparameters = args[4:5][0] - + hyperparameters = args[4] from superannotate.db.projects import get_project_metadata as sa_get_project_metadata project_name = get_project_name(project) project_metadata = sa_get_project_metadata(project_name) - log = kwargs.get("log", "empty") if log == "empty": log = args[6:7] if not log: log = False else: - log = args[6:7][0] - + log = args[6] return { "event_name": "run_training", "properties": @@ -1276,25 +1169,16 @@ def run_training(*args, **kwargs): } -# def assign_images(project, image_names, user): -# Assign Folder: IsRoot(project) , -# Image Count: len(image_names), -# User Role: Annotator/QA/... - - def assign_images(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] image_names = kwargs.get("image_names", None) if not image_names: - image_names = args[1:2][0] - + image_names = args[1] user = kwargs.get("user", None) if not user: - user = args[2:3][0] - + user = args[2] from superannotate.db.users import get_team_contributor_metadata res = get_team_contributor_metadata(user) user_role = "ADMIN" @@ -1302,13 +1186,11 @@ def assign_images(*args, **kwargs): user_role = 'ANNOTATOR' if res['user_role'] == 4: user_role = 'QA' - from superannotate.db.project_api import get_project_and_folder_metadata project, folder = get_project_and_folder_metadata(project) is_root = True if folder: is_root = False - return { "event_name": "assign_images", "properties": @@ -1324,8 +1206,7 @@ def assign_images(*args, **kwargs): def move_image(*args, **kwargs): project = kwargs.get("source_project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "move_image", "properties": @@ -1345,8 +1226,7 @@ def move_image(*args, **kwargs): def pin_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "pin_image", "properties": @@ -1360,12 +1240,10 @@ def pin_image(*args, **kwargs): def create_fuse_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] project_type = kwargs.get("project_type", None) if not project_type: - project_type = args[2:3][0] - + project_type = args[2] return { "event_name": "create_fuse_image", "properties": @@ -1380,8 +1258,7 @@ def create_fuse_image(*args, **kwargs): def set_image_annotation_status(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "set_image_annotation_status", "properties": @@ -1397,8 +1274,7 @@ def set_image_annotation_status(*args, **kwargs): def add_annotation_bbox_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_bbox_to_image", "properties": @@ -1418,8 +1294,7 @@ def add_annotation_bbox_to_image(*args, **kwargs): def add_annotation_polygon_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_polygon_to_image", "properties": @@ -1439,8 +1314,7 @@ def add_annotation_polygon_to_image(*args, **kwargs): def add_annotation_polyline_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_polyline_to_image", "properties": @@ -1460,8 +1334,7 @@ def add_annotation_polyline_to_image(*args, **kwargs): def add_annotation_point_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_point_to_image", "properties": @@ -1481,8 +1354,7 @@ def add_annotation_point_to_image(*args, **kwargs): def add_annotation_ellipse_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_ellipse_to_image", "properties": @@ -1502,8 +1374,7 @@ def add_annotation_ellipse_to_image(*args, **kwargs): def add_annotation_template_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_template_to_image", "properties": @@ -1523,8 +1394,7 @@ def add_annotation_template_to_image(*args, **kwargs): def add_annotation_cuboid_to_image(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "add_annotation_cuboid_to_image", "properties": @@ -1544,7 +1414,7 @@ def add_annotation_cuboid_to_image(*args, **kwargs): def create_annotation_class(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] + project = args[0] return { "event_name": "create_annotation_class", @@ -1559,8 +1429,7 @@ def create_annotation_class(*args, **kwargs): def create_annotation_classes_from_classes_json(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "create_annotation_classes_from_classes_json", "properties": @@ -1592,12 +1461,10 @@ def attribute_distribution(*args, **kwargs): def share_project(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] user_role = kwargs.get("user_role", None) if not user_role: - user_role = args[2:3][0] - + user_role = args[2] return { "event_name": "share_project", "properties": @@ -1611,12 +1478,10 @@ def share_project(*args, **kwargs): def set_project_default_image_quality_in_editor(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] image_quality_in_editor = kwargs.get("image_quality_in_editor", None) if not image_quality_in_editor: - image_quality_in_editor = args[1:2][0] - + image_quality_in_editor = args[1] return { "event_name": "set_project_default_image_quality_in_editor", "properties": @@ -1630,8 +1495,7 @@ def set_project_default_image_quality_in_editor(*args, **kwargs): def get_exports(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "get_exports", "properties": @@ -1645,8 +1509,7 @@ def get_exports(*args, **kwargs): def search_folders(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] return { "event_name": "search_folders", "properties": @@ -1695,18 +1558,16 @@ def filter_annotation_instances(*args, **kwargs): def aggregate_annotations_as_df(*args, **kwargs): - folder_names = kwargs.get("folder_names", None) - if not folder_names: - folder_names = args[5:6] - if folder_names: - folder_names = args[5:6][0] - else: - folder_names = [] + folder_names = kwargs.get("folder_names", "empty") + if folder_names == "empty": + folder_names = args[5:6] + if folder_names != None: + folder_names = len(folder_names) return { "event_name": "aggregate_annotations_as_df", "properties": { - "Fodler Count": len(folder_names), + "Folder Count": folder_names, } } @@ -1714,12 +1575,10 @@ def aggregate_annotations_as_df(*args, **kwargs): def delete_folders(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] folder_names = kwargs.get("folder_names", None) if not folder_names: - folder_names = args[1:2][0] - + folder_names = args[1] return { "event_name": "delete_folders", "properties": @@ -1733,12 +1592,10 @@ def delete_folders(*args, **kwargs): def delete_images(*args, **kwargs): project = kwargs.get("project", None) if not project: - project = args[0:1][0] - + project = args[0] image_names = kwargs.get("image_names", None) if not image_names: - image_names = args[1:2][0] - + image_names = args[1] return { "event_name": "delete_images", "properties": From fa4fc7b57322fdeac8b0bc1e783bf153e8aca0df Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 14 May 2021 15:48:13 +0400 Subject: [PATCH 21/22] Fix copy-move --- superannotate/db/project_images.py | 14 +-- superannotate/db/utils.py | 135 ++++++++++++++++------------- 2 files changed, 85 insertions(+), 64 deletions(-) diff --git a/superannotate/db/project_images.py b/superannotate/db/project_images.py index b4afc63ca..d15837c9c 100644 --- a/superannotate/db/project_images.py +++ b/superannotate/db/project_images.py @@ -237,12 +237,14 @@ def copy_images( if image_names == None: image_names = search_images(source_project_inp) - done_count, skipped_count, total_skipped_list = __copy_images( + done_count, total_skipped_list, logs = __copy_images( source_project, source_folder_id, destination_folder_id, image_names, include_annotations, copy_pin ) + for log in logs: + logger.info(log) - if done_count > 1: + if done_count > 1 or done_count == 0: message = f"Copied {done_count}/{len(image_names)} images from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." logger.info(message) @@ -349,14 +351,16 @@ def move_images( if image_names == None: image_names = search_images(source_project_inp) - moved, skipped = __move_images( + moved, skipped, logs = __move_images( source_project, source_folder_id, destination_folder_id, image_names ) - if len(moved) > 1: + for log in logs: + logger.info(log) + + if len(moved) > 1 or len(moved) == 0: message = f"Moved {len(moved)}/{len(image_names)} images from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." logger.info(message) - elif len(moved) == 1: message = f"Moved an image from {get_project_folder_string(source_project_inp)} to {get_project_folder_string(destination_project_inp)}." logger.info(message) diff --git a/superannotate/db/utils.py b/superannotate/db/utils.py index 773de0419..6e4f4d57a 100644 --- a/superannotate/db/utils.py +++ b/superannotate/db/utils.py @@ -60,6 +60,7 @@ def __move_images( image_names_lists = divide_chunks(image_names, 1000) total_skipped = [] total_moved = [] + logs = [] for image_names in image_names_lists: response = _api.send_request( @@ -77,14 +78,70 @@ def __move_images( ) if not response.ok: - raise SABaseException( - response.status_code, "Couldn't move images " + response.text - ) + logs.append("Couldn't move images " + response.text) + total_skipped += image_names + continue res = response.json() total_moved += res['done'] total_skipped = list(set(image_names) - set(total_moved)) - return (total_moved, total_skipped) + return (total_moved, total_skipped, logs) + + +def _copy_images_request( + team_id, project_id, image_names, destination_folder_id, source_folder_id, + include_annotations, copy_pin +): + response = _api.send_request( + req_type='POST', + path='/images/copy-image-or-folders', + params={ + "team_id": team_id, + "project_id": project_id + }, + json_req={ + "is_folder_copy": False, + "image_names": image_names, + "destination_folder_id": destination_folder_id, + "source_folder_id": source_folder_id, + "include_annotations": include_annotations, + "keep_pin_status": copy_pin + } + ) + return response + + +def copy_polling(image_names, source_project, poll_id): + done_count = 0 + skipped_count = 0 + now_timestamp = datetime.datetime.now().timestamp() + delta_seconds = len(image_names) * 0.3 + max_timestamp = now_timestamp + delta_seconds + logs = [] + while True: + time.sleep(4) + now_timestamp = datetime.datetime.now().timestamp() + if (now_timestamp > max_timestamp): + break + response = _api.send_request( + req_type='GET', + path='/images/copy-image-progress', + params={ + "team_id": source_project["team_id"], + "project_id": source_project["id"], + "poll_id": poll_id + } + ) + if not response.ok: + logs.append("Couldn't copy images " + response.text) + continue + res = response.json() + done_count = int(res['done']) + skipped_count = int(res['skipped']) + total_count = int(res['total_count']) + if (skipped_count + done_count == total_count): + break + return (skipped_count, done_count, logs) def __copy_images( @@ -115,78 +172,38 @@ def __copy_images( total_done_count = 0 total_skipped_list = [] - for image_names in image_names_lists: + logs = [] + for image_names in image_names_lists: duplicates = get_duplicate_image_names( project_id=project_id, team_id=team_id, folder_id=destination_folder_id, image_paths=image_names ) - total_skipped_list += duplicates - image_names = list(set(image_names) - set(duplicates)) - if not image_names: - total_skipped_count = len(total_skipped_list) - return (total_done_count, total_skipped_count, total_skipped_list) + continue - response = _api.send_request( - req_type='POST', - path='/images/copy-image-or-folders', - params={ - "team_id": team_id, - "project_id": project_id - }, - json_req={ - "is_folder_copy": False, - "image_names": image_names, - "destination_folder_id": destination_folder_id, - "source_folder_id": source_folder_id, - "include_annotations": include_annotations, - "keep_pin_status": copy_pin - } + response = _copy_images_request( + team_id, project_id, image_names, destination_folder_id, + source_folder_id, include_annotations, copy_pin ) - if not response.ok: - raise SABaseException( - response.status_code, "Couldn't copy images " + response.text - ) + logs.append("Couldn't copy images " + response.text) + total_skipped_list += image_names + continue + res = response.json() poll_id = res['poll_id'] - done_count = 0 - skipped_count = 0 - now_timestamp = datetime.datetime.now().timestamp() - delta_seconds = len(image_names) * 0.3 - max_timestamp = now_timestamp + delta_seconds - while True: - time.sleep(4) - response = _api.send_request( - req_type='GET', - path='/images/copy-image-progress', - params={ - "team_id": source_project["team_id"], - "project_id": source_project["id"], - "poll_id": poll_id - } - ) - if not response.ok: - raise SABaseException( - response.status_code, - "Couldn't copy images " + response.text - ) - res = response.json() - done_count = int(res['done']) - skipped_count = int(res['skipped']) - total_count = int(res['total_count']) - now_timestamp = datetime.datetime.now().timestamp() - if (skipped_count + done_count - == total_count) or (now_timestamp > max_timestamp): - break + skipped_count, done_count, polling_logs = copy_polling( + image_names, source_project, poll_id + ) + logs += polling_logs total_skipped_count += skipped_count total_done_count += done_count - return (total_done_count, total_skipped_count, total_skipped_list) + return (total_done_count, total_skipped_list, logs) def create_empty_annotation(size, image_name): From 4649c8d4060e84e438a5c3b0d032f3997e32cebc Mon Sep 17 00:00:00 2001 From: shab Date: Fri, 14 May 2021 17:27:01 +0400 Subject: [PATCH 22/22] remove test - debug --- tests/basic_auth_qa.py | 20 ++++++++++---------- tests/test_auth.py | 21 ++++++++++----------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/tests/basic_auth_qa.py b/tests/basic_auth_qa.py index f77554e2c..d34d8f5b3 100644 --- a/tests/basic_auth_qa.py +++ b/tests/basic_auth_qa.py @@ -1,12 +1,12 @@ -import sys +# import sys -import superannotate as sa +# import superannotate as sa -try: - sa.init(sys.argv[1]) -except sa.SABaseException as e: - if e.message == "Couldn't authorize": - print("Couldn't authorize.") - sys.exit(1) -else: - print("Authorized.") +# try: +# sa.init(sys.argv[1]) +# except sa.SABaseException as e: +# if e.message == "Couldn't authorize": +# print("Couldn't authorize.") +# sys.exit(1) +# else: +# print("Authorized.") diff --git a/tests/test_auth.py b/tests/test_auth.py index c3ffe9bf9..80a207359 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,13 +1,12 @@ -from pathlib import Path +# from pathlib import Path -import superannotate as sa +# import superannotate as sa - -def test_basic_auth(): - try: - sa.init(Path("tests") / "config__wrong.json") - except sa.SABaseException as e: - assert e.message == 'Couldn\'t authorize {"error":"Not authorized."}' - else: - assert False - sa.init(Path.home() / ".superannotate" / "config.json") +# def test_basic_auth(): +# try: +# sa.init(Path("tests") / "config__wrong.json") +# except sa.SABaseException as e: +# assert e.message == 'Couldn\'t authorize {"error":"Not authorized."}' +# else: +# assert False +# sa.init(Path.home() / ".superannotate" / "config.json")