From fc359ec4e8101d48b445183794f3536fa98fa24d Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:36:48 -0700 Subject: [PATCH 1/8] add new schema future Schema can be a first class thing but for now just ability to do this --- sdk/diffgram/core/core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdk/diffgram/core/core.py b/sdk/diffgram/core/core.py index bbdac5d..855742c 100644 --- a/sdk/diffgram/core/core.py +++ b/sdk/diffgram/core/core.py @@ -335,6 +335,23 @@ def set_default_directory(self, self.session.headers.update( {'directory_id': str(self.directory_id)}) + + + def new_schema(self, + name: str): + + endpoint = "/api/v1/project/" + self.project_string_id + \ + "/labels-schema/new" + + request_json_body = {'name': name} + + response = self.session.post(self.host + endpoint, + json = request_json_body) + + self.handle_errors(response) + return response.json() + + # TODO review not using this pattern anymore setattr(Project, "get_label_file_dict", get_label_file_dict) From 8083b2477c29c08306651788af041e07d0925b59 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Tue, 19 Jul 2022 18:44:25 -0700 Subject: [PATCH 2/8] add sending tag support --- sdk/diffgram/job/job.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/diffgram/job/job.py b/sdk/diffgram/job/job.py index d8f8b92..616492b 100644 --- a/sdk/diffgram/job/job.py +++ b/sdk/diffgram/job/job.py @@ -135,7 +135,8 @@ def serialize(self): 'attached_directories_dict': self.attached_directories_dict, 'launch_datetime': self.launch_datetime, 'label_file_list': label_file_list, - 'member_list_ids': self.member_list_ids + 'member_list_ids': self.member_list_ids, + 'tag_list': self.tag_list } def new(self, @@ -159,6 +160,7 @@ def new(self, single_copy_directories=[], members_list_ids = [], auto_launch=True, + tag_list = [], ): """ @@ -193,6 +195,7 @@ def new(self, job.label_schema_id = label_schema_id job.attached_directories = [] job.member_list_ids = members_list_ids + job.tag_list = tag_list if len(sync_directories) == 0 and len(single_copy_directories) == 0: raise ValueError( From 98ecf30f596b45ba9d8320a4b7317e9ea8d8a0b3 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:14:43 -0700 Subject: [PATCH 3/8] remove legacy file count we expect dataset sync mechanism to generate these numbers --- sdk/diffgram/job/job.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sdk/diffgram/job/job.py b/sdk/diffgram/job/job.py index 616492b..f1c87b5 100644 --- a/sdk/diffgram/job/job.py +++ b/sdk/diffgram/job/job.py @@ -131,7 +131,6 @@ def serialize(self): 'review_by_human_freqeuncy': self.review_by_human_freqeuncy, 'label_mode': self.label_mode, 'passes_per_file': self.passes_per_file, - 'file_count': self.file_count, 'attached_directories_dict': self.attached_directories_dict, 'launch_datetime': self.launch_datetime, 'label_file_list': label_file_list, @@ -154,7 +153,6 @@ def new(self, file_list=None, guide=None, launch_datetime=None, - file_count=None, label_file_list=None, sync_directories=[], single_copy_directories=[], @@ -209,12 +207,6 @@ def new(self, job.attached_directories_dict = {'attached_directories_list': self.attached_directories} - if not file_count: - if file_list: - file_count = len(file_list) - - job.file_count = file_count - endpoint = "/api/v1/project/{}/job/new".format(self.client.project_string_id) response = self.client.session.post( From b6569fbdfdf6d83b230d7c4a11f35083cd080ddd Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:21:21 -0700 Subject: [PATCH 4/8] remove legacy file_list, was replaced with dataset sync --- sdk/diffgram/job/job.py | 53 ----------------------------------------- 1 file changed, 53 deletions(-) diff --git a/sdk/diffgram/job/job.py b/sdk/diffgram/job/job.py index f1c87b5..9e580f4 100644 --- a/sdk/diffgram/job/job.py +++ b/sdk/diffgram/job/job.py @@ -150,7 +150,6 @@ def new(self, review_by_human_freqeuncy=None, label_mode=None, passes_per_file=None, - file_list=None, guide=None, launch_datetime=None, label_file_list=None, @@ -221,12 +220,6 @@ def new(self, # TODO review better way to update fields job.id = data["job"]["id"] - if file_list: - # Careful we want to call job here not self - # Since job will have a different id - # self is constructor - job.file_update(file_list=file_list) - if guide: job.guide_update(guide=guide) @@ -243,53 +236,7 @@ def new(self, return job - def file_update( - self, - file_list, - add_or_remove="add" - ): - """ - - Arguments - self, - file_list, list of files, - add_or_remove, either "add" or "remove" - - Expects - - Returns - - Assumptions - - The API will use the project default if None is supplied - but if we are not in the default we must supply valid - directory_id - - Otherwise when it checks permissions it will error - ie {"file_link":"File link not in incoming directory"} - """ - - endpoint = "/api/v1/project/" + self.client.project_string_id + \ - "/job/file/attach" - - file_list = [file.serialize() for file in file_list] - - update_dict = { - 'directory_id': self.client.directory_id, - 'file_list_selected': file_list, - 'job_id': self.id, - 'add_or_remove': add_or_remove} - - response = self.client.session.post(self.client.host + endpoint, - json=update_dict) - - self.client.handle_errors(response) - - data = response.json() - - if data["log"]["success"] == True: - print("File update success") def launch( self From e3d75da509733c30b7e50dd78a1ba34b9071dd47 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:36:52 -0700 Subject: [PATCH 5/8] Add support to declare directory, move to json for k8s Context that current k8s ingress requires custom headers So these could get ignored And there should be way to pass directory from this method --- sdk/diffgram/file/file_constructor.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sdk/diffgram/file/file_constructor.py b/sdk/diffgram/file/file_constructor.py index 099843b..66e77c5 100644 --- a/sdk/diffgram/file/file_constructor.py +++ b/sdk/diffgram/file/file_constructor.py @@ -39,6 +39,7 @@ def file_from_response( def from_local( self, path: str, + directory_id: int = None, instance_list: list = None, frame_packet_map: dict = None, assume_new_instances_machine_made: bool = True, @@ -54,35 +55,35 @@ def from_local( files = {'file': (os.path.basename(path), open(path, 'rb'), 'application/octet-stream')} - headers = { - 'immediate_mode': 'True', - } + json_payload = {} + + if directory_id is None: + directory_id = self.client.directory_id - payload = {} + json_payload['directory_id'] = directory_id if instance_list: - payload['instance_list'] = self.__validate_and_format_instance_list( + json_payload['instance_list'] = self.__validate_and_format_instance_list( instance_list = instance_list, assume_new_instances_machine_made = assume_new_instances_machine_made, convert_names_to_label_files = convert_names_to_label_files ) if frame_packet_map: - payload['frame_packet_map'] = self.__validate_and_format_frame_packet_map( + json_payload['frame_packet_map'] = self.__validate_and_format_frame_packet_map( frame_packet_map = frame_packet_map, assume_new_instances_machine_made = assume_new_instances_machine_made, convert_names_to_label_files = convert_names_to_label_files ) - files['json'] = (None, json.dumps(payload), 'application/json') + files['json'] = (None, json.dumps(json_payload), 'application/json') endpoint = "/api/walrus/v1/project/" + self.client.project_string_id \ + "/input/from_local" response = self.client.session.post( self.client.host + endpoint, - files = files, - headers = headers) + files = files) self.client.handle_errors(response) From ecdadc4acd633b6742d6362610493b3870d8b8f0 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:52:29 -0700 Subject: [PATCH 6/8] make id available since that's expected having the .directory_id is just a legacy thing as far as I know I don't see any harm in having both and it's more friendly to new person / logical to just do .id --- sdk/diffgram/core/directory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/diffgram/core/directory.py b/sdk/diffgram/core/directory.py index a6c25b3..3a042c6 100644 --- a/sdk/diffgram/core/directory.py +++ b/sdk/diffgram/core/directory.py @@ -70,6 +70,8 @@ def convert_json_to_sdk_object(self, directory_list_json): ) refresh_from_dict(new_directory, directory_json) + new_directory.id = new_directory.directory_id + # note timing issue, this needs to happen after id is refreshed new_directory.init_files() new_directory.start_iterator( From 14eefdd561163bbcb4ba1c1648e91fedc6ae5b17 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:07:48 -0700 Subject: [PATCH 7/8] Add default label schema --- sdk/diffgram/job/job.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/diffgram/job/job.py b/sdk/diffgram/job/job.py index 9e580f4..8b78b05 100644 --- a/sdk/diffgram/job/job.py +++ b/sdk/diffgram/job/job.py @@ -177,6 +177,10 @@ def new(self, raise ValueError('Please provide at least one member_id in members_list_ids.') job = Job(client=self.client) + if label_schema_id is None: + if self.client.label_schema_list: + label_schema_id = self.client.label_schema_list[0].get('id') + job.name = name job.instance_type = instance_type job.share = share From d19b9a4a14c6af3eec20ebe3f617eaddcdc85bf8 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:09:22 -0700 Subject: [PATCH 8/8] remove legacy code, this was replaced with Schema --- sdk/diffgram/job/job.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/diffgram/job/job.py b/sdk/diffgram/job/job.py index 8b78b05..976ef7b 100644 --- a/sdk/diffgram/job/job.py +++ b/sdk/diffgram/job/job.py @@ -152,7 +152,6 @@ def new(self, passes_per_file=None, guide=None, launch_datetime=None, - label_file_list=None, sync_directories=[], single_copy_directories=[], members_list_ids = [], @@ -192,7 +191,6 @@ def new(self, job.label_mode = label_mode job.passes_per_file = passes_per_file job.launch_datetime = launch_datetime - job.label_file_list = label_file_list job.label_schema_id = label_schema_id job.attached_directories = [] job.member_list_ids = members_list_ids