Skip to content
Merged

Tags #37

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions sdk/diffgram/core/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,23 @@ def set_default_directory(self,
self.session.headers.update(
{'directory_id': str(self.directory_id)})



def new_schema(self,
name: str):

endpoint = "/api/v1/project/" + self.project_string_id + \
"/labels-schema/new"

request_json_body = {'name': name}

response = self.session.post(self.host + endpoint,
json = request_json_body)

self.handle_errors(response)
return response.json()


# TODO review not using this pattern anymore

setattr(Project, "get_label_file_dict", get_label_file_dict)
Expand Down
2 changes: 2 additions & 0 deletions sdk/diffgram/core/directory.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ def convert_json_to_sdk_object(self, directory_list_json):
)
refresh_from_dict(new_directory, directory_json)

new_directory.id = new_directory.directory_id

# note timing issue, this needs to happen after id is refreshed
new_directory.init_files()
new_directory.start_iterator(
Expand Down
19 changes: 10 additions & 9 deletions sdk/diffgram/file/file_constructor.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def file_from_response(
def from_local(
self,
path: str,
directory_id: int = None,
instance_list: list = None,
frame_packet_map: dict = None,
assume_new_instances_machine_made: bool = True,
Expand All @@ -54,35 +55,35 @@ def from_local(

files = {'file': (os.path.basename(path), open(path, 'rb'), 'application/octet-stream')}

headers = {
'immediate_mode': 'True',
}
json_payload = {}

if directory_id is None:
directory_id = self.client.directory_id

payload = {}
json_payload['directory_id'] = directory_id

if instance_list:
payload['instance_list'] = self.__validate_and_format_instance_list(
json_payload['instance_list'] = self.__validate_and_format_instance_list(
instance_list = instance_list,
assume_new_instances_machine_made = assume_new_instances_machine_made,
convert_names_to_label_files = convert_names_to_label_files
)

if frame_packet_map:
payload['frame_packet_map'] = self.__validate_and_format_frame_packet_map(
json_payload['frame_packet_map'] = self.__validate_and_format_frame_packet_map(
frame_packet_map = frame_packet_map,
assume_new_instances_machine_made = assume_new_instances_machine_made,
convert_names_to_label_files = convert_names_to_label_files
)

files['json'] = (None, json.dumps(payload), 'application/json')
files['json'] = (None, json.dumps(json_payload), 'application/json')

endpoint = "/api/walrus/v1/project/" + self.client.project_string_id \
+ "/input/from_local"

response = self.client.session.post(
self.client.host + endpoint,
files = files,
headers = headers)
files = files)

self.client.handle_errors(response)

Expand Down
72 changes: 8 additions & 64 deletions sdk/diffgram/job/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,11 @@ def serialize(self):
'review_by_human_freqeuncy': self.review_by_human_freqeuncy,
'label_mode': self.label_mode,
'passes_per_file': self.passes_per_file,
'file_count': self.file_count,
'attached_directories_dict': self.attached_directories_dict,
'launch_datetime': self.launch_datetime,
'label_file_list': label_file_list,
'member_list_ids': self.member_list_ids
'member_list_ids': self.member_list_ids,
'tag_list': self.tag_list
}

def new(self,
Expand All @@ -150,15 +150,13 @@ def new(self,
review_by_human_freqeuncy=None,
label_mode=None,
passes_per_file=None,
file_list=None,
guide=None,
launch_datetime=None,
file_count=None,
label_file_list=None,
sync_directories=[],
single_copy_directories=[],
members_list_ids = [],
auto_launch=True,
tag_list = [],
):
"""

Expand All @@ -178,6 +176,10 @@ def new(self,
raise ValueError('Please provide at least one member_id in members_list_ids.')
job = Job(client=self.client)

if label_schema_id is None:
if self.client.label_schema_list:
label_schema_id = self.client.label_schema_list[0].get('id')

job.name = name
job.instance_type = instance_type
job.share = share
Expand All @@ -189,10 +191,10 @@ def new(self,
job.label_mode = label_mode
job.passes_per_file = passes_per_file
job.launch_datetime = launch_datetime
job.label_file_list = label_file_list
job.label_schema_id = label_schema_id
job.attached_directories = []
job.member_list_ids = members_list_ids
job.tag_list = tag_list

if len(sync_directories) == 0 and len(single_copy_directories) == 0:
raise ValueError(
Expand All @@ -206,12 +208,6 @@ def new(self,

job.attached_directories_dict = {'attached_directories_list': self.attached_directories}

if not file_count:
if file_list:
file_count = len(file_list)

job.file_count = file_count

endpoint = "/api/v1/project/{}/job/new".format(self.client.project_string_id)

response = self.client.session.post(
Expand All @@ -226,12 +222,6 @@ def new(self,
# TODO review better way to update fields
job.id = data["job"]["id"]

if file_list:
# Careful we want to call job here not self
# Since job will have a different id
# self is constructor
job.file_update(file_list=file_list)

if guide:
job.guide_update(guide=guide)

Expand All @@ -248,53 +238,7 @@ def new(self,

return job

def file_update(
self,
file_list,
add_or_remove="add"
):
"""

Arguments
self,
file_list, list of files,
add_or_remove, either "add" or "remove"

Expects

Returns

Assumptions

The API will use the project default if None is supplied
but if we are not in the default we must supply valid
directory_id

Otherwise when it checks permissions it will error
ie {"file_link":"File link not in incoming directory"}

"""

endpoint = "/api/v1/project/" + self.client.project_string_id + \
"/job/file/attach"

file_list = [file.serialize() for file in file_list]

update_dict = {
'directory_id': self.client.directory_id,
'file_list_selected': file_list,
'job_id': self.id,
'add_or_remove': add_or_remove}

response = self.client.session.post(self.client.host + endpoint,
json=update_dict)

self.client.handle_errors(response)

data = response.json()

if data["log"]["success"] == True:
print("File update success")

def launch(
self
Expand Down