From 9cb3a77f81f7894b550eb84880a84fb17fd939e8 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 27 Sep 2021 16:04:40 -0600 Subject: [PATCH 1/8] wip: File3d class for point clouds upload --- sdk/diffgram/file/file_3d.py | 84 ++++++++++++++++++++++++++++ sdk/diffgram/file/test_pcd_upload.py | 22 ++++++++ 2 files changed, 106 insertions(+) create mode 100644 sdk/diffgram/file/file_3d.py create mode 100644 sdk/diffgram/file/test_pcd_upload.py diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py new file mode 100644 index 0000000..9cd96d3 --- /dev/null +++ b/sdk/diffgram/file/file_3d.py @@ -0,0 +1,84 @@ +from diffgram.core.core import Project +import json +import os +import requests +import io +import math +import uuid + +class File3D: + point_list: list + client: Project + + def __init__(self, client, point_list = []): + self.client = client + self.point_list = point_list + + def add_point(self, x: float, y: float, z: float): + self.point_list.append({ + 'x': x, + 'y': y, + 'z': z + }) + return self.point_list + + def upload(self, dataset_name = None): + """ + Builds a JSON file from current point list and uploads it to + Diffgram. + :return: + """ + file_data = { + 'point_list': self.point_list + } + headers = { + 'content-type': 'multipart/form-data', + + } + json_data = json.dumps(file_data) + + endpoint = "/api/walrus/v1/project/{}/upload/large".format( + self.client.project_string_id + ) + # chunk_size = 5000000 # 5 MB chunks + chunk_size = 1000000 # 1 MB chunks + dataset_id = self.client.default_directory['id'] + if dataset_name is not None: + dataset_id = self.client.directory.get(dataset_name).id + + with io.StringIO(json_data) as s: + pos = s.tell() + s.seek(0, os.SEEK_END) + file_size = s.tell() + print('FILE SIZE', file_size) + s.seek(pos) + + # requests.post('http://some.url/streamed', data = f) + num_chunks = int(math.ceil(file_size / chunk_size)) + last_chunk_size = -1 + if file_size % chunk_size != 0: + last_chunk_size = file_size % chunk_size + + print('Num chunks', num_chunks) + print('chunk_size', chunk_size) + print('last_chunk_size', last_chunk_size) + uid_upload = str(uuid.uuid4()) + for i in range(0, num_chunks): + # print('File Size: {} bytes.'.format(file_size)) + payload = { + 'dzuid': uid_upload, + 'dzchunkindex': i, + 'dztotalfilesize': file_size, + 'dzchunksize': chunk_size, + 'dztotalchunkcount': num_chunks, + 'dzchunkbyteoffset': i * chunk_size, + 'directory_id': dataset_id, + 'source': 'api', + 'file': 'binary data here', + } + + if i == (num_chunks - 1) and last_chunk_size != -1: + # Read last chunk size here... + payload['dzchunksize'] = last_chunk_size + print(payload) + return file_size diff --git a/sdk/diffgram/file/test_pcd_upload.py b/sdk/diffgram/file/test_pcd_upload.py new file mode 100644 index 0000000..7b8a696 --- /dev/null +++ b/sdk/diffgram/file/test_pcd_upload.py @@ -0,0 +1,22 @@ + +import numpy as np +import open3d as o3d +from diffgram.file.file_3d import File3D +from diffgram.core.core import Project + +pcd = o3d.io.read_point_cloud("/home/pablo/Downloads/lidar_ascii_v5.pcd") +out_arr = np.asarray(pcd.points) + +project = Project( project_string_id = "glorybiter", + debug= True, + client_id = "LIVE__6bn2dsiitc4vbmnwlihz", + client_secret = "kzwlrg20pdvjzfutz2oitx2aakqvf0216j201pv47i53nw8v52x1unqtjt3h") + +diffgram_3d_file = File3D(client = project) + +for point in out_arr: + diffgram_3d_file.add_point( + x = point[0], + y = point[1], + z = point[2], + ) \ No newline at end of file From ab15e8ccf7cb7ecb361d15b13a15e43d850bf3c5 Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 28 Sep 2021 11:47:03 -0600 Subject: [PATCH 2/8] wip: sensor fusion JSON upload --- sdk/diffgram/file/file_3d.py | 47 ++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py index 9cd96d3..3956ba8 100644 --- a/sdk/diffgram/file/file_3d.py +++ b/sdk/diffgram/file/file_3d.py @@ -6,6 +6,7 @@ import math import uuid + class File3D: point_list: list client: Project @@ -22,26 +23,23 @@ def add_point(self, x: float, y: float, z: float): }) return self.point_list - def upload(self, dataset_name = None): + def upload(self, dataset_name = None, chunk_size = 5000000): """ Builds a JSON file from current point list and uploads it to Diffgram. + :param dataset_name: + :param chunk_size: Size of each chunk of the JSON file. Default is 5MB :return: """ file_data = { 'point_list': self.point_list } - headers = { - 'content-type': 'multipart/form-data', - - } json_data = json.dumps(file_data) - endpoint = "/api/walrus/v1/project/{}/upload/large".format( + endpoint = "/api/walrus/project/{}/upload/large".format( self.client.project_string_id ) - # chunk_size = 5000000 # 5 MB chunks - chunk_size = 1000000 # 1 MB chunks + chunk_size = 5000000 # 5 MB chunks dataset_id = self.client.default_directory['id'] if dataset_name is not None: dataset_id = self.client.directory.get(dataset_name).id @@ -49,36 +47,43 @@ def upload(self, dataset_name = None): with io.StringIO(json_data) as s: pos = s.tell() s.seek(0, os.SEEK_END) - file_size = s.tell() - print('FILE SIZE', file_size) + file_size = s.tell() s.seek(pos) - - # requests.post('http://some.url/streamed', data = f) num_chunks = int(math.ceil(file_size / chunk_size)) last_chunk_size = -1 if file_size % chunk_size != 0: last_chunk_size = file_size % chunk_size - print('Num chunks', num_chunks) - print('chunk_size', chunk_size) - print('last_chunk_size', last_chunk_size) uid_upload = str(uuid.uuid4()) for i in range(0, num_chunks): # print('File Size: {} bytes.'.format(file_size)) + payload = { - 'dzuid': uid_upload, + 'dzuuid': uid_upload, 'dzchunkindex': i, 'dztotalfilesize': file_size, 'dzchunksize': chunk_size, 'dztotalchunkcount': num_chunks, 'dzchunkbyteoffset': i * chunk_size, 'directory_id': dataset_id, - 'source': 'api', - 'file': 'binary data here', + 'source': 'from_sensor_fusion_json', } - + # Adjust final chunk size if i == (num_chunks - 1) and last_chunk_size != -1: # Read last chunk size here... payload['dzchunksize'] = last_chunk_size - print(payload) - return file_size + + # Read file Chunk + s.seek(payload['dzchunkbyteoffset']) + file_chunk = s.read(payload['dzchunksize']) + files = {'file': ('{}_sensor_fusion_file.json'.format(uid_upload), file_chunk)} + # Make request to server + url = self.client.host + endpoint + response = self.client.session.post(url, + data = payload, + files = files) + + self.client.handle_errors(response) + # print(payload) + + return True From c68067f67ae1e5ac61c56d620e0171ba0b9e2d97 Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 28 Sep 2021 12:38:01 -0600 Subject: [PATCH 3/8] wip: improve performance by decoupling animate from render --- sdk/diffgram/file/file_3d.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py index 3956ba8..1de08a7 100644 --- a/sdk/diffgram/file/file_3d.py +++ b/sdk/diffgram/file/file_3d.py @@ -84,6 +84,5 @@ def upload(self, dataset_name = None, chunk_size = 5000000): files = files) self.client.handle_errors(response) - # print(payload) return True From 5c70facc6d59ab0d35c6428047258a67bc97a448 Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 28 Sep 2021 16:03:59 -0600 Subject: [PATCH 4/8] wip: support adding intensity and device ID values --- sdk/diffgram/file/file_3d.py | 19 +++++++++++++++++-- sdk/diffgram/file/test_pcd_upload.py | 24 +++++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py index 1de08a7..aae335d 100644 --- a/sdk/diffgram/file/file_3d.py +++ b/sdk/diffgram/file/file_3d.py @@ -15,11 +15,26 @@ def __init__(self, client, point_list = []): self.client = client self.point_list = point_list - def add_point(self, x: float, y: float, z: float): + def add_point(self, + x: float, + y: float, + z: float, + intensity = None, + device_id = None, + timestamp = None, + is_ground = False): + + if intensity is not None and intensity > 1.0 or intensity < 0: + raise Exception('Intensity point must be between 0 and 1. Value is: {}'.format(intensity)) + self.point_list.append({ 'x': x, 'y': y, - 'z': z + 'z': z, + 'intensity': intensity, + 'device_id': device_id, + 'timestamp': timestamp, + 'is_ground': is_ground, }) return self.point_list diff --git a/sdk/diffgram/file/test_pcd_upload.py b/sdk/diffgram/file/test_pcd_upload.py index 7b8a696..56573ec 100644 --- a/sdk/diffgram/file/test_pcd_upload.py +++ b/sdk/diffgram/file/test_pcd_upload.py @@ -4,8 +4,7 @@ from diffgram.file.file_3d import File3D from diffgram.core.core import Project -pcd = o3d.io.read_point_cloud("/home/pablo/Downloads/lidar_ascii_v5.pcd") -out_arr = np.asarray(pcd.points) + project = Project( project_string_id = "glorybiter", debug= True, @@ -14,9 +13,28 @@ diffgram_3d_file = File3D(client = project) -for point in out_arr: +out_arr = [] +with open('/home/pablo/Downloads/lidar_ascii_v5.pcd') as f: + lines = f.readlines() + is_on_points = False + for line in lines: + if not is_on_points: + if line.startswith('DATA'): + is_on_points = True + else: + data = line.split(' ') + row = [] + for elm in data: + row.append(float(elm)) + row[3] = min((row[3] / 100, 1.0)) + out_arr.append(row) + +for i in range(0, len(out_arr)): + point = out_arr[i] + color = out_arr[i] diffgram_3d_file.add_point( x = point[0], y = point[1], z = point[2], + intensity = point[3] ) \ No newline at end of file From a06567c3e8f095ba05579620eb32190940a594ee Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 1 Oct 2021 13:28:26 -0600 Subject: [PATCH 5/8] fix: original filename generation --- sdk/diffgram/file/file_3d.py | 11 ++-- sdk/diffgram/file/test_pcd_upload.py | 94 +++++++++++++++++----------- 2 files changed, 66 insertions(+), 39 deletions(-) diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py index aae335d..e1db9fa 100644 --- a/sdk/diffgram/file/file_3d.py +++ b/sdk/diffgram/file/file_3d.py @@ -11,8 +11,9 @@ class File3D: point_list: list client: Project - def __init__(self, client, point_list = []): + def __init__(self, client, name, point_list = []): self.client = client + self.original_filename = name self.point_list = point_list def add_point(self, @@ -24,8 +25,9 @@ def add_point(self, timestamp = None, is_ground = False): - if intensity is not None and intensity > 1.0 or intensity < 0: - raise Exception('Intensity point must be between 0 and 1. Value is: {}'.format(intensity)) + if intensity is not None: + if intensity > 1.0 or intensity < 0: + raise Exception('Intensity point must be between 0 and 1. Value is: {}'.format(intensity)) self.point_list.append({ 'x': x, @@ -75,6 +77,7 @@ def upload(self, dataset_name = None, chunk_size = 5000000): payload = { 'dzuuid': uid_upload, + 'original_filename': self.original_filename, 'dzchunkindex': i, 'dztotalfilesize': file_size, 'dzchunksize': chunk_size, @@ -91,7 +94,7 @@ def upload(self, dataset_name = None, chunk_size = 5000000): # Read file Chunk s.seek(payload['dzchunkbyteoffset']) file_chunk = s.read(payload['dzchunksize']) - files = {'file': ('{}_sensor_fusion_file.json'.format(uid_upload), file_chunk)} + files = {'file': ('{}_sf.json'.format(self.original_filename), file_chunk)} # Make request to server url = self.client.host + endpoint response = self.client.session.post(url, diff --git a/sdk/diffgram/file/test_pcd_upload.py b/sdk/diffgram/file/test_pcd_upload.py index 56573ec..c0ec97f 100644 --- a/sdk/diffgram/file/test_pcd_upload.py +++ b/sdk/diffgram/file/test_pcd_upload.py @@ -1,40 +1,64 @@ - -import numpy as np -import open3d as o3d from diffgram.file.file_3d import File3D from diffgram.core.core import Project +import numpy as np +import open3d as o3d + + +def read_pcd_o3d(file_name): + pcd = o3d.io.read_point_cloud(file_name) + out_arr = np.asarray(pcd.points) + return out_arr + + +project = Project(project_string_id = "glorybiter", + debug = True, + client_id = "LIVE__6bn2dsiitc4vbmnwlihz", + client_secret = "kzwlrg20pdvjzfutz2oitx2aakqvf0216j201pv47i53nw8v52x1unqtjt3h") + + +def upload_test_file1(): + diffgram_3d_file = File3D(client = project, name = 'lidar_ascii_v5.pcd') + + points_arr = [] + with open('/home/pablo/Downloads/lidar_ascii_v5.pcd') as f: + lines = f.readlines() + is_on_points = False + for line in lines: + if not is_on_points: + if line.startswith('DATA'): + is_on_points = True + else: + data = line.split(' ') + row = [] + for elm in data: + row.append(float(elm)) + row[3] = min((row[3] / 100, 1.0)) + points_arr.append(row) + + for i in range(0, len(points_arr)): + point = points_arr[i] + diffgram_3d_file.add_point( + x = point[0], + y = point[1], + z = point[2], + intensity = point[3] + ) + diffgram_3d_file.upload() + + +# File 2 test +def upload_test_file2(): + points_arr = read_pcd_o3d('/home/pablo/Downloads/model.pcd') + diffgram_3d_file = File3D(client = project, name = 'face.pcd') + for i in range(0, len(points_arr)): + point = points_arr[i] + diffgram_3d_file.add_point( + x = point[0], + y = point[1], + z = point[2], + ) -project = Project( project_string_id = "glorybiter", - debug= True, - client_id = "LIVE__6bn2dsiitc4vbmnwlihz", - client_secret = "kzwlrg20pdvjzfutz2oitx2aakqvf0216j201pv47i53nw8v52x1unqtjt3h") - -diffgram_3d_file = File3D(client = project) - -out_arr = [] -with open('/home/pablo/Downloads/lidar_ascii_v5.pcd') as f: - lines = f.readlines() - is_on_points = False - for line in lines: - if not is_on_points: - if line.startswith('DATA'): - is_on_points = True - else: - data = line.split(' ') - row = [] - for elm in data: - row.append(float(elm)) - row[3] = min((row[3] / 100, 1.0)) - out_arr.append(row) - -for i in range(0, len(out_arr)): - point = out_arr[i] - color = out_arr[i] - diffgram_3d_file.add_point( - x = point[0], - y = point[1], - z = point[2], - intensity = point[3] - ) \ No newline at end of file + print('num points: {}'.format(len(diffgram_3d_file.point_list))) + diffgram_3d_file.upload() From c6c4236f0c94d6e478238d69a627c2813db52b97 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 18 Nov 2021 09:33:54 -0600 Subject: [PATCH 6/8] wip --- sdk/diffgram/file/test_pcd_upload.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/diffgram/file/test_pcd_upload.py b/sdk/diffgram/file/test_pcd_upload.py index c0ec97f..9a31342 100644 --- a/sdk/diffgram/file/test_pcd_upload.py +++ b/sdk/diffgram/file/test_pcd_upload.py @@ -10,10 +10,10 @@ def read_pcd_o3d(file_name): return out_arr -project = Project(project_string_id = "glorybiter", - debug = True, - client_id = "LIVE__6bn2dsiitc4vbmnwlihz", - client_secret = "kzwlrg20pdvjzfutz2oitx2aakqvf0216j201pv47i53nw8v52x1unqtjt3h") +project = Project(project_string_id = "forestfox", + client_id = "LIVE__hyibdfom3kqia8ks6jcp", + client_secret = "zx0gzw4yznimtfg4x4gdyqoc0sq6t6d73uc6dvkys7f07k9mvt9mzffm101m", + debug = True) def upload_test_file1(): @@ -48,7 +48,7 @@ def upload_test_file1(): # File 2 test def upload_test_file2(): - points_arr = read_pcd_o3d('/home/pablo/Downloads/model.pcd') + points_arr = read_pcd_o3d('/home/pablo/Downloads/Zaghetto.pcd') diffgram_3d_file = File3D(client = project, name = 'face.pcd') From 18deb53b6523e821607e9d269febb3f24a0f011f Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 30 Nov 2021 11:08:33 -0600 Subject: [PATCH 7/8] fix: add checks for empty point cloud list --- sdk/diffgram/file/file_3d.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py index e1db9fa..e4887e1 100644 --- a/sdk/diffgram/file/file_3d.py +++ b/sdk/diffgram/file/file_3d.py @@ -48,6 +48,10 @@ def upload(self, dataset_name = None, chunk_size = 5000000): :param chunk_size: Size of each chunk of the JSON file. Default is 5MB :return: """ + + if len(self.point_list) == 0: + print('At least 1 point should be provided in the point list. Please add point using add_point()') + return False file_data = { 'point_list': self.point_list } From 7c4fbddda46f773806d82d279a29d1cb73652c9a Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 14 Dec 2021 15:13:12 -0600 Subject: [PATCH 8/8] cleanup --- sdk/diffgram/file/file_3d.py | 4 +- sdk/diffgram/file/test_pcd_upload.py | 64 ---------------------------- 2 files changed, 3 insertions(+), 65 deletions(-) delete mode 100644 sdk/diffgram/file/test_pcd_upload.py diff --git a/sdk/diffgram/file/file_3d.py b/sdk/diffgram/file/file_3d.py index e4887e1..37f7b5b 100644 --- a/sdk/diffgram/file/file_3d.py +++ b/sdk/diffgram/file/file_3d.py @@ -12,6 +12,7 @@ class File3D: client: Project def __init__(self, client, name, point_list = []): + self.point_list = [] self.client = client self.original_filename = name self.point_list = point_list @@ -57,6 +58,8 @@ def upload(self, dataset_name = None, chunk_size = 5000000): } json_data = json.dumps(file_data) + with open('data.json', 'w') as outfile: + json.dump(json_data, outfile) endpoint = "/api/walrus/project/{}/upload/large".format( self.client.project_string_id ) @@ -77,7 +80,6 @@ def upload(self, dataset_name = None, chunk_size = 5000000): uid_upload = str(uuid.uuid4()) for i in range(0, num_chunks): - # print('File Size: {} bytes.'.format(file_size)) payload = { 'dzuuid': uid_upload, diff --git a/sdk/diffgram/file/test_pcd_upload.py b/sdk/diffgram/file/test_pcd_upload.py deleted file mode 100644 index 9a31342..0000000 --- a/sdk/diffgram/file/test_pcd_upload.py +++ /dev/null @@ -1,64 +0,0 @@ -from diffgram.file.file_3d import File3D -from diffgram.core.core import Project -import numpy as np -import open3d as o3d - - -def read_pcd_o3d(file_name): - pcd = o3d.io.read_point_cloud(file_name) - out_arr = np.asarray(pcd.points) - return out_arr - - -project = Project(project_string_id = "forestfox", - client_id = "LIVE__hyibdfom3kqia8ks6jcp", - client_secret = "zx0gzw4yznimtfg4x4gdyqoc0sq6t6d73uc6dvkys7f07k9mvt9mzffm101m", - debug = True) - - -def upload_test_file1(): - diffgram_3d_file = File3D(client = project, name = 'lidar_ascii_v5.pcd') - - points_arr = [] - with open('/home/pablo/Downloads/lidar_ascii_v5.pcd') as f: - lines = f.readlines() - is_on_points = False - for line in lines: - if not is_on_points: - if line.startswith('DATA'): - is_on_points = True - else: - data = line.split(' ') - row = [] - for elm in data: - row.append(float(elm)) - row[3] = min((row[3] / 100, 1.0)) - points_arr.append(row) - - for i in range(0, len(points_arr)): - point = points_arr[i] - diffgram_3d_file.add_point( - x = point[0], - y = point[1], - z = point[2], - intensity = point[3] - ) - diffgram_3d_file.upload() - - -# File 2 test -def upload_test_file2(): - points_arr = read_pcd_o3d('/home/pablo/Downloads/Zaghetto.pcd') - - diffgram_3d_file = File3D(client = project, name = 'face.pcd') - - for i in range(0, len(points_arr)): - point = points_arr[i] - diffgram_3d_file.add_point( - x = point[0], - y = point[1], - z = point[2], - ) - - print('num points: {}'.format(len(diffgram_3d_file.point_list))) - diffgram_3d_file.upload()