From 92daf09066ad4ddf912da43fe9bc47b8b5e14000 Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Fri, 31 Jan 2020 06:10:10 -0700 Subject: [PATCH 1/7] Ignoring generated calib file and dataset images --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..5309e933a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +dataset/ +depthai.calib From d17fcb4902a6e5190441e9213d229839a07bd359 Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Sun, 2 Feb 2020 07:21:45 -0700 Subject: [PATCH 2/7] Adds a number of modifications to the calibration flow: * Renamed script from calibration_pipeline.py to just calibrate.py * Introduced a number of arguments to execute portiions of the pipeline and only for specified polygon positions * Doesn't throw an error if the datasets/ dir already exists * Defaults to 1 image per polygon position (vs. 5) so total default images = 13 vs. 65 for a faster flow. * Adds more incremental logging messages --- python-api/calibrate.py | 164 ++++++++++++++++++++++++++++ python-api/calibration_pipeline.py | 167 ----------------------------- python-api/calibration_utils.py | 83 ++++++++++++-- 3 files changed, 241 insertions(+), 173 deletions(-) create mode 100644 python-api/calibrate.py delete mode 100644 python-api/calibration_pipeline.py diff --git a/python-api/calibrate.py b/python-api/calibrate.py new file mode 100644 index 000000000..921e7f70d --- /dev/null +++ b/python-api/calibrate.py @@ -0,0 +1,164 @@ +import depthai +from calibration_utils import * +from argparse import ArgumentParser +from time import time +import numpy as np +import os +from pathlib import Path +import shutil +import consts.resource_paths + +use_cv = True +try: + import cv2 +except ImportError: + use_cv = False + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("-p", "--polygons", default=list(np.arange(len(setPolygonCoordinates(1000,600)))), nargs='*', + type=int, required=False, + help="Space-separated list of polygons (ex: 0 5 7) to restrict image capture. Default is all polygons.") + parser.add_argument("-c", "--count", default=1, + type=int, required=False, + help="Number of images per polygon to capture. Default is 1.") + parser.add_argument("-s", "--square_size_cm", default="2.5", + type=float, required=False, + help="Square size of calibration pattern used in centimeters. Default is 2.5.") + parser.add_argument("-i", "--image_op", default="modify", + type=str, required=False, + help="Whether existing images should be modified or all images should be deleted before running image capture. The default is 'modify'. Change to 'delete' to delete all image files.") + parser.add_argument("-m", "--mode", default=['capture','process'], nargs='*', + type=str, required=False, + help="Space-separaed list of calibration options to run. By default, executes the full 'capture process' pipeline. To execute a single step, enter just that step (ex: 'process').") + options = parser.parse_args() + + return options + +args = vars(parse_args()) +print("args:",args) + +if 'capture' in args['mode']: + + # Delete Dataset directory if asked + if args['image_op'] == 'delete': + shutil.rmtree('dataset/') + + # Creates dirs to save captured images + try: + for path in ["left","right"]: + Path("dataset/"+path).mkdir(parents=True, exist_ok=True) + except OSError as e: + print ("An error occurred trying to create image dataset directories:",e) + exit(0) + + cmd_file = consts.resource_paths.device_depth_cmd_fpath + + # Create Depth AI Pipeline to start video streaming + streams_list = ['left', 'right', 'depth'] + pipieline = depthai.create_pipeline( + streams=streams_list, + cmd_file=cmd_file, + calibration_file=consts.resource_paths.calib_fpath, + config_file=consts.resource_paths.pipeline_config_fpath + ) + + num_of_polygons = 0 + polygons_coordinates = [] + + image_per_polygon_counter = 0 # variable to track how much images were captured per each polygon + complete = False # Indicates if images have been captured for all polygons + + polygon_index = args['polygons'][0] # number to track which polygon is currently using + total_num_of_captured_images = 0 # variable to hold total number of captured images + + capture_images = False # value to track the state of capture button (spacebar) + captured_left_image = False # value to check if image from the left camera was capture + captured_right_image = False # value to check if image from the right camera was capture + + run_capturing_images = True # value becames False and stop the main loop when all polygon indexes were used + + calculate_coordinates = False # track if coordinates of polynoms was calculated + total_images = args['count']*len(args['polygons']) + + while run_capturing_images: + data_list = pipieline.get_available_data_packets() + for packet in data_list: + if packet.stream_name == 'left' or packet.stream_name == 'right': + frame = packet.getData() + if calculate_coordinates == False: + height, width = frame.shape + polygons_coordinates = setPolygonCoordinates(height, width) + # polygons_coordinates = select_polygon_coords(polygons_coordinates,args['polygons']) + num_of_polygons = len(args['polygons']) + print("Will take %i total images, %i per each polygon." % (total_images,args['count'])) + calculate_coordinates = True + + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + if capture_images == True: + if packet.stream_name == 'left': + filename = image_filename(packet.stream_name,polygon_index,total_num_of_captured_images) + cv2.imwrite("dataset/left/" + str(filename), frame) + print("py: Saved image as: " + str(filename)) + captured_left_image = True + + elif packet.stream_name == 'right': + filename = image_filename(packet.stream_name,polygon_index,total_num_of_captured_images) + cv2.imwrite("dataset/right/" + str(filename), frame) + print("py: Saved image as: " + str(filename)) + captured_right_image = True + + if captured_right_image == True and captured_left_image == True: + capture_images = False + captured_left_image = False + captured_right_image = False + total_num_of_captured_images += 1 + image_per_polygon_counter += 1 + + if image_per_polygon_counter == args['count']: + image_per_polygon_counter = 0 + try: + polygon_index = args['polygons'][args['polygons'].index(polygon_index)+1] + except IndexError: + complete = True + + if complete == False: + cv2.putText(frame, "Align cameras with callibration board and press spacebar to capture the image", (0, 25), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0)) + cv2.putText(frame, "Polygon Position: %i. " % (polygon_index) + "Captured %i of %i images." % (total_num_of_captured_images,total_images), (0, 700), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0)) + cv2.polylines(frame, np.array([getPolygonCoordinates(polygon_index, polygons_coordinates)]), True, (0, 0, 255), 4) + cv2.imshow(packet.stream_name, frame) + else: + # all polygons used, stop the loop + run_capturing_images = False + + key = cv2.waitKey(1) + + if key == ord(" "): + capture_images = True + + elif key == ord("q"): + print("py: Calibration has been interrupted!") + exit(0) + + + del pipieline # need to manualy delete the object, because of size of HostDataPacket queue runs out (Not enough free space to save {stream}) + + cv2.destroyWindow("left") + cv2.destroyWindow("right") + +else: + print("Skipping capture.") + +if 'process' in args['mode']: + print("py: Starting calibration based on captured images") + cal_data = StereoCalibration() + try: + cal_data.calibrate("dataset", args['square_size_cm'], "./depthai.calib") + except AssertionError as e: + print("[ERROR] " + str(e)) + exit(0) +else: + print("Skipping process.") + +print('py: DONE.') diff --git a/python-api/calibration_pipeline.py b/python-api/calibration_pipeline.py deleted file mode 100644 index ff0cbf7c5..000000000 --- a/python-api/calibration_pipeline.py +++ /dev/null @@ -1,167 +0,0 @@ -import depthai -import calibration_utils - -from time import time -import numpy as np -import os - -import consts.resource_paths - -use_cv = True -try: - import cv2 -except ImportError: - use_cv = False - - -def setPolygonCoordinates(height, width): - horizontal_shift = width//4 - vertical_shift = height//4 - - margin = 60 - slope = 150 - - p_coordinates = [ - [[margin,0], [margin,height], [width//2, height-slope], [width//2, slope]], - [[horizontal_shift, 0], [horizontal_shift, height], [width//2 + horizontal_shift, height-slope], [width//2 + horizontal_shift, slope]], - [[horizontal_shift*2-margin, 0], [horizontal_shift*2-margin, height], [width//2 + horizontal_shift*2-margin, height-slope], [width//2 + horizontal_shift*2-margin, slope]], - - [[margin,margin], [margin, height-margin], [width-margin, height-margin], [width-margin, margin]], - - [[width-margin, 0], [width-margin, height], [width//2, height-slope], [width//2, slope]], - [[width-horizontal_shift, 0], [width-horizontal_shift, height], [width//2-horizontal_shift, height-slope], [width//2-horizontal_shift, slope]], - [[width-horizontal_shift*2+margin, 0], [width-horizontal_shift*2+margin, height], [width//2-horizontal_shift*2+margin, height-slope], [width//2-horizontal_shift*2+margin, slope]], - - [[0,margin], [width, margin], [width-slope, height//2], [slope, height//2]], - [[0,vertical_shift], [width, vertical_shift], [width-slope, height//2+vertical_shift], [slope, height//2+vertical_shift]], - [[0,vertical_shift*2-margin], [width, vertical_shift*2-margin], [width-slope, height//2+vertical_shift*2-margin], [slope, height//2+vertical_shift*2-margin]], - - [[0,height-margin], [width, height-margin], [width-slope, height//2], [slope, height//2]], - [[0,height-vertical_shift], [width, height-vertical_shift], [width-slope, height//2-vertical_shift], [slope, height//2-vertical_shift]], - [[0,height-vertical_shift*2+margin], [width, height-vertical_shift*2+margin], [width-slope, height//2-vertical_shift*2+margin], [slope, height//2-vertical_shift*2+margin]] - ] - return p_coordinates - -def getPolygonCoordinates(idx, p_coordinates): - return p_coordinates[idx] - -def getNumOfPolygons(p_coordinates): - return len(p_coordinates) - - - -# creates dirs to save captured images -try: - os.mkdir("dataset") - os.chdir("dataset") - os.mkdir("left") - os.mkdir("right") - os.chdir("../") -except OSError: - print ("py: Creation of the directories for images from left and right cameras have failed") - exit(0) - -# get size of calibration pattern, uses to calibrate cameras -calib_puttern_size = input("Enter size of calibration pattern used, Please use [cm] (Default value: 2.5 cm): ") - -if calib_puttern_size == "": - calib_puttern_size = 2.5 -else: - calib_puttern_size = float(calib_puttern_size) - -print("py: Size of calibration = " + str(calib_puttern_size)) - - -cmd_file = consts.resource_paths.device_depth_cmd_fpath - -streams_list = ['left', 'right', 'depth'] -pipieline = depthai.create_pipeline( - streams=streams_list, - cmd_file=cmd_file, - calibration_file=consts.resource_paths.calib_fpath, - config_file=consts.resource_paths.pipeline_config_fpath - ) - -num_of_polygons = 0 -polygons_coordinates = [] - -num_image_per_polygon = 5 # number of images captured per polygon, this value can be changed, the bigger value, the more images per each polygon will be captured -image_per_polygon_counter = 0 # variable to track how much images were captured per each polygon - -polygon_index = 0 # number to track which polygon is currently using -total_num_of_captured_images = 0 # variable to hold total number of captured images - -capture_images = False # value to track the state of capture button (spacebar) -captured_left_image = False # value to check if image from the left camera was capture -captured_right_image = False # value to check if image from the right camera was capture - -run_capturing_images = True # value becames False and stop the main loop when all polygon indexes were used - -calculate_coordinates = False # track if coordinates of polynoms was calculated - -while run_capturing_images: - data_list = pipieline.get_available_data_packets() - for packet in data_list: - if packet.stream_name == 'left' or packet.stream_name == 'right': - frame = packet.getData() - if calculate_coordinates == False: - height, width = frame.shape - polygons_coordinates = setPolygonCoordinates(height, width) - num_of_polygons = getNumOfPolygons(polygons_coordinates) - calculate_coordinates = True - - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - - if capture_images == True: - if packet.stream_name == 'left': - filename = "left_" + str(total_num_of_captured_images) + ".png" - cv2.imwrite("dataset/left/" + str(filename), frame) - print("py: Saved image as: " + str(filename)) - captured_left_image = True - - elif packet.stream_name == 'right': - filename = "right_" + str(total_num_of_captured_images) + ".png" - cv2.imwrite("dataset/right/" + str(filename), frame) - print("py: Saved image as: " + str(filename)) - captured_right_image = True - - if captured_right_image == True and captured_left_image == True: - capture_images = False - captured_left_image = False - captured_right_image = False - total_num_of_captured_images += 1 - image_per_polygon_counter += 1 - - if image_per_polygon_counter == num_image_per_polygon: - polygon_index += 1 - image_per_polygon_counter = 0 - - if num_of_polygons != polygon_index: - cv2.putText(frame, "Align cameras with callibration board and press spacebar to capture the image", (0, 25), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0)) - cv2.putText(frame, "Number of saved images: " + str(total_num_of_captured_images), (750, 700), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0)) - cv2.polylines(frame, np.array([getPolygonCoordinates(polygon_index, polygons_coordinates)]), True, (0, 0, 255), 4) - cv2.imshow(packet.stream_name, frame) - else: - # all polygons used, stop the loop - run_capturing_images = False - - key = cv2.waitKey(1) - - if key == ord(" "): - capture_images = True - - elif key == ord("q"): - print("py: Calibration has been interrupted!") - exit(0) - - -del pipieline # need to manualy delete the object, because of size of HostDataPacket queue runs out (Not enough free space to save {stream}) - -cv2.destroyWindow("left") -cv2.destroyWindow("right") -print("py: Starting calibration based on captured images") -cal_data = calibration_utils.StereoCalibration() -cal_data.calibrate("dataset", calib_puttern_size, "./depthai.calib") - -print('py: DONE.') - diff --git a/python-api/calibration_utils.py b/python-api/calibration_utils.py index c1d92b0b4..e60bb9aac 100644 --- a/python-api/calibration_utils.py +++ b/python-api/calibration_utils.py @@ -3,6 +3,8 @@ import os import shutil import numpy as np +import re +import time from argparse import ArgumentParser @@ -40,6 +42,56 @@ def parse_args(): return options +# Creates a set of 13 polygon coordinates +def setPolygonCoordinates(height, width): + horizontal_shift = width//4 + vertical_shift = height//4 + + margin = 60 + slope = 150 + + p_coordinates = [ + [[margin,0], [margin,height], [width//2, height-slope], [width//2, slope]], + [[horizontal_shift, 0], [horizontal_shift, height], [width//2 + horizontal_shift, height-slope], [width//2 + horizontal_shift, slope]], + [[horizontal_shift*2-margin, 0], [horizontal_shift*2-margin, height], [width//2 + horizontal_shift*2-margin, height-slope], [width//2 + horizontal_shift*2-margin, slope]], + + [[margin,margin], [margin, height-margin], [width-margin, height-margin], [width-margin, margin]], + + [[width-margin, 0], [width-margin, height], [width//2, height-slope], [width//2, slope]], + [[width-horizontal_shift, 0], [width-horizontal_shift, height], [width//2-horizontal_shift, height-slope], [width//2-horizontal_shift, slope]], + [[width-horizontal_shift*2+margin, 0], [width-horizontal_shift*2+margin, height], [width//2-horizontal_shift*2+margin, height-slope], [width//2-horizontal_shift*2+margin, slope]], + + [[0,margin], [width, margin], [width-slope, height//2], [slope, height//2]], + [[0,vertical_shift], [width, vertical_shift], [width-slope, height//2+vertical_shift], [slope, height//2+vertical_shift]], + [[0,vertical_shift*2-margin], [width, vertical_shift*2-margin], [width-slope, height//2+vertical_shift*2-margin], [slope, height//2+vertical_shift*2-margin]], + + [[0,height-margin], [width, height-margin], [width-slope, height//2], [slope, height//2]], + [[0,height-vertical_shift], [width, height-vertical_shift], [width-slope, height//2-vertical_shift], [slope, height//2-vertical_shift]], + [[0,height-vertical_shift*2+margin], [width, height-vertical_shift*2+margin], [width-slope, height//2-vertical_shift*2+margin], [slope, height//2-vertical_shift*2+margin]] + ] + return p_coordinates + +def getPolygonCoordinates(idx, p_coordinates): + return p_coordinates[idx] + +def getNumOfPolygons(p_coordinates): + return len(p_coordinates) + +# Filters polygons to just those at the given indexes. +def select_polygon_coords(p_coordinates,indexes): + if indexes == None: + # The default + return p_coordinates + else: + print("Filtering polygons to those at indexes=",indexes) + return [p_coordinates[i] for i in indexes] + +def image_filename(stream_name,polygon_index,total_num_of_captured_images): + return "{stream_name}_p{polygon_index}_{total_num_of_captured_images}.png".format(stream_name=stream_name,polygon_index=polygon_index,total_num_of_captured_images=total_num_of_captured_images) + +def polygon_from_image_name(image_name): + """Returns the polygon index from an image name (ex: "left_p10_0.png" => 10)""" + return int(re.findall("p(\d+)",image_name)[0]) class StereoCalibration(object): """Class to Calculate Calibration and Rectify a Stereo Camera.""" @@ -75,6 +127,7 @@ def process_images(self, filepath): self.objpoints = [] # 3d point in real world space self.imgpoints_l = [] # 2d points in image plane. self.imgpoints_r = [] # 2d points in image plane. + self.calib_successes = [] # polygon ids of left/right image sets with checkerboard corners. images_left = glob.glob(filepath + "/left/*") images_right = glob.glob(filepath + "/right/*") @@ -96,6 +149,9 @@ def process_images(self, filepath): assert img_l is not None, "ERROR: Images not read correctly" assert img_r is not None, "ERROR: Images not read correctly" + print("Finding chessboard corners for %s and %s..." % (image_left,image_right)) + start_time = time.time() + # Find the chess board corners flags = 0 flags |= cv2.CALIB_CB_ADAPTIVE_THRESH @@ -116,15 +172,30 @@ def process_images(self, filepath): rt = cv2.cornerSubPix(img_r, corners_r, (5, 5), (-1, -1), self.criteria) self.imgpoints_r.append(corners_r) + self.calib_successes.append(polygon_from_image_name(image_left)) + print("\t[OK]. Took %i seconds." % (round(time.time() - start_time, 2))) else: - print("Corners not detected for", - str(os.path.basename(image_left)), - str(os.path.basename(image_right))) + print("\t[ERROR] - Corners not detected. Took %i seconds." % (round(time.time() - start_time, 2))) self.img_shape = img_r.shape[::-1] print(str(len(self.objpoints)) + " of " + str(len(images_left)) + " images being used for calibration") - assert len(self.objpoints) > 4, "ERROR: Not enough valid image sets, please re-capture" + self.ensure_valid_images() + + def ensure_valid_images(self): + """ + Ensures there is one set of left/right images for each polygon. If not, raises an raises an + AssertionError with instructions on re-running calibration for the invalid polygons. + """ + expected_polygons = len(setPolygonCoordinates(1000,600)) # inseted values are placeholders + unique_calib_successes = set(self.calib_successes) + if len(unique_calib_successes) != expected_polygons: + valid = set(np.arange(0,expected_polygons)) + missing = valid - unique_calib_successes + arg_value = ' '.join(map(str, missing)) + raise AssertionError("Missing valid image sets for %i polygons. Re-run calibration with the\n'-p %s' argument to re-capture images for these polygons." % (len(missing), arg_value)) + else: + return true def stereo_calibrate(self): """Calibrate camera and construct Homography.""" @@ -155,7 +226,7 @@ def stereo_calibrate(self): self.objpoints, self.imgpoints_l, self.imgpoints_r, self.M1, self.d1, self.M2, self.d2, self.img_shape, criteria=stereocalib_criteria, flags=flags) - + assert ret < 1.0, "ERROR: Calibration no succesfull, please re-capture" print("Calibration successful, RMS error: " + str(ret)) @@ -292,4 +363,4 @@ def rectify_dataset(self, dataset_dir, calibration_file): cv2.imwrite(os.path.join(left_result_dir, os.path.basename(image_left)), img_l) cv2.imwrite(os.path.join(right_result_dir, - os.path.basename(image_right)), img_r) \ No newline at end of file + os.path.basename(image_right)), img_r) From 5ead867249caeee08e9559e93bee7499dcd58faa Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Mon, 3 Feb 2020 06:34:58 -0700 Subject: [PATCH 3/7] Removing unused arg parsing --- python-api/calibrate.py | 2 +- python-api/calibration_utils.py | 29 ----------------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/python-api/calibrate.py b/python-api/calibrate.py index 921e7f70d..d2fa99bc2 100644 --- a/python-api/calibrate.py +++ b/python-api/calibrate.py @@ -30,7 +30,7 @@ def parse_args(): help="Whether existing images should be modified or all images should be deleted before running image capture. The default is 'modify'. Change to 'delete' to delete all image files.") parser.add_argument("-m", "--mode", default=['capture','process'], nargs='*', type=str, required=False, - help="Space-separaed list of calibration options to run. By default, executes the full 'capture process' pipeline. To execute a single step, enter just that step (ex: 'process').") + help="Space-separated list of calibration options to run. By default, executes the full 'capture process' pipeline. To execute a single step, enter just that step (ex: 'process').") options = parser.parse_args() return options diff --git a/python-api/calibration_utils.py b/python-api/calibration_utils.py index e60bb9aac..5ebaf261b 100644 --- a/python-api/calibration_utils.py +++ b/python-api/calibration_utils.py @@ -5,8 +5,6 @@ import numpy as np import re import time -from argparse import ArgumentParser - def mkdir_overwrite(dir): if not os.path.exists(dir): @@ -15,33 +13,6 @@ def mkdir_overwrite(dir): shutil.rmtree(dir) os.makedirs(dir) - -def parse_args(): - parser = ArgumentParser() - parser.add_argument("-d", "--dataset", action="store", - type=str, dest="dataset_dir", required=True, - help="Path to calibration dataset, NB must conatin /left and /right dirs") - parser.add_argument("-s", "--square_size_cm", action="store", - type=float, dest="square_size_cm", required=False, - help="Sqaure size of calibration pattern used, NB Please use [cm]") - parser.add_argument("-c", "--calib_file", action="store", - type=str, dest="calib_filepath", default="./calibration.bin", - help="output filepath for calibration") - parser.add_argument("-a", "--apply_calibration", action="store_true", - dest="apply_calibration", - help="Instead of calibrating, aplly calibration file to dataset") - - options = parser.parse_args() - - if options.apply_calibration is False and options.square_size_cm is None: - print("\nError: Attempting to calibrate but no square size provided.\n") - parser.print_help() - raise SystemExit() - elif options.apply_calibration is True and options.square_size_cm is not None: - print("\nWarning: Sqaure size arg not used.\n") - - return options - # Creates a set of 13 polygon coordinates def setPolygonCoordinates(height, width): horizontal_shift = width//4 From 0377b98e84718fe0345f702d562d13a168d33e5b Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Mon, 3 Feb 2020 07:09:59 -0700 Subject: [PATCH 4/7] Working calibration w/full image set --- python-api/calibrate.py | 4 ++-- python-api/calibration_utils.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/python-api/calibrate.py b/python-api/calibrate.py index d2fa99bc2..f773e5897 100644 --- a/python-api/calibrate.py +++ b/python-api/calibrate.py @@ -36,7 +36,7 @@ def parse_args(): return options args = vars(parse_args()) -print("args:",args) +print("Using Arguments=",args) if 'capture' in args['mode']: @@ -151,7 +151,7 @@ def parse_args(): print("Skipping capture.") if 'process' in args['mode']: - print("py: Starting calibration based on captured images") + print("Starting image processing") cal_data = StereoCalibration() try: cal_data.calibrate("dataset", args['square_size_cm'], "./depthai.calib") diff --git a/python-api/calibration_utils.py b/python-api/calibration_utils.py index 5ebaf261b..51991a962 100644 --- a/python-api/calibration_utils.py +++ b/python-api/calibration_utils.py @@ -72,6 +72,7 @@ def __init__(self): def calibrate(self, filepath, square_size, out_filepath): """Function to calculate calibration for stereo camera.""" + start_time = time.time() # init object data self.objp = np.zeros((9 * 6, 3), np.float32) self.objp[:, :2] = np.mgrid[0:9, 0:6].T.reshape(-1, 2) @@ -86,8 +87,9 @@ def calibrate(self, filepath, square_size, out_filepath): # save data to binary file self.H.tofile(out_filepath) - print("Result written to: " + out_filepath) + print("Calibration file written to %s.\nRename this file to `default.calib` and move it the `resources` folder." % (out_filepath)) + print("\tTook %i to run image processing." % (round(time.time() - start_time, 2))) # show debug output for visual inspection print("\nRectifying dataset for visual inspection") self.show_rectified_images(filepath, out_filepath) @@ -120,7 +122,7 @@ def process_images(self, filepath): assert img_l is not None, "ERROR: Images not read correctly" assert img_r is not None, "ERROR: Images not read correctly" - print("Finding chessboard corners for %s and %s..." % (image_left,image_right)) + print("Finding chessboard corners for %s and %s..." % (os.path.basename(image_left), os.path.basename(image_right))) start_time = time.time() # Find the chess board corners @@ -166,7 +168,7 @@ def ensure_valid_images(self): arg_value = ' '.join(map(str, missing)) raise AssertionError("Missing valid image sets for %i polygons. Re-run calibration with the\n'-p %s' argument to re-capture images for these polygons." % (len(missing), arg_value)) else: - return true + return True def stereo_calibrate(self): """Calibrate camera and construct Homography.""" @@ -198,8 +200,8 @@ def stereo_calibrate(self): self.M1, self.d1, self.M2, self.d2, self.img_shape, criteria=stereocalib_criteria, flags=flags) - assert ret < 1.0, "ERROR: Calibration no succesfull, please re-capture" - print("Calibration successful, RMS error: " + str(ret)) + assert ret < 1.0, "[ERROR] Calibration RMS error < 1.0 (%i). Re-try image capture." % (ret) + print("[OK] Calibration successful w/ RMS error=" + str(ret)) # construct Homography plane_depth = 40.0 # arbitrary plane depth @@ -212,7 +214,7 @@ def stereo_calibrate(self): disparity = (self.M1[0, 0] * T[0] / plane_depth) self.H[0, 2] -= disparity self.H = self.H.astype(np.float32) - print("Rectifying Homography:") + print("Rectifying Homography...") print(self.H) def show_rectified_images(self, dataset_dir, calibration_file): @@ -288,6 +290,7 @@ def show_rectified_images(self, dataset_dir, calibration_file): line_row += 30 # show image + print("Displaying Stereo Pair for visual inspection. Press the [ESC] key to exit.") while(1): cv2.imshow('Stereo Pair', img_concat) k = cv2.waitKey(33) From 6dfa3f5c57d7e382638abf1063fb4467a17c9496 Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Mon, 3 Feb 2020 07:26:53 -0700 Subject: [PATCH 5/7] Reduce image for display by 1.5x. The video stream image is to large to display both left/right on my external monitor. It's not possible to allow window resize w/o Qt. Reduce size by 1.5x before displaying. --- python-api/calibrate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python-api/calibrate.py b/python-api/calibrate.py index f773e5897..45884459f 100644 --- a/python-api/calibrate.py +++ b/python-api/calibrate.py @@ -127,7 +127,11 @@ def parse_args(): cv2.putText(frame, "Align cameras with callibration board and press spacebar to capture the image", (0, 25), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0)) cv2.putText(frame, "Polygon Position: %i. " % (polygon_index) + "Captured %i of %i images." % (total_num_of_captured_images,total_images), (0, 700), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0)) cv2.polylines(frame, np.array([getPolygonCoordinates(polygon_index, polygons_coordinates)]), True, (0, 0, 255), 4) - cv2.imshow(packet.stream_name, frame) + # original image is 1280x720. reduce by 2x so it fits better. + aspect_ratio = 1.5 + new_x, new_y = int(frame.shape[1]/aspect_ratio), int(frame.shape[0]/aspect_ratio) + resized_image = cv2.resize(frame,(new_x,new_y)) + cv2.imshow(packet.stream_name, resized_image) else: # all polygons used, stop the loop run_capturing_images = False From 9ade9459e876a00d02e7c89479b4c50be1e51c10 Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Mon, 3 Feb 2020 07:56:07 -0700 Subject: [PATCH 6/7] Added epilog --- python-api/calibrate.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/python-api/calibrate.py b/python-api/calibrate.py index 45884459f..59103efc5 100644 --- a/python-api/calibrate.py +++ b/python-api/calibrate.py @@ -1,5 +1,6 @@ import depthai from calibration_utils import * +import argparse from argparse import ArgumentParser from time import time import numpy as np @@ -15,7 +16,37 @@ use_cv = False def parse_args(): - parser = ArgumentParser() + epilog_text = ''' + Captures and processes images for disparity depth calibration, generating a `dephtai.calib` file + that should be loaded when initializing depthai. By default, captures one image across 13 polygon positions. + + Image capture requires the use of a printed 7x9 OpenCV checkerboard applied to a flat surface (ex: sturdy cardboard). + When taking photos, ensure the checkerboard fits within both the left and right image displays. The board does not need + to fit within each drawn red polygon shape, but it should mimic the display of the polygon. + + If the checkerboard does not fit within a captured image, the calibration will generate an error message with instructions + on re-running calibration for polygons w/o valid checkerboard captures. + + The script requires a RMS error < 1.0 to generate a calibration file. If RMS exceeds this threshold, an error is displayed. + + Example usage: + + Run calibration with a checkerboard square size of 2.35 cm: + python calibrate.py -s 2.35 + + Run calibration for only the first and 3rd polygon positions: + python calibrate.py -p 0 2 + + Only run image processing (not image capture) w/ a 2.35 cm square size. Requires a set of polygon images: + python calibrate.py -s 2.35 -m process + + Delete all existing images before starting image capture: + python calibrate.py -i delete + + Capture 3 images per polygon: + python calibrate.py -c 3 + ''' + parser = ArgumentParser(epilog=epilog_text,formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("-p", "--polygons", default=list(np.arange(len(setPolygonCoordinates(1000,600)))), nargs='*', type=int, required=False, help="Space-separated list of polygons (ex: 0 5 7) to restrict image capture. Default is all polygons.") From 1d4e45616a5a909d6006beb9cf38b02927dc33ca Mon Sep 17 00:00:00 2001 From: Derek Haynes Date: Mon, 3 Feb 2020 08:01:39 -0700 Subject: [PATCH 7/7] Updating README --- python-api/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/python-api/README.md b/python-api/README.md index 7d1eb4779..9c5d9d18f 100644 --- a/python-api/README.md +++ b/python-api/README.md @@ -5,7 +5,7 @@ This is python API of DepthAI and examples. Files with extention `.so` are python modules: - `depthai.cpython-36m-x86_64-linux-gnu.so` built for Ubuntu 18.04 & Python 3.6 - `depthai.cpython-37m-arm-linux-gnueabihf.so` built for Raspbian 10 & Python 3.7 - + # Examples `test.py` - depth example `test_cnn.py` - CNN inference example @@ -24,10 +24,10 @@ Example of the conversion: # Disparity Depth Calibration For better depth image quality, you need a stereo calibration. To do it, you have to: -1. Print the chessboard for calibration. The picture can be found in the `resources` folder (resources/calibration-chess-board.png) -2. Start python3 script: type `python3 calibration_pipeline.py` in the terminal. Two streams left and right will show up. Each window will contain a polygon. +1. Print the chessboard for calibration. The picture can be found in the `resources` folder (resources/calibration-chess-board.png). Measure the square size in centimeters and insert the value in the command below. +2. Start python3 script: type `python3 calibrate.py -s [SQUARE_SIZE_IN_CM]` in the terminal. Two streams left and right will show up. Each window will contain a polygon. 3. Put a printed chessboard within the polygon and press barspace. It will take a photo. There will be 13 positions of polygons. -4. After it, the calibration will automatically start based on taken pictures. If calibration is a successful file named `depthai.calib` will be generated. +4. After it, the calibration will automatically start based on taken pictures. If calibration is a successful file named `depthai.calib` will be generated. Depthai has the default calibration file. There are two ways to change it: 1. Easy way: rename your calibration file to `default.calib` and move it the resources folder. @@ -36,7 +36,7 @@ Depthai has the default calibration file. There are two ways to change it: # Issue reporting We are developing depthai framework, and it's crucial for us to know what kind of problems users are facing. So thanks for testing DepthAI! The information you give us, the faster we will help you and make depthai better! - + Please, do the following steps: 1. Run script `log_system_information.sh` and provide us the output (`log_system_information.txt`, it's system version & modules versions); 2. Take a photo of a device you are using (or provide us a device model); @@ -44,4 +44,3 @@ Please, do the following steps: 4. Describe the actual running results (what you see after started your script with depthai); 5. Provide us information about how you are using the depthai python API (code snippet, for example); 6. Send us consol outputs; -