Circle Selector

.gitignore

@@ -11,7 +11,7 @@ Python/.ipynb_checkpoints/
 Python/__pycache__/
 Python/cmake_deprecated_functions.log
 Python/Notebooks/.ipynb_checkpoints/
-
+Python/docs/__pycache__/
 ## Other
 Screenshots/
 userSettings.json

Python/preprocessing/abs_preprocessor.py

@@ -71,8 +71,12 @@ def __init__(self, args):
         # the input image index range of OmniPhotos
         self.image_start_idx = self.config["preprocessing.frame_index_start"]
         self.image_end_idx = self.config["preprocessing.frame_index_end"]
+        self.find_stable_circle = self.config["preprocessing.find_stable_circle"]
         if self.image_start_idx < 0 or self.image_start_idx > self.frame_number:
             self.show_info("preprocessing.frame_index_start set error", "error")
+        if self.find_stable_circle:
+            self.image_start_idx = 0
+            self.image_end_idx = -1
 
         if self.image_end_idx == -1:
             self.image_end_idx = self.frame_number - 1
@@ -84,14 +88,13 @@ def __init__(self, args):
 
         # create the images list
         self.original_filename_expression = self.config["preprocessing.original_filename_expression"]
-        self.op_filename_expression = self.config["preprocessing.op_filename_expression"]
 
         self.trajectory_images_list = [] # used to storage the processed original image file name
         self.op_image_list = [] # used to storage the mp ready image file name
 
         for idx in range(self.image_start_idx, self.image_end_idx + 1):
             self.trajectory_images_list.append(self.original_filename_expression % idx)
-            self.op_image_list.append(self.op_filename_expression % idx)
+            self.op_image_list.append(self.original_filename_expression % idx)
 
         self.image_list = self.op_image_list
 

Python/preprocessing/circleselector/__init__.py

@@ -0,0 +1,5 @@
+import circleselector.cv_utils
+import circleselector.datatypes
+import circleselector.loader
+import circleselector.metrics
+import circleselector.plotting_utils

Python/preprocessing/circleselector/cv_utils.py

@@ -0,0 +1,162 @@
+import os
+
+import cv2
+import numpy as np
+from matplotlib import pyplot as plt
+from skimage.metrics import structural_similarity
+
+import flownet2.utils.computeColor as computeColor
+
+
+def resize(img, scale_percent, verbose=False):
+    if verbose:
+        print('Original Dimensions : ', img.shape)
+    width = int(img.shape[1] * scale_percent / 100)
+    height = int(img.shape[0] * scale_percent / 100)
+    dim = (width, height)
+
+    # resize image
+    resized = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
+    if verbose:
+        print('Resized Dimensions : ', resized.shape)
+    return resized
+
+
+# remap images using calculated flows
+def warp_flow(img, flow):
+    h, w = flow.shape[:2]
+    flow = -flow
+    flow[:, :, 0] += np.arange(w)
+    flow[:, :, 1] += np.arange(h)[:, np.newaxis]
+    res = cv2.remap(img, flow, None, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
+    return res
+
+
+def calculate_psnr(img1, img2, max_value=255):
+    """"Calculating peak signal-to-noise ratio (PSNR) between two images."""
+    mse = np.mean((np.array(img1, dtype=np.float32) - np.array(img2, dtype=np.float32)) ** 2)
+    if mse == 0:
+        return 100
+    return 20 * np.log10(max_value / (np.sqrt(mse)))
+
+
+# calculate_psnr(gt_img, ours_img)
+
+
+def calculate_ssim(img1, img2):
+    return structural_similarity(img1, img2, multichannel=True)
+
+
+def warp_images(img1, img2, savedir: str = None, look_at_angle: float = 0):
+    # pad image on which flow will be calculated
+    padding = 180
+    img1 = slice_eqimage(img1, look_at_angle, padding=padding)
+    img2 = slice_eqimage(img2, look_at_angle, padding=padding)
+
+    output_im1 = resize(img1, 50)
+    output_im2 = resize(img2, 50)
+
+    resized1 = resize(img1, 25)
+    resized2 = resize(img2, 25)
+
+    # cast image to greyscale
+    prvs = cv2.cvtColor(resized1, cv2.COLOR_BGR2GRAY)
+    next = cv2.cvtColor(resized2, cv2.COLOR_BGR2GRAY)
+    dis = cv2.DISOpticalFlow_create()
+
+    # calculate forward flow
+    flow_forward = dis.calc(prvs, next, None)
+    flow_forward = cv2.resize(flow_forward, (output_im1.shape[1], output_im1.shape[0]),
+                              interpolation=cv2.INTER_LINEAR) * 2
+    # calculate backward flow
+    flow_backward = dis.calc(next, prvs, None)
+    flow_backward = cv2.resize(flow_backward, (output_im1.shape[1], output_im1.shape[0]),
+                               interpolation=cv2.INTER_LINEAR) * 2
+
+    next_img = warp_flow(output_im1, flow_forward)
+    prvs_img = warp_flow(output_im2, flow_backward)
+
+    # unpad the images
+    unpadded_next = next_img[:, padding // 2:next_img.shape[1] - padding // 2]
+    unpadded_prvs = prvs_img[:, padding // 2:prvs_img.shape[1] - padding // 2]
+
+    if savedir is not None:
+        plt.imsave(os.path.join(savedir, "forward_flow.jpg"),
+                   computeColor.computeImg(flow_forward)[:, padding:next_img.shape[1] - (padding)])
+        plt.imsave(os.path.join(savedir, "backward_flow.jpg"),
+                   computeColor.computeImg(flow_backward)[:, padding:next_img.shape[1] - (padding)])
+        plt.imsave(os.path.join(savedir, "2_output2.jpg"), np.flip(unpadded_prvs, axis=2))
+        plt.imsave(os.path.join(savedir, "4_output1.jpg"), np.flip(unpadded_next, axis=2))
+        plt.imsave(os.path.join(savedir, "1_input_img1.jpg"),
+                   np.flip(output_im1[:, padding // 2:next_img.shape[1] - padding // 2], axis=2))
+        plt.imsave(os.path.join(savedir, "3_input_img2.jpg"),
+                   np.flip(output_im2[:, padding // 2:prvs_img.shape[1] - padding // 2], axis=2))
+    return unpadded_next, unpadded_prvs
+
+
+def slice_eqimage(img: np.array, look_at_angle: float, padding: int = 0):
+    """
+
+    :param img: equirectangular image
+    :param look_at_angle: radians
+    :param padding: number of indicies to add to each end of the eq image slicing. (e.g 60)
+    :return: img hemisphere, centered at lookatang
+    """
+    hemisphere_width = img.shape[1] // 4
+    padded_img = np.hstack((img, img, img))
+    # 3 stacked equirectangular images leads to a total angle of 6 pi. The equirectangular images are stacked
+    # for when we are looking at the outer edge of the equirectangular image. (where the wrap around occurs).
+    lookatindex = round(padded_img.shape[1] * (3 * np.pi + look_at_angle) / (6 * np.pi))
+    lower = lookatindex - hemisphere_width - padding
+    upper = lookatindex + hemisphere_width + padding
+    return padded_img[:, lower:upper]
+
+
+def calculate_metrics(interval: tuple, dataset_path: str, savedir: str = None, rel_input_image_path='Input',
+                      look_at_angle: float = 0) -> tuple:
+    """
+
+    :param interval: tuple containing interaval metrics are being calculated for
+    :param dataset_path: path to dataset folder (e.g path/to/GenoaCathedral)
+    :param rel_input_image_path: input image path relative to dataset_path
+    :param savedir: will save OF output to savedir if not None
+    :param look_at_angle: direction relative to center in radians
+    :return: ssim, psnr
+    """
+
+    # read in the images according to the indexes in the interval
+    # NOTE: the code assumes all the images are listed, in order, in the image directory.
+    input_path = os.path.join(dataset_path, rel_input_image_path)
+    images = os.listdir(input_path)
+
+    # remove any file that are not images from the list.
+    for filename in images:
+        if os.path.splitext(filename)[-1] not in [".png", ".jpg"]:
+            images.remove(filename)
+    path1 = os.path.join(input_path, images[interval[0]])
+    path2 = os.path.join(input_path, images[interval[1]])
+    img1 = cv2.imread(path1, 1)
+    img2 = cv2.imread(path2, 1)
+
+    for enum, array in enumerate([img1, img2]):
+        if not array.size:
+            raise FileNotFoundError("array was empty for " + [path1, path2][enum])
+
+    remap1, remap2 = warp_images(img1, img2, savedir, look_at_angle)
+    img1 = slice_eqimage(resize(img1, 50), look_at_angle)
+    img2 = slice_eqimage(resize(img2, 50), look_at_angle)
+
+    # crop poles to remove distortions
+    remap1 = crop_poles(remap1)
+    remap2 = crop_poles(remap2)
+    img1 = crop_poles(img1)
+    img2 = crop_poles(img2)
+
+    ssim = (calculate_ssim(img1, remap2) + calculate_ssim(img2, remap1)) / 2
+    psnr = (calculate_psnr(img1, remap2) + calculate_psnr(img2, remap1)) / 2
+
+    return ssim, psnr
+
+def crop_poles(img):
+    margin = round(0.05 * img.shape[0])
+    return img[margin:img.shape[0] - margin]

Python/preprocessing/circleselector/datatypes.py

@@ -0,0 +1,165 @@
+import csv
+import json
+
+import numpy as np
+from skimage.feature import peak_local_max
+
+
+class PointDict(list):
+    """
+    A class to contain the calculated data from Metrics.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.keys = []
+        if len(self) > 0:
+            self.keys = list(self[0].keys())
+
+    def __str__(self):
+        output = []
+        if self.keys is None:
+            self.keys = list(self[0].keys())
+        headings = self.keys[0]
+        for key in self.keys[1:]:
+            headings += ' ' + key
+        output.append(headings)
+        for dct in self:
+            lst = []
+            for key in dct:
+                lst.append(str(dct[key]))
+            output.append(', '.join(lst))
+        return '\n'.join(output)
+
+    def split_interval(self):
+        """
+        Returns a point dict with the keys: interval_start, interval_end, errors
+        :return: PointDict
+        """
+        intervals = self.get("interval")
+        data = PointDict()
+        data.set("interval_start", [interval[0] for interval in intervals])
+        data.set("interval_end", [interval[1] for interval in intervals])
+        for key in self.keys[1:]:
+            data.set(key, self.get(key))
+        return data
+
+    def toCSV(self, filename):
+        self.keys = self[0].keys()
+        with open(filename, 'w', newline='') as output_file:
+            dict_writer = csv.DictWriter(output_file, self.keys)
+            dict_writer.writeheader()
+            dict_writer.writerows(self)
+
+    def toJSON(self, filename):
+        with open(filename, 'w') as fout:
+            json.dump(self, fout)
+
+    def fromJSON(self, filename):
+        with open(filename, 'r') as fin:
+            data = PointDict(json.load(fin))
+            self.extend(data)
+            self.keys = data.keys
+
+    def get(self, key: str) -> list:
+        """
+        will return a list of the specified error if found in self.keys
+
+        :param key: one of errors in point dict (check self.keys)
+        :return: lst
+        """
+        return [dct[key] for dct in self]
+
+    def set(self, key: str, vals: list):
+        """
+        will assign the values given in vals to the keys in self
+
+        :param key: interval,endpoint_error,...etc
+        :param vals: list of vals (floats)
+        :return: None
+        """
+        if key not in self.keys:
+            self.keys.append(key)
+        if len(self) == 0:
+            for val in vals:
+                self.append({key: float(val)})
+            return
+        for idx in range(len(self)):
+            self[idx][key] = float(vals[idx])
+
+    def find_local_minima(self, numpoints=None, errors: list = None, save_sum=True):
+        """
+        Will find the intervals in the data that score best on the metrics calculated on run_on_interval method
+        in the Metrics class.
+
+        :param numpoints: ~1k points in path. if None, will calculate it by iterating through the point dict and finding
+        the max value in "intervals".
+        :param errors: what errors to include
+        :param save_sum: will save the sum under "summed_errors" in the list
+        :return: PointDict
+        """
+        # find the total number of cameras in the camera path (e.g when the frame_trajectory.txt file is unavailable).
+        if numpoints is None:
+            numpoints = 0
+            for dct in self:
+                numpoints = max((dct["interval"][0], dct['interval'][1], numpoints)) + 1
+
+        if errors is None:
+            errors = list(self.keys)[1:]  # ignores the 'interval' key.
+
+        # generate a list of summed errors
+        lst = []
+        for dct in self:
+            lst.append(sum([dct[error] for error in errors]))
+        if save_sum:
+            self.set("summed_errors", lst)
+
+        # generate the heatmap
+        arr = np.zeros((numpoints, numpoints))
+        for enum, dct in enumerate(self):
+            arr[dct["interval"][0], dct["interval"][1]] = 1 / lst[enum]
+
+        # get the coordinates of the local maxima
+        coords = peak_local_max(arr, min_distance=10, threshold_rel=0.5)
+
+        # set up a boolean array to speed up finding what interval the local maxima
+        # correspond to.
+        bool_arr = np.zeros(arr.shape)
+        for coord in coords:
+            bool_arr[coord[0], coord[1]] = 1
+
+        intervals = []
+        for dct in self:
+            if bool_arr[dct["interval"][0], dct["interval"][1]]:
+                intervals.append(dct)
+
+        return PointDict(intervals)
+
+    def find_best_interval(self) -> dict:
+        """
+        Should only be called once the cv metrics have been run.
+
+        :return: the dict that has lowest ssim + psnr
+        """
+
+        for item in ["ssim", "psnr"]:
+            if item not in self.keys:
+                raise Exception(item + " not found in keys.")
+
+        combined_cv_error = []
+        for item in self:
+            combined_cv_error.append(item["ssim"] + item["psnr"] / 100)
+
+        self.set("combined_cv_error", combined_cv_error)
+        self.sort(key=lambda dct: dct["combined_cv_error"], reverse=True)
+
+        return self[0]
+
+
+class CameraCenters(list):
+    """
+    A list of 3d array-like values representing the locations
+    of the camera centers with their orientations (in Quats).
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.orientations = []