coxa location videos for BRSA templates

examples/df3d_coxa_location_annotations.py

@@ -0,0 +1,109 @@
+import os.path
+import importlib.util
+import pickle
+
+import numpy as np
+
+import utils_video
+import utils_video.utils
+import utils_video.generators
+
+import deepfly.plot_util
+from deepfly.procrustes import procrustes_seperate
+
+from df3dPostProcessing import df3dPostProcess
+
+spec = importlib.util.spec_from_file_location(
+    "module.name", "/home/aymanns/BRSA/examples/annotations.py"
+)
+annotations = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(annotations)
+
+my_points = []
+clc_points = []
+sg_points = []
+
+for beh, data in annotations.annotations["my annotations"].items():
+    for exp in data:
+        directory = exp["directory"]
+        path_to_processed_points = os.path.join(
+            directory, "behData/images/df3d/points3D.csv"
+        )
+        points3D = np.genfromtxt(path_to_processed_points, delimiter=",")
+        points3D = points3D.reshape((points3D.shape[0], -1, 3))
+        my_points.append(points3D[:10])
+
+
+for beh, data in annotations.annotations["Chin-Lin's data"].items():
+    for exp in data:
+        directory = exp["directory"]
+        path_to_processed_points = os.path.join(directory, "images/df3d_2/points3D.csv")
+        points3D = np.genfromtxt(path_to_processed_points, delimiter=",")
+        points3D = points3D.reshape((points3D.shape[0], -1, 3))
+        clc_points.append(points3D[:10])
+
+
+with open("/home/aymanns/BRSA/examples/pose_result_smooth.pkl", "rb") as f:
+    semihs_data = pickle.load(f)
+
+for beh, data in annotations.annotations["semih's annotations"].items():
+    for exp in data:
+        key = exp["key"]
+        points3D = semihs_data[key]
+        # points3D = procrustes_seperate(points3D.copy())
+        # points3D = deepfly.plot_util.rotate_points3d(points3D)
+        sg_points.append(points3D[:10])
+
+# print(np.array(points).shape)
+# exit()
+
+points = np.array(my_points + clc_points + sg_points)
+coxa_indices = np.array([0, 5, 10, 19, 24, 29])
+coxa_points = points[:, :, coxa_indices, :]
+coxa_points = coxa_points.reshape((-1, 3))
+centroid = np.mean(coxa_points, axis=0)
+coxa_points = coxa_points - centroid
+U, S, VT = np.linalg.svd(np.transpose(coxa_points))
+projected_coxa_points = np.transpose(np.dot(np.transpose(U), np.transpose(coxa_points)))
+mins = np.min(projected_coxa_points, axis=0)
+maxs = np.max(projected_coxa_points, axis=0)
+
+
+def coxa_locations(points3d, U, mins, maxs, labels=None):
+    # allow for multiple experiments to be shown
+    if points3d.ndim == 3:
+        points3d = points3d[
+            np.newaxis,
+        ]
+
+    n_exp = points3d.shape[0]
+
+    coxa_indices = np.array([0, 5, 10, 19, 24, 29])
+    coxa_points = points3d[:, :, coxa_indices, :]
+    coxa_points = coxa_points.reshape((-1, 3))
+    centroid = np.mean(coxa_points, axis=0)
+    coxa_points = coxa_points - centroid
+    # U, S, VT = np.linalg.svd(np.transpose(coxa_points))
+    # print("U:", U)
+    projected_coxa_points = np.transpose(
+        np.dot(np.transpose(U), np.transpose(coxa_points))
+    )
+    # mins = np.min(projected_coxa_points, axis=0)
+    # maxs = np.max(projected_coxa_points, axis=0)
+    projected_coxa_points = projected_coxa_points.reshape(
+        [n_exp, -1, len(coxa_indices), 3]
+    )
+    for frame_idx in range(projected_coxa_points.shape[1]):
+        yield utils_video.utils.plot_coxa_positions(
+            projected_coxa_points[:, frame_idx], mins, maxs, labels
+        )
+
+
+# generator = coxa_locations(np.array(points), labels=None)
+my_generator = coxa_locations(np.array(my_points), U, mins, maxs, labels=None)
+clc_generator = coxa_locations(np.array(clc_points), U, mins, maxs, labels=None)
+sg_generator = coxa_locations(np.array(sg_points), U, mins, maxs, labels=None)
+generator = utils_video.generators.stack([my_generator, clc_generator, sg_generator])
+utils_video.make_video(
+    "coxa_location_all_of_my_and_clcs_annotations.mp4", generator, fps=30
+)

examples/df3d_coxa_location_annotations_df3dPostProcessing.py

@@ -0,0 +1,232 @@
+import os.path
+import importlib.util
+import pickle
+import glob
+
+import numpy as np
+
+import utils_video
+import utils_video.utils
+import utils_video.generators
+
+import deepfly.plot_util
+from deepfly.procrustes import procrustes_seperate
+
+from df3dPostProcessing import df3dPostProcess
+
+
+spec = importlib.util.spec_from_file_location(
+    "module.name", "/home/aymanns/BRSA/examples/annotations.py"
+)
+annotations = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(annotations)
+
+keys = []
+for beh, data in annotations.annotations["semih's annotations"].items():
+    for exp in data:
+        key = exp["key"]
+        keys.append(key)
+files = ""
+keys = set(keys)
+for key in keys:
+    key = key[len("pose_result__data_paper_") :]
+    fly_index = key.find("Fly")
+    path = f"/data/paper/{key[:fly_index-1]}/{key[fly_index:fly_index+4]}/{key[fly_index+5:fly_index+12]}/behData/images.zip"
+    files += path + "\n"
+    print(path)
+with open("df3d_paper_annotated_experiments.sh", "w") as f:
+    f.write(files)
+exit()
+
+
+def get_most_recent_pose_result(folder):
+    possible_pose_results = glob.glob(os.path.join(directory, "pose_result*.pkl"))
+    change_times = [os.stat(path).st_mtime for path in possible_pose_results]
+    try:
+        most_recent_pose_result = possible_pose_results[np.argmax(change_times)]
+    except ValueError:
+        print("skipped because df3d output is missing")
+        most_recent_pose_result = None
+    return most_recent_pose_result
+
+
+def convert_to_df3d_output_format(aligned):
+    points3D = []
+    points3D.append(aligned["LF_leg"]["Coxa"]["raw_pos_aligned"])
+    points3D.append(aligned["LF_leg"]["Femur"]["raw_pos_aligned"])
+    points3D.append(aligned["LF_leg"]["Tibia"]["raw_pos_aligned"])
+    points3D.append(aligned["LF_leg"]["Tarsus"]["raw_pos_aligned"])
+    points3D.append(aligned["LF_leg"]["Claw"]["raw_pos_aligned"])
+
+    points3D.append(aligned["LM_leg"]["Coxa"]["raw_pos_aligned"])
+    points3D.append(aligned["LM_leg"]["Femur"]["raw_pos_aligned"])
+    points3D.append(aligned["LM_leg"]["Tibia"]["raw_pos_aligned"])
+    points3D.append(aligned["LM_leg"]["Tarsus"]["raw_pos_aligned"])
+    points3D.append(aligned["LM_leg"]["Claw"]["raw_pos_aligned"])
+
+    points3D.append(aligned["LH_leg"]["Coxa"]["raw_pos_aligned"])
+    points3D.append(aligned["LH_leg"]["Femur"]["raw_pos_aligned"])
+    points3D.append(aligned["LH_leg"]["Tibia"]["raw_pos_aligned"])
+    points3D.append(aligned["LH_leg"]["Tarsus"]["raw_pos_aligned"])
+    points3D.append(aligned["LH_leg"]["Claw"]["raw_pos_aligned"])
+
+    # Left antenna
+    points3D.append(np.zeros_like(aligned["LH_leg"]["Claw"]["raw_pos_aligned"]))
+    # Left stripe 1
+    points3D.append(np.zeros_like(aligned["LH_leg"]["Claw"]["raw_pos_aligned"]))
+    # Left stripe 2
+    points3D.append(np.zeros_like(aligned["LH_leg"]["Claw"]["raw_pos_aligned"]))
+    # Left stripe 3
+    points3D.append(np.zeros_like(aligned["LH_leg"]["Claw"]["raw_pos_aligned"]))
+
+    points3D.append(aligned["RF_leg"]["Coxa"]["raw_pos_aligned"])
+    points3D.append(aligned["RF_leg"]["Femur"]["raw_pos_aligned"])
+    points3D.append(aligned["RF_leg"]["Tibia"]["raw_pos_aligned"])
+    points3D.append(aligned["RF_leg"]["Tarsus"]["raw_pos_aligned"])
+    points3D.append(aligned["RF_leg"]["Claw"]["raw_pos_aligned"])
+
+    points3D.append(aligned["RM_leg"]["Coxa"]["raw_pos_aligned"])
+    points3D.append(aligned["RM_leg"]["Femur"]["raw_pos_aligned"])
+    points3D.append(aligned["RM_leg"]["Tibia"]["raw_pos_aligned"])
+    points3D.append(aligned["RM_leg"]["Tarsus"]["raw_pos_aligned"])
+    points3D.append(aligned["RM_leg"]["Claw"]["raw_pos_aligned"])
+
+    points3D.append(aligned["RH_leg"]["Coxa"]["raw_pos_aligned"])
+    points3D.append(aligned["RH_leg"]["Femur"]["raw_pos_aligned"])
+    points3D.append(aligned["RH_leg"]["Tibia"]["raw_pos_aligned"])
+    points3D.append(aligned["RH_leg"]["Tarsus"]["raw_pos_aligned"])
+    points3D.append(aligned["RH_leg"]["Claw"]["raw_pos_aligned"])
+
+    # Right antenna
+    points3D.append(np.zeros_like(aligned["RH_leg"]["Claw"]["raw_pos_aligned"]))
+    # Right stripe 1
+    points3D.append(np.zeros_like(aligned["RH_leg"]["Claw"]["raw_pos_aligned"]))
+    # Right stripe 2
+    points3D.append(np.zeros_like(aligned["RH_leg"]["Claw"]["raw_pos_aligned"]))
+    # Right stripe 3
+    points3D.append(np.zeros_like(aligned["RH_leg"]["Claw"]["raw_pos_aligned"]))
+
+    points3D = np.array(points3D)
+    points3D = np.swapaxes(points3D, 0, 1)
+    return points3D
+
+
+my_points = []
+clc_points = []
+sg_points = []
+
+for beh, data in annotations.annotations["my annotations"].items():
+    for exp in data:
+        directory = exp["directory"]
+        directory = os.path.join(directory, "behData/images/df3d/")
+        print(directory)
+        most_recent_pose_result = get_most_recent_pose_result(directory)
+        if most_recent_pose_result is None:
+            continue
+        df3dPost = df3dPostProcess(most_recent_pose_result)
+        aligned = df3dPost.align_3d_data()
+        points3D = convert_to_df3d_output_format(aligned)
+        my_points.append(points3D[:10])
+        # print(type(aligned))
+        # print(aligned.keys())
+        # print(type(aligned["LF_leg"]))
+        # print(aligned["LF_leg"].keys())
+        # print(type(aligned["LF_leg"]["Coxa"]))
+        # print(aligned["LF_leg"]["Coxa"].keys())
+        # print(aligned["LF_leg"]["Coxa"]["fixed_pos_aligned"])
+        # print(aligned["LF_leg"]["Coxa"]["raw_pos_aligned"])
+        # print(aligned["LF_leg"]["Coxa"]["mean_length"])
+        # print(aligned["LF_leg"]["Tibia"].keys())
+        ## print(aligned["LF_leg"]["Tibia"]["fixed_pos_aligned"])
+        # print(aligned["LF_leg"]["Tibia"]["raw_pos_aligned"])
+        # print(aligned["LF_leg"]["Tibia"]["mean_length"])
+        # points3D = aligned["points3D"]
+        # print(points3D.shape)
+        # exit()
+
+
+for beh, data in annotations.annotations["Chin-Lin's data"].items():
+    for exp in data:
+        directory = exp["directory"]
+        directory = os.path.join(directory, "images/df3d_2/")
+        print(directory)
+        most_recent_pose_result = get_most_recent_pose_result(directory)
+        if most_recent_pose_result is None:
+            continue
+        df3dPost = df3dPostProcess(most_recent_pose_result)
+        aligned = df3dPost.align_3d_data()
+        points3D = convert_to_df3d_output_format(aligned)
+        clc_points.append(points3D[:10])
+
+
+for beh, data in annotations.annotations["semih's annotations"].items():
+    for exp in data:
+        key = exp["key"]
+        print(key)
+        pose_result_file = os.path.join(
+            "/home/aymanns/utils_video/examples/pose_result_df3d_paper", key
+        )
+        df3dPost = df3dPostProcess(pose_result_file)
+        aligned = df3dPost.align_3d_data(rescale=False)
+        points3D = convert_to_df3d_output_format(aligned)
+        sg_points.append(points3D[:10])
+
+# print(np.array(points).shape)
+# exit()
+
+# print(type(my_points), len(my_points))
+# print(type(clc_points), len(clc_points))
+# print(type(sg_points), len(sg_points))
+# print(my_points[0].shape)
+# print(clc_points[0].shape)
+# print(sg_points[0].shape)
+points = np.array(my_points + clc_points + sg_points)
+coxa_indices = np.array([0, 5, 10, 19, 24, 29])
+coxa_points = points[:, :, coxa_indices, :]
+coxa_points = coxa_points.reshape((-1, 3))
+centroid = np.mean(coxa_points, axis=0)
+coxa_points = coxa_points - centroid
+U, S, VT = np.linalg.svd(np.transpose(coxa_points))
+projected_coxa_points = np.transpose(np.dot(np.transpose(U), np.transpose(coxa_points)))
+mins = np.min(projected_coxa_points, axis=0)
+maxs = np.max(projected_coxa_points, axis=0)
+
+
+def coxa_locations(points3d, U, mins, maxs, labels=None):
+    # allow for multiple experiments to be shown
+    if points3d.ndim == 3:
+        points3d = points3d[
+            np.newaxis,
+        ]
+
+    n_exp = points3d.shape[0]
+
+    coxa_indices = np.array([0, 5, 10, 19, 24, 29])
+    coxa_points = points3d[:, :, coxa_indices, :]
+    coxa_points = coxa_points.reshape((-1, 3))
+    centroid = np.mean(coxa_points, axis=0)
+    coxa_points = coxa_points - centroid
+    # U, S, VT = np.linalg.svd(np.transpose(coxa_points))
+    # print("U:", U)
+    projected_coxa_points = np.transpose(
+        np.dot(np.transpose(U), np.transpose(coxa_points))
+    )
+    # mins = np.min(projected_coxa_points, axis=0)
+    # maxs = np.max(projected_coxa_points, axis=0)
+    projected_coxa_points = projected_coxa_points.reshape(
+        [n_exp, -1, len(coxa_indices), 3]
+    )
+    for frame_idx in range(projected_coxa_points.shape[1]):
+        yield utils_video.utils.plot_coxa_positions(
+            projected_coxa_points[:, frame_idx], mins, maxs, labels
+        )
+
+
+# generator = coxa_locations(np.array(points), labels=None)
+my_generator = coxa_locations(np.array(my_points), U, mins, maxs, labels=None)
+clc_generator = coxa_locations(np.array(clc_points), U, mins, maxs, labels=None)
+sg_generator = coxa_locations(np.array(sg_points), U, mins, maxs, labels=None)
+generator = utils_video.generators.stack([my_generator, clc_generator, sg_generator])
+utils_video.make_video(
+    "coxa_location_all_of_my_and_clcs_annotations.mp4", generator, fps=30
+)

utils_video/main.py

@@ -21,6 +21,12 @@ def make_video(video_path, frame_generator, fps, output_shape=(-1, 2880), n_fram
     fps : int
         Frame rate in frames per second.
     """
+    if fps.is_integer() and int(fps) != 1 and (int(fps) & (int(fps) - 1)) == 0:
+        import warnings
+
+        warnings.warn(
+            f"Frame rate {fps} is a power of 2. This can result in faulty video files."
+        )
     fourcc = cv2.VideoWriter_fourcc(*"mp4v")
     first_frame = next(frame_generator)
     frame_generator = itertools.chain([first_frame], frame_generator)