Roi plugin

.gitignore

@@ -55,4 +55,5 @@ deployment/pupil_v*
 *.pyd
 *.dll
 win_drv
-.vs
+.vs
+.venv

pupil_src/launchables/eye.py

@@ -129,7 +129,6 @@ def eye(
         from gl_utils import make_coord_system_pixel_based
         from gl_utils import make_coord_system_norm_based
         from gl_utils import is_window_visible, glViewport
-        from ui_roi import UIRoi
 
         # monitoring
         import psutil
@@ -145,6 +144,7 @@ def eye(
         from av_writer import JPEG_Writer, MPEG_Writer
         from ndsi import H264Writer
         from video_capture import source_classes, manager_classes
+        from roi import Roi
 
         from background_helper import IPC_Logging_Task_Proxy
         from pupil_detector_plugins import available_detector_plugins
@@ -175,7 +175,6 @@ def interrupt_handler(sig, frame):
 
         icon_bar_width = 50
         window_size = None
-        camera_render_size = None
         hdpi_factor = 1.0
 
         # g_pool holds variables for this process
@@ -188,6 +187,7 @@ def interrupt_handler(sig, frame):
         g_pool.eye_id = eye_id
         g_pool.process = f"eye{eye_id}"
         g_pool.timebase = timebase
+        g_pool.camera_render_size = None
 
         g_pool.ipc_pub = ipc_socket
 
@@ -202,7 +202,7 @@ def get_timestamp():
             manager_classes
             + source_classes
             + available_detectors
-            + [PupilDetectorManager]
+            + [PupilDetectorManager, Roi]
         )
         g_pool.plugin_by_name = {p.__name__: p for p in plugins}
 
@@ -229,20 +229,20 @@ def get_timestamp():
             # Detector needs to be loaded first to set `g_pool.pupil_detector`
             (default_detector_cls.__name__, {}),
             ("PupilDetectorManager", {}),
+            ("Roi", {}),
         ]
 
         # Callback functions
         def on_resize(window, w, h):
             nonlocal window_size
-            nonlocal camera_render_size
             nonlocal hdpi_factor
 
             active_window = glfw.glfwGetCurrentContext()
             glfw.glfwMakeContextCurrent(window)
             hdpi_factor = glfw.getHDPIFactor(window)
             g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
             window_size = w, h
-            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
+            g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
             g_pool.gui.update_window(w, h)
             g_pool.gui.collect_menus()
             for g in g_pool.graphs:
@@ -264,16 +264,18 @@ def on_window_mouse_button(window, button, action, mods):
             g_pool.gui.update_button(button, action, mods)
 
         def on_pos(window, x, y):
-            x *= hdpi_factor
-            y *= hdpi_factor
+            x, y = x * hdpi_factor, y * hdpi_factor
             g_pool.gui.update_mouse(x, y)
 
-            if g_pool.u_r.active_edit_pt:
-                pos = normalize((x, y), camera_render_size)
-                if g_pool.flip:
-                    pos = 1 - pos[0], 1 - pos[1]
-                pos = denormalize(pos, g_pool.capture.frame_size)
-                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
+            pos = x, y
+            pos = normalize(pos, g_pool.camera_render_size)
+            if g_pool.flip:
+                pos = 1 - pos[0], 1 - pos[1]
+            # Position in img pixels
+            pos = denormalize(pos, g_pool.capture.frame_size)
+
+            for p in g_pool.plugins:
+                p.on_pos(pos)
 
         def on_scroll(window, x, y):
             g_pool.gui.update_scroll(x, y * scroll_factor)
@@ -371,32 +373,6 @@ def set_window_size():
             f_width += int(icon_bar_width * g_pool.gui.scale)
             glfw.glfwSetWindowSize(main_window, f_width, f_height)
 
-        def uroi_on_mouse_button(button, action, mods):
-            if g_pool.display_mode == "roi":
-                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
-                    g_pool.u_r.active_edit_pt = False
-                    # if the roi interacts we dont want
-                    # the gui to interact as well
-                    return
-                elif action == glfw.GLFW_PRESS:
-                    x, y = glfw.glfwGetCursorPos(main_window)
-                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
-                    x *= hdpi_factor
-                    y *= hdpi_factor
-                    pos = normalize((x, y), camera_render_size)
-                    if g_pool.flip:
-                        pos = 1 - pos[0], 1 - pos[1]
-                    # Position in img pixels
-                    pos = denormalize(
-                        pos, g_pool.capture.frame_size
-                    )  # Position in img pixels
-                    if g_pool.u_r.mouse_over_edit_pt(
-                        pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
-                    ):
-                        # if the roi interacts we dont want
-                        # the gui to interact as well
-                        return
-
         general_settings.append(ui.Button("Reset window size", set_window_size))
         general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
         general_settings.append(
@@ -439,11 +415,6 @@ def uroi_on_mouse_button(button, action, mods):
 
         g_pool.writer = None
 
-        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
-        roi_user_settings = session_settings.get("roi")
-        if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
-            g_pool.u_r.set(roi_user_settings)
-
         # Register callbacks main_window
         glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
         glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
@@ -563,19 +534,6 @@ def window_should_update():
 
             frame = event.get("frame")
             if frame:
-                f_width, f_height = g_pool.capture.frame_size
-                # TODO: Roi should be its own plugin. This way we could put it at the
-                # appropriate order for recent_events() to process frame resolution
-                # changes immediately after the backend.
-                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
-                    f_height,
-                    f_width,
-                ):
-                    g_pool.pupil_detector.on_resolution_change(
-                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
-                        g_pool.capture.frame_size,
-                    )
-                    g_pool.u_r = UIRoi((f_height, f_width))
                 if should_publish_frames:
                     try:
                         if frame_publish_format == "jpeg":
@@ -631,16 +589,10 @@ def window_should_update():
                     glfw.glfwMakeContextCurrent(main_window)
                     clear_gl_screen()
 
-                    glViewport(0, 0, *camera_render_size)
+                    glViewport(0, 0, *g_pool.camera_render_size)
                     for p in g_pool.plugins:
                         p.gl_display()
 
-                    glViewport(0, 0, *camera_render_size)
-                    # render the ROI
-                    g_pool.u_r.draw(g_pool.gui.scale)
-                    if g_pool.display_mode == "roi":
-                        g_pool.u_r.draw_points(g_pool.gui.scale)
-
                     glViewport(0, 0, *window_size)
                     # render graphs
                     fps_graph.draw()
@@ -662,13 +614,12 @@ def window_should_update():
                     for button, action, mods in user_input.buttons:
                         x, y = glfw.glfwGetCursorPos(main_window)
                         pos = x * hdpi_factor, y * hdpi_factor
-                        pos = normalize(pos, camera_render_size)
+                        pos = normalize(pos, g_pool.camera_render_size)
+                        if g_pool.flip:
+                            pos = 1 - pos[0], 1 - pos[1]
                         # Position in img pixels
                         pos = denormalize(pos, g_pool.capture.frame_size)
 
-                        # TODO: remove when ROI is plugin
-                        uroi_on_mouse_button(button, action, mods)
-
                         for plugin in g_pool.plugins:
                             if plugin.on_click(pos, button, action):
                                 break
@@ -698,7 +649,6 @@ def window_should_update():
         session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
         # save session persistent settings
         session_settings["gui_scale"] = g_pool.gui_user_scale
-        session_settings["roi"] = g_pool.u_r.get()
         session_settings["flip"] = g_pool.flip
         session_settings["display_mode"] = g_pool.display_mode
         session_settings["ui_config"] = g_pool.gui.configuration

pupil_src/shared_modules/methods.py

@@ -54,54 +54,6 @@ def get_dt():
         yield dt
 
 
-class Roi(object):
-    """this is a simple 2D Region of Interest class
-    it is applied on numpy arrays for convenient slicing
-    like this:
-
-    roi_array_slice = full_array[r.view]
-    # do something with roi_array_slice
-
-    this creates a view, no data copying done
-    """
-
-    def __init__(self, array_shape):
-        self.array_shape = array_shape
-        self.lX = 0
-        self.lY = 0
-        self.uX = array_shape[1]
-        self.uY = array_shape[0]
-        self.nX = 0
-        self.nY = 0
-
-    @property
-    def view(self):
-        return slice(self.lY, self.uY), slice(self.lX, self.uX)
-
-    def add_vector(self, vector):
-        """
-        adds the roi offset to a len2 vector
-        """
-        x, y = vector
-        return (self.lX + x, self.lY + y)
-
-    def sub_vector(self, vector):
-        """
-        subs the roi offset to a len2 vector
-        """
-        x, y = vector
-        return (x - self.lX, y - self.lY)
-
-    def set(self, vals):
-        if vals is not None and len(vals) is 5:
-            self.lX, self.lY, self.uX, self.uY, self.array_shape = vals
-        elif vals is not None and len(vals) is 4:
-            self.lX, self.lY, self.uX, self.uY = vals
-
-    def get(self):
-        return self.lX, self.lY, self.uX, self.uY, self.array_shape
-
-
 def project_distort_pts(
     pts_xyz,
     camera_matrix,

pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py

@@ -47,17 +47,8 @@ def __init__(
         self.proxy = PropertyProxy(self.detector_2d)
 
     def detect(self, frame):
-        roi = Roi(*self.g_pool.u_r.get()[:4])
-        if (
-            not 0 <= roi.x_min <= roi.x_max < frame.width
-            or not 0 <= roi.y_min <= roi.y_max < frame.height
-        ):
-            # TODO: Invalid ROIs can occur when switching camera resolutions, because we
-            # adjust the roi only after all plugin recent_events() have been called.
-            # Optimally we make a plugin out of the ROI and call its recent_events()
-            # immediately after the backend, before the detection.
-            logger.debug(f"Invalid Roi {roi} for img {frame.width}x{frame.height}!")
-            return None
+        # convert roi-plugin to detector roi
+        roi = Roi(*self.g_pool.roi.bounds)
 
         debug_img = frame.bgr if self.g_pool.display_mode == "algorithm" else None
         result = self.detector_2d.detect(

pupil_src/shared_modules/pupil_detector_plugins/detector_3d_plugin.py

@@ -49,17 +49,8 @@ def __init__(
         self.debugVisualizer3D = Eye_Visualizer(g_pool, self.detector_3d.focal_length())
 
     def detect(self, frame):
-        roi = Roi(*self.g_pool.u_r.get()[:4])
-        if (
-            not 0 <= roi.x_min <= roi.x_max < frame.width
-            or not 0 <= roi.y_min <= roi.y_max < frame.height
-        ):
-            # TODO: Invalid ROIs can occur when switching camera resolutions, because we
-            # adjust the roi only after all plugin recent_events() have been called.
-            # Optimally we make a plugin out of the ROI and call its recent_events()
-            # immediately after the backend, before the detection.
-            logger.debug(f"Invalid Roi {roi} for img {frame.width}x{frame.height}!")
-            return None
+        # convert roi-plugin to detector roi
+        roi = Roi(*self.g_pool.roi.bounds)
 
         debug_img = frame.bgr if self.g_pool.display_mode == "algorithm" else None
         result = self.detector_3d.detect(

pupil_src/shared_modules/pupil_detector_plugins/detector_base_plugin.py

@@ -54,13 +54,20 @@ def __init__(self, g_pool):
             "pupil_detector.broadcast_properties": self.handle_broadcast_properties_notification,
             "pupil_detector.set_property": self.handle_set_property_notification,
         }
+        self._last_frame_size = None
 
     def recent_events(self, event):
         frame = event.get("frame")
         if not frame:
             self._recent_detection_result = None
             return
 
+        frame_size = (frame.width, frame.height)
+        if frame_size != self._last_frame_size:
+            if self._last_frame_size is not None:
+                self.on_resolution_change(self._last_frame_size, frame_size)
+            self._last_frame_size = frame_size
+
         detection_result = self.detect(frame=frame)
         event["pupil_detection_result"] = detection_result
         self._recent_detection_result = detection_result
@@ -105,27 +112,35 @@ def handle_set_property_notification(self, notification):
                 )
             elif property_name == "roi":
                 # Modify the ROI with the values sent over network
+
                 try:
-                    minX, maxX, minY, maxY = property_value
+                    minX, minY, maxX, maxY = property_value
                 except (ValueError, TypeError) as err:
                     # NOTE: ValueError gets throws when length of the tuple does not
                     # match. TypeError gets thrown when it is not a tuple.
                     raise ValueError(
-                        "ROI needs to be 4 integers: (minX, maxX, minY, maxY)"
+                        "ROI needs to be 4 integers: (minX, minY, maxX, maxY)"
                     ) from err
-                if minX > maxX or minY > maxY:
-                    raise ValueError("ROI malformed: minX > maxX or minY > maxY!")
-                ui_roi = self.g_pool.u_r
-                ui_roi.lX = max(ui_roi.min_x, int(minX))
-                ui_roi.lY = max(ui_roi.min_y, int(minY))
-                ui_roi.uX = min(ui_roi.max_x, int(maxX))
-                ui_roi.uY = min(ui_roi.max_y, int(maxY))
+
+                # Apply very strict error checking here, although roi deal with invalid
+                # values, so the user gets immediate feedback and does not wonder why
+                # something did not work as expected.
+                width, height = self.g_pool.roi.frame_size
+                if not ((0 <= minX < maxX < width) and (0 <= minY < maxY <= height)):
+                    raise ValueError(
+                        "Received ROI with invalid dimensions!"
+                        f" (minX={minX}, minY={minY}, maxX={maxX}, maxY={maxY})"
+                        f" for frame size ({width} x {height})"
+                    )
+
+                self.g_pool.roi.bounds = (minX, minY, maxX, maxY)
+
             else:
                 raise KeyError(
                     "Notification subject does not "
                     "specifiy detector type nor modify ROI."
                 )
-            logger.debug(f"`{property_name}` property set to {property_value}")
+            logger.debug(f"'{property_name}' property set to {property_value}")
         except KeyError:
             logger.error("Malformed notification received")
             logger.debug(traceback.format_exc())