Add exposure mode (auto/manual) for Pupil Cam 2

pupil_src/shared_modules/video_capture/utils.py

@@ -10,8 +10,52 @@
 '''
 
 import numpy as np
+import cv2
 
 
+class Exposure_Time(object):
+    def __init__(self, max_ET, frame_rate, mode="manual"):
+        self.mode = mode
+        self.ET_thres = 1, min(10000 / frame_rate, max_ET)
+        self.last_ET = self.ET_thres[1]
+
+        self.targetY_thres = 90, 150
+
+        self.AE_Win = np.array([[3, 1, 1, 1, 1, 1, 1, 3],
+                                [3, 1, 1, 1, 1, 1, 1, 3],
+                                [2, 1, 1, 1, 1, 1, 1, 2],
+                                [2, 1, 1, 1, 1, 1, 1, 2],
+                                [2, 1, 1, 1, 1, 1, 1, 2],
+                                [2, 1, 1, 1, 1, 1, 1, 2],
+                                [3, 1, 1, 1, 1, 1, 1, 3],
+                                [3, 1, 1, 1, 1, 1, 1, 3]])
+        self.smooth = 1/3
+        self.check_freq = 0.1/3
+        self.last_check_timestamp = None
+
+    def calculate_based_on_frame(self, frame):
+        if self.last_check_timestamp is None:
+            self.last_check_timestamp = frame.timestamp
+
+        if frame.timestamp - self.last_check_timestamp > self.check_freq:
+            if self.mode == "manual":
+                self.last_ET = self.ET_thres[1]
+                return self.ET_thres[1]
+            elif self.mode == "auto":
+                image_block = cv2.resize(frame.gray, dsize=self.AE_Win.shape)
+                YTotal = max(np.multiply(self.AE_Win, image_block).sum() / self.AE_Win.sum(), 1)
+
+                if YTotal < self.targetY_thres[0]:
+                    targetET = self.last_ET * self.targetY_thres[0] / YTotal
+                elif YTotal > self.targetY_thres[1]:
+                    targetET = self.last_ET * self.targetY_thres[1] / YTotal
+                else:
+                    targetET = self.last_ET
+
+                next_ET = np.clip(self.last_ET + (targetET - self.last_ET) * self.smooth, self.ET_thres[0], self.ET_thres[1])
+                self.last_ET = next_ET
+                return next_ET
+
 class Check_Frame_Stripes(object):
     def __init__(self, check_freq_init=0.1, check_freq_upperbound=5, check_freq_lowerbound=0.00001, factor=0.8):
         self.check_freq_init = check_freq_init

pupil_src/shared_modules/video_capture/uvc_backend.py

@@ -15,7 +15,7 @@
 from version_utils import VersionFormat
 from .base_backend import InitialisationError, Base_Source, Base_Manager
 from camera_models import load_intrinsics
-from .utils import Check_Frame_Stripes
+from .utils import Check_Frame_Stripes, Exposure_Time
 
 # check versions for our own depedencies as they are fast-changing
 assert VersionFormat(uvc.__version__) >= VersionFormat('0.13')
@@ -29,7 +29,7 @@ class UVC_Source(Base_Source):
     """
     Camera Capture is a class that encapsualtes uvc.Capture:
     """
-    def __init__(self, g_pool, frame_size, frame_rate, name=None, preferred_names=(), uid=None, uvc_controls={}, check_stripes=True):
+    def __init__(self, g_pool, frame_size, frame_rate, name=None, preferred_names=(), uid=None, uvc_controls={}, check_stripes=True, exposure_mode="manual"):
         import platform
 
         super().__init__(g_pool)
@@ -79,20 +79,25 @@ def __init__(self, g_pool, frame_size, frame_rate, name=None, preferred_names=()
 
         # checkframestripes will be initialized accordingly in configure_capture()
         self.check_stripes = check_stripes
+        self.exposure_mode = exposure_mode
         self.checkframestripes = None
+        self.preferred_exposure_time = None
 
         # check if we were sucessfull
         if not self.uvc_capture:
             logger.error("Init failed. Capture is started in ghost mode. No images will be supplied.")
             self.name_backup = preferred_names
             self.frame_size_backup = frame_size
             self.frame_rate_backup = frame_rate
+            self.exposure_time_backup = None
             self._intrinsics = load_intrinsics(self.g_pool.user_dir, self.name, self.frame_size)
         else:
             self.configure_capture(frame_size, frame_rate, uvc_controls)
             self.name_backup = (self.name,)
             self.frame_size_backup = frame_size
             self.frame_rate_backup = frame_rate
+            controls_dict = dict([(c.display_name, c) for c in self.uvc_capture.controls])
+            self.exposure_time_backup = controls_dict['Absolute Exposure Time'].value
         self.backup_uvc_controls = {}
 
     def verify_drivers(self):
@@ -181,12 +186,14 @@ def configure_capture(self, frame_size, frame_rate, uvc_controls):
                 except KeyError: pass
 
         elif ("Pupil Cam2" in self.uvc_capture.name):
+            if self.exposure_mode == "auto":
+                special_settings = {200: 28, 180: 31}
+                controls_dict = dict([(c.display_name, c) for c in self.uvc_capture.controls])
+                self.preferred_exposure_time = Exposure_Time(max_ET=special_settings.get(self.frame_rate, 32), frame_rate=self.frame_rate, mode=self.exposure_mode)
+
             if self.check_stripes:
                 self.checkframestripes = Check_Frame_Stripes()
 
-            try: controls_dict['Auto Exposure Mode'].value = 1
-            except KeyError: pass
-
             try:controls_dict['Saturation'].value = 0
             except KeyError: pass
 
@@ -253,6 +260,11 @@ def recent_events(self, events):
         try:
             frame = self.uvc_capture.get_frame(0.05)
 
+            if self.preferred_exposure_time:
+                target = self.preferred_exposure_time.calculate_based_on_frame(frame)
+                if target is not None:
+                    self.exposure_time = target
+
             if self.checkframestripes and self.checkframestripes.require_restart(frame):
                 # set the self.frame_rate in order to restart
                 self.frame_rate = self.frame_rate
@@ -285,6 +297,7 @@ def get_init_dict(self):
         d['frame_size'] = self.frame_size
         d['frame_rate'] = self.frame_rate
         d['check_stripes'] = self.check_stripes
+        d['exposure_mode'] = self.exposure_mode
         if self.uvc_capture:
             d['name'] = self.name
             d['uvc_controls'] = self._get_uvc_controls()
@@ -342,15 +355,31 @@ def frame_rate(self, new_rate):
         self.frame_rate_backup = rate
 
         if ("Pupil Cam2" in self.uvc_capture.name):
-            controls_dict = dict([(c.display_name, c) for c in self.uvc_capture.controls])
-
             special_settings = {200: 28, 180: 31}
-            try: controls_dict['Absolute Exposure Time'].value = special_settings.get(new_rate, 32)
-            except KeyError: pass
+            if self.exposure_mode == "auto":
+                self.preferred_exposure_time = Exposure_Time(max_ET=special_settings.get(new_rate, 32), frame_rate=new_rate, mode=self.exposure_mode)
+            else:
+                self.exposure_time = min(self.exposure_time, special_settings.get(new_rate, 32))
 
             if self.check_stripes:
                 self.checkframestripes = Check_Frame_Stripes()
 
+    @property
+    def exposure_time(self):
+        if self.uvc_capture:
+            controls_dict = dict([(c.display_name, c) for c in self.uvc_capture.controls])
+            return controls_dict['Absolute Exposure Time'].value
+        else:
+            return self.exposure_time_backup
+
+    @exposure_time.setter
+    def exposure_time(self, new_et):
+        try:
+            controls_dict = dict([(c.display_name, c) for c in self.uvc_capture.controls])
+            if abs(new_et - controls_dict['Absolute Exposure Time'].value) >= 1:
+                controls_dict['Absolute Exposure Time'].value = new_et
+        except KeyError: pass
+
     @property
     def jpeg_support(self):
         return True
@@ -387,6 +416,10 @@ def gui_update_from_device():
         def set_frame_size(new_size):
             self.frame_size = new_size
 
+        def set_frame_rate(new_rate):
+            self.frame_rate = new_rate
+            self.update_menu()
+
         if self.uvc_capture is None:
             ui_elements.append(ui.Info_Text('Capture initialization failed.'))
             self.menu.extend(ui_elements)
@@ -399,16 +432,32 @@ def set_frame_size(new_size):
         image_processing = ui.Growing_Menu(label='Image Post Processing')
         image_processing.collapsed = True
 
-        sensor_control.append(ui.Selector(
-            'frame_size', self,
-            setter=set_frame_size,
-            selection=self.uvc_capture.frame_sizes,
-            label='Resolution'
-        ))
+        sensor_control.append(ui.Selector('frame_size', self, setter=set_frame_size, selection=self.uvc_capture.frame_sizes, label='Resolution'))
 
         def frame_rate_getter():
             return (self.uvc_capture.frame_rates, [str(fr) for fr in self.uvc_capture.frame_rates])
-        sensor_control.append(ui.Selector('frame_rate', self, selection_getter=frame_rate_getter, label='Frame rate'))
+        sensor_control.append(ui.Selector('frame_rate', self, selection_getter=frame_rate_getter, setter=set_frame_rate, label='Frame rate'))
+
+        if ("Pupil Cam2" in self.uvc_capture.name):
+            special_settings = {200: 28, 180: 31}
+
+            def set_exposure_mode(exposure_mode):
+                self.exposure_mode = exposure_mode
+                if self.exposure_mode == "auto":
+                    self.preferred_exposure_time = Exposure_Time(max_ET=special_settings.get(self.frame_rate, 32), frame_rate=self.frame_rate, mode=self.exposure_mode)
+                else:
+                    self.preferred_exposure_time = None
+
+                logger.info("Exposure mode for camera {0} is now set to {1} mode".format(self.uvc_capture.name, exposure_mode))
+                self.update_menu()
+
+            def exposure_mode_getter():
+                return ["manual", "auto"], ["manual mode", "auto mode"]
+            sensor_control.append(ui.Selector('exposure_mode', self, setter=set_exposure_mode, selection_getter=exposure_mode_getter, selection=self.exposure_mode, label="Exposure Mode"))
+
+            sensor_control.append(ui.Slider('exposure_time', self, label='Absolute Exposure Time', min=1, max=special_settings.get(self.frame_rate, 32), step=1))
+            if self.exposure_mode == "auto":
+                sensor_control[-1].read_only = True
 
         if ("Pupil Cam" in self.uvc_capture.name):
             blacklist = ['Auto Focus', 'Absolute Focus', 'Absolute Iris ',
@@ -418,6 +467,9 @@ def frame_rate_getter():
         else:
             blacklist = []
 
+        if ("Pupil Cam2" in self.uvc_capture.name):
+            blacklist += ['Auto Exposure Mode', 'Auto Exposure Priority', 'Absolute Exposure Time']
+
         for control in self.uvc_capture.controls:
             c = None
             ctl_name = control.display_name
@@ -448,6 +500,7 @@ def frame_rate_getter():
                     sensor_control.append(c)
 
         ui_elements.append(sensor_control)
+
         if image_processing.elements:
             ui_elements.append(image_processing)
         ui_elements.append(ui.Button("refresh",gui_update_from_device))