Improvements in frame selection modules

PynPoint · Dec 8, 2019 · 1b936c8 · 1b936c8
1 parent 826a2a9
commit 1b936c8
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 87 deletions.
diff --git a/pynpoint/processing/extract.py b/pynpoint/processing/extract.py
@@ -161,7 +161,7 @@ def _crop_around_star(image, position, im_size, fwhm):
         history = f'fwhm_star [pix] = {self.m_fwhm_star}'
 
         if self.m_index_out_port is not None:
-            self.m_index_out_port.set_all(np.transpose(np.asarray(index)))
+            self.m_index_out_port.set_all(index, data_dim=1)
             self.m_index_out_port.copy_attributes(self.m_image_in_port)
             self.m_index_out_port.add_history('StarExtractionModule', history)
 

diff --git a/pynpoint/processing/frameselection.py b/pynpoint/processing/frameselection.py
@@ -101,30 +101,20 @@ def run(self) -> None:
 
         if np.size(np.where(self.m_frames >= self.m_image_in_port.get_shape()[0])) > 0:
             raise ValueError(f'Some values in \'frames\' are larger than the total number of '
-                             f'available frames, {self.m_image_in_port.get_shape()[0]}')
+                             f'available frames, {self.m_image_in_port.get_shape()[0]}.')
 
         write_selected_data(memory=self._m_config_port.get_attribute('MEMORY'),
                             indices=self.m_frames,
                             image_in_port=self.m_image_in_port,
                             selected_out_port=self.m_selected_out_port,
                             removed_out_port=self.m_removed_out_port)
 
-        history = f'frames removed = {np.size(self.m_frames)}'
-
-        if self.m_selected_out_port is not None:
-            # Copy attributes before write_selected_attributes is used
-            self.m_selected_out_port.copy_attributes(self.m_image_in_port)
-            self.m_selected_out_port.add_history('RemoveFramesModule', history)
-
-        if self.m_removed_out_port is not None:
-            # Copy attributes before write_selected_attributes is used
-            self.m_removed_out_port.copy_attributes(self.m_image_in_port)
-            self.m_removed_out_port.add_history('RemoveFramesModule', history)
-
         write_selected_attributes(indices=self.m_frames,
                                   image_in_port=self.m_image_in_port,
                                   selected_out_port=self.m_selected_out_port,
-                                  removed_out_port=self.m_removed_out_port)
+                                  removed_out_port=self.m_removed_out_port,
+                                  module_type='RemoveFramesModule',
+                                  history=f'frames removed = {np.size(self.m_frames)}')
 
         self.m_image_in_port.close_port()
 
@@ -330,50 +320,40 @@ def run(self) -> None:
         phot_std = np.nanstd(phot)
         print(f'Standard deviation = {phot_std:.2f}')
 
-        indices = np.logical_or((phot > phot_ref + self.m_threshold*phot_std),
-                                (phot < phot_ref - self.m_threshold*phot_std))
+        index_del = np.logical_or((phot > phot_ref + self.m_threshold*phot_std),
+                                  (phot < phot_ref - self.m_threshold*phot_std))
+
+        index_del[np.isnan(phot)] = True
+        index_del = np.where(index_del)[0]
 
-        indices[np.isnan(phot)] = True
-        indices_del = np.asarray(np.where(indices)[0], dtype=np.int)
+        index_sel = np.ones(nimages, dtype=bool)
+        index_sel[index_del] = False
 
         write_selected_data(memory=self._m_config_port.get_attribute('MEMORY'),
-                            indices=indices_del,
+                            indices=index_del,
                             image_in_port=self.m_image_in_port,
                             selected_out_port=self.m_selected_out_port,
                             removed_out_port=self.m_removed_out_port)
 
-        history = f'frames removed = {np.size(indices_del)}'
+        history = f'frames removed = {np.size(index_del)}'
 
         if self.m_index_out_port is not None:
-            self.m_index_out_port.set_all(np.transpose(indices_del))
+            self.m_index_out_port.set_all(index_del, data_dim=1)
             self.m_index_out_port.copy_attributes(self.m_image_in_port)
             self.m_index_out_port.add_attribute('STAR_POSITION', starpos, static=False)
             self.m_index_out_port.add_history('FrameSelectionModule', history)
 
-        # Copy attributes before write_selected_attributes is used
-        self.m_selected_out_port.copy_attributes(self.m_image_in_port)
-
-        # Copy attributes before write_selected_attributes is used
-        self.m_removed_out_port.copy_attributes(self.m_image_in_port)
-
-        write_selected_attributes(indices=indices_del,
+        write_selected_attributes(indices=index_del,
                                   image_in_port=self.m_image_in_port,
                                   selected_out_port=self.m_selected_out_port,
-                                  removed_out_port=self.m_removed_out_port)
-
-        indices_sel = np.ones(nimages, dtype=bool)
-        indices_sel[indices] = False
-
-        self.m_selected_out_port.add_attribute('STAR_POSITION',
-                                               starpos[indices_sel],
-                                               static=False)
+                                  removed_out_port=self.m_removed_out_port,
+                                  module_type='FrameSelectionModule',
+                                  history=history)
 
+        self.m_selected_out_port.add_attribute('STAR_POSITION', starpos[index_sel], static=False)
         self.m_selected_out_port.add_history('FrameSelectionModule', history)
 
-        self.m_removed_out_port.add_attribute('STAR_POSITION',
-                                              starpos[indices],
-                                              static=False)
-
+        self.m_removed_out_port.add_attribute('STAR_POSITION', starpos[index_del], static=False)
         self.m_removed_out_port.add_history('FrameSelectionModule', history)
 
         self.m_image_in_port.close_port()
@@ -1006,26 +986,19 @@ def run(self) -> None:
                             selected_out_port=self.m_selected_out_port,
                             removed_out_port=self.m_removed_out_port)
 
-        if self.m_selected_out_port is not None:
-            # Copy attributes before write_selected_attributes is used
-            self.m_selected_out_port.copy_attributes(self.m_image_in_port)
-
-        if self.m_removed_out_port is not None:
-            # Copy attributes before write_selected_attributes is used
-            self.m_removed_out_port.copy_attributes(self.m_image_in_port)
-
-        # write the selected and removed data to the respective output ports
         write_selected_attributes(indices=index_del,
                                   image_in_port=self.m_image_in_port,
                                   selected_out_port=self.m_selected_out_port,
-                                  removed_out_port=self.m_removed_out_port)
+                                  removed_out_port=self.m_removed_out_port,
+                                  module_type='SelectByAttributeModule',
+                                  history=f'selected tag = {self.m_attribute_tag}')
 
         self.m_image_in_port.close_port()
 
 
 class ResidualSelectionModule(ProcessingModule):
     """
-    Pipeline module for applying a frame selection after the PSF subtraction.
+    Pipeline module for applying a frame selection on the residuals of the PSF subtraction.
     """
 
     __author__ = 'Tomas Stolker'
@@ -1052,7 +1025,7 @@ def __init__(self,
         percentage : float
             The percentage of best frames that is selected.
         annulus_radii : tuple(float, float)
-            Inner and outer radius (arcsec) of the annulus.
+            Inner and outer radius of the annulus (arcsec).
 
         Returns
         -------
@@ -1107,8 +1080,11 @@ def run(self) -> None:
 
             phot_annulus[i] = np.sum(np.abs(self.m_image_in_port[i][pixel_select]))
 
-        n_select = int(nimages*self.m_percentage/100.)
+        print(f'Minimum, maximum = {np.amin(phot_annulus):.2f}, {np.amax(phot_annulus):.2f}')
+        print(f'Mean, median = {np.nanmean(phot_annulus):.2f}, {np.nanmedian(phot_annulus):.2f}')
+        print(f'Standard deviation = {np.nanstd(phot_annulus):.2f}')
 
+        n_select = int(nimages*self.m_percentage/100.)
         index_del = np.argsort(phot_annulus)[n_select:]
 
         write_selected_data(memory=self._m_config_port.get_attribute('MEMORY'),
@@ -1117,21 +1093,11 @@ def run(self) -> None:
                             selected_out_port=self.m_selected_out_port,
                             removed_out_port=self.m_removed_out_port)
 
-        history = f'frames removed = {index_del.size}'
-
-        # Copy attributes before write_selected_attributes is used
-        self.m_selected_out_port.copy_attributes(self.m_image_in_port)
-
-        # Copy attributes before write_selected_attributes is used
-        self.m_removed_out_port.copy_attributes(self.m_image_in_port)
-
-        self.m_selected_out_port.add_history('ResidualsSelectionModule', history)
-        self.m_removed_out_port.add_history('ResidualsSelectionModule', history)
-
-        # write the selected and removed data to the respective output ports
         write_selected_attributes(indices=index_del,
                                   image_in_port=self.m_image_in_port,
                                   selected_out_port=self.m_selected_out_port,
-                                  removed_out_port=self.m_removed_out_port)
+                                  removed_out_port=self.m_removed_out_port,
+                                  module_type='ResidualSelectionModule',
+                                  history=f'frames removed = {index_del.size}')
 
         self.m_image_in_port.close_port()
diff --git a/pynpoint/util/remove.py b/pynpoint/util/remove.py
@@ -1,5 +1,5 @@
 """
-Functions to write selected data and attributes to the central database.
+Functions to write selected data and attributes to the database.
 """
 
 import time
@@ -18,23 +18,23 @@
 def write_selected_data(memory: Union[int, np.int64],
                         indices: np.ndarray,
                         image_in_port: InputPort,
-                        selected_out_port: OutputPort = None,
-                        removed_out_port: OutputPort = None) -> None:
+                        selected_out_port: Union[OutputPort, None],
+                        removed_out_port: Union[OutputPort, None]) -> None:
     """
-    Function to write a selected number of images from a data set.
+    Function to write the selected and removed data.
 
     Parameters
     ----------
     memory : int
-        Number of images that is simultaneously loaded into the memory
+        Number of images that is simultaneously loaded into the memory.
     indices : numpy.ndarray
         Image indices that will be removed.
     image_in_port : pynpoint.core.dataio.InputPort
         Port to the input images.
     selected_out_port : pynpoint.core.dataio.OutputPort, None
-        Port to store the selected images.
+        Port to store the selected images. No data is written if set to None.
     removed_out_port : pynpoint.core.dataio.OutputPort, None
-        Port to store the removed images.
+        Port to store the removed images. No data is written if set to None.
 
     Returns
     -------
@@ -55,28 +55,28 @@ def write_selected_data(memory: Union[int, np.int64],
 
         images = image_in_port[frames[i]:frames[i+1], ]
 
-        subset_del = np.where(np.logical_and(indices >= frames[i],
-                                             indices < frames[i+1]))[0]
-
+        subset_del = np.where(np.logical_and(indices >= frames[i], indices < frames[i+1]))[0]
         index_del = indices[subset_del] % memory
 
         index_sel = np.ones(images.shape[0], np.bool)
-        index_sel[index_del] = 0
+        index_sel[index_del] = False
 
         if selected_out_port is not None and index_sel.size > 0:
             selected_out_port.append(images[index_sel])
 
-        if removed_out_port is not None and indices.size != images.shape[0]:
+        if removed_out_port is not None and index_del.size > 0:
             removed_out_port.append(images[index_del])
 
 
 @typechecked
 def write_selected_attributes(indices: np.ndarray,
                               image_in_port: InputPort,
-                              selected_out_port: OutputPort = None,
-                              removed_out_port: OutputPort = None) -> None:
+                              selected_out_port: Union[OutputPort, None],
+                              removed_out_port: Union[OutputPort, None],
+                              module_type: str,
+                              history: str) -> None:
     """
-    Function to write the attributes of a selected number of images.
+    Function to write the attributes of the selected and removed data.
 
     Parameters
     ----------
@@ -88,17 +88,29 @@ def write_selected_attributes(indices: np.ndarray,
         Port to store the attributes of the selected images. Not written if set to None.
     removed_out_port : pynpoint.core.dataio.OutputPort, None
         Port to store the attributes of the removed images. Not written if set to None.
+    module_type : str
+    history : str
 
     Returns
     -------
     NoneType
         None
     """
 
+    if selected_out_port is not None:
+        # First copy the existing attributes to the selected_out_port
+        selected_out_port.copy_attributes(image_in_port)
+        selected_out_port.add_history(module_type, history)
+
+    if removed_out_port is not None:
+        # First copy the existing attributes to the removed_out_port
+        removed_out_port.copy_attributes(image_in_port)
+        removed_out_port.add_history(module_type, history)
+
     non_static = image_in_port.get_all_non_static_attributes()
 
     index_sel = np.ones(image_in_port.get_shape()[0], np.bool)
-    index_sel[indices] = 0
+    index_sel[indices] = False
 
     for i, attr_item in enumerate(non_static):
         values = image_in_port.get_attribute(attr_item)
@@ -118,15 +130,13 @@ def write_selected_attributes(indices: np.ndarray,
         nframes_del = np.zeros(nframes.shape, dtype=np.int)
 
         for i, frames in enumerate(nframes):
-            total = np.sum(nframes[:i])
-
             if indices.size == 0:
                 nframes_sel[i] = frames
                 nframes_del[i] = 0
 
             else:
-                index_del = np.where(np.logical_and(indices >= total,
-                                                    indices < total+frames))[0]
+                sum_n = np.sum(nframes[:i])
+                index_del = np.where(np.logical_and(indices >= sum_n, indices < sum_n+frames))[0]
 
                 nframes_sel[i] = frames - index_del.size
                 nframes_del[i] = index_del.size