Merge pull request #31 from mehta-lab/calibration_plugin

Calibration/Acquisition Plugin

recOrder/io/utils.py

@@ -2,6 +2,54 @@
 import os
 import tifffile as tiff
 import numpy as np
+from waveorder.waveorder_reconstructor import fluorescence_microscopy, waveorder_microscopy
+
+
+def extract_reconstruction_parameters(reconstructor, magnification=None):
+    """
+    Function that extracts the reconstruction parameters from a waveorder reconstructor.  Works
+    for both fluorescence_microscopy class and waveorder_microscopy class.
+
+    Parameters
+    ----------
+    reconstructor:      (waveorder reconstructor object) initalized reconstructor
+    magnification:      (float or None) magnification of the microscope setup (value not saved in reconstructor)
+
+    Returns
+    -------
+    attr_dict           (dict) dictionary of reconstruction parameters in their native units
+
+    """
+
+    ps = reconstructor.ps
+    if ps:
+        ps = ps * magnification if magnification else ps
+
+    if isinstance(reconstructor, waveorder_microscopy):
+        attr_dict = {'phase_dimension': reconstructor.phase_deconv,
+                     'wavelength (nm)': np.round(reconstructor.lambda_illu * 1000 * reconstructor.n_media,1),
+                     'pad_z': reconstructor.pad_z,
+                     'n_objective_media': reconstructor.n_media,
+                     'bg_correction_option': reconstructor.bg_option,
+                     'objective_NA': reconstructor.NA_obj * reconstructor.n_media,
+                     'condenser_NA': reconstructor.NA_illu * reconstructor.n_media,
+                     'magnification': magnification,
+                     'swing': reconstructor.chi if reconstructor.N_channel == 4 else reconstructor.chi / 2 / np.pi,
+                     'pixel_size': ps}
+
+    elif isinstance(reconstructor, fluorescence_microscopy):
+        attr_dict = {'fluor_wavelength (nm)': list(reconstructor.lambda_emiss * reconstructor.n_media * 1000),
+                     'pad_z': reconstructor.pad_z,
+                     'n_objective_media': reconstructor.n_media,
+                     'objective_NA': reconstructor.NA_obj * reconstructor.n_media,
+                     'magnification': magnification,
+                     'pixel_size': ps}
+
+    else:
+        attr_dict = dict()
+
+    return attr_dict
+
 
 def load_bg(bg_path, height, width, ROI=None):
     """
@@ -17,7 +65,7 @@ def load_bg(bg_path, height, width, ROI=None):
     bg_data   : (ndarray) Array of background data w/ dimensions (N_channel, Y, X)
     """
 
-    bg_paths = glob.glob(os.path.join(bg_path,'*.tif'))
+    bg_paths = glob.glob(os.path.join(bg_path, '*.tif'))
     bg_paths.sort()
     bg_data = np.zeros([len(bg_paths), height, width])
 
@@ -32,7 +80,22 @@ def load_bg(bg_path, height, width, ROI=None):
 
     return bg_data
 
+
 def create_grid_from_coordinates(xy_coords, rows, columns):
+    """
+    Function to create a grid from XY-position coordinates.  Useful for generating HCS Zarr metadata.
+
+    Parameters
+    ----------
+    xy_coords:          (list) XY Stage position list in the order in which it was acquired: (X, Y) tuple.
+    rows:               (int) number of rows in the grid-like acquisition
+    columns:            (int) number of columns in the grid-like acquisition
+
+    Returns
+    -------
+    pos_index_grid      (array) A grid-like array mimicking the shape of the acquisition where the value in the array
+                                corresponds to the position index at that location.
+    """
 
     coords = dict()
     coords_list = []
@@ -58,6 +121,10 @@ def create_grid_from_coordinates(xy_coords, rows, columns):
 
     return pos_index_grid
 
+class MockEmitter:
+    def emit(self, value):
+        pass
+
 def get_unimodal_threshold(input_image):
     """Determines optimal unimodal threshold
     https://users.cs.cf.ac.uk/Paul.Rosin/resources/papers/unimodal2.pdf

recOrder/io/zarr_converter.py

@@ -19,7 +19,7 @@ class ZarrConverter:
     for tiled acquisitions)
     """
 
-    def __init__(self, input, output, data_type, replace_position_names=False, format_hcs=False):
+    def __init__(self, input, output, data_type=None, replace_position_names=False, format_hcs=False):
 
         # Add Initial Checks
         if len(glob.glob(os.path.join(input, '*.tif'))) == 0:
@@ -39,7 +39,7 @@ def __init__(self, input, output, data_type, replace_position_names=False, forma
         self.meta_file = None
 
         print('Initializing Data...')
-        self.reader = WaveorderReader(self.data_directory, self.data_type, extract_data=False)
+        self.reader = WaveorderReader(self.data_directory, data_type, extract_data=False)
         print('Finished initializing data')
 
         self.summary_metadata = self.reader.mm_meta['Summary'] if self.reader.mm_meta else None
@@ -79,35 +79,12 @@ def __init__(self, input, output, data_type, replace_position_names=False, forma
         self.metadata = dict()
         self.metadata['recOrder_Converter_Version'] = self.version
         self.metadata['Summary'] = self.summary_metadata
-        self.metadata['ImagePlaneMetadata'] = dict()
 
         # initialize metadata if HCS desired, init writer
         self.hcs_meta = self._generate_hcs_metadata() if self.format_hcs else None
         self.writer = WaveorderWriter(self.save_directory, hcs=self.format_hcs, hcs_meta=self.hcs_meta, verbose=False)
         self.writer.create_zarr_root(self.save_name)
 
-    # def _gather_index_maps(self):
-    #     """
-    #     Will return a dictionary of {coord: (filepath, page)} of length(N_Images) to later query
-    #     Returns
-    #     -------
-    #     """
-    #     if self.data_type == 'ometiff':
-    #         for file in self.files:
-    #             tf = tiff.TiffFile(file)
-    #             meta = tf.micromanager_metadata['IndexMap']
-    #
-    #             for page in range(len(meta['Channel'])):
-    #                 coord = [0, 0, 0, 0]
-    #                 coord[self.p_dim] = meta['Position'][page]
-    #                 coord[self.t_dim] = meta['Frame'][page]
-    #                 coord[self.c_dim] = meta['Channel'][page]
-    #                 coord[self.z_dim] = meta['Slice'][page]
-    #
-    #                 self.coord_map[tuple(coord)] = (file, page)
-    #     else:
-    #         pass
-
     def _gen_coordset(self):
         """
         generates a coordinate set in the dimensional order to which the data was acquired.
@@ -242,7 +219,7 @@ def _perform_image_check(self, tiff_image, coord):
 
         """
 
-        zarr_array = self.writer.sub_writer.current_pos_group['array']
+        zarr_array = self.writer.sub_writer.current_pos_group['arr_0']
         zarr_img = zarr_array[coord[self.dim_order.index('time')],
                               coord[self.dim_order.index('channel')],
                               coord[self.dim_order.index('z')]]
@@ -341,7 +318,7 @@ def get_channel_clims(self, pos):
     def init_zarr_structure(self):
         """
         Initiates the zarr store.  Will create a zarr store with user-specified name or original name of data
-        if not provided.  Store will contain a group called 'array' with contains an array of original
+        if not provided.  Store will contain a group called 'arr_0' with contains an array of original
         data dtype of dimensions (T, C, Z, Y, X).  Appends OME-zarr metadata with clims,chan_names
 
         Current compressor is Blosc zstd w/ bitshuffle (~1.5x compression, faster compared to best 1.6x compressor)
@@ -418,13 +395,7 @@ def run_conversion(self):
                 plane_meta = self._generate_plane_metadata(tf, page)
                 meta[f'{coord_reorder}'] = plane_meta
 
-                # only write the plane metadata for the first time-point to avoid huge i/o overhead
-                # rest will be placed in json file with the zarr store
-                if coord[self.t_dim] == 0:
-                    self.metadata['ImagePlaneMetadata'][f'{coord_reorder}'] = plane_meta
-                    json.dump(meta, self.meta_file, indent=1)
-                else:
-                    json.dump(meta, self.meta_file, indent=1)
+                json.dump(meta, self.meta_file, indent=1)
             # get the memory mapped image
             img_raw = self.get_image_array(coord[self.p_dim], coord[self.t_dim], coord[self.c_dim], coord[self.z_dim])
 

recOrder/tests/converter_tests/test_converter.py

@@ -0,0 +1,97 @@
+import pytest
+import os
+import shutil
+import zarr
+import tifffile as tiff
+from waveorder.io import WaveorderReader, WaveorderWriter
+import glob
+import numpy as np
+from recOrder.io.zarr_converter import ZarrConverter
+
+def test_converter_initialize(setup_data_save_folder, setup_test_data):
+
+    folder, ometiff_data, zarr_data, bf_data = setup_test_data
+    save_folder = setup_data_save_folder
+
+    input = ometiff_data
+    output = os.path.join(save_folder,'2T_3P_81Z_231Y_498X_Kazansky.zarr')
+
+    if os.path.exists(output):
+        shutil.rmtree(output)
+
+    converter = ZarrConverter(input, output, 'ometiff')
+    tf = tiff.TiffFile(os.path.join(ometiff_data, '2T_3P_81Z_231Y_498X_Kazansky_2_MMStack.ome.tif'))
+
+    assert(converter.files == glob.glob(os.path.join(ometiff_data, '*.ome.tif')))
+    assert(converter.dtype == 'uint16')
+    assert(isinstance(converter.reader, WaveorderReader))
+    assert(isinstance(converter.writer, WaveorderWriter))
+    assert(converter.summary_metadata == tf.micromanager_metadata['Summary'])
+
+    converter._gen_coordset()
+    coords = []
+    for t in range(2):
+        for p in range(3):
+            for c in range(4):
+                for z in range(81):
+                    coords.append((t, p, c, z))
+
+    assert(converter.coords == coords)
+    assert(converter.p_dim == 1)
+    assert(converter.t_dim == 0)
+    assert(converter.c_dim == 2)
+    assert(converter.z_dim == 3)
+    assert(converter.p == 3)
+    assert(converter.t == 2)
+    assert(converter.c == 4)
+    assert(converter.z == 81)
+
+def test_converter_run(setup_data_save_folder, setup_test_data):
+
+    folder, ometiff_data, zarr_data, bf_data = setup_test_data
+    save_folder = setup_data_save_folder
+
+    input = ometiff_data
+    output = os.path.join(save_folder, '2T_3P_81Z_231Y_498X_Kazansky.zarr')
+
+    if os.path.exists(output):
+        shutil.rmtree(output)
+
+    converter = ZarrConverter(input, output, 'ometiff')
+    tf = tiff.TiffFile(os.path.join(ometiff_data, '2T_3P_81Z_231Y_498X_Kazansky_2_MMStack.ome.tif'))
+
+    converter.run_conversion()
+
+    zs = zarr.open(output, 'r')
+
+    assert(os.path.exists(os.path.join(save_folder, '2T_3P_81Z_231Y_498X_Kazansky_ImagePlaneMetadata.txt')))
+
+    coords = []
+    for t in range(2):
+        for p in range(3):
+            for c in range(4):
+                for z in range(81):
+                    coords.append((t, p, c, z))
+
+    assert(converter.coords == coords)
+
+    cnt = 0
+    for t in range(2):
+        for p in range(3):
+            for c in range(4):
+                for z in range(81):
+                    image = zs['Row_0'][f'Col_{p}'][f'Pos_00{p}']['arr_0'][t, c, z]
+                    tiff_image = tf.pages.get(cnt).asarray()
+                    assert(np.array_equal(image, tiff_image))
+                    cnt += 1
+
+
+# def test_converter_upti():
+#
+#     input = '/Users/cameron.foltz/Desktop/Test_Data/upti/CM_FOV1/data'
+#     output = '/Users/cameron.foltz/Desktop/Test_Data/converter_test/CM_FOV1_upti.zarr'
+#     if os.path.exists(output):
+#         shutil.rmtree(output)
+#     data_type = 'upti'
+#     converter = ZarrConverter(input, output, data_type)
+#     converter.run_conversion()