seg_suffix Requires Explicit File Extension (#848)

* seg_suffix now requires the extension in addition to the suffix name

* pycodestyle

* added ArrowInvalid to exclude_lines in .coveragerc

.coveragerc

@@ -7,6 +7,8 @@ exclude_lines =
     # Don't complain if tests don't hit defensive assertion code:
     raise AssertionError
     raise NotImplementedError
+    (ArrowInvalid, OSError, IOError)
+
 
 ignore_errors = True
 fail_under = 45

ark/phenotyping/post_cluster_utils.py

@@ -4,7 +4,8 @@
 import numpy as np
 import pandas as pd
 
-from ark.utils import data_utils, load_utils, plot_utils
+from ark.settings import EXTENSION_TYPES
+from ark.utils import data_utils, load_utils, misc_utils, plot_utils
 
 
 def plot_hist_thresholds(cell_table, populations, marker, pop_col='cell_meta_cluster',
@@ -52,7 +53,7 @@ def plot_hist_thresholds(cell_table, populations, marker, pop_col='cell_meta_clu
 
 def create_mantis_project(cell_table, fovs, seg_dir, pop_col,
                           mask_dir, image_dir, mantis_dir,
-                          seg_suffix_name: str = "_whole_cell") -> None:
+                          seg_suffix_name: str = "_whole_cell.tiff") -> None:
     """Create a complete Mantis project for viewing cell labels
 
     Args:
@@ -64,9 +65,18 @@ def create_mantis_project(cell_table, fovs, seg_dir, pop_col,
         image_dir (path): path to the directory containing the raw image data
         mantis_dir (path): path to the directory where the mantis project will be created
         seg_suffix_name (str, optional):
-            The suffix of the segmentation file. Defaults to "_whole_cell".
+            The suffix of the segmentation file and it's file extension.
+            Defaults to "_whole_cell.tiff".
     """
 
+    # Validate image extension input.
+    seg_suffix_ext: str = seg_suffix_name.split(".")[-1]
+    misc_utils.verify_in_list(seg_suffix_ext=seg_suffix_ext,
+                              supported_image_extensions=EXTENSION_TYPES["IMAGE"])
+
+    # split the file extension from the suffix name
+    seg_suffix_name_no_ext: str = seg_suffix_name.split(".")[0]
+
     if not os.path.exists(mask_dir):
         os.makedirs(mask_dir)
 
@@ -78,12 +88,12 @@ def create_mantis_project(cell_table, fovs, seg_dir, pop_col,
 
     # label and save the cell mask for each FOV
     for fov in fovs:
-        whole_cell_files = [fov + seg_suffix_name + ".tiff"]
+        whole_cell_files = [fov + seg_suffix_name]
 
         # load the segmentation labels in for the FOVs
         label_map = load_utils.load_imgs_from_dir(
             data_dir=seg_dir, files=whole_cell_files, xr_dim_name='compartments',
-            xr_channel_names=[seg_suffix_name], trim_suffix=seg_suffix_name
+            xr_channel_names=[seg_suffix_name_no_ext], trim_suffix=seg_suffix_name_no_ext
         ).loc[fov, ...]
 
         # use label_cells_by_cluster to create cell masks
@@ -92,7 +102,7 @@ def create_mantis_project(cell_table, fovs, seg_dir, pop_col,
             cell_label_column='label', cluster_column='pop_vals'
         )
 
-        # save the cell mask for each FOV
+        # save the cell mask for each FOV -- (saves with ".tiff" extension)
         data_utils.save_fov_mask(
             fov,
             mask_dir,

ark/phenotyping/post_cluster_utils_test.py

@@ -73,7 +73,7 @@ def test_create_mantis_project(tmp_path):
                                              seg_dir=seg_dir, pop_col='cell_meta_cluster',
                                              mask_dir=mask_dir, image_dir=image_dir,
                                              mantis_dir=mantis_dir,
-                                             seg_suffix_name="_whole_cell_test")
+                                             seg_suffix_name="_whole_cell_test.tiff")
 
     # make sure that the mask found in each mantis directory is correct
     for fov in fovs:

ark/settings.py

@@ -1,3 +1,5 @@
+from typing import Dict, List
+
 # hope u like capital letters
 
 # default cell table column names
@@ -56,3 +58,9 @@
 LDA_PLOT_TYPES = ["adjacency", "topic_assignment"]
 # mibitracker
 MIBITRACKER_BACKEND = 'https://backend-dot-mibitracker-angelolab.appspot.com'
+
+EXTENSION_TYPES: Dict[str, List[str]] = {
+    "IMAGE": ["tiff", "tif", "png", "jpg", "jpeg", "ome.tiff"],
+    "ARCHIVE": ["tar", "gz", "zip"],
+    "DATA": ["csv", "feather", "bin", "json"],
+}

ark/utils/io_utils.py

@@ -1,6 +1,10 @@
+import itertools
 import os
 import pathlib
 import warnings
+from typing import List
+
+from ark.settings import EXTENSION_TYPES
 
 
 def validate_paths(paths, data_prefix=False):
@@ -115,8 +119,8 @@ def remove_file_extensions(files):
     # remove the file extension
     names = [os.path.splitext(name) for name in files]
     names_corrected = []
-    extension_types = ["tiff", "tif", "png", "jpg", "jpeg", "tar", "gz", "csv", "feather",
-                       "bin", "json"]
+    extension_types: List[str] = list(itertools.chain(*EXTENSION_TYPES.values()))
+
     for name in names:
         # We want everything after the "." for the extension
         ext = name[-1][1:]

ark/utils/plot_utils.py

@@ -15,6 +15,7 @@
 from skimage.exposure import rescale_intensity
 from skimage.segmentation import find_boundaries
 
+from ark.settings import EXTENSION_TYPES
 from ark.utils import io_utils, load_utils, misc_utils
 # plotting functions
 from ark.utils.misc_utils import verify_in_list, verify_same_elements
@@ -395,7 +396,7 @@ def create_mantis_dir(fovs: List[str], mantis_project_path: Union[str, pathlib.P
                       mapping: Union[str, pathlib.Path, pd.DataFrame],
                       seg_dir: Union[str, pathlib.Path],
                       mask_suffix: str = "_mask",
-                      seg_suffix_name: str = "_whole_cell",
+                      seg_suffix_name: str = "_whole_cell.tiff",
                       img_sub_folder: str = ""):
     """Creates a mantis project directory so that it can be opened by the mantis viewer.
     Copies fovs, segmentation files, masks, and mapping csv's into a new directory structure.
@@ -438,7 +439,8 @@ def create_mantis_dir(fovs: List[str], mantis_project_path: Union[str, pathlib.P
         mask_suffix (str, optional):
             The suffix used to find the mask tiffs. Defaults to "_mask".
         seg_suffix_name (str, optional):
-            The suffix of the segmentation file. Defaults to "_whole_cell".
+            The suffix of the segmentation file and it's file extension.
+            Defaults to "_whole_cell.tiff".
         img_sub_folder (str, optional):
             The subfolder where the channels exist within the `img_data_path`.
             Defaults to "normalized".
@@ -447,6 +449,11 @@ def create_mantis_dir(fovs: List[str], mantis_project_path: Union[str, pathlib.P
     if not os.path.exists(mantis_project_path):
         os.makedirs(mantis_project_path)
 
+    # `seg_suffix` file extension validation
+    seg_suffix_ext = seg_suffix_name.split(".")[-1]
+    misc_utils.verify_in_list(seg_suffix_ext=seg_suffix_ext,
+                              supported_image_extensions=EXTENSION_TYPES["IMAGE"])
+
     # create key from cluster number to cluster name
     if type(mapping) in {pathlib.Path, str}:
         map_df = pd.read_csv(mapping)
@@ -489,17 +496,17 @@ def create_mantis_dir(fovs: List[str], mantis_project_path: Union[str, pathlib.P
             os.makedirs(output_dir)
 
             # copy all channels into new folder
-            chans = io_utils.list_files(img_source_dir, '.tiff')
+            chans = io_utils.list_files(img_source_dir, substrs=f".{seg_suffix_ext}")
             for chan in chans:
                 shutil.copy(os.path.join(img_source_dir, chan), os.path.join(output_dir, chan))
 
         # copy mask into new folder
-        mask_name: str = mn + mask_suffix + '.tiff'
+        mask_name: str = mn + mask_suffix + f".{seg_suffix_ext}"
         shutil.copy(os.path.join(mask_output_dir, mask_name),
                     os.path.join(output_dir, 'population{}.tiff'.format(mask_suffix)))
 
         # copy the segmentation files into the output directory
-        seg_name: str = fov + seg_suffix_name + '.tiff'
+        seg_name: str = fov + seg_suffix_name
         shutil.copy(os.path.join(seg_dir, seg_name),
                     os.path.join(output_dir, 'cell_segmentation.tiff'))
 

ark/utils/plot_utils_test.py

@@ -337,7 +337,7 @@ def test_create_mantis_dir():
                 mask_suffix=mask_suffix,
                 mapping=mapping,
                 seg_dir=image_segmentation_full_path,
-                seg_suffix_name="_whole_cell_test",
+                seg_suffix_name="_whole_cell_test.tiff",
                 img_sub_folder=img_sub_folder
             )
 

templates/2_Cluster_Pixels.ipynb

@@ -886,7 +886,8 @@
     "    mask_output_dir=os.path.join(base_dir, pixel_output_dir, \"pixel_masks\"),\n",
     "    mapping = os.path.join(base_dir, pixel_meta_cluster_remap_name),\n",
     "    seg_dir=os.path.join(base_dir, deepcell_output_dir),\n",
-    "    mask_suffix=\"_pixel_mask\")"
+    "    mask_suffix=\"_pixel_mask\",\n",
+    "    seg_suffix_name=seg_suffix)"
    ]
   }
  ],

templates/3_Cluster_Cells.ipynb

@@ -731,7 +731,8 @@
     "    mask_output_dir=os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n",
     "    mapping = os.path.join(base_dir, cell_meta_cluster_remap_name),\n",
     "    seg_dir=os.path.join(base_dir, \"segmentation\", \"deepcell_output\"),\n",
-    "    mask_suffix=\"_cell_mask\")"
+    "    mask_suffix=\"_cell_mask\",\n",
+    "    seg_suffix_name=seg_suffix)"
    ]
   }
  ],