Merge c7d313b into 3084a09

README.md

@@ -63,6 +63,30 @@ You will need to authenticate. Note the last URL (the one with `127.0.0.1:8888`
 
 You can shut down the notebooks and close docker by entering control-c in the terminal window.
 
+### Mantis Viewer
+
+Mantis Project Structure:
+```sh
+mantis_project
+├── fov0
+│   ├── cell_segmentation.tiff
+│   ├── chan0.tiff
+│   ├── chan1.tiff
+│   ├── chan2.tiff
+│   ├── ...
+│   ├── population_mask.csv
+│   └── population_mask.tiff
+└── fov1
+│   ├── cell_segmentation.tiff
+│   ├── chan0.tiff
+│   ├── chan1.tiff
+│   ├── chan2.tiff
+│   ├── ...
+│   ├── population_mask.csv
+│   └── population_mask.tiff
+└── ...
+```
+
 ## External Hard Drives and Google File Stream
 
 To configure external hard drive (or google file stream) access, you will have to add this to Dockers file paths in the Preferences menu. 

ark/utils/plot_utils.py

@@ -1,17 +1,18 @@
+from typing import Union, List
 import numpy as np
 import matplotlib.cm as cm
 import matplotlib.colors as colors
 import matplotlib.pyplot as plt
 from mpl_toolkits.axes_grid1 import make_axes_locatable
 import os
+import shutil
 import pandas as pd
+import pathlib
 import xarray as xr
-
 from skimage.segmentation import find_boundaries
 from skimage.exposure import rescale_intensity
 
-from ark.utils import load_utils
-from ark.utils import misc_utils
+from ark.utils import load_utils, misc_utils, io_utils
 
 # plotting functions
 from ark.utils.misc_utils import verify_in_list, verify_same_elements
@@ -356,3 +357,112 @@ def create_overlay(fov, segmentation_dir, data_dir,
         rescaled[alternate_contour_mask > 0, 1:] = 0
 
     return rescaled
+
+
+def create_mantis_project(fovs: List[str], mantis_project_path: Union[str, pathlib.Path],
+                          img_data_path: Union[str, pathlib.Path],
+                          mask_output_dir: Union[str, pathlib.Path],
+                          mapping: Union[str, pathlib.Path, pd.DataFrame],
+                          seg_dir: Union[str, pathlib.Path],
+                          mask_suffix: str = "_mask", img_sub_folder: str = "normalized"):
+    """Creates a mantis project directory so that it can be opened by the mantis viewer.
+    Copies fovs, segmentation files, masks, and mapping csv's into a new directory structure.
+    Here is how the contents of the mantis project folder will look like.
+
+    ```{code-block} sh
+    mantis_project
+    ├── fov0
+    │   ├── cell_segmentation.tiff
+    │   ├── chan0.tiff
+    │   ├── chan1.tiff
+    │   ├── chan2.tiff
+    │   ├── ...
+    │   ├── population_mask.csv
+    │   └── population_mask.tiff
+    └── fov1
+    │   ├── cell_segmentation.tiff
+    │   ├── chan0.tiff
+    │   ├── chan1.tiff
+    │   ├── chan2.tiff
+    │   ├── ...
+    │   ├── population_mask.csv
+    │   └── population_mask.tiff
+    └── ...
+    ```
+
+    Args:
+        fovs (List[str]):
+            A list of FOVs to create a Mantis Project for.
+        mantis_project_path (Union[str, pathlib.Path]):
+            The folder where the mantis project will be created.
+        img_data_path (Union[str, pathlib.Path]):
+            The location of the all the fovs you wish to create a project from.
+        mask_output_dir (Union[str, pathlib.Path]):
+            The folder containing all the masks of the fovs.
+        mapping (Union[str, pathlib.Path, pd.DataFrame]):
+            The location of the mapping file, or the mapping Pandas DataFrame itself.
+        seg_dir (Union[str, pathlib.Path]):
+            The location of the segmentation directory for the fovs.
+        mask_suffix (str, optional):
+            The suffix used to find the mask tiffs. Defaults to "_mask".
+        img_sub_folder (str, optional):
+            The subfolder where the channels exist within the `img_data_path`.
+            Defaults to "normalized".
+    """
+
+    if not os.path.exists(mantis_project_path):
+        os.makedirs(mantis_project_path)
+
+    # create key from cluster number to cluster name
+    if type(mapping) in {pathlib.Path, str}:
+        map_df = pd.read_csv(mapping)
+    elif type(mapping) is pd.DataFrame:
+        map_df = mapping
+    else:
+        ValueError("Mapping must either be a path to an already saved mapping csv, \
+                   or a DataFrame that is already loaded in.")
+
+    map_df = map_df.loc[:, ['metacluster', 'mc_name']]
+    # remove duplicates from df
+    map_df = map_df.drop_duplicates()
+    map_df = map_df.sort_values(by=['metacluster'])
+
+    # rename for mantis names
+    map_df = map_df.rename({'metacluster': 'region_id', 'mc_name': 'region_name'}, axis=1)
+
+    # get names of fovs with masks
+    mask_names = io_utils.list_files(mask_output_dir, mask_suffix)
+    total_fov_names = io_utils.extract_delimited_names(mask_names, delimiter=mask_suffix)
+
+    # use `fovs`, a subset of the FOVs in `total_fov_names` which
+    # is a list of FOVs in `img_data_path`
+    verify_in_list(fovs=fovs, img_data_fovs=total_fov_names)
+
+    # create a folder with image data, pixel masks, and segmentation mask
+    for idx, val in enumerate(fovs):
+        # set up paths
+        img_source_dir = os.path.join(img_data_path, val, img_sub_folder)
+        output_dir = os.path.join(mantis_project_path, val)
+
+        # copy image data if not already copied in from previous round of clustering
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+            # copy all channels into new folder
+            chans = io_utils.list_files(img_source_dir, '.tiff')
+            for chan in chans:
+                shutil.copy(os.path.join(img_source_dir, chan), os.path.join(output_dir, chan))
+
+        # copy mask into new folder
+        mask_name = mask_names[idx]
+        shutil.copy(os.path.join(mask_output_dir, mask_name),
+                    os.path.join(output_dir, 'population{}.tiff'.format(mask_suffix)))
+
+        # copy the segmentation files into the output directory
+        seg_name = val + '_feature_0.tiff'
+        shutil.copy(os.path.join(seg_dir, seg_name),
+                    os.path.join(output_dir, 'cell_segmentation.tiff'))
+
+        # copy mapping into directory
+        map_df.to_csv(os.path.join(output_dir, 'population{}.csv'.format(mask_suffix)),
+                      index=False)

ark/utils/plot_utils_test.py

@@ -8,10 +8,10 @@
 import xarray as xr
 import pytest
 
-from ark.utils import plot_utils
+from ark.utils import plot_utils, test_utils
 from skimage.draw import circle
 
-from ark.utils.plot_utils import plot_clustering_result
+from pathlib import Path
 
 
 def _generate_segmentation_labels(img_dims, num_cells=20):
@@ -230,3 +230,146 @@ def test_create_overlay():
                 img_overlay_chans=['nuclear_channel', 'membrane_channel'],
                 seg_overlay_comp='whole_cell',
                 alternate_segmentation=alternate_labels[:100, :100])
+
+
+def test_create_mantis_project():
+
+    # Initial data
+    example_labels = _generate_segmentation_labels((1024, 1024))
+    example_masks = _generate_image_data((1024, 1024, 1))
+
+    # Misc paths used
+    segmentation_dir = "seg_dir"
+    mask_dir = "masks"
+    cell_output_dir = "cell_output"
+    img_data_path = "img_data"
+    img_sub_folder = "normalized"
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+
+        # create the folders
+        os.makedirs(os.path.join(temp_dir, cell_output_dir))
+        os.makedirs(os.path.join(temp_dir, segmentation_dir))
+        os.makedirs(os.path.join(temp_dir, cell_output_dir, mask_dir))
+        os.makedirs(os.path.join(temp_dir, img_data_path))
+
+        # mantis project path
+        mantis_project_path = os.path.join(temp_dir, 'mantis')
+
+        # mask output dir path
+        mask_output_dir = os.path.join(temp_dir, cell_output_dir, mask_dir)
+
+        # image data path, create 2 fovs, with 4 channels each
+        fovs, channels = test_utils.gen_fov_chan_names(num_fovs=6, num_chans=4,
+                                                       use_delimiter=False, return_imgs=False)
+
+        fov_path = os.path.join(temp_dir, img_data_path)
+        filelocs, data_xr = test_utils.create_paired_xarray_fovs(
+            fov_path, fovs, channels, img_shape=(10, 10), mode='tiff', delimiter=None,
+            sub_dir=img_sub_folder, fills=True, dtype=np.int16
+        )
+
+        # Loop over the xarray, save each fov's channels,
+        # segmentation label compartments, and sample masks
+        fovs = data_xr.fovs.values
+        fovs_subset = fovs[:3]
+
+        for fov in fovs:
+
+            # Save the segmentation label compartments for each fov
+            io.imsave(os.path.join(temp_dir, segmentation_dir, '%s_feature_0.tiff' % fov),
+                      example_labels, check_contrast=False)
+
+            # Save the sample masks
+            io.imsave(os.path.join(mask_output_dir, '%s_mask.tiff' % fov),
+                      example_masks, check_contrast=False)
+
+            # Save each channel per fov
+            for idx, chan in enumerate(channels):
+                io.imsave(filelocs[fov][idx] + ".tiff",
+                          data_xr.loc[fov, :, :, chan], check_contrast=False)
+
+        # create the mapping path, and the sample mapping file
+        mapping_path = os.path.join(temp_dir, cell_output_dir, 'sample_mapping_path.csv')
+
+        df = pd.DataFrame.from_dict({
+            'cluster': np.arange(20),
+            'metacluster': np.repeat(np.arange(5), 4),
+            'mc_name': ['meta' + str(i) for i in np.repeat(np.arange(5), 4)]
+        })
+        df.to_csv(mapping_path, index=False)
+
+        # The suffix for finding masks
+        mask_suffix = "_mask"
+
+        # Image segmentation full path
+        image_segmentation_full_path = os.path.join(temp_dir, segmentation_dir)
+
+        # Test mapping csv, and df
+        for mapping in [df, mapping_path]:
+            plot_utils.create_mantis_project(
+                fovs=fovs_subset,
+                mantis_project_path=mantis_project_path,
+                img_data_path=fov_path,
+                mask_output_dir=mask_output_dir,
+                mask_suffix=mask_suffix,
+                mapping=mapping,
+                seg_dir=image_segmentation_full_path,
+                img_sub_folder=img_sub_folder
+            )
+
+            # Testing file existence and correctness
+            for fov in fovs_subset:
+                # output path for testing
+                output_path = os.path.join(mantis_project_path, fov)
+
+                # Mask tiff tests
+                mask_path = os.path.join(output_path, "population{}.tiff".format(mask_suffix))
+                original_mask_path = os.path.join(mask_output_dir, '%s_mask.tiff' % fov)
+                assert os.path.exists(mask_path)
+
+                # Cell Segmentation tiff tests
+                cell_seg_path = os.path.join(output_path, "cell_segmentation.tiff")
+                # Assert that the segmentation label compartments exist in the new directory
+                assert os.path.exists(cell_seg_path)
+
+                # Assert that the `cell_segmentation` file is equal to `fov8_feature_0`
+                original_cell_seg_path = os.path.join(temp_dir, segmentation_dir,
+                                                      '%s_feature_0.tiff' % fov)
+                cell_seg_img = io.imread(cell_seg_path)
+                original_cell_seg_img = io.imread(original_cell_seg_path)
+                np.testing.assert_equal(cell_seg_img, original_cell_seg_img)
+
+                # Assert that the mask is the same file
+                mask_img = io.imread(mask_path)
+                original_mask_img = io.imread(original_mask_path)
+                np.testing.assert_equal(mask_img, original_mask_img)
+
+                # mapping csv tests
+                if type(mapping) is pd.DataFrame:
+                    original_mapping_df = df
+                else:
+                    original_mapping_df = pd.read_csv(mapping_path)
+                new_mapping_df = pd.read_csv(
+                    os.path.join(output_path, "population{}.csv".format(mask_suffix)))
+
+                # Assert that metacluster col equals the region_id col
+                metacluster_col = original_mapping_df[["metacluster"]]
+                region_id_col = new_mapping_df[["region_id"]]
+                metacluster_col.eq(region_id_col)
+
+                # Assert that mc_name col equals the region_name col
+                mc_name_col = original_mapping_df[["mc_name"]]
+                region_name = new_mapping_df[["region_name"]]
+                mc_name_col.eq(region_name)
+
+                mantis_fov_channels = sorted(list(Path(output_path).glob("chan*.tiff")))
+
+                # Test that all fov channels exist and are correct
+                for chan_path in mantis_fov_channels:
+                    new_chan = io.imread(chan_path)
+
+                    # get the channel name
+                    chan, _ = chan_path.name.split('.')
+                    original_chan = data_xr.loc[fov, :, :, chan].values
+                    np.testing.assert_equal(new_chan, original_chan)

ark/utils/tiff_utils_test.py

@@ -17,16 +17,16 @@
 def test_read_mibitiff():
     img_data, all_channels = tiff_utils.read_mibitiff(EXAMPLE_MIBITIFF_PATH)
 
-    assert(img_data.shape == (1024, 1024, len(all_channels)))
+    assert img_data.shape == (1024, 1024, len(all_channels))
 
     channel_names = [chan_tup[1] for chan_tup in all_channels]
 
     subset_imgdata, subset_chan = tiff_utils.read_mibitiff(EXAMPLE_MIBITIFF_PATH,
                                                            channels=channel_names[:3])
 
-    assert(subset_chan == all_channels[:3])
+    assert subset_chan == all_channels[:3]
 
-    assert(np.all(img_data[:, :, :3] == subset_imgdata))
+    assert np.all(img_data[:, :, :3] == subset_imgdata)
 
     # should throw error on standard tif load
     with pytest.raises(ValueError):
@@ -61,4 +61,4 @@ def test_write_mibitiff():
         # validate correct tiff writeout via read_mibitiff
         load_data, chan_tups = tiff_utils.read_mibitiff(filepaths[fovs[0]])
 
-        assert(np.all(true_data[0, :, :, :].values == load_data))
+        assert np.all(true_data[0, :, :, :].values == load_data)

templates_ark/example_cell_clustering.ipynb

@@ -673,6 +673,31 @@
     "    base_dir, cell_table_path, cell_data_name\n",
     ")"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3.6 Save Images a Mantis Viewer Project\n",
+    "\n",
+    "Mantis Viewer is a visualization tool for multi-dimensional pathology imaging. Learn more about Mantis Viewer in the [README](../README.md#mantis-viewer)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plot_utils.create_mantis_project(\n",
+    "    fovs=cell_fovs\n",
+    "    mantis_project_path=os.path.join(base_dir, cell_output_dir, 'mantis'),\n",
+    "    img_data_path=segmentation_dir,\n",
+    "    mask_output_dir=os.path.join(base_dir, cell_output_dir),\n",
+    "    mask_suffix='_cell_mask',\n",
+    "    mapping_path = os.path.join(base_dir, cell_output_dir, cell_cluster_prefix + '_cell_meta_cluster_mapping.csv'),\n",
+    "    seg_dir=segmentation_dir)"
+   ]
   }
  ],
  "metadata": {