Merge branch 'main' into ome_tiff

.travis.yml

@@ -12,9 +12,10 @@ python:
 # contains a very old version of importlib-metadata which is incompatible for 3.8+
 # We add python path to enable testing jupyter notebooks
 install:
-  - pip install -U importlib_metadata setuptools
-  - travis_retry pip install -r requirements.txt
-  - travis_retry pip install -r requirements-test.txt
+  - pip install -U --no-cache-dir importlib-metadata setuptools pip
+  - pip install -U --no-cache-dir charset-normalizer==2.1.1
+  - travis_retry pip install --no-cache-dir -r requirements.txt
+  - travis_retry pip install --no-cache-dir -r requirements-test.txt
   - travis_retry export PYTHONPATH=$PWD
 
 env:

README.md

@@ -86,11 +86,11 @@ After the Pixie Pipeline, the user can inspect and fine tune their results with
 We recommend using the latest release of `ark`. You can find all the versions available in the [Releases Section](https://github.com/angelolab/ark-analysis/releases).
 Open terminal and navigate to where you want the code stored. 
 
-Currently, the latest release is `v0.4.4`.
+Currently, the latest release is `v0.5.0`.
 Then install the latest release with:
 
 ```
-git clone https://github.com/angelolab/ark-analysis.git --depth 1 --branch v0.4.4
+git clone https://github.com/angelolab/ark-analysis.git --depth 1 --branch v0.5.0
 ```
 
 You may also install previous releases by simply changing the version after the `--branch`.

ark/analysis/dimensionality_reduction.py

@@ -4,8 +4,7 @@
 from sklearn.decomposition import PCA
 from sklearn.manifold import TSNE
 from sklearn.preprocessing import StandardScaler
-
-from ark.utils import misc_utils
+from tmi import misc_utils
 
 
 def plot_dim_reduced_data(component_one, component_two, fig_id, hue, cell_data,

ark/analysis/spatial_analysis.py

@@ -4,10 +4,11 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
+from tmi import io_utils, load_utils, misc_utils
 from tqdm.notebook import tqdm
 
 import ark.settings as settings
-from ark.utils import io_utils, load_utils, misc_utils, spatial_analysis_utils
+from ark.utils import spatial_analysis_utils
 
 
 def generate_channel_spatial_enrichment_stats(label_dir, dist_mat_dir, marker_thresholds, all_data,

ark/analysis/spatial_analysis_test.py

@@ -4,10 +4,12 @@
 import numpy as np
 import pandas as pd
 import pytest
+from tmi import load_utils
+from tmi.test_utils import _write_labels
 
 import ark.settings as settings
 from ark.analysis import spatial_analysis
-from ark.utils import load_utils, spatial_analysis_utils, test_utils
+from ark.utils import spatial_analysis_utils, test_utils
 
 EXCLUDE_CHANNELS = [
     "Background",
@@ -38,9 +40,10 @@ def test_generate_channel_spatial_enrichment_stats():
     # only the number of elements returned and the included_fovs argument needs testing
     marker_thresholds = test_utils._make_threshold_mat(in_utils=False)
 
-    with tempfile.TemporaryDirectory() as label_dir, tempfile.TemporaryDirectory() as dist_mat_dir:
-        test_utils._write_labels(label_dir, ["fov8", "fov9"], ["segmentation_label"], (10, 10),
-                                 '', True, np.uint8, suffix='_whole_cell')
+    with tempfile.TemporaryDirectory() as label_dir, \
+         tempfile.TemporaryDirectory() as dist_mat_dir:
+        _write_labels(label_dir, ["fov8", "fov9"], ["segmentation_label"], (10, 10),
+                      '', True, np.uint8, suffix='_whole_cell')
 
         spatial_analysis_utils.calc_dist_matrix(label_dir, dist_mat_dir)
         label_maps = load_utils.load_imgs_from_dir(label_dir, trim_suffix="_whole_cell",
@@ -74,9 +77,10 @@ def test_generate_channel_spatial_enrichment_stats():
 def test_generate_cluster_spatial_enrichment_stats():
     # since the functionality if channel spatial enrichment is tested later,
     # only the number of elements returned and the included_fovs argument needs testing
-    with tempfile.TemporaryDirectory() as label_dir, tempfile.TemporaryDirectory() as dist_mat_dir:
-        test_utils._write_labels(label_dir, ["fov8", "fov9"], ["segmentation_label"], (10, 10),
-                                 '', True, np.uint8, suffix='_whole_cell')
+    with tempfile.TemporaryDirectory() as label_dir, \
+         tempfile.TemporaryDirectory() as dist_mat_dir:
+        _write_labels(label_dir, ["fov8", "fov9"], ["segmentation_label"], (10, 10),
+                      '', True, np.uint8, suffix='_whole_cell')
 
         spatial_analysis_utils.calc_dist_matrix(label_dir, dist_mat_dir)
         label_maps = load_utils.load_imgs_from_dir(label_dir, trim_suffix="_whole_cell",

ark/analysis/visualize.py

@@ -3,8 +3,8 @@
 import pandas as pd
 import seaborn as sns
 import spatial_lda.visualization as sv
+from tmi import misc_utils
 
-from ark.utils import misc_utils
 from ark.utils.spatial_lda_utils import make_plot_fn
 
 

ark/analysis/visualize_test.py

@@ -10,7 +10,6 @@
 import ark.spLDA.processing as pros
 from ark.analysis import visualize
 from ark.utils import test_utils
-from ark.utils.test_utils import make_cell_table
 
 
 def test_draw_heatmap():
@@ -165,7 +164,7 @@ def test_visualize_neighbor_cluster_metrics():
 
 def test_visualize_topic_eda():
     # Create/format/featurize testing cell table
-    cell_table = make_cell_table(num_cells=1000)
+    cell_table = test_utils.make_cell_table(num_cells=1000)
     all_clusters = list(np.unique(cell_table[settings.CELL_TYPE]))
     cell_table_format = pros.format_cell_table(cell_table, clusters=all_clusters)
     cell_table_features = pros.featurize_cell_table(cell_table_format)
@@ -201,7 +200,7 @@ def test_visualize_topic_eda():
 
 def test_visualize_fov_stats():
     # Create/format/featurize testing cell table
-    cell_table = make_cell_table(num_cells=1000)
+    cell_table = test_utils.make_cell_table(num_cells=1000)
     all_clusters = list(np.unique(cell_table[settings.CELL_TYPE]))
     cell_table_format = pros.format_cell_table(cell_table, clusters=all_clusters)
 
@@ -226,7 +225,7 @@ def test_visualize_fov_stats():
 
 
 def test_visualize_fov_graphs():
-    cell_table = make_cell_table(num_cells=1000)
+    cell_table = test_utils.make_cell_table(num_cells=1000)
     all_clusters = list(np.unique(cell_table[settings.CELL_TYPE]))
     cell_table_format = pros.format_cell_table(cell_table, clusters=all_clusters)
     cell_table_features = pros.featurize_cell_table(cell_table_format)

ark/phenotyping/cell_cluster_utils.py

@@ -1,15 +1,14 @@
 import os
-import subprocess
 
 import feather
 import matplotlib.patches as patches
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import scipy.stats as stats
+from tmi import io_utils, misc_utils
 
 from ark.analysis import visualize
-from ark.utils import io_utils, misc_utils
 from ark.phenotyping import cluster_helpers
 
 

ark/phenotyping/cell_cluster_utils_test.py

@@ -1,16 +1,16 @@
-from copy import deepcopy
 import os
 import tempfile
+from copy import deepcopy
 
 import feather
 import numpy as np
 import pandas as pd
 import pytest
 from sklearn.utils import shuffle
+from tmi import misc_utils
 
-import ark.phenotyping.cluster_helpers as cluster_helpers
 import ark.phenotyping.cell_cluster_utils as cell_cluster_utils
-import ark.utils.misc_utils as misc_utils
+import ark.phenotyping.cluster_helpers as cluster_helpers
 
 parametrize = pytest.mark.parametrize
 

ark/phenotyping/cluster_helpers.py

@@ -1,19 +1,19 @@
-from abc import ABC, abstractmethod
 import bisect
-import feather
-from itertools import combinations
-import numpy as np
 import os
 import pathlib
+import warnings
+from abc import ABC, abstractmethod
+from itertools import combinations
+from typing import List, Protocol, runtime_checkable
+
+import feather
+import numpy as np
 import pandas as pd
 from pyFlowSOM import map_data_to_nodes, som
 from scipy.stats import zscore
 from sklearn.cluster import AgglomerativeClustering
-from typing import Callable, Generic, List, Protocol, runtime_checkable
-import warnings
-
-from ark.utils.io_utils import list_files, validate_paths
-from ark.utils.misc_utils import verify_in_list
+from tmi.io_utils import list_files, validate_paths
+from tmi.misc_utils import verify_in_list
 
 
 class PixieSOMCluster(ABC):
@@ -57,7 +57,6 @@ def normalize_data(self) -> pd.DataFrame:
             pandas.DataFrame:
                 The data with `columns` normalized by the values in `norm_data`
         """
-        pass
 
     def train_som(self, data: pd.DataFrame):
         """Trains the SOM on the data provided and saves the weights generated

ark/phenotyping/cluster_helpers_test.py

@@ -1,15 +1,17 @@
+import os
+import pathlib
+import random
 from copy import deepcopy
+from typing import Iterator, Tuple
+
 import feather
 import numpy as np
-import os
 import pandas as pd
-import pathlib
 import pytest
-import random
-from typing import Iterator, List, Tuple
+from tmi.misc_utils import verify_same_elements
 
-from ark.phenotyping.cluster_helpers import PixieConsensusCluster, PixelSOMCluster, CellSOMCluster
-from ark.utils.misc_utils import verify_same_elements
+from ark.phenotyping.cluster_helpers import (CellSOMCluster, PixelSOMCluster,
+                                             PixieConsensusCluster)
 
 
 @pytest.fixture(scope="session")

ark/phenotyping/pixel_cluster_utils.py

@@ -1,20 +1,19 @@
-from functools import partial
 import multiprocessing
 import os
-from shutil import rmtree
-import subprocess
+import random
 import warnings
+from functools import partial
+from shutil import rmtree
 
 import feather
 import numpy as np
 import pandas as pd
-from pyarrow.lib import ArrowInvalid
-import random
 import scipy.ndimage as ndimage
-from skimage.io import imread, imsave
+from pyarrow.lib import ArrowInvalid
+from skimage.io import imread
+from tmi import image_utils, io_utils, load_utils, misc_utils
 
 from ark.phenotyping import cluster_helpers
-from ark.utils import io_utils, load_utils, misc_utils
 
 multiprocessing.set_start_method('spawn', force=True)
 
@@ -226,8 +225,10 @@ def smooth_channels(fovs, tiff_dir, img_sub_folder, channels, smooth_vals):
             img = load_utils.load_imgs_from_tree(data_dir=tiff_dir, img_sub_folder=img_sub_folder,
                                                  fovs=[fov], channels=[chan]).values[0, :, :, 0]
             chan_out = ndimage.gaussian_filter(img, sigma=smooth_vals[idx])
-            imsave(os.path.join(tiff_dir, fov, img_sub_folder, chan + '_smoothed.tiff'),
-                   chan_out, check_contrast=False)
+            image_utils.save_image(
+                os.path.join(tiff_dir, fov, img_sub_folder, chan + '_smoothed.tiff'),
+                chan_out
+            )
 
 
 def filter_with_nuclear_mask(fovs, tiff_dir, seg_dir, channel,
@@ -285,8 +286,7 @@ def filter_with_nuclear_mask(fovs, tiff_dir, seg_dir, channel,
         img[seg_mask] = 0
 
         # save filtered image
-        imsave(os.path.join(tiff_dir, fov, img_sub_folder, channel + suffix), img,
-               check_contrast=False)
+        image_utils.save_image(os.path.join(tiff_dir, fov, img_sub_folder, channel + suffix), img)
 
 
 def compute_pixel_cluster_channel_avg(fovs, channels, base_dir, pixel_cluster_col,

ark/phenotyping/pixel_cluster_utils_test.py

@@ -1,7 +1,7 @@
-from copy import deepcopy
 import os
 import tempfile
 import warnings
+from copy import deepcopy
 from shutil import rmtree
 
 import feather
@@ -12,13 +12,10 @@
 import skimage.io as io
 from pytest_cases import parametrize_with_cases
 from skimage.draw import disk
+from tmi import image_utils, io_utils, load_utils, misc_utils, test_utils
 
 import ark.phenotyping.cluster_helpers as cluster_helpers
 import ark.phenotyping.pixel_cluster_utils as pixel_cluster_utils
-import ark.utils.io_utils as io_utils
-import ark.utils.load_utils as load_utils
-import ark.utils.misc_utils as misc_utils
-import ark.utils.test_utils as test_utils
 
 parametrize = pytest.mark.parametrize
 
@@ -207,7 +204,7 @@ def test_calculate_pixel_intensity_percentile():
                 else:
                     divisor = 100
                 # saved modified channel
-                io.imsave(chan_path, img / divisor)
+                image_utils.save_image(chan_path, img / divisor)
 
         channel_percentiles = pd.DataFrame(np.array([[1, 1, 1]]),
                                            columns=['chan1', 'chan2', 'chan3'])
@@ -414,8 +411,7 @@ def test_filter_with_nuclear_mask(sub_dir, exclude, capsys):
 
             # save the nuclear segmetation
             file_name = fov + "_nuclear.tiff"
-            io.imsave(os.path.join(seg_dir, file_name), rand_img,
-                      check_contrast=False)
+            image_utils.save_image(os.path.join(seg_dir, file_name), rand_img)
 
         # create sample image data
         test_utils.create_paired_xarray_fovs(
@@ -711,8 +707,7 @@ def test_preprocess_fov(mocker):
         for fov in ['fov0', 'fov1']:
             rand_img = np.random.randint(0, 16, size=(10, 10))
             file_name = fov + "_whole_cell.tiff"
-            io.imsave(os.path.join(seg_dir, file_name), rand_img,
-                      check_contrast=False)
+            image_utils.save_image(os.path.join(seg_dir, file_name), rand_img)
 
         channel_norm_df = pd.DataFrame(
             np.expand_dims(np.repeat(10, repeats=len(chans)), axis=0),
@@ -831,8 +826,7 @@ def test_create_pixel_matrix_base(fovs, chans, sub_dir, seg_dir_include,
             for fov in fovs:
                 rand_img = np.random.randint(0, 16, size=(10, 10))
                 file_name = fov + "_whole_cell.tiff"
-                io.imsave(os.path.join(seg_dir, file_name), rand_img,
-                          check_contrast=False)
+                image_utils.save_image(os.path.join(seg_dir, file_name), rand_img)
         # otherwise, set seg_dir to None
         else:
             seg_dir = None
@@ -979,7 +973,7 @@ def test_create_pixel_matrix_base(fovs, chans, sub_dir, seg_dir_include,
             fov_dir = os.path.join(new_tiff_dir, fov, sub_dir)
             os.makedirs(fov_dir)
             for chan in chans:
-                io.imsave(os.path.join(fov_dir, chan + '.tiff'), img)
+                image_utils.save_image(os.path.join(fov_dir, chan + '.tiff'), img)
 
         # recreate the output directory
         rmtree(sample_pixel_output_dir)

ark/phenotyping/post_cluster_utils.py

@@ -4,9 +4,10 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+from tmi import load_utils, misc_utils
+from tmi.settings import EXTENSION_TYPES
 
-from ark.settings import EXTENSION_TYPES
-from ark.utils import data_utils, load_utils, misc_utils, plot_utils
+from ark.utils import data_utils, plot_utils
 
 
 def plot_hist_thresholds(cell_table, populations, marker, pop_col='cell_meta_cluster',

ark/phenotyping/post_cluster_utils_test.py

@@ -4,9 +4,9 @@
 import pandas as pd
 import pytest
 import skimage.io as io
+from tmi import image_utils, test_utils
 
 from ark.phenotyping import post_cluster_utils
-from ark.utils import test_utils
 
 
 def test_plot_hist_thresholds():
@@ -59,7 +59,7 @@ def test_create_mantis_project(tmp_path):
     # create random segmentation masks
     for fov in fovs:
         data = np.random.randint(0, 5, 100).reshape(10, 10)
-        io.imsave(os.path.join(seg_dir, fov + '_whole_cell_test.tiff'), data, check_contrast=False)
+        image_utils.save_image(os.path.join(seg_dir, fov + '_whole_cell_test.tiff'), data)
 
     # create cell table with two clusters
     cell_label = np.tile(np.arange(1, 5), len(fovs))