Skip to content

Commit

Permalink
Fix loading of TIF 16bits grayscale files (#1107)
Browse files Browse the repository at this point in the history 
* segmentation_pair.py: Fix loading of gray TIF files

* test_loader.py: Add test_read_png_tif

* test_loader.py: Add keywords

* Remove unnecessary length_2D and stride_2D in test_read_png_tif

* Update data-testing link in download_data.py
  • Loading branch information
@mariehbourget
mariehbourget committed Mar 28, 2022
1 parent b5d6721 commit 165918d
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 23 deletions.
4 changes: 3 additions & 1 deletion ivadomed/loader/segmentation_pair.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,9 @@ def convert_file_to_nifti(self, filename, extension, is_gt=False):
# For '.png', '.tif', '.tiff', '.jpg' and 'jpeg' extentions
# Read image as 8 bit grayscale in numpy array (behavior TBD in ivadomed for RGB, RBGA or higher bit depth)
if "tif" in extension:
img = np.expand_dims(imageio.imread(filename, format='tiff-pil', as_gray=True), axis=-1).astype(np.uint8)
img = np.expand_dims(imageio.imread(filename, format='tiff-pil'), axis=-1).astype(np.uint8)
if len(img.shape) > 3:
img = np.expand_dims(imageio.imread(filename, format='tiff-pil', as_gray=True), axis=-1).astype(np.uint8)
else:
img = np.expand_dims(imageio.imread(filename, as_gray=True), axis=-1).astype(np.uint8)

Expand Down
2 changes: 1 addition & 1 deletion ivadomed/scripts/download_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
`Spine Generic <https://github.com/spine-generic/data-multi-subject>`_.
Used for Tutorial and example in Ivadomed."""},
"data_testing": {
"url": ["https://github.com/ivadomed/data-testing/archive/r20220221.zip"],
"url": ["https://github.com/ivadomed/data-testing/archive/r20220328.zip"],
"description": "Data Used for integration/unit test in Ivadomed."},
"data_multi_testing": {
"url": ["https://github.com/MotionCorrect/data_multi-sessions-contrasts/archive/refs/tags/v2022-01-06.zip"],
Expand Down
84 changes: 63 additions & 21 deletions testing/unit_tests/test_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
from ivadomed.loader.bids_dataframe import BidsDataframe
from testing.unit_tests.t_utils import create_tmp_dir, __data_testing_dir__, __tmp_dir__, download_data_testing_test_files, path_repo_root
from testing.common_testing_util import remove_tmp_dir
from ivadomed import utils as imed_utils
from ivadomed.loader import loader as imed_loader
import ivadomed.loader.utils as imed_loader_utils
from ivadomed.keywords import MetadataKW
from ivadomed.loader import mri2d_segmentation_dataset as imed_loader_mri2dseg
from ivadomed.keywords import LoaderParamsKW, MetadataKW, ModelParamsKW, TransformationKW
from pathlib import Path


Expand All @@ -35,8 +37,8 @@ def test_bids_df_microscopy_png(download_data_testing_test_files, loader_paramet
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [], 'columns_added': [], 'columns_removed': []}
Expand All @@ -60,8 +62,8 @@ def test_bids_df_anat(download_data_testing_test_files, loader_parameters):
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [],
Expand All @@ -84,8 +86,8 @@ def test_bids_df_multi(download_data_testing_test_files, loader_parameters):
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref_multi.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test_multi.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref_multi.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test_multi.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [],
Expand All @@ -109,8 +111,8 @@ def test_bids_df_ctscan(download_data_testing_test_files, loader_parameters):
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [], 'columns_added': [], 'columns_removed': []}
Expand Down Expand Up @@ -164,7 +166,7 @@ def test_load_dataset_2d_png(download_data_testing_test_files,
Test to make sure load_dataset runs with 2D PNG files, writes corresponding NIfTI files,
and binarizes ground-truth values to 0 and 1.
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
ds = imed_loader.load_dataset(bids_df,
Expand Down Expand Up @@ -213,7 +215,7 @@ def test_2d_patches_and_resampling(download_data_testing_test_files,
Test that 2d patching is done properly.
Test that microscopy pixelsize and resampling are applied on the right dimensions.
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
ds = imed_loader.load_dataset(bids_df,
Expand Down Expand Up @@ -251,7 +253,7 @@ def test_get_target_filename_list(loader_parameters, model_parameters, transform
"""
Test that all target_suffix are considered for target filename when list
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
test_ds = imed_loader.load_dataset(bids_df,
Expand All @@ -260,7 +262,7 @@ def test_get_target_filename_list(loader_parameters, model_parameters, transform
'dataset_type': 'training'}})
target_filename = test_ds.filename_pairs[0][1]

assert len(target_filename) == len(loader_parameters["target_suffix"])
assert len(target_filename) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX])


@pytest.mark.parametrize('loader_parameters', [{
Expand Down Expand Up @@ -288,7 +290,7 @@ def test_get_target_filename_list_multiple_raters(loader_parameters, model_param
"""
Test that all target_suffix are considered for target filename when list
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
test_ds = imed_loader.load_dataset(bids_df,
Expand All @@ -297,9 +299,9 @@ def test_get_target_filename_list_multiple_raters(loader_parameters, model_param
'dataset_type': 'training'}})
target_filename = test_ds.filename_pairs[0][1]

assert len(target_filename) == len(loader_parameters["target_suffix"])
assert len(target_filename[0]) == len(loader_parameters["target_suffix"][0])
assert len(target_filename[1]) == len(loader_parameters["target_suffix"][1])
assert len(target_filename) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX])
assert len(target_filename[0]) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX][0])
assert len(target_filename[1]) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX][1])


@pytest.mark.parametrize('loader_parameters', [{
Expand All @@ -326,12 +328,12 @@ def test_microscopy_pixelsize(download_data_testing_test_files, loader_parameter
Test that PixelSize and PixelSizeUnits microscopy metadata
are handled properly for PixelSizeUnits: "mm", "um" and "nm"
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)

# PixelSizeUnits: "mm"
data_lst = ['sub-rat2_sample-data5_SEM.png']
transform_parameters = {"Resample": {"wspace": 0.000093, "hspace": 0.000093}}
transform_parameters = {TransformationKW.RESAMPLE: {"wspace": 0.000093, "hspace": 0.000093}}
ds = imed_loader.load_dataset(bids_df,
**{**loader_parameters, **{'data_list': data_lst,
'transforms_params': transform_parameters,
Expand All @@ -340,7 +342,7 @@ def test_microscopy_pixelsize(download_data_testing_test_files, loader_parameter

# PixelSizeUnits: "um"
data_lst = ['sub-rat3_ses-02_sample-data11_run-1_SEM.png']
transform_parameters = {"Resample": {"wspace": 0.0001, "hspace": 0.0001}}
transform_parameters = {TransformationKW.RESAMPLE: {"wspace": 0.0001, "hspace": 0.0001}}
ds = imed_loader.load_dataset(bids_df,
**{**loader_parameters, **{'data_list': data_lst,
'transforms_params': transform_parameters,
Expand All @@ -349,13 +351,53 @@ def test_microscopy_pixelsize(download_data_testing_test_files, loader_parameter

# PixelSizeUnits: "nm"
data_lst = ['sub-rat3_ses-02_sample-data10_SEM.png']
transform_parameters = {"Resample": {"wspace": 0.0001, "hspace": 0.0001}}
transform_parameters = {TransformationKW.RESAMPLE: {"wspace": 0.0001, "hspace": 0.0001}}
ds = imed_loader.load_dataset(bids_df,
**{**loader_parameters, **{'data_list': data_lst,
'transforms_params': transform_parameters,
'dataset_type': 'training'}})
assert ds[0]['input'].shape == (1, 758, 737)


@pytest.mark.parametrize('loader_parameters', [{
"path_data": [str(Path(__data_testing_dir__, "data_test_png_tif"))],
"bids_config": f"{path_repo_root}/ivadomed/config/config_bids.json",
"target_suffix": ["_seg-myelin-manual"],
"extensions": [".png", ".tif"],
"roi_params": {"suffix": None, "slice_filter_roi": None},
"contrast_params": {"contrast_lst": [], "balance": {}},
"slice_axis": "axial",
"slice_filter_params": {"filter_empty_mask": False, "filter_empty_input": True},
"patch_filter_params": {"filter_empty_mask": False, "filter_empty_input": False},
"multichannel": False
}])
@pytest.mark.parametrize('model_parameters', [{
"name": "Unet",
"dropout_rate": 0.3,
"bn_momentum": 0.1,
"final_activation": "sigmoid",
"depth": 3
}])
def test_read_png_tif(download_data_testing_test_files, loader_parameters, model_parameters):
"""
Test to make sure all combinaitions of PNG/TIF, 8/16 bits, Grayscale/RGB/RGBA files
can be loaded without errors.
"""
metadata = {}
metadata[MetadataKW.PIXEL_SIZE] = [0.07, 0.07]
metadata[MetadataKW.PIXEL_SIZE_UNITS] = "um"
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=False)
file_lst = bids_df.df['path'].tolist()
filename_pairs = [(file_lst, None, None, metadata if isinstance(metadata, list) else [metadata])]
slice_axis = imed_utils.AXIS_DCT[loader_parameters[LoaderParamsKW.SLICE_AXIS]]
ds = imed_loader_mri2dseg.MRI2DSegmentationDataset(filename_pairs,
slice_axis=slice_axis,
cache=True,
transform=[None, None],
slice_filter_fn=None)
ds.load_filenames()


def teardown_function():
remove_tmp_dir()

0 comments on commit 165918d

Please sign in to comment.