Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix loading of TIF 16bits grayscale files #1107

Merged
merged 6 commits into from
Mar 28, 2022
Merged

Fix loading of TIF 16bits grayscale files #1107

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f6b242d
segmentation_pair.py: Fix loading of gray TIF files
mariehbourget Mar 17, 2022
c07d481
test_loader.py: Add test_read_png_tif
mariehbourget Mar 17, 2022
baa238b
test_loader.py: Add keywords
mariehbourget Mar 17, 2022
6e1e87b
Remove unnecessary length_2D and stride_2D in test_read_png_tif
mariehbourget Mar 17, 2022
8f88551
Update data-testing link in download_data.py
mariehbourget Mar 28, 2022
56521c8
Merge branch 'master' into mhb/1105-fix-tif-16b
mariehbourget Mar 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion ivadomed/loader/segmentation_pair.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,9 @@ def convert_file_to_nifti(self, filename, extension, is_gt=False):
# For '.png', '.tif', '.tiff', '.jpg' and 'jpeg' extentions
# Read image as 8 bit grayscale in numpy array (behavior TBD in ivadomed for RGB, RBGA or higher bit depth)
if "tif" in extension:
img = np.expand_dims(imageio.imread(filename, format='tiff-pil', as_gray=True), axis=-1).astype(np.uint8)
img = np.expand_dims(imageio.imread(filename, format='tiff-pil'), axis=-1).astype(np.uint8)
if len(img.shape) > 3:
img = np.expand_dims(imageio.imread(filename, format='tiff-pil', as_gray=True), axis=-1).astype(np.uint8)
else:
img = np.expand_dims(imageio.imread(filename, as_gray=True), axis=-1).astype(np.uint8)

Expand Down
84 changes: 63 additions & 21 deletions testing/unit_tests/test_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@
from ivadomed.loader.bids_dataframe import BidsDataframe
from testing.unit_tests.t_utils import create_tmp_dir, __data_testing_dir__, __tmp_dir__, download_data_testing_test_files, path_repo_root
from testing.common_testing_util import remove_tmp_dir
from ivadomed import utils as imed_utils
from ivadomed.loader import utils as imed_loader_utils
from ivadomed.loader import loader as imed_loader
from ivadomed.keywords import MetadataKW
from ivadomed.loader import mri2d_segmentation_dataset as imed_loader_mri2dseg
from ivadomed.keywords import LoaderParamsKW, MetadataKW, ModelParamsKW, TransformationKW
from pathlib import Path
logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -37,8 +39,8 @@ def test_bids_df_microscopy_png(download_data_testing_test_files, loader_paramet
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test.csv")
mariehbourget marked this conversation as resolved.
Show resolved Hide resolved
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [], 'columns_added': [], 'columns_removed': []}
Expand All @@ -62,8 +64,8 @@ def test_bids_df_anat(download_data_testing_test_files, loader_parameters):
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [],
Expand All @@ -86,8 +88,8 @@ def test_bids_df_multi(download_data_testing_test_files, loader_parameters):
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref_multi.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test_multi.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref_multi.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test_multi.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [],
Expand All @@ -111,8 +113,8 @@ def test_bids_df_ctscan(download_data_testing_test_files, loader_parameters):
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
df_test = bids_df.df.drop(columns=['path'])
df_test = df_test.sort_values(by=['filename']).reset_index(drop=True)
csv_ref = Path(loader_parameters["path_data"][0], "df_ref.csv")
csv_test = Path(loader_parameters["path_data"][0], "df_test.csv")
csv_ref = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_ref.csv")
csv_test = Path(loader_parameters[LoaderParamsKW.PATH_DATA][0], "df_test.csv")
df_test.to_csv(csv_test, index=False)
diff = csv_diff.compare(csv_diff.load_csv(open(csv_ref)), csv_diff.load_csv(open(csv_test)))
assert diff == {'added': [], 'removed': [], 'changed': [], 'columns_added': [], 'columns_removed': []}
Expand Down Expand Up @@ -166,7 +168,7 @@ def test_load_dataset_2d_png(download_data_testing_test_files,
Test to make sure load_dataset runs with 2D PNG files, writes corresponding NIfTI files,
and binarizes ground-truth values to 0 and 1.
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
ds = imed_loader.load_dataset(bids_df,
Expand Down Expand Up @@ -215,7 +217,7 @@ def test_2d_patches_and_resampling(download_data_testing_test_files,
Test that 2d patching is done properly.
Test that microscopy pixelsize and resampling are applied on the right dimensions.
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
ds = imed_loader.load_dataset(bids_df,
Expand Down Expand Up @@ -253,7 +255,7 @@ def test_get_target_filename_list(loader_parameters, model_parameters, transform
"""
Test that all target_suffix are considered for target filename when list
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
test_ds = imed_loader.load_dataset(bids_df,
Expand All @@ -262,7 +264,7 @@ def test_get_target_filename_list(loader_parameters, model_parameters, transform
'dataset_type': 'training'}})
target_filename = test_ds.filename_pairs[0][1]

assert len(target_filename) == len(loader_parameters["target_suffix"])
assert len(target_filename) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX])


@pytest.mark.parametrize('loader_parameters', [{
Expand Down Expand Up @@ -290,7 +292,7 @@ def test_get_target_filename_list_multiple_raters(loader_parameters, model_param
"""
Test that all target_suffix are considered for target filename when list
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)
data_lst = ['sub-rat3_ses-01_sample-data9_SEM.png']
test_ds = imed_loader.load_dataset(bids_df,
Expand All @@ -299,9 +301,9 @@ def test_get_target_filename_list_multiple_raters(loader_parameters, model_param
'dataset_type': 'training'}})
target_filename = test_ds.filename_pairs[0][1]

assert len(target_filename) == len(loader_parameters["target_suffix"])
assert len(target_filename[0]) == len(loader_parameters["target_suffix"][0])
assert len(target_filename[1]) == len(loader_parameters["target_suffix"][1])
assert len(target_filename) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX])
assert len(target_filename[0]) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX][0])
assert len(target_filename[1]) == len(loader_parameters[LoaderParamsKW.TARGET_SUFFIX][1])


@pytest.mark.parametrize('loader_parameters', [{
Expand All @@ -328,12 +330,12 @@ def test_microscopy_pixelsize(download_data_testing_test_files, loader_parameter
Test that PixelSize and PixelSizeUnits microscopy metadata
are handled properly for PixelSizeUnits: "mm", "um" and "nm"
"""
loader_parameters.update({"model_params": model_parameters})
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=True)

# PixelSizeUnits: "mm"
data_lst = ['sub-rat2_sample-data5_SEM.png']
transform_parameters = {"Resample": {"wspace": 0.000093, "hspace": 0.000093}}
transform_parameters = {TransformationKW.RESAMPLE: {"wspace": 0.000093, "hspace": 0.000093}}
ds = imed_loader.load_dataset(bids_df,
**{**loader_parameters, **{'data_list': data_lst,
'transforms_params': transform_parameters,
Expand All @@ -342,7 +344,7 @@ def test_microscopy_pixelsize(download_data_testing_test_files, loader_parameter

# PixelSizeUnits: "um"
data_lst = ['sub-rat3_ses-02_sample-data11_run-1_SEM.png']
transform_parameters = {"Resample": {"wspace": 0.0001, "hspace": 0.0001}}
transform_parameters = {TransformationKW.RESAMPLE: {"wspace": 0.0001, "hspace": 0.0001}}
ds = imed_loader.load_dataset(bids_df,
**{**loader_parameters, **{'data_list': data_lst,
'transforms_params': transform_parameters,
Expand All @@ -351,13 +353,53 @@ def test_microscopy_pixelsize(download_data_testing_test_files, loader_parameter

# PixelSizeUnits: "nm"
data_lst = ['sub-rat3_ses-02_sample-data10_SEM.png']
transform_parameters = {"Resample": {"wspace": 0.0001, "hspace": 0.0001}}
transform_parameters = {TransformationKW.RESAMPLE: {"wspace": 0.0001, "hspace": 0.0001}}
ds = imed_loader.load_dataset(bids_df,
**{**loader_parameters, **{'data_list': data_lst,
'transforms_params': transform_parameters,
'dataset_type': 'training'}})
assert ds[0]['input'].shape == (1, 758, 737)


@pytest.mark.parametrize('loader_parameters', [{
"path_data": [str(Path(__data_testing_dir__, "data_test_png_tif"))],
"bids_config": f"{path_repo_root}/ivadomed/config/config_bids.json",
"target_suffix": ["_seg-myelin-manual"],
"extensions": [".png", ".tif"],
"roi_params": {"suffix": None, "slice_filter_roi": None},
"contrast_params": {"contrast_lst": [], "balance": {}},
"slice_axis": "axial",
"slice_filter_params": {"filter_empty_mask": False, "filter_empty_input": True},
"patch_filter_params": {"filter_empty_mask": False, "filter_empty_input": False},
"multichannel": False
}])
@pytest.mark.parametrize('model_parameters', [{
"name": "Unet",
"dropout_rate": 0.3,
"bn_momentum": 0.1,
"final_activation": "sigmoid",
"depth": 3
}])
def test_read_png_tif(download_data_testing_test_files, loader_parameters, model_parameters):
"""
Test to make sure all combinaitions of PNG/TIF, 8/16 bits, Grayscale/RGB/RGBA files
can be loaded without errors.
"""
metadata = {}
metadata[MetadataKW.PIXEL_SIZE] = [0.07, 0.07]
metadata[MetadataKW.PIXEL_SIZE_UNITS] = "um"
loader_parameters.update({LoaderParamsKW.MODEL_PARAMS: model_parameters})
bids_df = BidsDataframe(loader_parameters, __tmp_dir__, derivatives=False)
file_lst = bids_df.df['path'].tolist()
filename_pairs = [(file_lst, None, None, metadata if isinstance(metadata, list) else [metadata])]
slice_axis = imed_utils.AXIS_DCT[loader_parameters[LoaderParamsKW.SLICE_AXIS]]
ds = imed_loader_mri2dseg.MRI2DSegmentationDataset(filename_pairs,
slice_axis=slice_axis,
cache=True,
transform=[None, None],
slice_filter_fn=None)
ds.load_filenames()


def teardown_function():
remove_tmp_dir()