From 3a9c981f43faaf71ee963e9bacd9bdf3854781e4 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 17 Feb 2022 14:42:14 +0000 Subject: [PATCH] fixes #3708 Signed-off-by: Wenqi Li --- tests/test_download_and_extract.py | 32 +++++----- tests/test_global_mutual_information_loss.py | 11 +++- tests/test_integration_classification_2d.py | 13 +++-- tests/test_masked_inference_wsi_dataset.py | 13 +++-- tests/test_patch_wsi_dataset.py | 11 ++-- tests/test_senet.py | 14 ++--- tests/test_smartcache_patch_wsi_dataset.py | 11 ++-- tests/test_warp.py | 11 +++- tests/test_wsireader.py | 11 ++-- tests/testing_data/data_config.json | 61 ++++++++++++++++++++ tests/utils.py | 15 ++++- 11 files changed, 151 insertions(+), 52 deletions(-) create mode 100644 tests/testing_data/data_config.json diff --git a/tests/test_download_and_extract.py b/tests/test_download_and_extract.py index 435b280022..e6045cada9 100644 --- a/tests/test_download_and_extract.py +++ b/tests/test_download_and_extract.py @@ -15,21 +15,24 @@ from pathlib import Path from urllib.error import ContentTooShortError, HTTPError +from parameterized import parameterized + from monai.apps import download_and_extract, download_url, extractall -from tests.utils import skip_if_downloading_fails, skip_if_quick +from tests.utils import skip_if_downloading_fails, skip_if_quick, testing_data_config class TestDownloadAndExtract(unittest.TestCase): @skip_if_quick def test_actions(self): testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data") - url = "https://drive.google.com/uc?id=1QsnnkvZyJPcbRoV_ArW8SnE1OTuoVbKE" + config_dict = testing_data_config("images", "mednist") + url = config_dict["url"] filepath = Path(testing_dir) / "MedNIST.tar.gz" output_dir = Path(testing_dir) - md5_value = "0bc7306e7427e00ad1c5526a6677552d" + hash_val, hash_type = config_dict["hash_val"], config_dict["hash_type"] with skip_if_downloading_fails(): - download_and_extract(url, filepath, output_dir, md5_value) - download_and_extract(url, filepath, output_dir, md5_value) + download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type) + download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type) wrong_md5 = "0" with self.assertLogs(logger="monai.apps", level="ERROR"): @@ -47,22 +50,17 @@ def test_actions(self): self.assertTrue(str(e).startswith("md5 check")) @skip_if_quick - def test_default(self): + @parameterized.expand((("icon", "tar"), ("favicon", "zip"))) + def test_default(self, key, file_type): with tempfile.TemporaryDirectory() as tmp_dir: with skip_if_downloading_fails(): - # icon.tar.gz https://drive.google.com/file/d/1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn/view?usp=sharing - download_and_extract( - "https://drive.google.com/uc?id=1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn", - output_dir=tmp_dir, - hash_val="a55d11ad26ed9eb7277905d796205531", - file_type="tar", - ) - # favicon.ico.zip https://drive.google.com/file/d/1TqBTJap621NO9arzXRrYi04lr9NTVF8H/view?usp=sharing + img_spec = testing_data_config("images", key) download_and_extract( - "https://drive.google.com/uc?id=1TqBTJap621NO9arzXRrYi04lr9NTVF8H", + img_spec["url"], output_dir=tmp_dir, - hash_val="ac6e167ee40803577d98237f2b0241e5", - file_type="zip", + hash_val=img_spec["hash_val"], + hash_type=img_spec["hash_type"], + file_type=file_type, ) diff --git a/tests/test_global_mutual_information_loss.py b/tests/test_global_mutual_information_loss.py index 6aa5455352..b2a52f139b 100644 --- a/tests/test_global_mutual_information_loss.py +++ b/tests/test_global_mutual_information_loss.py @@ -16,11 +16,10 @@ from monai import transforms from monai.losses.image_dissimilarity import GlobalMutualInformationLoss -from tests.utils import SkipIfBeforePyTorchVersion, download_url_or_skip_test, skip_if_quick +from tests.utils import SkipIfBeforePyTorchVersion, download_url_or_skip_test, skip_if_quick, testing_data_config device = "cuda" if torch.cuda.is_available() else "cpu" -FILE_URL = "https://drive.google.com/uc?id=17tsDLvG_GZm7a4fCVMCv-KyDx0hqq1ji" FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + "mri.nii") EXPECTED_VALUE = { @@ -54,7 +53,13 @@ @skip_if_quick class TestGlobalMutualInformationLoss(unittest.TestCase): def setUp(self): - download_url_or_skip_test(FILE_URL, FILE_PATH) + config = testing_data_config("images", "Prostate_T2W_AX_1") + download_url_or_skip_test( + url=config["url"], + filepath=FILE_PATH, + hash_val=config.get("hash_val"), + hash_type=config.get("hash_type", "sha256"), + ) @SkipIfBeforePyTorchVersion((1, 9)) def test_bspline(self): diff --git a/tests/test_integration_classification_2d.py b/tests/test_integration_classification_2d.py index 0e79a26ea7..5a742ce4f9 100644 --- a/tests/test_integration_classification_2d.py +++ b/tests/test_integration_classification_2d.py @@ -38,10 +38,8 @@ ) from monai.utils import set_determinism from tests.testing_data.integration_answers import test_integration_value -from tests.utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick +from tests.utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick, testing_data_config -TEST_DATA_URL = "https://drive.google.com/uc?id=1QsnnkvZyJPcbRoV_ArW8SnE1OTuoVbKE" -MD5_VALUE = "0bc7306e7427e00ad1c5526a6677552d" TASK = "integration_classification_2d" @@ -186,7 +184,14 @@ def setUp(self): if not os.path.exists(data_dir): with skip_if_downloading_fails(): - download_and_extract(TEST_DATA_URL, dataset_file, self.data_dir, MD5_VALUE) + data_spec = testing_data_config("images", "mednist") + download_and_extract( + data_spec["url"], + dataset_file, + self.data_dir, + hash_val=data_spec["hash_val"], + hash_type=data_spec["hash_type"], + ) assert os.path.exists(data_dir) diff --git a/tests/test_masked_inference_wsi_dataset.py b/tests/test_masked_inference_wsi_dataset.py index 64e6d981c7..c29b95a2d8 100644 --- a/tests/test_masked_inference_wsi_dataset.py +++ b/tests/test_masked_inference_wsi_dataset.py @@ -19,14 +19,15 @@ from monai.apps.pathology.data import MaskedInferenceWSIDataset from monai.utils import optional_import -from tests.utils import download_url_or_skip_test, skip_if_quick +from tests.utils import download_url_or_skip_test, skip_if_quick, testing_data_config _, has_cim = optional_import("cucim", name="CuImage") _, has_osl = optional_import("openslide") -FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe" -base_name, extension = FILE_URL.split("id=")[1], ".tiff" -FILE_NAME = "temp_" + base_name +FILE_KEY = "wsi_img" +FILE_URL = testing_data_config("images", FILE_KEY, "url") +base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff" +FILE_NAME = f"temp_{base_name}" FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", FILE_NAME + extension) MASK1 = os.path.join(os.path.dirname(__file__), "testing_data", "temp_tissue_mask1.npy") @@ -159,7 +160,9 @@ def prepare_data(): class TestMaskedInferenceWSIDataset(unittest.TestCase): def setUp(self): prepare_data() - download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f") + hash_type = testing_data_config("images", FILE_KEY, "hash_type") + hash_val = testing_data_config("images", FILE_KEY, "hash_val") + download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val) @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4]) @skipUnless(has_cim, "Requires CuCIM") diff --git a/tests/test_patch_wsi_dataset.py b/tests/test_patch_wsi_dataset.py index fb7c3dd242..c351ce5f79 100644 --- a/tests/test_patch_wsi_dataset.py +++ b/tests/test_patch_wsi_dataset.py @@ -19,14 +19,15 @@ from monai.apps.pathology.data import PatchWSIDataset from monai.utils import optional_import -from tests.utils import download_url_or_skip_test +from tests.utils import download_url_or_skip_test, testing_data_config _cucim, has_cim = optional_import("cucim") has_cim = has_cim and hasattr(_cucim, "CuImage") _, has_osl = optional_import("openslide") -FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe" -base_name, extension = FILE_URL.split("id=")[1], ".tiff" +FILE_KEY = "wsi_img" +FILE_URL = testing_data_config("images", FILE_KEY, "url") +base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff" FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension) TEST_CASE_0 = [ @@ -108,7 +109,9 @@ class TestPatchWSIDataset(unittest.TestCase): def setUp(self): - download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f") + hash_type = testing_data_config("images", FILE_KEY, "hash_type") + hash_val = testing_data_config("images", FILE_KEY, "hash_val") + download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val) @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3]) @skipUnless(has_cim, "Requires CuCIM") diff --git a/tests/test_senet.py b/tests/test_senet.py index be7f571e0b..57ca49237d 100644 --- a/tests/test_senet.py +++ b/tests/test_senet.py @@ -21,7 +21,7 @@ from monai.networks import eval_mode from monai.networks.nets import SENet154, SEResNet50, SEResNet101, SEResNet152, SEResNext50, SEResNext101 from monai.utils import optional_import -from tests.utils import test_is_quick, test_pretrained_networks, test_script_save +from tests.utils import test_is_quick, test_pretrained_networks, test_script_save, testing_data_config if TYPE_CHECKING: import pretrainedmodels @@ -69,27 +69,27 @@ def setUp(self): testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data") testing_data_urls = { "senet154": { - "url": "https://drive.google.com/uc?id=1e10LFGVIV9L8_Q5Fhwi3X5nU6mDRrCDh", + "url": testing_data_config("models", "senet154-c7b49a05", "url"), "filename": "senet154-c7b49a05.pth", }, "se_resnet50": { - "url": "https://drive.google.com/uc?id=1WCeveS0tvjta4Wcp1wAGRi_uyXRfXAGA", + "url": testing_data_config("models", "se_resnet50-ce0d4300", "url"), "filename": "se_resnet50-ce0d4300.pth", }, "se_resnet101": { - "url": "https://drive.google.com/uc?id=1Bh0PmLISUltsY8FevtlTbt6vT35clzWg", + "url": testing_data_config("models", "se_resnet101-7e38fcc6", "url"), "filename": "se_resnet101-7e38fcc6.pth", }, "se_resnet152": { - "url": "https://drive.google.com/uc?id=1fcqpP0ITOcALy_TZAcBdkyf7HcH687J-", + "url": testing_data_config("models", "se_resnet152-d17c99b7", "url"), "filename": "se_resnet152-d17c99b7.pth", }, "se_resnext50_32x4d": { - "url": "https://drive.google.com/uc?id=1kRKW8YjGaEwYdQUyhoCIDg1H9ZAoJ-jI", + "url": testing_data_config("models", "se_resnext50_32x4d-a260b3a4", "url"), "filename": "se_resnext50_32x4d-a260b3a4.pth", }, "se_resnext101_32x4d": { - "url": "https://drive.google.com/uc?id=1Tg6Zim1lXgmYgH7FyTXAgihbkq5Jegni", + "url": testing_data_config("models", "se_resnext101_32x4d-3b2fe3d8", "url"), "filename": "se_resnext101_32x4d-3b2fe3d8.pth", }, } diff --git a/tests/test_smartcache_patch_wsi_dataset.py b/tests/test_smartcache_patch_wsi_dataset.py index b9c01c3298..e2150edce5 100644 --- a/tests/test_smartcache_patch_wsi_dataset.py +++ b/tests/test_smartcache_patch_wsi_dataset.py @@ -19,13 +19,14 @@ from monai.apps.pathology.data import SmartCachePatchWSIDataset from monai.utils import optional_import -from tests.utils import download_url_or_skip_test +from tests.utils import download_url_or_skip_test, testing_data_config _cucim, has_cim = optional_import("cucim") has_cim = has_cim and hasattr(_cucim, "CuImage") -FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe" -base_name, extension = FILE_URL.split("id=")[1], ".tiff" +FILE_KEY = "wsi_img" +FILE_URL = testing_data_config("images", FILE_KEY, "url") +base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff" FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension) TEST_CASE_0 = [ @@ -134,7 +135,9 @@ class TestSmartCachePatchWSIDataset(unittest.TestCase): def setUp(self): - download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f") + hash_type = testing_data_config("images", FILE_KEY, "hash_type") + hash_val = testing_data_config("images", FILE_KEY, "hash_val") + download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val) @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2]) @skipUnless(has_cim, "Requires CuCIM") diff --git a/tests/test_warp.py b/tests/test_warp.py index 2efd790ceb..c039b57211 100644 --- a/tests/test_warp.py +++ b/tests/test_warp.py @@ -20,7 +20,7 @@ from monai.networks.blocks.warp import Warp from monai.transforms import LoadImaged from monai.utils import GridSampleMode, GridSamplePadMode -from tests.utils import SkipIfBeforePyTorchVersion, SkipIfNoModule, download_url_or_skip_test +from tests.utils import SkipIfBeforePyTorchVersion, SkipIfNoModule, download_url_or_skip_test, testing_data_config LOW_POWER_TEST_CASES = [ # run with BUILD_MONAI=1 to test csrc/resample, BUILD_MONAI=0 to test native grid_sample [ @@ -98,7 +98,13 @@ class TestWarp(unittest.TestCase): def setUp(self): - download_url_or_skip_test(FILE_URL, FILE_PATH) + config = testing_data_config("images", "Prostate_T2W_AX_1") + download_url_or_skip_test( + url=config["url"], + filepath=FILE_PATH, + hash_val=config.get("hash_val"), + hash_type=config.get("hash_type", "sha256"), + ) @SkipIfNoModule("itk") def test_itk_benchmark(self): @@ -141,7 +147,6 @@ def test_grad(self): gradcheck(warp_layer, (input_image, ddf), atol=1e-2, eps=1e-2) -FILE_URL = "https://drive.google.com/uc?id=17tsDLvG_GZm7a4fCVMCv-KyDx0hqq1ji" FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + "mri.nii") diff --git a/tests/test_wsireader.py b/tests/test_wsireader.py index 0e93f7fc04..589bd32369 100644 --- a/tests/test_wsireader.py +++ b/tests/test_wsireader.py @@ -23,7 +23,7 @@ from monai.transforms import Compose, LoadImaged, ToTensord from monai.utils import first, optional_import from monai.utils.enums import PostFix -from tests.utils import download_url_or_skip_test +from tests.utils import download_url_or_skip_test, testing_data_config cucim, has_cucim = optional_import("cucim") has_cucim = has_cucim and hasattr(cucim, "CuImage") @@ -32,8 +32,9 @@ _, has_codec = optional_import("imagecodecs") has_tiff = has_tiff and has_codec -FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe" -base_name, extension = FILE_URL.split("id=")[1], ".tiff" +FILE_KEY = "wsi_img" +FILE_URL = testing_data_config("images", FILE_KEY, "url") +base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff" FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension) HEIGHT = 32914 @@ -104,7 +105,9 @@ def save_rgba_tiff(array: np.ndarray, filename: str, mode: str): @skipUnless(has_cucim or has_osl or has_tiff, "Requires cucim, openslide, or tifffile!") def setUpModule(): # noqa: N802 - download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f") + hash_type = testing_data_config("images", FILE_KEY, "hash_type") + hash_val = testing_data_config("images", FILE_KEY, "hash_val") + download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val) class WSIReaderTests: diff --git a/tests/testing_data/data_config.json b/tests/testing_data/data_config.json new file mode 100644 index 0000000000..bf5fffe3bf --- /dev/null +++ b/tests/testing_data/data_config.json @@ -0,0 +1,61 @@ +{ + "images": { + "wsi_img": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CMU-1.tiff", + "hash_type": "sha256", + "hash_val": "73a7e89bc15576587c3d68e55d9bf92f09690280166240b48ff4b48230b13bcd" + }, + "favicon": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/favicon.ico.zip", + "hash_type": "sha256", + "hash_val": "3a3635c8d8adb81feebc5926b4106e8eb643a24a4be2a69a9d35f9a578acadb5" + }, + "icon": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/icon.tar.gz", + "hash_type": "sha256", + "hash_val": "90f24cd8f20f3932624da95190ce384302261acf0ea15b358f7832e3b6becac0" + }, + "mednist": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MedNIST.tar.gz", + "hash_type": "sha256", + "hash_val": "f2f4881ff8799a170b10a403495f0ce0ad7486491901cde67a647e6627e7f916" + }, + "Prostate_T2W_AX_1": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/Prostate_T2W_AX_1.nii", + "hash_type": "sha256", + "hash_val": "a14231f539c0f365a5f83f2a046969a9b9870e56ffd126fd8e7242364d25938a" + } + }, + "models": { + "senet154-c7b49a05": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/senet154-c7b49a05.pth", + "hash_type": "sha256", + "hash_val": "c7b49a056b98b0bed65b0237c27acdead655e599669215573d357ad337460413" + }, + "se_resnet101-7e38fcc6": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet101-7e38fcc6.pth", + "hash_type": "sha256", + "hash_val": "7e38fcc64eff3225a3ea4e6081efeb6087e8d5a61c204d94edc2ed1aab0b9d70" + }, + "se_resnet152-d17c99b7": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet152-d17c99b7.pth", + "hash_type": "sha256", + "hash_val": "d17c99b703dcca2d2507ddfb68f72625a2f7e23ee64396eb992f1b2cf7e6bdc1" + }, + "se_resnet50-ce0d4300": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet50-ce0d4300.pth", + "hash_type": "sha256", + "hash_val": "ce0d430017d3f4aa6b5658c72209f3bfffb060207fd26a2ef0b203ce592eba01" + }, + "se_resnext101_32x4d-3b2fe3d8": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext101_32x4d-3b2fe3d8.pth", + "hash_type": "sha256", + "hash_val": "3b2fe3d8acb8de7d5976c4baf518f24a0237509272a69366e816682d3e57b989" + }, + "se_resnext50_32x4d-a260b3a4": { + "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext50_32x4d-a260b3a4.pth", + "hash_type": "sha256", + "hash_val": "a260b3a40f82dfe37c58d26a612bcf7bef0d27c6fed096226b0e4e9fb364168e" + } + } +} diff --git a/tests/utils.py b/tests/utils.py index 0af019f0b0..c186ca1675 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -13,6 +13,8 @@ import datetime import functools import importlib +import json +import operator import os import queue import sys @@ -22,7 +24,7 @@ import unittest import warnings from contextlib import contextmanager -from functools import partial +from functools import partial, reduce from subprocess import PIPE, Popen from typing import Callable, Optional, Tuple from urllib.error import ContentTooShortError, HTTPError @@ -45,6 +47,17 @@ quick_test_var = "QUICKTEST" _tf32_enabled = None +_test_data_config: dict = {} + + +def testing_data_config(*keys): + """get _test_data_config[keys0][keys1]...[keysN]""" + if not _test_data_config: + with open(os.path.join(os.path.dirname(__file__), "testing_data", "data_config.json")) as c: + _config = json.load(c) + for k, v in _config.items(): + _test_data_config[k] = v + return reduce(operator.getitem, keys, _test_data_config) def clone(data: NdarrayTensor) -> NdarrayTensor: