3708 - refactor testing data fetching and migrate to release assets

tests/test_download_and_extract.py

@@ -15,21 +15,24 @@
 from pathlib import Path
 from urllib.error import ContentTooShortError, HTTPError
 
+from parameterized import parameterized
+
 from monai.apps import download_and_extract, download_url, extractall
-from tests.utils import skip_if_downloading_fails, skip_if_quick
+from tests.utils import skip_if_downloading_fails, skip_if_quick, testing_data_config
 
 
 class TestDownloadAndExtract(unittest.TestCase):
     @skip_if_quick
     def test_actions(self):
         testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
-        url = "https://drive.google.com/uc?id=1QsnnkvZyJPcbRoV_ArW8SnE1OTuoVbKE"
+        config_dict = testing_data_config("images", "mednist")
+        url = config_dict["url"]
         filepath = Path(testing_dir) / "MedNIST.tar.gz"
         output_dir = Path(testing_dir)
-        md5_value = "0bc7306e7427e00ad1c5526a6677552d"
+        hash_val, hash_type = config_dict["hash_val"], config_dict["hash_type"]
         with skip_if_downloading_fails():
-            download_and_extract(url, filepath, output_dir, md5_value)
-            download_and_extract(url, filepath, output_dir, md5_value)
+            download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type)
+            download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type)
 
         wrong_md5 = "0"
         with self.assertLogs(logger="monai.apps", level="ERROR"):
@@ -47,22 +50,17 @@ def test_actions(self):
             self.assertTrue(str(e).startswith("md5 check"))
 
     @skip_if_quick
-    def test_default(self):
+    @parameterized.expand((("icon", "tar"), ("favicon", "zip")))
+    def test_default(self, key, file_type):
         with tempfile.TemporaryDirectory() as tmp_dir:
             with skip_if_downloading_fails():
-                # icon.tar.gz https://drive.google.com/file/d/1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn/view?usp=sharing
-                download_and_extract(
-                    "https://drive.google.com/uc?id=1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn",
-                    output_dir=tmp_dir,
-                    hash_val="a55d11ad26ed9eb7277905d796205531",
-                    file_type="tar",
-                )
-                # favicon.ico.zip https://drive.google.com/file/d/1TqBTJap621NO9arzXRrYi04lr9NTVF8H/view?usp=sharing
+                img_spec = testing_data_config("images", key)
                 download_and_extract(
-                    "https://drive.google.com/uc?id=1TqBTJap621NO9arzXRrYi04lr9NTVF8H",
+                    img_spec["url"],
                     output_dir=tmp_dir,
-                    hash_val="ac6e167ee40803577d98237f2b0241e5",
-                    file_type="zip",
+                    hash_val=img_spec["hash_val"],
+                    hash_type=img_spec["hash_type"],
+                    file_type=file_type,
                 )
 
 

tests/test_global_mutual_information_loss.py

@@ -16,11 +16,10 @@
 
 from monai import transforms
 from monai.losses.image_dissimilarity import GlobalMutualInformationLoss
-from tests.utils import SkipIfBeforePyTorchVersion, download_url_or_skip_test, skip_if_quick
+from tests.utils import SkipIfBeforePyTorchVersion, download_url_or_skip_test, skip_if_quick, testing_data_config
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
-FILE_URL = "https://drive.google.com/uc?id=17tsDLvG_GZm7a4fCVMCv-KyDx0hqq1ji"
 FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + "mri.nii")
 
 EXPECTED_VALUE = {
@@ -54,7 +53,13 @@
 @skip_if_quick
 class TestGlobalMutualInformationLoss(unittest.TestCase):
     def setUp(self):
-        download_url_or_skip_test(FILE_URL, FILE_PATH)
+        config = testing_data_config("images", "Prostate_T2W_AX_1")
+        download_url_or_skip_test(
+            url=config["url"],
+            filepath=FILE_PATH,
+            hash_val=config.get("hash_val"),
+            hash_type=config.get("hash_type", "sha256"),
+        )
 
     @SkipIfBeforePyTorchVersion((1, 9))
     def test_bspline(self):

tests/test_integration_classification_2d.py

@@ -38,10 +38,8 @@
 )
 from monai.utils import set_determinism
 from tests.testing_data.integration_answers import test_integration_value
-from tests.utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick
+from tests.utils import DistTestCase, TimedCall, skip_if_downloading_fails, skip_if_quick, testing_data_config
 
-TEST_DATA_URL = "https://drive.google.com/uc?id=1QsnnkvZyJPcbRoV_ArW8SnE1OTuoVbKE"
-MD5_VALUE = "0bc7306e7427e00ad1c5526a6677552d"
 TASK = "integration_classification_2d"
 
 
@@ -186,7 +184,14 @@ def setUp(self):
 
         if not os.path.exists(data_dir):
             with skip_if_downloading_fails():
-                download_and_extract(TEST_DATA_URL, dataset_file, self.data_dir, MD5_VALUE)
+                data_spec = testing_data_config("images", "mednist")
+                download_and_extract(
+                    data_spec["url"],
+                    dataset_file,
+                    self.data_dir,
+                    hash_val=data_spec["hash_val"],
+                    hash_type=data_spec["hash_type"],
+                )
 
         assert os.path.exists(data_dir)
 

tests/test_masked_inference_wsi_dataset.py

@@ -19,14 +19,15 @@
 
 from monai.apps.pathology.data import MaskedInferenceWSIDataset
 from monai.utils import optional_import
-from tests.utils import download_url_or_skip_test, skip_if_quick
+from tests.utils import download_url_or_skip_test, skip_if_quick, testing_data_config
 
 _, has_cim = optional_import("cucim", name="CuImage")
 _, has_osl = optional_import("openslide")
 
-FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
-base_name, extension = FILE_URL.split("id=")[1], ".tiff"
-FILE_NAME = "temp_" + base_name
+FILE_KEY = "wsi_img"
+FILE_URL = testing_data_config("images", FILE_KEY, "url")
+base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
+FILE_NAME = f"temp_{base_name}"
 FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", FILE_NAME + extension)
 
 MASK1 = os.path.join(os.path.dirname(__file__), "testing_data", "temp_tissue_mask1.npy")
@@ -159,7 +160,9 @@ def prepare_data():
 class TestMaskedInferenceWSIDataset(unittest.TestCase):
     def setUp(self):
         prepare_data()
-        download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+        hash_type = testing_data_config("images", FILE_KEY, "hash_type")
+        hash_val = testing_data_config("images", FILE_KEY, "hash_val")
+        download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
 
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4])
     @skipUnless(has_cim, "Requires CuCIM")

tests/test_patch_wsi_dataset.py

@@ -19,14 +19,15 @@
 
 from monai.apps.pathology.data import PatchWSIDataset
 from monai.utils import optional_import
-from tests.utils import download_url_or_skip_test
+from tests.utils import download_url_or_skip_test, testing_data_config
 
 _cucim, has_cim = optional_import("cucim")
 has_cim = has_cim and hasattr(_cucim, "CuImage")
 _, has_osl = optional_import("openslide")
 
-FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
-base_name, extension = FILE_URL.split("id=")[1], ".tiff"
+FILE_KEY = "wsi_img"
+FILE_URL = testing_data_config("images", FILE_KEY, "url")
+base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
 FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension)
 
 TEST_CASE_0 = [
@@ -108,7 +109,9 @@
 
 class TestPatchWSIDataset(unittest.TestCase):
     def setUp(self):
-        download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+        hash_type = testing_data_config("images", FILE_KEY, "hash_type")
+        hash_val = testing_data_config("images", FILE_KEY, "hash_val")
+        download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
 
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
     @skipUnless(has_cim, "Requires CuCIM")

tests/test_senet.py

@@ -21,7 +21,7 @@
 from monai.networks import eval_mode
 from monai.networks.nets import SENet154, SEResNet50, SEResNet101, SEResNet152, SEResNext50, SEResNext101
 from monai.utils import optional_import
-from tests.utils import test_is_quick, test_pretrained_networks, test_script_save
+from tests.utils import test_is_quick, test_pretrained_networks, test_script_save, testing_data_config
 
 if TYPE_CHECKING:
     import pretrainedmodels
@@ -69,27 +69,27 @@ def setUp(self):
             testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
             testing_data_urls = {
                 "senet154": {
-                    "url": "https://drive.google.com/uc?id=1e10LFGVIV9L8_Q5Fhwi3X5nU6mDRrCDh",
+                    "url": testing_data_config("models", "senet154-c7b49a05", "url"),
                     "filename": "senet154-c7b49a05.pth",
                 },
                 "se_resnet50": {
-                    "url": "https://drive.google.com/uc?id=1WCeveS0tvjta4Wcp1wAGRi_uyXRfXAGA",
+                    "url": testing_data_config("models", "se_resnet50-ce0d4300", "url"),
                     "filename": "se_resnet50-ce0d4300.pth",
                 },
                 "se_resnet101": {
-                    "url": "https://drive.google.com/uc?id=1Bh0PmLISUltsY8FevtlTbt6vT35clzWg",
+                    "url": testing_data_config("models", "se_resnet101-7e38fcc6", "url"),
                     "filename": "se_resnet101-7e38fcc6.pth",
                 },
                 "se_resnet152": {
-                    "url": "https://drive.google.com/uc?id=1fcqpP0ITOcALy_TZAcBdkyf7HcH687J-",
+                    "url": testing_data_config("models", "se_resnet152-d17c99b7", "url"),
                     "filename": "se_resnet152-d17c99b7.pth",
                 },
                 "se_resnext50_32x4d": {
-                    "url": "https://drive.google.com/uc?id=1kRKW8YjGaEwYdQUyhoCIDg1H9ZAoJ-jI",
+                    "url": testing_data_config("models", "se_resnext50_32x4d-a260b3a4", "url"),
                     "filename": "se_resnext50_32x4d-a260b3a4.pth",
                 },
                 "se_resnext101_32x4d": {
-                    "url": "https://drive.google.com/uc?id=1Tg6Zim1lXgmYgH7FyTXAgihbkq5Jegni",
+                    "url": testing_data_config("models", "se_resnext101_32x4d-3b2fe3d8", "url"),
                     "filename": "se_resnext101_32x4d-3b2fe3d8.pth",
                 },
             }

tests/test_smartcache_patch_wsi_dataset.py

@@ -19,13 +19,14 @@
 
 from monai.apps.pathology.data import SmartCachePatchWSIDataset
 from monai.utils import optional_import
-from tests.utils import download_url_or_skip_test
+from tests.utils import download_url_or_skip_test, testing_data_config
 
 _cucim, has_cim = optional_import("cucim")
 has_cim = has_cim and hasattr(_cucim, "CuImage")
 
-FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
-base_name, extension = FILE_URL.split("id=")[1], ".tiff"
+FILE_KEY = "wsi_img"
+FILE_URL = testing_data_config("images", FILE_KEY, "url")
+base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
 FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension)
 
 TEST_CASE_0 = [
@@ -134,7 +135,9 @@
 
 class TestSmartCachePatchWSIDataset(unittest.TestCase):
     def setUp(self):
-        download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+        hash_type = testing_data_config("images", FILE_KEY, "hash_type")
+        hash_val = testing_data_config("images", FILE_KEY, "hash_val")
+        download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
 
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2])
     @skipUnless(has_cim, "Requires CuCIM")

tests/test_warp.py

@@ -20,7 +20,7 @@
 from monai.networks.blocks.warp import Warp
 from monai.transforms import LoadImaged
 from monai.utils import GridSampleMode, GridSamplePadMode
-from tests.utils import SkipIfBeforePyTorchVersion, SkipIfNoModule, download_url_or_skip_test
+from tests.utils import SkipIfBeforePyTorchVersion, SkipIfNoModule, download_url_or_skip_test, testing_data_config
 
 LOW_POWER_TEST_CASES = [  # run with BUILD_MONAI=1 to test csrc/resample, BUILD_MONAI=0 to test native grid_sample
     [
@@ -98,7 +98,13 @@
 
 class TestWarp(unittest.TestCase):
     def setUp(self):
-        download_url_or_skip_test(FILE_URL, FILE_PATH)
+        config = testing_data_config("images", "Prostate_T2W_AX_1")
+        download_url_or_skip_test(
+            url=config["url"],
+            filepath=FILE_PATH,
+            hash_val=config.get("hash_val"),
+            hash_type=config.get("hash_type", "sha256"),
+        )
 
     @SkipIfNoModule("itk")
     def test_itk_benchmark(self):
@@ -141,7 +147,6 @@ def test_grad(self):
                 gradcheck(warp_layer, (input_image, ddf), atol=1e-2, eps=1e-2)
 
 
-FILE_URL = "https://drive.google.com/uc?id=17tsDLvG_GZm7a4fCVMCv-KyDx0hqq1ji"
 FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + "mri.nii")
 
 

tests/test_wsireader.py

@@ -23,7 +23,7 @@
 from monai.transforms import Compose, LoadImaged, ToTensord
 from monai.utils import first, optional_import
 from monai.utils.enums import PostFix
-from tests.utils import download_url_or_skip_test
+from tests.utils import download_url_or_skip_test, testing_data_config
 
 cucim, has_cucim = optional_import("cucim")
 has_cucim = has_cucim and hasattr(cucim, "CuImage")
@@ -32,8 +32,9 @@
 _, has_codec = optional_import("imagecodecs")
 has_tiff = has_tiff and has_codec
 
-FILE_URL = "https://drive.google.com/uc?id=1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe"
-base_name, extension = FILE_URL.split("id=")[1], ".tiff"
+FILE_KEY = "wsi_img"
+FILE_URL = testing_data_config("images", FILE_KEY, "url")
+base_name, extension = os.path.basename(f"{FILE_URL}"), ".tiff"
 FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + base_name + extension)
 
 HEIGHT = 32914
@@ -104,7 +105,9 @@ def save_rgba_tiff(array: np.ndarray, filename: str, mode: str):
 
 @skipUnless(has_cucim or has_osl or has_tiff, "Requires cucim, openslide, or tifffile!")
 def setUpModule():  # noqa: N802
-    download_url_or_skip_test(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+    hash_type = testing_data_config("images", FILE_KEY, "hash_type")
+    hash_val = testing_data_config("images", FILE_KEY, "hash_val")
+    download_url_or_skip_test(FILE_URL, FILE_PATH, hash_type=hash_type, hash_val=hash_val)
 
 
 class WSIReaderTests:

tests/testing_data/data_config.json

@@ -0,0 +1,61 @@
+{
+    "images": {
+        "wsi_img": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/CMU-1.tiff",
+            "hash_type": "sha256",
+            "hash_val": "73a7e89bc15576587c3d68e55d9bf92f09690280166240b48ff4b48230b13bcd"
+        },
+        "favicon": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/favicon.ico.zip",
+            "hash_type": "sha256",
+            "hash_val": "3a3635c8d8adb81feebc5926b4106e8eb643a24a4be2a69a9d35f9a578acadb5"
+        },
+        "icon": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/icon.tar.gz",
+            "hash_type": "sha256",
+            "hash_val": "90f24cd8f20f3932624da95190ce384302261acf0ea15b358f7832e3b6becac0"
+        },
+        "mednist": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MedNIST.tar.gz",
+            "hash_type": "sha256",
+            "hash_val": "f2f4881ff8799a170b10a403495f0ce0ad7486491901cde67a647e6627e7f916"
+        },
+        "Prostate_T2W_AX_1": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/Prostate_T2W_AX_1.nii",
+            "hash_type": "sha256",
+            "hash_val": "a14231f539c0f365a5f83f2a046969a9b9870e56ffd126fd8e7242364d25938a"
+        }
+    },
+    "models": {
+        "senet154-c7b49a05": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/senet154-c7b49a05.pth",
+            "hash_type": "sha256",
+            "hash_val": "c7b49a056b98b0bed65b0237c27acdead655e599669215573d357ad337460413"
+        },
+        "se_resnet101-7e38fcc6": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet101-7e38fcc6.pth",
+            "hash_type": "sha256",
+            "hash_val": "7e38fcc64eff3225a3ea4e6081efeb6087e8d5a61c204d94edc2ed1aab0b9d70"
+        },
+        "se_resnet152-d17c99b7": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet152-d17c99b7.pth",
+            "hash_type": "sha256",
+            "hash_val": "d17c99b703dcca2d2507ddfb68f72625a2f7e23ee64396eb992f1b2cf7e6bdc1"
+        },
+        "se_resnet50-ce0d4300": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnet50-ce0d4300.pth",
+            "hash_type": "sha256",
+            "hash_val": "ce0d430017d3f4aa6b5658c72209f3bfffb060207fd26a2ef0b203ce592eba01"
+        },
+        "se_resnext101_32x4d-3b2fe3d8": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext101_32x4d-3b2fe3d8.pth",
+            "hash_type": "sha256",
+            "hash_val": "3b2fe3d8acb8de7d5976c4baf518f24a0237509272a69366e816682d3e57b989"
+        },
+        "se_resnext50_32x4d-a260b3a4": {
+            "url": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/se_resnext50_32x4d-a260b3a4.pth",
+            "hash_type": "sha256",
+            "hash_val": "a260b3a40f82dfe37c58d26a612bcf7bef0d27c6fed096226b0e4e9fb364168e"
+        }
+    }
+}