From 22fe733d1b501dba02f5a11dc95d9c9f1d194948 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 15:16:07 +0100 Subject: [PATCH 1/8] Skip big model in test --- test/test_models.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index adbf7e2819e..ef65bdfa126 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -20,12 +20,12 @@ ACCEPT = os.getenv("EXPECTTEST_ACCEPT", "0") == "1" -def get_models_from_module(module): +def get_models_from_module(module, skipped_models): # TODO add a registration mechanism to torchvision.models return [ v for k, v in module.__dict__.items() - if callable(v) and k[0].lower() == k[0] and k[0] != "_" and k != "get_weight" + if callable(v) and k[0].lower() == k[0] and k[0] != "_" and k != "get_weight" and k not in skipped_models ] @@ -327,6 +327,11 @@ def _check_input_backprop(model, inputs): for m in slow_models: _model_params[m] = {"input_shape": (1, 3, 64, 64)} +# skip big models to reduce memory usage on CI test +skipped_models = { + "vit_h_14", +} + # The following contains configuration and expected values to be used tests that are model specific _model_tests_values = { @@ -582,7 +587,7 @@ def test_vitc_models(model_fn, dev): test_classification_model(model_fn, dev) -@pytest.mark.parametrize("model_fn", get_models_from_module(models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models, skipped_models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_classification_model(model_fn, dev): set_rng_seed(0) @@ -616,7 +621,7 @@ def test_classification_model(model_fn, dev): _check_input_backprop(model, x) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.segmentation)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.segmentation, skipped_models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_segmentation_model(model_fn, dev): set_rng_seed(0) @@ -678,7 +683,7 @@ def check_out(out): _check_input_backprop(model, x) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection, skipped_models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_detection_model(model_fn, dev): set_rng_seed(0) @@ -776,7 +781,7 @@ def compute_mean_std(tensor): _check_input_backprop(model, model_input) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection, skipped_models)) def test_detection_model_validation(model_fn): set_rng_seed(0) model = model_fn(num_classes=50, weights=None, weights_backbone=None) @@ -805,7 +810,7 @@ def test_detection_model_validation(model_fn): model(x, targets=targets) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.video)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.video, skipped_models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_video_model(model_fn, dev): # the default input shape is @@ -837,7 +842,7 @@ def test_video_model(model_fn, dev): ), reason="This Pytorch Build has not been built with fbgemm and qnnpack", ) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.quantization)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.quantization, skipped_models)) def test_quantized_classification_model(model_fn): set_rng_seed(0) defaults = { @@ -886,7 +891,7 @@ def test_quantized_classification_model(model_fn): torch.ao.quantization.convert(model, inplace=True) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection, skipped_models)) def test_detection_model_trainable_backbone_layers(model_fn, disable_weight_loading): model_name = model_fn.__name__ max_trainable = _model_tests_values[model_name]["max_trainable"] From 132061615fc3e13325e26085a577cb9d5ba8cc16 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 15:26:35 +0100 Subject: [PATCH 2/8] Let get_models_from_module func to read skipped_models from global directly instead of function param to reduce changes --- test/test_models.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index ef65bdfa126..cc8a45286a5 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -19,8 +19,13 @@ ACCEPT = os.getenv("EXPECTTEST_ACCEPT", "0") == "1" +# skip big models to reduce memory usage on CI test +skipped_models = { + "vit_h_14", +} + -def get_models_from_module(module, skipped_models): +def get_models_from_module(module): # TODO add a registration mechanism to torchvision.models return [ v @@ -327,11 +332,6 @@ def _check_input_backprop(model, inputs): for m in slow_models: _model_params[m] = {"input_shape": (1, 3, 64, 64)} -# skip big models to reduce memory usage on CI test -skipped_models = { - "vit_h_14", -} - # The following contains configuration and expected values to be used tests that are model specific _model_tests_values = { @@ -587,7 +587,7 @@ def test_vitc_models(model_fn, dev): test_classification_model(model_fn, dev) -@pytest.mark.parametrize("model_fn", get_models_from_module(models, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_classification_model(model_fn, dev): set_rng_seed(0) @@ -621,7 +621,7 @@ def test_classification_model(model_fn, dev): _check_input_backprop(model, x) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.segmentation, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.segmentation)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_segmentation_model(model_fn, dev): set_rng_seed(0) @@ -683,7 +683,7 @@ def check_out(out): _check_input_backprop(model, x) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_detection_model(model_fn, dev): set_rng_seed(0) @@ -781,7 +781,7 @@ def compute_mean_std(tensor): _check_input_backprop(model, model_input) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection)) def test_detection_model_validation(model_fn): set_rng_seed(0) model = model_fn(num_classes=50, weights=None, weights_backbone=None) @@ -810,7 +810,7 @@ def test_detection_model_validation(model_fn): model(x, targets=targets) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.video, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.video)) @pytest.mark.parametrize("dev", cpu_and_gpu()) def test_video_model(model_fn, dev): # the default input shape is @@ -842,7 +842,7 @@ def test_video_model(model_fn, dev): ), reason="This Pytorch Build has not been built with fbgemm and qnnpack", ) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.quantization, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.quantization)) def test_quantized_classification_model(model_fn): set_rng_seed(0) defaults = { @@ -891,7 +891,7 @@ def test_quantized_classification_model(model_fn): torch.ao.quantization.convert(model, inplace=True) -@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection, skipped_models)) +@pytest.mark.parametrize("model_fn", get_models_from_module(models.detection)) def test_detection_model_trainable_backbone_layers(model_fn, disable_weight_loading): model_name = model_fn.__name__ max_trainable = _model_tests_values[model_name]["max_trainable"] From c9005b30906b9266ac68d7059f0daa366d511da8 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 15:38:23 +0100 Subject: [PATCH 3/8] Also skip regnet_y_128gf --- test/test_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_models.py b/test/test_models.py index cc8a45286a5..c137699261b 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -22,6 +22,7 @@ # skip big models to reduce memory usage on CI test skipped_models = { "vit_h_14", + "regnet_y_128gf", } From 15a02a3bd8cc8a931e913c7123d84f3ab3b6eb2e Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 16:19:50 +0100 Subject: [PATCH 4/8] Only skip test for test_classification_model and create toggle using env var --- test/test_models.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index c137699261b..f3078319b89 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -18,12 +18,7 @@ from torchvision import models ACCEPT = os.getenv("EXPECTTEST_ACCEPT", "0") == "1" - -# skip big models to reduce memory usage on CI test -skipped_models = { - "vit_h_14", - "regnet_y_128gf", -} +SKIP_BIG_MODEL = os.getenv("SKIP_BIG_MODEL", "1") == "1" def get_models_from_module(module): @@ -31,7 +26,7 @@ def get_models_from_module(module): return [ v for k, v in module.__dict__.items() - if callable(v) and k[0].lower() == k[0] and k[0] != "_" and k != "get_weight" and k not in skipped_models + if callable(v) and k[0].lower() == k[0] and k[0] != "_" and k != "get_weight" ] @@ -334,6 +329,18 @@ def _check_input_backprop(model, inputs): _model_params[m] = {"input_shape": (1, 3, 64, 64)} +# skip big models to reduce memory usage on CI test +skipped_big_models = { + "vit_h_14", + "regnet_y_128gf", +} + + +def do_skip_big_model(model_name): + if model_name in skipped_big_models: + pytest.skip(f"Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model") + + # The following contains configuration and expected values to be used tests that are model specific _model_tests_values = { "retinanet_resnet50_fpn": { @@ -597,6 +604,9 @@ def test_classification_model(model_fn, dev): "input_shape": (1, 3, 224, 224), } model_name = model_fn.__name__ + if dev == torch.device("cuda") and SKIP_BIG_MODEL: + # We skip model if model_name is inside skipped_big_models + do_skip_big_model(model_name) kwargs = {**defaults, **_model_params.get(model_name, {})} num_classes = kwargs.get("num_classes") input_shape = kwargs.pop("input_shape") From a3f58196fa3da75950d88a6123ec787915d9cbf5 Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 16:21:17 +0100 Subject: [PATCH 5/8] Remove unnecessary comment --- test/test_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_models.py b/test/test_models.py index f3078319b89..59bba8bfe8b 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -605,7 +605,6 @@ def test_classification_model(model_fn, dev): } model_name = model_fn.__name__ if dev == torch.device("cuda") and SKIP_BIG_MODEL: - # We skip model if model_name is inside skipped_big_models do_skip_big_model(model_name) kwargs = {**defaults, **_model_params.get(model_name, {})} num_classes = kwargs.get("num_classes") From 404aeb3c45d4c4964f80338d19286d8bedaf877b Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 15:32:29 +0000 Subject: [PATCH 6/8] Fix comparison of device to use str --- test/test_models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index 59bba8bfe8b..7389e6b4f3c 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -604,7 +604,7 @@ def test_classification_model(model_fn, dev): "input_shape": (1, 3, 224, 224), } model_name = model_fn.__name__ - if dev == torch.device("cuda") and SKIP_BIG_MODEL: + if dev == "cuda" and SKIP_BIG_MODEL: do_skip_big_model(model_name) kwargs = {**defaults, **_model_params.get(model_name, {})} num_classes = kwargs.get("num_classes") @@ -620,7 +620,7 @@ def test_classification_model(model_fn, dev): _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None), eager_out=out) _check_fx_compatible(model, x, eager_out=out) - if dev == torch.device("cuda"): + if dev == "cuda": with torch.cuda.amp.autocast(): out = model(x) # See autocast_flaky_numerics comment at top of file. @@ -673,7 +673,7 @@ def check_out(out): _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None), eager_out=out) _check_fx_compatible(model, x, eager_out=out) - if dev == torch.device("cuda"): + if dev == "cuda": with torch.cuda.amp.autocast(): out = model(x) # See autocast_flaky_numerics comment at top of file. @@ -771,7 +771,7 @@ def compute_mean_std(tensor): full_validation = check_out(out) _check_jit_scriptable(model, ([x],), unwrapper=script_model_unwrapper.get(model_name, None), eager_out=out) - if dev == torch.device("cuda"): + if dev == "cuda": with torch.cuda.amp.autocast(): out = model(model_input) # See autocast_flaky_numerics comment at top of file. @@ -837,7 +837,7 @@ def test_video_model(model_fn, dev): _check_fx_compatible(model, x, eager_out=out) assert out.shape[-1] == 50 - if dev == torch.device("cuda"): + if dev == "cuda": with torch.cuda.amp.autocast(): out = model(x) assert out.shape[-1] == 50 From 99c6fb383dc8ec79f37ff1e0ae0d7973eab1146d Mon Sep 17 00:00:00 2001 From: Yosua Michael Maranatha Date: Wed, 27 Apr 2022 16:37:42 +0100 Subject: [PATCH 7/8] Add logic to test_classification_model directly --- test/test_models.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/test_models.py b/test/test_models.py index 7389e6b4f3c..d9c8b707eeb 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -335,12 +335,6 @@ def _check_input_backprop(model, inputs): "regnet_y_128gf", } - -def do_skip_big_model(model_name): - if model_name in skipped_big_models: - pytest.skip(f"Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model") - - # The following contains configuration and expected values to be used tests that are model specific _model_tests_values = { "retinanet_resnet50_fpn": { @@ -604,8 +598,8 @@ def test_classification_model(model_fn, dev): "input_shape": (1, 3, 224, 224), } model_name = model_fn.__name__ - if dev == "cuda" and SKIP_BIG_MODEL: - do_skip_big_model(model_name) + if dev == "cuda" and SKIP_BIG_MODEL and model_name in skipped_big_models: + pytest.skip("Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model") kwargs = {**defaults, **_model_params.get(model_name, {})} num_classes = kwargs.get("num_classes") input_shape = kwargs.pop("input_shape") From c50d48cecff06227f6796c6968fdad4c732a78c6 Mon Sep 17 00:00:00 2001 From: Vasilis Vryniotis Date: Thu, 28 Apr 2022 01:36:10 +0100 Subject: [PATCH 8/8] Add kprcnn in autocast flaky list --- test/test_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_models.py b/test/test_models.py index ecc2e926e89..18d01921be4 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -232,6 +232,7 @@ def _check_input_backprop(model, inputs): "lraspp_mobilenet_v3_large", "maskrcnn_resnet50_fpn", "maskrcnn_resnet50_fpn_v2", + "keypointrcnn_resnet50_fpn", ) # The tests for the following quantized models are flaky possibly due to inconsistent