From 54c0d077a1c8c92f185408b8dc2066fa2ba30a14 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 7 Dec 2020 11:01:35 +0100 Subject: [PATCH 1/9] enable testing DDP examples --- pl_examples/test_examples.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py index 96fca5bd2042e..62921fda27db3 100644 --- a/pl_examples/test_examples.py +++ b/pl_examples/test_examples.py @@ -72,22 +72,21 @@ def test_examples_dp(tmpdir, import_cli, cli_args): module.cli_main() -# ToDo: fix this failing example -# @pytest.mark.parametrize('import_cli', [ -# 'pl_examples.basic_examples.simple_image_classifier', -# 'pl_examples.basic_examples.backbone_image_classifier', -# 'pl_examples.basic_examples.autoencoder', -# ]) -# @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") -# @pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP_AMP]) -# def test_examples_ddp(tmpdir, import_cli, cli_args): -# -# module = importlib.import_module(import_cli) -# # update the temp dir -# cli_args = cli_args % {'tmpdir': tmpdir} -# -# with mock.patch("argparse._sys.argv", ["any.py"] + cli_args.strip().split()): -# module.cli_main() +@pytest.mark.parametrize('import_cli', [ + 'pl_examples.basic_examples.simple_image_classifier', + 'pl_examples.basic_examples.backbone_image_classifier', + 'pl_examples.basic_examples.autoencoder', +]) +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") +@pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP_AMP]) +def test_examples_ddp(tmpdir, import_cli, cli_args): + + module = importlib.import_module(import_cli) + # update the temp dir + cli_args = cli_args % {'tmpdir': tmpdir} + + with mock.patch("argparse._sys.argv", ["any.py"] + cli_args.strip().split()): + module.cli_main() @pytest.mark.parametrize( From ed6b367a8d372638964150717eaefd41eb68e7a7 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 7 Dec 2020 12:11:18 +0100 Subject: [PATCH 2/9] args --- pl_examples/test_examples.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py index 62921fda27db3..0cbd41c0bac02 100644 --- a/pl_examples/test_examples.py +++ b/pl_examples/test_examples.py @@ -38,17 +38,12 @@ --accelerator dp \ """ -ARGS_DP_AMP = ARGS_DP + """ ---precision 16 \ -""" - ARGS_DDP = ARGS_DEFAULT + """ --gpus 2 \ --accelerator ddp \ ---precision 16 \ """ -ARGS_DDP_AMP = ARGS_DEFAULT + """ +ARGS_AMP = """ --precision 16 \ """ @@ -61,7 +56,7 @@ ] ) @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") -@pytest.mark.parametrize('cli_args', [ARGS_DP, ARGS_DP_AMP]) +@pytest.mark.parametrize('cli_args', [ARGS_DP, ARGS_DP + ARGS_AMP]) def test_examples_dp(tmpdir, import_cli, cli_args): module = importlib.import_module(import_cli) @@ -78,7 +73,7 @@ def test_examples_dp(tmpdir, import_cli, cli_args): 'pl_examples.basic_examples.autoencoder', ]) @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") -@pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP_AMP]) +@pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP + ARGS_AMP]) def test_examples_ddp(tmpdir, import_cli, cli_args): module = importlib.import_module(import_cli) From 6339d338c88f2253b554e3947e9d2c6e44ad249a Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 7 Dec 2020 22:02:27 +0100 Subject: [PATCH 3/9] ddp_spawn --- pl_examples/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py index 0cbd41c0bac02..7af619eb51e6a 100644 --- a/pl_examples/test_examples.py +++ b/pl_examples/test_examples.py @@ -40,7 +40,7 @@ ARGS_DDP = ARGS_DEFAULT + """ --gpus 2 \ ---accelerator ddp \ +--accelerator ddp_spawn \ """ ARGS_AMP = """ From 61b299cb08718aa63357e149aea18855b35fcd0d Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 17 Dec 2020 17:16:05 +0100 Subject: [PATCH 4/9] ddp as extra script --- azure-pipelines.yml | 4 ++++ pl_examples/run_ddp-example.sh | 12 +++++++++++ pl_examples/test_examples.py | 38 +++++++++++++++++----------------- 3 files changed, 35 insertions(+), 19 deletions(-) create mode 100644 pl_examples/run_ddp-example.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0bb4f5cfffd82..96fde2de24e9a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,3 +88,7 @@ jobs: - script: | python -m pytest benchmarks pl_examples -v --maxfail=2 --durations=0 displayName: 'Testing: extended' + + - script: | + bash pl_examples/run_ddp-example.sh + displayName: 'Examples' diff --git a/pl_examples/run_ddp-example.sh b/pl_examples/run_ddp-example.sh new file mode 100644 index 0000000000000..f0c7695e766f2 --- /dev/null +++ b/pl_examples/run_ddp-example.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +ARGS_DEFAULT=" --default_root_dir %(tmpdir)s --max_epochs 1 --batch_size 32 --limit_train_batches 2 --limit_val_batches 2" +ARGS_EXTRA_DDP=" --gpus 2 --accelerator ddp" +ARGS_EXTRA_AMP=" --precision 16" + +python pl_examples/basic_examples/simple_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} +python pl_examples/basic_examples/simple_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP} +python pl_examples/basic_examples/backbone_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} +python pl_examples/basic_examples/backbone_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP} +python pl_examples/basic_examples/autoencoder.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} +python pl_examples/basic_examples/autoencoder.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP} diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py index 7af619eb51e6a..ad544a3f274f0 100644 --- a/pl_examples/test_examples.py +++ b/pl_examples/test_examples.py @@ -38,10 +38,10 @@ --accelerator dp \ """ -ARGS_DDP = ARGS_DEFAULT + """ ---gpus 2 \ ---accelerator ddp_spawn \ -""" +# ARGS_DDP = ARGS_DEFAULT + """ +# --gpus 2 \ +# --accelerator ddp_spawn \ +# """ ARGS_AMP = """ --precision 16 \ @@ -67,21 +67,21 @@ def test_examples_dp(tmpdir, import_cli, cli_args): module.cli_main() -@pytest.mark.parametrize('import_cli', [ - 'pl_examples.basic_examples.simple_image_classifier', - 'pl_examples.basic_examples.backbone_image_classifier', - 'pl_examples.basic_examples.autoencoder', -]) -@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") -@pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP + ARGS_AMP]) -def test_examples_ddp(tmpdir, import_cli, cli_args): - - module = importlib.import_module(import_cli) - # update the temp dir - cli_args = cli_args % {'tmpdir': tmpdir} - - with mock.patch("argparse._sys.argv", ["any.py"] + cli_args.strip().split()): - module.cli_main() +# @pytest.mark.parametrize('import_cli', [ +# 'pl_examples.basic_examples.simple_image_classifier', +# 'pl_examples.basic_examples.backbone_image_classifier', +# 'pl_examples.basic_examples.autoencoder', +# ]) +# @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") +# @pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP + ARGS_AMP]) +# def test_examples_ddp(tmpdir, import_cli, cli_args): +# +# module = importlib.import_module(import_cli) +# # update the temp dir +# cli_args = cli_args % {'tmpdir': tmpdir} +# +# with mock.patch("argparse._sys.argv", ["any.py"] + cli_args.strip().split()): +# module.cli_main() @pytest.mark.parametrize( From df15e9453537fa97789c1a6a3c9942a0b82f071a Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 12 Feb 2021 09:21:18 +0100 Subject: [PATCH 5/9] path # Conflicts: # .drone.yml --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 96fde2de24e9a..41d5d2223dc49 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -91,4 +91,6 @@ jobs: - script: | bash pl_examples/run_ddp-example.sh + env: + PYTHONPATH: "$PWD:$PYTHONPATH" displayName: 'Examples' From f836bd61e4dbf9e067d4b942b6f418007d90f265 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 12 Feb 2021 09:52:10 +0100 Subject: [PATCH 6/9] install --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 41d5d2223dc49..fc5c228e1fd7f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -90,7 +90,6 @@ jobs: displayName: 'Testing: extended' - script: | + python setup.py install bash pl_examples/run_ddp-example.sh - env: - PYTHONPATH: "$PWD:$PYTHONPATH" displayName: 'Examples' From 5cffd4764a615709f989909582554d592b04e62b Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 12 Feb 2021 15:34:13 +0100 Subject: [PATCH 7/9] -u --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc5c228e1fd7f..a659908c6229a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -90,6 +90,6 @@ jobs: displayName: 'Testing: extended' - script: | - python setup.py install + python setup.py install --user bash pl_examples/run_ddp-example.sh displayName: 'Examples' From 4e5f3cb660c9370746d338dfb86ebe020c96953b Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 12 Feb 2021 18:02:04 +0100 Subject: [PATCH 8/9] q --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a659908c6229a..68e63771b3892 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -90,6 +90,7 @@ jobs: displayName: 'Testing: extended' - script: | - python setup.py install --user + python setup.py install --user --quiet bash pl_examples/run_ddp-example.sh + pip uninstall pytorch-lightning displayName: 'Examples' From 978bdf2ed72df2409ac43a201192660a608ab76a Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 12 Feb 2021 20:10:06 +0100 Subject: [PATCH 9/9] -y --- azure-pipelines.yml | 2 +- pl_examples/test_examples.py | 22 ---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 68e63771b3892..8a6f1324521b0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -92,5 +92,5 @@ jobs: - script: | python setup.py install --user --quiet bash pl_examples/run_ddp-example.sh - pip uninstall pytorch-lightning + pip uninstall -y pytorch-lightning displayName: 'Examples' diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py index ad544a3f274f0..b930957a26346 100644 --- a/pl_examples/test_examples.py +++ b/pl_examples/test_examples.py @@ -38,11 +38,6 @@ --accelerator dp \ """ -# ARGS_DDP = ARGS_DEFAULT + """ -# --gpus 2 \ -# --accelerator ddp_spawn \ -# """ - ARGS_AMP = """ --precision 16 \ """ @@ -67,23 +62,6 @@ def test_examples_dp(tmpdir, import_cli, cli_args): module.cli_main() -# @pytest.mark.parametrize('import_cli', [ -# 'pl_examples.basic_examples.simple_image_classifier', -# 'pl_examples.basic_examples.backbone_image_classifier', -# 'pl_examples.basic_examples.autoencoder', -# ]) -# @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") -# @pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP + ARGS_AMP]) -# def test_examples_ddp(tmpdir, import_cli, cli_args): -# -# module = importlib.import_module(import_cli) -# # update the temp dir -# cli_args = cli_args % {'tmpdir': tmpdir} -# -# with mock.patch("argparse._sys.argv", ["any.py"] + cli_args.strip().split()): -# module.cli_main() - - @pytest.mark.parametrize( 'import_cli', [ 'pl_examples.basic_examples.simple_image_classifier',