From 57ad254332270b508727ae32f6f4b3daf8ca2960 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 09:47:05 +0200 Subject: [PATCH 01/41] enable caching of model weights for prototype CI --- .circleci/config.yml | 8 ++++++++ .circleci/config.yml.in | 8 ++++++++ test/test_prototype_models.py | 6 +++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a17410b9a6f..b3af3714cb8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,9 +271,17 @@ jobs: - run: name: Install test utilities command: pip install --user --progress-bar=off pytest + - restore_cache: + keys: + - torch-hub - run: name: Run tests command: pytest test/test_prototype_*.py + - save_cache: + keys: + - torch-hub + paths: + - ~/.cache/torch/hub binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4b0bc8d7e1d..c79675167b5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -271,9 +271,17 @@ jobs: - run: name: Install test utilities command: pip install --user --progress-bar=off pytest + - restore_cache: + keys: + - torch-hub - run: name: Run tests command: pytest test/test_prototype_*.py + - save_cache: + keys: + - torch-hub + paths: + - ~/.cache/torch/hub binary_linux_wheel: <<: *binary_common diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index 4c1694789f1..e8fca88dcd2 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -33,21 +33,21 @@ def test_get_weight(): @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +#@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") def test_classification_model(model_fn, dev): TM.test_classification_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models.segmentation)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +#@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") def test_segmentation_model(model_fn, dev): TM.test_segmentation_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models) + TM.get_models_from_module(models.segmentation)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +#@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") def test_old_vs_new_factory(model_fn, dev): defaults = { "pretrained": True, From 54e88cad376f43896a1691e8a33323d698a32ef7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 09:53:14 +0200 Subject: [PATCH 02/41] syntax --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3af3714cb8..9276e413b2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -278,7 +278,7 @@ jobs: name: Run tests command: pytest test/test_prototype_*.py - save_cache: - keys: + key: - torch-hub paths: - ~/.cache/torch/hub diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c79675167b5..f00105dd948 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -278,7 +278,7 @@ jobs: name: Run tests command: pytest test/test_prototype_*.py - save_cache: - keys: + key: - torch-hub paths: - ~/.cache/torch/hub From 603f615178073faee9864d81413bb37421240e17 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 09:54:29 +0200 Subject: [PATCH 03/41] syntax --- .circleci/config.yml | 3 +-- .circleci/config.yml.in | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9276e413b2d..911800a721f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -278,8 +278,7 @@ jobs: name: Run tests command: pytest test/test_prototype_*.py - save_cache: - key: - - torch-hub + key: torch-hub paths: - ~/.cache/torch/hub diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index f00105dd948..f8f970fe497 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -278,8 +278,7 @@ jobs: name: Run tests command: pytest test/test_prototype_*.py - save_cache: - key: - - torch-hub + key: torch-hub paths: - ~/.cache/torch/hub From 35a7a46abe63c979f96e002a7675370822f1018d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 10:13:27 +0200 Subject: [PATCH 04/41] make cache dir dynamic --- .circleci/config.yml | 5 ++++- .circleci/config.yml.in | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 911800a721f..90966f999a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,10 +277,13 @@ jobs: - run: name: Run tests command: pytest test/test_prototype_*.py + - run: + name: Find torch.hub dir for caching + command: export HUB_DIR=$(python -c "import torch; print(torch.hub.get_dir())") - save_cache: key: torch-hub paths: - - ~/.cache/torch/hub + - $HUB_DIR binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index f8f970fe497..db0d01a16ac 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -277,10 +277,13 @@ jobs: - run: name: Run tests command: pytest test/test_prototype_*.py + - run: + name: Find torch.hub dir for caching + command: export HUB_DIR=$(python -c "import torch; print(torch.hub.get_dir())") - save_cache: key: torch-hub paths: - - ~/.cache/torch/hub + - $HUB_DIR binary_linux_wheel: <<: *binary_common From df386b9b0cd11bc53d03316fb1a31b007ff3071a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 10:13:47 +0200 Subject: [PATCH 05/41] increase verbosity --- .circleci/config.yml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index db0d01a16ac..488e0336e98 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -276,7 +276,7 @@ jobs: - torch-hub - run: name: Run tests - command: pytest test/test_prototype_*.py + command: pytest -vvv test/test_prototype_*.py - run: name: Find torch.hub dir for caching command: export HUB_DIR=$(python -c "import torch; print(torch.hub.get_dir())") From 96b741118575d208c44e0eee492b08987c089a35 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 10:15:36 +0200 Subject: [PATCH 06/41] fix --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90966f999a4..5b6a8da5d69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,10 +276,10 @@ jobs: - torch-hub - run: name: Run tests - command: pytest test/test_prototype_*.py + command: pytest -vvv test/test_prototype_*.py - run: name: Find torch.hub dir for caching - command: export HUB_DIR=$(python -c "import torch; print(torch.hub.get_dir())") + command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV - save_cache: key: torch-hub paths: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 488e0336e98..c3714a068af 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -279,7 +279,7 @@ jobs: command: pytest -vvv test/test_prototype_*.py - run: name: Find torch.hub dir for caching - command: export HUB_DIR=$(python -c "import torch; print(torch.hub.get_dir())") + command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV - save_cache: key: torch-hub paths: From adf0c24e41bb303627a6a27740948f55546ddddc Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 10:47:07 +0200 Subject: [PATCH 07/41] use larget CI machine --- .circleci/config.yml | 3 ++- .circleci/config.yml.in | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b6a8da5d69..83ad2da501c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -257,6 +257,7 @@ jobs: prototype_test: docker: - image: circleci/python:3.7 + resource_class: large steps: - run: name: Install torch @@ -276,7 +277,7 @@ jobs: - torch-hub - run: name: Run tests - command: pytest -vvv test/test_prototype_*.py + command: pytest -svvv test/test_prototype_*.py - run: name: Find torch.hub dir for caching command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c3714a068af..4f6e5e6be79 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -257,6 +257,7 @@ jobs: prototype_test: docker: - image: circleci/python:3.7 + resource_class: large steps: - run: name: Install torch @@ -276,7 +277,7 @@ jobs: - torch-hub - run: name: Run tests - command: pytest -vvv test/test_prototype_*.py + command: pytest -svvv test/test_prototype_*.py - run: name: Find torch.hub dir for caching command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV From 0fc0bdd8a4cb456e952a5b89a0332a1be29b775a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 10:55:57 +0200 Subject: [PATCH 08/41] revert debug output --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83ad2da501c..ea91e82a5f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,7 +277,7 @@ jobs: - torch-hub - run: name: Run tests - command: pytest -svvv test/test_prototype_*.py + command: pytest --durations=20 test/test_prototype_*.py - run: name: Find torch.hub dir for caching command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4f6e5e6be79..a6c00706ef4 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -277,7 +277,7 @@ jobs: - torch-hub - run: name: Run tests - command: pytest -svvv test/test_prototype_*.py + command: pytest --durations=20 test/test_prototype_*.py - run: name: Find torch.hub dir for caching command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV From b9335aa4eac9454aa7bd94aebba7150657dbf044 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 10:59:28 +0200 Subject: [PATCH 09/41] [DEBUG] test env var usage in save_cache --- .circleci/config.yml | 18 +++++++++++------- .circleci/config.yml.in | 18 +++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea91e82a5f1..ee09f33c794 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -262,6 +262,17 @@ jobs: - run: name: Install torch command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + - run: + name: Find torch.hub dir for caching + command: | + HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') + echo $HUB_DIR + echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV + - save_cache: + key: torch-hub + paths: + - ${HUB_DIR} + - run: exit 1 - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git @@ -278,13 +289,6 @@ jobs: - run: name: Run tests command: pytest --durations=20 test/test_prototype_*.py - - run: - name: Find torch.hub dir for caching - command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV - - save_cache: - key: torch-hub - paths: - - $HUB_DIR binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a6c00706ef4..62f5bf88399 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -262,6 +262,17 @@ jobs: - run: name: Install torch command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + - run: + name: Find torch.hub dir for caching + command: | + HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') + echo $HUB_DIR + echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV + - save_cache: + key: torch-hub + paths: + - ${HUB_DIR} + - run: exit 1 - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git @@ -278,13 +289,6 @@ jobs: - run: name: Run tests command: pytest --durations=20 test/test_prototype_*.py - - run: - name: Find torch.hub dir for caching - command: echo "export HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())')" >> $BASH_ENV - - save_cache: - key: torch-hub - paths: - - $HUB_DIR binary_linux_wheel: <<: *binary_common From 7bebe1aaa68214722cf096cbd9986e4f9209d5c3 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 11:04:28 +0200 Subject: [PATCH 10/41] retry --- .circleci/config.yml | 29 +++++++++++++++-------------- .circleci/config.yml.in | 29 +++++++++++++++-------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ee09f33c794..90af75eb767 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -261,25 +261,16 @@ jobs: steps: - run: name: Install torch - command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - run: - name: Find torch.hub dir for caching command: | - HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') - echo $HUB_DIR - echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV - - save_cache: - key: torch-hub - paths: - - ${HUB_DIR} - - run: exit 1 + pip install --user --progress-bar=off numpy + pip install --user --progress-bar=off --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - checkout - - run: - name: Install torchvision - command: pip install --user --progress-bar off --no-build-isolation . +# - run: +# name: Install torchvision +# command: pip install --user --progress-bar off --no-build-isolation . - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -289,6 +280,16 @@ jobs: - run: name: Run tests command: pytest --durations=20 test/test_prototype_*.py + - run: + name: Find torch.hub dir for caching + command: | + HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') + echo "HUB_DIR=$HUB_DIR" + echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV + - save_cache: + key: torch-hub + paths: + - ${HUB_DIR} binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 62f5bf88399..0912cdc3469 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -261,25 +261,16 @@ jobs: steps: - run: name: Install torch - command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - run: - name: Find torch.hub dir for caching command: | - HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') - echo $HUB_DIR - echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV - - save_cache: - key: torch-hub - paths: - - ${HUB_DIR} - - run: exit 1 + pip install --user --progress-bar=off numpy + pip install --user --progress-bar=off --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - checkout - - run: - name: Install torchvision - command: pip install --user --progress-bar off --no-build-isolation . +# - run: +# name: Install torchvision +# command: pip install --user --progress-bar off --no-build-isolation . - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -289,6 +280,16 @@ jobs: - run: name: Run tests command: pytest --durations=20 test/test_prototype_*.py + - run: + name: Find torch.hub dir for caching + command: | + HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') + echo "HUB_DIR=$HUB_DIR" + echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV + - save_cache: + key: torch-hub + paths: + - ${HUB_DIR} binary_linux_wheel: <<: *binary_common From a134e0335b0ca31c8cd3508f5833e8a75b8e22dc Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 11:36:10 +0200 Subject: [PATCH 11/41] use checksum for caching --- .circleci/config.yml | 34 +++++++++++++++++++++++----------- .circleci/config.yml.in | 34 +++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90af75eb767..c1867fcc67c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,24 +272,36 @@ jobs: # name: Install torchvision # command: pip install --user --progress-bar off --no-build-isolation . - run: - name: Install test utilities - command: pip install --user --progress-bar=off pytest + name: Prepare caching + command: | + CACHE_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') + echo "CACHE_DIR=CACHE_DIR" + echo "export CACHE_DIR=CACHE_DIR" >> $BASH_ENV + + CACHE_FALLBACK_KEY=model-weights + echo "CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" + echo "export CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" >> $BASH_ENV + + # Although CircleCI provides a checksum functionality, + # we can't use that here, because it only works for single files + CHECKSUM=($(tar -cf - torchvision/prototype/models | sha256sum)) + CACHE_KEY=$CACHE_FALLBACK_KEY-$CHECKSUM + echo "CACHE_KEY=CACHE_KEY" + echo "export CACHE_KEY=CACHE_KEY" >> $BASH_ENV - restore_cache: keys: - - torch-hub + - ${CACHE_KEY} + - ${CACHE_FALLBACK_KEY} + - run: + name: Install test utilities + command: pip install --user --progress-bar=off pytest - run: name: Run tests command: pytest --durations=20 test/test_prototype_*.py - - run: - name: Find torch.hub dir for caching - command: | - HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') - echo "HUB_DIR=$HUB_DIR" - echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV - save_cache: - key: torch-hub + key: ${CACHE_KEY} paths: - - ${HUB_DIR} + - ${CACHE_DIR} binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 0912cdc3469..9869c479e8a 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -272,24 +272,36 @@ jobs: # name: Install torchvision # command: pip install --user --progress-bar off --no-build-isolation . - run: - name: Install test utilities - command: pip install --user --progress-bar=off pytest + name: Prepare caching + command: | + CACHE_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') + echo "CACHE_DIR=CACHE_DIR" + echo "export CACHE_DIR=CACHE_DIR" >> $BASH_ENV + + CACHE_FALLBACK_KEY=model-weights + echo "CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" + echo "export CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" >> $BASH_ENV + + # Although CircleCI provides a checksum functionality, + # we can't use that here, because it only works for single files + CHECKSUM=($(tar -cf - torchvision/prototype/models | sha256sum)) + CACHE_KEY=$CACHE_FALLBACK_KEY-$CHECKSUM + echo "CACHE_KEY=CACHE_KEY" + echo "export CACHE_KEY=CACHE_KEY" >> $BASH_ENV - restore_cache: keys: - - torch-hub + - ${CACHE_KEY} + - ${CACHE_FALLBACK_KEY} + - run: + name: Install test utilities + command: pip install --user --progress-bar=off pytest - run: name: Run tests command: pytest --durations=20 test/test_prototype_*.py - - run: - name: Find torch.hub dir for caching - command: | - HUB_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') - echo "HUB_DIR=$HUB_DIR" - echo "export HUB_DIR=$HUB_DIR" >> $BASH_ENV - save_cache: - key: torch-hub + key: ${CACHE_KEY} paths: - - ${HUB_DIR} + - ${CACHE_DIR} binary_linux_wheel: <<: *binary_common From c6bdeeaca173ece80ed4e542e8d026ea18cad543 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 11:47:40 +0200 Subject: [PATCH 12/41] remove env vars because expansion is not working --- .circleci/config.yml | 28 +++++++++------------------- .circleci/config.yml.in | 28 +++++++++------------------- 2 files changed, 18 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c1867fcc67c..12849eb93ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -273,25 +273,13 @@ jobs: # command: pip install --user --progress-bar off --no-build-isolation . - run: name: Prepare caching - command: | - CACHE_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') - echo "CACHE_DIR=CACHE_DIR" - echo "export CACHE_DIR=CACHE_DIR" >> $BASH_ENV - - CACHE_FALLBACK_KEY=model-weights - echo "CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" - echo "export CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" >> $BASH_ENV - - # Although CircleCI provides a checksum functionality, - # we can't use that here, because it only works for single files - CHECKSUM=($(tar -cf - torchvision/prototype/models | sha256sum)) - CACHE_KEY=$CACHE_FALLBACK_KEY-$CHECKSUM - echo "CACHE_KEY=CACHE_KEY" - echo "export CACHE_KEY=CACHE_KEY" >> $BASH_ENV + command: tar -cf weights.lock torchvision/prototype/models - restore_cache: keys: - - ${CACHE_KEY} - - ${CACHE_FALLBACK_KEY} + + - weights-{{ checksum weights.lock }} + + - weights- - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -299,9 +287,11 @@ jobs: name: Run tests command: pytest --durations=20 test/test_prototype_*.py - save_cache: - key: ${CACHE_KEY} + + key: weights-{{ checksum weights.lock}} + paths: - - ${CACHE_DIR} + - ~/.cache/torch/hub binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 9869c479e8a..24334b07d4e 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -273,25 +273,13 @@ jobs: # command: pip install --user --progress-bar off --no-build-isolation . - run: name: Prepare caching - command: | - CACHE_DIR=$(python -c 'import torch; print(torch.hub.get_dir())') - echo "CACHE_DIR=CACHE_DIR" - echo "export CACHE_DIR=CACHE_DIR" >> $BASH_ENV - - CACHE_FALLBACK_KEY=model-weights - echo "CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" - echo "export CACHE_FALLBACK_KEY=CACHE_FALLBACK_KEY" >> $BASH_ENV - - # Although CircleCI provides a checksum functionality, - # we can't use that here, because it only works for single files - CHECKSUM=($(tar -cf - torchvision/prototype/models | sha256sum)) - CACHE_KEY=$CACHE_FALLBACK_KEY-$CHECKSUM - echo "CACHE_KEY=CACHE_KEY" - echo "export CACHE_KEY=CACHE_KEY" >> $BASH_ENV + command: tar -cf weights.lock torchvision/prototype/models - restore_cache: keys: - - ${CACHE_KEY} - - ${CACHE_FALLBACK_KEY} + {% raw %} + - weights-{{ checksum weights.lock }} + {% endraw %} + - weights- - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -299,9 +287,11 @@ jobs: name: Run tests command: pytest --durations=20 test/test_prototype_*.py - save_cache: - key: ${CACHE_KEY} + {% raw %} + key: weights-{{ checksum weights.lock}} + {% endraw %} paths: - - ${CACHE_DIR} + - ~/.cache/torch/hub binary_linux_wheel: <<: *binary_common From f443f2ac4032d04e72a48758b7f85b6567b0dcc6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 11:49:59 +0200 Subject: [PATCH 13/41] syntax --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12849eb93ee..e2e9d694f84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,7 +277,7 @@ jobs: - restore_cache: keys: - - weights-{{ checksum weights.lock }} + - weights-{{ checksum "weights.lock" }} - weights- - run: @@ -288,7 +288,7 @@ jobs: command: pytest --durations=20 test/test_prototype_*.py - save_cache: - key: weights-{{ checksum weights.lock}} + key: weights-{{ "checksum weights.lock" }} paths: - ~/.cache/torch/hub diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 24334b07d4e..a5656587b87 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -277,7 +277,7 @@ jobs: - restore_cache: keys: {% raw %} - - weights-{{ checksum weights.lock }} + - weights-{{ checksum "weights.lock" }} {% endraw %} - weights- - run: @@ -288,7 +288,7 @@ jobs: command: pytest --durations=20 test/test_prototype_*.py - save_cache: {% raw %} - key: weights-{{ checksum weights.lock}} + key: weights-{{ "checksum weights.lock" }} {% endraw %} paths: - ~/.cache/torch/hub From d497f21631134a51c03aaa343727d83db4860c8f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 12:02:27 +0200 Subject: [PATCH 14/41] cleanup --- .circleci/config.yml | 13 ++++++++----- .circleci/config.yml.in | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2e9d694f84..d97390787ef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -263,16 +263,19 @@ jobs: name: Install torch command: | pip install --user --progress-bar=off numpy - pip install --user --progress-bar=off --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - checkout -# - run: -# name: Install torchvision -# command: pip install --user --progress-bar off --no-build-isolation . + - run: + name: Install torchvision + command: pip install --user --progress-bar off --no-build-isolation . - run: name: Prepare caching + # CircleCI's builtin {{ checksum ... }} utility + # only works for single files, so we create an archive here for all the files + # that should be used for the cache key. command: tar -cf weights.lock torchvision/prototype/models - restore_cache: keys: @@ -285,7 +288,7 @@ jobs: command: pip install --user --progress-bar=off pytest - run: name: Run tests - command: pytest --durations=20 test/test_prototype_*.py + command: pytest -v --durations=20 test/test_prototype_*.py - save_cache: key: weights-{{ "checksum weights.lock" }} diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a5656587b87..35c2abf0846 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -263,16 +263,19 @@ jobs: name: Install torch command: | pip install --user --progress-bar=off numpy - pip install --user --progress-bar=off --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - checkout -# - run: -# name: Install torchvision -# command: pip install --user --progress-bar off --no-build-isolation . + - run: + name: Install torchvision + command: pip install --user --progress-bar off --no-build-isolation . - run: name: Prepare caching + # CircleCI's builtin {% raw %} {{ checksum ... }} {% endraw %} utility + # only works for single files, so we create an archive here for all the files + # that should be used for the cache key. command: tar -cf weights.lock torchvision/prototype/models - restore_cache: keys: @@ -285,7 +288,7 @@ jobs: command: pip install --user --progress-bar=off pytest - run: name: Run tests - command: pytest --durations=20 test/test_prototype_*.py + command: pytest -v --durations=20 test/test_prototype_*.py - save_cache: {% raw %} key: weights-{{ "checksum weights.lock" }} From 4a9f1c158b62f50601aad571d375b469fcf8c57a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 14:17:13 +0200 Subject: [PATCH 15/41] base caching on model-urls --- .circleci/config.yml | 11 ++++------- .circleci/config.yml.in | 11 ++++------- scripts/collect_model_urls.py | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 scripts/collect_model_urls.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d97390787ef..9a19ae1f481 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -273,16 +273,13 @@ jobs: command: pip install --user --progress-bar off --no-build-isolation . - run: name: Prepare caching - # CircleCI's builtin {{ checksum ... }} utility - # only works for single files, so we create an archive here for all the files - # that should be used for the cache key. - command: tar -cf weights.lock torchvision/prototype/models + command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - restore_cache: keys: - - weights-{{ checksum "weights.lock" }} + - hub-{{ checksum "prototype_model_urls.txt" }} - - weights- + - hub- - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -291,7 +288,7 @@ jobs: command: pytest -v --durations=20 test/test_prototype_*.py - save_cache: - key: weights-{{ "checksum weights.lock" }} + key: hub-{{ checksum "prototype_model_urls.txt" }} paths: - ~/.cache/torch/hub diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 35c2abf0846..7d630e83982 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -273,16 +273,13 @@ jobs: command: pip install --user --progress-bar off --no-build-isolation . - run: name: Prepare caching - # CircleCI's builtin {% raw %} {{ checksum ... }} {% endraw %} utility - # only works for single files, so we create an archive here for all the files - # that should be used for the cache key. - command: tar -cf weights.lock torchvision/prototype/models + command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - restore_cache: keys: {% raw %} - - weights-{{ checksum "weights.lock" }} + - hub-{{ checksum "prototype_model_urls.txt" }} {% endraw %} - - weights- + - hub- - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -291,7 +288,7 @@ jobs: command: pytest -v --durations=20 test/test_prototype_*.py - save_cache: {% raw %} - key: weights-{{ "checksum weights.lock" }} + key: hub-{{ checksum "prototype_model_urls.txt" }} {% endraw %} paths: - ~/.cache/torch/hub diff --git a/scripts/collect_model_urls.py b/scripts/collect_model_urls.py new file mode 100644 index 00000000000..bb3d9d709d7 --- /dev/null +++ b/scripts/collect_model_urls.py @@ -0,0 +1,22 @@ +import pathlib +import re +import sys + +MODEL_URL_PATTERN = re.compile(r"https://download.pytorch.org/models/\w+-[0-9a-f]{8}[.]pth") + + +def main(root): + model_urls = set() + for path in pathlib.Path(root).glob("**/*"): + if path.name.startswith("_") or not path.suffix == ".py": + continue + + with open(path, "r") as file: + for line in file: + model_urls.update(MODEL_URL_PATTERN.findall(line)) + + print("\n".join(sorted(model_urls))) + + +if __name__ == "__main__": + main(sys.argv[1]) From bd4c84d144d9d6ce1ea66eb707e8fd60e8be8bed Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 15:04:42 +0200 Subject: [PATCH 16/41] relax regex --- scripts/collect_model_urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_model_urls.py b/scripts/collect_model_urls.py index bb3d9d709d7..4b3fb1673f5 100644 --- a/scripts/collect_model_urls.py +++ b/scripts/collect_model_urls.py @@ -2,7 +2,7 @@ import re import sys -MODEL_URL_PATTERN = re.compile(r"https://download.pytorch.org/models/\w+-[0-9a-f]{8}[.]pth") +MODEL_URL_PATTERN = re.compile(r"https://download.pytorch.org/models/.*?[.]pth") def main(root): From b5bf51a4ea5598ff5082284cb848bfbe3adb4502 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 15:19:53 +0200 Subject: [PATCH 17/41] cleanup skips --- .circleci/config.yml.in | 4 +++- test/common_utils.py | 21 ++++++++++++++++++--- test/test_prototype_models.py | 13 ++++++++----- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 7d630e83982..61974d3b15b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -285,7 +285,9 @@ jobs: command: pip install --user --progress-bar=off pytest - run: name: Run tests - command: pytest -v --durations=20 test/test_prototype_*.py + command: | + export PYTORCH_TEST_WITH_PROTOTYPE=1 + pytest -v --durations=20 test/test_prototype_*.py - save_cache: {% raw %} key: hub-{{ checksum "prototype_model_urls.txt" }} diff --git a/test/common_utils.py b/test/common_utils.py index f782613971c..1e9766330f5 100644 --- a/test/common_utils.py +++ b/test/common_utils.py @@ -4,8 +4,10 @@ import random import shutil import tempfile +from distutils.util import strtobool import numpy as np +import pytest import torch from PIL import Image from torchvision import io @@ -13,9 +15,18 @@ import __main__ # noqa: 401 -IN_CIRCLE_CI = os.getenv("CIRCLECI", False) == "true" -IN_RE_WORKER = os.environ.get("INSIDE_RE_WORKER") is not None -IN_FBCODE = os.environ.get("IN_FBCODE_TORCHVISION") == "1" +def get_bool_env_var(name, *, exist_ok=False, default=False): + value = os.getenv(name) + if value is None: + return default + if exist_ok: + return True + return bool(strtobool(value)) + + +IN_CIRCLE_CI = get_bool_env_var("CIRCLECI") +IN_RE_WORKER = get_bool_env_var("INSIDE_RE_WORKER", exist_ok=True) +IN_FBCODE = get_bool_env_var("IN_FBCODE_TORCHVISION") CUDA_NOT_AVAILABLE_MSG = "CUDA device not available" CIRCLECI_GPU_NO_CUDA_MSG = "We're in a CircleCI GPU machine, and this test doesn't need cuda." @@ -202,3 +213,7 @@ def _test_fn_on_batch(batch_tensors, fn, scripted_fn_atol=1e-8, **fn_kwargs): # scriptable function test s_transformed_batch = scripted_fn(batch_tensors, **fn_kwargs) torch.testing.assert_close(transformed_batch, s_transformed_batch, rtol=1e-5, atol=scripted_fn_atol) + + +def skip_on_env_var(name, *, reason, exist_ok=False, default=False): + return pytest.mark.skipif(get_bool_env_var(name, exist_ok=exist_ok, default=default), reason=reason) diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index e8fca88dcd2..98e83e609a2 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -1,12 +1,15 @@ import importlib -import os import pytest import test_models as TM import torch -from common_utils import cpu_and_gpu +from common_utils import cpu_and_gpu, skip_on_env_var from torchvision.prototype import models +skip_if_not_test_with_prototype = skip_on_env_var( + "PYTORCH_TEST_WITH_PROTOTYPE", reason="Prototype code tests are disabled" +) + def _get_original_model(model_fn): original_module_name = model_fn.__module__.replace(".prototype", "") @@ -33,21 +36,21 @@ def test_get_weight(): @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -#@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +@skip_if_not_test_with_prototype def test_classification_model(model_fn, dev): TM.test_classification_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models.segmentation)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -#@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +@skip_if_not_test_with_prototype def test_segmentation_model(model_fn, dev): TM.test_segmentation_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models) + TM.get_models_from_module(models.segmentation)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -#@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +@skip_if_not_test_with_prototype def test_old_vs_new_factory(model_fn, dev): defaults = { "pretrained": True, From fe4537b8333e7099c820656107a08671e73a6bf1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 16:58:27 +0200 Subject: [PATCH 18/41] cleanup --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a19ae1f481..7ed79ffc126 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -285,7 +285,9 @@ jobs: command: pip install --user --progress-bar=off pytest - run: name: Run tests - command: pytest -v --durations=20 test/test_prototype_*.py + command: | + export PYTORCH_TEST_WITH_PROTOTYPE=1 + pytest -v --durations=20 test/test_prototype_*.py - save_cache: key: hub-{{ checksum "prototype_model_urls.txt" }} From 0e76cf9871407946762384a028f1cc7a834af224 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 18:23:58 +0200 Subject: [PATCH 19/41] fix skipping logic --- test/common_utils.py | 4 ++-- test/test_prototype_models.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/common_utils.py b/test/common_utils.py index 79933025d9d..e20e2c658bc 100644 --- a/test/common_utils.py +++ b/test/common_utils.py @@ -215,5 +215,5 @@ def _test_fn_on_batch(batch_tensors, fn, scripted_fn_atol=1e-8, **fn_kwargs): torch.testing.assert_close(transformed_batch, s_transformed_batch, rtol=1e-5, atol=scripted_fn_atol) -def skip_on_env_var(name, *, reason, exist_ok=False, default=False): - return pytest.mark.skipif(get_bool_env_var(name, exist_ok=exist_ok, default=default), reason=reason) +def run_on_env_var(name, *, skip_reason=None, exist_ok=False, default=False): + return pytest.mark.skipif(not get_bool_env_var(name, exist_ok=exist_ok, default=default), reason=skip_reason) diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index 4e3c99c4ea9..23ac5173f24 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -3,11 +3,12 @@ import pytest import test_models as TM import torch -from common_utils import cpu_and_gpu, skip_on_env_var +from common_utils import cpu_and_gpu, run_on_env_var from torchvision.prototype import models -skip_if_not_test_with_prototype = skip_on_env_var( - "PYTORCH_TEST_WITH_PROTOTYPE", reason="Prototype code tests are disabled" +run_if_test_with_prototype = run_on_env_var( + "PYTORCH_TEST_WITH_PROTOTYPE", + skip_reason="Prototype code tests are disabled by default. Set PYTORCH_TEST_WITH_PROTOTYPE=1 to run it.", ) @@ -41,21 +42,21 @@ def test_get_weight(): @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@skip_if_not_test_with_prototype +@run_if_test_with_prototype def test_classification_model(model_fn, dev): TM.test_classification_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models.segmentation)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@skip_if_not_test_with_prototype +@run_if_test_with_prototype def test_segmentation_model(model_fn, dev): TM.test_segmentation_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models.video)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@skip_if_not_test_with_prototype +@run_if_test_with_prototype def test_video_model(model_fn, dev): TM.test_video_model(model_fn, dev) @@ -67,7 +68,7 @@ def test_video_model(model_fn, dev): + get_models_with_module_names(models.video), ) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@skip_if_not_test_with_prototype +@run_if_test_with_prototype def test_old_vs_new_factory(model_fn, module_name, dev): defaults = { "models": { From 7c652b7b5548ad6730d40926927b1c77a7fddee4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 19:49:19 +0200 Subject: [PATCH 20/41] improve step name --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ed79ffc126..d8299c7002f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,7 +272,7 @@ jobs: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . - run: - name: Prepare caching + name: Generate cache key command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - restore_cache: keys: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 61974d3b15b..4a5bfb58d99 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -272,7 +272,7 @@ jobs: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . - run: - name: Prepare caching + name: Generate cache key command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - restore_cache: keys: From a1086b79da29ab34f40861fa41dad90c0514cc5d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Oct 2021 19:50:36 +0200 Subject: [PATCH 21/41] benchmark without caching --- .circleci/config.yml | 30 +++++++++++++++--------------- .circleci/config.yml.in | 30 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8299c7002f..5ae0555638a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,15 +271,15 @@ jobs: - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . - - run: - name: Generate cache key - command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - - restore_cache: - keys: - - - hub-{{ checksum "prototype_model_urls.txt" }} - - - hub- +# - run: +# name: Generate cache key +# command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt +# - restore_cache: +# keys: +# +# - hub-{{ checksum "prototype_model_urls.txt" }} +# +# - hub- - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -288,12 +288,12 @@ jobs: command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 pytest -v --durations=20 test/test_prototype_*.py - - save_cache: - - key: hub-{{ checksum "prototype_model_urls.txt" }} - - paths: - - ~/.cache/torch/hub +# - save_cache: +# +# key: hub-{{ checksum "prototype_model_urls.txt" }} +# +# paths: +# - ~/.cache/torch/hub binary_linux_wheel: <<: *binary_common diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4a5bfb58d99..aeff95df049 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -271,15 +271,15 @@ jobs: - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . - - run: - name: Generate cache key - command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - - restore_cache: - keys: - {% raw %} - - hub-{{ checksum "prototype_model_urls.txt" }} - {% endraw %} - - hub- +# - run: +# name: Generate cache key +# command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt +# - restore_cache: +# keys: +# {% raw %} +# - hub-{{ checksum "prototype_model_urls.txt" }} +# {% endraw %} +# - hub- - run: name: Install test utilities command: pip install --user --progress-bar=off pytest @@ -288,12 +288,12 @@ jobs: command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 pytest -v --durations=20 test/test_prototype_*.py - - save_cache: - {% raw %} - key: hub-{{ checksum "prototype_model_urls.txt" }} - {% endraw %} - paths: - - ~/.cache/torch/hub +# - save_cache: +# {% raw %} +# key: hub-{{ checksum "prototype_model_urls.txt" }} +# {% endraw %} +# paths: +# - ~/.cache/torch/hub binary_linux_wheel: <<: *binary_common From 9d2bb9c171c887e6c6dc169dba0c109c44010664 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 29 Oct 2021 07:36:03 +0200 Subject: [PATCH 22/41] benchmark with external download --- .circleci/config.yml | 12 +++++++++--- .circleci/config.yml.in | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ae0555638a..8d444c0d4ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,9 +271,15 @@ jobs: - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . -# - run: -# name: Generate cache key -# command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt + - run: + name: Generate cache key + command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt + - run: + name: Download model weights + command: | + HUB_DIR="~/.cache/torch/hub" + wget -P $HUB_DIR -i prototype_model_urls.txt + du -hs $HUB_DIR # - restore_cache: # keys: # diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index aeff95df049..55099b98939 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -271,9 +271,15 @@ jobs: - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . -# - run: -# name: Generate cache key -# command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt + - run: + name: Generate cache key + command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt + - run: + name: Download model weights + command: | + HUB_DIR="~/.cache/torch/hub" + wget -P $HUB_DIR -i prototype_model_urls.txt + du -hs $HUB_DIR # - restore_cache: # keys: # {% raw %} From 0a7d1f65008b4116731027f81d63e54c49b891b1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 29 Oct 2021 07:50:12 +0200 Subject: [PATCH 23/41] debug --- .circleci/config.yml | 7 ++----- .circleci/config.yml.in | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d444c0d4ce..8755380e4bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,10 +276,7 @@ jobs: command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights - command: | - HUB_DIR="~/.cache/torch/hub" - wget -P $HUB_DIR -i prototype_model_urls.txt - du -hs $HUB_DIR + command: wget --no-verbose -P ~/.cache/torch/hub -i prototype_model_urls.txt # - restore_cache: # keys: # @@ -293,7 +290,7 @@ jobs: name: Run tests command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 - pytest -v --durations=20 test/test_prototype_*.py + pytest -svvv --durations=20 test/test_prototype_*.py # - save_cache: # # key: hub-{{ checksum "prototype_model_urls.txt" }} diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 55099b98939..a50033b63c1 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -276,10 +276,7 @@ jobs: command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights - command: | - HUB_DIR="~/.cache/torch/hub" - wget -P $HUB_DIR -i prototype_model_urls.txt - du -hs $HUB_DIR + command: wget --no-verbose -P ~/.cache/torch/hub -i prototype_model_urls.txt # - restore_cache: # keys: # {% raw %} @@ -293,7 +290,7 @@ jobs: name: Run tests command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 - pytest -v --durations=20 test/test_prototype_*.py + pytest -svvv --durations=20 test/test_prototype_*.py # - save_cache: # {% raw %} # key: hub-{{ checksum "prototype_model_urls.txt" }} From a6f0eb1618a229a99d1a015b40f6d04fc366d91c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 29 Oct 2021 08:09:25 +0200 Subject: [PATCH 24/41] fix manual download location --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8755380e4bd..6e5aafcf0ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,7 +276,7 @@ jobs: command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights - command: wget --no-verbose -P ~/.cache/torch/hub -i prototype_model_urls.txt + command: wget --no-verbose -P ~/.cache/torch/hub/checkpoints -i prototype_model_urls.txt # - restore_cache: # keys: # @@ -290,7 +290,7 @@ jobs: name: Run tests command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 - pytest -svvv --durations=20 test/test_prototype_*.py + pytest -v --durations=20 test/test_prototype_*.py # - save_cache: # # key: hub-{{ checksum "prototype_model_urls.txt" }} diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a50033b63c1..e18d82ae071 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -276,7 +276,7 @@ jobs: command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights - command: wget --no-verbose -P ~/.cache/torch/hub -i prototype_model_urls.txt + command: wget --no-verbose -P ~/.cache/torch/hub/checkpoints -i prototype_model_urls.txt # - restore_cache: # keys: # {% raw %} @@ -290,7 +290,7 @@ jobs: name: Run tests command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 - pytest -svvv --durations=20 test/test_prototype_*.py + pytest -v --durations=20 test/test_prototype_*.py # - save_cache: # {% raw %} # key: hub-{{ checksum "prototype_model_urls.txt" }} From cfe4a7161a6fb234b5bb6fbbbf31045c4a039056 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 29 Oct 2021 08:24:52 +0200 Subject: [PATCH 25/41] debug again --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e5aafcf0ba..57b871c932e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,7 +290,7 @@ jobs: name: Run tests command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 - pytest -v --durations=20 test/test_prototype_*.py + pytest -sv --durations=20 test/test_prototype_*.py # - save_cache: # # key: hub-{{ checksum "prototype_model_urls.txt" }} diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index e18d82ae071..cd662331ade 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -290,7 +290,7 @@ jobs: name: Run tests command: | export PYTORCH_TEST_WITH_PROTOTYPE=1 - pytest -v --durations=20 test/test_prototype_*.py + pytest -sv --durations=20 test/test_prototype_*.py # - save_cache: # {% raw %} # key: hub-{{ checksum "prototype_model_urls.txt" }} From df2097a877531841bab2b42fb2d93bb824fb971a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 08:53:14 +0100 Subject: [PATCH 26/41] download weights in the background --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 57b871c932e..b48df60efa2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,6 +276,7 @@ jobs: command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights + background: true command: wget --no-verbose -P ~/.cache/torch/hub/checkpoints -i prototype_model_urls.txt # - restore_cache: # keys: From 789446ea92acfe3e2eca26ed9487fc841cbbaebf Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:24:26 +0100 Subject: [PATCH 27/41] try parallel download --- .circleci/config.yml | 8 ++++---- .circleci/config.yml.in | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2adb9d9dea..f7a1a060529 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,13 +277,13 @@ jobs: - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . - - run: - name: Generate cache key - command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights background: true - command: wget --no-verbose -P ~/.cache/torch/hub/checkpoints -i prototype_model_urls.txt + command: | + sudo apt update -qy && sudo apt install -qy parallel + python scripts/collect_model_urls.py torchvision/prototype/models \ + | parallel -j4 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - run: name: Install test requirements command: pip install --user --progress-bar=off pytest pytest-mock scipy iopath diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 6b41e45fd47..2d68f0de1ec 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -277,13 +277,13 @@ jobs: - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . - - run: - name: Generate cache key - command: python scripts/collect_model_urls.py torchvision/prototype/models > prototype_model_urls.txt - run: name: Download model weights background: true - command: wget --no-verbose -P ~/.cache/torch/hub/checkpoints -i prototype_model_urls.txt + command: | + sudo apt update -qy && sudo apt install -qy parallel + python scripts/collect_model_urls.py torchvision/prototype/models \ + | parallel -j4 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - run: name: Install test requirements command: pip install --user --progress-bar=off pytest pytest-mock scipy iopath From 3d1eac32beadb45b8919d78ad92bf83f6511f306 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:25:38 +0100 Subject: [PATCH 28/41] add missing import --- test/test_prototype_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index f89ea9cc599..857b28906e1 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -1,4 +1,5 @@ import importlib +import os import pytest import test_models as TM From ef217038c2f3377595b206e9591c5324da434fee Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:26:35 +0100 Subject: [PATCH 29/41] use correct decoractor --- test/test_prototype_models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index 857b28906e1..4ddef639e98 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -1,5 +1,4 @@ import importlib -import os import pytest import test_models as TM @@ -60,13 +59,13 @@ def test_classification_model(model_fn, dev): @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models.detection)) @pytest.mark.parametrize("dev", cpu_and_gpu()) -@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +@run_if_test_with_prototype def test_detection_model(model_fn, dev): TM.test_detection_model(model_fn, dev) @pytest.mark.parametrize("model_fn", TM.get_models_from_module(models.quantization)) -@pytest.mark.skipif(os.getenv("PYTORCH_TEST_WITH_PROTOTYPE", "0") == "0", reason="Prototype code tests are disabled") +@run_if_test_with_prototype def test_quantized_classification_model(model_fn): TM.test_quantized_classification_model(model_fn) From 037aeb14e44481e1c2e1b23d0d5752164785ad98 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:42:41 +0100 Subject: [PATCH 30/41] up resource_class --- .circleci/config.yml | 10 +++++----- .circleci/config.yml.in | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7a1a060529..29959737b33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -263,7 +263,7 @@ jobs: prototype_test: docker: - image: circleci/python:3.7 - resource_class: large + resource_class: xlarge steps: - run: name: Install torch @@ -273,10 +273,6 @@ jobs: - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - - checkout - - run: - name: Install torchvision - command: pip install --user --progress-bar off --no-build-isolation . - run: name: Download model weights background: true @@ -284,6 +280,10 @@ jobs: sudo apt update -qy && sudo apt install -qy parallel python scripts/collect_model_urls.py torchvision/prototype/models \ | parallel -j4 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + - checkout + - run: + name: Install torchvision + command: pip install --user --progress-bar off --no-build-isolation . - run: name: Install test requirements command: pip install --user --progress-bar=off pytest pytest-mock scipy iopath diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 2d68f0de1ec..48c6e8226fe 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -263,7 +263,7 @@ jobs: prototype_test: docker: - image: circleci/python:3.7 - resource_class: large + resource_class: xlarge steps: - run: name: Install torch @@ -273,10 +273,6 @@ jobs: - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git - - checkout - - run: - name: Install torchvision - command: pip install --user --progress-bar off --no-build-isolation . - run: name: Download model weights background: true @@ -284,6 +280,10 @@ jobs: sudo apt update -qy && sudo apt install -qy parallel python scripts/collect_model_urls.py torchvision/prototype/models \ | parallel -j4 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + - checkout + - run: + name: Install torchvision + command: pip install --user --progress-bar off --no-build-isolation . - run: name: Install test requirements command: pip install --user --progress-bar=off pytest pytest-mock scipy iopath From 70b09b002ea7e6c68d355c55e11c32049d5f6175 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:44:44 +0100 Subject: [PATCH 31/41] fix wording --- test/test_prototype_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index 4ddef639e98..2cb378520a4 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -8,7 +8,7 @@ run_if_test_with_prototype = run_on_env_var( "PYTORCH_TEST_WITH_PROTOTYPE", - skip_reason="Prototype code tests are disabled by default. Set PYTORCH_TEST_WITH_PROTOTYPE=1 to run it.", + skip_reason="Prototype tests are disabled by default. Set PYTORCH_TEST_WITH_PROTOTYPE=1 to run them.", ) From a45f56dcac68e9fd008768f072b6a6a357b0739e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:48:34 +0100 Subject: [PATCH 32/41] enable stdout passthrough to see download during test --- .circleci/config.yml | 6 +++--- .circleci/config.yml.in | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 29959737b33..340a8a35eed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,9 +277,9 @@ jobs: name: Download model weights background: true command: | - sudo apt update -qy && sudo apt install -qy parallel + sudo apt update -qy && sudo apt install -qy parallel wget python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j4 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - checkout - run: name: Install torchvision @@ -291,7 +291,7 @@ jobs: name: Run tests environment: PYTORCH_TEST_WITH_PROTOTYPE: 1 - command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py + command: pytest --junitxml=test-results/junit.xml -vs --durations 20 test/test_prototype_*.py - store_test_results: path: test-results diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 48c6e8226fe..4bfee26c621 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -277,9 +277,9 @@ jobs: name: Download model weights background: true command: | - sudo apt update -qy && sudo apt install -qy parallel + sudo apt update -qy && sudo apt install -qy parallel wget python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j4 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - checkout - run: name: Install torchvision @@ -291,7 +291,7 @@ jobs: name: Run tests environment: PYTORCH_TEST_WITH_PROTOTYPE: 1 - command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py + command: pytest --junitxml=test-results/junit.xml -vs --durations 20 test/test_prototype_*.py - store_test_results: path: test-results From a9eb42507faafa7f06d1d57ad5dc5e3bab83d909 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:50:25 +0100 Subject: [PATCH 33/41] remove linebreak --- .circleci/config.yml | 3 +-- .circleci/config.yml.in | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 340a8a35eed..78af21b44a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -278,8 +278,7 @@ jobs: background: true command: | sudo apt update -qy && sudo apt install -qy parallel wget - python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + python scripts/collect_model_urls.py torchvision/prototype/models | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - checkout - run: name: Install torchvision diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4bfee26c621..51c87bcd1f5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -278,8 +278,7 @@ jobs: background: true command: | sudo apt update -qy && sudo apt install -qy parallel wget - python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + python scripts/collect_model_urls.py torchvision/prototype/models | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - checkout - run: name: Install torchvision From 0c816110624c59a67c5032398b843001d553249c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 09:52:21 +0100 Subject: [PATCH 34/41] move checkout up --- .circleci/config.yml | 5 +++-- .circleci/config.yml.in | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78af21b44a3..84452eae312 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -273,13 +273,14 @@ jobs: - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git + - checkout - run: name: Download model weights background: true command: | sudo apt update -qy && sudo apt install -qy parallel wget - python scripts/collect_model_urls.py torchvision/prototype/models | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - - checkout + python scripts/collect_model_urls.py torchvision/prototype/models \ + | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 51c87bcd1f5..ce17523743d 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -273,13 +273,14 @@ jobs: - run: name: Install prototype dependencies command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git + - checkout - run: name: Download model weights background: true command: | sudo apt update -qy && sudo apt install -qy parallel wget - python scripts/collect_model_urls.py torchvision/prototype/models | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - - checkout + python scripts/collect_model_urls.py torchvision/prototype/models \ + | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . From 8b1ebafae927377cf9d0d26532c3e5dfb95e1410 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 10:05:18 +0100 Subject: [PATCH 35/41] cleanup --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84452eae312..e84de857834 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -280,7 +280,7 @@ jobs: command: | sudo apt update -qy && sudo apt install -qy parallel wget python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + | parallel -j0 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . @@ -291,7 +291,7 @@ jobs: name: Run tests environment: PYTORCH_TEST_WITH_PROTOTYPE: 1 - command: pytest --junitxml=test-results/junit.xml -vs --durations 20 test/test_prototype_*.py + command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py - store_test_results: path: test-results diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index ce17523743d..088628a3fbf 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -280,7 +280,7 @@ jobs: command: | sudo apt update -qy && sudo apt install -qy parallel wget python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j8 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} + | parallel -j0 wget --no-verbose -P ~/.cache/torch/hub/checkpoints {} - run: name: Install torchvision command: pip install --user --progress-bar off --no-build-isolation . @@ -291,7 +291,7 @@ jobs: name: Run tests environment: PYTORCH_TEST_WITH_PROTOTYPE: 1 - command: pytest --junitxml=test-results/junit.xml -vs --durations 20 test/test_prototype_*.py + command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py - store_test_results: path: test-results From 133933ec0e9b81cb42313186fbcbfed77f2dc2e0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 11:03:48 +0100 Subject: [PATCH 36/41] debug failing test --- .circleci/config.yml | 5 ++++- test/test_prototype_models.py | 2 ++ torchvision/models/quantization/mobilenetv3.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e84de857834..e9c0da387fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -291,7 +291,10 @@ jobs: name: Run tests environment: PYTORCH_TEST_WITH_PROTOTYPE: 1 - command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py + command: > + pytest --junitxml=test-results/junit.xml -v --durations 20 -rP + test/test_prototype_models.py::test_old_vs_new_factory + -k "cpu and mobilenet_v3_large and quantization" - store_test_results: path: test-results diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index 2cb378520a4..ae21b7a7ef6 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -126,6 +126,8 @@ def test_old_vs_new_factory(model_fn, module_name, dev): model_new = _build_model(model_fn, **kwargs).to(device=dev) torch.testing.assert_close(model_new(x), model_old(x), rtol=0.0, atol=0.0, check_dtype=False) + assert False + def test_smoke(): import torchvision.prototype.models # noqa: F401 diff --git a/torchvision/models/quantization/mobilenetv3.py b/torchvision/models/quantization/mobilenetv3.py index 6eaa9114d74..ac273176764 100644 --- a/torchvision/models/quantization/mobilenetv3.py +++ b/torchvision/models/quantization/mobilenetv3.py @@ -41,9 +41,18 @@ def _load_from_state_dict( unexpected_keys, error_msgs, ): + print("=" * 80) + print(f"# {prefix}") + print("=" * 80) + print(local_metadata) + version = local_metadata.get("version", None) + has_qconfig = hasattr(self, "qconfig") + print(f"has_qconfig={has_qconfig}") if hasattr(self, "qconfig") and (version is None or version < 2): + print("-" * 80) + print("# fixing state dict") default_state_dict = { "scale_activation.activation_post_process.scale": torch.tensor([1.0]), "scale_activation.activation_post_process.zero_point": torch.tensor([0], dtype=torch.int32), @@ -53,6 +62,7 @@ def _load_from_state_dict( for k, v in default_state_dict.items(): full_key = prefix + k if full_key not in state_dict: + print(full_key) state_dict[full_key] = v super()._load_from_state_dict( From 60b2716a99d45b7388cc94b99b26e5081ae5b941 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 11:48:53 +0100 Subject: [PATCH 37/41] temp fix --- torchvision/models/quantization/mobilenetv3.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/torchvision/models/quantization/mobilenetv3.py b/torchvision/models/quantization/mobilenetv3.py index ac273176764..33f4a155be3 100644 --- a/torchvision/models/quantization/mobilenetv3.py +++ b/torchvision/models/quantization/mobilenetv3.py @@ -41,21 +41,16 @@ def _load_from_state_dict( unexpected_keys, error_msgs, ): - print("=" * 80) - print(f"# {prefix}") - print("=" * 80) - print(local_metadata) - version = local_metadata.get("version", None) - has_qconfig = hasattr(self, "qconfig") - print(f"has_qconfig={has_qconfig}") if hasattr(self, "qconfig") and (version is None or version < 2): - print("-" * 80) - print("# fixing state dict") default_state_dict = { "scale_activation.activation_post_process.scale": torch.tensor([1.0]), + "scale_activation.activation_post_process.activation_post_process.scale": torch.tensor([1.0]), "scale_activation.activation_post_process.zero_point": torch.tensor([0], dtype=torch.int32), + "scale_activation.activation_post_process.activation_post_process.zero_point": torch.tensor( + [0], dtype=torch.int32 + ), "scale_activation.activation_post_process.fake_quant_enabled": torch.tensor([1]), "scale_activation.activation_post_process.observer_enabled": torch.tensor([1]), } From 0a247d4a47915ceb500766a61541d47bd457da5c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 11:56:51 +0100 Subject: [PATCH 38/41] fix --- test/test_prototype_models.py | 2 -- torchvision/models/quantization/mobilenetv3.py | 1 - 2 files changed, 3 deletions(-) diff --git a/test/test_prototype_models.py b/test/test_prototype_models.py index ae21b7a7ef6..2cb378520a4 100644 --- a/test/test_prototype_models.py +++ b/test/test_prototype_models.py @@ -126,8 +126,6 @@ def test_old_vs_new_factory(model_fn, module_name, dev): model_new = _build_model(model_fn, **kwargs).to(device=dev) torch.testing.assert_close(model_new(x), model_old(x), rtol=0.0, atol=0.0, check_dtype=False) - assert False - def test_smoke(): import torchvision.prototype.models # noqa: F401 diff --git a/torchvision/models/quantization/mobilenetv3.py b/torchvision/models/quantization/mobilenetv3.py index 33f4a155be3..c831a443d78 100644 --- a/torchvision/models/quantization/mobilenetv3.py +++ b/torchvision/models/quantization/mobilenetv3.py @@ -57,7 +57,6 @@ def _load_from_state_dict( for k, v in default_state_dict.items(): full_key = prefix + k if full_key not in state_dict: - print(full_key) state_dict[full_key] = v super()._load_from_state_dict( From 7ed8fe3ee85421566e5e5c913cdd9f488bee5489 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 12:12:23 +0100 Subject: [PATCH 39/41] cleanup --- .circleci/config.yml | 5 +---- torchvision/models/quantization/mobilenetv3.py | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9c0da387fc..e84de857834 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -291,10 +291,7 @@ jobs: name: Run tests environment: PYTORCH_TEST_WITH_PROTOTYPE: 1 - command: > - pytest --junitxml=test-results/junit.xml -v --durations 20 -rP - test/test_prototype_models.py::test_old_vs_new_factory - -k "cpu and mobilenet_v3_large and quantization" + command: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py - store_test_results: path: test-results diff --git a/torchvision/models/quantization/mobilenetv3.py b/torchvision/models/quantization/mobilenetv3.py index c831a443d78..6eaa9114d74 100644 --- a/torchvision/models/quantization/mobilenetv3.py +++ b/torchvision/models/quantization/mobilenetv3.py @@ -46,11 +46,7 @@ def _load_from_state_dict( if hasattr(self, "qconfig") and (version is None or version < 2): default_state_dict = { "scale_activation.activation_post_process.scale": torch.tensor([1.0]), - "scale_activation.activation_post_process.activation_post_process.scale": torch.tensor([1.0]), "scale_activation.activation_post_process.zero_point": torch.tensor([0], dtype=torch.int32), - "scale_activation.activation_post_process.activation_post_process.zero_point": torch.tensor( - [0], dtype=torch.int32 - ), "scale_activation.activation_post_process.fake_quant_enabled": torch.tensor([1]), "scale_activation.activation_post_process.observer_enabled": torch.tensor([1]), } From a2f975852cb1246dad447ba59e39da947b07b23d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 13:57:03 +0100 Subject: [PATCH 40/41] fix regex --- scripts/collect_model_urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_model_urls.py b/scripts/collect_model_urls.py index 4b3fb1673f5..3554e80b1ed 100644 --- a/scripts/collect_model_urls.py +++ b/scripts/collect_model_urls.py @@ -2,7 +2,7 @@ import re import sys -MODEL_URL_PATTERN = re.compile(r"https://download.pytorch.org/models/.*?[.]pth") +MODEL_URL_PATTERN = re.compile(r"https://download[.]pytorch[.]org/models/.*?[.]pth") def main(root): From 31f26a68342a16ef60c146f34ac410430b4569f0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 26 Nov 2021 14:14:00 +0100 Subject: [PATCH 41/41] remove explicit install of numpy --- .circleci/config.yml | 1 - .circleci/config.yml.in | 1 - 2 files changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e84de857834..a3bacfbe969 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -268,7 +268,6 @@ jobs: - run: name: Install torch command: | - pip install --user --progress-bar=off numpy pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - run: name: Install prototype dependencies diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 088628a3fbf..6c6ad438929 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -268,7 +268,6 @@ jobs: - run: name: Install torch command: | - pip install --user --progress-bar=off numpy pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - run: name: Install prototype dependencies