From 39d04482f208c79fb70f50e1118549b44aacb25a Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 12:48:53 +0200 Subject: [PATCH 01/11] add versions --- tests/legacy/back-compatible-versions.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/legacy/back-compatible-versions.txt b/tests/legacy/back-compatible-versions.txt index c2bc418eecd93..e68c4e29b83a7 100644 --- a/tests/legacy/back-compatible-versions.txt +++ b/tests/legacy/back-compatible-versions.txt @@ -77,3 +77,12 @@ 1.8.4 1.8.5 1.8.6 +1.9.0 +1.9.1 +1.9.2 +1.9.3 +1.9.4 +1.9.5 +2.0.0 +2.0.1 +2.0.2 From 5da97379b86cdda639a6aa8170889ed1f3a4e89a Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 15:20:58 +0200 Subject: [PATCH 02/11] make local --- .actions/pull_legacy_checkpoints.sh | 2 ++ .azure/gpu-tests-pytorch.yml | 9 ++++++--- .github/workflows/ci-tests-pytorch.yml | 11 +++++++---- .../checkpointing/test_legacy_checkpoints.py | 2 ++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.actions/pull_legacy_checkpoints.sh b/.actions/pull_legacy_checkpoints.sh index cf8a0fdf4e574..b61647a96e50c 100644 --- a/.actions/pull_legacy_checkpoints.sh +++ b/.actions/pull_legacy_checkpoints.sh @@ -1,9 +1,11 @@ #!/bin/bash + # Run this script from the project root. URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip" mkdir -p tests/legacy # wget is simpler but does not work on Windows python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'tests/legacy/checkpoints.zip')" ls -l tests/legacy/ + unzip -o tests/legacy/checkpoints.zip -d tests/legacy/ ls -l tests/legacy/checkpoints/ diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 4a381ffc16372..35c2d29dabbe5 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -123,13 +123,16 @@ jobs: python requirements/pytorch/check-avail-extras.py displayName: 'Env details' - - bash: bash .actions/pull_legacy_checkpoints.sh - displayName: 'Get legacy checkpoints' - - bash: python -m pytest pytorch_lightning workingDirectory: src displayName: 'Testing: PyTorch doctests' + - bash: | + bash tests/legacy/generate_checkpoints.sh + mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ + bash .actions/pull_legacy_checkpoints.sh + displayName: 'Get legacy checkpoints' + - bash: python -m coverage run --source pytorch_lightning -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 env: PL_RUN_CUDA_TESTS: "1" diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index f0205930f0552..f92301a6fe20d 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -98,10 +98,6 @@ jobs: if: ${{ matrix.requires == 'oldest' }} run: python .actions/assistant.py replace_oldest_ver - - name: Pull legacy checkpoints - if: ${{ matrix.type != 'slow' }} - run: bash .actions/pull_legacy_checkpoints.sh - - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | @@ -161,6 +157,13 @@ jobs: --source_import="pytorch_lightning,lightning_fabric" \ --target_import="lightning.pytorch,lightning.fabric" + - name: Pull legacy checkpoints + if: ${{ matrix.type != 'slow' }} + run: | + bash tests/legacy/generate_checkpoints.sh + mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ + bash .actions/pull_legacy_checkpoints.sh + - name: Testing Warnings # the stacklevel can only be set on >=3.7 if: matrix.python-version != '3.7' diff --git a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py index f00ca383d3149..57f537acfa26d 100644 --- a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py +++ b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py @@ -32,6 +32,8 @@ # load list of all back compatible versions with open(os.path.join(_PATH_LEGACY, "back-compatible-versions.txt")) as fp: LEGACY_BACK_COMPATIBLE_PL_VERSIONS = [ln.strip() for ln in fp.readlines()] +# This shall be created for each CI run +LEGACY_BACK_COMPATIBLE_PL_VERSIONS += ['local'] @pytest.mark.parametrize("pl_version", LEGACY_BACK_COMPATIBLE_PL_VERSIONS) From 18ab39d6917d2a95c51ba5994c82d5d747353137 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 May 2023 13:22:11 +0000 Subject: [PATCH 03/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py index 57f537acfa26d..f4781b66fd2e6 100644 --- a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py +++ b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py @@ -33,7 +33,7 @@ with open(os.path.join(_PATH_LEGACY, "back-compatible-versions.txt")) as fp: LEGACY_BACK_COMPATIBLE_PL_VERSIONS = [ln.strip() for ln in fp.readlines()] # This shall be created for each CI run -LEGACY_BACK_COMPATIBLE_PL_VERSIONS += ['local'] +LEGACY_BACK_COMPATIBLE_PL_VERSIONS += ["local"] @pytest.mark.parametrize("pl_version", LEGACY_BACK_COMPATIBLE_PL_VERSIONS) From d9869396746097326b7247bb8b56e49b40105c0e Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 15:52:13 +0200 Subject: [PATCH 04/11] imports --- tests/legacy/simple_classif_training.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 00e51627efad9..7f8fdfe58e05f 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -15,9 +15,9 @@ import torch -import pytorch_lightning as pl -from pytorch_lightning import seed_everything -from pytorch_lightning.callbacks import EarlyStopping +import lightning.pytorch as pl +from lightning.pytorch import seed_everything +from lightning.pytorch.callbacks import EarlyStopping from tests_pytorch.helpers.datamodules import ClassifDataModule from tests_pytorch.helpers.simple_models import ClassificationModel From 34658007e2f51b1b3ccc66fd84a0cc1cdde0dbae Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 16:19:44 +0200 Subject: [PATCH 05/11] local --- .azure/gpu-tests-pytorch.yml | 1 - .github/workflows/ci-tests-pytorch.yml | 1 - tests/legacy/generate_checkpoints.sh | 20 +++++++++----------- tests/legacy/requirements.txt | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 35c2d29dabbe5..4d1d3f5be877f 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -129,7 +129,6 @@ jobs: - bash: | bash tests/legacy/generate_checkpoints.sh - mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ bash .actions/pull_legacy_checkpoints.sh displayName: 'Get legacy checkpoints' diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index f92301a6fe20d..57a8fb2695cf7 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -161,7 +161,6 @@ jobs: if: ${{ matrix.type != 'slow' }} run: | bash tests/legacy/generate_checkpoints.sh - mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ bash .actions/pull_legacy_checkpoints.sh - name: Testing Warnings diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 7873daf3fe56f..08e9b7a78e19d 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -7,11 +7,12 @@ set -e LEGACY_PATH=$(cd $(dirname $0); pwd -P) -ENV_PATH=$LEGACY_PATH/vEnv +ENV_PATH="$LEGACY_PATH/vEnv" export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` -echo LEGACY_PATH: $LEGACY_PATH -echo ENV_PATH: $ENV_PATH -echo PYTHONPATH: $PYTHONPATH +print "LEGACY_PATH: $LEGACY_PATH" +print "ENV_PATH: $ENV_PATH" +print "PYTHONPATH: $PYTHONPATH" +rm -rf $ENV_PATH function create_and_save_checkpoint { python --version @@ -28,11 +29,9 @@ function create_and_save_checkpoint { # iterate over all arguments assuming that each argument is version for pl_ver in "$@" do - echo processing version: $pl_ver + print "processing version: $pl_ver" - # Don't install/update anything before activating venv - # to avoid breaking any existing environment. - rm -rf $ENV_PATH + # Don't install/update anything before activating venv to avoid breaking any existing environment. python -m venv $ENV_PATH source $ENV_PATH/bin/activate @@ -47,10 +46,9 @@ done # use the PL installed in the environment if no PL version is specified if [[ -z "$@" ]]; then - pl_ver=$(python -c "import pytorch_lightning as pl; print(pl.__version__)") - echo processing version: $pl_ver + print "processing local version" python -m pip install -r $LEGACY_PATH/requirements.txt - + pl_ver="local" create_and_save_checkpoint fi diff --git a/tests/legacy/requirements.txt b/tests/legacy/requirements.txt index fd2281e591624..43cd22c429f8b 100644 --- a/tests/legacy/requirements.txt +++ b/tests/legacy/requirements.txt @@ -1,2 +1,2 @@ -torchmetrics # necessary because old PL verions don't have it as dependency +torchmetrics # necessary because old PL versions don't have it as dependency scikit-learn From 294b80b9953db9d2fd2d09b8cbdf1b117e7fda5f Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 16:27:29 +0200 Subject: [PATCH 06/11] et --- .azure/gpu-tests-pytorch.yml | 4 +++- .github/workflows/ci-tests-pytorch.yml | 6 ++++-- tests/legacy/generate_checkpoints.sh | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 4d1d3f5be877f..043909d793efd 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -128,8 +128,10 @@ jobs: displayName: 'Testing: PyTorch doctests' - bash: | - bash tests/legacy/generate_checkpoints.sh bash .actions/pull_legacy_checkpoints.sh + cd tests/legacy + bash generate_checkpoints.sh + ls -l checkpoints/ displayName: 'Get legacy checkpoints' - bash: python -m coverage run --source pytorch_lightning -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 57a8fb2695cf7..86dd0009aad89 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -157,11 +157,13 @@ jobs: --source_import="pytorch_lightning,lightning_fabric" \ --target_import="lightning.pytorch,lightning.fabric" - - name: Pull legacy checkpoints + - name: Get legacy checkpoints if: ${{ matrix.type != 'slow' }} run: | - bash tests/legacy/generate_checkpoints.sh bash .actions/pull_legacy_checkpoints.sh + cd tests/legacy + bash generate_checkpoints.sh + ls -l checkpoints/ - name: Testing Warnings # the stacklevel can only be set on >=3.7 diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 08e9b7a78e19d..da9269ef6ea39 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -7,7 +7,7 @@ set -e LEGACY_PATH=$(cd $(dirname $0); pwd -P) -ENV_PATH="$LEGACY_PATH/vEnv" +ENV_PATH=$LEGACY_PATH/vEnv export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` print "LEGACY_PATH: $LEGACY_PATH" print "ENV_PATH: $ENV_PATH" From c11a25cd4e2ca059f0078e4623ddd920279caaee Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 17:06:31 +0200 Subject: [PATCH 07/11] printf --- tests/legacy/generate_checkpoints.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index da9269ef6ea39..83a72bf82672e 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -9,9 +9,9 @@ set -e LEGACY_PATH=$(cd $(dirname $0); pwd -P) ENV_PATH=$LEGACY_PATH/vEnv export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` -print "LEGACY_PATH: $LEGACY_PATH" -print "ENV_PATH: $ENV_PATH" -print "PYTHONPATH: $PYTHONPATH" +printf "LEGACY_PATH: $LEGACY_PATH" +printf "ENV_PATH: $ENV_PATH" +printf "PYTHONPATH: $PYTHONPATH" rm -rf $ENV_PATH function create_and_save_checkpoint { @@ -29,7 +29,7 @@ function create_and_save_checkpoint { # iterate over all arguments assuming that each argument is version for pl_ver in "$@" do - print "processing version: $pl_ver" + printf "processing version: $pl_ver" # Don't install/update anything before activating venv to avoid breaking any existing environment. python -m venv $ENV_PATH @@ -46,7 +46,7 @@ done # use the PL installed in the environment if no PL version is specified if [[ -z "$@" ]]; then - print "processing local version" + printf "processing local version" python -m pip install -r $LEGACY_PATH/requirements.txt pl_ver="local" From 8c571d1ef9ddb5fb452bb9e8285cc6b1f855fd0f Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 17:21:26 +0200 Subject: [PATCH 08/11] argv --- tests/legacy/generate_checkpoints.sh | 2 +- tests/legacy/simple_classif_training.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 83a72bf82672e..3091d7948fca8 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -19,7 +19,7 @@ function create_and_save_checkpoint { python -m pip --version python -m pip list - python $LEGACY_PATH/simple_classif_training.py + python $LEGACY_PATH/simple_classif_training.py $pl_ver cp $LEGACY_PATH/simple_classif_training.py $LEGACY_PATH/checkpoints/$pl_ver mv $LEGACY_PATH/checkpoints/$pl_ver/lightning_logs/version_0/checkpoints/*.ckpt $LEGACY_PATH/checkpoints/$pl_ver/ diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 7f8fdfe58e05f..2732255a9a895 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import sys import torch @@ -50,5 +51,6 @@ def main_train(dir_path, max_epochs: int = 20): if __name__ == "__main__": - path_dir = os.path.join(PATH_LEGACY, "checkpoints", str(pl.__version__)) + name = sys.argv[1] if len(sys.argv) > 1 else str(pl.__version__) + path_dir = os.path.join(PATH_LEGACY, "checkpoints", name) main_train(path_dir) From 9a36e8cb448b155c248be468a1e81e2c57357918 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 17:34:20 +0200 Subject: [PATCH 09/11] local --- tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py b/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py index 07a79dccd975b..3bfdc4b339c5a 100644 --- a/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py +++ b/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py @@ -58,7 +58,7 @@ def test_imports_unified(pl_version: str): path_ckpt = path_ckpts[-1] # only below version 1.5.0 we pickled stuff in checkpoints - if Version(pl_version) < Version("1.5.0"): + if pl_version != "local" and Version(pl_version) < Version("1.5.0"): context = pytest.warns(UserWarning, match="Redirecting import of") else: context = no_warning_call(match="Redirecting import of*") From dfee809c3f31704dfa3b844fd8de01264282117c Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Thu, 4 May 2023 17:43:37 +0200 Subject: [PATCH 10/11] Apply suggestions from code review --- tests/legacy/back-compatible-versions.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/legacy/back-compatible-versions.txt b/tests/legacy/back-compatible-versions.txt index e68c4e29b83a7..4dc21c75b39c0 100644 --- a/tests/legacy/back-compatible-versions.txt +++ b/tests/legacy/back-compatible-versions.txt @@ -83,6 +83,3 @@ 1.9.3 1.9.4 1.9.5 -2.0.0 -2.0.1 -2.0.2 From 6f229f3730b8aff8c32cf53313bbba5cfc005477 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Thu, 4 May 2023 17:44:23 +0200 Subject: [PATCH 11/11] Apply suggestions from code review --- tests/legacy/simple_classif_training.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 2732255a9a895..b4116ee6c5444 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -16,9 +16,9 @@ import torch -import lightning.pytorch as pl -from lightning.pytorch import seed_everything -from lightning.pytorch.callbacks import EarlyStopping +import pytorch_lightning as pl +from pytorch_lightning import seed_everything +from pytorch_lightning.callbacks import EarlyStopping from tests_pytorch.helpers.datamodules import ClassifDataModule from tests_pytorch.helpers.simple_models import ClassificationModel