From df02054eca517ddcf9c447af26b4c2172edb347b Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 12:48:53 +0200 Subject: [PATCH 1/9] add versions --- tests/legacy/back-compatible-versions.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/legacy/back-compatible-versions.txt b/tests/legacy/back-compatible-versions.txt index c2bc418eecd93..e68c4e29b83a7 100644 --- a/tests/legacy/back-compatible-versions.txt +++ b/tests/legacy/back-compatible-versions.txt @@ -77,3 +77,12 @@ 1.8.4 1.8.5 1.8.6 +1.9.0 +1.9.1 +1.9.2 +1.9.3 +1.9.4 +1.9.5 +2.0.0 +2.0.1 +2.0.2 From 386d9aa7b38492c4ab50887082405e663dcd7e52 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 15:20:58 +0200 Subject: [PATCH 2/9] make local --- .actions/pull_legacy_checkpoints.sh | 2 ++ .azure/gpu-tests-pytorch.yml | 9 ++++++--- .github/workflows/_legacy-checkpoints.yml | 1 + .github/workflows/ci-tests-pytorch.yml | 9 ++++++--- .../checkpointing/test_legacy_checkpoints.py | 2 ++ 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.actions/pull_legacy_checkpoints.sh b/.actions/pull_legacy_checkpoints.sh index cf8a0fdf4e574..b61647a96e50c 100644 --- a/.actions/pull_legacy_checkpoints.sh +++ b/.actions/pull_legacy_checkpoints.sh @@ -1,9 +1,11 @@ #!/bin/bash + # Run this script from the project root. URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip" mkdir -p tests/legacy # wget is simpler but does not work on Windows python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'tests/legacy/checkpoints.zip')" ls -l tests/legacy/ + unzip -o tests/legacy/checkpoints.zip -d tests/legacy/ ls -l tests/legacy/checkpoints/ diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 5df5dad4b4179..47eaab9f72294 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -120,9 +120,6 @@ jobs: python requirements/pytorch/check-avail-extras.py displayName: 'Env details' - - bash: bash .actions/pull_legacy_checkpoints.sh - displayName: 'Get legacy checkpoints' - - bash: python -m pytest pytorch_lightning workingDirectory: src condition: eq(variables['PACKAGE_NAME'], 'pytorch') @@ -138,6 +135,12 @@ jobs: condition: eq(variables['PACKAGE_NAME'], 'pytorch') displayName: 'Adjust tests & examples' + - bash: | + bash tests/legacy/generate_checkpoints.sh + mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ + bash .actions/pull_legacy_checkpoints.sh + displayName: 'Get legacy checkpoints' + - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest --ignore benchmarks -v --durations=50 workingDirectory: tests/tests_pytorch env: diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 649c3f590a208..93d06894ecbde 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -131,5 +131,6 @@ jobs: title: Adding test for legacy checkpoint created with ${{ needs.create-legacy-ckpts.outputs.pl-version }} delete-branch: true labels: | + checkpointing tests pl diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 923b9a27c6080..bef41dafe59d3 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -85,9 +85,6 @@ jobs: if: ${{ matrix.requires == 'oldest' }} run: python .actions/assistant.py replace_oldest_ver - - name: Pull legacy checkpoints - run: bash .actions/pull_legacy_checkpoints.sh - - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }} run: | @@ -154,6 +151,12 @@ jobs: - name: Prevent using raw source run: rm -rf src/ + - name: Pull legacy checkpoints + run: | + bash tests/legacy/generate_checkpoints.sh + mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ + bash .actions/pull_legacy_checkpoints.sh + - name: Testing Warnings working-directory: tests/tests_pytorch # needs to run outside of `pytest` diff --git a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py index 86dd5c6cfe9b7..f652dc66bad6f 100644 --- a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py +++ b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py @@ -32,6 +32,8 @@ # load list of all back compatible versions with open(os.path.join(_PATH_LEGACY, "back-compatible-versions.txt")) as fp: LEGACY_BACK_COMPATIBLE_PL_VERSIONS = [ln.strip() for ln in fp.readlines()] +# This shall be created for each CI run +LEGACY_BACK_COMPATIBLE_PL_VERSIONS += ['local'] @pytest.mark.parametrize("pl_version", LEGACY_BACK_COMPATIBLE_PL_VERSIONS) From 852cfc89e3be1d9f6c702e98527e3b3884c2ca19 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 May 2023 13:22:11 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py index f652dc66bad6f..a171e92e8bd59 100644 --- a/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py +++ b/tests/tests_pytorch/checkpointing/test_legacy_checkpoints.py @@ -33,7 +33,7 @@ with open(os.path.join(_PATH_LEGACY, "back-compatible-versions.txt")) as fp: LEGACY_BACK_COMPATIBLE_PL_VERSIONS = [ln.strip() for ln in fp.readlines()] # This shall be created for each CI run -LEGACY_BACK_COMPATIBLE_PL_VERSIONS += ['local'] +LEGACY_BACK_COMPATIBLE_PL_VERSIONS += ["local"] @pytest.mark.parametrize("pl_version", LEGACY_BACK_COMPATIBLE_PL_VERSIONS) From 25c957537e2c17e8d894c125961dc052ce3f104a Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 15:52:13 +0200 Subject: [PATCH 4/9] imports --- tests/legacy/simple_classif_training.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 00e51627efad9..7f8fdfe58e05f 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -15,9 +15,9 @@ import torch -import pytorch_lightning as pl -from pytorch_lightning import seed_everything -from pytorch_lightning.callbacks import EarlyStopping +import lightning.pytorch as pl +from lightning.pytorch import seed_everything +from lightning.pytorch.callbacks import EarlyStopping from tests_pytorch.helpers.datamodules import ClassifDataModule from tests_pytorch.helpers.simple_models import ClassificationModel From 38ee194412d364405ea7c66b4607d03606926a8a Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 16:19:44 +0200 Subject: [PATCH 5/9] local --- .azure/gpu-tests-pytorch.yml | 1 - .github/workflows/ci-tests-pytorch.yml | 1 - tests/legacy/generate_checkpoints.sh | 20 +++++++++----------- tests/legacy/requirements.txt | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 47eaab9f72294..5eefee3f9af01 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -137,7 +137,6 @@ jobs: - bash: | bash tests/legacy/generate_checkpoints.sh - mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ bash .actions/pull_legacy_checkpoints.sh displayName: 'Get legacy checkpoints' diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index bef41dafe59d3..31acf46fdd401 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -154,7 +154,6 @@ jobs: - name: Pull legacy checkpoints run: | bash tests/legacy/generate_checkpoints.sh - mv tests/legacy/checkpoints/*/ tests/legacy/checkpoints/local/ bash .actions/pull_legacy_checkpoints.sh - name: Testing Warnings diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 7873daf3fe56f..08e9b7a78e19d 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -7,11 +7,12 @@ set -e LEGACY_PATH=$(cd $(dirname $0); pwd -P) -ENV_PATH=$LEGACY_PATH/vEnv +ENV_PATH="$LEGACY_PATH/vEnv" export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` -echo LEGACY_PATH: $LEGACY_PATH -echo ENV_PATH: $ENV_PATH -echo PYTHONPATH: $PYTHONPATH +print "LEGACY_PATH: $LEGACY_PATH" +print "ENV_PATH: $ENV_PATH" +print "PYTHONPATH: $PYTHONPATH" +rm -rf $ENV_PATH function create_and_save_checkpoint { python --version @@ -28,11 +29,9 @@ function create_and_save_checkpoint { # iterate over all arguments assuming that each argument is version for pl_ver in "$@" do - echo processing version: $pl_ver + print "processing version: $pl_ver" - # Don't install/update anything before activating venv - # to avoid breaking any existing environment. - rm -rf $ENV_PATH + # Don't install/update anything before activating venv to avoid breaking any existing environment. python -m venv $ENV_PATH source $ENV_PATH/bin/activate @@ -47,10 +46,9 @@ done # use the PL installed in the environment if no PL version is specified if [[ -z "$@" ]]; then - pl_ver=$(python -c "import pytorch_lightning as pl; print(pl.__version__)") - echo processing version: $pl_ver + print "processing local version" python -m pip install -r $LEGACY_PATH/requirements.txt - + pl_ver="local" create_and_save_checkpoint fi diff --git a/tests/legacy/requirements.txt b/tests/legacy/requirements.txt index fd2281e591624..43cd22c429f8b 100644 --- a/tests/legacy/requirements.txt +++ b/tests/legacy/requirements.txt @@ -1,2 +1,2 @@ -torchmetrics # necessary because old PL verions don't have it as dependency +torchmetrics # necessary because old PL versions don't have it as dependency scikit-learn From 480369243292e59b69f3caad723ebf57c464c2f1 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 16:27:29 +0200 Subject: [PATCH 6/9] et --- .azure/gpu-tests-pytorch.yml | 4 +++- .github/workflows/ci-tests-pytorch.yml | 6 ++++-- tests/legacy/generate_checkpoints.sh | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 5eefee3f9af01..07d3fd61c2da5 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -136,8 +136,10 @@ jobs: displayName: 'Adjust tests & examples' - bash: | - bash tests/legacy/generate_checkpoints.sh bash .actions/pull_legacy_checkpoints.sh + cd tests/legacy + bash generate_checkpoints.sh + ls -l checkpoints/ displayName: 'Get legacy checkpoints' - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest --ignore benchmarks -v --durations=50 diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 31acf46fdd401..5bf4307cdf764 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -151,10 +151,12 @@ jobs: - name: Prevent using raw source run: rm -rf src/ - - name: Pull legacy checkpoints + - name: Get legacy checkpoints run: | - bash tests/legacy/generate_checkpoints.sh bash .actions/pull_legacy_checkpoints.sh + cd tests/legacy + bash generate_checkpoints.sh + ls -l checkpoints/ - name: Testing Warnings working-directory: tests/tests_pytorch diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 08e9b7a78e19d..da9269ef6ea39 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -7,7 +7,7 @@ set -e LEGACY_PATH=$(cd $(dirname $0); pwd -P) -ENV_PATH="$LEGACY_PATH/vEnv" +ENV_PATH=$LEGACY_PATH/vEnv export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` print "LEGACY_PATH: $LEGACY_PATH" print "ENV_PATH: $ENV_PATH" From 746412dd9df6c8a9428506689524e634884dab63 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 17:06:31 +0200 Subject: [PATCH 7/9] printf --- tests/legacy/generate_checkpoints.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index da9269ef6ea39..83a72bf82672e 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -9,9 +9,9 @@ set -e LEGACY_PATH=$(cd $(dirname $0); pwd -P) ENV_PATH=$LEGACY_PATH/vEnv export PYTHONPATH=$(dirname $LEGACY_PATH) # for `import tests_pytorch` -print "LEGACY_PATH: $LEGACY_PATH" -print "ENV_PATH: $ENV_PATH" -print "PYTHONPATH: $PYTHONPATH" +printf "LEGACY_PATH: $LEGACY_PATH" +printf "ENV_PATH: $ENV_PATH" +printf "PYTHONPATH: $PYTHONPATH" rm -rf $ENV_PATH function create_and_save_checkpoint { @@ -29,7 +29,7 @@ function create_and_save_checkpoint { # iterate over all arguments assuming that each argument is version for pl_ver in "$@" do - print "processing version: $pl_ver" + printf "processing version: $pl_ver" # Don't install/update anything before activating venv to avoid breaking any existing environment. python -m venv $ENV_PATH @@ -46,7 +46,7 @@ done # use the PL installed in the environment if no PL version is specified if [[ -z "$@" ]]; then - print "processing local version" + printf "processing local version" python -m pip install -r $LEGACY_PATH/requirements.txt pl_ver="local" From 35d6760ea93216babfe781c53f71ee838c806e7c Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 17:21:26 +0200 Subject: [PATCH 8/9] argv --- tests/legacy/generate_checkpoints.sh | 2 +- tests/legacy/simple_classif_training.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 83a72bf82672e..3091d7948fca8 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -19,7 +19,7 @@ function create_and_save_checkpoint { python -m pip --version python -m pip list - python $LEGACY_PATH/simple_classif_training.py + python $LEGACY_PATH/simple_classif_training.py $pl_ver cp $LEGACY_PATH/simple_classif_training.py $LEGACY_PATH/checkpoints/$pl_ver mv $LEGACY_PATH/checkpoints/$pl_ver/lightning_logs/version_0/checkpoints/*.ckpt $LEGACY_PATH/checkpoints/$pl_ver/ diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 7f8fdfe58e05f..2732255a9a895 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import sys import torch @@ -50,5 +51,6 @@ def main_train(dir_path, max_epochs: int = 20): if __name__ == "__main__": - path_dir = os.path.join(PATH_LEGACY, "checkpoints", str(pl.__version__)) + name = sys.argv[1] if len(sys.argv) > 1 else str(pl.__version__) + path_dir = os.path.join(PATH_LEGACY, "checkpoints", name) main_train(path_dir) From 2d9271567293a9df6357ace01b4c57adc7cb0d18 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 17:34:20 +0200 Subject: [PATCH 9/9] local --- tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py b/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py index c820283abe724..e19f6c07b5e55 100644 --- a/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py +++ b/tests/tests_pytorch/graveyard/test_legacy_import_unpickler.py @@ -61,7 +61,7 @@ def test_imports_unified(pl_version: str): path_ckpt = path_ckpts[-1] # only below version 1.5.0 we pickled stuff in checkpoints - if Version(pl_version) < Version("1.5.0"): + if pl_version != "local" and Version(pl_version) < Version("1.5.0"): context = pytest.warns(UserWarning, match="Redirecting import of") else: context = no_warning_call(match="Redirecting import of*")