From e25fde82cf79a3f5e6b422f6cb10454fa79a3e13 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 14 Apr 2023 10:39:23 +0200 Subject: [PATCH 01/17] Adding tests for legacy checkpoints --- docs/source-pytorch/past_versions.rst | 4 +++- tests/legacy/back-compatible-versions.txt | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source-pytorch/past_versions.rst b/docs/source-pytorch/past_versions.rst index 3b036ae5fbefd..7314dd56bd079 100644 --- a/docs/source-pytorch/past_versions.rst +++ b/docs/source-pytorch/past_versions.rst @@ -19,7 +19,9 @@ TO help you with keeping up to spead, check :doc:`Migration guide <./upgrade/mig - `1.9.0 `_, `1.9.1 `_, `1.9.2 `_, - `1.9.3 `_ + `1.9.3 `_, + `1.9.4 `_, + `1.9.5 `_ - :doc:`from 1.9 to 2.0 ` * - `1.8 `_ diff --git a/tests/legacy/back-compatible-versions.txt b/tests/legacy/back-compatible-versions.txt index b6c18738712ce..56fdb6f74d496 100644 --- a/tests/legacy/back-compatible-versions.txt +++ b/tests/legacy/back-compatible-versions.txt @@ -71,3 +71,14 @@ 1.7.6 1.7.7 1.8.0 +1.8.1 +1.8.2 +1.8.3 +1.8.4 +1.8.5 +1.8.6 +1.9.0 +1.9.1 +1.9.2 +1.9.3 +1.9.4 From e0b48780b4351bf1ba089e96550c3569a010177d Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 14 Apr 2023 10:42:34 +0200 Subject: [PATCH 02/17] Apply suggestions from code review --- tests/legacy/back-compatible-versions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/legacy/back-compatible-versions.txt b/tests/legacy/back-compatible-versions.txt index 56fdb6f74d496..4dc21c75b39c0 100644 --- a/tests/legacy/back-compatible-versions.txt +++ b/tests/legacy/back-compatible-versions.txt @@ -82,3 +82,4 @@ 1.9.2 1.9.3 1.9.4 +1.9.5 From d721c9bbfbdd3889cf3ce62dff379f771e97c842 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 28 Apr 2023 11:51:45 +0200 Subject: [PATCH 03/17] Apply suggestions from code review --- tests/legacy/back-compatible-versions.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/legacy/back-compatible-versions.txt b/tests/legacy/back-compatible-versions.txt index 4dc21c75b39c0..c2bc418eecd93 100644 --- a/tests/legacy/back-compatible-versions.txt +++ b/tests/legacy/back-compatible-versions.txt @@ -77,9 +77,3 @@ 1.8.4 1.8.5 1.8.6 -1.9.0 -1.9.1 -1.9.2 -1.9.3 -1.9.4 -1.9.5 From e502063c009033bba5a6e7ba6564baf2e0a94b17 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 28 Apr 2023 14:57:01 +0200 Subject: [PATCH 04/17] fix simple --- tests/tests_pytorch/helpers/simple_models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/tests_pytorch/helpers/simple_models.py b/tests/tests_pytorch/helpers/simple_models.py index 4e82ba00f1900..ffd73c05bdbe9 100644 --- a/tests/tests_pytorch/helpers/simple_models.py +++ b/tests/tests_pytorch/helpers/simple_models.py @@ -11,13 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import operator + import torch import torch.nn.functional as F +from lightning_utilities import compare_version from torch import nn from torchmetrics import Accuracy, MeanSquaredError from pytorch_lightning import LightningModule -from pytorch_lightning.utilities.imports import _TORCHMETRICS_GREATER_EQUAL_0_11 as _TM_GE_0_11 + +_TM_GE_0_11 = compare_version("torchmetrics", operator.ge, "0.11.0") # using new API with task class ClassificationModel(LightningModule): From f465aa3389d7736281af4d6261cce0263dc293bb Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 28 Apr 2023 15:06:59 +0200 Subject: [PATCH 05/17] utils --- .github/workflows/_legacy-checkpoints.yml | 15 ++++----------- tests/legacy/generate_checkpoints.sh | 2 +- tests/tests_pytorch/helpers/simple_models.py | 3 ++- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 99f592da2ade2..66cec355c5c5d 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -68,16 +68,11 @@ jobs: env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 - run: | - pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip list + run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html if: inputs.pl_version == '' - name: Install PL version - run: | - pip install "pytorch-lightning==${{ inputs.pl_version }}" \ - -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip list + run: pip install "pytorch-lightning==${{ inputs.pl_version }}" -f https://download.pytorch.org/whl/cpu/torch_stable.html if: inputs.pl_version != '' - name: Pull legacy checkpoints @@ -86,12 +81,10 @@ jobs: - name: Decide PL version to create a PR with id: decide-version - run: | - python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT + run: python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT - name: Generate checkpoints - run: | - bash generate_checkpoints.sh ${{ inputs.pl_version }} + run: bash generate_checkpoints.sh ${{ inputs.pl_version }} - name: Keep artifact id: keep-artifact diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 9f597161acfc9..28e3117959e43 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -35,7 +35,7 @@ do python -m venv $ENV_PATH source $ENV_PATH/bin/activate - python -m pip install pytorch_lightning==$pl_ver -r $LEGACY_PATH/requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html + python -m pip install "pytorch_lightning==$pl_ver" "lightning-utilities" -r $LEGACY_PATH/requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html create_and_save_checkpoint diff --git a/tests/tests_pytorch/helpers/simple_models.py b/tests/tests_pytorch/helpers/simple_models.py index ffd73c05bdbe9..52322d5a258e6 100644 --- a/tests/tests_pytorch/helpers/simple_models.py +++ b/tests/tests_pytorch/helpers/simple_models.py @@ -21,7 +21,8 @@ from pytorch_lightning import LightningModule -_TM_GE_0_11 = compare_version("torchmetrics", operator.ge, "0.11.0") # using new API with task +# using new API with task +_TM_GE_0_11 = compare_version("torchmetrics", operator.ge, "0.11.0") class ClassificationModel(LightningModule): From ee5a4624da60bdc34642dfc42a8f3d7e2bd80d36 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 28 Apr 2023 15:13:38 +0200 Subject: [PATCH 06/17] import --- tests/tests_pytorch/helpers/simple_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_pytorch/helpers/simple_models.py b/tests/tests_pytorch/helpers/simple_models.py index 52322d5a258e6..678cf4b0472ff 100644 --- a/tests/tests_pytorch/helpers/simple_models.py +++ b/tests/tests_pytorch/helpers/simple_models.py @@ -15,7 +15,7 @@ import torch import torch.nn.functional as F -from lightning_utilities import compare_version +from lightning_utilities.core.imports import compare_version from torch import nn from torchmetrics import Accuracy, MeanSquaredError From 3fef1e1e293e47cccf26dcb0f48bef588d33ad4c Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 28 Apr 2023 15:19:29 +0200 Subject: [PATCH 07/17] pl --- tests/legacy/generate_checkpoints.sh | 2 +- tests/tests_pytorch/helpers/simple_models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 28e3117959e43..0da3e0f60ef5e 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -35,7 +35,7 @@ do python -m venv $ENV_PATH source $ENV_PATH/bin/activate - python -m pip install "pytorch_lightning==$pl_ver" "lightning-utilities" -r $LEGACY_PATH/requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html + python -m pip install "pytorch_lightning==$pl_ver" -r $LEGACY_PATH/requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html create_and_save_checkpoint diff --git a/tests/tests_pytorch/helpers/simple_models.py b/tests/tests_pytorch/helpers/simple_models.py index 678cf4b0472ff..36b0c7bcd7177 100644 --- a/tests/tests_pytorch/helpers/simple_models.py +++ b/tests/tests_pytorch/helpers/simple_models.py @@ -15,11 +15,11 @@ import torch import torch.nn.functional as F -from lightning_utilities.core.imports import compare_version from torch import nn from torchmetrics import Accuracy, MeanSquaredError from pytorch_lightning import LightningModule +from pytorch_lightning.utilities.imports import compare_version # using new API with task _TM_GE_0_11 = compare_version("torchmetrics", operator.ge, "0.11.0") From e2dc026cb61a839d6e6d439331e62b96c06c74a9 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 3 May 2023 21:53:10 +0200 Subject: [PATCH 08/17] num_features=24 --- tests/legacy/simple_classif_training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index fd62bc0963c0f..1dc0fc4635632 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -39,7 +39,7 @@ def main_train(dir_path, max_epochs: int = 20): ) dm = ClassifDataModule() - model = ClassificationModel() + model = ClassificationModel(num_features=24, num_classes=3, lr=0.01) trainer.fit(model, datamodule=dm) res = trainer.test(model, datamodule=dm) assert res[0]["test_loss"] <= 0.85, str(res[0]["test_loss"]) From aa6024a305e30e7e59456341525387340d6b1bef Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 3 May 2023 22:03:04 +0200 Subject: [PATCH 09/17] num_features=24 --- tests/legacy/simple_classif_training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 1dc0fc4635632..af12e5de9a27f 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -38,7 +38,7 @@ def main_train(dir_path, max_epochs: int = 20): deterministic=True, ) - dm = ClassifDataModule() + dm = ClassifDataModule(num_features=24, length=800, num_classes=3) model = ClassificationModel(num_features=24, num_classes=3, lr=0.01) trainer.fit(model, datamodule=dm) res = trainer.test(model, datamodule=dm) From c332ff546e8e1cb46191995850cec0ffece4e71b Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 3 May 2023 22:14:58 +0200 Subject: [PATCH 10/17] length=6000 --- tests/legacy/simple_classif_training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index af12e5de9a27f..0953024ff3ad4 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -38,7 +38,7 @@ def main_train(dir_path, max_epochs: int = 20): deterministic=True, ) - dm = ClassifDataModule(num_features=24, length=800, num_classes=3) + dm = ClassifDataModule(num_features=24, length=6000, num_classes=3, batch_size=128) model = ClassificationModel(num_features=24, num_classes=3, lr=0.01) trainer.fit(model, datamodule=dm) res = trainer.test(model, datamodule=dm) From bdb3e54f838b311aa81249b882e5f2d56a5ca11e Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 3 May 2023 22:24:54 +0200 Subject: [PATCH 11/17] other --- tests/legacy/simple_classif_training.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/legacy/simple_classif_training.py b/tests/legacy/simple_classif_training.py index 0953024ff3ad4..00e51627efad9 100644 --- a/tests/legacy/simple_classif_training.py +++ b/tests/legacy/simple_classif_training.py @@ -38,7 +38,9 @@ def main_train(dir_path, max_epochs: int = 20): deterministic=True, ) - dm = ClassifDataModule(num_features=24, length=6000, num_classes=3, batch_size=128) + dm = ClassifDataModule( + num_features=24, length=6000, num_classes=3, batch_size=128, n_clusters_per_class=2, n_informative=int(24 / 3) + ) model = ClassificationModel(num_features=24, num_classes=3, lr=0.01) trainer.fit(model, datamodule=dm) res = trainer.test(model, datamodule=dm) From 5ac2e8be12650d46a7c9543459ae761098aeb02f Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 3 May 2023 23:19:42 +0200 Subject: [PATCH 12/17] rm --- tests/legacy/generate_checkpoints.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 0da3e0f60ef5e..571a9c2d9b924 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -32,6 +32,7 @@ do # Don't install/update anything before activating venv # to avoid breaking any existing environment. + rm -rf $ENV_PATH python -m venv $ENV_PATH source $ENV_PATH/bin/activate From 6963c028f36f3a2069093f7d4f3442afa000161d Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 09:50:37 +0200 Subject: [PATCH 13/17] dru run --- .github/workflows/_legacy-checkpoints.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 66cec355c5c5d..cde197ca29644 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -88,7 +88,10 @@ jobs: - name: Keep artifact id: keep-artifact - run: python -c "print('DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT + run: | + python -c "print('DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT + echo "${inputs.push_to_s3}}" + python -c "print('AWS_RUN=' + str('--dryrun' if '${inputs.push_to_s3}}' == 'true' else ''))" >> $GITHUB_ENV - name: Upload checkpoints to GitHub Actions artifact uses: actions/upload-artifact@v3 @@ -97,18 +100,12 @@ jobs: path: tests/legacy/checkpoints/ retention-days: ${{ steps.keep-artifact.outputs.DAYS }} - - name: Upload checkpoints to S3 (dryrun) - run: | - aws s3 sync --dryrun checkpoints/ s3://pl-public-data/legacy/checkpoints/ - zip -r checkpoints.zip checkpoints - aws s3 cp --dryrun checkpoints.zip s3://pl-public-data/legacy/ --acl public-read - - name: Upload checkpoints to S3 run: | - aws s3 sync checkpoints/ s3://pl-public-data/legacy/checkpoints/ + aws s3 sync $AWS_RUN checkpoints/ s3://pl-public-data/legacy/checkpoints/ zip -r checkpoints.zip checkpoints - aws s3 cp checkpoints.zip s3://pl-public-data/legacy/ --acl public-read - if: inputs.push_to_s3 + aws s3 cp $AWS_RUN checkpoints.zip s3://pl-public-data/legacy/ --acl public-read + adding-ckpt-test: runs-on: ubuntu-20.04 From 37903c2a9878486f895ceff42592b1c3455b3f5d Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 09:54:56 +0200 Subject: [PATCH 14/17] dru run --- .github/workflows/_legacy-checkpoints.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index cde197ca29644..deb5c0a88c79b 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -90,8 +90,8 @@ jobs: id: keep-artifact run: | python -c "print('DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT - echo "${inputs.push_to_s3}}" - python -c "print('AWS_RUN=' + str('--dryrun' if '${inputs.push_to_s3}}' == 'true' else ''))" >> $GITHUB_ENV + echo "${{inputs.push_to_s3}}" + python -c "print('AWS_RUN=' + str('--dryrun' if '${{inputs.push_to_s3}}' == 'true' else ''))" >> $GITHUB_ENV - name: Upload checkpoints to GitHub Actions artifact uses: actions/upload-artifact@v3 From bfed43cfb97a9f2f91916674a170085a70c7a062 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 10:02:50 +0200 Subject: [PATCH 15/17] dru run --- .github/workflows/_legacy-checkpoints.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index deb5c0a88c79b..7a76a0803212f 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -86,12 +86,11 @@ jobs: - name: Generate checkpoints run: bash generate_checkpoints.sh ${{ inputs.pl_version }} - - name: Keep artifact + - name: Keep artifact & DryRun id: keep-artifact run: | python -c "print('DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT - echo "${{inputs.push_to_s3}}" - python -c "print('AWS_RUN=' + str('--dryrun' if '${{inputs.push_to_s3}}' == 'true' else ''))" >> $GITHUB_ENV + python -c "print('AWS_RUN=' + str('' if '${{inputs.push_to_s3}}' == 'true' else '--dryrun'))" >> $GITHUB_ENV - name: Upload checkpoints to GitHub Actions artifact uses: actions/upload-artifact@v3 From 70668d4a7de190c90fd2fc07a2f675778ce376cf Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 10:38:41 +0200 Subject: [PATCH 16/17] rm --- .github/workflows/_legacy-checkpoints.yml | 5 ++--- tests/legacy/generate_checkpoints.sh | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 7a76a0803212f..191b517cfcf8c 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -87,9 +87,8 @@ jobs: run: bash generate_checkpoints.sh ${{ inputs.pl_version }} - name: Keep artifact & DryRun - id: keep-artifact run: | - python -c "print('DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT + python -c "print('KEEP_DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_ENV python -c "print('AWS_RUN=' + str('' if '${{inputs.push_to_s3}}' == 'true' else '--dryrun'))" >> $GITHUB_ENV - name: Upload checkpoints to GitHub Actions artifact @@ -97,7 +96,7 @@ jobs: with: name: checkpoints-${{ github.sha }} path: tests/legacy/checkpoints/ - retention-days: ${{ steps.keep-artifact.outputs.DAYS }} + retention-days: ${{ env.KEEP_DAYS }} - name: Upload checkpoints to S3 run: | diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 571a9c2d9b924..7873daf3fe56f 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -38,6 +38,7 @@ do python -m pip install "pytorch_lightning==$pl_ver" -r $LEGACY_PATH/requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html + rm -rf $LEGACY_PATH/checkpoints/$pl_ver create_and_save_checkpoint deactivate From ed885644eca4c5446add85fd78b9db960ff16671 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 4 May 2023 11:43:20 +0200 Subject: [PATCH 17/17] tag --- .github/workflows/_legacy-checkpoints.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 191b517cfcf8c..73386c71dbf71 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -123,5 +123,6 @@ jobs: title: Adding test for legacy checkpoint created with ${{ needs.create-legacy-ckpts.outputs.pl-version }} delete-branch: true labels: | + checkpointing tests pl