Patch release for stable [rebase & merge]

.actions/assistant.py

@@ -158,7 +158,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
     '...PyTorch Lightning is just organized PyTorch...'
     """
     path_readme = os.path.join(path_dir, "README.md")
-    text = open(path_readme, encoding="utf-8").read()
+    with open(path_readme, encoding="utf-8") as fo:
+        text = fo.read()
 
     # drop images from readme
     text = text.replace(
@@ -331,7 +332,7 @@ def copy_replace_imports(
             if not isfile(fp_new):
                 shutil.copy(fp, fp_new)
             continue
-        elif ext in (".pyc",):
+        if ext in (".pyc",):
             continue
         # Try to parse everything else
         with open(fp, encoding="utf-8") as fo:
@@ -394,8 +395,10 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
 
     @staticmethod
     def _replace_min(fname: str) -> None:
-        req = open(fname, encoding="utf-8").read().replace(">=", "==")
-        open(fname, "w", encoding="utf-8").write(req)
+        with open(fname, encoding="utf-8") as fo:
+            req = fo.read().replace(">=", "==")
+        with open(fname, "w", encoding="utf-8") as fw:
+            fw.write(req)
 
     @staticmethod
     def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:

.actions/pull_legacy_checkpoints.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
+
 # Run this script from the project root.
 URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip"
 mkdir -p tests/legacy
 # wget is simpler but does not work on Windows
 python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'tests/legacy/checkpoints.zip')"
 ls -l tests/legacy/
+
 unzip -o tests/legacy/checkpoints.zip -d tests/legacy/
 ls -l tests/legacy/checkpoints/

.azure/app-cloud-e2e.yml

@@ -60,7 +60,7 @@ jobs:
     pool: "azure-cpus"
     container:
       # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags
-      image: mcr.microsoft.com/playwright/python:v1.30.0-focal
+      image: mcr.microsoft.com/playwright/python:v1.32.1-focal
       options: "--shm-size=4gb"
     strategy:
       matrix:
@@ -135,7 +135,7 @@ jobs:
 
     # TODO: we are testing it as `lightning`, so add also version for `lightning_app`
     - bash: |
-        pip install -e .[test] \
+        pip install -e .[app-dev] \
           -f https://download.pytorch.org/whl/cpu/torch_stable.html
       displayName: 'Install Lightning & dependencies'
 

.azure/gpu-benchmark.yml

@@ -59,7 +59,6 @@ jobs:
     - bash: |
         echo $CUDA_VISIBLE_DEVICES
         echo $TORCH_URL
-        lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
         which python && which pip
@@ -76,7 +75,6 @@ jobs:
 
     - bash: |
         set -e
-        pip list
         python requirements/collect_env_details.py
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
       displayName: 'Env details'

.azure/gpu-tests-fabric.yml

@@ -71,7 +71,7 @@ jobs:
         cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
         echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver"
         echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
-        scope=$( python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))' )
+        scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
         echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
       displayName: 'set env. vars'
 
@@ -81,7 +81,6 @@ jobs:
         echo $CUDA_VERSION_MM
         echo $TORCH_URL
         echo $COVERAGE_SOURCE
-        lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
         which python && which pip
@@ -92,17 +91,20 @@ jobs:
 
     - bash: |
         PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
+        pip install -q wget packaging
+        python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
         for fpath in `ls requirements/**/*.txt`; do \
-          python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \
+          python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
         done
       displayName: 'Adjust dependencies'
 
-    - bash: pip install -e .[dev,strategies,examples] -U --find-links ${TORCH_URL}
+    - bash: |
+        extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
+        pip install -e ".[${extra}dev]" pytest-timeout -U --find-links ${TORCH_URL}
       displayName: 'Install package & dependencies'
 
     - bash: |
         set -e
-        pip list
         python requirements/collect_env_details.py
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
       displayName: 'Env details'
@@ -138,6 +140,13 @@ jobs:
       displayName: 'Testing: fabric standalone tests'
       timeoutInMinutes: "10"
 
+    - bash: bash run_standalone_tasks.sh
+      workingDirectory: tests/tests_fabric
+      env:
+        PL_RUN_CUDA_TESTS: "1"
+      displayName: 'Testing: fabric standalone tasks'
+      timeoutInMinutes: "10"
+
     - bash: |
         python -m coverage report
         python -m coverage xml

.azure/gpu-tests-pytorch.yml

@@ -51,10 +51,6 @@ jobs:
     cancelTimeoutInMinutes: "2"
     strategy:
       matrix:
-        'PyTorch & strategies':  # this uses torch 1.12 as not all strategies support 1.13 yet
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
-          scope: "strategies"
-          PACKAGE_NAME: "pytorch"
         'PyTorch | latest':
           image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1"
           scope: ""
@@ -82,7 +78,7 @@ jobs:
         cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
         echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver"
         echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
-        scope=$( python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))' )
+        scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
         echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
       displayName: 'set env. vars'
 
@@ -92,7 +88,6 @@ jobs:
         echo $CUDA_VERSION_MM
         echo $TORCH_URL
         echo $COVERAGE_SOURCE
-        lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
         which python && which pip
@@ -103,47 +98,44 @@ jobs:
 
     - bash: |
         PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
+        pip install -q wget packaging
+        python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
         for fpath in `ls requirements/**/*.txt`; do \
-          python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \
+          python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \
         done
+        # prune packages with installation issues
+        pip install -q -r .actions/requirements.txt
+        python .actions/assistant.py requirements_prune_pkgs \
+          --packages="[lightning-colossalai,lightning-bagua]" \
+          --req_files="[requirements/_integrations/strategies.txt]"
       displayName: 'Adjust dependencies'
 
-    - bash: pip install -e .[extra,test,examples] -U --find-links ${TORCH_URL}
-      displayName: 'Install package & dependencies'
-
     - bash: |
-        pip uninstall -y -r requirements/pytorch/strategies.txt \
-                         -r requirements/_integrations/strategies.txt
-      condition: ne(variables['scope'], 'strategies')
-      displayName: 'Uninstall strategies'
+        extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
+        pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links ${TORCH_URL}
+      displayName: 'Install package & dependencies'
 
-    - bash: |
-        set -e
-        pip install -r requirements/pytorch/strategies.txt \
-                    -r requirements/_integrations/strategies.txt \
-          --find-links ${TORCH_URL}
-        python requirements/pytorch/check-avail-strategies.py
-      condition: eq(variables['scope'], 'strategies')
-      displayName: 'Install strategies'
+    - bash: pip uninstall -y lightning
+      condition: eq(variables['PACKAGE_NAME'], 'pytorch')
+      # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it
+      displayName: 'Drop LAI from extensions'
+    - bash: pip uninstall -y pytorch-lightning
+      condition: eq(variables['PACKAGE_NAME'], 'lightning')
+      displayName: 'Drop PL for LAI'
 
     - bash: |
         set -e
-        pip list
         python requirements/collect_env_details.py
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
         python requirements/pytorch/check-avail-extras.py
       displayName: 'Env details'
 
-    - bash: bash .actions/pull_legacy_checkpoints.sh
-      displayName: 'Get legacy checkpoints'
-
     - bash: python -m pytest pytorch_lightning
       workingDirectory: src
       condition: eq(variables['PACKAGE_NAME'], 'pytorch')
       displayName: 'Testing: PyTorch doctests'
 
     - bash: |
-        pip install -q -r .actions/requirements.txt
         python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
           --source_import="lightning.fabric,lightning.pytorch" \
           --target_import="lightning_fabric,pytorch_lightning"
@@ -153,6 +145,13 @@ jobs:
       condition: eq(variables['PACKAGE_NAME'], 'pytorch')
       displayName: 'Adjust tests & examples'
 
+    - bash: |
+        bash .actions/pull_legacy_checkpoints.sh
+        cd tests/legacy
+        bash generate_checkpoints.sh
+        ls -l checkpoints/
+      displayName: 'Get legacy checkpoints'
+
     - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest --ignore benchmarks -v --durations=50
       workingDirectory: tests/tests_pytorch
       env:

.azure/hpu-tests.yml

@@ -41,7 +41,7 @@ pr:
 jobs:
   - job: testing
     # how long to run the job before automatically cancelling
-    timeoutInMinutes: "10"
+    timeoutInMinutes: "20"
     # how much time to give 'run always even if cancelled tasks' before stopping them
     cancelTimeoutInMinutes: "2"
     pool: intel-hpus

.codecov.yml

@@ -36,6 +36,7 @@ coverage:
     # https://codecov.readme.io/v1.0/docs/commit-status
     project:
       default:
+        informational: true
         target: 99% # specify the target coverage for each commit status
         threshold: 30% # allow this little decrease on project
         # https://github.com/codecov/support/wiki/Filtering-Branches
@@ -44,6 +45,7 @@ coverage:
     # https://github.com/codecov/support/wiki/Patch-Status
     patch:
       default:
+        informational: true
         target: 50% # specify the target "X%" coverage to hit
         threshold: 5% # allow this much decrease on patch
     changes: false

.github/BECOMING_A_CORE_CONTRIBUTOR.md

@@ -58,7 +58,7 @@ There's also [GitHub discussions](https://github.com/Lightning-AI/lightning/disc
 ## Applying
 
 There are no precise targets for becoming a core contributor. In the past, community members have become core after fitting the previous expectations consistently.
-We are on the lookout for new people to join, however, if you feel like you meet the expectations already and we haven't reached out to you yet, feel free to ping us privately on [![Discord](https://img.shields.io/discord/1077906959069626439?style=plastic)](https://discord.gg/VptPCZkGNa)!.
+We are on the lookout for new people to join, however, if you feel like you meet the expectations already and we haven't reached out to you yet, feel free to ping us privately on [Discord](https://discord.gg/XbUZ3xbdQk).
 
 ## Employment
 

.github/ISSUE_TEMPLATE/1_bug_report.yaml

@@ -26,6 +26,22 @@ body:
     validations:
       required: true
 
+  - type: dropdown
+    id: versions
+    attributes:
+      label: What version are you seeing the problem on?
+      description: select all version where you have experienced this issue
+      multiple: true
+      options:
+        - "v1.6"
+        - "v1.7"
+        - "v1.8"
+        - "v1.9"
+        - "v2.0"
+        - "master"
+    validations:
+      required: true
+
   - type: markdown
     attributes:
       value: "**Note: The rest of this form is optional, but filling it out may help us to provide better support.**"

.github/advanced-issue-labeler.yml

@@ -0,0 +1,19 @@
+# syntax - https://github.com/redhat-plumbers-in-action/advanced-issue-labeler#policy
+---
+
+policy:
+  - section:
+    - id: ['versions']
+      label:
+        - name: "ver: 1.6.x"
+          keys: ['v1_6', 'v1.6', '1.6.x']
+        - name: "ver: 1.7.x"
+          keys: ['v1_7', 'v1.7', '1.7.x']
+        - name: "ver: 1.8.x"
+          keys: ['v1_8', 'v1.8', '1.8.x']
+        - name: "ver: 1.9.x"
+          keys: ['v1_9', 'v1.9', '1.9.x']
+        - name: "ver: 2.0.x"
+          keys: ['v2_0', 'v2.0', '2.0+']
+        - name: "ver: 2.1.x"
+          keys: ['master']

.github/checkgroup.yml

@@ -338,27 +338,28 @@ subprojects:
       - "Store.cloud (test_store Mac)"
       - "Store.cloud (test_store Windows)"
 
-  - id: "lightning_app: Azure"
-    paths:
-      - ".actions/**"
-      - ".azure/app-cloud-e2e.yml"
-      - "src/lightning/__about__.py"
-      - "src/lightning/__init__.py"
-      - "src/lightning/__main__.py"
-      - "src/lightning/__setup__.py"
-      - "src/lightning/__version__.py"
-      - "src/lightning/app/**"
-      - "src/lightning_app/*"
-      - "examples/app/**"
-      - "requirements/app/**"
-      - "tests/integrations_app/**"
-      - "!tests/integrations_app/flagship/**"
-      - "setup.py"
-      - "!requirements/*/docs.txt"
-      - "!*.md"
-      - "!**/*.md"
-    checks:
-      - "App.cloud-e2e"
+  # FixMe: re-enable when BE stabilize
+  # - id: "lightning_app: Azure"
+  #   paths:
+  #     - ".actions/**"
+  #     - ".azure/app-cloud-e2e.yml"
+  #     - "src/lightning/__about__.py"
+  #     - "src/lightning/__init__.py"
+  #     - "src/lightning/__main__.py"
+  #     - "src/lightning/__setup__.py"
+  #     - "src/lightning/__version__.py"
+  #     - "src/lightning/app/**"
+  #     - "src/lightning_app/*"
+  #     - "examples/app/**"
+  #     - "requirements/app/**"
+  #     - "tests/integrations_app/**"
+  #     - "!tests/integrations_app/flagship/**"
+  #     - "setup.py"
+  #     - "!requirements/*/docs.txt"
+  #     - "!*.md"
+  #     - "!**/*.md"
+  #   checks:
+  #     - "App.cloud-e2e"
 
   - id: "lightning_app: Docs"
     paths:

.github/workflows/_flagship-apps.yml

@@ -29,7 +29,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: mcr.microsoft.com/playwright/python:v1.30.0-focal
+      image: mcr.microsoft.com/playwright/python:v1.32.1-focal
     strategy:
       fail-fast: false
       matrix: