diff --git a/.actions/assistant.py b/.actions/assistant.py index d791793af9490..c5043addffeec 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -158,7 +158,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str: '...PyTorch Lightning is just organized PyTorch...' """ path_readme = os.path.join(path_dir, "README.md") - text = open(path_readme, encoding="utf-8").read() + with open(path_readme, encoding="utf-8") as fo: + text = fo.read() # drop images from readme text = text.replace( @@ -331,7 +332,7 @@ def copy_replace_imports( if not isfile(fp_new): shutil.copy(fp, fp_new) continue - elif ext in (".pyc",): + if ext in (".pyc",): continue # Try to parse everything else with open(fp, encoding="utf-8") as fo: @@ -394,8 +395,10 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None: @staticmethod def _replace_min(fname: str) -> None: - req = open(fname, encoding="utf-8").read().replace(">=", "==") - open(fname, "w", encoding="utf-8").write(req) + with open(fname, encoding="utf-8") as fo: + req = fo.read().replace(">=", "==") + with open(fname, "w", encoding="utf-8") as fw: + fw.write(req) @staticmethod def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None: diff --git a/.actions/pull_legacy_checkpoints.sh b/.actions/pull_legacy_checkpoints.sh index cf8a0fdf4e574..b61647a96e50c 100644 --- a/.actions/pull_legacy_checkpoints.sh +++ b/.actions/pull_legacy_checkpoints.sh @@ -1,9 +1,11 @@ #!/bin/bash + # Run this script from the project root. URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip" mkdir -p tests/legacy # wget is simpler but does not work on Windows python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'tests/legacy/checkpoints.zip')" ls -l tests/legacy/ + unzip -o tests/legacy/checkpoints.zip -d tests/legacy/ ls -l tests/legacy/checkpoints/ diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index e1b6711a20b34..89480c4ebd950 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -60,7 +60,7 @@ jobs: pool: "azure-cpus" container: # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags - image: mcr.microsoft.com/playwright/python:v1.30.0-focal + image: mcr.microsoft.com/playwright/python:v1.32.1-focal options: "--shm-size=4gb" strategy: matrix: @@ -135,7 +135,7 @@ jobs: # TODO: we are testing it as `lightning`, so add also version for `lightning_app` - bash: | - pip install -e .[test] \ + pip install -e .[app-dev] \ -f https://download.pytorch.org/whl/cpu/torch_stable.html displayName: 'Install Lightning & dependencies' diff --git a/.azure/gpu-benchmark.yml b/.azure/gpu-benchmark.yml index 34942c8928f53..f9580fb595e59 100644 --- a/.azure/gpu-benchmark.yml +++ b/.azure/gpu-benchmark.yml @@ -59,7 +59,6 @@ jobs: - bash: | echo $CUDA_VISIBLE_DEVICES echo $TORCH_URL - lspci | egrep 'VGA|3D' whereis nvidia nvidia-smi which python && which pip @@ -76,7 +75,6 @@ jobs: - bash: | set -e - pip list python requirements/collect_env_details.py python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'" displayName: 'Env details' diff --git a/.azure/gpu-tests-fabric.yml b/.azure/gpu-tests-fabric.yml index 64088cc255f9c..f951c13de2431 100644 --- a/.azure/gpu-tests-fabric.yml +++ b/.azure/gpu-tests-fabric.yml @@ -71,7 +71,7 @@ jobs: cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))") echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver" echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html" - scope=$( python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))' ) + scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))') echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope" displayName: 'set env. vars' @@ -81,7 +81,6 @@ jobs: echo $CUDA_VERSION_MM echo $TORCH_URL echo $COVERAGE_SOURCE - lspci | egrep 'VGA|3D' whereis nvidia nvidia-smi which python && which pip @@ -92,17 +91,20 @@ jobs: - bash: | PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])") + pip install -q wget packaging + python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ - python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \ + python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \ done displayName: 'Adjust dependencies' - - bash: pip install -e .[dev,strategies,examples] -U --find-links ${TORCH_URL} + - bash: | + extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))") + pip install -e ".[${extra}dev]" pytest-timeout -U --find-links ${TORCH_URL} displayName: 'Install package & dependencies' - bash: | set -e - pip list python requirements/collect_env_details.py python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'" displayName: 'Env details' @@ -138,6 +140,13 @@ jobs: displayName: 'Testing: fabric standalone tests' timeoutInMinutes: "10" + - bash: bash run_standalone_tasks.sh + workingDirectory: tests/tests_fabric + env: + PL_RUN_CUDA_TESTS: "1" + displayName: 'Testing: fabric standalone tasks' + timeoutInMinutes: "10" + - bash: | python -m coverage report python -m coverage xml diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 6d17e19e6dda5..e509add125385 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -51,10 +51,6 @@ jobs: cancelTimeoutInMinutes: "2" strategy: matrix: - 'PyTorch & strategies': # this uses torch 1.12 as not all strategies support 1.13 yet - image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1" - scope: "strategies" - PACKAGE_NAME: "pytorch" 'PyTorch | latest': image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.7.1" scope: "" @@ -82,7 +78,7 @@ jobs: cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))") echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver" echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html" - scope=$( python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))' ) + scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))') echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope" displayName: 'set env. vars' @@ -92,7 +88,6 @@ jobs: echo $CUDA_VERSION_MM echo $TORCH_URL echo $COVERAGE_SOURCE - lspci | egrep 'VGA|3D' whereis nvidia nvidia-smi which python && which pip @@ -103,47 +98,44 @@ jobs: - bash: | PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])") + pip install -q wget packaging + python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ - python ./requirements/pytorch/adjust-versions.py $fpath ${PYTORCH_VERSION}; \ + python ./adjust-torch-versions.py $fpath ${PYTORCH_VERSION}; \ done + # prune packages with installation issues + pip install -q -r .actions/requirements.txt + python .actions/assistant.py requirements_prune_pkgs \ + --packages="[lightning-colossalai,lightning-bagua]" \ + --req_files="[requirements/_integrations/strategies.txt]" displayName: 'Adjust dependencies' - - bash: pip install -e .[extra,test,examples] -U --find-links ${TORCH_URL} - displayName: 'Install package & dependencies' - - bash: | - pip uninstall -y -r requirements/pytorch/strategies.txt \ - -r requirements/_integrations/strategies.txt - condition: ne(variables['scope'], 'strategies') - displayName: 'Uninstall strategies' + extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))") + pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links ${TORCH_URL} + displayName: 'Install package & dependencies' - - bash: | - set -e - pip install -r requirements/pytorch/strategies.txt \ - -r requirements/_integrations/strategies.txt \ - --find-links ${TORCH_URL} - python requirements/pytorch/check-avail-strategies.py - condition: eq(variables['scope'], 'strategies') - displayName: 'Install strategies' + - bash: pip uninstall -y lightning + condition: eq(variables['PACKAGE_NAME'], 'pytorch') + # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it + displayName: 'Drop LAI from extensions' + - bash: pip uninstall -y pytorch-lightning + condition: eq(variables['PACKAGE_NAME'], 'lightning') + displayName: 'Drop PL for LAI' - bash: | set -e - pip list python requirements/collect_env_details.py python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'" python requirements/pytorch/check-avail-extras.py displayName: 'Env details' - - bash: bash .actions/pull_legacy_checkpoints.sh - displayName: 'Get legacy checkpoints' - - bash: python -m pytest pytorch_lightning workingDirectory: src condition: eq(variables['PACKAGE_NAME'], 'pytorch') displayName: 'Testing: PyTorch doctests' - bash: | - pip install -q -r .actions/requirements.txt python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \ --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" @@ -153,6 +145,13 @@ jobs: condition: eq(variables['PACKAGE_NAME'], 'pytorch') displayName: 'Adjust tests & examples' + - bash: | + bash .actions/pull_legacy_checkpoints.sh + cd tests/legacy + bash generate_checkpoints.sh + ls -l checkpoints/ + displayName: 'Get legacy checkpoints' + - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest --ignore benchmarks -v --durations=50 workingDirectory: tests/tests_pytorch env: diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml index 7b0b72313ef46..9493ff24436e3 100644 --- a/.azure/hpu-tests.yml +++ b/.azure/hpu-tests.yml @@ -41,7 +41,7 @@ pr: jobs: - job: testing # how long to run the job before automatically cancelling - timeoutInMinutes: "10" + timeoutInMinutes: "20" # how much time to give 'run always even if cancelled tasks' before stopping them cancelTimeoutInMinutes: "2" pool: intel-hpus diff --git a/.codecov.yml b/.codecov.yml index 8544950a6acd9..301eba5ba4de5 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -36,6 +36,7 @@ coverage: # https://codecov.readme.io/v1.0/docs/commit-status project: default: + informational: true target: 99% # specify the target coverage for each commit status threshold: 30% # allow this little decrease on project # https://github.com/codecov/support/wiki/Filtering-Branches @@ -44,6 +45,7 @@ coverage: # https://github.com/codecov/support/wiki/Patch-Status patch: default: + informational: true target: 50% # specify the target "X%" coverage to hit threshold: 5% # allow this much decrease on patch changes: false diff --git a/.github/BECOMING_A_CORE_CONTRIBUTOR.md b/.github/BECOMING_A_CORE_CONTRIBUTOR.md index 51d326d9eb299..3042c318cb4aa 100644 --- a/.github/BECOMING_A_CORE_CONTRIBUTOR.md +++ b/.github/BECOMING_A_CORE_CONTRIBUTOR.md @@ -58,7 +58,7 @@ There's also [GitHub discussions](https://github.com/Lightning-AI/lightning/disc ## Applying There are no precise targets for becoming a core contributor. In the past, community members have become core after fitting the previous expectations consistently. -We are on the lookout for new people to join, however, if you feel like you meet the expectations already and we haven't reached out to you yet, feel free to ping us privately on [![Discord](https://img.shields.io/discord/1077906959069626439?style=plastic)](https://discord.gg/VptPCZkGNa)!. +We are on the lookout for new people to join, however, if you feel like you meet the expectations already and we haven't reached out to you yet, feel free to ping us privately on [Discord](https://discord.gg/XbUZ3xbdQk). ## Employment diff --git a/.github/ISSUE_TEMPLATE/1_bug_report.yaml b/.github/ISSUE_TEMPLATE/1_bug_report.yaml index b8a394453be1a..6fcddb55cb73e 100644 --- a/.github/ISSUE_TEMPLATE/1_bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/1_bug_report.yaml @@ -26,6 +26,22 @@ body: validations: required: true + - type: dropdown + id: versions + attributes: + label: What version are you seeing the problem on? + description: select all version where you have experienced this issue + multiple: true + options: + - "v1.6" + - "v1.7" + - "v1.8" + - "v1.9" + - "v2.0" + - "master" + validations: + required: true + - type: markdown attributes: value: "**Note: The rest of this form is optional, but filling it out may help us to provide better support.**" diff --git a/.github/advanced-issue-labeler.yml b/.github/advanced-issue-labeler.yml new file mode 100644 index 0000000000000..e958c3f892515 --- /dev/null +++ b/.github/advanced-issue-labeler.yml @@ -0,0 +1,19 @@ +# syntax - https://github.com/redhat-plumbers-in-action/advanced-issue-labeler#policy +--- + +policy: + - section: + - id: ['versions'] + label: + - name: "ver: 1.6.x" + keys: ['v1_6', 'v1.6', '1.6.x'] + - name: "ver: 1.7.x" + keys: ['v1_7', 'v1.7', '1.7.x'] + - name: "ver: 1.8.x" + keys: ['v1_8', 'v1.8', '1.8.x'] + - name: "ver: 1.9.x" + keys: ['v1_9', 'v1.9', '1.9.x'] + - name: "ver: 2.0.x" + keys: ['v2_0', 'v2.0', '2.0+'] + - name: "ver: 2.1.x" + keys: ['master'] diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 13680c0c841b9..95353f31999b9 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -338,27 +338,28 @@ subprojects: - "Store.cloud (test_store Mac)" - "Store.cloud (test_store Windows)" - - id: "lightning_app: Azure" - paths: - - ".actions/**" - - ".azure/app-cloud-e2e.yml" - - "src/lightning/__about__.py" - - "src/lightning/__init__.py" - - "src/lightning/__main__.py" - - "src/lightning/__setup__.py" - - "src/lightning/__version__.py" - - "src/lightning/app/**" - - "src/lightning_app/*" - - "examples/app/**" - - "requirements/app/**" - - "tests/integrations_app/**" - - "!tests/integrations_app/flagship/**" - - "setup.py" - - "!requirements/*/docs.txt" - - "!*.md" - - "!**/*.md" - checks: - - "App.cloud-e2e" + # FixMe: re-enable when BE stabilize + # - id: "lightning_app: Azure" + # paths: + # - ".actions/**" + # - ".azure/app-cloud-e2e.yml" + # - "src/lightning/__about__.py" + # - "src/lightning/__init__.py" + # - "src/lightning/__main__.py" + # - "src/lightning/__setup__.py" + # - "src/lightning/__version__.py" + # - "src/lightning/app/**" + # - "src/lightning_app/*" + # - "examples/app/**" + # - "requirements/app/**" + # - "tests/integrations_app/**" + # - "!tests/integrations_app/flagship/**" + # - "setup.py" + # - "!requirements/*/docs.txt" + # - "!*.md" + # - "!**/*.md" + # checks: + # - "App.cloud-e2e" - id: "lightning_app: Docs" paths: diff --git a/.github/workflows/_flagship-apps.yml b/.github/workflows/_flagship-apps.yml index bdcf92c14135f..5ed0d2f92a2c1 100644 --- a/.github/workflows/_flagship-apps.yml +++ b/.github/workflows/_flagship-apps.yml @@ -29,7 +29,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: mcr.microsoft.com/playwright/python:v1.30.0-focal + image: mcr.microsoft.com/playwright/python:v1.32.1-focal strategy: fail-fast: false matrix: diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 380866b42580c..93d06894ecbde 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -67,16 +67,11 @@ jobs: env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 - run: | - pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip list + run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html if: inputs.pl_version == '' - name: Install PL version - run: | - pip install "pytorch-lightning==${{ inputs.pl_version }}" \ - -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip list + run: pip install "pytorch-lightning==${{ inputs.pl_version }}" -f https://download.pytorch.org/whl/cpu/torch_stable.html if: inputs.pl_version != '' - name: Adjust tests -> PL @@ -92,39 +87,31 @@ jobs: - name: Decide PL version to create a PR with id: decide-version - run: | - python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT + run: python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT - name: Generate checkpoints working-directory: ${{ env.legacy_dir }} - run: | - bash generate_checkpoints.sh ${{ inputs.pl_version }} + run: bash generate_checkpoints.sh ${{ inputs.pl_version }} - - name: Keep artifact - id: keep-artifact - run: python -c "print('DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT + - name: Keep artifact & DryRun + run: | + python -c "print('KEEP_DAYS=' + str(30 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_ENV + python -c "print('AWS_RUN=' + str('' if '${{inputs.push_to_s3}}' == 'true' else '--dryrun'))" >> $GITHUB_ENV - name: Upload checkpoints to GitHub Actions artifact uses: actions/upload-artifact@v3 with: name: checkpoints-${{ github.sha }} path: ${{ env.legacy_dir }}/checkpoints/ - retention-days: ${{ steps.keep-artifact.outputs.DAYS }} - - - name: Upload checkpoints to S3 (dryrun) - working-directory: ${{ env.legacy_dir }} - run: | - aws s3 sync --dryrun checkpoints/ s3://pl-public-data/legacy/checkpoints/ - zip -r checkpoints.zip checkpoints - aws s3 cp --dryrun checkpoints.zip s3://pl-public-data/legacy/ --acl public-read + retention-days: ${{ env.KEEP_DAYS }} - name: Upload checkpoints to S3 working-directory: ${{ env.legacy_dir }} run: | - aws s3 sync checkpoints/ s3://pl-public-data/legacy/checkpoints/ + aws s3 sync $AWS_RUN checkpoints/ s3://pl-public-data/legacy/checkpoints/ zip -r checkpoints.zip checkpoints - aws s3 cp checkpoints.zip s3://pl-public-data/legacy/ --acl public-read - if: inputs.push_to_s3 + aws s3 cp $AWS_RUN checkpoints.zip s3://pl-public-data/legacy/ --acl public-read + add-ckpt-test: runs-on: ubuntu-20.04 @@ -144,5 +131,6 @@ jobs: title: Adding test for legacy checkpoint created with ${{ needs.create-legacy-ckpts.outputs.pl-version }} delete-branch: true labels: | + checkpointing tests pl diff --git a/.github/workflows/ci-dockers-pytorch.yml b/.github/workflows/ci-dockers-pytorch.yml index 1d5eb9d62ea4f..5454b18ddff27 100644 --- a/.github/workflows/ci-dockers-pytorch.yml +++ b/.github/workflows/ci-dockers-pytorch.yml @@ -62,41 +62,41 @@ jobs: push: ${{ env.PUSH_RELEASE }} # pushed in release-docker.yml only when PL is released timeout-minutes: 50 - build-xla: - if: github.event.pull_request.draft == false - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - # the config used in '.github/workflows/tpu-tests.yml' - python_version: ["3.8"] - xla_version: ["1.12"] - steps: - - uses: actions/checkout@v3 - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - if: env.PUSH_NIGHTLY == 'true' - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - uses: docker/build-push-action@v4 - with: - build-args: | - PYTHON_VERSION=${{ matrix.python_version }} - XLA_VERSION=${{ matrix.xla_version }} - file: dockers/base-xla/Dockerfile - push: ${{ env.PUSH_NIGHTLY }} - tags: pytorchlightning/pytorch_lightning:base-xla-py${{ matrix.python_version }}-torch${{ matrix.xla_version }} - timeout-minutes: 60 - - uses: ravsamhq/notify-slack-action@v2 - if: failure() && env.PUSH_NIGHTLY == 'true' - with: - status: ${{ job.status }} - token: ${{ secrets.GITHUB_TOKEN }} - notification_title: ${{ format('XLA; {0} py{1} for *{2}*', runner.os, matrix.python_version, matrix.xla_version) }} - message_format: '{emoji} *{workflow}* {status_message}, see <{run_url}|detail>, cc: <@U01GD29QCAV>' # kaushikb11 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + #build-xla: + # if: github.event.pull_request.draft == false + # runs-on: ubuntu-20.04 + # strategy: + # fail-fast: false + # matrix: + # # the config used in '.github/workflows/tpu-tests.yml' + # python_version: ["3.8"] + # xla_version: ["1.12"] + # steps: + # - uses: actions/checkout@v3 + # - uses: docker/setup-buildx-action@v2 + # - uses: docker/login-action@v2 + # if: env.PUSH_NIGHTLY == 'true' + # with: + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # - uses: docker/build-push-action@v4 + # with: + # build-args: | + # PYTHON_VERSION=${{ matrix.python_version }} + # XLA_VERSION=${{ matrix.xla_version }} + # file: dockers/base-xla/Dockerfile + # push: ${{ env.PUSH_NIGHTLY }} + # tags: pytorchlightning/pytorch_lightning:base-xla-py${{ matrix.python_version }}-torch${{ matrix.xla_version }} + # timeout-minutes: 60 + # - uses: ravsamhq/notify-slack-action@v2 + # if: failure() && env.PUSH_NIGHTLY == 'true' + # with: + # status: ${{ job.status }} + # token: ${{ secrets.GITHUB_TOKEN }} + # notification_title: ${{ format('XLA; {0} py{1} for *{2}*', runner.os, matrix.python_version, matrix.xla_version) }} + # message_format: '{emoji} *{workflow}* {status_message}, see <{run_url}|detail>, cc: <@U01GD29QCAV>' # kaushikb11 + # env: + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} build-cuda: if: github.event.pull_request.draft == false @@ -221,19 +221,19 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - build-NGC: - if: github.event.pull_request.draft == false - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Build Conda Docker - # publish master/release - uses: docker/build-push-action@v4 - with: - file: dockers/nvidia/Dockerfile - push: false - timeout-minutes: 55 + #build-NGC: + # if: github.event.pull_request.draft == false + # runs-on: ubuntu-20.04 + # steps: + # - name: Checkout + # uses: actions/checkout@v3 + # - name: Build Conda Docker + # # publish master/release + # uses: docker/build-push-action@v4 + # with: + # file: dockers/nvidia/Dockerfile + # push: false + # timeout-minutes: 55 build-docs: if: github.event.pull_request.draft == false diff --git a/.github/workflows/ci-examples-app.yml b/.github/workflows/ci-examples-app.yml index 76c930b093357..d16c5d3750f0a 100644 --- a/.github/workflows/ci-examples-app.yml +++ b/.github/workflows/ci-examples-app.yml @@ -52,6 +52,7 @@ jobs: FREEZE_REQUIREMENTS: 1 TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" PYPI_CACHE_DIR: "_pip-wheels" + LIGHTING_TESTING: "1" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -74,8 +75,9 @@ jobs: - name: Install Lightning package & dependencies run: | + extra=$(python -c "print({'lightning': 'app-'}.get('${{ matrix.pkg-name }}', ''))") # do not use `-e` because it will make both packages available since it adds `src` to `sys.path` automatically - pip install .[dev] -U -f ${TORCH_URL} -f ${PYPI_CACHE_DIR} --prefer-binary + pip install ".[${extra}dev]" -U -f ${TORCH_URL} -f ${PYPI_CACHE_DIR} --prefer-binary pip list - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' diff --git a/.github/workflows/ci-flagship-apps.yml b/.github/workflows/ci-flagship-apps.yml index a21853ca47ddb..34230a53748ce 100644 --- a/.github/workflows/ci-flagship-apps.yml +++ b/.github/workflows/ci-flagship-apps.yml @@ -4,13 +4,14 @@ name: Test App - flagships on: push: branches: ["release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] - paths: - - ".github/workflows/ci-flagship-apps.yml" - - "github/actions/prep-apps/action.yml" - - "tests/integrations_app/flagship/**" + # todo: skip this as we are not validation them ATM + #pull_request: + # branches: [master, "release/*"] + # types: [opened, reopened, ready_for_review, synchronize] + # paths: + # - ".github/workflows/ci-flagship-apps.yml" + # - "github/actions/prep-apps/action.yml" + # - "tests/integrations_app/flagship/**" schedule: # on Sundays - cron: "0 0 * * 0" diff --git a/.github/workflows/ci-tests-app.yml b/.github/workflows/ci-tests-app.yml index b4c7f6d5c0945..57b127dacc264 100644 --- a/.github/workflows/ci-tests-app.yml +++ b/.github/workflows/ci-tests-app.yml @@ -47,11 +47,12 @@ jobs: - {os: "ubuntu-20.04", pkg-name: "app", python-version: "3.9", requires: "latest"} - {os: "windows-2022", pkg-name: "app", python-version: "3.8", requires: "latest"} # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 40 + timeout-minutes: 30 env: PACKAGE_NAME: ${{ matrix.pkg-name }} FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} PYPI_CACHE_DIR: "_pip-wheels" + LIGHTING_TESTING: "1" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -81,7 +82,8 @@ jobs: - name: Install package & dependencies run: | python -m pip install -q pip -U - pip install -e .[dev] -U -f ${TORCH_URL} -f ${PYPI_CACHE_DIR} --prefer-binary + extra=$(python -c "print({'lightning': 'app-'}.get('${{ matrix.pkg-name }}', ''))") + pip install -e ".[${extra}dev]" -U -f ${TORCH_URL} -f ${PYPI_CACHE_DIR} --prefer-binary pip list - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index f03c7ca8b4e65..0b8c562f6acef 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -84,8 +84,10 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }} run: | + pip install -q wget packaging + python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ - python ./requirements/pytorch/adjust-versions.py $fpath ${{ matrix.pytorch-version }}; \ + python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ done cat requirements/fabric/base.txt @@ -109,7 +111,8 @@ jobs: - name: Install package & dependencies run: | python -m pip install -q pip -U - pip install -e .[test] "pytest-timeout" -U -f ${TORCH_URL} ${TORCH_PREINSTALL} -f ${PYPI_CACHE_DIR} --prefer-binary + extra=$(python -c "print({'lightning': 'fabric-'}.get('${{ matrix.pkg-name }}', ''))") + pip install -e ".[${extra}test]" "pytest-timeout" -U -f ${TORCH_URL} ${TORCH_PREINSTALL} -f ${PYPI_CACHE_DIR} --prefer-binary pip install -r requirements/fabric/strategies.txt -f ${PYPI_CACHE_DIR} --prefer-binary pip list - name: Dump handy wheels diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index f999cdc9dfe01..0bc8c85002564 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -85,14 +85,13 @@ jobs: if: ${{ matrix.requires == 'oldest' }} run: python .actions/assistant.py replace_oldest_ver - - name: Pull legacy checkpoints - run: bash .actions/pull_legacy_checkpoints.sh - - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }} run: | + pip install -q wget packaging + python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ - python ./requirements/pytorch/adjust-versions.py $fpath ${{ matrix.pytorch-version }}; \ + python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ done cat requirements/pytorch/base.txt @@ -116,10 +115,17 @@ jobs: - name: Install package & dependencies run: | python -m pip install -q pip -U - pip install .[extra,test] -U \ - "pytest-timeout" \ + extra=$(python -c "print({'lightning': 'pytorch-'}.get('${{ matrix.pkg-name }}', ''))") + pip install ".[${extra}extra,${extra}test]" -U "pytest-timeout" \ -f ${TORCH_URL} ${TORCH_PREINSTALL} -f ${PYPI_CACHE_DIR} --prefer-binary pip list + - name: Drop LAI from extensions + if: ${{ matrix.pkg-name != 'lightning' }} + # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it + run: pip uninstall -y lightning + - name: Drop PL for LAI + if: ${{ matrix.pkg-name == 'lightning' }} + run: pip uninstall -y pytorch-lightning - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true @@ -148,6 +154,13 @@ jobs: --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" + - name: Get legacy checkpoints + run: | + bash .actions/pull_legacy_checkpoints.sh + cd tests/legacy + bash generate_checkpoints.sh + ls -l checkpoints/ + - name: Testing Warnings working-directory: tests/tests_pytorch # needs to run outside of `pytest` diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 3cc066fb380d9..2c3abccda6b7b 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -44,8 +44,7 @@ jobs: env: FREEZE_REQUIREMENTS: 1 run: | - # todo: adjust requirements for both code-bases - pip install -e '.[extra,ui,cloud]' -r requirements/typing.txt + pip install -e '.[all]' -r requirements/typing.txt pip list - name: Check typing diff --git a/.github/workflows/labeler-issue.yml b/.github/workflows/labeler-issue.yml new file mode 100644 index 0000000000000..c640f3578fd4d --- /dev/null +++ b/.github/workflows/labeler-issue.yml @@ -0,0 +1,34 @@ +name: Label Issues +on: + issues: + types: [opened, edited] + +permissions: + contents: read + +jobs: + label-component: + runs-on: ubuntu-latest + + permissions: + # required for all workflows + issues: write + + # only required for workflows in private repositories + actions: read + contents: read + + steps: + - uses: actions/checkout@v3 + + - name: Parse issue form + uses: stefanbuck/github-issue-parser@v3 + id: issue-parser + with: + template-path: .github/ISSUE_TEMPLATE/1_bug_report.yaml + + - name: Set labels based on severity field + uses: redhat-plumbers-in-action/advanced-issue-labeler@v2 + with: + issue-form: ${{ steps.issue-parser.outputs.jsonString }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler-pr.yml similarity index 76% rename from .github/workflows/labeler.yml rename to .github/workflows/labeler-pr.yml index 11cd619105e8e..e67d7e3e83943 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler-pr.yml @@ -1,6 +1,5 @@ name: Label Pull Requests -on: - - pull_request_target +on: [pull_request_target] jobs: triage: @@ -12,3 +11,4 @@ jobs: - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" + configuration-path: .github/labeler.yml diff --git a/.github/workflows/tpu-tests.yml b/.github/workflows/tpu-tests.yml index 85912db076ee6..3bdfbcb8d7541 100644 --- a/.github/workflows/tpu-tests.yml +++ b/.github/workflows/tpu-tests.yml @@ -142,7 +142,6 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 375d77d5ef4f8..17c18b06ee99a 100644 --- a/.gitignore +++ b/.gitignore @@ -53,9 +53,9 @@ wheels/ .installed.cfg *.egg src/*/version.info -src/lightning_app/ -src/lightning_fabric/ -src/pytorch_lightning/ +src/lightning_app/* +src/lightning_fabric/* +src/pytorch_lightning/* !src/*/__about__.py !src/*/__main__.py !src/*/__setup__.py @@ -63,7 +63,7 @@ src/pytorch_lightning/ !src/*/MANIFEST.in !src/*/py.typed !src/*/README.md -!src/*/*.info +!src/*/shell-folder_code-lives-lightning.info # PyInstaller # Usually these files are written by a python script from a template diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bde8e42de2f7f..e965aaf83bda4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: name: Upgrade code - repo: https://github.com/PyCQA/docformatter - rev: v1.4 + rev: v1.6.3 hooks: - id: docformatter args: [--in-place, --wrap-summaries=115, --wrap-descriptions=120] @@ -67,6 +67,12 @@ repos: hooks: - id: yesqa name: Unused noqa + additional_dependencies: + #- pep8-naming + - flake8-pytest-style + - flake8-bandit + - flake8-simplify + - flake8-return - repo: https://github.com/PyCQA/isort rev: 5.12.0 @@ -76,7 +82,7 @@ repos: exclude: docs/source-app - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.3.0 hooks: - id: black name: Format code @@ -106,7 +112,7 @@ repos: )$ - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.261' + rev: 'v0.0.262' hooks: - id: ruff args: ["--fix"] diff --git a/README.md b/README.md index db35544b5ccbc..ba27507cd4f17 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,12 @@ ______________________________________________________________________ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pytorch-lightning)](https://pypi.org/project/pytorch-lightning/) [![PyPI Status](https://badge.fury.io/py/pytorch-lightning.svg)](https://badge.fury.io/py/pytorch-lightning) [![PyPI Status](https://pepy.tech/badge/pytorch-lightning)](https://pepy.tech/project/pytorch-lightning) -[![Conda](https://img.shields.io/conda/v/conda-forge/pytorch-lightning?label=conda&color=success)](https://anaconda.org/conda-forge/pytorch-lightning) +[![Conda](https://img.shields.io/conda/v/conda-forge/lightning?label=conda&color=success)](https://anaconda.org/conda-forge/lightning) [![DockerHub](https://img.shields.io/docker/pulls/pytorchlightning/pytorch_lightning.svg)](https://hub.docker.com/r/pytorchlightning/pytorch_lightning) [![codecov](https://codecov.io/gh/Lightning-AI/lightning/branch/master/graph/badge.svg?token=SmzX8mnKlA)](https://codecov.io/gh/Lightning-AI/lightning) -[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=stable)](https://lightning.ai/docs/pytorch/stable/) [![Discord](https://img.shields.io/discord/1077906959069626439?style=plastic)](https://discord.gg/VptPCZkGNa) +![GitHub commit activity](https://img.shields.io/github/commit-activity/w/lightning-ai/lightning) [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/lightning/blob/master/LICENSE)