Merge branch 'master' into bugfix/collection_getitem

Lightning-AI · Apr 10, 2024 · 60fe120 · 60fe120
2 parents c66948b + e5b1135
commit 60fe120
Show file tree

Hide file tree

Showing 16 changed files with 66 additions and 59 deletions.
diff --git a/.azure/gpu-integrations.yml b/.azure/gpu-integrations.yml
@@ -19,11 +19,11 @@ jobs:
       matrix:
         "torch | 1.x":
           docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda11.8.0-py3.9-torch1.13"
-          torch-ver: "1.13.1"
+          torch-ver: "1.13"
           requires: "oldest"
         "torch | 2.x":
-          docker-image: "pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime"
-          torch-ver: "2.2.1"
+          docker-image: "pytorch/pytorch:2.2.2-cuda12.1-cudnn8-runtime"
+          torch-ver: "2.2"
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "40"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -91,7 +91,7 @@ jobs:
       - bash: |
           set -e
           pip list
-          python -c "from torch import __version__ as ver ; assert str(ver).split('+')[0] == '$(torch-ver)', f'PyTorch: {ver}'"
+          python -c "from torch import __version__ as ver ; assert '.'.join(str(ver).split('.')[:2]) == '$(torch-ver)', f'PyTorch: {ver}'"
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'found GPUs: {mgpu}'"
         displayName: "Sanity check"
 

diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml
@@ -17,16 +17,19 @@ jobs:
   - job: unitest_GPU
     strategy:
       matrix:
-        "PyTorch | 1.10":
+        "PyTorch | 1.10 oldest":
           # Torch does not have build wheels with old Torch versions for newer CUDA
-          docker-image: "pytorchlightning/torchmetrics:ubuntu20.04-cuda11.3.1-py3.9-torch1.10"
-          torch-ver: "1.10.2"
-        "PyTorch | 1.X":
-          docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda11.8.0-py3.9-torch1.13"
-          torch-ver: "1.13.1"
-        "PyTorch | 2.X":
-          docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda12.1.1-py3.11-torch2.2"
-          torch-ver: "2.2.1"
+          docker-image: "ubuntu20.04-cuda11.3.1-py3.9-torch1.10"
+          torch-ver: "1.10"
+        "PyTorch | 1.X LTS":
+          docker-image: "ubuntu22.04-cuda11.8.0-py3.9-torch1.13"
+          torch-ver: "1.13"
+        "PyTorch | 2.X stable":
+          docker-image: "ubuntu22.04-cuda12.1.1-py3.11-torch2.2"
+          torch-ver: "2.2"
+        "PyTorch | 2.X future":
+          docker-image: "ubuntu22.04-cuda12.1.1-py3.11-torch2.3"
+          torch-ver: "2.3"
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "180"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -47,11 +50,12 @@ jobs:
       # MKL_THREADING_LAYER: "GNU"
       MKL_SERVICE_FORCE_INTEL: "1"
       TEST_DIRS: "unittests"
+      CACHED_REFERENCES: "/var/tmp/cached-references.zip"
       # todo: consider unfreeze for master too
       FREEZE_REQUIREMENTS: 1
 
     container:
-      image: "$(docker-image)"
+      image: "pytorchlightning/torchmetrics:$(docker-image)"
       options: "--gpus=all --shm-size=8g -v /var/tmp:/var/tmp"
 
     workspace:
@@ -119,24 +123,30 @@ jobs:
           set -e
           pip list
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'found GPUs: {mgpu}'"
-          python -c "from torch import __version__ as ver ; assert str(ver).split('+')[0] == '$(torch-ver)', f'PyTorch: installed {ver} but expected $(torch-ver)'"
+          python -c "from torch import __version__ as ver ; assert '.'.join(str(ver).split('.')[:2]) == '$(torch-ver)', f'PyTorch: installed {ver} but expected $(torch-ver)'"
         displayName: "Sanity check"
 
       - bash: |
           pip install -q py-tree
           py-tree /var/tmp/torch
           py-tree /var/tmp/hf
-          # this gives more the 60k lines and takes a few minutes to run
-          #py-tree $(PYTEST_REFERENCE_CACHE) --show_hidden
-          # make sure the cache exists even it is empty
-          mkdir -p /var/tmp/cached-references
         displayName: "Show caches"
 
       - bash: |
-          cp -r /var/tmp/cached-references tests/_cache-references
+          # Check if the file references exists
+          if [ -f $(CACHED_REFERENCES) ]; then
+              # Create a directory if it doesn't already exist
+              mkdir -p tests/_cache-references
+              # Unzip 'ref.zip' into the directory inside tests folder
+              unzip -q $(CACHED_REFERENCES) -d tests/_cache-references
+              ls -lh tests/_cache-references/
+          else
+              echo "The file '$(CACHED_REFERENCES)' does not exist."
+          fi
           du -h --max-depth=1 tests/
-        condition: eq(variables['Build.Reason'], 'PullRequest')
+        timeoutInMinutes: "5"
         # if pull request, copy the cache to the tests folder to be used in the next steps
+        condition: eq(variables['Build.Reason'], 'PullRequest')
         displayName: "Copy cached refs"
 
       - bash: |
@@ -147,23 +157,23 @@ jobs:
         env:
           DOCTEST_DOWNLOAD_TIMEOUT: "180"
           SKIP_SLOW_DOCTEST: "1"
-        workingDirectory: src
+        workingDirectory: "src/"
         timeoutInMinutes: "40"
         displayName: "DocTesting"
 
       - bash: |
           wget https://pl-public-data.s3.amazonaws.com/metrics/data.zip
           unzip -o data.zip
           ls -l _data/*
-        workingDirectory: tests
+        workingDirectory: "tests/"
         displayName: "Pull testing data from S3"
 
       - bash: |
           python -m pytest $(TEST_DIRS) \
             -m "not DDP" --numprocesses=5 --dist=loadfile \
             --cov=torchmetrics --timeout=240 --durations=100 \
             --reruns 3 --reruns-delay 1
-        workingDirectory: tests
+        workingDirectory: "tests/"
         # skip for PR if there is nothing to test, note that outside PR there is default 'unittests'
         condition: and(succeeded(), ne(variables['TEST_DIRS'], ''))
         timeoutInMinutes: "90"
@@ -175,16 +185,16 @@ jobs:
             --timeout=240 --durations=100
         env:
           USE_PYTEST_POOL: "1"
-        workingDirectory: tests
+        workingDirectory: "tests/"
         # skip for PR if there is nothing to test, note that outside PR there is default 'unittests'
         condition: and(succeeded(), ne(variables['TEST_DIRS'], ''))
         timeoutInMinutes: "90"
         displayName: "UnitTesting DDP"
 
       - bash: |
+          # archive potentially updated cache to the machine filesystem to be reused with next jobs
+          zip -q -r $(CACHED_REFERENCES) tests/_cache-references
           du -h --max-depth=1 tests/
-          # copy potentially updated cache to the machine filesystem to be reused with next jobs
-          cp -r --update tests/_cache-references /var/tmp/cached-references
         # set as extra step to not pollute general cache when jobs fails or crashes
         # so do this update only with successful jobs on master
         condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
@@ -196,7 +206,7 @@ jobs:
           python -m codecov --token=$(CODECOV_TOKEN) --name="GPU-coverage" \
             --commit=$(Build.SourceVersion) --flags=gpu,unittest --env=linux,azure
           ls -l
-        workingDirectory: tests
+        workingDirectory: "tests/"
         # skip for PR if there is nothing to test, note that outside PR there is default 'unittests'
         condition: and(succeeded(), ne(variables['TEST_DIRS'], ''))
         displayName: "Statistics"
@@ -209,7 +219,7 @@ jobs:
             echo "Processing $fn example..."
             python $fn
           done
-        workingDirectory: examples
+        workingDirectory: "examples/"
         # skip for PR if there is nothing to test, note that outside PR there is default 'unittests'
         condition: and(succeeded(), ne(variables['TEST_DIRS'], ''))
         displayName: "Examples"

diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml
@@ -13,19 +13,19 @@ concurrency:
 
 jobs:
   check-code:
-    uses: Lightning-AI/utilities/.github/workflows/check-typing.yml@v0.10.1
+    uses: Lightning-AI/utilities/.github/workflows/check-typing.yml@v0.11.2
     with:
-      actions-ref: v0.10.1
+      actions-ref: v0.11.2
       extra-typing: "typing"
 
   check-schema:
-    uses: Lightning-AI/utilities/.github/workflows/check-schema.yml@v0.10.1
+    uses: Lightning-AI/utilities/.github/workflows/check-schema.yml@v0.11.2
 
   check-package:
     if: github.event.pull_request.draft == false
-    uses: Lightning-AI/utilities/.github/workflows/check-package.yml@v0.10.1
+    uses: Lightning-AI/utilities/.github/workflows/check-package.yml@v0.11.2
     with:
-      actions-ref: v0.10.1
+      actions-ref: v0.11.2
       artifact-name: dist-packages-${{ github.sha }}
       import-name: "torchmetrics"
       testing-matrix: |
@@ -35,7 +35,7 @@ jobs:
         }
 
   check-md-links:
-    uses: Lightning-AI/utilities/.github/workflows/check-md-links.yml@v0.10.1
+    uses: Lightning-AI/utilities/.github/workflows/check-md-links.yml@v0.11.2
     with:
       base-branch: master
       config-file: ".github/markdown-links-config.json"
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -40,24 +40,24 @@ jobs:
           - "1.13.1"
           - "2.0.1"
           - "2.1.2"
-          - "2.2.1"
+          - "2.2.2"
         include:
           # cover additional python nad PR combinations
           - { os: "ubuntu-22.04", python-version: "3.8", pytorch-version: "1.13.1" }
           - { os: "ubuntu-22.04", python-version: "3.10", pytorch-version: "2.0.1" }
-          - { os: "ubuntu-22.04", python-version: "3.10", pytorch-version: "2.2.1" }
-          - { os: "ubuntu-22.04", python-version: "3.11", pytorch-version: "2.2.1" }
+          - { os: "ubuntu-22.04", python-version: "3.10", pytorch-version: "2.2.2" }
+          - { os: "ubuntu-22.04", python-version: "3.11", pytorch-version: "2.2.2" }
           # standard mac machine, not the M1
           - { os: "macOS-12", python-version: "3.8", pytorch-version: "1.13.1" }
           - { os: "macOS-12", python-version: "3.10", pytorch-version: "2.0.1" }
-          - { os: "macOS-12", python-version: "3.11", pytorch-version: "2.2.1" }
+          - { os: "macOS-12", python-version: "3.11", pytorch-version: "2.2.2" }
           # using the ARM based M1 machine
           - { os: "macOS-14", python-version: "3.10", pytorch-version: "2.0.1" }
-          - { os: "macOS-14", python-version: "3.11", pytorch-version: "2.2.1" }
+          - { os: "macOS-14", python-version: "3.11", pytorch-version: "2.2.2" }
           # some windows
           - { os: "windows-2022", python-version: "3.8", pytorch-version: "1.13.1" }
           - { os: "windows-2022", python-version: "3.10", pytorch-version: "2.0.1" }
-          - { os: "windows-2022", python-version: "3.11", pytorch-version: "2.2.1" }
+          - { os: "windows-2022", python-version: "3.11", pytorch-version: "2.2.2" }
     env:
       PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
       FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}

diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml
@@ -12,12 +12,12 @@ on:
 jobs:
   cron-clear:
     if: github.event_name == 'schedule'
-    uses: Lightning-AI/utilities/.github/workflows/clear-cache.yml@v0.10.1
+    uses: Lightning-AI/utilities/.github/workflows/clear-cache.yml@v0.11.2
     with:
       pattern: "pip-latest"
 
   direct-clear:
     if: github.event_name == 'workflow_dispatch'
-    uses: Lightning-AI/utilities/.github/workflows/clear-cache.yml@v0.10.1
+    uses: Lightning-AI/utilities/.github/workflows/clear-cache.yml@v0.11.2
     with:
       pattern: ${{ inputs.pattern }}
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -67,13 +67,11 @@ jobs:
           # These are the base images for PL release docker images,
           #  so include at least all the combinations in release-dockers.yml.
           - { python: "3.9", pytorch: "1.10", cuda: "11.3.1", ubuntu: "20.04" }
-          - { python: "3.9", pytorch: "1.11", cuda: "11.8.0", ubuntu: "22.04" }
+          #- { python: "3.9", pytorch: "1.11", cuda: "11.8.0", ubuntu: "22.04" }
           - { python: "3.9", pytorch: "1.13", cuda: "11.8.0", ubuntu: "22.04" }
-          - { python: "3.10", pytorch: "2.0", cuda: "11.8.0", ubuntu: "22.04" }
-          - { python: "3.10", pytorch: "2.0", cuda: "12.1.1", ubuntu: "22.04" }
-          - { python: "3.10", pytorch: "2.1", cuda: "12.1.1", ubuntu: "22.04" }
-          - { python: "3.11", pytorch: "2.1", cuda: "12.1.1", ubuntu: "22.04" }
+          - { python: "3.10", pytorch: "2.2", cuda: "12.1.1", ubuntu: "22.04" }
           - { python: "3.11", pytorch: "2.2", cuda: "12.1.1", ubuntu: "22.04" }
+          - { python: "3.11", pytorch: "2.3", cuda: "12.1.1", ubuntu: "22.04" }
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/publish-pkg.yml b/.github/workflows/publish-pkg.yml
@@ -67,7 +67,7 @@ jobs:
       - run: ls -lh dist/
       # We do this, since failures on test.pypi aren't that bad
       - name: Publish to Test PyPI
-        uses: pypa/gh-action-pypi-publish@v1.8.12
+        uses: pypa/gh-action-pypi-publish@v1.8.14
         with:
           user: __token__
           password: ${{ secrets.test_pypi_password }}
@@ -94,7 +94,7 @@ jobs:
           path: dist
       - run: ls -lh dist/
       - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.8.12
+        uses: pypa/gh-action-pypi-publish@v1.8.14
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -83,6 +83,7 @@ repos:
     rev: v3.1.0
     hooks:
       - id: prettier
+        files: \.(json|yml|yaml|toml)
         # https://prettier.io/docs/en/options.html#print-width
         args: ["--print-width=120"]
 

diff --git a/.prettierignore b/.prettierignore
diff --git a/requirements/audio.txt b/requirements/audio.txt
@@ -4,5 +4,5 @@
 # this need to be the same as used inside speechmetrics
 pesq @ git+https://github.com/ludlows/python-pesq
 pystoi >=0.3.0, <0.5.0
-torchaudio >=0.10.0, <=2.2.1
+torchaudio >=0.10.0, <2.3.0
 gammatone @ https://github.com/detly/gammatone/archive/master.zip#egg=Gammatone
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -3,6 +3,6 @@
 
 numpy >1.20.0
 packaging >17.1
-torch >=1.10.0, <=2.2.1
+torch >=1.10.0, <2.3.0
 typing-extensions; python_version < '3.9'
 lightning-utilities >=0.8.0, <0.12.0
diff --git a/requirements/multimodal.txt b/requirements/multimodal.txt
@@ -1,5 +1,5 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-transformers >=4.10.0, <4.39.0
+transformers >=4.10.0, <4.40.0
 piq <=0.8.0
diff --git a/requirements/text.txt b/requirements/text.txt
@@ -4,7 +4,7 @@
 nltk >=3.6, <=3.8.1
 tqdm >=4.41.0, <4.67.0
 regex >=2021.9.24, <=2023.12.25
-transformers >4.4.0, <4.39.0
+transformers >4.4.0, <4.40.0
 mecab-python3 >=1.0.6, <1.1.0
 mecab-ko >=1.0.0, <1.1.0
 mecab-ko-dic >=1.0.0, <1.1.0

diff --git a/requirements/text_test.txt b/requirements/text_test.txt
@@ -4,5 +4,5 @@
 jiwer >=2.3.0, <3.1.0
 rouge-score >0.1.0, <=0.1.2
 bert_score ==0.3.13
-huggingface-hub <0.22
+huggingface-hub <0.23
 sacrebleu >=2.3.0, <2.5.0
diff --git a/requirements/typing.txt b/requirements/typing.txt
@@ -1,5 +1,5 @@
 mypy ==1.9.0
-torch ==2.2.1
+torch ==2.2.2
 
 types-PyYAML
 types-emoji

diff --git a/src/torchmetrics/classification/specificity.py b/src/torchmetrics/classification/specificity.py
@@ -453,7 +453,7 @@ class Specificity(_ClassificationTaskWrapper):
     .. math:: \text{Specificity} = \frac{\text{TN}}{\text{TN} + \text{FP}}
 
     Where :math:`\text{TN}` and :math:`\text{FP}` represent the number of true negatives and false positives
-    respectively. The metric is only proper defined when :math:`\text{TP} + \text{FP} \neq 0`. If this case is
+    respectively. The metric is only proper defined when :math:`\text{TN} + \text{FP} \neq 0`. If this case is
     encountered for any class/label, the metric for that class/label will be set to 0 and the overall metric may
     therefore be affected in turn.