bump min py3.8 (#1021)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.azure/gpu-tests.yml

@@ -13,6 +13,16 @@ pr:
 
 jobs:
   - job: testing
+    strategy:
+      matrix:
+        'torch v1.13 | py3.8':
+          PYTHON_VERSION: '3.8'
+          TORCH_VERSION: '1.13.1'
+          CUDA_VERSION_MM: '117'
+        'torch v2.0 | py3.9':
+          PYTHON_VERSION: '3.9'
+          TORCH_VERSION: '2.0.1'
+          CUDA_VERSION_MM: '117'
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "45"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -28,63 +38,80 @@ jobs:
 
     container:
       #image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
-      image: "pytorch/pytorch:1.12.1-cuda11.3-cudnn8-runtime"
+      #image: "pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime"
+      image: "nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu20.04"
       #endpoint: azureContainerRegistryConnection
       options: "--gpus=all --shm-size=16g -v /usr/bin/docker:/tmp/docker:ro -v /var/tmp:/var/tmp"
-    pool: lit-rtx-3090
+    pool: "lit-rtx-3090"
 
     workspace:
       clean: all
 
     steps:
 
-    - bash: |
-        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
-        echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)"
-        ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-      displayName: 'Set environment variables'
-
     - script: |
-        /tmp/docker exec -t -u 0 $CONTAINER_ID \
+        container_id=$(head -1 /proc/self/cgroup|cut -d/ -f3)
+        /tmp/docker exec -t -u 0 $container_id \
           sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+        echo "##vso[task.setvariable variable=CONTAINER_ID]$container_id"
       displayName: 'Install Sudo in container (thanks Microsoft!)'
 
+    - bash: |
+        set -e
+        #add-apt-repository ppa:deadsnakes/ppa
+        apt-get -y update -qq --fix-missing
+        apt-get -y install --no-install-recommends \
+          build-essential \
+          python$PYTHON_VERSION \
+          python$PYTHON_VERSION-dev \
+          python3-distutils \
+          unrar \
+          cmake \
+          pciutils \
+          swig \
+          zlib1g-dev \
+          wget curl
+        update-alternatives --install /usr/bin/python python /usr/bin/python$PYTHON_VERSION 1
+
+        wget https://bootstrap.pypa.io/get-pip.py
+        python get-pip.py
+
+        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
+      displayName: 'Install sys & python'  # CUDA image if completely blind
+
     - bash: |
         whereis nvidia
         nvidia-smi
         echo $CUDA_VISIBLE_DEVICES
         echo $CONTAINER_ID
         python --version
+        pip --version
         pip list
       displayName: 'Image info & NVIDIA'
 
-    - bash: |
-        python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
-      displayName: 'Sanity check'
+    #- bash: |
+    #    # Box2D requirement for swig
+    #    sudo apt-get remove -y swig
+    #    sudo apt-get install -y swig3.0
+    #    sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
+    #  displayName: 'bypass/fix swig'
 
     - bash: |
-        set -e
-        # Box2D requirement for swig
-        sudo apt-get remove -y swig
-        sudo apt-get install -y swig3.0
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-
-        sudo apt-get update --fix-missing
-        sudo apt-get install -y curl unrar cmake zlib1g-dev
-
-        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
-        TORCH_VERSION=$(python -c "import torch ; print(torch.__version__)")
-
         pip install "pip==21.2.3"  # todo: drop after resolving extras
-        pip install . "torch==$TORCH_VERSION" -r ./requirements/devel.txt \
-            --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
-        pip list
+        pip install ".[dev]" "torch==$TORCH_VERSION" \
+          --prefer-binary -f https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
       displayName: 'Install pkg & dependencies'
 
+    - bash: |
+        pip --version
+        pip list
+        python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
+      displayName: 'Sanity check'
+
     - script: |
         set -ex
-        mkdir -p datasets
-        cd datasets
+        mkdir -p _datasets
+        cd _datasets
         curl http://www.atarimania.com/roms/Roms.rar -o Roms.rar
         unrar x -y Roms.rar
         python -m atari_py.import_roms ROMS
@@ -111,14 +138,18 @@ jobs:
         testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)'
       condition: succeededOrFailed()
 
-#    - task: PublishCodeCoverageResults@1
-#      displayName: 'Publish coverage report'
-#      inputs:
-#        codeCoverageTool: 'Cobertura'
-#        summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
-#        reportDirectory: '$(Build.SourcesDirectory)/htmlcov'
-#        testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
-#      condition: succeededOrFailed()
+    #- task: PublishCodeCoverageResults@1
+    #  displayName: 'Publish coverage report'
+    #  inputs:
+    #    codeCoverageTool: 'Cobertura'
+    #    summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
+    #    reportDirectory: '$(Build.SourcesDirectory)/htmlcov'
+    #    testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
+    #  condition: succeededOrFailed()
+
+    - bash: |
+        python -m pytest tests/models/test_detection.py tests/models/yolo -v
+      displayName: 'Test Detection'
 
     - bash: rm -rf $(Build.SourcesDirectory)
       condition: succeededOrFailed()

.github/workflows/ci-tests.yml

@@ -24,7 +24,7 @@ jobs:
         # Something is wrong with windows tests, will need to debug separately.
         os: [ubuntu-20.04, macOS-11]
         # todo: seems there are some issues with py3.10 while install dependencies
-        python-version: ["3.7", "3.9"]
+        python-version: ["3.8", "3.9"]
         requires: ['oldest', 'latest']
     env:
       FREEZE_REQUIREMENTS: 1
@@ -66,7 +66,8 @@ jobs:
     - name: Install package
       run: |
         pip install "pip==22.2.1"  # todo: drop after resolving extras
-        pip install -e . "coverage[toml]" --upgrade --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -e . "coverage[toml]" --upgrade --prefer-binary \
+          -f https://download.pytorch.org/whl/cpu/torch_stable.html
         pip list
 
     - name: Test Package [only]
@@ -94,21 +95,21 @@ jobs:
 
     - name: Install dependencies
       run: |
-        pip install --requirement requirements/devel.txt --upgrade --quiet \
+        pip install -r requirements/devel.txt --upgrade --quiet \
           --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
         pip list
 
     - name: Cache datasets
       uses: actions/cache@v3
       with:
-        path: ./datasets
+        path: ./_datasets
         key: pl-datasets-${{ hashFiles('tests/conftest.py') }}
 
     - name: Download ROMs
       # continue-on-error: true
       run: |
-        mkdir -p datasets
-        cd datasets
+        mkdir -p _datasets
+        cd _datasets
         curl http://www.atarimania.com/roms/Roms.rar -o Roms.rar
         unrar x -y Roms.rar
         python -m atari_py.import_roms ROMS

.gitignore

@@ -145,6 +145,7 @@ dmypy.json
 .DS_Store
 
 # Datasets
+/_datasets/
 /datasets/
 MNIST/
 CIFAR10/

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     rev: v3.4.0
     hooks:
       - id: pyupgrade
-        args: [--py37-plus]
+        args: [--py38-plus]
         name: Upgrade code
 
   - repo: https://github.com/PyCQA/docformatter

CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Improved YOLO model includes YOLOv4, YOLOv5, and YOLOX networks and training algorithms ([#552](https://github.com/PyTorchLightning/pytorch-lightning-bolts/pull/817))
 
 
+- Bumped support of min python version to py3.8 ([#1021](https://github.com/PyTorchLightning/pytorch-lightning-bolts/pull/1021))
+
+
 ### Deprecated
 
 

Makefile

@@ -13,7 +13,7 @@ clean:
 	rm -rf ./docs/source/generated
 	rm -rf ./docs/source/*/generated
 	rm -rf ./docs/source/api
-	rm -rf ./datasets
+	rm -rf ./_datasets
 
 test: clean env
 

requirements/models.txt

@@ -1,7 +1,7 @@
-torchvision >=0.10
+torchvision >=0.10.0
 scikit-learn >=1.0.2
 Pillow >9.0.0
 gym[atari] >=0.17.2, <0.20.0  # strict
-atari_py >0.2, <0.3  # strict
+atari-py >0.2, !=0.2.6, <0.3  # strict
 box2d-py >2.3, <2.4  # strict
 opencv-python-headless >=4.5.5.62

requirements/test.txt

@@ -1,9 +1,8 @@
-coverage[toml] >7.0
-pytest >7.0.0, <=7.3.1
-pytest-cov >=3.0.0, <5.0.0
-# pytest-flake8
-atari-py ==0.2.9  # needed for RL
+coverage[toml] >7.0.0, <8.0.0
+pytest ==7.3.1
+pytest-cov ==4.1.0
+
 scikit-learn >=1.0.2, <=1.2.2
-sparseml >1.0.0, <=1.2.0
+sparseml >1.0.0, <1.3.0
 ale-py >=0.7, <=0.8.1
 jsonargparse[signatures] >4.0.0  # for LightningCLI

setup.py

@@ -151,7 +151,7 @@ def _prepare_extras():
     include_package_data=True,
     zip_safe=False,
     keywords=["deep learning", "pytorch", "AI"],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     setup_requires=["wheel"],
     install_requires=_load_requirements(_PATH_ROOT, "requirements.txt"),
     extras_require=_prepare_extras(),
@@ -177,8 +177,8 @@ def _prepare_extras():
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
 )

tests/__init__.py

@@ -5,7 +5,7 @@
 
 TEST_ROOT = os.path.realpath(os.path.dirname(__file__))
 PACKAGE_ROOT = os.path.dirname(TEST_ROOT)
-DATASETS_PATH = os.path.join(PACKAGE_ROOT, "datasets")
+DATASETS_PATH = os.path.join(PACKAGE_ROOT, "_datasets")
 # generate a list of random seeds for each test
 ROOT_SEED = 1234
 

tests/callbacks/test_data_monitor.py

@@ -11,9 +11,11 @@
 from pl_bolts.models import LitMNIST
 
 
-@pytest.mark.parametrize(("log_every_n_steps", "max_steps", "expected_calls"), [pytest.param(3, 10, 3)])
+# @pytest.mark.parametrize(("log_every_n_steps", "max_steps", "expected_calls"), [pytest.param(3, 10, 3)])
 @mock.patch("pl_bolts.callbacks.data_monitor.DataMonitorBase.log_histogram")
-def test_base_log_interval_override(log_histogram, tmpdir, log_every_n_steps, max_steps, expected_calls, datadir):
+def test_base_log_interval_override(
+    log_histogram, tmpdir, datadir, log_every_n_steps=3, max_steps=10, expected_calls=3
+):
     """Test logging interval set by log_every_n_steps argument."""
     monitor = TrainingDataMonitor(log_every_n_steps=log_every_n_steps)
     model = LitMNIST(num_workers=0)

tests/conftest.py

@@ -46,6 +46,7 @@ def restore_env_variables():  # noqa: PT004
     allowlist = {
         "CUBLAS_WORKSPACE_CONFIG",  # enabled with deterministic flag
         "CUDA_DEVICE_ORDER",
+        "CUDA_MODULE_LOADING",
         "LOCAL_RANK",
         "NODE_RANK",
         "WORLD_SIZE",