ci: skip dataset downloads (#1023)

Lightning-Universe · May 31, 2023 · a746a76 · a746a76
1 parent 514b22a
commit a746a76
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 31 deletions.
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -28,6 +28,7 @@ jobs:
         requires: ['oldest', 'latest']
     env:
       FREEZE_REQUIREMENTS: 1
+      TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     # the reason for high number is MUCH slower tests on macOS and py3.8
     timeout-minutes: 50
@@ -39,20 +40,6 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
-    - name: Setup macOS
-      if: runner.os == 'macOS'
-      run: |
-        brew update  # Todo: find a better way...
-        brew install libomp  # https://github.com/pytorch/pytorch/issues/20030
-        brew install rar
-
-    - name: Setup Ubuntu
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y unrar
-
     - name: Set min. dependencies
       if: matrix.requires == 'oldest'
       run: |
@@ -66,19 +53,17 @@ jobs:
     - name: Install package
       run: |
         pip install "pip==22.2.1"  # todo: drop after resolving extras
-        pip install -e . "coverage[toml]" --upgrade --prefer-binary \
-          -f https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -e . -U --prefer-binary -f ${TORCH_URL}
         pip list
 
     - name: Test Package [only]
       working-directory: ./src
       run: |
-        pip install pytest -q
+        pip install "coverage[toml]" pytest -q
         # TODO: package shall be fine to run full without any ignores
         python -m pytest . \
           --ignore=pl_bolts/datamodules \
           --ignore=pl_bolts/datasets \
-          --ignore=pl_bolts/models/self_supervised/amdim/transforms.py \
           --ignore=pl_bolts/models/rl
 
     - name: Get pip cache dir
@@ -89,34 +74,50 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements/*.txt') }}
+        key: ${{ runner.os }}-pip-${{ matrix.requires }}-${{ hashFiles('requirements/*.txt') }}
         restore-keys: |
-          ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-
+          ${{ runner.os }}-pip-${{ matrix.requires }}-
 
     - name: Install dependencies
       run: |
-        pip install -r requirements/devel.txt --upgrade --quiet \
-          --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -r requirements/devel.txt -U -q -f ${TORCH_URL}
         pip list
 
     - name: Cache datasets
+      id: cache-datasets
       uses: actions/cache@v3
       with:
         path: ./_datasets
-        key: pl-datasets-${{ hashFiles('tests/conftest.py') }}
+        # bump this date if you need update cache
+        key: datasets-20230601
+
+    - name: Setup macOS
+      if: runner.os == 'macOS' && steps.cache-datasets.outputs.cache-hit != 'true'
+      run: |
+        brew update
+        brew install rar
+
+    - name: Setup Ubuntu
+      if: runner.os == 'Linux'&& steps.cache-datasets.outputs.cache-hit != 'true'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y unrar
 
     - name: Download ROMs
-      # continue-on-error: true
+      if: steps.cache-datasets.outputs.cache-hit != 'true'
       run: |
         mkdir -p _datasets
         cd _datasets
         curl http://www.atarimania.com/roms/Roms.rar -o Roms.rar
         unrar x -y Roms.rar
-        python -m atari_py.import_roms ROMS
+        rm Roms.rar
+
+    - name: Init ROMs
+      working-directory: ./_datasets
+      run: python -m atari_py.import_roms ROMS
 
     - name: Tests
-      run: |
-        python -m pytest tests -v --cov=pl_bolts
+      run: python -m pytest tests -v --cov=pl_bolts
 
     - name: Statistics
       if: success()

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -4,8 +4,8 @@
 from pytorch_lightning import seed_everything
 
 TEST_ROOT = os.path.realpath(os.path.dirname(__file__))
-PACKAGE_ROOT = os.path.dirname(TEST_ROOT)
-DATASETS_PATH = os.path.join(PACKAGE_ROOT, "_datasets")
+PROJECT_ROOT = os.path.dirname(TEST_ROOT)
+DATASETS_PATH = os.path.join(PROJECT_ROOT, "_datasets")
 # generate a list of random seeds for each test
 ROOT_SEED = 1234
 

diff --git a/tests/test_utilities.py b/tests/test_utilities.py
@@ -1,7 +1,7 @@
 import os
 
-from tests import PACKAGE_ROOT
+from tests import PROJECT_ROOT
 
 
 def test_paths():
-    assert os.path.isdir(PACKAGE_ROOT)
+    assert os.path.isdir(PROJECT_ROOT)