Cache test data in Actions

.github/workflows/docker.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build_and_pub_docker:
-    runs-on: [ ubuntu-latest ]
+    runs-on: [ ubuntu-22.04 ]
     steps:
       - name: Login to docker.io
         run: echo ${{ secrets.DOCKER_PWD }} | docker login -u ${{ secrets.DOCKER_LOGIN }} --password-stdin

.github/workflows/examples.yml

@@ -17,17 +17,33 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-22.04]
         python: ["3.8"]
         minimal: [false]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        id: cache-bikes-dataset
+        env:
+          cache-name: cache-bikes-dataset
+        with:
+          path: Bike-Sharing-Dataset.zip
+          key: cache-bikes-dataset
+      - name: Download test data
+        if: ${{ steps.cache-bikes-dataset.outputs.cache-hit != 'true' }}
+        run:
+          curl -k https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip -o Bike-Sharing-Dataset.zip
+      - name: Prepare test data
+        if: ${{ steps.cache-bikes-dataset.outputs.cache-hit != 'true' }}
+        run:
+          unzip Bike-Sharing-Dataset.zip -d Bike-Sharing-Dataset
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           architecture: "x64"
           cache: "pip"
+          cache-dependency-path: setup.py
       - name: Install package
         run: pip install -e .[dev,spark,fsspec]
       - name: Install minimal reqs
@@ -37,9 +53,5 @@ jobs:
         run: pip install catboost sentence-transformers
       - name: Export examples
         run: jupyter nbconvert --to python examples/*/*.ipynb --output-dir example_scripts
-      - name: Download test data
-        run:
-          curl -k https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip -o Bike-Sharing-Dataset.zip &&
-          unzip Bike-Sharing-Dataset.zip -d Bike-Sharing-Dataset
       - name: Run examples
         run: python example_test.py

.github/workflows/main.yml

@@ -25,7 +25,7 @@ concurrency:
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   changed_files:
-    runs-on: ubuntu-latest # windows-latest || macos-latest
+    runs-on: ubuntu-22.04 # windows-latest || macos-latest
     name: 👀 List Changed Evidently Files
     outputs:
       evidently_any_modified: ${{ steps.changed-files.outputs.evidently_python_any_modified == 'true' }}
@@ -59,7 +59,7 @@ jobs:
 
   linter:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: changed_files
     if: ${{ github.event.pull_request.draft == false && needs.changed_files.outputs.evidently_any_modified == 'true' }}
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -71,6 +71,7 @@ jobs:
           python-version: "3.9"
           architecture: "x64"
           cache: "pip"
+          cache-dependency-path: requirements.min.txt
       # Runs a single command using the runners shell
       - name: Install dependencies
         run: pip install -r requirements.min.txt && pip install -r requirements.dev.txt
@@ -79,10 +80,30 @@ jobs:
       - name: Run Mypy
         run: mypy
 
+  prepare-cache-data:
+    name: Prepare cache data
+    runs-on: ubuntu-22.04
+    needs: changed_files
+    if: ${{ github.event.pull_request.draft == false && needs.changed_files.outputs.evidently_any_modified == 'true' }}
+    steps:
+      - uses: actions/cache@v3
+        id: cache-bikes-dataset
+        env:
+          cache-name: cache-bikes-dataset
+        with:
+          path: Bike-Sharing-Dataset.zip
+          key: cache-bikes-dataset
+      - name: Download test data
+        if: ${{ steps.cache-bikes-dataset.outputs.cache-hit != 'true' }}
+        run:
+          curl -k https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip -o Bike-Sharing-Dataset.zip
+
   test-minimal:
     name: Test on minimal requirements
-    runs-on: ubuntu-latest
-    needs: changed_files
+    runs-on: ubuntu-22.04
+    needs:
+      - changed_files
+      - prepare-cache-data
     if: ${{ github.event.pull_request.draft == false && needs.changed_files.outputs.evidently_any_modified == 'true' }}
     steps:
       - uses: actions/checkout@v4
@@ -91,25 +112,39 @@ jobs:
           python-version: "3.8"
           architecture: "x64"
           cache: "pip"
+          cache-dependency-path: requirements.min.txt
+      - uses: actions/cache@v3
+        id: cache-bikes-dataset
+        env:
+          cache-name: cache-bikes-dataset
+        with:
+          path: Bike-Sharing-Dataset.zip
+          key: cache-bikes-dataset
+      - name: Download test data
+        if: ${{ steps.cache-bikes-dataset.outputs.cache-hit != 'true' }}
+        run:
+          curl -k https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip -o Bike-Sharing-Dataset.zip
       - name: Update pip
         run: python -m pip install --upgrade pip
-      - name: Install package
-        run: pip install -e .[dev,spark,fsspec]
       - name: Install minimal dependencies
         run: pip install -r requirements.min.txt
+      - name: Install package
+        run: pip install -e .[dev,spark,fsspec]
       - name: Run pip-audit
         run: pip-audit --ignore-vuln PYSEC-2024-48
       - name: Run Tests
         run: python -m pytest --durations=50
   test:
     # The type of runner that the job will run on
     name: Test ${{ matrix.os }} with py${{ matrix.python }}
-    needs: linter
+    needs:
+      - linter
+      - prepare-cache-data
     runs-on: ${{ matrix.os }}
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-13]
+        os: [ubuntu-22.04, windows-2022, macos-13]
         python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         exclude:
           - os: windows-latest
@@ -122,14 +157,31 @@ jobs:
           python-version: ${{ matrix.python }}
           architecture: "x64"
           cache: "pip"
+          cache-dependency-path: setup.py
+      - uses: actions/cache@v3
+        id: cache-scipy-data
+        with:
+          path: ~/scikit_learn_data
+          key: cache-scipy-data
+      - uses: actions/cache@v3
+        id: cache-bikes-dataset
+        env:
+          cache-name: cache-bikes-dataset
+        with:
+          path: Bike-Sharing-Dataset.zip
+          key: cache-bikes-dataset
+      - name: Download test data
+        if: ${{ steps.cache-bikes-dataset.outputs.cache-hit != 'true' }}
+        run:
+          curl -k https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip -o Bike-Sharing-Dataset.zip
       - name: Install package
         run: pip install -e .[dev,spark,fsspec]
       - name: Run Tests
         run: python -m pytest --durations=50
 
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: changed_files
     if: ${{ github.event.pull_request.draft == false && needs.changed_files.outputs.evidently_any_modified == 'true' }}
     steps:
@@ -139,8 +191,9 @@ jobs:
           python-version: "3.8"
           architecture: "x64"
           cache: "pip"
+          cache-dependency-path: setup.py
       - name: Install dependencies
-        run: pip install -r requirements.min.txt
+        run: pip install -e ".[dev]"
       - name: Install wheel
         run: pip install wheel
       - name: Build package

.github/workflows/release.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   check:
     name: Verifying preconditions for releasing Evidently
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Checking out sources
@@ -29,7 +29,7 @@ jobs:
 
   build:
     name: Build distribution
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs:
       - check
 
@@ -53,7 +53,7 @@ jobs:
 
   publish_to_pypi:
     name: Publish Python distributions to PyPI and TestPyPI
-    runs-on: [ubuntu-latest]
+    runs-on: [ubuntu-22.04]
     needs:
       - build
 
@@ -75,7 +75,7 @@ jobs:
 
   build_and_publish_docker:
     name: Build Docker image and publish it to DockerHub
-    runs-on: [ubuntu-latest]
+    runs-on: [ubuntu-22.04]
     needs:
       - build
 
@@ -113,7 +113,7 @@ jobs:
 
   create_release:
     name: Create release for the Tag
-    runs-on: [ubuntu-latest]
+    runs-on: [ubuntu-22.04]
     needs:
       - publish_to_pypi
 

.github/workflows/ui.yml

@@ -20,7 +20,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   changed_files:
-    runs-on: ubuntu-latest # windows-latest || macos-latest
+    runs-on: ubuntu-22.04 # windows-latest || macos-latest
     name: 👀 List Changed UI Files
     outputs:
       ui_any_modified: ${{ steps.changed-files.outputs.ui_any_modified == 'true' }}
@@ -58,7 +58,7 @@ jobs:
 
   ui-build:
     name: UI type-check and build with node v${{ matrix.node-version }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: changed_files
     if: ${{ github.event.pull_request.draft == false && needs.changed_files.outputs.ui_any_modified == 'true' }}
     strategy:
@@ -105,7 +105,7 @@ jobs:
 
   ui-test:
     name: Playwright tests for Service
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: changed_files
     if: ${{ github.event.pull_request.draft == false && needs.changed_files.outputs.evidently_python_any_modified == 'true' }}
 

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 venv
-.venv
+.venv*
 .vscode
 .DS_Store
 examples/.DS_Store

src/evidently/ui/demo_projects/bikes.py

@@ -1,4 +1,5 @@
 import io
+import os
 import zipfile
 from datetime import datetime
 from datetime import time
@@ -27,10 +28,14 @@
 
 
 def create_data():
-    content = requests.get(
-        "https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip",
-        verify=False,
-    ).content
+    if os.path.exists("Bike-Sharing-Dataset.zip"):
+        with open("Bike-Sharing-Dataset.zip", "rb") as f:
+            content = f.read()
+    else:
+        content = requests.get(
+            "https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip",
+            verify=False,
+        ).content
     with zipfile.ZipFile(io.BytesIO(content)) as arc:
         raw_data = pd.read_csv(
             arc.open("hour.csv"),

tests/spark/metrics/test_data_drift.py

@@ -1,3 +1,4 @@
+import sys
 from typing import Callable
 from typing import List
 
@@ -20,6 +21,10 @@
 
 
 @slow
+@pytest.mark.skipif(
+    (sys.version_info.major, sys.version_info.minor) == (3, 12) and sys.platform.startswith("win"),
+    reason="on Windows 2022 with python 3.12 pyspark package is broken",
+)
 @pytest.mark.parametrize(
     "metric,column_mapping,result_adjust",
     [

tests/ui/test_demo_project.py

@@ -9,6 +9,7 @@
 @pytest.mark.parametrize("demo_project", list(DEMO_PROJECTS.keys()))
 def test_create_demo_project(demo_project, tmp_path):
     dp = DEMO_PROJECTS[demo_project]
+    dp.count = 2
     dp.create(str(tmp_path))
 
     ws = Workspace(path=str(tmp_path))