ci-tests.yml

name: CI Tests
on:
  push:
    branches:
      - main
      - dev
      - bugfix
      - "release/*"
  pull_request:
    branches:
      - main
      - dev
      - bugfix
      - "release/*"

env:
  DEFAULT_PYTHON: 3.8
  CI: "true"
  # Increase this value to reset cache if environment.yml has not changed
  CACHE_VERSION: 6

concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
  cancel-in-progress: true

jobs:

  lint:
    name: Linters (${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    defaults:
      run:
        shell: bash -l {0}

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip  # ubuntu location
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          python -m pip install nox pre_commit \
            mypy==0.982 \
            types-click \
            types-pyyaml \
            types-pkg_resources \
            types-requests \
            types-pytz
      - name: Pip info
        run: python -m pip list

      - name: Check requirements
        run: >
          nox
          -db virtualenv -r
          --non-interactive
          --python ${{ matrix.python-version }}
          --session requirements-${{ matrix.python-version }}

      - name: Sort Imports
        run: pre-commit run isort --all-files

      - name: Black
        if: always()
        run: pre-commit run black --all-files

      - name: Pylint
        if: always()
        run: pre-commit run pylint --all-files

      - name: Mypy Type Checking
        if: always()
        run: pre-commit run mypy --all-files

  tests:
    name: >
      CI Tests (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }})
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: pwsh
    env:
      PYTHONUTF8: 1
      NOX_FLAGS: -v --no-install --no-venv --non-interactive
      PYTEST_FLAGS: --cov=pandera --cov-report=term-missing --cov-report=xml --cov-append
      HYPOTHESIS_FLAGS: -n=auto -q --hypothesis-profile=ci
    strategy:
      fail-fast: true
      matrix:
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
        pandas-version: ["1.5.3", "2.0.3"]
        pydantic-version: ["1.10.11", "2.0.2"]
        exclude:
        - python-version: "3.7"
          pandas-version: "2.0.3"
        - python-version: "3.7"
          pandas-version: "1.5.2"
        - python-version: "3.10"
          pandas-version: "1.3.0"
        - python-version: "3.11"
          pandas-version: "1.3.0"
        include:
        - os: ubuntu-latest
          pip-cache: ~/.cache/pip
        - os: macos-latest
          pip-cache: ~/Library/Caches/pip
        - os: windows-latest
          pip-cache: ~/AppData/Local/pip/Cache

    steps:
      - uses: actions/checkout@v3

      - name: Cache conda
        uses: actions/cache@v3
        env:
          # Increase this value to reset cache if etc/environment.yml has not changed
          CACHE_NUMBER: 2
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
        id: cache

      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ${{ matrix.pip-cache }}
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          python-version: ${{ matrix.python-version }}
          miniforge-version: latest
          miniforge-variant: Mambaforge
          use-mamba: true
          activate-environment: pandera-dev
          channels: conda-forge
          channel-priority: true
          use-only-tar-bz2: true

      # ray currently cannot be installed on python 3.11
      - name: Remove Ray from Deps
        if: ${{ matrix.python-version == '3.11' && matrix.os != 'macos-latest' }}
        run: sed -i '/ray/d' environment.yml

      - name: Remove Ray from Deps
        if: ${{ matrix.python-version == '3.11' && matrix.os == 'macos-latest' }}
        run: sed -i .bak '/ray/d' environment.yml

      # need to install pandas via pip: conda installation is on the fritz
      - name: Install Conda Deps [pandas 2]
        if: ${{ matrix.pandas-version == '2.0.3' }}
        run: |
          mamba install -c conda-forge asv pandas geopandas bokeh
          mamba env update -n pandera-dev -f environment.yml
          pip install pandas==${{ matrix.pandas-version }}
          pip install --user dask>=2023.3.2

      - name: Install Conda Deps
        if: ${{ matrix.pandas-version != '2.0.3' }}
        run: |
          mamba install -c conda-forge asv pandas==${{ matrix.pandas-version }} geopandas bokeh
          mamba env update -n pandera-dev -f environment.yml

      - name: Install Pydantic Deps
        run: pip install -U --upgrade-strategy only-if-needed pydantic==${{ matrix.pydantic-version }}

      - name: Install Pydantic v2 Deps
        if : ${{ matrix.pydantic-version == '2.0.2' }}
        run: pip install pydantic-settings

      - run: |
          conda info
          conda list
          conda config --show-sources
          conda config --show
          printenv | sort

      - name: Unit Tests - Core
        run: pytest tests/core ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - Hypotheses
        run: pytest tests/hypotheses ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - IO
        run: pytest tests/io ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - Mypy
        if: ${{ matrix.python-version != '3.7' }}
        run: pytest -v tests/mypy ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - Strategies
        run: pytest tests/strategies ${{ env.PYTEST_FLAGS }} ${{ env.HYPOTHESIS_FLAGS }}

      - name: Unit Tests - FastAPI
        # there's an issue with the fastapi tests in CI that's not reproducible locally
        # when pydantic > v2
        if: ${{ matrix.python-version != '3.7'  && matrix.pydantic-version != '2.0.2' }}
        run: pytest tests/fastapi ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - GeoPandas
        run: pytest tests/geopandas ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - Dask
        if: ${{ matrix.pandas-version != '2.0.3' }}
        run: pytest tests/dask ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - Pyspark
        if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) && matrix.pandas-version != '2.0.3' }}
        run: pytest tests/pyspark ${{ env.PYTEST_FLAGS }}

      - name: Unit Tests - Modin-Dask
        if: ${{ !contains(fromJson('["3.11"]'), matrix.python-version) && matrix.pandas-version != '2.0.3' }}
        run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
        env:
          CI_MODIN_ENGINES: dask

      - name: Unit Tests - Modin-Ray
        # ray CI issues with the following:
        # - windows, python 3.10
        # - mac, python 3.7
        # Tracking issue: https://github.com/modin-project/modin/issues/5466
        if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version)  && matrix.pandas-version != '2.0.3' && matrix.pydantic-version != '2.0.2' }}
        run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
        env:
          CI_MODIN_ENGINES: ray

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3

      - name: Check Docstrings
        if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) && matrix.pydantic-version != '2.0.2' }}
        run: nox ${{ env.NOX_FLAGS }} --session doctests

      - name: Check Docs
        if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) && matrix.pydantic-version != '2.0.2' }}
        run: nox ${{ env.NOX_FLAGS }} --session docs