Support native PySpark.sql on Pandera #1814
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - dev | |
| - bugfix | |
| - "release/*" | |
| pull_request: | |
| branches: | |
| - main | |
| - dev | |
| - bugfix | |
| - "release/*" | |
| env: | |
| DEFAULT_PYTHON: 3.8 | |
| CI: "true" | |
| # Increase this value to reset cache if environment.yml has not changed | |
| CACHE_VERSION: 6 | |
| concurrency: | |
| group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| name: Linters (${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/cache@v2 | |
| with: | |
| path: ~/.cache/pip # ubuntu location | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install nox pre_commit \ | |
| mypy==0.982 \ | |
| types-click \ | |
| types-pyyaml \ | |
| types-pkg_resources \ | |
| types-requests \ | |
| types-pytz | |
| - name: Pip info | |
| run: python -m pip list | |
| - name: Check requirements | |
| run: > | |
| nox | |
| -db virtualenv -r | |
| --non-interactive | |
| --python ${{ matrix.python-version }} | |
| --session requirements-${{ matrix.python-version }} | |
| - name: Sort Imports | |
| run: pre-commit run isort --all-files | |
| - name: Black | |
| if: always() | |
| run: pre-commit run black --all-files | |
| - name: Pylint | |
| if: always() | |
| run: pre-commit run pylint --all-files | |
| - name: Mypy Type Checking | |
| if: always() | |
| run: pre-commit run mypy --all-files | |
| tests: | |
| name: > | |
| CI Tests (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }}) | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: pwsh | |
| env: | |
| PYTHONUTF8: 1 | |
| NOX_FLAGS: -v --no-install --no-venv --non-interactive | |
| PYTEST_FLAGS: --cov=pandera --cov-report=term-missing --cov-report=xml --cov-append | |
| HYPOTHESIS_FLAGS: -n=auto -q --hypothesis-profile=ci | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] | |
| pandas-version: ["1.3.0", "1.5.2", "2.0.1"] | |
| exclude: | |
| - python-version: "3.7" | |
| pandas-version: "2.0.1" | |
| - python-version: "3.7" | |
| pandas-version: "1.5.2" | |
| - python-version: "3.10" | |
| pandas-version: "1.3.0" | |
| - python-version: "3.11" | |
| pandas-version: "1.3.0" | |
| include: | |
| - os: ubuntu-latest | |
| pip-cache: ~/.cache/pip | |
| - os: macos-latest | |
| pip-cache: ~/Library/Caches/pip | |
| - os: windows-latest | |
| pip-cache: ~/AppData/Local/pip/Cache | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Cache conda | |
| uses: actions/cache@v2 | |
| env: | |
| # Increase this value to reset cache if etc/environment.yml has not changed | |
| CACHE_NUMBER: 1 | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} | |
| id: cache | |
| - name: Cache pip | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ matrix.pip-cache }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: true | |
| python-version: ${{ matrix.python-version }} | |
| miniforge-version: latest | |
| miniforge-variant: Mambaforge | |
| use-mamba: true | |
| activate-environment: pandera-dev | |
| channels: conda-forge | |
| channel-priority: true | |
| use-only-tar-bz2: true | |
| # ray currently cannot be installed on python 3.11 | |
| - name: Remove Ray from Deps | |
| if: ${{ matrix.python-version == '3.11' && matrix.os != 'macos-latest' }} | |
| run: sed -i '/ray/d' environment.yml | |
| - name: Remove Ray from Deps | |
| if: ${{ matrix.python-version == '3.11' && matrix.os == 'macos-latest' }} | |
| run: sed -i .bak '/ray/d' environment.yml | |
| # need to install pandas via pip: conda installation is on the fritz | |
| - name: Install Conda Deps [pandas 2] | |
| if: ${{ matrix.pandas-version == '2.0.1' }} | |
| run: | | |
| mamba install -c conda-forge asv pandas geopandas bokeh | |
| mamba env update -n pandera-dev -f environment.yml | |
| pip install pandas==${{ matrix.pandas-version }} | |
| pip install --user dask>=2023.3.2 | |
| - name: Install Conda Deps | |
| if: ${{ matrix.pandas-version != '2.0.1' }} | |
| run: | | |
| mamba install -c conda-forge asv pandas==${{ matrix.pandas-version }} geopandas bokeh | |
| mamba env update -n pandera-dev -f environment.yml | |
| - run: | | |
| conda info | |
| conda list | |
| conda config --show-sources | |
| conda config --show | |
| printenv | sort | |
| - name: Unit Tests - Core | |
| run: pytest tests/core ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Hypotheses | |
| run: pytest tests/hypotheses ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - IO | |
| run: pytest tests/io ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Mypy | |
| if: ${{ matrix.python-version != '3.7' }} | |
| run: pytest -v tests/mypy ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Strategies | |
| run: pytest tests/strategies ${{ env.PYTEST_FLAGS }} ${{ env.HYPOTHESIS_FLAGS }} | |
| - name: Unit Tests - FastAPI | |
| run: pytest tests/fastapi ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - GeoPandas | |
| run: pytest tests/geopandas ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Dask | |
| if: ${{ matrix.pandas-version != '2.0.1' }} | |
| run: pytest tests/dask ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Pyspark | |
| if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) && matrix.pandas-version != '2.0.1' }} | |
| run: pytest tests/pyspark ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Modin-Dask | |
| if: ${{ !contains(fromJson('["3.11"]'), matrix.python-version) && matrix.pandas-version != '2.0.1' }} | |
| run: pytest tests/modin ${{ env.PYTEST_FLAGS }} | |
| env: | |
| CI_MODIN_ENGINES: dask | |
| - name: Unit Tests - Modin-Ray | |
| # ray CI issues with the following: | |
| # - windows, python 3.10 | |
| # - mac, python 3.7 | |
| # Tracking issue: https://github.com/modin-project/modin/issues/5466 | |
| if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) && matrix.pandas-version != '2.0.1' }} | |
| run: pytest tests/modin ${{ env.PYTEST_FLAGS }} | |
| env: | |
| CI_MODIN_ENGINES: ray | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| - name: Check Docstrings | |
| if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) }} | |
| run: nox ${{ env.NOX_FLAGS }} --session doctests | |
| - name: Check Docs | |
| if: ${{ matrix.os != 'windows-latest' && !contains(fromJson('["3.7", "3.10", "3.11"]'), matrix.python-version) }} | |
| run: nox ${{ env.NOX_FLAGS }} --session docs |