ci[python]: Separate linting from testing (#4795)

.github/workflows/benchmark.yaml

@@ -25,7 +25,7 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
-          cache-dependency-path: 'py-polars/build.requirements.txt'
+          cache-dependency-path: 'py-polars/requirements-dev.txt'
       - name: Set up R
         uses: r-lib/actions/setup-r@v1
         with:
@@ -37,7 +37,7 @@ jobs:
           pip install --upgrade pip
           python -m venv venv
           source venv/bin/activate
-          pip install -r py-polars/build.requirements.txt
+          pip install -r py-polars/requirements-dev.txt
           cd py-polars
           maturin develop --release -- -C codegen-units=8 -C lto=thin -C target-cpu=native
           cd tests/db-benchmark

.github/workflows/lint-python.yaml

@@ -0,0 +1,51 @@
+name: Lint Python
+
+on:
+  pull_request:
+    paths:
+      - 'py-polars/**'
+      - '.github/workflows/lint-python.yaml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: py-polars
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: 'py-polars/requirements-lint.txt'
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements-lint.txt
+      - name: Lint Python
+        run: |
+          black --check .
+          blackdoc --check .
+          isort --check .
+          pyupgrade --py37-plus `find . -name "*.py" -type f`
+          flake8
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2022-08-22
+          components: rustfmt, clippy
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+      - name: Lint Rust
+        run: |
+          cargo fmt --all -- --check
+          make clippy

.github/workflows/test-python.yaml

@@ -1,4 +1,4 @@
-name: Test python
+name: Test Python
 
 on:
   pull_request:
@@ -12,8 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test-python:
-    name: Build and test Python
+  main:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
@@ -25,47 +24,46 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Rust
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: nightly-2022-08-22
-          components: rustfmt, clippy
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
-          cache-dependency-path: 'py-polars/build.requirements.txt'
+          cache-dependency-path: 'py-polars/requirements-dev.txt'
+      - name: Create virtual environment
+        run: |
+          python -m venv venv
+          echo "$GITHUB_WORKSPACE/py-polars/venv/bin" >> $GITHUB_PATH
       - name: Install dependencies
         run: |
           pip install --upgrade pip
-          pip install -r build.requirements.txt
-      - name: Run formatting checks
-        run: |
-          black --check .
-          blackdoc --check .
-          isort --check .
-          pyupgrade --py37-plus `find . -name "*.py" -type f`
-          cargo fmt --all -- --check
-      - name: Run linting
-        run: flake8
+          pip install -r requirements-dev.txt
       # Allow untyped calls for older Python versions
-      - name: Run type checking
-        run: mypy  ${{ (matrix.python-version == '3.7') && '--allow-untyped-calls' || '' }}
-      - name: Run tests
+      - name: Run mypy
+        run: mypy ${{ (matrix.python-version == '3.7') && '--allow-untyped-calls' || '' }}
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2022-08-22
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+      - name: Install Polars
         env:
           RUSTFLAGS: -C debuginfo=0  # Do not produce debug symbols to keep memory usage down
         run: |
-          make venv && make test-with-cov
-          make clippy
-      - name: Check doc examples
-        run: make doctest
-      # test if we can import polars without any requirements
-      - name: Import polars
+          source activate
+          maturin develop
+      - name: Run tests and report coverage
+        run: pytest --cov=polars
+      - name: Run doctests
+        run: python tests/docs/run_doc_examples.py
+      - name: Check import without optional dependencies
         run: |
           pip uninstall pandas -y
-          python -c "import polars"
+          python -c 'import polars'
           pip uninstall numpy -y
-          python -c "import polars"
+          python -c 'import polars'
           pip uninstall pyarrow -y
-          python -c "import polars"
+          python -c 'import polars'

.github/workflows/test-windows-python.yaml

@@ -29,11 +29,11 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
-          cache-dependency-path: 'py-polars/build.requirements.txt'
+          cache-dependency-path: 'py-polars/requirements-dev.txt'
       - name: Install dependencies
         run: |
           pip install --upgrade pip
-          pip install -r build.requirements.txt
+          pip install -r requirements-dev.txt
       - name: Run tests
         shell: bash
         env:

CONTRIBUTING.md

@@ -134,7 +134,7 @@ You have to follow these steps:
 
   - Install Rust nightly via [rustup](https://www.rust-lang.org/tools/install)
   - run `$ rustup override set nightly` from the root of the repo.
-  - from [./py-polars](./py-polars) run `$ pip3 install -r build.requirements.txt`
+  - from [./py-polars](./py-polars) run `$ make venv`
   - **tests:** from [./py-polars](./py-polars) run `$ make test`
   - **formatting + linting:** from [./py-polars](./py-polars) run `$ make pre-commit` before committing.
   - **docs:** from [./py-polars/docs](./py-polars/docs) run `$ pip3 install -r requirements-docs.txt` followed by `make html`

py-polars/Makefile

@@ -6,20 +6,23 @@ PYTHON_BIN=venv/bin
 
 venv:  ## Set up virtual environment
 	@python -m venv venv
-	@venv/bin/pip install -U pip
-	@venv/bin/pip install -r build.requirements.txt
+	@venv/bin/pip install --upgrade pip
+	@venv/bin/pip install -r requirements-dev.txt
+	@venv/bin/pip install -r requirements-lint.txt
 	@unset CONDA_PREFIX && source venv/bin/activate && maturin develop
 
 .PHONY: clean
 clean:  ## Clean up caches and build artifacts
 	@rm -rf venv/
+	@rm -rf target/
 	@rm -rf docs/build/
 	@rm -rf docs/source/reference/api/
 	@rm -rf .hypothesis/
 	@rm -rf .mypy_cache/
 	@rm -rf .pytest_cache/
 	@rm -f .coverage
 	@rm -f coverage.xml
+	@rm -f polars/polars.abi3.so
 	@find -type f -name '*.py[co]' -delete -or -type d -name __pycache__ -delete
 	@cargo clean
 

py-polars/build.requirements.txt → py-polars/requirements-dev.txt

@@ -4,28 +4,18 @@
 
 # Dependencies
 numpy
-pyarrow
 pandas
+pyarrow
 pytz
-types-pytz
 xlsx2csv
 
 # Tooling
+hypothesis==6.54.5
 maturin==0.13.2
+mypy==0.971
 pytest==7.1.3
 pytest-cov==3.0.0
-hypothesis==6.54.5
-black==22.8.0
-blackdoc==0.3.6
-isort==5.10.1
-mypy==0.971
-flake8==5.0.4
-flake8-bugbear==22.8.23
-flake8-comprehensions==3.10.0
-flake8-docstrings==1.6.0
-flake8-simplify==0.19.3
-flake8-tidy-imports==4.8.0
-pyupgrade==2.37.3
 
 # Stub files
 pandas-stubs==1.2.0.61
+types-pytz==2022.2.1.0

py-polars/requirements-lint.txt

@@ -0,0 +1,10 @@
+black==22.8.0
+blackdoc==0.3.6
+flake8==5.0.4
+flake8-bugbear==22.8.23
+flake8-comprehensions==3.10.0
+flake8-docstrings==1.6.0
+flake8-simplify==0.19.3
+flake8-tidy-imports==4.8.0
+isort==5.10.1
+pyupgrade==2.37.3