.github/workflows/conda-lock.yml

@@ -28,21 +28,27 @@ jobs:
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          mamba-version: "*"
-          miniforge-version: latest
-          miniforge-variant: Mambaforge
+          miniconda-version: latest
           activate-environment: conda-lock
-          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          channel-priority: strict
+          python-version: "3.11"
           condarc-file: ci/conda-lock/condarc
 
+      - name: install conda libmamba solver
+        run: conda install -n base conda-libmamba-solver
+
+      - name: set solver to libmamba
+        run: conda config --set solver libmamba
+
       - name: install conda-lock
-        run: mamba install 'conda-lock>=1.4'
+        run: conda install conda-lock=1.4
 
       - name: generate lock file
         run: ./ci/conda-lock/generate.sh "${{ matrix.python-version }}"
 
       - name: create conda environment
-        run: mamba create --name ibis${{ matrix.python-version }} --file conda-lock/linux-64-${{ matrix.python-version }}.lock
+        run: conda create --name ibis${{ matrix.python-version }} --file conda-lock/linux-64-${{ matrix.python-version }}.lock
 
       - name: upload conda lock files
         uses: actions/upload-artifact@v3
@@ -78,7 +84,7 @@ jobs:
           name: conda-lock-files-${{ github.run_attempt }}
           path: conda-lock
 
-      - uses: peter-evans/create-pull-request@v4
+      - uses: peter-evans/create-pull-request@v5
         id: create_pr
         with:
           token: ${{ steps.generate_pr_token.outputs.token }}
@@ -93,13 +99,13 @@ jobs:
           labels: |
             dependencies
 
-      - uses: juliangruber/approve-pull-request-action@v2.0.3
+      - uses: juliangruber/approve-pull-request-action@v2.0.4
         if: steps.create_pr.outputs.pull-request-operation == 'created'
         with:
           github-token: ${{ steps.generate_pr_approval_token.outputs.token }}
           number: ${{ steps.create_pr.outputs.pull-request-number }}
 
-      - uses: peter-evans/enable-pull-request-automerge@v2
+      - uses: peter-evans/enable-pull-request-automerge@v3
         if: steps.create_pr.outputs.pull-request-operation == 'created'
         with:
           token: ${{ steps.generate_pr_token.outputs.token }}
@@ -151,11 +157,3 @@ jobs:
             git pull --rebase -s recursive -X ours
             git push
           fi
-
-      - name: react on success
-        uses: peter-evans/create-or-update-comment@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
-          comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
-          reaction-type: hooray

.github/workflows/ibis-backends.yml

@@ -99,6 +99,17 @@ jobs:
               - postgres
             sys-deps:
               - libgeos-dev
+          - name: postgres
+            title: PostgreSQL + PyArrow
+            extras:
+              - postgres
+              - geospatial
+            additional_deps:
+              - pyarrow
+            services:
+              - postgres
+            sys-deps:
+              - libgeos-dev
           - name: impala
             title: Impala
             serial: true
@@ -133,30 +144,6 @@ jobs:
               - druid
             services:
               - druid
-        include:
-          - os: ubuntu-latest
-            python-version: "3.8"
-            backend:
-              name: pyspark
-              title: PySpark
-              serial: true
-              extras:
-                - pyspark
-          - os: ubuntu-latest
-            python-version: "3.10"
-            backend:
-              name: pyspark
-              title: PySpark
-              serial: true
-              extras:
-                - pyspark
-          - os: ubuntu-latest
-            python-version: "3.11"
-            backend:
-              name: pyspark
-              title: PySpark
-              extras:
-                - pyspark
         exclude:
           - os: windows-latest
             backend:
@@ -188,6 +175,19 @@ jobs:
                 - postgres
               sys-deps:
                 - libgeos-dev
+          - os: windows-latest
+            backend:
+              name: postgres
+              title: PostgreSQL + PyArrow
+              extras:
+                - postgres
+                - geospatial
+              additional_deps:
+                - pyarrow
+              services:
+                - postgres
+              sys-deps:
+                - libgeos-dev
           - os: windows-latest
             backend:
               name: impala
@@ -243,12 +243,6 @@ jobs:
         if: matrix.os == 'windows-latest' && matrix.backend.name == 'sqlite'
         run: choco install sqlite
 
-      - uses: actions/setup-java@v3
-        if: matrix.backend.name == 'pyspark'
-        with:
-          distribution: microsoft
-          java-version: 17
-
       - name: checkout
         uses: actions/checkout@v3
 
@@ -284,6 +278,10 @@ jobs:
       - name: install ibis
         run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }}"
 
+      - name: install other deps
+        if: matrix.backend.additional_deps != null
+        run: poetry run pip install "${{ join(matrix.backend.additional_deps, ' ') }}"
+
       - name: "run parallel tests: ${{ matrix.backend.name }}"
         if: ${{ !matrix.backend.serial }}
         run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup
@@ -334,6 +332,7 @@ jobs:
             title: Dask
             deps:
               - "dask[array,dataframe]@2022.9.1"
+              - "pandas@1.5.3"
             extras:
               - dask
           - name: postgres
@@ -439,6 +438,81 @@ jobs:
         if: ${{ failure() }}
         run: docker compose logs
 
+  test_pyspark:
+    name: PySpark ${{ matrix.os }} python-${{ matrix.python-version }} pandas-${{ matrix.pandas-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        python-version:
+          - "3.10"
+        pandas-version:
+          - "1.5.*"
+          - "2.*.*"
+        include:
+          - os: ubuntu-latest
+            python-version: "3.8"
+            pandas-version: "1.5.*"
+          - os: ubuntu-latest
+            python-version: "3.11"
+            pandas-version: "2.*.*"
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: microsoft
+          java-version: 17
+
+      - uses: extractions/setup-just@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: download backend data
+        run: just download-data
+
+      - name: install python
+        uses: actions/setup-python@v4
+        id: install_python
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - run: python -m pip install --upgrade pip 'poetry<1.4'
+
+      - name: install minimum versions
+        run: poetry add --lock 'pandas@${{ matrix.pandas-version }}' 'numpy@1.23.*'
+
+      - name: checkout the lock file
+        run: git checkout poetry.lock
+
+      - name: lock with no updates
+        # poetry add is aggressive and will update other dependencies like
+        # numpy and pandas so we keep the pyproject.toml edits and then relock
+        # without updating anything except the requested versions
+        run: poetry lock --no-update
+
+      - name: install ibis
+        run: poetry install --without dev --without docs --extras pyspark
+
+      - name: run tests
+        run: just ci-check -m pyspark
+
+      - name: upload code coverage
+        if: success()
+        uses: codecov/codecov-action@v3
+        with:
+          flags: backend,pyspark,${{ runner.os }},python-${{ steps.install_python.outputs.python-version }},pandas-${{ matrix.pandas-version }}
+
+      - name: publish test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: pyspark-${{ matrix.os }}-${{ matrix.python-version }}
+          path: junit.xml
+
   gen_lockfile_sqlalchemy2:
     name: Generate Poetry Lockfile for SQLAlchemy 2
     runs-on: ubuntu-latest
@@ -454,7 +528,7 @@ jobs:
       - run: python -m pip install --upgrade pip 'poetry<1.4'
 
       - name: remove deps that are not compatible with sqlalchemy 2
-        run: poetry remove duckdb-engine snowflake-sqlalchemy
+        run: poetry remove snowflake-sqlalchemy
 
       - name: add sqlalchemy 2
         run: poetry add --lock --optional 'sqlalchemy>=2,<3'
@@ -522,6 +596,10 @@ jobs:
             extras:
               - trino
               - postgres
+          - name: duckdb
+            title: DuckDB
+            extras:
+              - duckdb
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -602,5 +680,6 @@ jobs:
       - test_backends_min_version
       - test_backends
       - test_backends_sqlalchemy2
+      - test_pyspark
     steps:
       - run: exit 0

.github/workflows/nix.yml

@@ -50,6 +50,9 @@ jobs:
           extra_nix_config: |
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
 
+      - name: check poetry version
+        run: nix run '.#check-poetry-version' -- "1.3"
+
       - name: setup cachix
         uses: cachix/cachix-action@v12
         with:
@@ -64,7 +67,7 @@ jobs:
           nix build ".#ibis${version//./}" --fallback --keep-going --print-build-logs
 
       - name: nix build devShell
-        if: github.event_name == 'push'
+        if: github.event_name == 'push' && matrix.os != 'macos-latest'
         run: |
           set -euo pipefail
 

.github/workflows/update-deps.yml

@@ -86,7 +86,7 @@ jobs:
           basehead: ${{ steps.get_current_commit.outputs.rev }}...${{ steps.get_new_commit.outputs.rev }}
           include-merge-commits: false
 
-      - uses: peter-evans/create-pull-request@v4
+      - uses: peter-evans/create-pull-request@v5
         if: fromJSON(steps.needs_pr.outputs.did_change)
         id: create_pr
         with:
@@ -99,13 +99,13 @@ jobs:
           body: ${{ steps.compare_commits.outputs.differences }}
           labels: dependencies,nix
 
-      - uses: juliangruber/approve-pull-request-action@v2.0.3
+      - uses: juliangruber/approve-pull-request-action@v2.0.4
         if: fromJSON(steps.needs_pr.outputs.did_change)
         with:
           github-token: ${{ steps.generate_pr_approval_token.outputs.token }}
           number: ${{ steps.create_pr.outputs.pull-request-number }}
 
-      - uses: peter-evans/enable-pull-request-automerge@v2
+      - uses: peter-evans/enable-pull-request-automerge@v3
         if: fromJSON(steps.needs_pr.outputs.did_change)
         with:
           token: ${{ steps.generate_pr_token.outputs.token }}