Showing 658 changed files with 19,355 additions and 13,525 deletions.
diff --git a/.codespellrc b/.codespellrc
@@ -1,6 +1,6 @@
 [codespell]
 # local codespell matches `./docs`, pre-commit codespell matches `docs`
-skip = *.lock,.direnv,.git,./docs/_freeze,./docs/_output/**,./docs/_inv/**,docs/_freeze/**,*.svg,*.css,*.html,*.js
+skip = *.lock,.direnv,.git,./docs/_freeze,./docs/_output/**,./docs/_inv/**,docs/_freeze/**,*.svg,*.css,*.html,*.js,ibis/backends/tests/tpc/queries/duckdb/ds/44.sql
 ignore-regex = \b(i[if]f|I[IF]F|AFE)\b
 builtin = clear,rare,names
 ignore-words-list = tim,notin,ang
diff --git a/.github/workflows/create-rotate-key-issue.yml b/.github/workflows/create-rotate-key-issue.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Generate a GitHub token
-        uses: actions/create-github-app-token@v1.10.1
+        uses: actions/create-github-app-token@v1.10.3
         id: generate_token
         with:
           app-id: ${{ secrets.SQUAWK_BOT_APP_ID }}

diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml
@@ -13,7 +13,7 @@ jobs:
       cancel-in-progress: true
     if: github.event.label.name == 'docs-preview'
     steps:
-      - uses: actions/create-github-app-token@v1.10.1
+      - uses: actions/create-github-app-token@v1.10.3
         id: generate_token
         with:
           app-id: ${{ secrets.DOCS_BOT_APP_ID }}
@@ -44,17 +44,38 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: generate url alias
+        id: get_alias
+        run: |
+          echo "id=pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
+
+      - name: Get all changed qmd files
+        id: changed-qmd-files
+        uses: tj-actions/changed-files@v44
+        with:
+          # Avoid using single or double quotes for multiline patterns
+          files: |
+            **.qmd
+
+      - name: get preview links
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-qmd-files.outputs.all_changed_files }}
+          PREVIEW_URL: "https://${{ steps.get_alias.outputs.id }}--ibis-quarto.netlify.app"
+        run: |
+          {
+            echo 'Docs preview: https://${{ steps.get_alias.outputs.id }}--ibis-quarto.netlify.app'
+            for file in ${ALL_CHANGED_FILES}; do
+              link="${file#docs/}"
+              echo "- [${file}](${PREVIEW_URL}/${link%.qmd})"
+            done
+          } | tee /tmp/comment
+
       - name: build docs
         run: nix develop --ignore-environment --keep HOME -c just docs-build-all
 
       - name: install netlify cli
         run: npm install -g netlify-cli
 
-      - name: generate url alias
-        id: get_alias
-        run: |
-          echo "id=pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
-
       - name: build and push quarto docs to preview url
         run: netlify deploy --dir=docs/_output --alias="${{ steps.get_alias.outputs.id }}"
         env:
@@ -67,5 +88,4 @@ jobs:
         with:
           token: ${{ steps.generate_token.outputs.token }}
           issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            Docs preview: https://${{ steps.get_alias.outputs.id }}--ibis-quarto.netlify.app
+          body-path: /tmp/comment
diff --git a/.github/workflows/ibis-backends-cloud.yml b/.github/workflows/ibis-backends-cloud.yml
@@ -80,7 +80,7 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - uses: actions/create-github-app-token@v1.10.1
+      - uses: actions/create-github-app-token@v1.10.3
         id: generate_token
         with:
           app-id: ${{ secrets.DOCS_BOT_APP_ID }}

diff --git a/.github/workflows/ibis-backends-skip-helper.yml b/.github/workflows/ibis-backends-skip-helper.yml
@@ -27,6 +27,10 @@ on:
       - "*.x.x"
   merge_group:
 jobs:
+  test_bigquery_lite:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "No build required"
   test_backends:
     runs-on: ubuntu-latest
     steps:
@@ -43,6 +47,7 @@ jobs:
     # this job exists so that we can use a single job from this workflow to gate merging
     runs-on: ubuntu-latest
     needs:
+      - test_bigquery_lite
       - test_backends_min_version
       - test_backends
       - test_pyspark

diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml
@@ -126,7 +126,7 @@ jobs:
             extras:
               - sqlite
           - name: datafusion
-            title: Datafusion
+            title: DataFusion
             extras:
               - datafusion
           - name: polars
@@ -166,7 +166,7 @@ jobs:
             sys-deps:
               - libgeos-dev
           - name: risingwave
-            title: Risingwave
+            title: RisingWave
             serial: true
             services:
               - risingwave
@@ -304,7 +304,7 @@ jobs:
           - os: windows-latest
             backend:
               name: risingwave
-              title: Risingwave
+              title: RisingWave
               serial: true
               services:
                 - risingwave
@@ -531,7 +531,7 @@ jobs:
             title: Dask
             deps:
               required:
-                - "numpy@1.23.2"
+                - "numpy@1.23.5"
                 - "pyarrow@10.0.1"
               optional:
                 - "dask[array,dataframe]@2022.9.1"
@@ -542,7 +542,7 @@ jobs:
             title: PostgreSQL
             deps:
               required:
-                - "numpy@1.23.2"
+                - "numpy@1.23.5"
                 - "pyarrow@10.0.1"
               optional:
                 - "psycopg2@2.8.4"
@@ -560,7 +560,7 @@ jobs:
               title: PostgreSQL
               deps:
                 required:
-                  - "numpy@1.23.2"
+                  - "numpy@1.23.5"
                   - "pyarrow@10.0.1"
                 optional:
                   - "psycopg2@2.8.4"
@@ -577,7 +577,7 @@ jobs:
               title: PostgreSQL
               deps:
                 required:
-                  - "numpy@1.23.2"
+                  - "numpy@1.23.5"
                   - "pyarrow@10.0.1"
                 optional:
                   - "psycopg2@2.8.4"
@@ -594,7 +594,7 @@ jobs:
               title: Dask
               deps:
                 required:
-                  - "numpy@1.23.2"
+                  - "numpy@1.23.5"
                   - "pyarrow@10.0.1"
                 optional:
                   - "dask[array,dataframe]@2022.9.1"
@@ -680,7 +680,7 @@ jobs:
       matrix:
         include:
           - python-version: "3.10"
-            pyspark-version: "3.3"
+            pyspark-version: "3.3.3"
             deps:
               - "'pandas@<2'"
               - "'numpy@<1.24'"
@@ -743,6 +743,11 @@ jobs:
         if: matrix.pyspark-version == '3.5'
         run: poetry run pip install delta-spark
 
+      - name: install iceberg
+        shell: bash
+        if: matrix.pyspark-version == '3.5'
+        run: pushd "$(poetry run python -c "import pyspark; print(pyspark.__file__.rsplit('/', 1)[0])")/jars" && curl -LO https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.5.2/iceberg-spark-runtime-3.5_2.12-1.5.2.jar
+
       - name: run tests
         run: just ci-check -m pyspark
 
@@ -763,6 +768,7 @@ jobs:
     # this job exists so that we can use a single job from this workflow to gate merging
     runs-on: ubuntu-latest
     needs:
+      - test_bigquery_lite
       - test_backends_min_version
       - test_backends
       - test_pyspark

diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml
@@ -37,7 +37,7 @@ env:
 
 jobs:
   test_core:
-    name: Test ${{ matrix.os }} python-${{ matrix.python-version }}
+    name: Test ${{ matrix.os }} python-${{ matrix.python-version }} pyarrow-${{ matrix.pyarrow }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -49,6 +49,9 @@ jobs:
           - "3.10"
           - "3.11"
           - "3.12"
+        pyarrow:
+          - true
+          - false
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -77,17 +80,25 @@ jobs:
       - name: install ibis
         run: poetry install --without dev --without docs --extras "visualization decompiler"
 
+      - name: install pyarrow
+        if: matrix.pyarrow
+        run: poetry run pip install pyarrow pyarrow-hotfix
+
+      - name: check pyarrow import
+        if: matrix.pyarrow
+        run: poetry run python -c 'import pyarrow, pyarrow_hotfix'
+
       - uses: extractions/setup-just@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: run all core tests and run benchmarks once parallel
         if: matrix.os != 'windows-latest'
-        run: just ci-check -m "'core or benchmarks'" --numprocesses auto
+        run: just ci-check -m "'core or benchmarks'" --numprocesses auto -rs
 
       - name: run all core tests and run benchmarks once serial
         if: matrix.os == 'windows-latest'
-        run: just ci-check -m "'core or benchmarks'"
+        run: just ci-check -m "'core or benchmarks'" -rs
 
       - name: upload code coverage
         if: success()

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,7 +14,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/create-github-app-token@v1.10.1
+      - uses: actions/create-github-app-token@v1.10.3
         id: generate_token
         with:
           app-id: ${{ secrets.APP_ID }}

diff --git a/.github/workflows/upload-algolia-api.py b/.github/workflows/upload-algolia-api.py
@@ -1,5 +1,7 @@
 from __future__ import annotations  # noqa: INP001
 
+import glob
+import json
 import os
 import re
 from functools import partial
@@ -74,6 +76,7 @@ def main():
     records = []
     for qmd in API_QMDS:
         # For each QMD file, get the table-section of the methods, anchors, and descriptions
+        print(f"Scraping {qmd} for API methods...")  # noqa:T201
         with open(qmd) as f:
             methods = _grab_qmd_methods(f.read())
 
@@ -90,6 +93,21 @@ def main():
     # If the object IDs are new (which typically should be) this adds a new
     # record to the Algolia index.  If the object ID already exists, it gets
     # updated with the new fields in the record dict
+    print(f"Uploading {len(records)} records to {index.name=}")  # noqa:T201
+    index.save_objects(records)
+
+    # Methods documented on backend-specific docs pages aren't scraped by Quarto
+    # since we construct them programmatically.
+    # There is a hook in docs/backends/_templates/api.qmd that calls
+    # `dump_methods_to_json_for_algolia` that serializes all the backend methods
+    # to a backend-specific json file in docs/backends/
+    # (Not Pandas and Impala because those backend pages don't use the template)
+    #
+    # Here, we load those records and upload them to the Algolia index
+    records = []
+    for record_json in glob.glob("docs/backends/*.json"):
+        with open(record_json) as f:
+            records.extend(json.load(f))
     index.save_objects(records)
 
 

diff --git a/ci/check_disallowed_imports.py b/ci/check_disallowed_imports.py
diff --git a/ci/make_geography_db.py b/ci/make_geography_db.py
@@ -90,7 +90,7 @@ def main() -> None:
 
     args = parser.parse_args()
 
-    response = requests.get(args.input_data_url)
+    response = requests.get(args.input_data_url, timeout=600)
     response.raise_for_status()
     input_data = response.json()
     db_path = Path(args.output_directory).joinpath("geography.duckdb")