.env

@@ -1,4 +1,4 @@
-FLINK_VERSION=1.19.1
+FLINK_VERSION=1.20.0
 CLOUDSDK_ACTIVE_CONFIG_NAME=ibis-gbq
 GOOGLE_CLOUD_PROJECT="$CLOUDSDK_ACTIVE_CONFIG_NAME"
 PGPASSWORD="postgres"

.github/actionlint.yaml

@@ -1,3 +1,3 @@
 self-hosted-runner:
   labels:
-    - ubuntu-arm64-small
+    - ubuntu-arm64-24.04

.github/workflows/docs-preview.yml

@@ -51,7 +51,7 @@ jobs:
 
       - name: Get all changed qmd files
         id: changed-qmd-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           # Avoid using single or double quotes for multiline patterns
           files: |

.github/workflows/ibis-backends.yml

@@ -227,7 +227,7 @@ jobs:
             extras:
               - flink
             additional_deps:
-              - "'apache-flink==1.19.1'"
+              - "'apache-flink==1.20.0'"
               - "'pandas<2.2'"
               - setuptools
             services:
@@ -249,7 +249,7 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - "'apache-flink==1.19.1'"
+                - "'apache-flink==1.20.0'"
                 - "'pandas<2.2'"
                 - setuptools
               services:
@@ -400,7 +400,7 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - "'apache-flink==1.19.1'"
+                - "'apache-flink==1.20.0'"
                 - "'pandas<2.2'"
                 - setuptools
               services:
@@ -413,7 +413,7 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - "'apache-flink==1.19.1'"
+                - "'apache-flink==1.20.0'"
                 - "'pandas<2.2'"
                 - setuptools
               services:
@@ -481,6 +481,10 @@ jobs:
       - name: show installed deps
         run: poetry run pip list
 
+      - name: show version of python-linked sqlite
+        if: matrix.backend.name == 'sqlite'
+        run: poetry run python -c 'import sqlite3; print(sqlite3.sqlite_version)'
+
       - name: "run parallel tests: ${{ matrix.backend.name }}"
         if: ${{ !matrix.backend.serial }}
         run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup

.github/workflows/ibis-docs-main.yml

@@ -67,7 +67,7 @@ jobs:
 
       - name: Install Algolia API Dependencies
         run: |
-          python -m pip install --upgrade algoliasearch
+          python -m pip install --upgrade 'algoliasearch<4'
 
       - name: Create and Upload Base Index
         run: |

.github/workflows/ibis-main.yml

@@ -169,7 +169,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: run doctests
-        run: just ci-doctest --junitxml=junit.xml --cov=ibis --cov-report=xml:coverage.xml
+        run: just doctest --junitxml=junit.xml --cov=ibis --cov-report=xml:coverage.xml
+
+      - name: check for dead fixtures
+        run: poetry run pytest --dead-fixtures -m 'not flink'
 
       - name: upload code coverage
         if: success()

.github/workflows/nix-skip-helper.yml

@@ -37,7 +37,7 @@ jobs:
           - "3.11"
           - "3.12"
         include:
-          - os: ubuntu-arm64-small
+          - os: ubuntu-arm64-24.04
             python-version: "3.12"
           - os: macos-14
             python-version: "3.10"

.github/workflows/nix.yml

@@ -38,7 +38,7 @@ jobs:
           - "3.11"
           - "3.12"
         include:
-          - os: ubuntu-arm64-small
+          - os: ubuntu-arm64-24.04
             python-version: "3.12"
           - os: macos-14
             python-version: "3.10"
@@ -59,19 +59,13 @@ jobs:
           name: ibis
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
           extraPullNames: nix-community,poetry2nix
-      - name: nix build and test core
-        run: |
-          set -euo pipefail
-
-          version='${{ matrix.python-version }}'
-          nix build ".#ibisCore${version//./}" --fallback --keep-going --print-build-logs
 
-      - name: nix build and test local
+      - name: nix build and test
         run: |
           set -euo pipefail
 
           version='${{ matrix.python-version }}'
-          nix build ".#ibisLocal${version//./}" --fallback --keep-going --print-build-logs
+          nix build ".#ibis${version//./}" --fallback --keep-going --print-build-logs
 
       - name: nix build devShell
         run: |

compose.yaml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:24.7.2.13-alpine
+    image: clickhouse/clickhouse-server:24.7.3.42-alpine
     ports:
       - 8123:8123 # http port
       - 9000:9000 # native protocol port
@@ -28,7 +28,7 @@ services:
         - CMD
         - mariadb-admin
         - ping
-    image: mariadb:11.4.2
+    image: mariadb:11.5.2
     ports:
       - 3306:3306
     networks:
@@ -82,7 +82,7 @@ services:
       - mssql
 
   hive-metastore-db:
-    image: postgres:16.3-alpine
+    image: postgres:16.4-alpine
     environment:
       POSTGRES_USER: admin
       POSTGRES_PASSWORD: admin
@@ -99,7 +99,7 @@ services:
       - trino
 
   minio:
-    image: bitnami/minio:2024.8.3
+    image: bitnami/minio:2024.8.29
     environment:
       MINIO_ROOT_USER: accesskey
       MINIO_ROOT_PASSWORD: secretkey
@@ -161,7 +161,7 @@ services:
       test:
         - CMD-SHELL
         - trino --output-format null --execute 'show schemas in hive; show schemas in memory'
-    image: trinodb/trino:449
+    image: trinodb/trino:455
     ports:
       - 8080:8080
     networks:
@@ -172,7 +172,7 @@ services:
       - $PWD/docker/trino/jvm.config:/etc/trino/jvm.config:ro
 
   druid-postgres:
-    image: postgres:16.3-alpine
+    image: postgres:16.4-alpine
     container_name: druid-postgres
     environment:
       POSTGRES_PASSWORD: FoolishPassword
@@ -367,7 +367,7 @@ services:
       - oracle:/opt/oracle/data
 
   exasol:
-    image: exasol/docker-db:8.29.1
+    image: exasol/docker-db:8.30.0
     privileged: true
     ports:
       - 8563:8563
@@ -547,7 +547,7 @@ services:
       - impala
 
   risingwave:
-    image: ghcr.io/risingwavelabs/risingwave:nightly-20240204
+    image: ghcr.io/risingwavelabs/risingwave:v1.10.1
     command: "standalone --meta-opts=\" \
       --advertise-addr 0.0.0.0:5690 \
       --backend mem \

conda/environment-arm64-flink.yml

@@ -96,4 +96,4 @@ dependencies:
   - py4j =0.10.9.7
   - pip
   - pip:
-      - apache-flink =1.19.1
+      - apache-flink =1.20.0

conda/environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
 dependencies:
   # runtime dependencies
-  - apache-flink
+  - apache-flink =1.20.0
   - atpublic >=2.3
   - black >=22.1.0,<25
   - clickhouse-connect >=0.5.23

docker/flink/Dockerfile

@@ -1,8 +1,8 @@
-ARG FLINK_VERSION=1.19.1
+ARG FLINK_VERSION=1.20.0
 FROM flink:${FLINK_VERSION}
 
 # ibis-flink requires PyFlink dependency
-ARG FLINK_VERSION=1.19.1
+ARG FLINK_VERSION=1.20.0
 RUN wget -nv -P $FLINK_HOME/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-python/${FLINK_VERSION}/flink-python-${FLINK_VERSION}.jar
 
 # install python3 and pip3

docker/mysql/startup.sql

@@ -1,4 +1,4 @@
 CREATE USER 'ibis'@'localhost' IDENTIFIED BY 'ibis';
 CREATE SCHEMA IF NOT EXISTS test_schema;
-GRANT CREATE,SELECT,DROP ON *.* TO 'ibis'@'%';
+GRANT CREATE,SELECT,DROP,INSERT ON *.* TO 'ibis'@'%';
 FLUSH PRIVILEGES;

docs/_quarto.yml

@@ -168,6 +168,7 @@ website:
         - section: UDFs
           contents:
             - reference/scalar-udfs.qmd
+            - reference/aggregate-udfs.qmd
 
         - section: Connection APIs
           contents:

docs/_tabsets/install.qmd

@@ -10,7 +10,6 @@ from textwrap import dedent
 backends = [
     {"name": "BigQuery", "module": "bigquery"},
     {"name": "ClickHouse", "module": "clickhouse"},
-    {"name": "Dask", "module": "dask"},
     {"name": "DataFusion", "module": "datafusion"},
     {"name": "Druid", "module": "druid"},
     {"name": "DuckDB", "module": "duckdb"},
@@ -20,7 +19,6 @@ backends = [
     {"name": "MSSQL", "module": "mssql"},
     {"name": "MySQL", "module": "mysql"},
     {"name": "Oracle", "module": "oracle"},
-    {"name": "pandas", "module": "pandas"},
     {"name": "Polars", "module": "polars"},
     {"name": "PostgreSQL", "module": "postgres"},
     {"name": "PySpark", "module": "pyspark"},

docs/backends/dask.qmd

@@ -4,6 +4,13 @@
 
 ![](https://img.shields.io/badge/memtables-native-green?style=flat-square) ![](https://img.shields.io/badge/inputs-CSV | Parquet-blue?style=flat-square) ![](https://img.shields.io/badge/outputs-CSV | pandas | Parquet | PyArrow-orange?style=flat-square)
 
+::: {.callout-warning}
+## The Dask backend is slated for removal in Ibis 10.0
+We recommend using one of our other backends.
+
+Many workloads work well on the DuckDB and Polars backends, for example.
+:::
+
 ## Install
 
 Install Ibis and dependencies for the Dask backend:

docs/backends/pandas.qmd

@@ -4,6 +4,14 @@
 
 ![](https://img.shields.io/badge/memtables-native-green?style=flat-square) ![](https://img.shields.io/badge/inputs-CSV | Parquet-blue?style=flat-square) ![](https://img.shields.io/badge/outputs-CSV | pandas | Parquet | PyArrow-orange?style=flat-square)
 
+::: {.callout-warning}
+## The Pandas backend is slated for removal in Ibis 10.0
+We recommend using one of our other backends.
+
+Many workloads work well on the DuckDB and Polars backends, for example.
+:::
+
+
 ## Install
 
 Install Ibis and dependencies for the pandas backend:

docs/backends/snowflake.qmd

@@ -79,7 +79,8 @@ con = ibis.snowflake.connect(
     user="user",
     password="password",
     account="safpqpq-sq55555",
-    database="IBIS_TESTING/IBIS_TESTING",
+    database="my_database",
+    schema="my_schema",
 )
 ```
 
@@ -119,7 +120,8 @@ APIs. All values of `authenticator` are supported.
 con = ibis.snowflake.connect(
     user="user",
     account="safpqpq-sq55555",
-    database="my_database/my_schema",
+    database="my_database",
+    schema="my_schema",
     warehouse="my_warehouse",
     authenticator="externalbrowser",
 )
@@ -129,11 +131,41 @@ con = ibis.snowflake.connect(
 
 ```python
 con = ibis.connect(
-    f"snowflake://{user}@{account}/{database}?warehouse={warehouse}",
+    f"snowflake://{user}@{account}/{database}/{schema}?warehouse={warehouse}",
     authenticator="externalbrowser",
 )
 ```
 
+### Authenticating with Key Pair Authentication
+
+Ibis supports connecting to Snowflake warehouses using private keys.
+
+You can use it in the explicit-parameters-style or in the URL-style connection
+APIs.
+
+#### Explicit
+
+```python
+con = ibis.snowflake.connect(
+    user="user",
+    account="safpqpq-sq55555",
+    database="my_database",
+    schema="my_schema",
+    warehouse="my_warehouse",
+    # extracted private key from .p8 file
+    private_key=os.getenv(SNOWFLAKE_PKEY),
+)
+```
+
+#### URL
+
+```python
+con = ibis.connect(
+    f"snowflake://{user}@{account}/{database}/{schema}?warehouse={warehouse}",
+    private_key=os.getenv(SNOWFLAKE_PKEY),
+)
+```
+
 ### Looking up your Snowflake organization ID and user ID
 
 A [Snowflake account

docs/concepts/datatypes.qmd

@@ -100,6 +100,21 @@ t2 = ibis.examples.population.fetch().head(100)
 t2
 ```
 
+```{python}
+#| echo: false
+import IPython
+import rich
+import rich.traceback as rtb
+rtb.install(
+    suppress=[IPython, rich],
+    width=84,
+    word_wrap=True,
+    max_frames=1,
+    code_width=84,
+)
+del IPython, rich, rtb
+```
+
 ```{python}
 #| error: true
 t1.bill_depth_mm + t2.population