.envrc

@@ -2,3 +2,4 @@ use flake
 watch_file poetry.lock
 
 export CLOUDSDK_ACTIVE_CONFIG_NAME=ibis-gbq
+export SQLALCHEMY_WARN_20=1

.gitattributes

@@ -2,4 +2,5 @@ conda-lock/*.lock linguist-generated=true
 flake.lock linguist-generated=true
 poetry.lock linguist-generated=true
 requirements.txt linguist-generated=true
+docs/backends/app/requirements.txt linguist-generated=false
 docs/**/*.ipynb linguist-generated=true

.github/renovate.json

@@ -4,7 +4,18 @@
   "schedule": ["after 10pm and before 5am every weekday", "every weekend"],
   "semanticCommits": "enabled",
   "lockFileMaintenance": { "enabled": true },
-  "enabledManagers": ["docker-compose", "github-actions", "poetry"],
+  "enabledManagers": [
+    "docker-compose",
+    "dockerfile",
+    "github-actions",
+    "poetry"
+  ],
   "automerge": false,
-  "labels": ["dependencies"]
+  "labels": ["dependencies"],
+  "packageRules": [
+    {
+      "matchManagers": ["docker-compose", "dockerfile", "github-actions"],
+      "automerge": true
+    }
+  ]
 }

.github/workflows/check-generated-files.yml

@@ -46,7 +46,7 @@ jobs:
         run: poetry lock --check
 
       - name: generate requirements.txt
-        run: poetry export --with dev --with test --with docs --without-hashes --no-ansi > requirements.txt
+        run: poetry export --extras all --with dev --with test --with docs --without-hashes --no-ansi > requirements.txt
 
       - name: check requirements.txt
         run: git diff --exit-code requirements.txt

.github/workflows/ci-data.yml

@@ -55,15 +55,3 @@ jobs:
 
           bucket="gs://ibis-workflow-data/${yesterday}"
           gsutil cp -Z workflows.json jobs.json "${bucket}"
-
-          set +e
-
-          exit_code=0
-          for table in workflows jobs; do
-            if ! bq load --autodetect=true --source_format=NEWLINE_DELIMITED_JSON \
-              "workflows_native.${table}" "${bucket}/${table}.json"; then
-              ((++exit_code))
-            fi
-          done
-
-          exit "$exit_code"

.github/workflows/conda-lock.yml

@@ -5,9 +5,6 @@ on:
     # At minute 00:30 on Sunday
     - cron: "30 0 * * SUN"
   workflow_dispatch:
-  repository_dispatch:
-    types:
-      - condalock-command
 
 jobs:
   condalock:
@@ -38,7 +35,7 @@ jobs:
           condarc-file: ci/conda-lock/condarc
 
       - name: install conda-lock
-        run: mamba install conda-lock
+        run: mamba install 'conda-lock>=1.4'
 
       - name: generate lock file
         run: ./ci/conda-lock/generate.sh "${{ matrix.python-version }}"

.github/workflows/ibis-backends.yml

@@ -35,6 +35,8 @@ jobs:
   test_backends:
     name: ${{ matrix.backend.title }} ${{ matrix.os }} python-${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
+    env:
+      SQLALCHEMY_WARN_20: "1"
     strategy:
       fail-fast: false
       matrix:
@@ -286,6 +288,8 @@ jobs:
   test_backends_min_version:
     name: ${{ matrix.backend.title }} Min Version ${{ matrix.os }} python-${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
+    env:
+      SQLALCHEMY_WARN_20: "1"
     strategy:
       fail-fast: false
       matrix:

.github/workflows/ibis-docs-lint.yml

@@ -60,7 +60,7 @@ jobs:
 
       # run against the full shell.nix on push so it gets pushed to cachix
       - name: pre-commit checks
-        run: nix develop '.#preCommit' --ignore-environment --keep-going -c pre-commit run --all-files
+        run: nix develop '.#preCommit' --ignore-environment --keep-going -c pre-commit run --all-files --show-diff-on-failure --color=always
 
   benchmarks:
     runs-on: ubuntu-latest
@@ -147,7 +147,7 @@ jobs:
   docs_push:
     runs-on: ubuntu-latest
     if: github.event_name == 'push'
-    concurrency: docs-${{ github.repository }}-${{ github.head_ref || github.sha }}
+    concurrency: docs-${{ github.repository }}
     needs:
       # wait on benchmarks to prevent a race condition when pushing to the
       # gh-pages branch

.github/workflows/ibis-docs-release.yml

@@ -6,7 +6,7 @@ on:
       - published
 jobs:
   docs:
-    concurrency: docs-${{ github.repository }}-${{ github.head_ref || github.sha }}
+    concurrency: docs-${{ github.repository }}
     runs-on: ubuntu-latest
     steps:
       - name: install nix

.github/workflows/ibis-main.yml

@@ -30,6 +30,8 @@ concurrency:
 jobs:
   test_no_backends:
     name: Test ${{ matrix.os }} python-${{ matrix.python-version }}
+    env:
+      SQLALCHEMY_WARN_20: "1"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/update-deps.yml

@@ -76,7 +76,7 @@ jobs:
           app_id: ${{ secrets.PR_APPROVAL_BOT_APP_ID }}
           private_key: ${{ secrets.PR_APPROVAL_BOT_APP_PRIVATE_KEY }}
 
-      - uses: cpcloud/compare-commits-action@v5.0.27
+      - uses: cpcloud/compare-commits-action@v5.0.28
         if: fromJSON(steps.needs_pr.outputs.did_change)
         id: compare_commits
         with:

.pre-commit-config.yaml

@@ -14,21 +14,20 @@ default_stages:
   - commit
 repos:
   - repo: https://github.com/rhysd/actionlint
-    rev: v1.6.22
+    rev: v1.6.23
     hooks:
       - id: actionlint
   - repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
       - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.215
+    rev: v0.0.231
     hooks:
       - id: ruff
-        # exclude a file (if configured to do so), even if it's passed in explicitly
-        args: ["--force-exclude"]
+        args: ["--fix", "--show-source"]
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.28.0
+    rev: v1.29.0
     hooks:
       - id: yamllint
   - repo: https://github.com/pre-commit/pre-commit-hooks

.releaserc.js

@@ -63,7 +63,8 @@ module.exports = {
     [
       "@semantic-release/exec",
       {
-        verifyConditionsCmd: "ci/release/verify.sh ${options.dryRun}",
+        verifyConditionsCmd: "ci/release/verify_conditions.sh ${options.dryRun}",
+        verifyReleaseCmd: "ci/release/verify_release.sh ${nextRelease.version}",
         prepareCmd: "ci/release/prepare.sh ${nextRelease.version}",
         publishCmd: "ci/release/publish.sh",
       },

.streamlit/config.toml

@@ -0,0 +1,5 @@
+[browser]
+gatherUsageStats = false
+
+[theme]
+base = "dark"

ci/conda-lock/generate.sh

@@ -38,7 +38,8 @@ conda lock \
   --filename-template "${template}" \
   --filter-extras \
   --mamba \
-  "${extras[@]}" -e clickhouse
+  --category dev --category test --category docs \
+  "${extras[@]}" -e clickhouse -e datafusion
 
 conda lock \
   --file pyproject.toml \
@@ -49,4 +50,5 @@ conda lock \
   --filename-template "${template}" \
   --filter-extras \
   --mamba \
+  --category dev --category test --category docs \
   "${extras[@]}"

ci/release/verify_release.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import argparse
+import ast
+import concurrent.futures
+import itertools
+import sys
+from functools import partial
+from pathlib import Path
+from typing import NamedTuple
+
+from packaging import version as v
+
+
+class RemovedIn(NamedTuple):
+    version: v.Version
+    lineno: int
+
+
+class RemovedInVisitor(ast.NodeVisitor):
+    def __init__(self) -> None:
+        super().__init__()
+        self.removed_ins: list[RemovedIn] = []
+
+    def visit_keyword(self, node: ast.keyword) -> None:
+        if node.arg == "removed_in":
+            value = node.value
+            if isinstance(value, ast.Constant):
+                self.removed_ins.append(
+                    RemovedIn(
+                        version=normalize_version(value.value), lineno=node.lineno
+                    )
+                )
+
+
+def normalize_version(vstr: str) -> v.Version:
+    ver = v.parse(vstr)
+    return v.parse(f"{ver.major}.{ver.minor}.{ver.micro}")
+
+
+def find_invalid_removed_ins(
+    next_ver: v.Version, path: Path
+) -> list[tuple[Path, RemovedIn]]:
+    code = path.read_text()
+    node = ast.parse(code)
+    visitor = RemovedInVisitor()
+    visitor.visit(node)
+    return [
+        (path, removed_in)
+        for removed_in in visitor.removed_ins
+        if removed_in.version <= next_ver
+    ]
+
+
+def main(next_ver: v.Version, root: Path) -> int:
+    pyfiles = root.rglob("*.py")
+    find_invalid = partial(find_invalid_removed_ins, next_ver)
+
+    # this is noticeably faster than serial execution
+    with concurrent.futures.ProcessPoolExecutor() as e:
+        msgs = e.map(find_invalid, pyfiles)
+
+    infos = sorted(itertools.chain.from_iterable(msgs))
+
+    for path, removed_in in infos:
+        print(
+            f"{path}:{removed_in.lineno:d} (removed_in={removed_in.version} <= next={next_ver})",
+            file=sys.stderr,
+        )
+    return len(infos)
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(
+        description="Verify that removed_in= matches are greater than the next version."
+    )
+    p.add_argument("next_version", help="The next release version.")
+    p.add_argument(
+        "-r",
+        "--root",
+        type=Path,
+        default=Path(),
+        help="Root directory to search for Python modules. Defaults to the current directory.",
+    )
+
+    args = p.parse_args()
+
+    sys.exit(main(normalize_version(args.next_version), args.root))

ci/release/verify_release.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+next_version="${1}"
+python ci/release/verify_release.py "$next_version" --root "$PWD"

ci/schema/clickhouse.sql

@@ -1,55 +1,19 @@
-CREATE OR REPLACE TABLE diamonds (
-    carat Nullable(Float64),
-    cut Nullable(String),
-    color Nullable(String),
-    clarity Nullable(String),
-    depth Nullable(Float64),
-    `table` Nullable(Float64),
-    price Nullable(Int64),
-    x Nullable(Float64),
-    y Nullable(Float64),
-    z Nullable(Float64)
-) ENGINE = Memory;
+-- NB: The paths in this file are all relative to /var/lib/clickhouse/user_files
 
-CREATE OR REPLACE TABLE batting (
-    `playerID` Nullable(String),
-    `yearID` Nullable(Int64),
-    stint Nullable(Int64),
-    `teamID` Nullable(String),
-    `lgID` Nullable(String),
-    `G` Nullable(Int64),
-    `AB` Nullable(Int64),
-    `R` Nullable(Int64),
-    `H` Nullable(Int64),
-    `X2B` Nullable(Int64),
-    `X3B` Nullable(Int64),
-    `HR` Nullable(Int64),
-    `RBI` Nullable(Int64),
-    `SB` Nullable(Int64),
-    `CS` Nullable(Int64),
-    `BB` Nullable(Int64),
-    `SO` Nullable(Int64),
-    `IBB` Nullable(Int64),
-    `HBP` Nullable(Int64),
-    `SH` Nullable(Int64),
-    `SF` Nullable(Int64),
-    `GIDP` Nullable(Int64)
-) ENGINE = Memory;
+CREATE OR REPLACE TABLE diamonds ENGINE = Memory AS
+SELECT * FROM file('parquet/diamonds/diamonds.parquet', 'Parquet');
 
-CREATE OR REPLACE TABLE awards_players (
-    `playerID` Nullable(String),
-    `awardID` Nullable(String),
-    `yearID` Nullable(Int64),
-    `lgID` Nullable(String),
-    tie Nullable(String),
-    notes Nullable(String)
-) ENGINE = Memory;
+CREATE OR REPLACE TABLE batting ENGINE = Memory AS
+SELECT * FROM file('parquet/batting/batting.parquet', 'Parquet');
+
+CREATE OR REPLACE TABLE awards_players ENGINE = Memory AS
+SELECT * FROM file('parquet/awards_players/awards_players.parquet', 'Parquet');
 
 CREATE OR REPLACE TABLE functional_alltypes (
     `index` Nullable(Int64),
     `Unnamed: 0` Nullable(Int64),
     id Nullable(Int32),
-    bool_col Nullable(UInt8),
+    bool_col Nullable(Bool),
     tinyint_col Nullable(Int8),
     smallint_col Nullable(Int16),
     int_col Nullable(Int32),
@@ -58,10 +22,12 @@ CREATE OR REPLACE TABLE functional_alltypes (
     double_col Nullable(Float64),
     date_string_col Nullable(String),
     string_col Nullable(String),
+    -- TODO: clean this up when timestamp scale is supported
     timestamp_col Nullable(DateTime),
     year Nullable(Int32),
     month Nullable(Int32)
-) ENGINE = Memory;
+) ENGINE = Memory AS
+SELECT * FROM file('functional_alltypes.csv', 'CSVWithNames');
 
 CREATE OR REPLACE TABLE tzone (
     ts Nullable(DateTime),