.codespellrc

@@ -1,6 +1,6 @@
 [codespell]
 # local codespell matches `./docs`, pre-commit codespell matches `docs`
-skip = *.lock,.direnv,.git,./docs/_freeze,./docs/_output/**,./docs/_inv/**,docs/_freeze/**,*.svg,*.css,*.html,*.js,ibis/backends/tests/tpc/queries/duckdb/ds/44.sql
-ignore-regex = \b(i[if]f|I[IF]F|AFE)\b
+skip = *.lock,.direnv,.git,./docs/_freeze,./docs/_output/**,./docs/_inv/**,docs/_freeze/**,*.svg,*.css,*.html,*.js,ibis/backends/tests/tpc/queries/duckdb/ds/*.sql
+ignore-regex = \b(i[if]f|I[IF]F|AFE|alls)\b
 builtin = clear,rare,names
 ignore-words-list = tim,notin,ang

.env

@@ -1 +1,7 @@
-FLINK_VERSION=1.19.0
+FLINK_VERSION=1.19.1
+CLOUDSDK_ACTIVE_CONFIG_NAME=ibis-gbq
+GOOGLE_CLOUD_PROJECT="$CLOUDSDK_ACTIVE_CONFIG_NAME"
+PGPASSWORD="postgres"
+MYSQL_PWD="ibis"
+MSSQL_SA_PASSWORD="1bis_Testing!"
+DRUID_URL="druid://localhost:8082/druid/v2/sql"

.envrc

@@ -1,9 +1,8 @@
 use flake
 
+dotenv
+
 watch_file poetry.lock
 watch_file nix/overlay.nix
 watch_file nix/quarto/default.nix
 watch_file poetry-overrides.nix
-
-export CLOUDSDK_ACTIVE_CONFIG_NAME=ibis-gbq
-export GOOGLE_CLOUD_PROJECT="$CLOUDSDK_ACTIVE_CONFIG_NAME"

.github/workflows/algolia/configure-algolia-api.py

@@ -0,0 +1,35 @@
+from __future__ import annotations  # noqa: INP001
+
+import os
+
+from algoliasearch.search_client import SearchClient
+
+api_key = os.environ["ALGOLIA_WRITE_API_KEY"]
+app_id = os.environ["ALGOLIA_APP_ID"]
+index_name = os.environ["ALGOLIA_INDEX"]
+
+
+def main():
+    client = SearchClient.create(app_id, api_key)
+    index = client.init_index(index_name)
+
+    # Core is a custom attribute set to denote whether a record is part
+    # of the base expression API, we sort descending so those methods
+    # show up first in search instead of backend-specific methods
+    override_default_settings = {
+        "ranking": [
+            "typo",
+            "words",
+            "desc(core)",
+            "filters",
+            "proximity",
+            "attribute",
+            "exact",
+        ]
+    }
+
+    index.set_settings(override_default_settings)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/upload-algolia-api.py → .github/workflows/algolia/upload-algolia-api.py

@@ -62,13 +62,40 @@ def _create_api_record_from_method_line(base_url, method):
         "objectID": f"{base_url}{anchor}",
         "href": f"{base_url}{anchor}",
         "title": name,
-        "text": desc,
-        "crumbs": ["Expression API", "API", f"{section} expressions"],
+        "backend": "core",
+        "core": 1,
+        "crumbs": ["Expression API", "API", f"{section} expression"],
     }
+    if desc:
+        record["text"] = desc
 
     return record
 
 
+def adjust_backend_custom_attributes(backend_records):
+    """Adjusts attributes of the Algolia records.
+
+    Two custom attribute changes:
+        One is the name of the backend, which we can possibly use for grouping
+        or filtering results.
+
+        The other is a marker of whether the record is part of the core
+        expression API, which we can use to sort results so that generic table
+        expressions appear above backend-specific ones in the case of
+        name-collisions.
+
+    We also strip out the "text" attribute if it's empty
+    """
+    backend_name = backend_records[0]["title"].split(".", 1)[0]
+    for record in backend_records:
+        record["backend"] = backend_name
+        record["core"] = 0
+        if not record["text"]:
+            record.pop("text")
+
+    return backend_records
+
+
 def main():
     client = SearchClient.create(app_id, api_key)
     index = client.init_index(index_name)
@@ -106,8 +133,12 @@ def main():
     # Here, we load those records and upload them to the Algolia index
     records = []
     for record_json in glob.glob("docs/backends/*.json"):
+        print(f"Loading {record_json} methods...")  # noqa:T201
         with open(record_json) as f:
-            records.extend(json.load(f))
+            backend_records = json.load(f)
+            backend_records = adjust_backend_custom_attributes(backend_records)
+            records.extend(backend_records)
+    print(f"Uploading {len(records)} records to {index.name=}")  # noqa:T201
     index.save_objects(records)
 
 

.github/workflows/ibis-backends.yml

@@ -227,8 +227,8 @@ jobs:
             extras:
               - flink
             additional_deps:
-              - "'apache-flink < 1.20.0'"
-              - "'pandas < 2.2'"
+              - "'apache-flink==1.19.1'"
+              - "'pandas<2.2'"
               - setuptools
             services:
               - flink
@@ -249,8 +249,8 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - "'apache-flink < 1.20.0'"
-                - "'pandas < 2.2'"
+                - "'apache-flink==1.19.1'"
+                - "'pandas<2.2'"
                 - setuptools
               services:
                 - flink
@@ -400,8 +400,8 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - "'apache-flink < 1.20.0'"
-                - "'pandas < 2.2'"
+                - "'apache-flink==1.19.1'"
+                - "'pandas<2.2'"
                 - setuptools
               services:
                 - flink
@@ -413,8 +413,8 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - "'apache-flink < 1.20.0'"
-                - "'pandas < 2.2'"
+                - "'apache-flink==1.19.1'"
+                - "'pandas<2.2'"
                 - setuptools
               services:
                 - flink

.github/workflows/ibis-docs-main.yml

@@ -71,15 +71,23 @@ jobs:
 
       - name: Create and Upload Base Index
         run: |
-          python .github/workflows/upload-algolia.py
+          python .github/workflows/algolia/upload-algolia.py
         env:
           ALGOLIA_WRITE_API_KEY: ${{ secrets.ALGOLIA_WRITE_API_KEY }}
           ALGOLIA_APP_ID: TNU9HG3L41
           ALGOLIA_INDEX: prod_ibis
 
       - name: Create and Upload API Records to index
         run: |
-          python .github/workflows/upload-algolia-api.py
+          python .github/workflows/algolia/upload-algolia-api.py
+        env:
+          ALGOLIA_WRITE_API_KEY: ${{ secrets.ALGOLIA_WRITE_API_KEY }}
+          ALGOLIA_APP_ID: TNU9HG3L41
+          ALGOLIA_INDEX: prod_ibis
+
+      - name: Configure custom ranking on Algolia
+        run: |
+          python .github/workflows/algolia/configure-algolia-api.py
         env:
           ALGOLIA_WRITE_API_KEY: ${{ secrets.ALGOLIA_WRITE_API_KEY }}
           ALGOLIA_APP_ID: TNU9HG3L41

.github/workflows/ibis-main.yml

@@ -80,25 +80,25 @@ jobs:
       - name: install ibis
         run: poetry install --without dev --without docs --extras "visualization decompiler"
 
-      - name: install pyarrow
+      - name: install numpy/pandas/pyarrow
         if: matrix.pyarrow
-        run: poetry run pip install pyarrow pyarrow-hotfix
+        run: poetry run pip install numpy pandas pyarrow pyarrow-hotfix
 
-      - name: check pyarrow import
+      - name: check imports
         if: matrix.pyarrow
-        run: poetry run python -c 'import pyarrow, pyarrow_hotfix'
+        run: poetry run python -c 'import numpy, pandas, pyarrow, pyarrow_hotfix'
 
       - uses: extractions/setup-just@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: run all core tests and run benchmarks once parallel
         if: matrix.os != 'windows-latest'
-        run: just ci-check -m "'core or benchmarks'" --numprocesses auto -rs
+        run: just ci-check -m "'core or benchmark'" --numprocesses auto -rfEs
 
       - name: run all core tests and run benchmarks once serial
         if: matrix.os == 'windows-latest'
-        run: just ci-check -m "'core or benchmarks'" -rs
+        run: just ci-check -m "'core or benchmark'" -rfEs
 
       - name: upload code coverage
         if: success()

.github/workflows/nix.yml

@@ -59,12 +59,19 @@ jobs:
           name: ibis
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
           extraPullNames: nix-community,poetry2nix
-      - name: nix build and test
+      - name: nix build and test core
         run: |
           set -euo pipefail
 
           version='${{ matrix.python-version }}'
-          nix build ".#ibis${version//./}" --fallback --keep-going --print-build-logs
+          nix build ".#ibisCore${version//./}" --fallback --keep-going --print-build-logs
+
+      - name: nix build and test local
+        run: |
+          set -euo pipefail
+
+          version='${{ matrix.python-version }}'
+          nix build ".#ibisLocal${version//./}" --fallback --keep-going --print-build-logs
 
       - name: nix build devShell
         run: |

.github/workflows/set-milestone-on-issue.yml

@@ -0,0 +1,24 @@
+name: Set milestone on issues closed by PR
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+
+permissions:
+  issues: write
+  pull-requests: read
+
+jobs:
+  set-milestone:
+    if: github.repository == 'ibis-project/ibis'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: set milestone
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./ci/set-milestone-on-issue.sh "${GITHUB_SHA}"

.gitignore

@@ -152,3 +152,4 @@ docs/**/*.html
 # jupyterlite stuff
 .jupyterlite.doit.db
 docs/jupyter_lite_config.json
+*.quarto_ipynb

ci/linked-issues.gql

@@ -0,0 +1,11 @@
+query ($owner: String!, $repo: String!, $pr: Int!) {
+  repository(owner: $owner, name: $repo) {
+    pullRequest(number: $pr) {
+      closingIssuesReferences(first: 100) {
+        nodes {
+          number
+        }
+      }
+    }
+  }
+}

ci/schema/druid.sql

@@ -36,11 +36,24 @@ PARTITIONED BY ALL TIME;
 
 REPLACE INTO "functional_alltypes"
 OVERWRITE ALL
-SELECT *
+SELECT
+  "id",
+  "bool_col",
+  "tinyint_col",
+  "smallint_col",
+  "int_col",
+  "bigint_col",
+  "float_col",
+  "double_col",
+  "date_string_col",
+  "string_col",
+  TIME_PARSE(CONCAT(REPLACE("timestamp_col", ' ', 'T'), 'Z')) AS "timestamp_col",
+  "year",
+  "month"
 FROM TABLE(
   EXTERN(
-    '{"type":"local","files":["/data/functional_alltypes.parquet"]}',
-    '{"type":"parquet"}',
+    '{"type":"local","files":["/data/functional_alltypes.csv"]}',
+    '{"type":"csv","skipHeaderRows":1,"columns":["id","bool_col","tinyint_col","smallint_col","int_col","bigint_col","float_col","double_col","date_string_col","string_col","timestamp_col","year","month"]}',
     '[{"name":"id","type":"long"},{"name":"bool_col","type":"long"},{"name":"tinyint_col","type":"long"},{"name":"smallint_col","type":"long"},{"name":"int_col","type":"long"},{"name":"bigint_col","type":"long"},{"name":"float_col","type":"double"},{"name":"double_col","type":"double"},{"name":"date_string_col","type":"string"},{"name":"string_col","type":"string"},{"name":"timestamp_col","type":"string"},{"name":"year","type":"long"},{"name":"month","type":"long"}]'
   )
 )