Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,5 @@ Dockerfile* text
# crate; regenerated by `npm run build:native`. Tell git/GitHub they're
# machine-generated so they collapse in diffs and are excluded from
# blame and language stats.
native/sea/index.d.ts linguist-generated=true
native/sea/index.js linguist-generated=true
native/kernel/index.d.ts linguist-generated=true
native/kernel/index.js linguist-generated=true
54 changes: 27 additions & 27 deletions .github/workflows/kernel-e2e.yml
Original file line number Diff line number Diff line change
@@ -1,31 +1,31 @@
name: Kernel E2E Tests

# Runs the SEA backend e2e suite (tests/e2e/sea/**) against a real
# Runs the kernel backend e2e suite (tests/e2e/kernel/**) against a real
# Databricks warehouse with a freshly-built napi-rs kernel binding.
#
# The kernel is a private repo with no published binary artifact. We pin
# a kernel SHA in the `KERNEL_REV` file at the repo root, check the kernel
# out via a GitHub App token, and run `npm run build:native` to compile
# the napi binding into native/sea/ in the same checkout the tests run
# the napi binding into native/kernel/ in the same checkout the tests run
# against. Bumping `KERNEL_REV` is the ONLY way to pick up a new kernel
# version — this keeps the driver <-> kernel pair bisectable, so a driver
# change and the kernel revision it depends on always land together.
#
# Why this exists: the committed native/sea/index.d.ts + index.js are the
# Why this exists: the committed native/kernel/index.d.ts + index.js are the
# TypeScript declarations and the napi-rs platform router; the actual
# `.node` binary is gitignored (large, per-platform) and is NOT in the
# repo. The standard `main.yml` e2e job has no binary, so its SEA suite
# repo. The standard `main.yml` e2e job has no binary, so its kernel suite
# skips (it gates on DATABRICKS_PECOTESTING_* secrets it doesn't set).
# This workflow is what actually exercises the SEA path end-to-end against
# This workflow is what actually exercises the kernel path end-to-end against
# a known kernel revision.
#
# Gate semantics:
# - Plain PR events post a synthetic-success check so the required
# "Kernel E2E" check doesn't block PRs that don't touch the SEA path.
# "Kernel E2E" check doesn't block PRs that don't touch the kernel path.
# Real tests run in the merge queue.
# - `kernel-e2e` label triggers a preview run on the PR; the label is
# auto-removed on `synchronize` for the same security reason.
# - merge_group fires the real gate — runs when SEA-relevant files
# - merge_group fires the real gate — runs when kernel-relevant files
# changed, auto-passes otherwise.
#
# Required external setup (one-time, by a repo admin):
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:

# ───────────────────────────────────────────────────────────────
# Synthetic success on every non-label PR event so the required
# "Kernel E2E" check doesn't permablock PRs that don't touch SEA
# "Kernel E2E" check doesn't permablock PRs that don't touch kernel
# code. Real run happens in the merge queue (or via explicit label).
# ───────────────────────────────────────────────────────────────
skip-kernel-e2e-pr:
Expand Down Expand Up @@ -112,7 +112,7 @@ jobs:
});

# ───────────────────────────────────────────────────────────────
# Detect whether SEA-relevant files changed. Used by both the
# Detect whether kernel-relevant files changed. Used by both the
# labelled-PR path and the merge-queue path to decide between
# "really run the suite" and "auto-pass the check".
# ───────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -148,7 +148,7 @@ jobs:
ref: ${{ steps.refs.outputs.head_sha }}
fetch-depth: 0

- name: Detect SEA-relevant changes
- name: Detect kernel-relevant changes
id: changed
env:
HEAD_SHA: ${{ steps.refs.outputs.head_sha }}
Expand All @@ -157,18 +157,18 @@ jobs:
CHANGED=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")
echo "Changed files:"
echo "$CHANGED"
# Run when the SEA driver layer, the napi binding contract, SEA
# Run when the kernel driver layer, the napi binding contract, kernel
# e2e tests, this workflow, the kernel revision pin, or core deps
# move.
if echo "$CHANGED" | grep -qE "^(lib/sea/|native/sea/|tests/e2e/sea/|tests/unit/sea/|\.github/workflows/kernel-e2e\.yml|KERNEL_REV|package\.json|package-lock\.json)"; then
if echo "$CHANGED" | grep -qE "^(lib/kernel/|native/kernel/|tests/e2e/kernel/|tests/unit/kernel/|\.github/workflows/kernel-e2e\.yml|KERNEL_REV|package\.json|package-lock\.json)"; then
echo "run_tests=true" >> "$GITHUB_OUTPUT"
else
echo "run_tests=false" >> "$GITHUB_OUTPUT"
fi

# ───────────────────────────────────────────────────────────────
# Real test job. Builds the napi binding from the pinned kernel SHA
# and runs the SEA e2e suite against the dogfood warehouse.
# and runs the kernel e2e suite against the dogfood warehouse.
# ───────────────────────────────────────────────────────────────
run-kernel-e2e:
needs: detect-changes
Expand All @@ -182,17 +182,17 @@ jobs:
checks: write
id-token: write
env:
# The tests/e2e/sea suite reads creds from TWO sources, so we set both:
# The tests/e2e/kernel suite reads creds from TWO sources, so we set both:
#
# 1. Most SEA tests (execution, results, interval-*) read the
# 1. Most kernel tests (execution, results, interval-*) read the
# DATABRICKS_PECOTESTING_* vars directly and skip when absent.
DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
DATABRICKS_PECOTESTING_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_TOKEN }}
#
# 2. `e2e-smoke.test.ts` imports tests/e2e/utils/config.ts, which
# `process.exit(1)`s at module load if ANY E2E_* var is unset — which
# would abort the whole `tests/e2e/sea/**` mocha run before it starts.
# would abort the whole `tests/e2e/kernel/**` mocha run before it starts.
# Mirror main.yml's e2e-test job mapping so the suite loads + runs.
E2E_HOST: ${{ secrets.DATABRICKS_HOST }}
E2E_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
Expand Down Expand Up @@ -304,31 +304,31 @@ jobs:

- name: Build napi binding from pinned kernel
# build:native cd's into ${DATABRICKS_SQL_KERNEL_REPO}/napi, runs the
# napi-rs build, and copies index.* into native/sea/. Pointing it at
# napi-rs build, and copies index.* into native/kernel/. Pointing it at
# the SHA-pinned kernel checkout is what makes the binary match
# KERNEL_REV exactly.
env:
DATABRICKS_SQL_KERNEL_REPO: ${{ github.workspace }}/databricks-sql-kernel
run: npm run build:native

- name: Assert committed binding matches KERNEL_REV
# The committed native/sea/index.d.ts + index.js are the consumer-facing
# The committed native/kernel/index.d.ts + index.js are the consumer-facing
# type contract + platform router; they MUST correspond to the pinned
# kernel. build:native just regenerated them from the KERNEL_REV
# checkout, so any diff means the committed contract drifted from the
# pin — fail loudly and tell the author to commit the regenerated files.
# (The .node binaries are gitignored, so git diff only sees the contract.)
run: |
if ! git diff --exit-code -- native/sea/index.d.ts native/sea/index.js; then
echo "::error::native/sea/index.d.ts / index.js are out of sync with KERNEL_REV ($(tr -d '[:space:]' < KERNEL_REV)). Run 'npm run build:native' against that kernel SHA and commit native/sea/index.*."
if ! git diff --exit-code -- native/kernel/index.d.ts native/kernel/index.js; then
echo "::error::native/kernel/index.d.ts / index.js are out of sync with KERNEL_REV ($(tr -d '[:space:]' < KERNEL_REV)). Run 'npm run build:native' against that kernel SHA and commit native/kernel/index.*."
exit 1
fi
echo "Committed binding matches KERNEL_REV."

- name: Smoke-check binding loads
run: node -e "const b=require('./native/sea'); if(typeof b.version!=='function'){throw new Error('napi binding failed to load')} console.log('kernel binding ok:', b.version())"
run: node -e "const b=require('./native/kernel'); if(typeof b.version!=='function'){throw new Error('napi binding failed to load')} console.log('kernel binding ok:', b.version())"

- name: Run SEA e2e tests
- name: Run kernel e2e tests
# Run through `nyc` (not bare `mocha`) so the `require: ['ts-node/register']`
# in nyc.config.js loads the .ts specs via the CommonJS ts-node hook —
# exactly how main.yml's e2e job runs them. Bare `mocha` has no ts-node
Expand All @@ -340,7 +340,7 @@ jobs:
# index 4 (`mocha --config <file> <glob>`): tests/e2e/.mocharc.js derives
# `spec` from `process.argv.slice(4)`. We invoke mocha directly (not via
# `npm run e2e -- <glob>`, whose inner shell mangles `**` to zero files).
run: NODE_OPTIONS="--max-old-space-size=4096" npx nyc --report-dir coverage_kernel_e2e mocha --config tests/e2e/.mocharc.js "tests/e2e/sea/**/*.test.ts"
run: NODE_OPTIONS="--max-old-space-size=4096" npx nyc --report-dir coverage_kernel_e2e mocha --config tests/e2e/.mocharc.js "tests/e2e/kernel/**/*.test.ts"

- name: Post Kernel E2E check (success)
if: success()
Expand All @@ -358,7 +358,7 @@ jobs:
completed_at: new Date().toISOString(),
output: {
title: 'Kernel E2E passed',
summary: 'tests/e2e/sea ran green against the pinned kernel SHA.'
summary: 'tests/e2e/kernel ran green against the pinned kernel SHA.'
}
});

Expand All @@ -383,7 +383,7 @@ jobs:
});

# ───────────────────────────────────────────────────────────────
# Auto-pass the Kernel E2E check in the merge queue when no SEA-
# Auto-pass the Kernel E2E check in the merge queue when no kernel-
# relevant files changed.
# ───────────────────────────────────────────────────────────────
auto-pass-merge-queue:
Expand All @@ -409,7 +409,7 @@ jobs:
conclusion: 'success',
completed_at: new Date().toISOString(),
output: {
title: 'Skipped — no SEA-relevant changes',
summary: 'No files under lib/sea/, native/sea/, tests/e2e/sea/, tests/unit/sea/, KERNEL_REV, package.json, or package-lock.json changed.'
title: 'Skipped — no kernel-relevant changes',
summary: 'No files under lib/kernel/, native/kernel/, tests/e2e/kernel/, tests/unit/kernel/, KERNEL_REV, package.json, or package-lock.json changed.'
}
});
10 changes: 5 additions & 5 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ dist
*.DS_Store
lib/version.ts

# SEA native binding — copied/generated from kernel workspace by `npm run build:native`.
# The committed contract is `native/sea/index.d.ts` (TypeScript declarations) and
# `native/sea/index.js` (the napi-rs platform router — small, stable, and required in
# kernel native binding — copied/generated from kernel workspace by `npm run build:native`.
# The committed contract is `native/kernel/index.d.ts` (TypeScript declarations) and
# `native/kernel/index.js` (the napi-rs platform router — small, stable, and required in
# the publish tarball so a missing build step can't ship a tarball that can't load).
# The `.node` binaries are large per-platform artifacts and must NOT be committed;
# in production they arrive via the `@databricks/sql-kernel-<triple>` optional deps.
native/sea/index.node
native/sea/index.*.node
native/kernel/index.node
native/kernel/index.*.node
6 changes: 3 additions & 3 deletions .npmignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
!dist/**/*
!thrift/**/*

# SEA napi-rs router shim + TypeScript declarations. The router (index.js)
# kernel napi-rs router shim + TypeScript declarations. The router (index.js)
# selects the per-platform `.node` artifact from `@databricks/sql-kernel-*`
# optionalDependencies (populated when the kernel CI publishes them);
# the .d.ts is the consumer-facing type contract.
!native/sea/index.js
!native/sea/index.d.ts
!native/kernel/index.js
!native/kernel/index.d.ts

!LICENSE
!NOTICE
Expand Down
4 changes: 2 additions & 2 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,5 @@ package-lock.json
# Generated by napi-rs from the kernel's `napi-binding/napi/` crate;
# regenerated by `npm run build:native`. Format follows napi-rs's
# defaults (no semicolons), not this repo's prettier config.
native/sea/index.d.ts
native/sea/index.js
native/kernel/index.d.ts
native/kernel/index.js
10 changes: 5 additions & 5 deletions lib/DBSQLClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { buildUserAgentString } from './utils';
import IBackend from './contracts/IBackend';
import { InternalConnectionOptions } from './contracts/InternalConnectionOptions';
import ThriftBackend from './thrift-backend/ThriftBackend';
import SeaBackend from './sea/SeaBackend';
import KernelBackend from './kernel/KernelBackend';
import PlainHttpAuthentication from './connection/auth/PlainHttpAuthentication';
import DatabricksOAuth, { OAuthFlow } from './connection/auth/DatabricksOAuth';
import {
Expand Down Expand Up @@ -627,12 +627,12 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I

this.connectionProvider = this.createConnectionProvider(options);

// M0: `useSEA` is consumed via a non-exported internal-options cast so it
// doesn't ship in the public `.d.ts`. Mirrors Python's `kwargs.get("use_sea")`
// M0: `useKernel` is consumed via a non-exported internal-options cast so it
// doesn't ship in the public `.d.ts`. Mirrors Python's `kwargs.get("use_kernel")`
// pattern (see databricks-sql-python/src/databricks/sql/session.py).
const internalOptions = options as ConnectionOptions & InternalConnectionOptions;
const backend = internalOptions.useSEA
? new SeaBackend({ context: this })
const backend = internalOptions.useKernel
? new KernelBackend({ context: this })
: new ThriftBackend({
context: this,
onConnectionEvent: (event, payload) => this.forwardConnectionEvent(event, payload),
Expand Down
2 changes: 1 addition & 1 deletion lib/DBSQLLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export default class DBSQLLogger implements IDBSQLLogger {
file?: winston.transports.FileTransportInstance;
};

// Subscribers notified on `setLevel(...)` — used by the SEA/kernel backend to
// Subscribers notified on `setLevel(...)` — used by the kernel backend to
// keep the kernel-side log bridge's verbosity in lock-step with this logger.
private levelListeners: Array<(level: LogLevel) => void> = [];

Expand Down
2 changes: 1 addition & 1 deletion lib/DBSQLOperation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ export default class DBSQLOperation implements IOperation {
/**
* Requests operation status. Returns the Thrift wire response for
* back-compat with existing user code. On the Thrift backend the response
* is returned verbatim; on any other backend (e.g. SEA) the response is
* is returned verbatim; on any other backend (e.g. kernel) the response is
* synthesized from the neutral {@link IOperationBackend.status} result,
* with Thrift-only fields (`taskStatus`, `numModifiedRows`, etc.) left
* undefined.
Expand Down
4 changes: 2 additions & 2 deletions lib/contracts/IBackend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ import ISessionBackend from './ISessionBackend';

/**
* Top-level backend dispatch handle. One instance per `DBSQLClient`,
* chosen at `connect()` time based on the `useSEA` flag and never
* chosen at `connect()` time based on the `useKernel` flag and never
* re-selected per-call.
*/
export default interface IBackend {
/**
* Establish backend-level state before any session is opened. Implementations
* consume `options` to build backend-specific connection parameters (e.g. the
* SEA backend derives napi-binding `SeaNativeConnectionOptions` from the auth
* kernel backend derives napi-binding `KernelNativeConnectionOptions` from the auth
* + host fields here). Transport-layer connection providers are owned by
* `DBSQLClient` (via `IClientContext`) and exposed to backends through
* constructor injection.
Expand Down
4 changes: 2 additions & 2 deletions lib/contracts/IDBSQLLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export default interface IDBSQLLogger {
log(level: LogLevel, message: string): void;

/**
* Optional: the logger's current level. When implemented, the SEA/kernel
* Optional: the logger's current level. When implemented, the kernel
* backend uses it to set the verbosity of the kernel-side (Rust) log bridge,
* so kernel logs are filtered at the same level as the driver's own logs and
* land in the same sink. Loggers that don't implement it leave the kernel
Expand All @@ -17,7 +17,7 @@ export default interface IDBSQLLogger {

/**
* Optional: subscribe to runtime level changes. When implemented, the
* SEA/kernel backend subscribes so a runtime `setLevel(...)` retargets the
* kernel backend subscribes so a runtime `setLevel(...)` retargets the
* kernel-side log bridge too (not just the driver's own transports) — keeping
* kernel verbosity in lock-step with the driver's. Returns an unsubscribe
* function. Loggers that don't implement it still get the connect-time level;
Expand Down
6 changes: 3 additions & 3 deletions lib/contracts/IDBSQLSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export type ExecuteStatementOptions = {
* - **Thrift backend:** no-op. The Thrift path always submits asynchronously
* (`runAsync: true` on the wire) and polls during fetch; this option is not
* read.
* - **Kernel backend (`useSEA`):** selects the kernel execution path —
* - **Kernel backend (`useKernel`):** selects the kernel execution path —
* `false`/unset (default) runs the blocking direct-results path (faster,
* cancellable mid-compute); `true` submits and polls (returns a pending
* handle before completion). Default is sync, matching the python
Expand All @@ -39,13 +39,13 @@ export type ExecuteStatementOptions = {
*/
queryTags?: Record<string, string | null | undefined>;
/**
* SEA-only: server-side row cap for this statement (kernel `row_limit`). The
* kernel-only: server-side row cap for this statement (kernel `row_limit`). The
* Thrift backend has no execute-time server cap, so this is a no-op there;
* use `maxRows` for the cross-backend client-side fetch limit.
*/
rowLimit?: number;
/**
* SEA-only: per-statement Spark conf overlay (kernel `statement_conf`).
* kernel-only: per-statement Spark conf overlay (kernel `statement_conf`).
* Merged with the serialized `queryTags` (which land under the reserved
* `query_tags` key). Ignored by the Thrift backend.
*/
Expand Down
Loading
Loading