From 2716a938824ba9d12d7b0e3e50cc0bec10d47781 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 12:49:49 +0000 Subject: [PATCH 1/2] =?UTF-8?q?refactor(spec,drivers)!:=20retire=20IDataDr?= =?UTF-8?q?iver.findStream=20=E2=80=94=20required,=20uncalled,=20and=20inv?= =?UTF-8?q?erted=20in=20two=20of=20three=20impls=20(#4484)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `findStream` was a REQUIRED method on the driver contract, documented as the read "optimized for large datasets to avoid memory overflow". Three things were true of it at once: - Nothing called it. Repo-wide, outside the declaration and the three driver implementations, every hit was a test double — and ~20 of those satisfied the required method by throwing `not implemented`. No test ever went red. - `SqlDriver` and `InMemoryDriver` awaited `find()` for the ENTIRE result set and then yielded row by row, so the memory peak it promised to avoid was reached before the first yield. SqlDriver carried a `TODO: Use Knex .stream()`. - `MongoDBDriver._findStream` did stream, but was the one read there never routed through `buildFindOptions`, hardcoding `projection: { _id: 0 }` and silently dropping `query.fields` (the divergence #4459 recorded; subsumed, not fixed). Removed from `IDataDriver` and `DriverInterfaceSchema`, all three implementations deleted, and the ~38 stub lines that existed only to satisfy a required method. Registered as the `data-driver-find-stream-retired` semantic entry on the protocol-17 chain step (ADR-0087 D3) — a TS/API surface, never stored metadata, so no source rewrite and, deliberately, no tombstone: nothing ever `.parse()`d a driver object, so tsc is the only channel that can carry the prescription, and it carries it at the call site. `DriverCapabilities.streaming`, the unread flag whose only referent was this method, is left standing and filed as #4634 — removing it breaks every driver's capability literal, third-party included, and that audit should cover all ~30 flags in one pass. Fixes #4484 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012C2cd7tL8QDoZ2QKN3djJ5 --- .changeset/data-driver-find-stream-retired.md | 100 ++++++++++++++++++ docs/design/driver-turso.md | 1 - docs/protocol-upgrade-guide.md | 5 + .../src/loaders/database-loader.test.ts | 1 - .../objectql/src/datasource-mapping.test.ts | 1 - .../src/engine-aggregate-having.test.ts | 2 - .../src/engine-aggregate-timezone.test.ts | 1 - .../src/engine-ambient-transaction.test.ts | 1 - .../src/engine-audit-anchor-write.test.ts | 1 - .../src/engine-autonumber-batch.test.ts | 1 - .../objectql/src/engine-bulk-contract.test.ts | 1 - .../src/engine-cascade-delete.test.ts | 1 - .../engine-dangling-reference-audit.test.ts | 1 - .../src/engine-default-value-tokens.test.ts | 1 - .../engine-driver-connect-failfast.test.ts | 1 - .../objectql/src/engine-driver-health.test.ts | 1 - .../objectql/src/engine-filter-alias.test.ts | 1 - .../src/engine-findone-contract.test.ts | 1 - .../objectql/src/engine-insert-many.test.ts | 1 - .../src/engine-lifecycle-datasource.test.ts | 3 - ...ngine-lookup-referential-integrity.test.ts | 1 - .../objectql/src/engine-summary-retry.test.ts | 1 - .../src/engine-unknown-option.test.ts | 1 - .../src/engine-wire-alias-reject.test.ts | 1 - .../src/protocol-clone-real-engine.test.ts | 1 - .../src/protocol-registry-shadow.test.ts | 1 - ...ol-save-meta-repo-path-real-engine.test.ts | 1 - .../src/protocol-unknown-query-param.test.ts | 1 - .../src/protocol-unregistered-object.test.ts | 1 - .../src/query-expression-conformance.test.ts | 1 - packages/objectql/src/secret-fields.test.ts | 1 - packages/objectql/src/summary-rollup.test.ts | 1 - .../driver-memory/src/memory-driver.ts | 14 +-- .../driver-mongodb/src/mongodb-driver.test.ts | 23 ---- .../driver-mongodb/src/mongodb-driver.ts | 39 ++----- .../src/mongodb-findone-query.test.ts | 4 +- packages/plugins/driver-sql/src/sql-driver.ts | 16 +-- ...cord-lock-schedule-run.integration.test.ts | 1 - .../status-mirror-cascade.integration.test.ts | 1 - packages/rest/src/export-integration.test.ts | 1 - packages/rest/src/import-integration.test.ts | 1 - .../rest/src/import-job-integration.test.ts | 1 - .../runtime/src/degraded-boot-parity.test.ts | 1 - .../src/sandbox/nested-write.canary.test.ts | 1 - .../sandbox/nested-write.integration.test.ts | 1 - packages/spec/spec-changes.json | 14 +++ .../spec/src/contracts/data-driver.test.ts | 29 ++++- packages/spec/src/contracts/data-driver.ts | 14 +-- .../spec/src/contracts/data-engine.test.ts | 46 +------- packages/spec/src/data/driver.test.ts | 72 +++++++++++-- packages/spec/src/data/driver.zod.ts | 18 +--- .../spec/src/data/pagination-conformance.ts | 4 +- packages/spec/src/migrations/registry.ts | 53 +++++++++- .../src/record-change-integration.test.ts | 1 - 54 files changed, 298 insertions(+), 195 deletions(-) create mode 100644 .changeset/data-driver-find-stream-retired.md diff --git a/.changeset/data-driver-find-stream-retired.md b/.changeset/data-driver-find-stream-retired.md new file mode 100644 index 0000000000..29f701971b --- /dev/null +++ b/.changeset/data-driver-find-stream-retired.md @@ -0,0 +1,100 @@ +--- +"@objectstack/spec": minor +"@objectstack/driver-sql": minor +"@objectstack/driver-memory": minor +"@objectstack/driver-mongodb": minor +--- + +refactor(spec,drivers)!: retire `IDataDriver.findStream` — a required method with no caller, whose two main implementations did the opposite of what it promised (#4484, ADR-0049 enforce-or-remove) + +`findStream` was a **required** method on the driver contract — every driver and +every test double had to implement it — documented as the read + +> Optimized for large datasets to avoid memory overflow. + +Three things were true about it at once, and each is worse in the light of the +others. + +**Nothing called it.** Not the query engine (there is no `stream` entry on it), +not REST export, not import, not any bulk-read path. Repo-wide, outside the +contract declaration and the three driver implementations, every single hit was +a test double — and roughly twenty of those satisfied the required method like +this: + +```ts +findStream() { throw new Error('not implemented'); } +``` + +Twenty stubs that throw, across four packages, for years, and no test ever went +red. That is not an anecdote about test hygiene; it is the proof of absence. A +method whose every double throws is a method nothing reaches. + +**Two of the three implementations inverted its one guarantee.** `SqlDriver` and +`InMemoryDriver` both did this: + +```ts +const results = await this.find(object, query, options); // ← the entire result set +for (const row of results) yield row; +``` + +The whole table is resident in memory before the first `yield`. A caller who +believed the doc comment and reached for `findStream` precisely because a result +set was too large would have hit the overflow it existed to prevent, at exactly +the scale where it mattered. `SqlDriver` carried a `TODO: Use Knex .stream()` +admitting it. + +**The one real implementation dropped a parameter.** `MongoDBDriver._findStream` +did walk a cursor — but it was the only read in that driver never routed through +`buildFindOptions`, so it hardcoded `projection: { _id: 0 }` and silently +discarded `query.fields`. (#4459 unified `find`/`findOne` onto `buildFindOptions` +and recorded in its TSDoc that `_findStream` was left out. This removal subsumes +that divergence rather than fixing it — there is nothing left to fix it for.) + +Rather than manufacture a caller to justify three implementations, the method is +retired. If a cursor-based read is wanted, it should arrive **with** the caller +that needs it, so the contract can be shaped by a real requirement instead of +being reverse-engineered from a doc comment nobody could test. + +**Migration.** + +| Wrote | Write instead | +| --- | --- | +| `for await (const row of driver.findStream(obj, q)) { … }` | page `driver.find(obj, { ...q, limit, offset })` in a loop | +| `findStream(…) { … }` on your own driver | delete the method (see below) | +| `findStream() { throw new Error('ni'); }` in a test double | delete the line | + +Paging `find()` is not a downgrade from what `findStream` actually did: on SQL +and memory it is strictly better (bounded pages instead of one full +materialisation), and the paged read is the one with an **enforced** guarantee — +`IDataDriver.find` requires a total order across the whole walk, checked by the +shared `PAGINATION_CASES` / `PAGINATION_UNORDERED_CASES` fixtures in +`data/pagination-conformance.ts`. `findStream` never had a conformance case at +all. + +**Driver authors: nothing breaks on you.** An implementation left in place still +compiles — an extra method is not an error on a class or a widened object — it is +simply never reached, so deleting it is cleanup you can do whenever. The break is +on the **caller** side: `driver.findStream(...)` no longer type-checks, and there +were no callers. + +**No tombstone, deliberately.** The other v17 retirements tombstone their key so +authoring it fails loudly with a prescription. That would be noise here. +`DriverInterfaceSchema` describes a contract that code *implements*; nothing in +either repository ever ran a driver object through `.parse()`, so a +`retiredKey()` there would carry its prescription to no one. The channel that can +carry it is `tsc`, and `tsc` reports it where it is actionable — at a call site. +The key is removed from the schema and from `IDataDriver`, and the retirement is +registered as the `data-driver-find-stream-retired` semantic entry in the +protocol-17 chain step (ADR-0087 D3), so `spec-changes.json`, the generated +upgrade guide and the `spec_changes` MCP tool all carry it. There is no +`os migrate meta` step: a driver is code, never stack metadata, so the chain has +no source to rewrite. + +**Left standing on purpose:** `DriverCapabilities.streaming`, the capability flag +whose only referent was this method. It has no readers either (and the values +written into it were already wrong — `SqlDriver` declared `streaming: false` +while implementing `findStream`, `InMemoryDriver` declared `true` for the +copy-everything version), but removing a key from the capabilities literal breaks +every driver that writes it, third-party included, and the same audit should +cover the other ~30 flags in one pass rather than one at a time. Tracked as +#4634. diff --git a/docs/design/driver-turso.md b/docs/design/driver-turso.md index 9d3c6647b3..2d26c2115d 100644 --- a/docs/design/driver-turso.md +++ b/docs/design/driver-turso.md @@ -215,7 +215,6 @@ This does NOT require changes to existing client packages — it would be a new, | `getPoolStats()` | 🟡 | Concurrency tracking (no traditional pool) | | `execute()` | ✅ | `client.execute(sql, args)` | | `find()` | ✅ | SQL SELECT with QueryAST→SQL compiler | -| `findStream()` | 🟡 | Cursor-based pagination (no native streaming) | | `findOne()` | ✅ | `SELECT ... LIMIT 1` | | `create()` | ✅ | `INSERT INTO ... RETURNING *` | | `update()` | ✅ | `UPDATE ... WHERE id = ? RETURNING *` | diff --git a/docs/protocol-upgrade-guide.md b/docs/protocol-upgrade-guide.md index d53bbf56f2..bf7635fc8c 100644 --- a/docs/protocol-upgrade-guide.md +++ b/docs/protocol-upgrade-guide.md @@ -160,6 +160,8 @@ Closing the same audit on the data side, `datasource.readReplicas` is removed (# The `script` flow node converges on its one real path (#4343). It had four ways to name what it ran and only one of them ran anything: `config.actionType: 'email' | 'slack'` were logger-backed stubs that wrote a line, reported success and delivered nothing under any configuration — with `config.template` / `.recipients` / `.variables` feeding a message no channel ever sent; inline `config.script` was recognized and never executed (the built-in runtime has no server-side JS sandbox), so the node warned and no-op'd; and every other `actionType` value was shorthand for a registered-function name, a second spelling of `config.function`. All five keys are retired and `function` becomes required, which is also what finally made the contract PARSEABLE: while the legal key set depended on `actionType`, a flat parse would either reject valid shapes or wave everything through, so `script` (with `subflow`) now runs through the same execute-time contract parse #4277 gave the flat builtins. A shorthand `actionType` CONVERTS into `function` — that is what it meant — unless `function` is already set, in which case it was dead metadata the executor never reached. The other four are dropped outright: nothing read them, so there is no value to preserve, and rebuilding the intent is an authoring decision the tombstones prescribe per branch (a `notify` node for mail — it delivers through the messaging service, the in-app inbox by default and real email once `@objectstack/plugin-email` is installed; a `connector_action` with the Slack connector, or an `http` node posting to a webhook, for Slack; a registered function for an inline body). Retired from the load path for the same reason as the rest: absorbing `actionType: 'email'` silently would let an author keep believing the flow sends mail. +The same audit reaches the driver contract itself: `IDataDriver.findStream` is removed (#4484). It was REQUIRED — every driver and every test double had to implement it — and documented as the read "optimized for large datasets to avoid memory overflow", while two of its three implementations awaited `find()` for the whole result set and then yielded it row by row, reaching exactly the peak it promised to avoid; the third streamed for real but was the one read in that driver that skipped `buildFindOptions`, so it dropped `query.fields`. Nothing anywhere called it, which is why a contract method could carry an inverted guarantee for this long and why ~20 test doubles could satisfy it by throwing `not implemented`. Paged `find()` is the read that exists and is enforced (its total-order guarantee is checked by the shared pagination-conformance cases); a cursor-based read is worth building when a caller asks for one, which is the honest order. A TS/API surface, never stored — one semantic TODO for driver authors, no source rewrite, and no tombstone: `DriverInterfaceSchema` describes a contract that code IMPLEMENTS and nothing ever `.parse()`d a driver, so tsc is the only channel that could carry the prescription, and it carries it where it matters — at a call site. + ### Mechanical (applied for you) | Conversion | Surface | Change | Load window | @@ -227,6 +229,9 @@ The `script` flow node converges on its one real path (#4343). It had four ways - **`workflow-service-slot-retired`** — `CoreServiceName 'workflow' / IWorkflowService / WorkflowProtocol / discovery routes.workflow / RestApiRouteCategory workflow` → the live mechanisms the slot only ever pointed at: `state_machine` validation rules for record state machines, approval flow nodes on the approvals runtime (ADR-0019) for approvals, lifecycle hooks + `record_change` flows (service-automation) for record-triggered automation - Why not automatic: The workflow slot was declared end to end and implemented nowhere: no code in either repository ever registered or resolved it (ADR-0115 Evidence 5 — the only touches were plugin-dev's retired stub probe and the generic discovery walk), no implementation of any WorkflowProtocol method ever existed, and no host ever mounted `/api/v1/workflow` (the pre-#3586 DEFAULT_DISPATCHER_ROUTES listed it among routes that never existed). Every part of it was ADR-0078's silently-inert declaration: a CoreServiceName nothing filled, a contract nothing implemented, a protocol nothing served, a discovery route field no builder could truthfully populate. These are TS/API surfaces and a discovery RESPONSE field — never stored in stack metadata, so there is no source for the chain to rewrite; consumers of the deleted types move their imports themselves. ADR-0049 / ADR-0078, #4451. - Done when: No import of IWorkflowService, WorkflowProtocol or the Get/WorkflowState/Config/Transition types resolves; no code calls getService('workflow') or reads discovery `routes.workflow` / `services.workflow`; record state machines, approvals and record-triggered automation go through the replacement mechanisms. Discovery output on a default boot is unchanged (the slot was always reported unavailable; now it is simply absent). +- **`data-driver-find-stream-retired`** — `contracts.IDataDriver.findStream / data.DriverInterfaceSchema.findStream` → find() with limit/offset — the paged read whose determinism IS enforced (IDataDriver.find, data/pagination-conformance.ts) + - Why not automatic: `findStream` was a REQUIRED contract method documented as "optimized for large datasets to avoid memory overflow", and in two of its three implementations it delivered the opposite: `SqlDriver` and `InMemoryDriver` both awaited `find()` for the ENTIRE result set and then yielded it row by row, so the peak memory a caller was promised protection from was already reached before the first yield. The third (`MongoDBDriver._findStream`) did walk a cursor, but it was the one read path in that driver never routed through `buildFindOptions`, so it hardcoded `projection: { _id: 0 }` and silently discarded `query.fields`. None of it was ever observed, because the method had NO caller in either repository: the engine exposes no stream entry, and the REST export, import and bulk-read paths all go through `find()`. The ~20 driver test doubles that existed only to satisfy a required method almost all threw `not implemented`, and nothing ever noticed — which is the proof, not the anecdote. Being REQUIRED, it also taxed every new driver and every test double with an implementation of a capability the platform does not have. Rather than build a caller to justify three implementations, the method is retired; a real cursor-based read should return WITH the caller that needs it (ADR-0049 enforce-or-remove). This is a TS/API contract surface — a driver is CODE, never stack metadata — so there is no source for the chain to rewrite, and deliberately no schema tombstone either: nothing ever ran a driver object through `DriverInterfaceSchema.parse()`, so a prescription there would have no one to reach. The enforced channel is tsc, and it points at callers. ADR-0049 / ADR-0078, #4484. + - Done when: No code calls `driver.findStream(...)`; large reads page through `find()` with `limit`/`offset` (which guarantees a total order across the whole walk) or go through the export surface. Drivers and test doubles no longer implement the method — one left behind still compiles and is simply never reached, so removing it is cleanup rather than a break, while a CALLER of it no longer type-checks. --- diff --git a/packages/metadata/src/loaders/database-loader.test.ts b/packages/metadata/src/loaders/database-loader.test.ts index 97627d1227..0f30f8edcd 100644 --- a/packages/metadata/src/loaders/database-loader.test.ts +++ b/packages/metadata/src/loaders/database-loader.test.ts @@ -97,7 +97,6 @@ function createMockDriver(): IDataDriver { return Promise.resolve(null); }), - findStream: vi.fn(), create: vi.fn().mockImplementation((tableName: string, data: Record) => { const table = getTable(tableName); diff --git a/packages/objectql/src/datasource-mapping.test.ts b/packages/objectql/src/datasource-mapping.test.ts index 446d49f41d..c9c2805b73 100644 --- a/packages/objectql/src/datasource-mapping.test.ts +++ b/packages/objectql/src/datasource-mapping.test.ts @@ -21,7 +21,6 @@ const createMockDriver = (name: string) => ({ bulkUpdate: async () => [], bulkDelete: async () => {}, execute: async () => ({}), - findStream: async function* () {}, upsert: async (obj: string, data: any) => ({ id: '1', ...data }), beginTransaction: async () => ({}), commit: async () => {}, diff --git a/packages/objectql/src/engine-aggregate-having.test.ts b/packages/objectql/src/engine-aggregate-having.test.ts index a80d0938dd..fa79f6b789 100644 --- a/packages/objectql/src/engine-aggregate-having.test.ts +++ b/packages/objectql/src/engine-aggregate-having.test.ts @@ -30,7 +30,6 @@ function makeNativeDriver(rows: any[]) { supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find() { return rows.slice(); }, - findStream() { throw new Error('ni'); }, async findOne() { return rows[0] ?? null; }, async create(_o: string, d: any) { return d; }, async update(_o: string, _id: string, d: any) { return d; }, @@ -66,7 +65,6 @@ function makeRawDriver(rows: any[]) { supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find() { return rows.slice(); }, - findStream() { throw new Error('ni'); }, async findOne() { return rows[0] ?? null; }, async create(_o: string, d: any) { return d; }, async update(_o: string, _id: string, d: any) { return d; }, diff --git a/packages/objectql/src/engine-aggregate-timezone.test.ts b/packages/objectql/src/engine-aggregate-timezone.test.ts index 9d324a5717..322df73d4f 100644 --- a/packages/objectql/src/engine-aggregate-timezone.test.ts +++ b/packages/objectql/src/engine-aggregate-timezone.test.ts @@ -24,7 +24,6 @@ function makeBucketingDriver(rows: any[]) { supports: { queryDateGranularity: { day: true, week: true, month: true, quarter: true, year: true } }, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find() { return rows.slice(); }, - findStream() { throw new Error('ni'); }, async findOne() { return rows[0] ?? null; }, async create(_o: string, d: any) { return d; }, async update(_o: string, _id: string, d: any) { return d; }, diff --git a/packages/objectql/src/engine-ambient-transaction.test.ts b/packages/objectql/src/engine-ambient-transaction.test.ts index 0a03fce56b..4318169087 100644 --- a/packages/objectql/src/engine-ambient-transaction.test.ts +++ b/packages/objectql/src/engine-ambient-transaction.test.ts @@ -40,7 +40,6 @@ function makeRecordingDriver() { seen.find.push({ object, transaction: options?.transaction }); return Array.from(storeFor(object).values()); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string) { for (const r of storeFor(object).values()) return r; return null; diff --git a/packages/objectql/src/engine-audit-anchor-write.test.ts b/packages/objectql/src/engine-audit-anchor-write.test.ts index 95515f9dca..7083b4ce3c 100644 --- a/packages/objectql/src/engine-audit-anchor-write.test.ts +++ b/packages/objectql/src/engine-audit-anchor-write.test.ts @@ -47,7 +47,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/engine-autonumber-batch.test.ts b/packages/objectql/src/engine-autonumber-batch.test.ts index fc42e1c7dc..c214995d3c 100644 --- a/packages/objectql/src/engine-autonumber-batch.test.ts +++ b/packages/objectql/src/engine-autonumber-batch.test.ts @@ -20,7 +20,6 @@ function makeDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(object: string) { return Array.from(storeFor(object).values()); }, - findStream() { throw new Error('ni'); }, async findOne() { return null; }, async create(object: string, data: Record) { n += 1; diff --git a/packages/objectql/src/engine-bulk-contract.test.ts b/packages/objectql/src/engine-bulk-contract.test.ts index 3e6257fa49..e978abb615 100644 --- a/packages/objectql/src/engine-bulk-contract.test.ts +++ b/packages/objectql/src/engine-bulk-contract.test.ts @@ -20,7 +20,6 @@ function makeDriver(opts: { bulkCreate?: (object: string, rows: any[]) => Promis name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(object: string) { return Array.from(storeFor(object).values()); }, - findStream() { throw new Error('ni'); }, async findOne() { return null; }, async create(object: string, data: Record) { n += 1; diff --git a/packages/objectql/src/engine-cascade-delete.test.ts b/packages/objectql/src/engine-cascade-delete.test.ts index 62fe7f2b8a..f0fcfea9b5 100644 --- a/packages/objectql/src/engine-cascade-delete.test.ts +++ b/packages/objectql/src/engine-cascade-delete.test.ts @@ -73,7 +73,6 @@ function makeMemoryDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/objectql/src/engine-dangling-reference-audit.test.ts b/packages/objectql/src/engine-dangling-reference-audit.test.ts index f125349641..ec74f007a4 100644 --- a/packages/objectql/src/engine-dangling-reference-audit.test.ts +++ b/packages/objectql/src/engine-dangling-reference-audit.test.ts @@ -88,7 +88,6 @@ function makeMemoryDriver() { // audit's bounded-scan reporting untestable AND looser than production. return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of peek(object).values()) if (matches(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/engine-default-value-tokens.test.ts b/packages/objectql/src/engine-default-value-tokens.test.ts index 126e206b75..d9ee19eb0a 100644 --- a/packages/objectql/src/engine-default-value-tokens.test.ts +++ b/packages/objectql/src/engine-default-value-tokens.test.ts @@ -32,7 +32,6 @@ function makeMemoryDriver() { async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(object: string) { return Array.from(storeFor(object).values()); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string) { return storeFor(object).values().next().value ?? null; }, async create(object: string, data: Record) { nextId += 1; diff --git a/packages/objectql/src/engine-driver-connect-failfast.test.ts b/packages/objectql/src/engine-driver-connect-failfast.test.ts index d801888a9d..6fd845af76 100644 --- a/packages/objectql/src/engine-driver-connect-failfast.test.ts +++ b/packages/objectql/src/engine-driver-connect-failfast.test.ts @@ -25,7 +25,6 @@ function makeDriver(name: string, connect: () => Promise) { async checkHealth() { return connected; }, async execute() { return null; }, async find() { return []; }, - findStream() { throw new Error('ni'); }, async findOne() { return null; }, async create(_o: string, data: Record) { return { id: 'r_1', ...data }; }, async update(_o: string, id: string, data: Record) { return { ...data, id }; }, diff --git a/packages/objectql/src/engine-driver-health.test.ts b/packages/objectql/src/engine-driver-health.test.ts index df7b40f99c..26ec9cd69a 100644 --- a/packages/objectql/src/engine-driver-health.test.ts +++ b/packages/objectql/src/engine-driver-health.test.ts @@ -17,7 +17,6 @@ function makeDriver(name: string, checkHealth?: () => Promise) { async connect() {}, async disconnect() {}, async execute() { return null; }, async find() { return []; }, - findStream() { throw new Error('ni'); }, async findOne() { return null; }, async create(_o: string, data: Record) { return { id: 'r_1', ...data }; }, async update(_o: string, id: string, data: Record) { return { ...data, id }; }, diff --git a/packages/objectql/src/engine-filter-alias.test.ts b/packages/objectql/src/engine-filter-alias.test.ts index 97f45f6715..059170db5d 100644 --- a/packages/objectql/src/engine-filter-alias.test.ts +++ b/packages/objectql/src/engine-filter-alias.test.ts @@ -57,7 +57,6 @@ function makeMemoryDriver() { const rows = Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); return typeof ast?.limit === 'number' && ast.limit > 0 ? rows.slice(0, ast.limit) : rows; }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/objectql/src/engine-findone-contract.test.ts b/packages/objectql/src/engine-findone-contract.test.ts index efc51ff604..cef5c01648 100644 --- a/packages/objectql/src/engine-findone-contract.test.ts +++ b/packages/objectql/src/engine-findone-contract.test.ts @@ -100,7 +100,6 @@ function makeRecordingDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any, opts: any) { reads.push({ ast, opts }); return run(o, ast); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any, opts: any) { reads.push({ ast, opts }); return run(o, ast)[0] ?? null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/objectql/src/engine-insert-many.test.ts b/packages/objectql/src/engine-insert-many.test.ts index 823686c632..07326115dc 100644 --- a/packages/objectql/src/engine-insert-many.test.ts +++ b/packages/objectql/src/engine-insert-many.test.ts @@ -22,7 +22,6 @@ function makeDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(object: string) { return Array.from(storeFor(object).values()); }, - findStream() { throw new Error('ni'); }, async findOne() { return null; }, async create(object: string, data: Record) { calls.create += 1; diff --git a/packages/objectql/src/engine-lifecycle-datasource.test.ts b/packages/objectql/src/engine-lifecycle-datasource.test.ts index de9a6eebfe..6de85029c0 100644 --- a/packages/objectql/src/engine-lifecycle-datasource.test.ts +++ b/packages/objectql/src/engine-lifecycle-datasource.test.ts @@ -32,9 +32,6 @@ function stubDriver(name: string) { async find() { return []; }, - findStream() { - throw new Error('ns'); - }, async findOne() { return null; }, diff --git a/packages/objectql/src/engine-lookup-referential-integrity.test.ts b/packages/objectql/src/engine-lookup-referential-integrity.test.ts index 81bf1721da..fb044983cd 100644 --- a/packages/objectql/src/engine-lookup-referential-integrity.test.ts +++ b/packages/objectql/src/engine-lookup-referential-integrity.test.ts @@ -91,7 +91,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/engine-summary-retry.test.ts b/packages/objectql/src/engine-summary-retry.test.ts index 9bebc2590a..803d7df570 100644 --- a/packages/objectql/src/engine-summary-retry.test.ts +++ b/packages/objectql/src/engine-summary-retry.test.ts @@ -27,7 +27,6 @@ function makeDriver(opts: { onParentUpdate?: (parentId: string, callN: number) = async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('ni'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/engine-unknown-option.test.ts b/packages/objectql/src/engine-unknown-option.test.ts index 1850123298..5bcf57b9b8 100644 --- a/packages/objectql/src/engine-unknown-option.test.ts +++ b/packages/objectql/src/engine-unknown-option.test.ts @@ -77,7 +77,6 @@ function makeDriver() { async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any) { finds.push(ast); return run(o, ast); }, async findOne(o: string, ast: any) { finds.push(ast); return run(o, ast)[0] ?? null; }, - findStream() { throw new Error('ns'); }, async create(o: string, data: any) { nextId += 1; const id = data.id ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; }, async update(o: string, id: string, data: any) { const s = storeFor(o); const cur = s.get(id); if (!cur) throw new Error('nf'); const up = { ...cur, ...data, id }; s.set(id, up); return up; }, async updateMany(o: string, ast: any, data: any) { let n = 0; for (const [id, row] of storeFor(o)) { if (!matches(row, ast?.where)) continue; storeFor(o).set(id, { ...row, ...data, id }); n += 1; } return n; }, diff --git a/packages/objectql/src/engine-wire-alias-reject.test.ts b/packages/objectql/src/engine-wire-alias-reject.test.ts index 468adbea60..33ea89a628 100644 --- a/packages/objectql/src/engine-wire-alias-reject.test.ts +++ b/packages/objectql/src/engine-wire-alias-reject.test.ts @@ -64,7 +64,6 @@ function makeRecordingDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any) { finds.push(ast); return run(o, ast); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { finds.push(ast); return run(o, ast)[0] ?? null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/objectql/src/protocol-clone-real-engine.test.ts b/packages/objectql/src/protocol-clone-real-engine.test.ts index bdae421c51..f7bf2b645c 100644 --- a/packages/objectql/src/protocol-clone-real-engine.test.ts +++ b/packages/objectql/src/protocol-clone-real-engine.test.ts @@ -69,7 +69,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/protocol-registry-shadow.test.ts b/packages/objectql/src/protocol-registry-shadow.test.ts index 5aecc1c297..2c1adfb25d 100644 --- a/packages/objectql/src/protocol-registry-shadow.test.ts +++ b/packages/objectql/src/protocol-registry-shadow.test.ts @@ -102,7 +102,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/protocol-save-meta-repo-path-real-engine.test.ts b/packages/objectql/src/protocol-save-meta-repo-path-real-engine.test.ts index 790b7b7044..5acdc09da1 100644 --- a/packages/objectql/src/protocol-save-meta-repo-path-real-engine.test.ts +++ b/packages/objectql/src/protocol-save-meta-repo-path-real-engine.test.ts @@ -72,7 +72,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/protocol-unknown-query-param.test.ts b/packages/objectql/src/protocol-unknown-query-param.test.ts index 9a9ea3004b..e7b3c0b7aa 100644 --- a/packages/objectql/src/protocol-unknown-query-param.test.ts +++ b/packages/objectql/src/protocol-unknown-query-param.test.ts @@ -74,7 +74,6 @@ function makeMemoryDriver() { const from = typeof ast?.offset === 'number' ? ast.offset : 0; return typeof ast?.limit === 'number' ? all.slice(from, from + ast.limit) : all.slice(from); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/protocol-unregistered-object.test.ts b/packages/objectql/src/protocol-unregistered-object.test.ts index d45277afa9..d347b63066 100644 --- a/packages/objectql/src/protocol-unregistered-object.test.ts +++ b/packages/objectql/src/protocol-unregistered-object.test.ts @@ -72,7 +72,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/query-expression-conformance.test.ts b/packages/objectql/src/query-expression-conformance.test.ts index e1d92520a3..7c79840c89 100644 --- a/packages/objectql/src/query-expression-conformance.test.ts +++ b/packages/objectql/src/query-expression-conformance.test.ts @@ -164,7 +164,6 @@ function makeMemoryDriver() { const page = typeof ast?.limit === 'number' ? sorted.slice(from, from + ast.limit) : sorted.slice(from); return project(page, ast?.fields); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { const rows = await this.find(object, ast); return rows[0] ?? null; diff --git a/packages/objectql/src/secret-fields.test.ts b/packages/objectql/src/secret-fields.test.ts index b41782b620..cf70800246 100644 --- a/packages/objectql/src/secret-fields.test.ts +++ b/packages/objectql/src/secret-fields.test.ts @@ -40,7 +40,6 @@ function makeMemoryDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; return null; diff --git a/packages/objectql/src/summary-rollup.test.ts b/packages/objectql/src/summary-rollup.test.ts index fcf084c656..0e0c4b0542 100644 --- a/packages/objectql/src/summary-rollup.test.ts +++ b/packages/objectql/src/summary-rollup.test.ts @@ -51,7 +51,6 @@ function makeDriver() { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('ni'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; return null; diff --git a/packages/plugins/driver-memory/src/memory-driver.ts b/packages/plugins/driver-memory/src/memory-driver.ts index 10aa608901..ed79255461 100644 --- a/packages/plugins/driver-memory/src/memory-driver.ts +++ b/packages/plugins/driver-memory/src/memory-driver.ts @@ -199,7 +199,7 @@ export class InMemoryDriver implements IDataDriver { fullTextSearch: false, // @planned: Text tokenization + matching jsonQuery: false, geospatialQuery: false, - streaming: true, // Implemented via findStream() + streaming: true, // Unread by anything; described findStream(), retired in #4484 — see #4634 jsonFields: true, // Native JS object support arrayFields: true, // Native JS array support vectorSearch: false, // @planned: Cosine similarity search @@ -377,14 +377,10 @@ export class InMemoryDriver implements IDataDriver { return results; } - async *findStream(object: string, query: QueryAST, options?: DriverOptions) { - this.logger.debug('FindStream operation', { object }); - - const results = await this.find(object, query, options); - for (const record of results) { - yield record; - } - } + // `findStream` was removed with the contract method in 17.0.0 (#4484). Like the SQL + // driver's, this implementation awaited `find()` in full and then yielded row by + // row — the whole table was already in memory before the first `yield`. Nothing + // called it. Page through `find()` with `limit`/`offset`. async findOne(object: string, query: QueryAST, options?: DriverOptions) { this.logger.debug('FindOne operation', { object, query }); diff --git a/packages/plugins/driver-mongodb/src/mongodb-driver.test.ts b/packages/plugins/driver-mongodb/src/mongodb-driver.test.ts index a2ffb5079b..f084325484 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-driver.test.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-driver.test.ts @@ -296,29 +296,6 @@ describe.skipIf(!sharedMongod)('MongoDBDriver', () => { }); }); - // =========================================================================== - // Streaming - // =========================================================================== - - describe('findStream', () => { - it('should stream records', async () => { - await driver.bulkCreate('task', [ - { id: 's-1', title: 'One' }, - { id: 's-2', title: 'Two' }, - { id: 's-3', title: 'Three' }, - ]); - - const records: any[] = []; - for await (const record of driver.findStream('task', {})) { - records.push(record); - } - expect(records.length).toBe(3); - for (const r of records) { - expect(r).not.toHaveProperty('_id'); - } - }); - }); - // =========================================================================== // Aggregation // =========================================================================== diff --git a/packages/plugins/driver-mongodb/src/mongodb-driver.ts b/packages/plugins/driver-mongodb/src/mongodb-driver.ts index b18d5bf658..832bcf4329 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-driver.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-driver.ts @@ -226,9 +226,10 @@ export class MongoDBDriver implements IDataDriver { * * `singleRowLookup` marks the caller as `findOne`; see {@link buildSortSpec}. * - * Not used by `_findStream`, which deliberately projects the whole document - * regardless of `query.fields` — a separate divergence, and one that returns - * more data rather than the wrong data, so it is left as-is here. + * Every read path in this driver now goes through it. The one that did not — + * `_findStream`, which hardcoded `projection: { _id: 0 }` and so dropped + * `query.fields` on the floor — was retired with the contract method it served + * (#4484), which subsumes that divergence rather than fixing it. */ private buildFindOptions( query: QueryAST, @@ -295,32 +296,12 @@ export class MongoDBDriver implements IDataDriver { return result as Record | null; } - findStream(object: string, query: QueryAST, options?: DriverOptions): AsyncGenerator> { - return this._findStream(object, query, options); - } - - private async *_findStream(object: string, query: QueryAST, options?: DriverOptions): AsyncGenerator> { - const collection = this.getCollection(object); - const session = this.getSession(options); - - const filter = translateFilter(query.where, this.temporalKindFor(object)); - const findOptions: FindOptions = { - session, - projection: { _id: 0 }, - }; - - const sort = this.buildSortSpec(query); - if (sort) findOptions.sort = sort; - - if (query.offset !== undefined) findOptions.skip = query.offset; - if (query.limit !== undefined) findOptions.limit = query.limit; - - const cursor = collection.find(filter, findOptions); - - for await (const doc of cursor) { - yield doc as Record; - } - } + // `findStream` / `_findStream` were removed with the contract method in 17.0.0 + // (#4484). This was the only one of the three drivers that genuinely streamed — + // it walked the cursor — but it was also the only read here that never reached + // `buildFindOptions`, so `query.fields` was silently discarded on that path. With + // no caller anywhere in either repository there was nothing to fix it for. Page + // through `find()` with `limit`/`offset`. async create(object: string, data: Record, options?: DriverOptions): Promise> { const collection = this.getCollection(object); diff --git a/packages/plugins/driver-mongodb/src/mongodb-findone-query.test.ts b/packages/plugins/driver-mongodb/src/mongodb-findone-query.test.ts index 10af39255e..b86eb3b0f5 100644 --- a/packages/plugins/driver-mongodb/src/mongodb-findone-query.test.ts +++ b/packages/plugins/driver-mongodb/src/mongodb-findone-query.test.ts @@ -6,8 +6,8 @@ * * It used to issue `collection.findOne(translateFilter(query.where), { * projection: { _id: 0 } })` and nothing else: `orderBy`, `fields` and `offset` - * were accepted by the contract and dropped on the floor. `find` and - * `_findStream` in the same file had always handled all three, so this was a + * were accepted by the contract and dropped on the floor. `find` in the same + * file had always handled all three, so this was a * per-method divergence exactly like the engine-level one #4419 is about — * `findOne({ orderBy: [{ field: 'created_at', order: 'desc' }] })` did not * return the newest record, it returned whichever document the scan reached diff --git a/packages/plugins/driver-sql/src/sql-driver.ts b/packages/plugins/driver-sql/src/sql-driver.ts index 75a8f5a2ab..ce0bcba5a3 100644 --- a/packages/plugins/driver-sql/src/sql-driver.ts +++ b/packages/plugins/driver-sql/src/sql-driver.ts @@ -1533,17 +1533,11 @@ export class SqlDriver implements IDataDriver { return results[0] || null; } - /** - * Stream records matching a structured query. - * NOTE: Current implementation fetches all results then yields them. - * TODO: Use Knex .stream() for true cursor-based streaming on large datasets. - */ - async *findStream(object: string, query: QueryAST, options?: DriverOptions): AsyncGenerator> { - const results = await this.find(object, query, options); - for (const row of results) { - yield row; - } - } + // `findStream` was removed with the contract method in 17.0.0 (#4484). This driver's + // implementation awaited `find()` in full and then yielded row by row, so it never + // avoided the memory it was declared to avoid; nothing called it. Page through + // `find()` with `limit`/`offset` until a real Knex `.stream()` read is built to a + // caller's requirement. async create(object: string, data: Record, options?: DriverOptions): Promise { const { _id, ...rest } = data; diff --git a/packages/plugins/plugin-approvals/src/record-lock-schedule-run.integration.test.ts b/packages/plugins/plugin-approvals/src/record-lock-schedule-run.integration.test.ts index 2ed2753614..b2582d7d3d 100644 --- a/packages/plugins/plugin-approvals/src/record-lock-schedule-run.integration.test.ts +++ b/packages/plugins/plugin-approvals/src/record-lock-schedule-run.integration.test.ts @@ -74,7 +74,6 @@ function makeMemoryDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; diff --git a/packages/plugins/plugin-approvals/src/status-mirror-cascade.integration.test.ts b/packages/plugins/plugin-approvals/src/status-mirror-cascade.integration.test.ts index 5ce64eeb53..b2af91de47 100644 --- a/packages/plugins/plugin-approvals/src/status-mirror-cascade.integration.test.ts +++ b/packages/plugins/plugin-approvals/src/status-mirror-cascade.integration.test.ts @@ -74,7 +74,6 @@ function makeMemoryDriver(): any { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; return null; diff --git a/packages/rest/src/export-integration.test.ts b/packages/rest/src/export-integration.test.ts index edd38f499f..93793b5162 100644 --- a/packages/rest/src/export-integration.test.ts +++ b/packages/rest/src/export-integration.test.ts @@ -98,7 +98,6 @@ function makeMemoryDriver() { const sliced = limit != null ? sorted.slice(skip, skip + Number(limit)) : sorted.slice(skip); return sliced; }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/rest/src/import-integration.test.ts b/packages/rest/src/import-integration.test.ts index 249c40dfea..d0d768d2a0 100644 --- a/packages/rest/src/import-integration.test.ts +++ b/packages/rest/src/import-integration.test.ts @@ -54,7 +54,6 @@ function makeMemoryDriver() { const limit = ast?.limit ?? ast?.top; return limit != null ? rows.slice(skip, skip + Number(limit)) : rows.slice(skip); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/rest/src/import-job-integration.test.ts b/packages/rest/src/import-job-integration.test.ts index 6edf42f451..80990c0713 100644 --- a/packages/rest/src/import-job-integration.test.ts +++ b/packages/rest/src/import-job-integration.test.ts @@ -52,7 +52,6 @@ function makeMemoryDriver() { const limit = ast?.limit ?? ast?.top; return limit != null ? rows.slice(skip, skip + Number(limit)) : rows.slice(skip); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/runtime/src/degraded-boot-parity.test.ts b/packages/runtime/src/degraded-boot-parity.test.ts index da47d48621..55f11fb08a 100644 --- a/packages/runtime/src/degraded-boot-parity.test.ts +++ b/packages/runtime/src/degraded-boot-parity.test.ts @@ -46,7 +46,6 @@ function failingDriver(name: string) { async checkHealth() { return false; }, async execute() { return null; }, async find() { return []; }, - findStream() { throw new Error('ni'); }, async findOne() { return null; }, async create(_o: string, d: Record) { return { id: 'r_1', ...d }; }, async update(_o: string, id: string, d: Record) { return { ...d, id }; }, diff --git a/packages/runtime/src/sandbox/nested-write.canary.test.ts b/packages/runtime/src/sandbox/nested-write.canary.test.ts index a3ba278d4a..c8a3424dbb 100644 --- a/packages/runtime/src/sandbox/nested-write.canary.test.ts +++ b/packages/runtime/src/sandbox/nested-write.canary.test.ts @@ -76,7 +76,6 @@ function makeMemoryDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/runtime/src/sandbox/nested-write.integration.test.ts b/packages/runtime/src/sandbox/nested-write.integration.test.ts index c40ff813b5..2ff9cc3d45 100644 --- a/packages/runtime/src/sandbox/nested-write.integration.test.ts +++ b/packages/runtime/src/sandbox/nested-write.integration.test.ts @@ -62,7 +62,6 @@ function makeMemoryDriver() { name: 'memory', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('ns'); }, async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, async create(o: string, data: Record) { nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; diff --git a/packages/spec/spec-changes.json b/packages/spec/spec-changes.json index 5a0c00f6ab..aa192043ef 100644 --- a/packages/spec/spec-changes.json +++ b/packages/spec/spec-changes.json @@ -378,6 +378,13 @@ "migrationId": "workflow-service-slot-retired", "toMajor": 17, "rationale": "The workflow slot was declared end to end and implemented nowhere: no code in either repository ever registered or resolved it (ADR-0115 Evidence 5 — the only touches were plugin-dev's retired stub probe and the generic discovery walk), no implementation of any WorkflowProtocol method ever existed, and no host ever mounted `/api/v1/workflow` (the pre-#3586 DEFAULT_DISPATCHER_ROUTES listed it among routes that never existed). Every part of it was ADR-0078's silently-inert declaration: a CoreServiceName nothing filled, a contract nothing implemented, a protocol nothing served, a discovery route field no builder could truthfully populate. These are TS/API surfaces and a discovery RESPONSE field — never stored in stack metadata, so there is no source for the chain to rewrite; consumers of the deleted types move their imports themselves. ADR-0049 / ADR-0078, #4451." + }, + { + "surface": "contracts.IDataDriver.findStream / data.DriverInterfaceSchema.findStream", + "replacement": "find() with limit/offset — the paged read whose determinism IS enforced (IDataDriver.find, data/pagination-conformance.ts)", + "migrationId": "data-driver-find-stream-retired", + "toMajor": 17, + "rationale": "`findStream` was a REQUIRED contract method documented as \"optimized for large datasets to avoid memory overflow\", and in two of its three implementations it delivered the opposite: `SqlDriver` and `InMemoryDriver` both awaited `find()` for the ENTIRE result set and then yielded it row by row, so the peak memory a caller was promised protection from was already reached before the first yield. The third (`MongoDBDriver._findStream`) did walk a cursor, but it was the one read path in that driver never routed through `buildFindOptions`, so it hardcoded `projection: { _id: 0 }` and silently discarded `query.fields`. None of it was ever observed, because the method had NO caller in either repository: the engine exposes no stream entry, and the REST export, import and bulk-read paths all go through `find()`. The ~20 driver test doubles that existed only to satisfy a required method almost all threw `not implemented`, and nothing ever noticed — which is the proof, not the anecdote. Being REQUIRED, it also taxed every new driver and every test double with an implementation of a capability the platform does not have. Rather than build a caller to justify three implementations, the method is retired; a real cursor-based read should return WITH the caller that needs it (ADR-0049 enforce-or-remove). This is a TS/API contract surface — a driver is CODE, never stack metadata — so there is no source for the chain to rewrite, and deliberately no schema tombstone either: nothing ever ran a driver object through `DriverInterfaceSchema.parse()`, so a prescription there would have no one to reach. The enforced channel is tsc, and it points at callers. ADR-0049 / ADR-0078, #4484." } ], "removed": [] @@ -815,6 +822,13 @@ "migrationId": "workflow-service-slot-retired", "toMajor": 17, "rationale": "The workflow slot was declared end to end and implemented nowhere: no code in either repository ever registered or resolved it (ADR-0115 Evidence 5 — the only touches were plugin-dev's retired stub probe and the generic discovery walk), no implementation of any WorkflowProtocol method ever existed, and no host ever mounted `/api/v1/workflow` (the pre-#3586 DEFAULT_DISPATCHER_ROUTES listed it among routes that never existed). Every part of it was ADR-0078's silently-inert declaration: a CoreServiceName nothing filled, a contract nothing implemented, a protocol nothing served, a discovery route field no builder could truthfully populate. These are TS/API surfaces and a discovery RESPONSE field — never stored in stack metadata, so there is no source for the chain to rewrite; consumers of the deleted types move their imports themselves. ADR-0049 / ADR-0078, #4451." + }, + { + "surface": "contracts.IDataDriver.findStream / data.DriverInterfaceSchema.findStream", + "replacement": "find() with limit/offset — the paged read whose determinism IS enforced (IDataDriver.find, data/pagination-conformance.ts)", + "migrationId": "data-driver-find-stream-retired", + "toMajor": 17, + "rationale": "`findStream` was a REQUIRED contract method documented as \"optimized for large datasets to avoid memory overflow\", and in two of its three implementations it delivered the opposite: `SqlDriver` and `InMemoryDriver` both awaited `find()` for the ENTIRE result set and then yielded it row by row, so the peak memory a caller was promised protection from was already reached before the first yield. The third (`MongoDBDriver._findStream`) did walk a cursor, but it was the one read path in that driver never routed through `buildFindOptions`, so it hardcoded `projection: { _id: 0 }` and silently discarded `query.fields`. None of it was ever observed, because the method had NO caller in either repository: the engine exposes no stream entry, and the REST export, import and bulk-read paths all go through `find()`. The ~20 driver test doubles that existed only to satisfy a required method almost all threw `not implemented`, and nothing ever noticed — which is the proof, not the anecdote. Being REQUIRED, it also taxed every new driver and every test double with an implementation of a capability the platform does not have. Rather than build a caller to justify three implementations, the method is retired; a real cursor-based read should return WITH the caller that needs it (ADR-0049 enforce-or-remove). This is a TS/API contract surface — a driver is CODE, never stack metadata — so there is no source for the chain to rewrite, and deliberately no schema tombstone either: nothing ever ran a driver object through `DriverInterfaceSchema.parse()`, so a prescription there would have no one to reach. The enforced channel is tsc, and it points at callers. ADR-0049 / ADR-0078, #4484." } ], "removed": [] diff --git a/packages/spec/src/contracts/data-driver.test.ts b/packages/spec/src/contracts/data-driver.test.ts index 76f7792ae4..7ca6f7a0c2 100644 --- a/packages/spec/src/contracts/data-driver.test.ts +++ b/packages/spec/src/contracts/data-driver.test.ts @@ -44,7 +44,6 @@ describe('IDataDriver', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: () => (async function* () {})(), findOne: async () => null, create: async (_obj, data) => ({ id: '1', ...data }), update: async (_obj, _id, data) => ({ id: '1', ...data }), @@ -109,7 +108,6 @@ describe('IDataDriver', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: () => null, findOne: async () => null, create: async () => ({ id: '1' }), update: async () => ({ id: '1' }), @@ -176,7 +174,6 @@ describe('IDataDriver', () => { getPoolStats: () => ({ total: 10, idle: 5, active: 3, waiting: 2 }), execute: async () => ({}), find: async () => [], - findStream: () => null, findOne: async () => null, create: async () => ({ id: '1' }), update: async () => ({ id: '1' }), @@ -203,4 +200,30 @@ describe('IDataDriver', () => { expect(extendedDriver.deleteMany).toBeDefined(); expect(extendedDriver.explain).toBeDefined(); }); + + // =========================================================================== + // Retired surface (#4484, ADR-0049 enforce-or-remove) + // =========================================================================== + + describe('findStream (retired in 17.0.0)', () => { + it('is not declared on the contract, so calling it does not type-check', () => { + // The pin is the type, not the runtime: `keyof IDataDriver` is resolved by + // tsc, so re-adding `findStream(...)` to the interface makes `Retired` + // resolve to `never` and this line fails `pnpm typecheck` — which is the + // only channel that can catch a *contract* regression. The expect() below + // just gives the type assertion a home vitest will run. + type Retired = 'findStream' extends keyof IDataDriver ? never : 'absent'; + const retired: Retired = 'absent'; + expect(retired).toBe('absent'); + }); + + it('leaves an implementation that still defines it harmless', () => { + // A driver written against 16.x keeps compiling: an extra method is not an + // excess-property error on a class or on a widened object, it is simply + // never reached. The break is on the CALLER side — `driver.findStream(...)` + // no longer compiles — and there were no callers to break. + const legacyShaped = { findStream: () => undefined }; + expect('findStream' in legacyShaped).toBe(true); + }); + }); }); diff --git a/packages/spec/src/contracts/data-driver.ts b/packages/spec/src/contracts/data-driver.ts index 5ed7a6f9c8..0600eca20f 100644 --- a/packages/spec/src/contracts/data-driver.ts +++ b/packages/spec/src/contracts/data-driver.ts @@ -104,12 +104,14 @@ export interface IDataDriver { */ find(object: string, query: QueryAST, options?: DriverOptions): Promise[]>; - /** - * Stream records matching the structured query. - * Optimized for large datasets to avoid memory overflow. - * Returns an AsyncIterable or ReadableStream. - */ - findStream(object: string, query: QueryAST, options?: DriverOptions): unknown; + // `findStream` was removed in 17.0.0 (#4484, ADR-0049 enforce-or-remove). It was a + // REQUIRED method promising reads "optimized for large datasets to avoid memory + // overflow" that nothing in either repository ever called — and two of its three + // implementations awaited `find()` in full before yielding, so the one guarantee it + // existed to make was the one it inverted. Large reads go through `find()` with + // `limit`/`offset`, whose paged-read determinism IS enforced (see above and + // `data/pagination-conformance.ts`). A real cursor-based read should be + // reintroduced with the caller that needs it, not ahead of one. /** * Find a single record by query. diff --git a/packages/spec/src/contracts/data-engine.test.ts b/packages/spec/src/contracts/data-engine.test.ts index 8c920f200c..ff163aabc2 100644 --- a/packages/spec/src/contracts/data-engine.test.ts +++ b/packages/spec/src/contracts/data-engine.test.ts @@ -215,7 +215,6 @@ describe('Data Engine Contract', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: () => (async function* () {})(), findOne: async () => null, create: async (_obj, data) => ({ id: '1', ...data }), update: async (_obj, _id, data) => ({ id: '1', ...data }), @@ -254,7 +253,6 @@ describe('Data Engine Contract', () => { checkHealth: async () => connected, execute: async () => ({}), find: async () => [], - findStream: () => (async function* () {})(), findOne: async () => null, create: async (_obj, data) => ({ id: '1', ...data }), update: async (_obj, _id, data) => ({ id: '1', ...data }), @@ -288,7 +286,6 @@ describe('Data Engine Contract', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: () => (async function* () {})(), findOne: async () => null, create: async (_obj, data) => ({ id: '1', ...data }), update: async (_obj, _id, data) => ({ id: '1', ...data }), @@ -319,45 +316,8 @@ describe('Data Engine Contract', () => { expect(driver.explain).toBeDefined(); }); - it('should support findStream with yielded values', async () => { - const records = [{ id: '1', name: 'Alice' }, { id: '2', name: 'Bob' }]; - const driver: IDataDriver = { - name: 'streamer', - version: '1.0.0', - supports: { ...minimalCapabilities, streaming: true }, - connect: async () => {}, - disconnect: async () => {}, - checkHealth: async () => true, - execute: async () => ({}), - find: async () => records, - findStream: () => (async function* () { - for (const r of records) yield r; - })(), - findOne: async () => null, - create: async (_obj, data) => ({ id: '1', ...data }), - update: async (_obj, _id, data) => ({ id: '1', ...data }), - upsert: async (_obj, data) => ({ id: '1', ...data }), - delete: async () => true, - count: async () => records.length, - bulkCreate: async () => [], - bulkUpdate: async () => [], - bulkDelete: async () => {}, - beginTransaction: async () => ({}), - commit: async () => {}, - rollback: async () => {}, - syncSchema: async () => {}, - dropTable: async () => {}, - }; - - const stream = driver.findStream('users', {} as any); - const collected: any[] = []; - for await (const row of stream as AsyncIterable) { - collected.push(row); - } - - expect(collected).toHaveLength(2); - expect(collected[0].name).toBe('Alice'); - expect(collected[1].name).toBe('Bob'); - }); + // The `findStream` case that stood here was removed with the contract method + // in 17.0.0 (#4484) — it built an IDataDriver whose only job was to satisfy a + // required method no production code ever called. }); }); diff --git a/packages/spec/src/data/driver.test.ts b/packages/spec/src/data/driver.test.ts index 1b0cd24588..9f9ea8cc5b 100644 --- a/packages/spec/src/data/driver.test.ts +++ b/packages/spec/src/data/driver.test.ts @@ -83,7 +83,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: async function* () {}, findOne: async () => null, create: async () => ({}), update: async () => ({}), @@ -126,7 +125,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async (object: string, query: any) => [], - findStream: async function* (object: string, query: any) {}, findOne: async (object: string, query: any) => null, create: async (object: string, data: any) => data, update: async (object: string, id: any, data: any) => data, @@ -232,7 +230,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: async function* () {}, findOne: async () => null, create: async () => ({}), update: async () => ({}), @@ -313,7 +310,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: async function* () {}, findOne: async () => null, create: async () => ({}), update: async () => ({}), @@ -377,7 +373,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: async function* () {}, findOne: async () => null, create: async () => ({}), update: async () => ({}), @@ -419,7 +414,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async () => [], - findStream: async function* () {}, findOne: async () => null, create: async () => ({}), update: async () => ({}), @@ -463,7 +457,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async (object, query) => [], - findStream: async function* (object, query) {}, findOne: async (object, query) => null, create: async (object, data) => data, update: async (object, id, data) => data, @@ -505,7 +498,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async (object, query) => [], - findStream: async function* (object, query) {}, findOne: async (object, query) => null, create: async (object, data) => data, update: async (object, id, data) => data, @@ -547,7 +539,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async (object, query) => [], - findStream: async function* (object, query) {}, findOne: async (object, query) => null, create: async (object, data) => data, update: async (object, id, data) => data, @@ -589,7 +580,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async (object, query) => [], - findStream: async function* (object, query) {}, findOne: async (object, query) => null, create: async (object, data) => data, update: async (object, id, data) => data, @@ -631,7 +621,6 @@ describe('DriverInterfaceSchema', () => { checkHealth: async () => true, execute: async () => ({}), find: async (object, query) => [], - findStream: async function* (object, query) {}, findOne: async (object, query) => null, create: async (object, data) => data, update: async (object, id, data) => data, @@ -664,4 +653,65 @@ describe('DriverInterfaceSchema', () => { expect(() => DriverInterfaceSchema.parse(memoryDriver)).not.toThrow(); }); }); + + // =========================================================================== + // Retired surface (#4484, ADR-0049 enforce-or-remove) + // =========================================================================== + + describe('findStream (retired in 17.0.0)', () => { + it('is no longer part of the declared driver shape', () => { + expect(DriverInterfaceSchema.shape).not.toHaveProperty('findStream'); + }); + + it('deliberately carries no tombstone — nothing ever parsed a driver with it', () => { + // The other retirements in this major tombstone their key so authoring it + // fails loudly. That would be noise here: `DriverInterfaceSchema` describes + // a TypeScript contract that drivers IMPLEMENT, and nothing in either + // repository ever ran a driver object through `.parse()` — the prescription + // would have no one to reach. The enforced channel is tsc on `IDataDriver` + // (see contracts/data-driver.test.ts), which breaks callers, and there were + // none. A stray `findStream` on a driver object therefore just parses and + // is dropped, exactly as any other non-contract method on it always has. + const withLegacyMethod = { + name: 'legacy', + version: '1.0.0', + connect: async () => {}, + disconnect: async () => {}, + checkHealth: async () => true, + execute: async () => ({}), + find: async () => [], + findStream: async function* () {}, + findOne: async () => null, + create: async () => ({}), + update: async () => ({}), + upsert: async () => ({}), + delete: async () => true, + count: async () => 0, + bulkCreate: async () => [], + bulkUpdate: async () => [], + bulkDelete: async () => {}, + beginTransaction: async () => ({}), + commit: async () => {}, + rollback: async () => {}, + syncSchema: async () => {}, + dropTable: async () => {}, + supports: { + transactions: true, + queryFilters: true, + queryAggregations: true, + querySorting: true, + queryPagination: true, + queryWindowFunctions: false, + querySubqueries: false, + joins: false, + fullTextSearch: false, + jsonFields: true, + arrayFields: true, + }, + }; + + const parsed = DriverInterfaceSchema.parse(withLegacyMethod); + expect(parsed).not.toHaveProperty('findStream'); + }); + }); }); diff --git a/packages/spec/src/data/driver.zod.ts b/packages/spec/src/data/driver.zod.ts index 8200db066b..54ea8dd7b9 100644 --- a/packages/spec/src/data/driver.zod.ts +++ b/packages/spec/src/data/driver.zod.ts @@ -475,19 +475,11 @@ export const DriverInterfaceSchema = lazySchema(() => z.object({ .output(z.promise(z.array(z.record(z.string(), z.unknown())))) .describe('Find records'), - /** - * Stream records matching the structured query. - * Optimized for large datasets to avoid memory overflow. - * - * @param object - The name of the object. - * @param query - The structured QueryAST. - * @param options - Driver options. - * @returns AsyncIterable/ReadableStream of records. - */ - findStream: z.function() - .input(z.tuple([z.string(), QuerySchema, DriverOptionsSchema.optional()])) - .output(z.unknown()) - .describe('Stream records (AsyncIterable)'), + // `findStream` was removed in 17.0.0 (#4484, ADR-0049 enforce-or-remove) — see the + // matching note on `IDataDriver` in `contracts/data-driver.ts`. Nothing ever called + // it, and two of the three drivers implementing it materialised the whole result set + // before yielding, inverting the memory guarantee it was declared for. Page through + // `find()` with `limit`/`offset` instead. /** * Find a single record by query. diff --git a/packages/spec/src/data/pagination-conformance.ts b/packages/spec/src/data/pagination-conformance.ts index 5b1b464a86..1e24e1c76f 100644 --- a/packages/spec/src/data/pagination-conformance.ts +++ b/packages/spec/src/data/pagination-conformance.ts @@ -55,8 +55,8 @@ * * # Scope * - * The guarantee is on `find()` (and whatever a driver builds on it, e.g. a - * `findStream` that delegates), and only where the read is **paged**. It says + * The guarantee is on `find()` (and whatever a driver builds on it), and only + * where the read is **paged**. It says * nothing about an unpaged read with no `orderBy`: nothing is being sliced, so * no caller can be shown a partial view of the set, and imposing an order there * would change plan selection across the majority of reads to buy nothing. diff --git a/packages/spec/src/migrations/registry.ts b/packages/spec/src/migrations/registry.ts index 6e138577f4..b7d9f67e43 100644 --- a/packages/spec/src/migrations/registry.ts +++ b/packages/spec/src/migrations/registry.ts @@ -562,7 +562,22 @@ const step17: MigrationStep = { + 'installed; a `connector_action` with the Slack connector, or an `http` node posting to a ' + 'webhook, for Slack; a registered function for an inline body). Retired from the load path ' + 'for the same reason as the rest: absorbing `actionType: \'email\'` silently would let an ' - + 'author keep believing the flow sends mail.', + + 'author keep believing the flow sends mail.\n\n' + + 'The same audit reaches the driver contract itself: `IDataDriver.findStream` is removed ' + + '(#4484). It was REQUIRED — every driver and every test double had to implement it — and ' + + 'documented as the read "optimized for large datasets to avoid memory overflow", while ' + + 'two of its three implementations awaited `find()` for the whole result set and then ' + + 'yielded it row by row, reaching exactly the peak it promised to avoid; the third ' + + 'streamed for real but was the one read in that driver that skipped `buildFindOptions`, ' + + 'so it dropped `query.fields`. Nothing anywhere called it, which is why a contract ' + + 'method could carry an inverted guarantee for this long and why ~20 test doubles could ' + + 'satisfy it by throwing `not implemented`. Paged `find()` is the read that exists and is ' + + 'enforced (its total-order guarantee is checked by the shared pagination-conformance ' + + 'cases); a cursor-based read is worth building when a caller asks for one, which is the ' + + 'honest order. A TS/API surface, never stored — one semantic TODO for driver authors, no ' + + 'source rewrite, and no tombstone: `DriverInterfaceSchema` describes a contract that ' + + 'code IMPLEMENTS and nothing ever `.parse()`d a driver, so tsc is the only channel that ' + + 'could carry the prescription, and it carries it where it matters — at a call site.', conversionIds: [ 'action-execute-to-target', 'field-conditionalRequired-to-requiredWhen', @@ -806,6 +821,42 @@ const step17: MigrationStep = { + 'Discovery output on a default boot is unchanged (the slot was always reported ' + 'unavailable; now it is simply absent).', }, + { + id: 'data-driver-find-stream-retired', + surface: 'contracts.IDataDriver.findStream / data.DriverInterfaceSchema.findStream', + replacement: + 'find() with limit/offset — the paged read whose determinism IS enforced ' + + '(IDataDriver.find, data/pagination-conformance.ts)', + reason: + '`findStream` was a REQUIRED contract method documented as "optimized for large ' + + 'datasets to avoid memory overflow", and in two of its three implementations it ' + + 'delivered the opposite: `SqlDriver` and `InMemoryDriver` both awaited `find()` for ' + + 'the ENTIRE result set and then yielded it row by row, so the peak memory a caller ' + + 'was promised protection from was already reached before the first yield. The third ' + + '(`MongoDBDriver._findStream`) did walk a cursor, but it was the one read path in ' + + 'that driver never routed through `buildFindOptions`, so it hardcoded ' + + '`projection: { _id: 0 }` and silently discarded `query.fields`. None of it was ever ' + + 'observed, because the method had NO caller in either repository: the engine exposes ' + + 'no stream entry, and the REST export, import and bulk-read paths all go through ' + + '`find()`. The ~20 driver test doubles that existed only to satisfy a required ' + + 'method almost all threw `not implemented`, and nothing ever noticed — which is the ' + + 'proof, not the anecdote. Being REQUIRED, it also taxed every new driver and every ' + + 'test double with an implementation of a capability the platform does not have. ' + + 'Rather than build a caller to justify three implementations, the method is retired; ' + + 'a real cursor-based read should return WITH the caller that needs it (ADR-0049 ' + + 'enforce-or-remove). This is a TS/API contract surface — a driver is CODE, never ' + + 'stack metadata — so there is no source for the chain to rewrite, and deliberately ' + + 'no schema tombstone either: nothing ever ran a driver object through ' + + '`DriverInterfaceSchema.parse()`, so a prescription there would have no one to ' + + 'reach. The enforced channel is tsc, and it points at callers. ADR-0049 / ' + + 'ADR-0078, #4484.', + acceptanceCriteria: + 'No code calls `driver.findStream(...)`; large reads page through `find()` with ' + + '`limit`/`offset` (which guarantees a total order across the whole walk) or go ' + + 'through the export surface. Drivers and test doubles no longer implement the ' + + 'method — one left behind still compiles and is simply never reached, so removing ' + + 'it is cleanup rather than a break, while a CALLER of it no longer type-checks.', + }, ], }; diff --git a/packages/triggers/trigger-record-change/src/record-change-integration.test.ts b/packages/triggers/trigger-record-change/src/record-change-integration.test.ts index c9dd7abe64..27d6bf3202 100644 --- a/packages/triggers/trigger-record-change/src/record-change-integration.test.ts +++ b/packages/triggers/trigger-record-change/src/record-change-integration.test.ts @@ -55,7 +55,6 @@ function makeMemoryDriver(): any { async find(object: string, ast: any) { return Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; return null; From 672bf149fb54ff79dacafb2b2472c7d89464d1d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 14:15:14 +0000 Subject: [PATCH 2/2] fix(spec,objectql): changeset is a MAJOR bump, and sweep the last findStream stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects found reviewing the retirement against the `spec-property-retirement` checklist: - The changeset declared `minor` for all four packages. Removing a REQUIRED method from a published contract interface is breaking — the skill says `major` for `@objectstack/spec`, and it is the house convention for every other `!` spec change in this major (`session-dual-source-c4`, `notification-dual-source-c3`). The driver packages drop a public method too, so they go major with it. - `protocol-batch-atomic.test.ts` still carried a `findStream() { throw new Error('not implemented'); }` stub. It is typed `any`, so it compiles and is simply dead — but it is exactly the stub this issue exists to sweep, and leaving one behind lets the next reader infer the method still exists. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012C2cd7tL8QDoZ2QKN3djJ5 --- .changeset/data-driver-find-stream-retired.md | 8 ++++---- packages/objectql/src/protocol-batch-atomic.test.ts | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.changeset/data-driver-find-stream-retired.md b/.changeset/data-driver-find-stream-retired.md index 29f701971b..805e6f2100 100644 --- a/.changeset/data-driver-find-stream-retired.md +++ b/.changeset/data-driver-find-stream-retired.md @@ -1,8 +1,8 @@ --- -"@objectstack/spec": minor -"@objectstack/driver-sql": minor -"@objectstack/driver-memory": minor -"@objectstack/driver-mongodb": minor +"@objectstack/spec": major +"@objectstack/driver-sql": major +"@objectstack/driver-memory": major +"@objectstack/driver-mongodb": major --- refactor(spec,drivers)!: retire `IDataDriver.findStream` — a required method with no caller, whose two main implementations did the opposite of what it promised (#4484, ADR-0049 enforce-or-remove) diff --git a/packages/objectql/src/protocol-batch-atomic.test.ts b/packages/objectql/src/protocol-batch-atomic.test.ts index 8f4a1f8d4d..d1cfcf6a49 100644 --- a/packages/objectql/src/protocol-batch-atomic.test.ts +++ b/packages/objectql/src/protocol-batch-atomic.test.ts @@ -49,7 +49,6 @@ function makeSnapshotDriver() { async checkHealth() { return true; }, async execute() { return null; }, async find(object: string) { return Array.from(storeFor(object).values()); }, - findStream() { throw new Error('not implemented'); }, async findOne(object: string, ast: any, options: any) { seen.findOne.push({ object, transaction: options?.transaction }); const id = ast?.where?.id ?? ast?.filters?.id;