From be951591b06a5063f90702eb16c309f5b8ddbbf4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 06:44:55 +0000 Subject: [PATCH 1/5] feat: automatically append predicates to static queryKey in on-demand mode When using a static queryKey with syncMode: 'on-demand', the system now automatically appends serialized LoadSubsetOptions to create unique cache keys for different predicate combinations. This fixes an issue where static queryKeys in on-demand mode would cause all live queries with different predicates to share the same cache entry, defeating the purpose of predicate push-down. Changes: - Added serialization utilities for LoadSubsetOptions (serializeLoadSubsetOptions, serializeExpression, serializeValue) - Modified createQueryFromOpts to automatically append serialized predicates when queryKey is static and syncMode is 'on-demand' - Function-based queryKeys continue to work as before - Eager mode with static queryKeys unchanged (no automatic serialization) Tests: - Added comprehensive test suite for static queryKey with on-demand mode - Tests verify different predicates create separate cache entries - Tests verify identical predicates reuse the same cache entry - Tests verify eager mode behavior unchanged - All existing tests pass --- packages/query-db-collection/src/query.ts | 127 ++++++++++++- .../query-db-collection/tests/query.test.ts | 177 ++++++++++++++++++ 2 files changed, 303 insertions(+), 1 deletion(-) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index ee915aabf..4896c5feb 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -12,6 +12,7 @@ import type { ChangeMessage, CollectionConfig, DeleteMutationFnParams, + IR, InsertMutationFnParams, LoadSubsetOptions, SyncConfig, @@ -304,6 +305,119 @@ class QueryCollectionUtilsImpl { } } +/** + * Serializes LoadSubsetOptions into a stable, hashable format for query keys + * @internal + */ +function serializeLoadSubsetOptions( + options: LoadSubsetOptions | undefined +): unknown { + if (!options) { + return null + } + + const result: Record = {} + + if (options.where) { + result.where = serializeExpression(options.where) + } + + if (options.orderBy?.length) { + result.orderBy = options.orderBy.map((clause) => ({ + expression: serializeExpression(clause.expression), + direction: clause.compareOptions.direction, + nulls: clause.compareOptions.nulls, + })) + } + + if (options.limit !== undefined) { + result.limit = options.limit + } + + return JSON.stringify(Object.keys(result).length === 0 ? null : result) +} + +/** + * Recursively serializes an IR expression for stable hashing + * @internal + */ +function serializeExpression(expr: IR.BasicExpression | undefined): unknown { + if (!expr) { + return null + } + + switch (expr.type) { + case `val`: + return { + type: `val`, + value: serializeValue(expr.value), + } + case `ref`: + return { + type: `ref`, + path: [...expr.path], + } + case `func`: + return { + type: `func`, + name: expr.name, + args: expr.args.map((arg) => serializeExpression(arg)), + } + default: + return null + } +} + +/** + * Serializes special JavaScript values (undefined, NaN, Infinity, Date) + * @internal + */ +function serializeValue(value: unknown): unknown { + if (value === undefined) { + return { __type: `undefined` } + } + + if (typeof value === `number`) { + if (Number.isNaN(value)) { + return { __type: `nan` } + } + if (value === Number.POSITIVE_INFINITY) { + return { __type: `infinity`, sign: 1 } + } + if (value === Number.NEGATIVE_INFINITY) { + return { __type: `infinity`, sign: -1 } + } + } + + if ( + value === null || + typeof value === `string` || + typeof value === `number` || + typeof value === `boolean` + ) { + return value + } + + if (value instanceof Date) { + return { __type: `date`, value: value.toJSON() } + } + + if (Array.isArray(value)) { + return value.map((item) => serializeValue(item)) + } + + if (typeof value === `object`) { + return Object.fromEntries( + Object.entries(value as Record).map(([key, val]) => [ + key, + serializeValue(val), + ]) + ) + } + + return value +} + /** * Creates query collection options for use with a standard Collection. * This integrates TanStack Query with TanStack DB for automatic synchronization. @@ -627,7 +741,18 @@ export function queryCollectionOptions( queryFunction: typeof queryFn = queryFn ): true | Promise => { // Push the predicates down to the queryKey and queryFn - const key = typeof queryKey === `function` ? queryKey(opts) : queryKey + let key: QueryKey + if (typeof queryKey === `function`) { + // Function-based queryKey: use it to build the key from opts + key = queryKey(opts) + } else if (syncMode === `on-demand`) { + // Static queryKey in on-demand mode: automatically append serialized predicates + // to create separate cache entries for different predicate combinations + key = [...queryKey, serializeLoadSubsetOptions(opts)] + } else { + // Static queryKey in eager mode: use as-is + key = queryKey + } const hashedQueryKey = hashKey(key) const extendedMeta = { ...meta, loadSubsetOptions: opts } diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 2f218d73a..a909d7c0e 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -12,6 +12,7 @@ import type { Collection, DeleteMutationFnParams, InsertMutationFnParams, + LoadSubsetOptions, TransactionWithMutations, UpdateMutationFnParams, } from "@tanstack/db" @@ -3538,4 +3539,180 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(0) }) }) + + describe(`Static queryKey with on-demand mode`, () => { + it(`should automatically append serialized predicates to static queryKey in on-demand mode`, async () => { + const items: Array = [ + { id: `1`, name: `Item 1`, category: `A` }, + { id: `2`, name: `Item 2`, category: `A` }, + { id: `3`, name: `Item 3`, category: `B` }, + { id: `4`, name: `Item 4`, category: `B` }, + ] + + const queryFn = vi.fn((ctx: QueryFunctionContext) => { + const loadSubsetOptions = ctx.meta?.loadSubsetOptions as + | LoadSubsetOptions + | undefined + // Filter items based on the where clause if present + if (loadSubsetOptions?.where) { + // Simple mock filtering - in real use, you'd use parseLoadSubsetOptions + return Promise.resolve(items) + } + return Promise.resolve(items) + }) + + const staticQueryKey = [`static-on-demand-test`] + + const config: QueryCollectionConfig = { + id: `static-on-demand-test`, + queryClient, + queryKey: staticQueryKey, // Static queryKey (not a function) + queryFn, + getKey: (item: CategorisedItem) => item.id, + syncMode: `on-demand`, + startSync: true, + } + + const options = queryCollectionOptions(config) + const collection = createCollection(options) + + // Collection should start empty with on-demand sync mode + expect(collection.size).toBe(0) + + // Create first live query with category A filter + const queryA = createLiveQueryCollection({ + query: (q) => + q + .from({ item: collection }) + .where(({ item }) => eq(item.category, `A`)) + .select(({ item }) => item), + }) + + await queryA.preload() + + // Wait for first query to load + await vi.waitFor(() => { + expect(collection.size).toBeGreaterThan(0) + }) + + // Verify queryFn was called + expect(queryFn).toHaveBeenCalledTimes(1) + const firstCall = queryFn.mock.calls[0]?.[0] + expect(firstCall?.meta?.loadSubsetOptions).toBeDefined() + + // Create second live query with category B filter + const queryB = createLiveQueryCollection({ + query: (q) => + q + .from({ item: collection }) + .where(({ item }) => eq(item.category, `B`)) + .select(({ item }) => item), + }) + + await queryB.preload() + + // Wait for second query to trigger another queryFn call + await vi.waitFor(() => { + expect(queryFn).toHaveBeenCalledTimes(2) + }) + + // Verify the second call has different loadSubsetOptions + const secondCall = queryFn.mock.calls[1]?.[0] + expect(secondCall?.meta?.loadSubsetOptions).toBeDefined() + + // The two queries should have triggered separate cache entries + // because the static queryKey was automatically extended with serialized predicates + expect(queryFn).toHaveBeenCalledTimes(2) + + // Cleanup + await queryA.cleanup() + await queryB.cleanup() + }) + + it(`should create same cache key for identical predicates with static queryKey`, async () => { + const items: Array = [ + { id: `1`, name: `Item 1`, category: `A` }, + { id: `2`, name: `Item 2`, category: `A` }, + ] + + const queryFn = vi.fn().mockResolvedValue(items) + + const config: QueryCollectionConfig = { + id: `static-identical-predicates-test`, + queryClient, + queryKey: [`identical-test`], + queryFn, + getKey: (item: CategorisedItem) => item.id, + syncMode: `on-demand`, + startSync: true, + } + + const options = queryCollectionOptions(config) + const collection = createCollection(options) + + // Create two live queries with identical predicates + const query1 = createLiveQueryCollection({ + query: (q) => + q + .from({ item: collection }) + .where(({ item }) => eq(item.category, `A`)) + .select(({ item }) => item), + }) + + const query2 = createLiveQueryCollection({ + query: (q) => + q + .from({ item: collection }) + .where(({ item }) => eq(item.category, `A`)) + .select(({ item }) => item), + }) + + await query1.preload() + await query2.preload() + + await vi.waitFor(() => { + expect(collection.size).toBeGreaterThan(0) + }) + + // Should only call queryFn once because identical predicates + // should produce the same serialized cache key + expect(queryFn).toHaveBeenCalledTimes(1) + + // Cleanup + await query1.cleanup() + await query2.cleanup() + }) + + it(`should work correctly in eager mode with static queryKey (no automatic serialization)`, async () => { + const items: Array = [ + { id: `1`, name: `Item 1` }, + { id: `2`, name: `Item 2` }, + ] + + const queryFn = vi.fn().mockResolvedValue(items) + + const config: QueryCollectionConfig = { + id: `static-eager-test`, + queryClient, + queryKey: [`eager-test`], + queryFn, + getKey, + syncMode: `eager`, // Eager mode should NOT append predicates + startSync: true, + } + + const options = queryCollectionOptions(config) + const collection = createCollection(options) + + // Wait for initial load + await vi.waitFor(() => { + expect(collection.size).toBe(items.length) + }) + + // Should call queryFn once with empty predicates + expect(queryFn).toHaveBeenCalledTimes(1) + const call = queryFn.mock.calls[0]?.[0] + expect(call?.meta?.loadSubsetOptions).toEqual({}) + }) + }) }) From b1f303cf2431b04e7192846e0d7c9a7575e3be49 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 06:50:39 +0000 Subject: [PATCH 2/5] chore: add changeset for static queryKey on-demand mode fix --- .../automatic-static-querykey-ondemand.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .changeset/automatic-static-querykey-ondemand.md diff --git a/.changeset/automatic-static-querykey-ondemand.md b/.changeset/automatic-static-querykey-ondemand.md new file mode 100644 index 000000000..23e4e7fde --- /dev/null +++ b/.changeset/automatic-static-querykey-ondemand.md @@ -0,0 +1,35 @@ +--- +"@tanstack/query-db-collection": minor +--- + +Automatically append predicates to static queryKey in on-demand mode. + +When using a static `queryKey` with `syncMode: 'on-demand'`, the system now automatically appends serialized LoadSubsetOptions to create unique cache keys for different predicate combinations. This fixes an issue where all live queries with different predicates would share the same TanStack Query cache entry, causing data to be overwritten. + +**Before:** +```typescript +// This would cause conflicts between different queries +queryCollectionOptions({ + queryKey: ['products'], // Static key + syncMode: 'on-demand', + queryFn: async (ctx) => { + const { where, limit } = ctx.meta.loadSubsetOptions + return fetch(`/api/products?...`).then(r => r.json()) + } +}) +``` + +With different live queries filtering by `category='A'` and `category='B'`, both would share the same cache key `['products']`, causing the last query to overwrite the first. + +**After:** +Static queryKeys now work correctly in on-demand mode! The system automatically creates unique cache keys: +- Query with `category='A'` → `['products', '{"where":{...A...}}']` +- Query with `category='B'` → `['products', '{"where":{...B...}}']` + +**Key behaviors:** +- ✅ Static queryKeys now work correctly with on-demand mode (automatic serialization) +- ✅ Function-based queryKeys continue to work as before (no change) +- ✅ Eager mode with static queryKeys unchanged (no automatic serialization) +- ✅ Identical predicates correctly reuse the same cache entry + +This makes the documentation example work correctly without requiring users to manually implement function-based queryKeys for predicate push-down. From 186cb4d86d0c041c71d87f3108bc407a8b1eba49 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 06:51:13 +0000 Subject: [PATCH 3/5] chore: update changeset to patch instead of minor --- .changeset/automatic-static-querykey-ondemand.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/automatic-static-querykey-ondemand.md b/.changeset/automatic-static-querykey-ondemand.md index 23e4e7fde..34d1422f8 100644 --- a/.changeset/automatic-static-querykey-ondemand.md +++ b/.changeset/automatic-static-querykey-ondemand.md @@ -1,5 +1,5 @@ --- -"@tanstack/query-db-collection": minor +"@tanstack/query-db-collection": patch --- Automatically append predicates to static queryKey in on-demand mode. From 41d5a86371766cc6a9d931eb6c31aba50a90c30a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 06:53:03 +0000 Subject: [PATCH 4/5] chore: format changeset with prettier --- .changeset/automatic-static-querykey-ondemand.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.changeset/automatic-static-querykey-ondemand.md b/.changeset/automatic-static-querykey-ondemand.md index 34d1422f8..f9ae0e33a 100644 --- a/.changeset/automatic-static-querykey-ondemand.md +++ b/.changeset/automatic-static-querykey-ondemand.md @@ -7,15 +7,16 @@ Automatically append predicates to static queryKey in on-demand mode. When using a static `queryKey` with `syncMode: 'on-demand'`, the system now automatically appends serialized LoadSubsetOptions to create unique cache keys for different predicate combinations. This fixes an issue where all live queries with different predicates would share the same TanStack Query cache entry, causing data to be overwritten. **Before:** + ```typescript // This would cause conflicts between different queries queryCollectionOptions({ - queryKey: ['products'], // Static key - syncMode: 'on-demand', + queryKey: ["products"], // Static key + syncMode: "on-demand", queryFn: async (ctx) => { const { where, limit } = ctx.meta.loadSubsetOptions - return fetch(`/api/products?...`).then(r => r.json()) - } + return fetch(`/api/products?...`).then((r) => r.json()) + }, }) ``` @@ -23,10 +24,12 @@ With different live queries filtering by `category='A'` and `category='B'`, both **After:** Static queryKeys now work correctly in on-demand mode! The system automatically creates unique cache keys: + - Query with `category='A'` → `['products', '{"where":{...A...}}']` - Query with `category='B'` → `['products', '{"where":{...B...}}']` **Key behaviors:** + - ✅ Static queryKeys now work correctly with on-demand mode (automatic serialization) - ✅ Function-based queryKeys continue to work as before (no change) - ✅ Eager mode with static queryKeys unchanged (no automatic serialization) From 2895890c14b76a616cbd6fe0559c532dbe898c2d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Nov 2025 19:46:23 +0000 Subject: [PATCH 5/5] refactor: address PR review feedback Addresses Kevin's review comments: 1. Move serialization functions to dedicated utility file - Created src/serialization.ts with serializeLoadSubsetOptions, serializeExpression, and serializeValue functions - Keeps query.ts focused on query logic 2. Fix return type and use undefined instead of null - Changed serializeLoadSubsetOptions return type from `unknown` to `string | undefined` - Returns undefined instead of null for empty options - Updated usage to conditionally append only when serialized result is not undefined 3. Add missing CompareOptions properties to orderBy serialization - Now includes stringSort, locale, and localeOptions properties - Properly handles the StringCollationConfig union type with conditional serialization for locale-specific options All runtime tests pass (65/65 in query.test.ts). --- packages/query-db-collection/src/query.ts | 118 +--------------- .../query-db-collection/src/serialization.ts | 128 ++++++++++++++++++ 2 files changed, 131 insertions(+), 115 deletions(-) create mode 100644 packages/query-db-collection/src/serialization.ts diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 4896c5feb..d70128f3b 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -7,12 +7,12 @@ import { QueryKeyRequiredError, } from "./errors" import { createWriteUtils } from "./manual-sync" +import { serializeLoadSubsetOptions } from "./serialization" import type { BaseCollectionConfig, ChangeMessage, CollectionConfig, DeleteMutationFnParams, - IR, InsertMutationFnParams, LoadSubsetOptions, SyncConfig, @@ -305,119 +305,6 @@ class QueryCollectionUtilsImpl { } } -/** - * Serializes LoadSubsetOptions into a stable, hashable format for query keys - * @internal - */ -function serializeLoadSubsetOptions( - options: LoadSubsetOptions | undefined -): unknown { - if (!options) { - return null - } - - const result: Record = {} - - if (options.where) { - result.where = serializeExpression(options.where) - } - - if (options.orderBy?.length) { - result.orderBy = options.orderBy.map((clause) => ({ - expression: serializeExpression(clause.expression), - direction: clause.compareOptions.direction, - nulls: clause.compareOptions.nulls, - })) - } - - if (options.limit !== undefined) { - result.limit = options.limit - } - - return JSON.stringify(Object.keys(result).length === 0 ? null : result) -} - -/** - * Recursively serializes an IR expression for stable hashing - * @internal - */ -function serializeExpression(expr: IR.BasicExpression | undefined): unknown { - if (!expr) { - return null - } - - switch (expr.type) { - case `val`: - return { - type: `val`, - value: serializeValue(expr.value), - } - case `ref`: - return { - type: `ref`, - path: [...expr.path], - } - case `func`: - return { - type: `func`, - name: expr.name, - args: expr.args.map((arg) => serializeExpression(arg)), - } - default: - return null - } -} - -/** - * Serializes special JavaScript values (undefined, NaN, Infinity, Date) - * @internal - */ -function serializeValue(value: unknown): unknown { - if (value === undefined) { - return { __type: `undefined` } - } - - if (typeof value === `number`) { - if (Number.isNaN(value)) { - return { __type: `nan` } - } - if (value === Number.POSITIVE_INFINITY) { - return { __type: `infinity`, sign: 1 } - } - if (value === Number.NEGATIVE_INFINITY) { - return { __type: `infinity`, sign: -1 } - } - } - - if ( - value === null || - typeof value === `string` || - typeof value === `number` || - typeof value === `boolean` - ) { - return value - } - - if (value instanceof Date) { - return { __type: `date`, value: value.toJSON() } - } - - if (Array.isArray(value)) { - return value.map((item) => serializeValue(item)) - } - - if (typeof value === `object`) { - return Object.fromEntries( - Object.entries(value as Record).map(([key, val]) => [ - key, - serializeValue(val), - ]) - ) - } - - return value -} - /** * Creates query collection options for use with a standard Collection. * This integrates TanStack Query with TanStack DB for automatic synchronization. @@ -748,7 +635,8 @@ export function queryCollectionOptions( } else if (syncMode === `on-demand`) { // Static queryKey in on-demand mode: automatically append serialized predicates // to create separate cache entries for different predicate combinations - key = [...queryKey, serializeLoadSubsetOptions(opts)] + const serialized = serializeLoadSubsetOptions(opts) + key = serialized !== undefined ? [...queryKey, serialized] : queryKey } else { // Static queryKey in eager mode: use as-is key = queryKey diff --git a/packages/query-db-collection/src/serialization.ts b/packages/query-db-collection/src/serialization.ts new file mode 100644 index 000000000..ba22dc75a --- /dev/null +++ b/packages/query-db-collection/src/serialization.ts @@ -0,0 +1,128 @@ +import type { IR, LoadSubsetOptions } from "@tanstack/db" + +/** + * Serializes LoadSubsetOptions into a stable, hashable format for query keys + * @internal + */ +export function serializeLoadSubsetOptions( + options: LoadSubsetOptions | undefined +): string | undefined { + if (!options) { + return undefined + } + + const result: Record = {} + + if (options.where) { + result.where = serializeExpression(options.where) + } + + if (options.orderBy?.length) { + result.orderBy = options.orderBy.map((clause) => { + const baseOrderBy = { + expression: serializeExpression(clause.expression), + direction: clause.compareOptions.direction, + nulls: clause.compareOptions.nulls, + stringSort: clause.compareOptions.stringSort, + } + + // Handle locale-specific options when stringSort is 'locale' + if (clause.compareOptions.stringSort === `locale`) { + return { + ...baseOrderBy, + locale: clause.compareOptions.locale, + localeOptions: clause.compareOptions.localeOptions, + } + } + + return baseOrderBy + }) + } + + if (options.limit !== undefined) { + result.limit = options.limit + } + + return Object.keys(result).length === 0 ? undefined : JSON.stringify(result) +} + +/** + * Recursively serializes an IR expression for stable hashing + * @internal + */ +function serializeExpression(expr: IR.BasicExpression | undefined): unknown { + if (!expr) { + return null + } + + switch (expr.type) { + case `val`: + return { + type: `val`, + value: serializeValue(expr.value), + } + case `ref`: + return { + type: `ref`, + path: [...expr.path], + } + case `func`: + return { + type: `func`, + name: expr.name, + args: expr.args.map((arg) => serializeExpression(arg)), + } + default: + return null + } +} + +/** + * Serializes special JavaScript values (undefined, NaN, Infinity, Date) + * @internal + */ +function serializeValue(value: unknown): unknown { + if (value === undefined) { + return { __type: `undefined` } + } + + if (typeof value === `number`) { + if (Number.isNaN(value)) { + return { __type: `nan` } + } + if (value === Number.POSITIVE_INFINITY) { + return { __type: `infinity`, sign: 1 } + } + if (value === Number.NEGATIVE_INFINITY) { + return { __type: `infinity`, sign: -1 } + } + } + + if ( + value === null || + typeof value === `string` || + typeof value === `number` || + typeof value === `boolean` + ) { + return value + } + + if (value instanceof Date) { + return { __type: `date`, value: value.toJSON() } + } + + if (Array.isArray(value)) { + return value.map((item) => serializeValue(item)) + } + + if (typeof value === `object`) { + return Object.fromEntries( + Object.entries(value as Record).map(([key, val]) => [ + key, + serializeValue(val), + ]) + ) + } + + return value +}