From 8e9f0c1f384b3f5ed445188975bd5ccedfbe606c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 4 Sep 2025 16:59:22 +0200 Subject: [PATCH 1/4] Add unit test that reproduces the bug --- .../db/tests/collection-auto-index.test.ts | 119 ++++++++++++++++++ packages/db/tests/query/join-subquery.test.ts | 96 ++++++++++++++ 2 files changed, 215 insertions(+) diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index 3859d30bc..9bbab9a70 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -543,6 +543,125 @@ describe(`Collection Auto-Indexing`, () => { tracker.restore() }) + it(`should create auto-indexes for join key on lazy collection when joining subquery`, async () => { + const leftCollection = createCollection({ + getKey: (item) => item.id, + autoIndex: `eager`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const item of testData) { + write({ + type: `insert`, + value: item, + }) + } + commit() + markReady() + }, + }, + onInsert: async (_) => {}, + }) + + const rightCollection = createCollection({ + getKey: (item) => item.id2, + autoIndex: `eager`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { + id2: `1`, + name: `Other Active Item`, + age: 40, + status: `active`, + createdAt: new Date(), + }, + }) + write({ + type: `insert`, + value: { + id2: `other2`, + name: `Other Inactive Item`, + age: 35, + status: `inactive`, + createdAt: new Date(), + }, + }) + commit() + markReady() + }, + }, + }) + + await rightCollection.stateWhenReady() + + const liveQuery = createLiveQueryCollection({ + query: (q: any) => + q + .from({ item: leftCollection }) + .join( + { + other: q + .from({ other: rightCollection }) + .select(({ other }: any) => ({ + id2: other.id2, + name: other.name, + })), + }, + ({ item, other }: any) => eq(item.id, other.id2), + `left` + ) + .select(({ item, other }: any) => ({ + id: item.id, + name: item.name, + otherName: other.name, + })), + startSync: true, + }) + + await liveQuery.stateWhenReady() + + expect(liveQuery.size).toBe(testData.length) + + expect(rightCollection.indexes.size).toBe(1) + + const index = rightCollection.indexes.values().next().value! + expect(index.expression).toEqual({ + type: `ref`, + path: [`id2`], + }) + + const tracker = createIndexUsageTracker(rightCollection) + + // Now send another item through the left collection + // and check that it used the index to join it to items of the right collection + + leftCollection.insert({ + id: `other2`, + name: `New Item`, + age: 25, + status: `active`, + createdAt: new Date(), + }) + + expect(tracker.stats.queriesExecuted).toEqual([ + { + type: `index`, + operation: `eq`, + field: `id2`, + value: `other2`, + }, + ]) + + expect(liveQuery.size).toBe(testData.length + 1) + + tracker.restore() + }) + it(`should not create auto-indexes for unsupported operations`, async () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index fc00608a8..9afad35f3 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -102,6 +102,20 @@ const sampleUsers: Array = [ }, ] +const sampleProducts = [ + { id: 1, a: `8` }, + { id: 2, a: `6` }, + { id: 3, a: `0` }, + { id: 4, a: `5` }, +] + +const sampleTrials = [ + { id: 1, productId: 1, userId: 1, createdAt: new Date() }, + { id: 2, productId: 2, userId: 1, createdAt: new Date() }, + { id: 2, productId: 4, userId: 1, createdAt: null }, + { id: 3, productId: 3, userId: 2, createdAt: new Date() }, +] + function createIssuesCollection(autoIndex: `off` | `eager` = `eager`) { return createCollection( mockSyncCollectionOptions({ @@ -124,6 +138,50 @@ function createUsersCollection(autoIndex: `off` | `eager` = `eager`) { ) } +function createProductsCollection(autoIndex: `off` | `eager` = `eager`) { + return createCollection( + mockSyncCollectionOptions({ + id: `join-subquery-test-products`, + getKey: (product) => product.id, + initialData: sampleProducts, + autoIndex, + }) + ) + /* + return createCollection( + queryCollectionOptions({ + queryClient: queryClient, + queryKey: ["products"], + queryFn: async () => { + await new Promise((resolve) => setTimeout(resolve, 1)) + return sampleProducts + }, + getKey: (item) => item.id, + onInsert: async () => { + return { refetch: false } + }, + onUpdate: async () => { + return { refetch: false } + }, + onDelete: async () => { + return { refetch: false } + }, + }) as any + ) + */ +} + +function createTrialsCollection(autoIndex: `off` | `eager` = `eager`) { + return createCollection( + mockSyncCollectionOptions({ + id: `join-subquery-test-trials`, + getKey: (item) => `${item.productId}-${item.userId}`, + initialData: sampleTrials, + autoIndex, + }) + ) +} + function createJoinSubqueryTests(autoIndex: `off` | `eager`): void { describe(`with autoIndex ${autoIndex}`, () => { describe(`subqueries in FROM clause with joins`, () => { @@ -272,10 +330,14 @@ function createJoinSubqueryTests(autoIndex: `off` | `eager`): void { describe(`subqueries in JOIN clause`, () => { let issuesCollection: ReturnType let usersCollection: ReturnType + let productsCollection: ReturnType + let trialsCollection: ReturnType beforeEach(() => { issuesCollection = createIssuesCollection(autoIndex) usersCollection = createUsersCollection(autoIndex) + productsCollection = createProductsCollection(autoIndex) + trialsCollection = createTrialsCollection(autoIndex) }) test(`should use subquery in JOIN clause - inner join`, () => { @@ -358,6 +420,40 @@ function createJoinSubqueryTests(autoIndex: `off` | `eager`): void { }) }) + test(`should use subquery in JOIN clause - left join for trials`, () => { + const joinSubquery = createLiveQueryCollection({ + query: (q) => { + return q + .from({ product: productsCollection }) + .join( + { + tried: q + .from({ tried: trialsCollection }) + .where(({ tried }) => eq(tried.userId, 1)), + }, + ({ tried, product }) => eq(tried.productId, product.id), + `left` + ) + .where(({ product }) => eq(product.id, 1)) + .select(({ product, tried }) => ({ + product, + tried, + })) + }, + startSync: true, + }) + + const results = joinSubquery.toArray + expect(results).toHaveLength(1) + expect(results[0]!.product.id).toBe(1) + expect(results[0]!.tried).toBeDefined() + expect(results[0]!.tried!.userId).toBe(1) + expect(results[0]).toEqual({ + product: { id: 1, a: `8` }, + tried: sampleTrials[0], + }) + }) + test(`should handle subqueries with SELECT clauses in both FROM and JOIN`, () => { const joinQuery = createLiveQueryCollection({ startSync: true, From 0f5854dfc109fcc809beb8d0706a95664c62ca56 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 4 Sep 2025 17:02:34 +0200 Subject: [PATCH 2/4] Pass the lazy collection to followRef instead of the active collection --- packages/db/src/query/compiler/joins.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index c8ab79e1c..7da856158 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -208,13 +208,10 @@ function processJoin( ? (joinedExpr as PropRef) : (mainExpr as PropRef) - const activeColl = - activeCollection === `main` ? collections[mainTableId]! : lazyCollection - const followRefResult = followRef( rawQuery, lazyCollectionJoinExpr, - activeColl + lazyCollection )! const followRefCollection = followRefResult.collection From ed140eb5525be0e8b6e0692aba5061d44ecfd598 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 4 Sep 2025 17:05:06 +0200 Subject: [PATCH 3/4] Remove commented out code --- packages/db/tests/query/join-subquery.test.ts | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index 9afad35f3..0b686e5e8 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -147,28 +147,6 @@ function createProductsCollection(autoIndex: `off` | `eager` = `eager`) { autoIndex, }) ) - /* - return createCollection( - queryCollectionOptions({ - queryClient: queryClient, - queryKey: ["products"], - queryFn: async () => { - await new Promise((resolve) => setTimeout(resolve, 1)) - return sampleProducts - }, - getKey: (item) => item.id, - onInsert: async () => { - return { refetch: false } - }, - onUpdate: async () => { - return { refetch: false } - }, - onDelete: async () => { - return { refetch: false } - }, - }) as any - ) - */ } function createTrialsCollection(autoIndex: `off` | `eager` = `eager`) { From f5bfa90866b0a0881452405cd64f1cd197b0b8b2 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 4 Sep 2025 17:05:58 +0200 Subject: [PATCH 4/4] changeset --- .changeset/thin-rings-flow.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thin-rings-flow.md diff --git a/.changeset/thin-rings-flow.md b/.changeset/thin-rings-flow.md new file mode 100644 index 000000000..b6aba5a04 --- /dev/null +++ b/.changeset/thin-rings-flow.md @@ -0,0 +1,5 @@ +--- +"@tanstack/db": patch +--- + +Fixes a bug where optimized joins would miss data