diff --git a/docs/core-concepts/storage.md b/docs/core-concepts/storage.md index ae5a3bfd..8cb04b43 100644 --- a/docs/core-concepts/storage.md +++ b/docs/core-concepts/storage.md @@ -59,7 +59,7 @@ Those projections include: This is why search and recall can continue to work even when one retrieval path is incomplete. The storage layer keeps multiple doors into the same memory. -Semantic vectors are guarded by the retrieval text hash. The durable vector blob lives in `target_embeddings`; sqlite-vec rows are tracked separately in `vector_index_entries` and copied into `.konteks/vectors.sqlite`. The acceleration index can be rebuilt from the durable blob. If text changes, old vectors are not used for scoring; changed extraction repairs missing or stale vectors for changed sections and modules without rebuilding unrelated retrieval documents. Corpus-wide embedding generation and sqlite-vec repair run during `konteks init` and `konteks rebuild`, keeping automatic warm-up and save refresh work proportional to changed files. +Semantic vectors are guarded by the retrieval text hash. The durable vector blob lives in `target_embeddings`; sqlite-vec rows are tracked separately in `vector_index_entries` and copied into `.konteks/vectors.sqlite`. The acceleration index can be rebuilt from the durable blob. If text changes, old vectors are not used for scoring; changed extraction repairs missing or stale vectors for changed sections and modules without rebuilding unrelated retrieval documents. Corpus-wide embedding generation and sqlite-vec repair run during `konteks init` and `konteks rebuild`, keeping automatic warm-up and save refresh work proportional to changed files. Search validates sqlite-vec against durable vectors once per process. When native rows are incomplete, search returns the bounded durable-vector fallback immediately and repairs sqlite-vec in the background. Konteks loads sqlite-vec through Bun's SQLite extension loader when available, then falls back to Node's built-in SQLite loader. If neither runtime can load the native extension, indexing reports an actionable dependency error. diff --git a/docs/development/database-erd.md b/docs/development/database-erd.md index 76d23378..1a18af9a 100644 --- a/docs/development/database-erd.md +++ b/docs/development/database-erd.md @@ -247,7 +247,7 @@ but their table options live in `src/database/utils/migrations`. `memory_fts`. Semantic retrieval uses sqlite-vec virtual tables created per embedding dimension through Bun's SQLite extension loader when available, then falls back to Node's built-in SQLite extension loader. `target_embeddings` remains the -durable compatibility store and exact fallback source. `vector_index_entries` records which durable embeddings have been copied into the sqlite-vec virtual table for their dimension in `.konteks/vectors.sqlite`. A vector is considered fresh only when its model-aware `embedding_hash` matches the current retrieval text and the sqlite-vec metadata matches the same hash; stale or missing index rows are repaired from stored blobs during embedding generation and ignored during retrieval. +durable compatibility store and exact fallback source. `vector_index_entries` records which durable embeddings have been copied into the sqlite-vec virtual table for their dimension in `.konteks/vectors.sqlite`. A vector is considered fresh only when its model-aware `embedding_hash` matches the current retrieval text and the sqlite-vec metadata matches the same hash. Search validates native rows against durable vectors once per process; unhealthy groups use bounded exact fallback immediately and schedule best-effort background repair from stored blobs. ## Main Domains diff --git a/src/database/services/vector-index.ts b/src/database/services/vector-index.ts index e9d3576e..1882c1f3 100644 --- a/src/database/services/vector-index.ts +++ b/src/database/services/vector-index.ts @@ -5,6 +5,7 @@ import getDb from '@/database/actions/_db' import { vectorIndexEntries } from '@/database/schema' import { isSqliteTestRuntime } from '@/database/support/test-runtime' import { loadProjectContext } from '@/modules/project/context' +import { appendProjectErrorLog } from '@/support/error-log' type TargetType = 'section' | 'diary' | 'memory' | 'module' @@ -72,10 +73,24 @@ type VectorEmbeddingRow = { vector_blob: ArrayBuffer | Uint8Array } +type VectorIndexGroup = { + dimensions: number + model: string +} + +type VectorIndexRow = { + embedding_hash: string + target_id: string + target_type: TargetType +} + declare global { var __konteksVectorIndexConnectionFactoryForTests: | VectorIndexConnectionFactory | undefined + var __konteksWaitForVectorIndexRepairsForTests: + | (() => Promise) + | undefined } const BUN_SQLITE_MODULE = 'bun:sqlite' @@ -84,6 +99,11 @@ const SQLITE_BIND_CHUNK_SIZE = 250 let activeConnection: VectorIndexConnection | undefined let bunSqlitePromise: Promise | undefined let nodeSqlitePromise: Promise | undefined +const healthyGroups = new Set() +const repairPromises = new Map>() +globalThis.__konteksWaitForVectorIndexRepairsForTests = async () => { + await Promise.all(repairPromises.values()) +} class VectorIndexDependencyError extends Error { public constructor(message: string, options?: { cause?: unknown }) { @@ -103,6 +123,14 @@ export async function upsertVectorIndexTargets( if (!connection) { return false } + await upsertVectorIndexTargetsWithConnection(connection, targets) + return true +} + +async function upsertVectorIndexTargetsWithConnection( + connection: VectorIndexConnection, + targets: VectorIndexTarget[], +): Promise { for (const [dimensions, groupedTargets] of groupTargetsByDimensions( targets, )) { @@ -147,47 +175,23 @@ insert into ${vecTable} ( deleteStatement.finalize?.() insertStatement.finalize?.() } + invalidateVectorIndexGroups(connection.path, groupedTargets) } - - return true } export async function reconcileVectorIndexGroup(input: { dimensions: number model: string }): Promise { - const metadataCount = await vectorIndexEntryCount(input) const connection = await vectorConnection() if (!connection) { return false } - const tableName = tableNameForDimensions(input.dimensions) - if (!hasVectorTable(connection.database, tableName)) { - if (metadataCount > 0) { - await deleteVectorIndexEntriesForGroup(input) - } - return false - } - - const vectorCount = rowCount( - withStatement( - connection.database, - `select count(*) as count from ${tableName} where model = ?`, - statement => statement.get(input.model), - ), - ) - if (vectorCount === metadataCount) { + if (await validateVectorIndexGroup(connection, input)) { return true } - withNativeTransaction(connection.database, () => { - withStatement( - connection.database, - `delete from ${tableName} where model = ?`, - statement => statement.run(input.model), - ) - }) - await deleteVectorIndexEntriesForGroup(input) + await resetVectorIndexGroup(connection, input) return false } @@ -216,6 +220,7 @@ export async function deleteVectorIndexTargets( ) } }) + invalidateVectorIndexPath(connection.path) } export async function searchVectorIndex(input: { @@ -229,11 +234,15 @@ export async function searchVectorIndex(input: { return exactVectorSearch(input) } + if (!(await validateVectorIndexGroup(connection, input))) { + scheduleVectorIndexRepair(connection, input) + return await exactVectorSearch(input) + } + const vecTable = tableNameForDimensions(input.dimensions) if (!hasVectorTable(connection.database, vecTable)) { return await exactVectorSearch(input) } - const results = withStatement( connection.database, ` @@ -256,6 +265,135 @@ where embedding match ? return results.length > 0 ? results : await exactVectorSearch(input) } +async function validateVectorIndexGroup( + connection: VectorIndexConnection, + input: VectorIndexGroup, + options: { force?: boolean } = {}, +): Promise { + const key = vectorIndexGroupKey(connection.path, input) + if (!options.force && healthyGroups.has(key)) { + return true + } + + let cursor: Pick | undefined + const tableName = tableNameForDimensions(input.dimensions) + while (true) { + const durableRows = await loadDurableVectorPage(input, cursor) + const metadataRows = await loadVectorIndexEntryPage(input, cursor) + if (!hasVectorTable(connection.database, tableName)) { + if (durableRows.length > 0 || metadataRows.length > 0) { + return false + } + healthyGroups.add(key) + return true + } + const nativeRows = loadNativeVectorPage( + connection.database, + tableName, + input, + cursor, + ) + if ( + !sameVectorIndexRows(durableRows, metadataRows) || + !sameVectorIndexRows(durableRows, nativeRows) + ) { + return false + } + if (durableRows.length < SQLITE_BIND_CHUNK_SIZE) { + healthyGroups.add(key) + return true + } + cursor = durableRows.at(-1) + } +} + +function scheduleVectorIndexRepair( + connection: VectorIndexConnection, + input: VectorIndexGroup, +): void { + const key = vectorIndexGroupKey(connection.path, input) + if (repairPromises.has(key)) { + return + } + + const repair = Promise.resolve() + .then(() => repairVectorIndexGroup(connection, input)) + .catch(error => { + void appendProjectErrorLog({ + error, + metadata: { + dimensions: input.dimensions, + model: input.model, + operation: 'repair_vector_index', + vectorDatabase: connection.path, + }, + surface: 'background_maintenance', + }) + }) + .finally(() => { + repairPromises.delete(key) + }) + repairPromises.set(key, repair) +} + +async function repairVectorIndexGroup( + connection: VectorIndexConnection, + input: VectorIndexGroup, +): Promise { + await resetVectorIndexGroup(connection, input) + let cursor: + | Pick + | undefined + + while (true) { + const rows = await loadDurableVectorPage(input, cursor) + await upsertVectorIndexTargetsWithConnection( + connection, + rows.map(row => ({ + createdAt: new Date().toISOString(), + dimensions: input.dimensions, + embeddingHash: row.embedding_hash, + model: row.model, + targetId: row.target_id, + targetType: row.target_type, + vector: blobToFloat32Array(row.vector_blob), + })), + ) + if (rows.length < SQLITE_BIND_CHUNK_SIZE) { + break + } + cursor = rows.at(-1) + } + + if ( + !(await validateVectorIndexGroup(connection, input, { + force: true, + })) + ) { + throw new Error( + `Background sqlite-vec repair did not produce a healthy index for ${input.model} (${input.dimensions} dimensions).`, + ) + } +} + +async function resetVectorIndexGroup( + connection: VectorIndexConnection, + input: VectorIndexGroup, +): Promise { + healthyGroups.delete(vectorIndexGroupKey(connection.path, input)) + const tableName = tableNameForDimensions(input.dimensions) + if (hasVectorTable(connection.database, tableName)) { + withNativeTransaction(connection.database, () => { + withStatement( + connection.database, + `delete from ${tableName} where model = ?`, + statement => statement.run(input.model), + ) + }) + } + await deleteVectorIndexEntriesForGroup(input) +} + function ensureVectorTable( database: DatabaseSync, tableName: string, @@ -422,24 +560,12 @@ async function loadNodeSqlite(): Promise { return await nodeSqlitePromise } -async function exactVectorSearch(input: { - dimensions: number - limit: number - model: string - vector: Float32Array -}): Promise { - if (input.limit <= 0) { - return [] - } - +async function loadDurableVectorPage( + input: VectorIndexGroup, + cursor?: Pick, +): Promise { const db = await getDb() - const results: VectorSearchResult[] = [] - let cursor: - | Pick - | undefined - - while (true) { - const rows = await db.all(sql` + return await db.all(sql` select embedding_hash, model, @@ -449,20 +575,115 @@ select from target_embeddings where model = ${input.model} and dimensions = ${input.dimensions} - ${ - cursor - ? sql`and ( + ${vectorPageCursor(cursor)} +order by target_type, target_id +limit ${SQLITE_BIND_CHUNK_SIZE} +`) +} + +async function loadVectorIndexEntryPage( + input: VectorIndexGroup, + cursor?: Pick, +): Promise { + const db = await getDb() + return await db.all(sql` +select + embedding_hash, + target_id, + target_type +from vector_index_entries +where model = ${input.model} + and dimensions = ${input.dimensions} + ${vectorPageCursor(cursor)} +order by target_type, target_id +limit ${SQLITE_BIND_CHUNK_SIZE} +`) +} + +function loadNativeVectorPage( + database: DatabaseSync, + tableName: string, + input: VectorIndexGroup, + cursor?: Pick, +): VectorIndexRow[] { + const values: unknown[] = [input.model] + const cursorCondition = cursor + ? ` + and ( + target_type > ? + or ( + target_type = ? + and target_id > ? + ) + )` + : '' + if (cursor) { + values.push(cursor.target_type, cursor.target_type, cursor.target_id) + } + values.push(SQLITE_BIND_CHUNK_SIZE) + + return withStatement( + database, + ` +select + embedding_hash, + target_id, + target_type +from ${tableName} +where model = ?${cursorCondition} +order by target_type, target_id +limit ? +`, + statement => statement.all(...values) as VectorIndexRow[], + ) +} + +function vectorPageCursor( + cursor?: Pick, +) { + return cursor + ? sql`and ( target_type > ${cursor.target_type} or ( target_type = ${cursor.target_type} and target_id > ${cursor.target_id} ) )` - : sql`` + : sql`` +} + +function sameVectorIndexRows( + left: VectorIndexRow[], + right: VectorIndexRow[], +): boolean { + return ( + left.length === right.length && + left.every( + (row, index) => + row.embedding_hash === right[index]?.embedding_hash && + row.target_id === right[index]?.target_id && + row.target_type === right[index]?.target_type, + ) + ) } -order by target_type, target_id -limit ${SQLITE_BIND_CHUNK_SIZE} -`) + +async function exactVectorSearch(input: { + dimensions: number + limit: number + model: string + vector: Float32Array +}): Promise { + if (input.limit <= 0) { + return [] + } + + const results: VectorSearchResult[] = [] + let cursor: + | Pick + | undefined + + while (true) { + const rows = await loadDurableVectorPage(input, cursor) for (const row of rows) { results.push({ @@ -627,20 +848,6 @@ async function deleteVectorIndexEntriesForGroup(input: { ) } -async function vectorIndexEntryCount(input: { - dimensions: number - model: string -}): Promise { - const db = await getDb() - const row = await db.get<{ count: number }>(sql` -select count(*) as count -from vector_index_entries -where model = ${input.model} - and dimensions = ${input.dimensions} -`) - return rowCount(row) -} - function withStatement( database: DatabaseSync, query: string, @@ -688,7 +895,28 @@ function chunks(items: T[]): T[][] { return result } -function rowCount(row: unknown): number { - const count = (row as { count?: bigint | number } | undefined)?.count ?? 0 - return Number(count) +function invalidateVectorIndexGroups( + path: string, + targets: VectorIndexTarget[], +): void { + for (const target of targets) { + healthyGroups.delete( + vectorIndexGroupKey(path, { + dimensions: target.dimensions, + model: target.model, + }), + ) + } +} + +function invalidateVectorIndexPath(path: string): void { + for (const key of healthyGroups) { + if (key.startsWith(`${path}:`)) { + healthyGroups.delete(key) + } + } +} + +function vectorIndexGroupKey(path: string, input: VectorIndexGroup): string { + return `${path}:${input.model}:${input.dimensions}` } diff --git a/src/support/error-log.ts b/src/support/error-log.ts index 87e1b7d5..b92b3378 100644 --- a/src/support/error-log.ts +++ b/src/support/error-log.ts @@ -4,7 +4,11 @@ import { resolveProjectContext } from '@/modules/project/context' import { mkdir } from '@/support/file-manager' -type ErrorLogSurface = 'cli' | 'mcp_prompt' | 'mcp_tool' +type ErrorLogSurface = + | 'background_maintenance' + | 'cli' + | 'mcp_prompt' + | 'mcp_tool' type ErrorLogInput = { error: unknown diff --git a/tests/features/providers/protocol/retrieval-evals.test.ts b/tests/features/providers/protocol/retrieval-evals.test.ts index 75997c96..e5a5f9dc 100644 --- a/tests/features/providers/protocol/retrieval-evals.test.ts +++ b/tests/features/providers/protocol/retrieval-evals.test.ts @@ -56,6 +56,7 @@ async function withProjectRoot( } afterEach(async () => { + await globalThis.__konteksWaitForVectorIndexRepairsForTests?.() globalThis.__konteksEmbeddingProviderForTests = undefined globalThis.__konteksVectorIndexConnectionFactoryForTests = undefined await Promise.all(tempDirs.splice(0).map(path => rm(path))) diff --git a/tests/features/providers/protocol/vector-index.test.ts b/tests/features/providers/protocol/vector-index.test.ts index 118ea4a5..039937a5 100644 --- a/tests/features/providers/protocol/vector-index.test.ts +++ b/tests/features/providers/protocol/vector-index.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { mkdtemp, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { eq } from 'drizzle-orm' import getDb from '@/database/actions/_db' import { targetEmbeddings, vectorIndexEntries } from '@/database/schema' import { @@ -38,6 +39,7 @@ beforeEach(() => { }) afterEach(async () => { + await waitForVectorIndexRepairs() process.chdir(originalCwd) globalThis.__konteksVectorIndexConnectionFactoryForTests = undefined if (previousSqliteTestDatabase === undefined) { @@ -61,6 +63,20 @@ describe('vector index', () => { targetType: 'section' as const, vector: new Float32Array([index === 42 ? 1 : 0, 1, 0, 0]), })) + const db = await getDb() + await db.insert(targetEmbeddings).values( + targets.map(target => ({ + createdAt, + dimensions: target.dimensions, + dtype: 'float32', + embeddingHash: target.embeddingHash, + model: target.model, + normalized: 1, + targetId: target.targetId, + targetType: target.targetType, + vectorBlob: toBlob(target.vector), + })), + ) await expect(upsertVectorIndexTargets(targets)).resolves.toBe(true) await expect( @@ -73,6 +89,7 @@ describe('vector index', () => { ).resolves.toMatchObject([{ targetId: 'section-42' }]) await deleteVectorIndexTargets('section') + await db.delete(targetEmbeddings) await expect( searchVectorIndex({ dimensions: 4, @@ -130,6 +147,101 @@ describe('vector index', () => { }), ).resolves.toMatchObject([{ targetId: 'section-272' }]) }) + + it('falls back immediately and repairs partial sqlite-vec metadata', async () => { + await makeTempProject() + const db = await getDb() + const createdAt = new Date().toISOString() + const targets = Array.from({ length: 2 }, (_, index) => ({ + createdAt, + dimensions: 4, + embeddingHash: `repair-hash-${index}`, + model: 'fake/repair', + targetId: `section-${index}`, + targetType: 'section' as const, + vector: new Float32Array([index, 1, 0, 0]), + })) + await db.insert(targetEmbeddings).values( + targets.map(target => ({ + createdAt, + dimensions: target.dimensions, + dtype: 'float32', + embeddingHash: target.embeddingHash, + model: target.model, + normalized: 1, + targetId: target.targetId, + targetType: target.targetType, + vectorBlob: toBlob(target.vector), + })), + ) + await upsertVectorIndexTargets(targets) + await db + .delete(vectorIndexEntries) + .where(eq(vectorIndexEntries.targetId, 'section-1')) + + await expect( + searchVectorIndex({ + dimensions: 4, + limit: 1, + model: 'fake/repair', + vector: new Float32Array([1, 1, 0, 0]), + }), + ).resolves.toMatchObject([{ targetId: 'section-1' }]) + + await waitForVectorIndexRepairs() + const repairedEntries = await db + .select() + .from(vectorIndexEntries) + .where(eq(vectorIndexEntries.model, 'fake/repair')) + expect(repairedEntries).toHaveLength(2) + }) + + it('repairs vector metadata hash mismatches with equal row counts', async () => { + await makeTempProject() + const db = await getDb() + const createdAt = new Date().toISOString() + const target = { + createdAt, + dimensions: 4, + embeddingHash: 'repair-hash', + model: 'fake/hash-repair', + targetId: 'section-hash', + targetType: 'section' as const, + vector: new Float32Array([1, 1, 0, 0]), + } + await db.insert(targetEmbeddings).values({ + createdAt, + dimensions: target.dimensions, + dtype: 'float32', + embeddingHash: target.embeddingHash, + model: target.model, + normalized: 1, + targetId: target.targetId, + targetType: target.targetType, + vectorBlob: toBlob(target.vector), + }) + await upsertVectorIndexTargets([target]) + await db + .update(vectorIndexEntries) + .set({ embeddingHash: 'stale-hash' }) + .where(eq(vectorIndexEntries.targetId, target.targetId)) + + await expect( + searchVectorIndex({ + dimensions: 4, + limit: 1, + model: target.model, + vector: target.vector, + }), + ).resolves.toMatchObject([{ targetId: target.targetId }]) + + await waitForVectorIndexRepairs() + const [repairedEntry] = await db + .select() + .from(vectorIndexEntries) + .where(eq(vectorIndexEntries.targetId, target.targetId)) + expect(repairedEntry?.embeddingHash).toBe(target.embeddingHash) + }) }) async function makeTempProject(fileCount = 1): Promise { @@ -177,3 +289,7 @@ function useMissingVectorTableConnection(): void { path: 'missing-vector-table.sqlite', }) } + +async function waitForVectorIndexRepairs(): Promise { + await globalThis.__konteksWaitForVectorIndexRepairsForTests?.() +}