From 2f1c364de98d6f7894ce6789e2465030b0ebc08a Mon Sep 17 00:00:00 2001 From: Jason Nathan Date: Wed, 7 Feb 2024 23:58:46 +0800 Subject: [PATCH 1/7] created sqlite record manager and integration tests --- libs/langchain-community/package.json | 2 + .../langchain-community/src/indexes/sqlite.ts | 211 ++++++++++++++++++ .../src/indexes/tests/sqlite.int.test.ts | 82 +++++++ yarn.lock | 22 ++ 4 files changed, 317 insertions(+) create mode 100644 libs/langchain-community/src/indexes/sqlite.ts create mode 100644 libs/langchain-community/src/indexes/tests/sqlite.int.test.ts diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 956301c1a36..51ca5798d29 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -93,6 +93,7 @@ "@tensorflow/tfjs-converter": "^3.6.0", "@tensorflow/tfjs-core": "^3.6.0", "@tsconfig/recommended": "^1.0.2", + "@types/better-sqlite3": "^7.6.9", "@types/flat": "^5.0.2", "@types/html-to-text": "^9", "@types/jsdom": "^21.1.1", @@ -111,6 +112,7 @@ "@xata.io/client": "^0.28.0", "@xenova/transformers": "^2.5.4", "@zilliz/milvus2-sdk-node": ">=2.2.11", + "better-sqlite3": "^9.4.0", "cassandra-driver": "^4.7.2", "chromadb": "^1.5.3", "closevector-common": "0.1.3", diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts new file mode 100644 index 00000000000..d5a2af0fff8 --- /dev/null +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -0,0 +1,211 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import Database, { Database as DatabaseType, Statement } from "better-sqlite3"; +import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from "./base.js"; + +interface TimeRow { + epoch: number; +} + +interface Record { + k: string; + ex: boolean; +} + + +/** + * Options for configuring the SQLiteRecordManager class. + */ +export type SQLiteRecordManagerOptions = { + /** + * The file path or connection string of the SQLite database. + */ + filepathOrConnectionString: string; + + /** + * The name of the table in the SQLite database. + */ + tableName: string; +}; + +export class SQLiteRecordManager implements RecordManagerInterface { + lc_namespace = ["langchain", "recordmanagers", "sqlite"]; + + tableName: string + + db: DatabaseType; + + namespace: string + + constructor(namespace: string, config: SQLiteRecordManagerOptions) { + const { filepathOrConnectionString, tableName } = config; + this.namespace = namespace; + this.tableName = tableName; + this.db = new Database(filepathOrConnectionString); + } + + async createSchema(): Promise { + return new Promise((resolve, reject) => { + try { + this.db.exec(` + CREATE TABLE IF NOT EXISTS "${this.tableName}" ( + uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + key TEXT NOT NULL, + namespace TEXT NOT NULL, + updated_at REAL NOT NULL, + group_id TEXT, + UNIQUE (key, namespace) + ); + CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); + CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); + CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); + CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id); + `); + resolve(); + } catch (error: unknown) { + // Handle errors specific to SQLite + console.error('Error creating schema:', error); + reject(error); + } + }); + } + + async getTime(): Promise { + return new Promise((resolve, reject) => { + try { + const statement: Statement<[]> = this.db.prepare("SELECT strftime('%s', 'now') AS epoch"); + const {epoch} = statement.get() as TimeRow; + resolve(epoch); + } catch (error) { + reject(error); + } + }); + } + + + async update(keys: string[], updateOptions?: UpdateOptions): Promise { + return new Promise((resolve, reject) => { + if (keys.length === 0) { + resolve(); + return; + } + + this.getTime() + .then(async updatedAt => { + const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}; + + if (timeAtLeast && updatedAt < timeAtLeast) { + throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`); + } + + const groupIds = _groupIds ?? keys.map(() => null); + + if (groupIds.length !== keys.length) { + throw new Error(`Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})`); + } + + const recordsToUpsert = keys.map((key, i) => [ + key, + this.namespace, + updatedAt, + groupIds[i], + ]); + + for (const row of recordsToUpsert) { + // Prepare the statement for each row with a fixed number of anonymous placeholders + const individualStatement = this.db.prepare(` + INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) + VALUES (?, ?, ?, ?) + ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at; + `); + // Execute the prepared statement for the current row + individualStatement.run(...row); + } + + resolve(); + }) + .catch(error => reject(error)); + }); + } + + async exists(keys: string[]): Promise { + return new Promise((resolve, reject) => { + if (keys.length === 0) { + resolve([]); + return; + } + + const arrayPlaceholders = keys.map((_, i) => `$${i + 2}`).join(", "); + + const statement: Statement<[string, ...string[]]> = this.db.prepare(` + SELECT k, (key is not null) ex + FROM unnest(ARRAY[${arrayPlaceholders}]) k + LEFT JOIN "${this.tableName}" ON k=key AND namespace = ? + `); + + try { + const result = statement.all(this.namespace, ...keys) as Record[]; + + resolve(result.map(row => row.ex)); + } catch (error) { + reject(error); + } + }); + } + + async listKeys(options?: ListKeyOptions): Promise { + return new Promise((resolve, reject) => { + const { before, after, limit, groupIds } = options ?? {}; + let query = `SELECT key FROM "${this.tableName}" WHERE namespace = ?`; + const values: (string | number | (string | null)[])[] = [this.namespace]; + + if (before) { + query += ` AND updated_at < ?`; + values.push(before); + } + + if (after) { + query += ` AND updated_at > ?`; + values.push(after); + } + + if (limit) { + query += ` LIMIT ?`; + values.push(limit); + } + + if (groupIds && Array.isArray(groupIds)) { + query += ` AND group_id IN (${groupIds.map(() => '?').join(', ')})`; + values.push(...(groupIds as (string | number | (string | null)[])[])); + } + + query += ";"; + + try { + const result = this.db.prepare(query).all(...values) as { key: string }[]; + resolve(result.map(row => row.key)); + } catch (error) { + reject(error); + } + }); + } + + async deleteKeys(keys: string[]): Promise { + return new Promise((resolve, reject) => { + if (keys.length === 0) { + resolve(); + return; + } + + const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${keys.map(() => '?').join(', ')});`; + const values: (string | number)[] = [this.namespace, ...keys]; + + try { + this.db.prepare(query).run(...values); + resolve(); + } catch (error) { + reject(error); + } + }); + } + +} \ No newline at end of file diff --git a/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts b/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts new file mode 100644 index 00000000000..58aedb00ad3 --- /dev/null +++ b/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test, jest } from "@jest/globals"; +import { SQLiteRecordManager } from "../sqlite.js"; + +describe("SQLiteRecordManager", () => { + const tableName = "upsertion_record"; + let recordManager: SQLiteRecordManager; + + beforeAll(async () => { + // Initialize SQLiteRecordManager with a temporary filepath + const filepathOrConnectionString = ":memory:"; + recordManager = new SQLiteRecordManager('test', {tableName, filepathOrConnectionString}); + await recordManager.createSchema(); + }); + + afterEach(() => { + recordManager.db.exec(`DELETE FROM "${tableName}"`); + }); + + afterAll(() => { + // Close the database connection after all tests have finished + recordManager.db.close(); + }); + test("Test upsertion", async () => { + const keys = ["a", "b", "c"]; + await recordManager.update(keys); + const readKeys = await recordManager.listKeys(); + expect(readKeys).toEqual(expect.arrayContaining(keys)); + expect(readKeys).toHaveLength(keys.length); + }); + + test("Test upsertion with timeAtLeast", async () => { + // Mock getTime to return 100. + const unmockedGetTime = recordManager.getTime; + recordManager.getTime = jest.fn(() => Promise.resolve(100)); + + const keys = ["a", "b", "c"]; + await expect( + recordManager.update(keys, { timeAtLeast: 110 }) + ).rejects.toThrowError(); + const readKeys = await recordManager.listKeys(); + expect(readKeys).toHaveLength(0); + + // Set getTime back to normal. + recordManager.getTime = unmockedGetTime; + }); + + interface RecordRow { + // Define the structure of the rows returned from the database query + // Adjust the properties based on your table schema + id: number; + key: string; + updated_at: number; + group_id: string; + } + + test("Test update timestamp", async () => { + const unmockedGetTime = recordManager.getTime; + recordManager.getTime = jest.fn(() => Promise.resolve(100)); + try { + const keys = ["a", "b", "c"]; + await recordManager.update(keys); + const rows = recordManager.db.prepare(`SELECT * FROM "${tableName}"`).all() as RecordRow[]; + rows.forEach((row) => expect(row.updated_at).toEqual(100)); + + recordManager.getTime = jest.fn(() => Promise.resolve(200)); + await recordManager.update(keys); + const rows2 = await recordManager.db.prepare(`SELECT * FROM "${tableName}"`).all() as RecordRow[]; + rows2.forEach((row) => expect(row.updated_at).toEqual(200)); + } finally { + recordManager.getTime = unmockedGetTime; + } + }); + + test("Test update with groupIds", async () => { + const keys = ["a", "b", "c"]; + await recordManager.update(keys, { groupIds: ["group1", "group1", "group2"] }); + const rows = recordManager.db.prepare(`SELECT * FROM "${tableName}" WHERE group_id = ?`).all("group1") as RecordRow[]; + expect(rows.length).toEqual(2); + rows.forEach((row) => expect(row.group_id).toEqual("group1")); + }); + +}); \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index b06c219d91d..b27f76625fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8911,6 +8911,7 @@ __metadata: "@tensorflow/tfjs-converter": ^3.6.0 "@tensorflow/tfjs-core": ^3.6.0 "@tsconfig/recommended": ^1.0.2 + "@types/better-sqlite3": ^7.6.9 "@types/flat": ^5.0.2 "@types/html-to-text": ^9 "@types/jsdom": ^21.1.1 @@ -8929,6 +8930,7 @@ __metadata: "@xata.io/client": ^0.28.0 "@xenova/transformers": ^2.5.4 "@zilliz/milvus2-sdk-node": ">=2.2.11" + better-sqlite3: ^9.4.0 cassandra-driver: ^4.7.2 chromadb: ^1.5.3 closevector-common: 0.1.3 @@ -13664,6 +13666,15 @@ __metadata: languageName: node linkType: hard +"@types/better-sqlite3@npm:^7.6.9": + version: 7.6.9 + resolution: "@types/better-sqlite3@npm:7.6.9" + dependencies: + "@types/node": "*" + checksum: 6572076639dde1e65ad8fe0e319e797fa40793ca805ec39aa1d072d3f145f218775eafd27d7266fb4e42a6291d8b8b836278e7880b15ef728c750dfcbba2ee52 + languageName: node + linkType: hard + "@types/body-parser@npm:*": version: 1.19.2 resolution: "@types/body-parser@npm:1.19.2" @@ -16575,6 +16586,17 @@ __metadata: languageName: node linkType: hard +"better-sqlite3@npm:^9.4.0": + version: 9.4.0 + resolution: "better-sqlite3@npm:9.4.0" + dependencies: + bindings: ^1.5.0 + node-gyp: latest + prebuild-install: ^7.1.1 + checksum: a1a470fae20dfba82d6e74ae90b35ea8996c60922e95574162732d6e076e84c0c90fc4ff77ab8c27554671899eb15f284e2c8de5e4ee406aa9f7eb170eca5bee + languageName: node + linkType: hard + "big-integer@npm:^1.6.44": version: 1.6.51 resolution: "big-integer@npm:1.6.51" From c2fce2a1d281196812d4970ff897c0ef23b5aeea Mon Sep 17 00:00:00 2001 From: Jason Nathan Date: Thu, 8 Feb 2024 07:12:51 +0800 Subject: [PATCH 2/7] Updated tests and implementation --- .../langchain-community/src/indexes/sqlite.ts | 144 ++++++++++-------- .../src/indexes/tests/sqlite.int.test.ts | 116 ++++++++++++-- 2 files changed, 186 insertions(+), 74 deletions(-) diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts index d5a2af0fff8..b42cbf8d5de 100644 --- a/libs/langchain-community/src/indexes/sqlite.ts +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -1,6 +1,10 @@ // eslint-disable-next-line import/no-extraneous-dependencies import Database, { Database as DatabaseType, Statement } from "better-sqlite3"; -import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from "./base.js"; +import { + ListKeyOptions, + RecordManagerInterface, + UpdateOptions, +} from "./base.js"; interface TimeRow { epoch: number; @@ -10,7 +14,9 @@ interface Record { k: string; ex: boolean; } - +interface KeyRecord { + key: string; +} /** * Options for configuring the SQLiteRecordManager class. @@ -20,7 +26,7 @@ export type SQLiteRecordManagerOptions = { * The file path or connection string of the SQLite database. */ filepathOrConnectionString: string; - + /** * The name of the table in the SQLite database. */ @@ -30,15 +36,15 @@ export type SQLiteRecordManagerOptions = { export class SQLiteRecordManager implements RecordManagerInterface { lc_namespace = ["langchain", "recordmanagers", "sqlite"]; - tableName: string + tableName: string; db: DatabaseType; - namespace: string + namespace: string; constructor(namespace: string, config: SQLiteRecordManagerOptions) { const { filepathOrConnectionString, tableName } = config; - this.namespace = namespace; + this.namespace = namespace; this.tableName = tableName; this.db = new Database(filepathOrConnectionString); } @@ -63,7 +69,7 @@ export class SQLiteRecordManager implements RecordManagerInterface { resolve(); } catch (error: unknown) { // Handle errors specific to SQLite - console.error('Error creating schema:', error); + console.error("Error creating schema:", error); reject(error); } }); @@ -72,8 +78,10 @@ export class SQLiteRecordManager implements RecordManagerInterface { async getTime(): Promise { return new Promise((resolve, reject) => { try { - const statement: Statement<[]> = this.db.prepare("SELECT strftime('%s', 'now') AS epoch"); - const {epoch} = statement.get() as TimeRow; + const statement: Statement<[]> = this.db.prepare( + "SELECT strftime('%s', 'now') AS epoch" + ); + const { epoch } = statement.get() as TimeRow; resolve(epoch); } catch (error) { reject(error); @@ -81,50 +89,53 @@ export class SQLiteRecordManager implements RecordManagerInterface { }); } - async update(keys: string[], updateOptions?: UpdateOptions): Promise { return new Promise((resolve, reject) => { - if (keys.length === 0) { - resolve(); - return; - } - - this.getTime() - .then(async updatedAt => { - const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}; - - if (timeAtLeast && updatedAt < timeAtLeast) { - throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`); - } - - const groupIds = _groupIds ?? keys.map(() => null); - - if (groupIds.length !== keys.length) { - throw new Error(`Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})`); - } - - const recordsToUpsert = keys.map((key, i) => [ - key, - this.namespace, - updatedAt, - groupIds[i], - ]); - - for (const row of recordsToUpsert) { - // Prepare the statement for each row with a fixed number of anonymous placeholders - const individualStatement = this.db.prepare(` + if (keys.length === 0) { + resolve(); + return; + } + + this.getTime() + .then(async (updatedAt) => { + const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}; + + if (timeAtLeast && updatedAt < timeAtLeast) { + throw new Error( + `Time sync issue with database ${updatedAt} < ${timeAtLeast}` + ); + } + + const groupIds = _groupIds ?? keys.map(() => null); + + if (groupIds.length !== keys.length) { + throw new Error( + `Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})` + ); + } + + const recordsToUpsert = keys.map((key, i) => [ + key, + this.namespace, + updatedAt, + groupIds[i], + ]); + + for (const row of recordsToUpsert) { + // Prepare the statement for each row with a fixed number of anonymous placeholders + const individualStatement = this.db.prepare(` INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES (?, ?, ?, ?) ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at; `); - // Execute the prepared statement for the current row - individualStatement.run(...row); - } - - resolve(); - }) - .catch(error => reject(error)); - }); + // Execute the prepared statement for the current row + individualStatement.run(...row); + } + + resolve(); + }) + .catch((error) => reject(error)); + }); } async exists(keys: string[]): Promise { @@ -134,18 +145,22 @@ export class SQLiteRecordManager implements RecordManagerInterface { return; } - const arrayPlaceholders = keys.map((_, i) => `$${i + 2}`).join(", "); - - const statement: Statement<[string, ...string[]]> = this.db.prepare(` - SELECT k, (key is not null) ex - FROM unnest(ARRAY[${arrayPlaceholders}]) k - LEFT JOIN "${this.tableName}" ON k=key AND namespace = ? - `); + const placeholders = keys.map((_) => `?`).join(", "); + const sql = ` + SELECT key + FROM "${this.tableName}" + WHERE namespace = ? AND key IN (${placeholders}) + `; try { - const result = statement.all(this.namespace, ...keys) as Record[]; - - resolve(result.map(row => row.ex)); + const rows = this.db + .prepare(sql) + .all(this.namespace, ...keys) as KeyRecord[]; + // Create a set of existing keys for faster lookup + const existingKeysSet = new Set(rows.map((row) => row.key)); + // Map to boolean array based on presence in the existingKeysSet + const exists = keys.map((key) => existingKeysSet.has(key)); + resolve(exists); } catch (error) { reject(error); } @@ -174,15 +189,17 @@ export class SQLiteRecordManager implements RecordManagerInterface { } if (groupIds && Array.isArray(groupIds)) { - query += ` AND group_id IN (${groupIds.map(() => '?').join(', ')})`; + query += ` AND group_id IN (${groupIds.map(() => "?").join(", ")})`; values.push(...(groupIds as (string | number | (string | null)[])[])); } query += ";"; try { - const result = this.db.prepare(query).all(...values) as { key: string }[]; - resolve(result.map(row => row.key)); + const result = this.db.prepare(query).all(...values) as { + key: string; + }[]; + resolve(result.map((row) => row.key)); } catch (error) { reject(error); } @@ -196,7 +213,9 @@ export class SQLiteRecordManager implements RecordManagerInterface { return; } - const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${keys.map(() => '?').join(', ')});`; + const query = `DELETE FROM "${ + this.tableName + }" WHERE namespace = ? AND key IN (${keys.map(() => "?").join(", ")});`; const values: (string | number)[] = [this.namespace, ...keys]; try { @@ -207,5 +226,4 @@ export class SQLiteRecordManager implements RecordManagerInterface { } }); } - -} \ No newline at end of file +} diff --git a/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts b/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts index 58aedb00ad3..ea3cdfdaeb6 100644 --- a/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts +++ b/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts @@ -6,18 +6,20 @@ describe("SQLiteRecordManager", () => { let recordManager: SQLiteRecordManager; beforeAll(async () => { - // Initialize SQLiteRecordManager with a temporary filepath const filepathOrConnectionString = ":memory:"; - recordManager = new SQLiteRecordManager('test', {tableName, filepathOrConnectionString}); + recordManager = new SQLiteRecordManager("test", { + tableName, + filepathOrConnectionString, + }); await recordManager.createSchema(); }); - afterEach(() => { + afterEach(async () => { recordManager.db.exec(`DELETE FROM "${tableName}"`); + await recordManager.createSchema(); }); afterAll(() => { - // Close the database connection after all tests have finished recordManager.db.close(); }); test("Test upsertion", async () => { @@ -59,24 +61,116 @@ describe("SQLiteRecordManager", () => { try { const keys = ["a", "b", "c"]; await recordManager.update(keys); - const rows = recordManager.db.prepare(`SELECT * FROM "${tableName}"`).all() as RecordRow[]; + const rows = recordManager.db + .prepare(`SELECT * FROM "${tableName}"`) + .all() as RecordRow[]; rows.forEach((row) => expect(row.updated_at).toEqual(100)); - + recordManager.getTime = jest.fn(() => Promise.resolve(200)); await recordManager.update(keys); - const rows2 = await recordManager.db.prepare(`SELECT * FROM "${tableName}"`).all() as RecordRow[]; + const rows2 = (await recordManager.db + .prepare(`SELECT * FROM "${tableName}"`) + .all()) as RecordRow[]; rows2.forEach((row) => expect(row.updated_at).toEqual(200)); } finally { recordManager.getTime = unmockedGetTime; } }); - + test("Test update with groupIds", async () => { const keys = ["a", "b", "c"]; - await recordManager.update(keys, { groupIds: ["group1", "group1", "group2"] }); - const rows = recordManager.db.prepare(`SELECT * FROM "${tableName}" WHERE group_id = ?`).all("group1") as RecordRow[]; + await recordManager.update(keys, { + groupIds: ["group1", "group1", "group2"], + }); + const rows = recordManager.db + .prepare(`SELECT * FROM "${tableName}" WHERE group_id = ?`) + .all("group1") as RecordRow[]; expect(rows.length).toEqual(2); rows.forEach((row) => expect(row.group_id).toEqual("group1")); }); -}); \ No newline at end of file + test("Exists", async () => { + const keys = ["a", "b", "c"]; + await recordManager.update(keys); + const exists = await recordManager.exists(keys); + expect(exists).toEqual([true, true, true]); + + const nonExistentKeys = ["d", "e", "f"]; + const nonExists = await recordManager.exists(nonExistentKeys); + expect(nonExists).toEqual([false, false, false]); + + const mixedKeys = ["a", "e", "c"]; + const mixedExists = await recordManager.exists(mixedKeys); + expect(mixedExists).toEqual([true, false, true]); + }); + + test("Delete", async () => { + const keys = ["a", "b", "c"]; + await recordManager.update(keys); + await recordManager.deleteKeys(["a", "c"]); + const readKeys = await recordManager.listKeys(); + expect(readKeys).toEqual(["b"]); + }); + + test("List keys", async () => { + const unmockedGetTime = recordManager.getTime; + recordManager.getTime = jest.fn(() => Promise.resolve(100)); + try { + const keys = ["a", "b", "c"]; + await recordManager.update(keys); + const readKeys = await recordManager.listKeys(); + expect(readKeys).toEqual(expect.arrayContaining(keys)); + expect(readKeys).toHaveLength(keys.length); + + // All keys inserted after 90: should be all keys + const readKeysAfterInsertedAfter = await recordManager.listKeys({ + after: 90, + }); + expect(readKeysAfterInsertedAfter).toEqual(expect.arrayContaining(keys)); + + // All keys inserted after 110: should be none + const readKeysAfterInsertedBefore = await recordManager.listKeys({ + after: 110, + }); + expect(readKeysAfterInsertedBefore).toEqual([]); + + // All keys inserted before 110: should be all keys + const readKeysBeforeInsertedBefore = await recordManager.listKeys({ + before: 110, + }); + expect(readKeysBeforeInsertedBefore).toEqual( + expect.arrayContaining(keys) + ); + + // All keys inserted before 90: should be none + const readKeysBeforeInsertedAfter = await recordManager.listKeys({ + before: 90, + }); + expect(readKeysBeforeInsertedAfter).toEqual([]); + + // Set one key to updated at 120 and one at 80 + recordManager.getTime = jest.fn(() => Promise.resolve(120)); + await recordManager.update(["a"]); + recordManager.getTime = jest.fn(() => Promise.resolve(80)); + await recordManager.update(["b"]); + + // All keys updated after 90 and before 110: should only be "c" now + const readKeysBeforeAndAfter = await recordManager.listKeys({ + before: 110, + after: 90, + }); + expect(readKeysBeforeAndAfter).toEqual(["c"]); + } finally { + recordManager.getTime = unmockedGetTime; + } + }); + + test("List keys with groupIds", async () => { + const keys = ["a", "b", "c"]; + await recordManager.update(keys, { + groupIds: ["group1", "group1", "group2"], + }); + const readKeys = await recordManager.listKeys({ groupIds: ["group1"] }); + expect(readKeys).toEqual(["a", "b"]); + }); +}); From 65524412154949581c568a95363bebc4f33a983b Mon Sep 17 00:00:00 2001 From: Jason Nathan Date: Thu, 8 Feb 2024 07:12:51 +0800 Subject: [PATCH 3/7] Updated tests and implementation Updated implementation to make it cleaner --- .../langchain-community/src/indexes/sqlite.ts | 297 +++++++++--------- 1 file changed, 145 insertions(+), 152 deletions(-) diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts index b42cbf8d5de..eb44d588f1d 100644 --- a/libs/langchain-community/src/indexes/sqlite.ts +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -10,10 +10,6 @@ interface TimeRow { epoch: number; } -interface Record { - k: string; - ex: boolean; -} interface KeyRecord { key: string; } @@ -50,29 +46,25 @@ export class SQLiteRecordManager implements RecordManagerInterface { } async createSchema(): Promise { - return new Promise((resolve, reject) => { - try { - this.db.exec(` - CREATE TABLE IF NOT EXISTS "${this.tableName}" ( - uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), - key TEXT NOT NULL, - namespace TEXT NOT NULL, - updated_at REAL NOT NULL, - group_id TEXT, - UNIQUE (key, namespace) - ); - CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); - CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); - CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); - CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id); - `); - resolve(); - } catch (error: unknown) { - // Handle errors specific to SQLite - console.error("Error creating schema:", error); - reject(error); - } - }); + try { + this.db.exec(` + CREATE TABLE IF NOT EXISTS "${this.tableName}" ( + uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + key TEXT NOT NULL, + namespace TEXT NOT NULL, + updated_at REAL NOT NULL, + group_id TEXT, + UNIQUE (key, namespace) + ); + CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); + CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); + CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); + CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id); + `); + } catch (error) { + console.error("Error creating schema:", error); + throw error; // Re-throw the error to let the caller handle it + } } async getTime(): Promise { @@ -82,7 +74,7 @@ export class SQLiteRecordManager implements RecordManagerInterface { "SELECT strftime('%s', 'now') AS epoch" ); const { epoch } = statement.get() as TimeRow; - resolve(epoch); + resolve(Number(epoch)); } catch (error) { reject(error); } @@ -90,140 +82,141 @@ export class SQLiteRecordManager implements RecordManagerInterface { } async update(keys: string[], updateOptions?: UpdateOptions): Promise { - return new Promise((resolve, reject) => { - if (keys.length === 0) { - resolve(); - return; + if (keys.length === 0) { + return; + } + + const updatedAt = await this.getTime(); + const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}; + + if (timeAtLeast && updatedAt < timeAtLeast) { + throw new Error( + `Time sync issue with database ${updatedAt} < ${timeAtLeast}` + ); + } + + const groupIds = _groupIds ?? keys.map(() => null); + + if (groupIds.length !== keys.length) { + throw new Error( + `Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})` + ); + } + + const recordsToUpsert = keys.map((key, i) => [ + key, + this.namespace, + updatedAt, + groupIds[i] ?? null, // Ensure groupIds[i] is null if undefined + ]); + + // Consider using a transaction for batch operations + const updateTransaction = this.db.transaction(() => { + for (const row of recordsToUpsert) { + this.db + .prepare( + ` + INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) + VALUES (?, ?, ?, ?) + ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at + ` + ) + .run(...row); } - - this.getTime() - .then(async (updatedAt) => { - const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}; - - if (timeAtLeast && updatedAt < timeAtLeast) { - throw new Error( - `Time sync issue with database ${updatedAt} < ${timeAtLeast}` - ); - } - - const groupIds = _groupIds ?? keys.map(() => null); - - if (groupIds.length !== keys.length) { - throw new Error( - `Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})` - ); - } - - const recordsToUpsert = keys.map((key, i) => [ - key, - this.namespace, - updatedAt, - groupIds[i], - ]); - - for (const row of recordsToUpsert) { - // Prepare the statement for each row with a fixed number of anonymous placeholders - const individualStatement = this.db.prepare(` - INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) - VALUES (?, ?, ?, ?) - ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at; - `); - // Execute the prepared statement for the current row - individualStatement.run(...row); - } - - resolve(); - }) - .catch((error) => reject(error)); }); + + updateTransaction(); } async exists(keys: string[]): Promise { - return new Promise((resolve, reject) => { - if (keys.length === 0) { - resolve([]); - return; - } - - const placeholders = keys.map((_) => `?`).join(", "); - const sql = ` - SELECT key - FROM "${this.tableName}" - WHERE namespace = ? AND key IN (${placeholders}) - `; - - try { - const rows = this.db - .prepare(sql) - .all(this.namespace, ...keys) as KeyRecord[]; - // Create a set of existing keys for faster lookup - const existingKeysSet = new Set(rows.map((row) => row.key)); - // Map to boolean array based on presence in the existingKeysSet - const exists = keys.map((key) => existingKeysSet.has(key)); - resolve(exists); - } catch (error) { - reject(error); - } - }); + if (keys.length === 0) { + return []; + } + + // Prepare the placeholders and the query + const placeholders = keys.map((_, i) => `?`).join(", "); + const sql = ` + SELECT key + FROM "${this.tableName}" + WHERE namespace = ? AND key IN (${placeholders}) + `; + + // Initialize an array to fill with the existence checks + const existsArray = new Array(keys.length).fill(false); + + try { + // Execute the query + const rows = this.db + .prepare(sql) + .all(this.namespace, ...keys) as KeyRecord[]; + // Create a set of existing keys for faster lookup + const existingKeysSet = new Set(rows.map((row) => row.key)); + // Map the input keys to booleans indicating if they exist + keys.forEach((key, index) => { + existsArray[index] = existingKeysSet.has(key); + }); + return existsArray; + } catch (error) { + console.error("Error checking existence of keys:", error); + throw error; // Allow the caller to handle the error + } } async listKeys(options?: ListKeyOptions): Promise { - return new Promise((resolve, reject) => { - const { before, after, limit, groupIds } = options ?? {}; - let query = `SELECT key FROM "${this.tableName}" WHERE namespace = ?`; - const values: (string | number | (string | null)[])[] = [this.namespace]; - - if (before) { - query += ` AND updated_at < ?`; - values.push(before); - } - - if (after) { - query += ` AND updated_at > ?`; - values.push(after); - } - - if (limit) { - query += ` LIMIT ?`; - values.push(limit); - } - - if (groupIds && Array.isArray(groupIds)) { - query += ` AND group_id IN (${groupIds.map(() => "?").join(", ")})`; - values.push(...(groupIds as (string | number | (string | null)[])[])); - } - - query += ";"; - - try { - const result = this.db.prepare(query).all(...values) as { - key: string; - }[]; - resolve(result.map((row) => row.key)); - } catch (error) { - reject(error); - } - }); + const { before, after, limit, groupIds } = options ?? {}; + let query = `SELECT key FROM "${this.tableName}" WHERE namespace = ?`; + const values: (string | number | string[])[] = [this.namespace]; + + if (before) { + query += ` AND updated_at < ?`; + values.push(before); + } + + if (after) { + query += ` AND updated_at > ?`; + values.push(after); + } + + if (limit) { + query += ` LIMIT ?`; + values.push(limit); + } + + if (groupIds && Array.isArray(groupIds)) { + query += ` AND group_id IN (${groupIds + .filter((gid) => gid !== null) + .map(() => "?") + .join(", ")})`; + values.push(...groupIds.filter((gid): gid is string => gid !== null)); + } + + query += ";"; + + // Directly using try/catch with async/await for cleaner flow + try { + const result = this.db.prepare(query).all(...values) as { key: string }[]; + return result.map((row) => row.key); + } catch (error) { + console.error("Error listing keys:", error); + throw error; // Re-throw the error to be handled by the caller + } } async deleteKeys(keys: string[]): Promise { - return new Promise((resolve, reject) => { - if (keys.length === 0) { - resolve(); - return; - } - - const query = `DELETE FROM "${ - this.tableName - }" WHERE namespace = ? AND key IN (${keys.map(() => "?").join(", ")});`; - const values: (string | number)[] = [this.namespace, ...keys]; - - try { - this.db.prepare(query).run(...values); - resolve(); - } catch (error) { - reject(error); - } - }); + if (keys.length === 0) { + return; + } + + const placeholders = keys.map(() => "?").join(", "); + const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`; + const values: (string | number)[] = [this.namespace, ...keys]; + + // Directly using try/catch with async/await for cleaner flow + try { + this.db.prepare(query).run(...values); + } catch (error) { + console.error("Error deleting keys:", error); + throw error; // Re-throw the error to be handled by the caller + } } } From eb14ce0c98fffc8b1a4354b5156fdcf75a944653 Mon Sep 17 00:00:00 2001 From: Jason Nathan Date: Fri, 9 Feb 2024 15:17:27 +0800 Subject: [PATCH 4/7] some tiny refactors --- .../langchain-community/src/indexes/sqlite.ts | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts index eb44d588f1d..867a8261461 100644 --- a/libs/langchain-community/src/indexes/sqlite.ts +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -68,17 +68,16 @@ export class SQLiteRecordManager implements RecordManagerInterface { } async getTime(): Promise { - return new Promise((resolve, reject) => { - try { - const statement: Statement<[]> = this.db.prepare( - "SELECT strftime('%s', 'now') AS epoch" - ); - const { epoch } = statement.get() as TimeRow; - resolve(Number(epoch)); - } catch (error) { - reject(error); - } - }); + try { + const statement: Statement<[]> = this.db.prepare( + "SELECT strftime('%s', 'now') AS epoch" + ); + const { epoch } = statement.get() as TimeRow; + return Number(epoch); + } catch (error) { + console.error('Error getting time in SQLiteRecordManager:', error); + throw error; + } } async update(keys: string[], updateOptions?: UpdateOptions): Promise { @@ -134,7 +133,7 @@ export class SQLiteRecordManager implements RecordManagerInterface { } // Prepare the placeholders and the query - const placeholders = keys.map((_, i) => `?`).join(", "); + const placeholders = keys.map(() => `?`).join(", "); const sql = ` SELECT key FROM "${this.tableName}" From 1a3bc339011c5b12170cda2fa7b6ed195c932cb1 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 9 Feb 2024 14:57:30 -0800 Subject: [PATCH 5/7] cr --- .../langchain-community/src/indexes/sqlite.ts | 79 +++++++++++-------- .../src/indexes/tests/sqlite.int.test.ts | 5 +- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts index 867a8261461..1a26ab0c73c 100644 --- a/libs/langchain-community/src/indexes/sqlite.ts +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -19,10 +19,15 @@ interface KeyRecord { */ export type SQLiteRecordManagerOptions = { /** - * The file path or connection string of the SQLite database. + * The file path of the SQLite database. + * One of either `localPath` or `connectionString` is required. */ - filepathOrConnectionString: string; - + localPath?: string; + /** + * The connection string of the SQLite database. + * One of either `localPath` or `connectionString` is required. + */ + connectionString?: string; /** * The name of the table in the SQLite database. */ @@ -39,30 +44,39 @@ export class SQLiteRecordManager implements RecordManagerInterface { namespace: string; constructor(namespace: string, config: SQLiteRecordManagerOptions) { - const { filepathOrConnectionString, tableName } = config; + const { localPath, connectionString, tableName } = config; + if (!connectionString && !localPath) { + throw new Error( + "One of either `localPath` or `connectionString` is required." + ); + } + if (connectionString && localPath) { + throw new Error( + "Only one of either `localPath` or `connectionString` is allowed." + ); + } this.namespace = namespace; this.tableName = tableName; - this.db = new Database(filepathOrConnectionString); + this.db = new Database(connectionString ?? localPath); } async createSchema(): Promise { try { this.db.exec(` - CREATE TABLE IF NOT EXISTS "${this.tableName}" ( - uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), - key TEXT NOT NULL, - namespace TEXT NOT NULL, - updated_at REAL NOT NULL, - group_id TEXT, - UNIQUE (key, namespace) - ); - CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); - CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); - CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); - CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id); - `); +CREATE TABLE IF NOT EXISTS "${this.tableName}" ( + uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + key TEXT NOT NULL, + namespace TEXT NOT NULL, + updated_at REAL NOT NULL, + group_id TEXT, + UNIQUE (key, namespace) +); +CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at); +CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key); +CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace); +CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`); } catch (error) { - console.error("Error creating schema:", error); + console.error("Error creating schema"); throw error; // Re-throw the error to let the caller handle it } } @@ -113,17 +127,13 @@ export class SQLiteRecordManager implements RecordManagerInterface { const updateTransaction = this.db.transaction(() => { for (const row of recordsToUpsert) { this.db - .prepare( - ` - INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) - VALUES (?, ?, ?, ?) - ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at - ` - ) + .prepare(` +INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) +VALUES (?, ?, ?, ?) +ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`) .run(...row); } }); - updateTransaction(); } @@ -135,10 +145,9 @@ export class SQLiteRecordManager implements RecordManagerInterface { // Prepare the placeholders and the query const placeholders = keys.map(() => `?`).join(", "); const sql = ` - SELECT key - FROM "${this.tableName}" - WHERE namespace = ? AND key IN (${placeholders}) - `; +SELECT key +FROM "${this.tableName}" +WHERE namespace = ? AND key IN (${placeholders})`; // Initialize an array to fill with the existence checks const existsArray = new Array(keys.length).fill(false); @@ -156,7 +165,7 @@ export class SQLiteRecordManager implements RecordManagerInterface { }); return existsArray; } catch (error) { - console.error("Error checking existence of keys:", error); + console.error("Error checking existence of keys"); throw error; // Allow the caller to handle the error } } @@ -196,7 +205,7 @@ export class SQLiteRecordManager implements RecordManagerInterface { const result = this.db.prepare(query).all(...values) as { key: string }[]; return result.map((row) => row.key); } catch (error) { - console.error("Error listing keys:", error); + console.error("Error listing keys."); throw error; // Re-throw the error to be handled by the caller } } @@ -208,13 +217,13 @@ export class SQLiteRecordManager implements RecordManagerInterface { const placeholders = keys.map(() => "?").join(", "); const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`; - const values: (string | number)[] = [this.namespace, ...keys]; + const values = [this.namespace, ...keys].map((v) => typeof v !== "string" ? `${v}` : v); // Directly using try/catch with async/await for cleaner flow try { this.db.prepare(query).run(...values); } catch (error) { - console.error("Error deleting keys:", error); + console.error("Error deleting keys"); throw error; // Re-throw the error to be handled by the caller } } diff --git a/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts b/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts index ea3cdfdaeb6..251dc08cbcd 100644 --- a/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts +++ b/libs/langchain-community/src/indexes/tests/sqlite.int.test.ts @@ -6,10 +6,10 @@ describe("SQLiteRecordManager", () => { let recordManager: SQLiteRecordManager; beforeAll(async () => { - const filepathOrConnectionString = ":memory:"; + const localPath = ":memory:"; recordManager = new SQLiteRecordManager("test", { tableName, - filepathOrConnectionString, + localPath, }); await recordManager.createSchema(); }); @@ -22,6 +22,7 @@ describe("SQLiteRecordManager", () => { afterAll(() => { recordManager.db.close(); }); + test("Test upsertion", async () => { const keys = ["a", "b", "c"]; await recordManager.update(keys); From 06e5089f3a0eb069866b554d049cb1a91a940e82 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 9 Feb 2024 15:02:56 -0800 Subject: [PATCH 6/7] update peer deps, add to config file --- libs/langchain-community/.gitignore | 4 ++++ libs/langchain-community/langchain.config.js | 2 ++ libs/langchain-community/package.json | 17 +++++++++++++++++ libs/langchain-community/src/indexes/sqlite.ts | 2 +- yarn.lock | 3 +++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore index ced5a2948a4..416c6d5382c 100644 --- a/libs/langchain-community/.gitignore +++ b/libs/langchain-community/.gitignore @@ -666,6 +666,10 @@ indexes/memory.cjs indexes/memory.js indexes/memory.d.ts indexes/memory.d.cts +indexes/sqlite.cjs +indexes/sqlite.js +indexes/sqlite.d.ts +indexes/sqlite.d.cts util/convex.cjs util/convex.js util/convex.d.ts diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index c351659761b..e0d347a6460 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -209,6 +209,7 @@ export const config = { "indexes/base": "indexes/base", "indexes/postgres": "indexes/postgres", "indexes/memory": "indexes/memory", + "indexes/sqlite": "indexes/sqlite", // utils "util/convex": "utils/convex", "utils/event_source_parse": "utils/event_source_parse", @@ -334,6 +335,7 @@ export const config = { "util/convex", // indexes "indexes/postgres", + "indexes/sqlite", ], packageSuffix: "community", tsConfigPath: resolve("./tsconfig.json"), diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 51ca5798d29..920076c332f 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -212,6 +212,7 @@ "@xata.io/client": "^0.28.0", "@xenova/transformers": "^2.5.4", "@zilliz/milvus2-sdk-node": ">=2.2.7", + "better-sqlite3": "^9.4.0", "cassandra-driver": "^4.7.2", "chromadb": "*", "closevector-common": "0.1.3", @@ -383,6 +384,9 @@ "@zilliz/milvus2-sdk-node": { "optional": true }, + "better-sqlite3": { + "optional": true + }, "cassandra-driver": { "optional": true }, @@ -2002,6 +2006,15 @@ "import": "./indexes/memory.js", "require": "./indexes/memory.cjs" }, + "./indexes/sqlite": { + "types": { + "import": "./indexes/sqlite.d.ts", + "require": "./indexes/sqlite.d.cts", + "default": "./indexes/sqlite.d.ts" + }, + "import": "./indexes/sqlite.js", + "require": "./indexes/sqlite.cjs" + }, "./util/convex": { "types": { "import": "./util/convex.d.ts", @@ -2701,6 +2714,10 @@ "indexes/memory.js", "indexes/memory.d.ts", "indexes/memory.d.cts", + "indexes/sqlite.cjs", + "indexes/sqlite.js", + "indexes/sqlite.d.ts", + "indexes/sqlite.d.cts", "util/convex.cjs", "util/convex.js", "util/convex.d.ts", diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts index 1a26ab0c73c..bf3bdf86217 100644 --- a/libs/langchain-community/src/indexes/sqlite.ts +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -89,7 +89,7 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`); const { epoch } = statement.get() as TimeRow; return Number(epoch); } catch (error) { - console.error('Error getting time in SQLiteRecordManager:', error); + console.error('Error getting time in SQLiteRecordManager:'); throw error; } } diff --git a/yarn.lock b/yarn.lock index 78c76bc4a81..03c0916190d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9033,6 +9033,7 @@ __metadata: "@xata.io/client": ^0.28.0 "@xenova/transformers": ^2.5.4 "@zilliz/milvus2-sdk-node": ">=2.2.7" + better-sqlite3: ^9.4.0 cassandra-driver: ^4.7.2 chromadb: "*" closevector-common: 0.1.3 @@ -9159,6 +9160,8 @@ __metadata: optional: true "@zilliz/milvus2-sdk-node": optional: true + better-sqlite3: + optional: true cassandra-driver: optional: true chromadb: From 50e6461fb55902e3e036519885af12cde0b3aee5 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 9 Feb 2024 15:03:40 -0800 Subject: [PATCH 7/7] format --- libs/langchain-community/src/indexes/sqlite.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libs/langchain-community/src/indexes/sqlite.ts b/libs/langchain-community/src/indexes/sqlite.ts index bf3bdf86217..78d7f8faf34 100644 --- a/libs/langchain-community/src/indexes/sqlite.ts +++ b/libs/langchain-community/src/indexes/sqlite.ts @@ -89,7 +89,7 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`); const { epoch } = statement.get() as TimeRow; return Number(epoch); } catch (error) { - console.error('Error getting time in SQLiteRecordManager:'); + console.error("Error getting time in SQLiteRecordManager:"); throw error; } } @@ -127,10 +127,12 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`); const updateTransaction = this.db.transaction(() => { for (const row of recordsToUpsert) { this.db - .prepare(` + .prepare( + ` INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES (?, ?, ?, ?) -ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`) +ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at` + ) .run(...row); } }); @@ -217,7 +219,9 @@ WHERE namespace = ? AND key IN (${placeholders})`; const placeholders = keys.map(() => "?").join(", "); const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`; - const values = [this.namespace, ...keys].map((v) => typeof v !== "string" ? `${v}` : v); + const values = [this.namespace, ...keys].map((v) => + typeof v !== "string" ? `${v}` : v + ); // Directly using try/catch with async/await for cleaner flow try {