From 6877d2c94d5587198eafc48bd97aaa117d025861 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Tue, 28 Jan 2025 11:57:34 +0200 Subject: [PATCH 1/8] Introduced executeRaw to DBAdapter and WASQLiteConnection. --- .../src/client/AbstractPowerSyncDatabase.ts | 5 + packages/common/src/db/DBAdapter.ts | 5 +- .../sqlite/PowerSyncSQLitePreparedQuery.ts | 4 +- .../drizzle-driver/tests/sqlite/query.test.ts | 5 +- .../tests/sqlite/relationship.test.ts | 122 ++++++++++++++++++ .../src/db/OPSQLiteConnection.ts | 5 + .../src/db/OPSqliteAdapter.ts | 24 ++-- .../RNQSDBAdapter.ts | 8 ++ .../db/adapters/AsyncDatabaseConnection.ts | 1 + .../db/adapters/LockedAsyncDatabaseAdapter.ts | 20 ++- packages/web/src/db/adapters/SSRDBAdapter.ts | 4 + .../WorkerWrappedAsyncDatabaseConnection.ts | 4 + .../adapters/wa-sqlite/WASQLiteConnection.ts | 66 +++++++--- .../web/src/worker/db/WASQLiteDB.worker.ts | 1 + 14 files changed, 230 insertions(+), 44 deletions(-) create mode 100644 packages/drizzle-driver/tests/sqlite/relationship.test.ts diff --git a/packages/common/src/client/AbstractPowerSyncDatabase.ts b/packages/common/src/client/AbstractPowerSyncDatabase.ts index 8dc50477a..5396172ef 100644 --- a/packages/common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/common/src/client/AbstractPowerSyncDatabase.ts @@ -603,6 +603,11 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver Promise; + /** Execute a single write statement and return raw results. */ + executeRaw: (query: string, params?: any[] | undefined) => Promise; } export interface Transaction extends LockContext { @@ -95,6 +97,7 @@ export interface DBLockOptions { export interface DBAdapter extends BaseObserverInterface, DBGetUtils { close: () => void; execute: (query: string, params?: any[]) => Promise; + executeRaw: (query: string, params?: any[]) => Promise; executeBatch: (query: string, params?: any[][]) => Promise; name: string; readLock: (fn: (tx: LockContext) => Promise, options?: DBLockOptions) => Promise; @@ -103,7 +106,7 @@ export interface DBAdapter extends BaseObserverInterface, DBG writeTransaction: (fn: (tx: Transaction) => Promise, options?: DBLockOptions) => Promise; /** * This method refreshes the schema information across all connections. This is for advanced use cases, and should generally not be needed. - */ + */ refreshSchema: () => Promise; } diff --git a/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts b/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts index 5229caef0..51ba500bc 100644 --- a/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts +++ b/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts @@ -90,8 +90,8 @@ export class PowerSyncSQLitePreparedQuery< async values(placeholderValues?: Record): Promise { const params = fillPlaceholders(this.query.params, placeholderValues ?? {}); this.logger.logQuery(this.query.sql, params); - const rs = await this.db.execute(this.query.sql, params); - return rs.rows?._array ?? []; + + return await this.db.executeRaw(this.query.sql, params); } isResponseInArrayMode(): boolean { diff --git a/packages/drizzle-driver/tests/sqlite/query.test.ts b/packages/drizzle-driver/tests/sqlite/query.test.ts index 30e565911..63c48472d 100644 --- a/packages/drizzle-driver/tests/sqlite/query.test.ts +++ b/packages/drizzle-driver/tests/sqlite/query.test.ts @@ -56,9 +56,10 @@ describe('PowerSyncSQLitePreparedQuery', () => { const preparedQuery = new PowerSyncSQLitePreparedQuery(powerSyncDb, query, loggerMock, undefined, 'all', false); const values = await preparedQuery.values(); + expect(values).toEqual([ - { id: '1', name: 'Alice' }, - { id: '2', name: 'Bob' } + ['1', 'Alice'], + ['2', 'Bob'] ]); }); }); diff --git a/packages/drizzle-driver/tests/sqlite/relationship.test.ts b/packages/drizzle-driver/tests/sqlite/relationship.test.ts new file mode 100644 index 000000000..5dc5291b7 --- /dev/null +++ b/packages/drizzle-driver/tests/sqlite/relationship.test.ts @@ -0,0 +1,122 @@ +import { AbstractPowerSyncDatabase, column, Schema, Table } from '@powersync/common'; +import { PowerSyncDatabase } from '@powersync/web'; +import { eq, relations } from 'drizzle-orm'; +import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import * as SUT from '../../src/sqlite/PowerSyncSQLiteDatabase'; + +const users = new Table({ + name: column.text +}); + +const posts = new Table({ + content: column.text, + title: column.text, + user_id: column.text +}); + +const drizzleUsers = sqliteTable('users', { + id: text('id').primaryKey().notNull(), + name: text('name').notNull() +}); + +const drizzlePosts = sqliteTable('posts', { + id: text('id').primaryKey().notNull(), + content: text('content').notNull(), + title: text('title').notNull(), + user_id: text('user_id') + .notNull() + .references(() => drizzleUsers.id) +}); + +// Define relationships +const usersRelations = relations(drizzleUsers, ({ one, many }) => ({ + posts: many(drizzlePosts) // One user has many posts +})); + +const postsRelations = relations(drizzlePosts, ({ one }) => ({ + user: one(drizzleUsers, { + fields: [drizzlePosts.user_id], // Foreign key in posts + references: [drizzleUsers.id] // Primary key in users + }) +})); + +const PsSchema = new Schema({ users, posts }); +// const DrizzleSchema = { users: drizzleUsers, posts: drizzlePosts }; +const DrizzleSchema = { users: drizzleUsers, posts: drizzlePosts, usersRelations, postsRelations }; + +describe('Relationship tests', () => { + let powerSyncDb: AbstractPowerSyncDatabase; + let db: SUT.PowerSyncSQLiteDatabase; + + beforeEach(async () => { + powerSyncDb = new PowerSyncDatabase({ + database: { + dbFilename: 'test.db' + }, + schema: PsSchema + }); + db = SUT.wrapPowerSyncWithDrizzle(powerSyncDb, { schema: DrizzleSchema, logger: { logQuery: () => {} } }); + + await powerSyncDb.init(); + + await db.insert(drizzleUsers).values({ id: '1', name: 'Alice' }); + await db.insert(drizzlePosts).values({ id: '33', content: 'Post content', title: 'Post title', user_id: '1' }); + }); + + afterEach(async () => { + await powerSyncDb?.disconnectAndClear(); + }); + + it('should retrieve a user with posts', async () => { + const result = await db.query.users.findMany({ with: { posts: true } }); + + expect(result).toEqual([ + { id: '1', name: 'Alice', posts: [{ id: '33', content: 'Post content', title: 'Post title', user_id: '1' }] } + ]); + }); + + it('should retrieve a post with its user', async () => { + const result = await db.query.posts.findMany({ with: { user: true } }); + + expect(result).toEqual([ + { + id: '33', + content: 'Post content', + title: 'Post title', + user_id: '1', + user: { id: '1', name: 'Alice' } + } + ]); + }); + + it('should return a user and posts using leftJoin', async () => { + const result = await db + .select() + .from(drizzleUsers) + .leftJoin(drizzlePosts, eq(drizzleUsers.id, drizzlePosts.user_id)); + + expect(result[0].users).toEqual({ id: '1', name: 'Alice' }); + expect(result[0].posts).toEqual({ id: '33', content: 'Post content', title: 'Post title', user_id: '1' }); + }); + + it('should return a user and posts using rightJoin', async () => { + const result = await db + .select() + .from(drizzleUsers) + .rightJoin(drizzlePosts, eq(drizzleUsers.id, drizzlePosts.user_id)); + + expect(result[0].users).toEqual({ id: '1', name: 'Alice' }); + expect(result[0].posts).toEqual({ id: '33', content: 'Post content', title: 'Post title', user_id: '1' }); + }); + + it('should return a user and posts using fullJoin', async () => { + const result = await db + .select() + .from(drizzleUsers) + .fullJoin(drizzlePosts, eq(drizzleUsers.id, drizzlePosts.user_id)); + + expect(result[0].users).toEqual({ id: '1', name: 'Alice' }); + expect(result[0].posts).toEqual({ id: '33', content: 'Post content', title: 'Post title', user_id: '1' }); + }); +}); diff --git a/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts b/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts index bb95dd16e..498320348 100644 --- a/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts +++ b/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts @@ -97,6 +97,11 @@ export class OPSQLiteConnection extends BaseObserver { }; } + async executeRaw(query: string, params?: any[]): Promise { + // TODO CL: Test this + return await this.DB.executeRaw(query, params); + } + async executeBatch(query: string, params: any[][] = []): Promise { const tuple: SQLBatchTuple[] = [[query, params[0]]]; params.slice(1).forEach((p) => tuple.push([query, p])); diff --git a/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts b/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts index 6880963cc..bc10520a4 100644 --- a/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts +++ b/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts @@ -1,18 +1,5 @@ -import { - BaseObserver, - DBAdapter, - DBAdapterListener, - DBLockOptions, - QueryResult, - Transaction -} from '@powersync/common'; -import { - ANDROID_DATABASE_PATH, - getDylibPath, - IOS_LIBRARY_PATH, - open, - type DB -} from '@op-engineering/op-sqlite'; +import { BaseObserver, DBAdapter, DBAdapterListener, DBLockOptions, QueryResult, Transaction } from '@powersync/common'; +import { ANDROID_DATABASE_PATH, getDylibPath, IOS_LIBRARY_PATH, open, type DB } from '@op-engineering/op-sqlite'; import Lock from 'async-lock'; import { OPSQLiteConnection } from './OPSQLiteConnection'; import { Platform } from 'react-native'; @@ -247,6 +234,10 @@ export class OPSQLiteDBAdapter extends BaseObserver implement return this.writeLock((ctx) => ctx.execute(query, params)); } + executeRaw(query: string, params?: any[]) { + return this.writeLock((ctx) => ctx.executeRaw(query, params)); + } + async executeBatch(query: string, params: any[][] = []): Promise { return this.writeLock((ctx) => ctx.executeBatch(query, params)); } @@ -274,6 +265,7 @@ export class OPSQLiteDBAdapter extends BaseObserver implement await connection.execute('BEGIN'); const result = await fn({ execute: (query, params) => connection.execute(query, params), + executeRaw: (query, params) => connection.executeRaw(query, params), get: (query, params) => connection.get(query, params), getAll: (query, params) => connection.getAll(query, params), getOptional: (query, params) => connection.getOptional(query, params), @@ -292,7 +284,7 @@ export class OPSQLiteDBAdapter extends BaseObserver implement await this.initialized; await this.writeConnection!.refreshSchema(); - if(this.readConnections) { + if (this.readConnections) { for (let readConnection of this.readConnections) { await readConnection.connection.refreshSchema(); } diff --git a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts index 2b82d74f2..81339bb42 100644 --- a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts +++ b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts @@ -62,6 +62,10 @@ export class RNQSDBAdapter extends BaseObserver implements DB return this.baseDB.execute(query, params); } + executeRaw(query: string, params?: any[]): Promise { + throw new Error('Method not implemented.'); + } + async executeBatch(query: string, params: any[][] = []): Promise { const commands: any[] = []; @@ -85,6 +89,10 @@ export class RNQSDBAdapter extends BaseObserver implements DB return this.baseDB.readLock((ctx) => ctx.execute(sql, params)); } + private readOnlyExecuteRaw(sql: string, params?: any[]) { + return this.baseDB.readLock((ctx) => ctx.execute(sql, params)); + } + /** * Adds DB get utils to lock contexts and transaction contexts * @param tx diff --git a/packages/web/src/db/adapters/AsyncDatabaseConnection.ts b/packages/web/src/db/adapters/AsyncDatabaseConnection.ts index e581e984d..1bd832142 100644 --- a/packages/web/src/db/adapters/AsyncDatabaseConnection.ts +++ b/packages/web/src/db/adapters/AsyncDatabaseConnection.ts @@ -21,6 +21,7 @@ export interface AsyncDatabaseConnection; close(): Promise; execute(sql: string, params?: any[]): Promise; + executeRaw(sql: string, params?: any[]): Promise; executeBatch(sql: string, params?: any[]): Promise; registerOnTableChange(callback: OnTableChangeCallback): Promise<() => void>; getConfig(): Promise; diff --git a/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts b/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts index fe68b44eb..e8a138bd4 100644 --- a/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts +++ b/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts @@ -138,6 +138,10 @@ export class LockedAsyncDatabaseAdapter return this.writeLock((ctx) => ctx.execute(query, params)); } + async executeRaw(query: string, params?: any[] | undefined): Promise { + return this.writeLock((ctx) => ctx.executeRaw(query, params)); + } + async executeBatch(query: string, params?: any[][]): Promise { return this.writeLock((ctx) => this._executeBatch(query, params)); } @@ -169,12 +173,16 @@ export class LockedAsyncDatabaseAdapter async readLock(fn: (tx: LockContext) => Promise, options?: DBLockOptions | undefined): Promise { await this.waitForInitialized(); - return this.acquireLock(async () => fn(this.generateDBHelpers({ execute: this._execute }))); + return this.acquireLock(async () => + fn(this.generateDBHelpers({ execute: this._execute, executeRaw: this._executeRaw })) + ); } async writeLock(fn: (tx: LockContext) => Promise, options?: DBLockOptions | undefined): Promise { await this.waitForInitialized(); - return this.acquireLock(async () => fn(this.generateDBHelpers({ execute: this._execute }))); + return this.acquireLock(async () => + fn(this.generateDBHelpers({ execute: this._execute, executeRaw: this._executeRaw })) + ); } protected acquireLock(callback: () => Promise): Promise { @@ -283,6 +291,14 @@ export class LockedAsyncDatabaseAdapter }; }; + /** + * Wraps the worker executeRaw function, awaiting for it to be available + */ + private _executeRaw = async (sql: string, bindings?: any[]): Promise => { + await this.waitForInitialized(); + return await this.baseDB.executeRaw(sql, bindings); + }; + /** * Wraps the worker executeBatch function, awaiting for it to be available */ diff --git a/packages/web/src/db/adapters/SSRDBAdapter.ts b/packages/web/src/db/adapters/SSRDBAdapter.ts index b808d7477..f460da412 100644 --- a/packages/web/src/db/adapters/SSRDBAdapter.ts +++ b/packages/web/src/db/adapters/SSRDBAdapter.ts @@ -53,6 +53,10 @@ export class SSRDBAdapter extends BaseObserver implements DBA return this.writeMutex.runExclusive(async () => MOCK_QUERY_RESPONSE); } + async executeRaw(query: string, params?: any[]): Promise { + return this.writeMutex.runExclusive(async () => []); + } + async executeBatch(query: string, params?: any[][]): Promise { return this.writeMutex.runExclusive(async () => MOCK_QUERY_RESPONSE); } diff --git a/packages/web/src/db/adapters/WorkerWrappedAsyncDatabaseConnection.ts b/packages/web/src/db/adapters/WorkerWrappedAsyncDatabaseConnection.ts index bb26c59da..e4101f15c 100644 --- a/packages/web/src/db/adapters/WorkerWrappedAsyncDatabaseConnection.ts +++ b/packages/web/src/db/adapters/WorkerWrappedAsyncDatabaseConnection.ts @@ -67,6 +67,10 @@ export class WorkerWrappedAsyncDatabaseConnection { + return this.baseConnection.executeRaw(sql, params); + } + executeBatch(sql: string, params?: any[]): Promise { return this.baseConnection.executeBatch(sql, params); } diff --git a/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts b/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts index d73800d96..a7e3f90ff 100644 --- a/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts +++ b/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts @@ -333,6 +333,12 @@ export class WASqliteConnection }); } + async executeRaw(sql: string | TemplateStringsArray, bindings?: any[]): Promise { + return this.acquireExecuteLock(async () => { + return this.executeSingleStatementRaw(sql, bindings); + }); + } + async close() { this.broadcastChannel?.close(); await this.sqliteAPI.close(this.dbP); @@ -359,6 +365,44 @@ export class WASqliteConnection sql: string | TemplateStringsArray, bindings?: any[] ): Promise { + const results = await this._execute(sql, bindings); + + const rows: Record[] = []; + for (const resultSet of results) { + for (const row of resultSet.rows) { + const outRow: Record = {}; + resultSet.columns.forEach((key, index) => { + outRow[key] = row[index]; + }); + rows.push(outRow); + } + } + + const result = { + insertId: this.sqliteAPI.last_insert_id(this.dbP), + rowsAffected: this.sqliteAPI.changes(this.dbP), + rows: { + _array: rows, + length: rows.length + } + }; + + return result; + } + + /** + * This executes a single statement using SQLite3 and returns the results as an array of arrays. + */ + protected async executeSingleStatementRaw(sql: string | TemplateStringsArray, bindings?: any[]): Promise { + const results = await this._execute(sql, bindings); + + return results.flatMap((resultset) => resultset.rows.map((row) => resultset.columns.map((_, index) => row[index]))); + } + + private async _execute( + sql: string | TemplateStringsArray, + bindings?: any[] + ): Promise<{ columns: string[]; rows: SQLiteCompatibleType[][] }[]> { const results = []; for await (const stmt of this.sqliteAPI.statements(this.dbP, sql as string)) { let columns; @@ -394,26 +438,6 @@ export class WASqliteConnection } } - const rows: Record[] = []; - for (const resultSet of results) { - for (const row of resultSet.rows) { - const outRow: Record = {}; - resultSet.columns.forEach((key, index) => { - outRow[key] = row[index]; - }); - rows.push(outRow); - } - } - - const result = { - insertId: this.sqliteAPI.last_insert_id(this.dbP), - rowsAffected: this.sqliteAPI.changes(this.dbP), - rows: { - _array: rows, - length: rows.length - } - }; - - return result; + return results; } } diff --git a/packages/web/src/worker/db/WASQLiteDB.worker.ts b/packages/web/src/worker/db/WASQLiteDB.worker.ts index 5e2e169a8..7e28b024d 100644 --- a/packages/web/src/worker/db/WASQLiteDB.worker.ts +++ b/packages/web/src/worker/db/WASQLiteDB.worker.ts @@ -30,6 +30,7 @@ const openWorkerConnection = async (options: ResolvedWASQLiteOpenFactoryOptions) getConfig: Comlink.proxy(() => connection.getConfig()), close: Comlink.proxy(() => connection.close()), execute: Comlink.proxy(async (sql: string, params?: any[]) => connection.execute(sql, params)), + executeRaw: Comlink.proxy(async (sql: string, params?: any[]) => connection.executeRaw(sql, params)), executeBatch: Comlink.proxy(async (sql: string, params?: any[]) => connection.executeBatch(sql, params)), registerOnTableChange: Comlink.proxy(async (callback) => { // Proxy the callback remove function From 87ab8479a766a1563b6f8f05276d2eb84af60876 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Tue, 28 Jan 2025 14:51:50 +0200 Subject: [PATCH 2/8] Cleanup. --- packages/drizzle-driver/tests/sqlite/relationship.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/drizzle-driver/tests/sqlite/relationship.test.ts b/packages/drizzle-driver/tests/sqlite/relationship.test.ts index 5dc5291b7..ef178a9a5 100644 --- a/packages/drizzle-driver/tests/sqlite/relationship.test.ts +++ b/packages/drizzle-driver/tests/sqlite/relationship.test.ts @@ -29,20 +29,18 @@ const drizzlePosts = sqliteTable('posts', { .references(() => drizzleUsers.id) }); -// Define relationships const usersRelations = relations(drizzleUsers, ({ one, many }) => ({ - posts: many(drizzlePosts) // One user has many posts + posts: many(drizzlePosts) })); const postsRelations = relations(drizzlePosts, ({ one }) => ({ user: one(drizzleUsers, { - fields: [drizzlePosts.user_id], // Foreign key in posts - references: [drizzleUsers.id] // Primary key in users + fields: [drizzlePosts.user_id], + references: [drizzleUsers.id] }) })); const PsSchema = new Schema({ users, posts }); -// const DrizzleSchema = { users: drizzleUsers, posts: drizzlePosts }; const DrizzleSchema = { users: drizzleUsers, posts: drizzlePosts, usersRelations, postsRelations }; describe('Relationship tests', () => { From 20a758e50693e9dde8737817afbade92e37d889e Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 19 Mar 2025 13:26:53 +0200 Subject: [PATCH 3/8] Added executeRaw to RNQS db adapter, using .execute as fallback instead of depending on a native implementation. --- .../src/db/OPSQLiteConnection.ts | 1 - .../RNQSDBAdapter.ts | 54 +++++++++++++++---- .../db/adapters/LockedAsyncDatabaseAdapter.ts | 12 +++-- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts b/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts index 498320348..5704a48b1 100644 --- a/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts +++ b/packages/powersync-op-sqlite/src/db/OPSQLiteConnection.ts @@ -98,7 +98,6 @@ export class OPSQLiteConnection extends BaseObserver { } async executeRaw(query: string, params?: any[]): Promise { - // TODO CL: Test this return await this.DB.executeRaw(query, params); } diff --git a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts index 81339bb42..15e56b94a 100644 --- a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts +++ b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts @@ -8,7 +8,11 @@ import { DBGetUtils, QueryResult } from '@powersync/common'; -import type { QuickSQLiteConnection } from '@journeyapps/react-native-quick-sqlite'; +import type { + QuickSQLiteConnection, + LockContext as RNQSLockContext, + TransactionContext as RNQSTransactionContext +} from '@journeyapps/react-native-quick-sqlite'; /** * Adapter for React Native Quick SQLite @@ -43,27 +47,37 @@ export class RNQSDBAdapter extends BaseObserver implements DB } readLock(fn: (tx: PowerSyncLockContext) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.readLock((dbTx) => fn(this.generateDBHelpers(dbTx)), options); + return this.baseDB.readLock((dbTx) => fn(this.generateDBHelpers(this.generateLockContext(dbTx))), options); } readTransaction(fn: (tx: PowerSyncTransaction) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.readTransaction((dbTx) => fn(this.generateDBHelpers(dbTx)), options); + return this.baseDB.readTransaction( + (dbTx) => fn(this.generateDBHelpers(this.generateTransactionContext(dbTx))), + options + ); } writeLock(fn: (tx: PowerSyncLockContext) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.writeLock((dbTx) => fn(this.generateDBHelpers(dbTx)), options); + return this.baseDB.writeLock((dbTx) => fn(this.generateDBHelpers(this.generateLockContext(dbTx))), options); } writeTransaction(fn: (tx: PowerSyncTransaction) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.writeTransaction((dbTx) => fn(this.generateDBHelpers(dbTx)), options); + return this.baseDB.writeTransaction( + (dbTx) => fn(this.generateDBHelpers(this.generateTransactionContext(dbTx))), + options + ); } execute(query: string, params?: any[]) { return this.baseDB.execute(query, params); } - executeRaw(query: string, params?: any[]): Promise { - throw new Error('Method not implemented.'); + /** + * 'executeRaw' is not implemented in RNQS, this falls back to 'execute'. + */ + async executeRaw(query: string, params?: any[]): Promise { + const result = await this.baseDB.execute(query, params); + return result.rows?._array ?? []; } async executeBatch(query: string, params: any[][] = []): Promise { @@ -79,6 +93,28 @@ export class RNQSDBAdapter extends BaseObserver implements DB }; } + generateLockContext(ctx: RNQSLockContext) { + return { + ...ctx, + // 'executeRaw' is not implemented in RNQS, this falls back to 'execute'. + executeRaw: async (sql: string, params?: any[]) => { + const result = await ctx.execute(sql, params); + return result.rows?._array ?? []; + } + }; + } + + generateTransactionContext(ctx: RNQSTransactionContext) { + return { + ...ctx, + // 'executeRaw' is not implemented in RNQS, this falls back to 'execute'. + executeRaw: async (sql: string, params?: any[]) => { + const result = await ctx.execute(sql, params); + return result.rows?._array ?? []; + } + }; + } + /** * This provides a top-level read only execute method which is executed inside a read-lock. * This is necessary since the high level `execute` method uses a write-lock under @@ -89,10 +125,6 @@ export class RNQSDBAdapter extends BaseObserver implements DB return this.baseDB.readLock((ctx) => ctx.execute(sql, params)); } - private readOnlyExecuteRaw(sql: string, params?: any[]) { - return this.baseDB.readLock((ctx) => ctx.execute(sql, params)); - } - /** * Adds DB get utils to lock contexts and transaction contexts * @param tx diff --git a/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts b/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts index e8a138bd4..75f17e5fd 100644 --- a/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts +++ b/packages/web/src/db/adapters/LockedAsyncDatabaseAdapter.ts @@ -69,7 +69,8 @@ export class LockedAsyncDatabaseAdapter } this.dbGetHelpers = this.generateDBHelpers({ - execute: (query, params) => this.acquireLock(() => this._execute(query, params)) + execute: (query, params) => this.acquireLock(() => this._execute(query, params)), + executeRaw: (query, params) => this.acquireLock(() => this._executeRaw(query, params)) }); this.initPromise = this._init(); } @@ -197,9 +198,12 @@ export class LockedAsyncDatabaseAdapter return this.writeLock(this.wrapTransaction(fn)); } - private generateDBHelpers Promise }>( - tx: T - ): T & DBGetUtils { + private generateDBHelpers< + T extends { + execute: (sql: string, params?: any[]) => Promise; + executeRaw: (sql: string, params?: any[]) => Promise; + } + >(tx: T): T & DBGetUtils { return { ...tx, /** From cb3afd3e9793800cef418cad7d8a00a5fa0c22fa Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 19 Mar 2025 15:32:09 +0200 Subject: [PATCH 4/8] Added executeRaw to `@powersync/node`. --- packages/node/package.json | 2 + packages/node/src/db/AsyncDatabase.ts | 1 + .../node/src/db/BetterSQLite3DBAdapter.ts | 9 +- packages/node/src/db/RemoteConnection.ts | 4 + packages/node/src/db/SqliteWorker.ts | 11 +++ packages/node/tests/DrizzleNode.test.ts | 99 +++++++++++++++++++ packages/node/tests/utils.ts | 8 +- pnpm-lock.yaml | 6 ++ 8 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 packages/node/tests/DrizzleNode.test.ts diff --git a/packages/node/package.json b/packages/node/package.json index 1cba46988..90cbce04c 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -45,6 +45,8 @@ }, "devDependencies": { "@types/async-lock": "^1.4.0", + "drizzle-orm": "^0.35.2", + "@powersync/drizzle-driver": "workspace:*", "typescript": "^5.5.3", "vitest": "^3.0.5" }, diff --git a/packages/node/src/db/AsyncDatabase.ts b/packages/node/src/db/AsyncDatabase.ts index 61abae7ec..20816ab89 100644 --- a/packages/node/src/db/AsyncDatabase.ts +++ b/packages/node/src/db/AsyncDatabase.ts @@ -13,6 +13,7 @@ export interface AsyncDatabaseOpener { export interface AsyncDatabase { execute: (query: string, params: any[]) => Promise; + executeRaw: (query: string, params: any[]) => Promise; executeBatch: (query: string, params: any[][]) => Promise; close: () => Promise; // Collect table updates made since the last call to collectCommittedUpdates. diff --git a/packages/node/src/db/BetterSQLite3DBAdapter.ts b/packages/node/src/db/BetterSQLite3DBAdapter.ts index 7b09557b3..bf7c08d70 100644 --- a/packages/node/src/db/BetterSQLite3DBAdapter.ts +++ b/packages/node/src/db/BetterSQLite3DBAdapter.ts @@ -53,7 +53,9 @@ export class BetterSQLite3DBAdapter extends BaseObserver impl } const openWorker = async (isWriter: boolean) => { - const worker = new Worker(new URL('./SqliteWorker.js', import.meta.url), {name: isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`}); + const worker = new Worker(new URL('./SqliteWorker.js', import.meta.url), { + name: isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}` + }); const listeners = new WeakMap void>(); const comlink = Comlink.wrap({ @@ -216,6 +218,7 @@ export class BetterSQLite3DBAdapter extends BaseObserver impl await connection.execute('BEGIN'); const result = await fn({ execute: (query, params) => connection.execute(query, params), + executeRaw: (query, params) => connection.executeRaw(query, params), executeBatch: (query, params) => connection.executeBatch(query, params), get: (query, params) => connection.get(query, params), getAll: (query, params) => connection.getAll(query, params), @@ -252,6 +255,10 @@ export class BetterSQLite3DBAdapter extends BaseObserver impl return this.writeLock((ctx) => ctx.execute(query, params)); } + executeRaw(query: string, params?: any[] | undefined): Promise { + return this.writeLock((ctx) => ctx.executeRaw(query, params)); + } + executeBatch(query: string, params?: any[][]): Promise { return this.writeTransaction((ctx) => ctx.executeBatch(query, params)); } diff --git a/packages/node/src/db/RemoteConnection.ts b/packages/node/src/db/RemoteConnection.ts index 0a090f523..aa61d39c8 100644 --- a/packages/node/src/db/RemoteConnection.ts +++ b/packages/node/src/db/RemoteConnection.ts @@ -29,6 +29,10 @@ export class RemoteConnection implements LockContext { return RemoteConnection.wrapQueryResult(result); } + async executeRaw(query: string, params?: any[] | undefined): Promise { + return await this.database.executeRaw(query, params ?? []); + } + async getAll(sql: string, parameters?: any[]): Promise { const res = await this.execute(sql, parameters); return res.rows?._array ?? []; diff --git a/packages/node/src/db/SqliteWorker.ts b/packages/node/src/db/SqliteWorker.ts index 66ac01436..42e0b1a8b 100644 --- a/packages/node/src/db/SqliteWorker.ts +++ b/packages/node/src/db/SqliteWorker.ts @@ -65,6 +65,17 @@ class BlockingAsyncDatabase implements AsyncDatabase { } } + async executeRaw(query: string, params: any[]) { + const stmt = this.db.prepare(query); + + if (stmt.reader) { + return stmt.raw().all(params); + } else { + stmt.raw().run(params); + return []; + } + } + async executeBatch(query: string, params: any[][]) { params = params ?? []; diff --git a/packages/node/tests/DrizzleNode.test.ts b/packages/node/tests/DrizzleNode.test.ts new file mode 100644 index 000000000..9598aceb7 --- /dev/null +++ b/packages/node/tests/DrizzleNode.test.ts @@ -0,0 +1,99 @@ +import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { eq, relations } from 'drizzle-orm'; + +import { databaseTest } from './utils'; +import { wrapPowerSyncWithDrizzle } from '@powersync/drizzle-driver'; +import { PowerSyncDatabase } from '../lib'; +import { expect } from 'vitest'; + +export const drizzleLists = sqliteTable('lists', { + id: text('id'), + name: text('name') +}); + +export const drizzleTodos = sqliteTable('todos', { + id: text('id'), + content: text('content'), + list_id: text('list_id') +}); + +export const listsRelations = relations(drizzleLists, ({ one, many }) => ({ + todos: many(drizzleTodos) +})); + +export const todosRelations = relations(drizzleTodos, ({ one, many }) => ({ + list: one(drizzleLists, { + fields: [drizzleTodos.list_id], + references: [drizzleLists.id] + }) +})); + +export const drizzleSchema = { + lists: drizzleLists, + todos: drizzleTodos, + listsRelations, + todosRelations +}; + +const setupDrizzle = async (database: PowerSyncDatabase) => { + const db = wrapPowerSyncWithDrizzle(database, { + schema: drizzleSchema + }); + + await db.insert(drizzleLists).values({ id: '1', name: 'list 1' }); + await db.insert(drizzleTodos).values({ id: '33', content: 'Post content', list_id: '1' }); + return db; +}; + +databaseTest('should retrieve a list with todos', async ({ database }) => { + const db = await setupDrizzle(database); + + const result = await db.query.lists.findMany({ with: { todos: true } }); + + expect(result).toEqual([{ id: '1', name: 'list 1', todos: [{ id: '33', content: 'Post content', list_id: '1' }] }]); +}); + +databaseTest('should retrieve a todo with its list', async ({ database }) => { + const db = await setupDrizzle(database); + + const result = await db.query.todos.findMany({ with: { list: true } }); + + expect(result).toEqual([ + { + id: '33', + content: 'Post content', + list_id: '1', + list: { id: '1', name: 'list 1' } + } + ]); +}); + +databaseTest('should return a list and todos using leftJoin', async ({ database }) => { + const db = await setupDrizzle(database); + + const result = await db.select().from(drizzleLists).leftJoin(drizzleTodos, eq(drizzleLists.id, drizzleTodos.list_id)); + + expect(result[0].lists).toEqual({ id: '1', name: 'list 1' }); + expect(result[0].todos).toEqual({ id: '33', content: 'Post content', list_id: '1' }); +}); + +databaseTest('should return a list and todos using rightJoin', async ({ database }) => { + const db = await setupDrizzle(database); + + const result = await db + .select() + .from(drizzleLists) + .rightJoin(drizzleTodos, eq(drizzleLists.id, drizzleTodos.list_id)); + + expect(result[0].lists).toEqual({ id: '1', name: 'list 1' }); + expect(result[0].todos).toEqual({ id: '33', content: 'Post content', list_id: '1' }); +}); + +databaseTest('should return a list and todos using fullJoin', async ({ database }) => { + const db = await setupDrizzle(database); + + const result = await db.select().from(drizzleLists).fullJoin(drizzleTodos, eq(drizzleLists.id, drizzleTodos.list_id)); + + expect(result[0].lists).toEqual({ id: '1', name: 'list 1' }); + expect(result[0].todos).toEqual({ id: '33', content: 'Post content', list_id: '1' }); +}); diff --git a/packages/node/tests/utils.ts b/packages/node/tests/utils.ts index c8ce4345e..c0485e7ca 100644 --- a/packages/node/tests/utils.ts +++ b/packages/node/tests/utils.ts @@ -13,11 +13,17 @@ async function createTempDir() { export const LIST_TABLE = 'lists'; export const TODO_TABLE = 'todos'; +const lists = new Table({ + name: column.text +}); + const todos = new Table({ - content: column.text + content: column.text, + list_id: column.text }); export const AppSchema = new Schema({ + lists, todos }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac5283517..6d0ee62e1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1719,9 +1719,15 @@ importers: specifier: ^4.4.2 version: 4.4.2 devDependencies: + '@powersync/drizzle-driver': + specifier: workspace:* + version: link:../drizzle-driver '@types/async-lock': specifier: ^1.4.0 version: 1.4.2 + drizzle-orm: + specifier: ^0.35.2 + version: 0.35.2(@op-engineering/op-sqlite@11.4.4(react-native@0.77.0(@babel/core@7.26.8)(@babel/preset-env@7.26.8(@babel/core@7.26.8))(@react-native-community/cli-server-api@15.1.3)(@types/react@18.3.18)(react@18.3.1))(react@18.3.1))(@types/better-sqlite3@7.6.12)(@types/react@18.3.18)(better-sqlite3@11.7.2)(kysely@0.27.4)(react@18.3.1) typescript: specifier: ^5.5.3 version: 5.7.2 From a3beb5bc03735d254743c1b8a93e9f2a569e1f03 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 19 Mar 2025 15:59:52 +0200 Subject: [PATCH 5/8] Changeset entries. --- .changeset/beige-actors-flash.md | 8 ++++++++ .changeset/eleven-cups-rescue.md | 5 +++++ .changeset/fair-squids-chew.md | 8 ++++++++ 3 files changed, 21 insertions(+) create mode 100644 .changeset/beige-actors-flash.md create mode 100644 .changeset/eleven-cups-rescue.md create mode 100644 .changeset/fair-squids-chew.md diff --git a/.changeset/beige-actors-flash.md b/.changeset/beige-actors-flash.md new file mode 100644 index 000000000..aeacf8d6a --- /dev/null +++ b/.changeset/beige-actors-flash.md @@ -0,0 +1,8 @@ +--- +'@powersync/react-native': minor +--- + +Introduced `executeRaw` member to `RNQSDBAdapter` to match `DBAdapter` interface. +It handles SQLite query results differently to `execute` - to preserve all columns, preventing duplicate column names from being overwritten. + +The implementation for RNQS will currently fall back to `execute`, preserving current behavior. Users requiring this functionality should migrate to `@powersync/op-sqlite`. diff --git a/.changeset/eleven-cups-rescue.md b/.changeset/eleven-cups-rescue.md new file mode 100644 index 000000000..26cc28f75 --- /dev/null +++ b/.changeset/eleven-cups-rescue.md @@ -0,0 +1,5 @@ +--- +'@powersync/drizzle-driver': minor +--- + +Using `executeRaw` internally for queries instead of `execute`. This function processes SQLite query results differently to preserve all columns, preventing duplicate column names from being overwritten. diff --git a/.changeset/fair-squids-chew.md b/.changeset/fair-squids-chew.md new file mode 100644 index 000000000..5e32589d6 --- /dev/null +++ b/.changeset/fair-squids-chew.md @@ -0,0 +1,8 @@ +--- +'@powersync/op-sqlite': minor +'@powersync/common': minor +'@powersync/node': minor +'@powersync/web': minor +--- + +Introduced `executeRaw`, which processes SQLite query results differently to preserve all columns, preventing duplicate column names from being overwritten. From 3ab99a0f493c20e36765882563ff0fb97f4168aa Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Thu, 20 Mar 2025 11:03:17 +0200 Subject: [PATCH 6/8] Added jsdoc result example to executeRaw. Simplified RNQS adapter changes. --- packages/common/src/db/DBAdapter.ts | 15 ++++++++++- .../RNQSDBAdapter.ts | 27 ++++--------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/packages/common/src/db/DBAdapter.ts b/packages/common/src/db/DBAdapter.ts index 567cf0dfb..580603c4a 100644 --- a/packages/common/src/db/DBAdapter.ts +++ b/packages/common/src/db/DBAdapter.ts @@ -44,7 +44,20 @@ export interface DBGetUtils { export interface LockContext extends DBGetUtils { /** Execute a single write statement. */ execute: (query: string, params?: any[] | undefined) => Promise; - /** Execute a single write statement and return raw results. */ + /** + * Execute a single write statement and return raw results. + * Unlike `execute`, which returns an object with structured key-value pairs, + * `executeRaw` returns a nested array of raw values, where each row is + * represented as an array of column values without field names. + * + * Example result: + * + * ```[ [ '1', 'list 1', '33', 'Post content', '1' ] ]``` + * + * Where as `execute`'s `rows._array` would have been: + * + * ```[ { id: '33', name: 'list 1', content: 'Post content', list_id: '1' } ]``` + */ executeRaw: (query: string, params?: any[] | undefined) => Promise; } diff --git a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts index 15e56b94a..47c788f82 100644 --- a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts +++ b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts @@ -47,25 +47,19 @@ export class RNQSDBAdapter extends BaseObserver implements DB } readLock(fn: (tx: PowerSyncLockContext) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.readLock((dbTx) => fn(this.generateDBHelpers(this.generateLockContext(dbTx))), options); + return this.baseDB.readLock((dbTx) => fn(this.generateDBHelpers(this.generateContext(dbTx))), options); } readTransaction(fn: (tx: PowerSyncTransaction) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.readTransaction( - (dbTx) => fn(this.generateDBHelpers(this.generateTransactionContext(dbTx))), - options - ); + return this.baseDB.readTransaction((dbTx) => fn(this.generateDBHelpers(this.generateContext(dbTx))), options); } writeLock(fn: (tx: PowerSyncLockContext) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.writeLock((dbTx) => fn(this.generateDBHelpers(this.generateLockContext(dbTx))), options); + return this.baseDB.writeLock((dbTx) => fn(this.generateDBHelpers(this.generateContext(dbTx))), options); } writeTransaction(fn: (tx: PowerSyncTransaction) => Promise, options?: DBLockOptions): Promise { - return this.baseDB.writeTransaction( - (dbTx) => fn(this.generateDBHelpers(this.generateTransactionContext(dbTx))), - options - ); + return this.baseDB.writeTransaction((dbTx) => fn(this.generateDBHelpers(this.generateContext(dbTx))), options); } execute(query: string, params?: any[]) { @@ -93,18 +87,7 @@ export class RNQSDBAdapter extends BaseObserver implements DB }; } - generateLockContext(ctx: RNQSLockContext) { - return { - ...ctx, - // 'executeRaw' is not implemented in RNQS, this falls back to 'execute'. - executeRaw: async (sql: string, params?: any[]) => { - const result = await ctx.execute(sql, params); - return result.rows?._array ?? []; - } - }; - } - - generateTransactionContext(ctx: RNQSTransactionContext) { + generateContext(ctx: T) { return { ...ctx, // 'executeRaw' is not implemented in RNQS, this falls back to 'execute'. From 738a65e05dcb96115ceb2f76859279e6e4bd4f19 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Thu, 20 Mar 2025 13:28:28 +0200 Subject: [PATCH 7/8] Stripping column information from RNQS raw result. --- .../db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts index 47c788f82..36c05c23b 100644 --- a/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts +++ b/packages/react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBAdapter.ts @@ -71,7 +71,8 @@ export class RNQSDBAdapter extends BaseObserver implements DB */ async executeRaw(query: string, params?: any[]): Promise { const result = await this.baseDB.execute(query, params); - return result.rows?._array ?? []; + const rows = result.rows?._array ?? []; + return rows.map((row) => Object.values(row)); } async executeBatch(query: string, params: any[][] = []): Promise { @@ -93,7 +94,8 @@ export class RNQSDBAdapter extends BaseObserver implements DB // 'executeRaw' is not implemented in RNQS, this falls back to 'execute'. executeRaw: async (sql: string, params?: any[]) => { const result = await ctx.execute(sql, params); - return result.rows?._array ?? []; + const rows = result.rows?._array ?? []; + return rows.map((row) => Object.values(row)); } }; } From dbe68fe9d935f0b41a9e8a01143238319f3d32fd Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Thu, 20 Mar 2025 13:34:31 +0200 Subject: [PATCH 8/8] Removed redundant Object.values() calls, because executeRaw doesn't contain the columns as keys anymore - this was a no-op. --- .../src/sqlite/PowerSyncSQLitePreparedQuery.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts b/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts index 51ba500bc..8a5779aca 100644 --- a/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts +++ b/packages/drizzle-driver/src/sqlite/PowerSyncSQLitePreparedQuery.ts @@ -54,12 +54,12 @@ export class PowerSyncSQLitePreparedQuery< } const rows = (await this.values(placeholderValues)) as unknown[][]; - const valueRows = rows.map((row) => Object.values(row)); + if (customResultMapper) { - const mapped = customResultMapper(valueRows) as T['all']; + const mapped = customResultMapper(rows) as T['all']; return mapped; } - return valueRows.map((row) => mapResultRow(fields!, row, (this as any).joinsNotNullableMap)); + return rows.map((row) => mapResultRow(fields!, row, (this as any).joinsNotNullableMap)); } async get(placeholderValues?: Record): Promise { @@ -80,11 +80,10 @@ export class PowerSyncSQLitePreparedQuery< } if (customResultMapper) { - const valueRows = rows.map((row) => Object.values(row)); - return customResultMapper(valueRows) as T['get']; + return customResultMapper(rows) as T['get']; } - return mapResultRow(fields!, Object.values(row), joinsNotNullableMap); + return mapResultRow(fields!, row, joinsNotNullableMap); } async values(placeholderValues?: Record): Promise {