feat(dialect-wasqlite-worker): add support for OPFS

Author: subframe7536

packages/dialect-wasqlite-worker/README.md

@@ -1,6 +1,6 @@
 # WaSqlite Worker Dialect
 
-[kysely](https://github.com/kysely-org/kysely) dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite), execute sql in `Web Worker`, store data in IndexedDB
+[kysely](https://github.com/kysely-org/kysely) dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite), execute sql in `Web Worker`, store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
 
 ## install
 
@@ -31,6 +31,10 @@ export interface WaSqliteWorkerDialectConfig {
    * ```
    */
   worker?: Worker
+  /**
+   * prefer to store in OPFS, fallback to IndexedDB
+   */
+  preferOPFS?: boolean
   onCreateConnection?: (connection: DatabaseConnection) => Promise<void>
 }
 ```

packages/dialect-wasqlite-worker/package.json

@@ -40,9 +40,9 @@
     "kysely": "^0.26"
   },
   "dependencies": {
-    "zen-mitt": "^0.2.0"
+    "zen-mitt": "^0.3.0"
   },
   "devDependencies": {
-    "@subframe7536/wa-sqlite": "^0.9.6"
+    "@subframe7536/sqlite-wasm": "^0.1.1"
   }
 }

packages/dialect-wasqlite-worker/src/driver.ts

@@ -24,8 +24,9 @@ export class WaSqliteWorkerDriver implements Driver {
     }
     this.worker.postMessage({
       type: 'init',
-      dbName: this.config.dbName,
+      fileName: this.config.fileName,
       url: this.config.url,
+      preferOPFS: this.config.preferOPFS,
     } satisfies MainMsg)
     await new Promise<void>((resolve, reject) => {
       this.mitt?.once('init', ({ err }) => {
@@ -118,12 +119,8 @@ class WaSqliteWorkerConnection implements DatabaseConnection {
 
   async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
     const { parameters, sql, query } = compiledQuery
-    const mode = query.kind === 'SelectQueryNode'
-      ? 'query'
-      : query.kind === 'RawNode'
-        ? 'raw'
-        : 'exec'
-    this.worker.postMessage({ type: 'run', mode, sql, parameters } satisfies MainMsg)
+    const isSelect = query.kind === 'SelectQueryNode'
+    this.worker.postMessage({ type: 'run', isSelect, sql, parameters } satisfies MainMsg)
     return new Promise((resolve, reject) => {
       !this.mitt && reject('kysely instance has been destroyed')
 

packages/dialect-wasqlite-worker/src/index.ts

@@ -3,7 +3,10 @@ import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from 'kysely'
 import { WaSqliteWorkerDriver } from './driver'
 
 export interface WaSqliteWorkerDialectConfig {
-  dbName: string
+  /**
+   * db file name
+   */
+  fileName: string
   /**
    * the URL of `wa-sqlite` WASM
    * @example
@@ -22,6 +25,10 @@ export interface WaSqliteWorkerDialectConfig {
    * ```
    */
   worker?: Worker
+  /**
+   * prefer to use OPFS, fallback to IndexedDB
+   */
+  preferOPFS?: boolean
   onCreateConnection?: (connection: DatabaseConnection) => Promise<void>
 }
 export class WaSqliteWorkerDialect implements Dialect {

packages/dialect-wasqlite-worker/src/type.ts

@@ -1,13 +1,9 @@
 import type { QueryResult } from 'kysely'
 
-export type Promisable<T> = T | Promise<T>
-
-export type RunMode = 'exec' | 'query' | 'raw'
-
 export type MainMsg =
   | {
     type: 'run'
-    mode: RunMode
+    isSelect: boolean
     sql: string
     parameters?: readonly unknown[]
   }
@@ -17,7 +13,8 @@ export type MainMsg =
   | {
     type: 'init'
     url?: string
-    dbName: string
+    fileName: string
+    preferOPFS?: boolean
   }
 
 export type WorkerMsg = {

packages/dialect-wasqlite-worker/src/worker.ts

@@ -1,79 +1,29 @@
-import type { SQLiteAPI, SQLiteCompatibleType } from '@subframe7536/wa-sqlite'
-import { Factory, SQLITE_ROW } from '@subframe7536/wa-sqlite'
-import SQLiteAsyncESMFactory from '@subframe7536/wa-sqlite/dist/wa-sqlite-async.mjs'
-import { IDBBatchAtomicVFS } from '@subframe7536/wa-sqlite/src/examples/IDBBatchAtomicVFS'
-import type { MainMsg, RunMode, WorkerMsg } from './type'
-
-let sqlite: SQLiteAPI
-let db: number
-
-async function init(dbName: string, url?: string) {
-  const option = url ? { locateFile: () => url } : {}
-  const SQLiteAsyncModule = await SQLiteAsyncESMFactory(option)
-
-  sqlite = Factory(SQLiteAsyncModule)
-  sqlite.vfs_register(new IDBBatchAtomicVFS(dbName, { durability: 'relaxed' }))
-  db = await sqlite.open_v2(
-    dbName,
-    undefined,
-    dbName,
-  )
-}
-
-async function run(sql: string, parameters?: readonly unknown[]) {
-  const str = sqlite.str_new(db, sql)
-  const prepared = await sqlite.prepare_v2(
-    db,
-    sqlite.str_value(str),
-  )
-
-  if (prepared === null) {
-    return [] as any[]
+import type { SQLiteDB } from '@subframe7536/sqlite-wasm'
+import { initSQLite, isOpfsSupported } from '@subframe7536/sqlite-wasm'
+import type { QueryResult } from 'kysely'
+import type { MainMsg, WorkerMsg } from './type'
+
+let db: SQLiteDB
+
+async function init(fileName: string, url?: string, preferOPFS?: boolean) {
+  let storage
+  if (isOpfsSupported() && preferOPFS) {
+    storage = (await import('@subframe7536/sqlite-wasm/opfs')).useOpfsStorage(fileName, { url })
+  } else {
+    storage = (await import('@subframe7536/sqlite-wasm/idb')).useIdbStorage(fileName, { url })
   }
 
-  const stmt = prepared.stmt
-  try {
-    if (typeof parameters !== 'undefined') {
-      sqlite.bind_collection(
-        stmt,
-        parameters as any[],
-      )
-    }
-
-    const rows: Record<string, SQLiteCompatibleType>[] = []
-    let cols: string[] = []
-
-    while ((await sqlite.step(stmt)) === SQLITE_ROW) {
-      cols = cols.length === 0 ? sqlite.column_names(stmt) : cols
-      const row = sqlite.row(stmt)
-      rows.push(cols.reduce((acc, key, i) => {
-        acc[key] = row[i]
-        return acc
-      }, {} as Record<string, SQLiteCompatibleType>))
-    }
-    return rows
-  } finally {
-    await sqlite.finalize(stmt)
-  }
+  db = await initSQLite(storage)
 }
-
-async function exec(mode: RunMode, sql: string, parameters?: readonly unknown[]) {
-  const rows = await run(sql, parameters)
-  if (mode === 'query') {
+async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
+  const rows = await db.run(sql, parameters)
+  if (isSelect) {
     return { rows }
   }
-  const v = await run('SELECT last_insert_rowid() as id')
-  return {
-    insertId: BigInt(v[0].id),
-    numAffectedRows: BigInt(sqlite.changes(db)),
-    rows: mode === 'raw' ? rows : [],
-  }
-}
-
-async function close() {
-  await sqlite.close(db)
+  const insertId = BigInt(await db.lastInsertRowId())
+  const numAffectedRows = BigInt(db.changes())
+  return { rows, insertId, numAffectedRows }
 }
-
 onmessage = async (ev: MessageEvent<MainMsg>) => {
   const data = ev.data
   const ret: WorkerMsg = {
@@ -83,14 +33,14 @@ onmessage = async (ev: MessageEvent<MainMsg>) => {
   }
   try {
     switch (data.type) {
+      case 'init':
+        await init(data.fileName, data.url, data.preferOPFS)
+        break
       case 'run':
-        ret.data = await exec(data.mode, data.sql, data.parameters)
+        ret.data = await exec(data.isSelect, data.sql, data.parameters)
         break
       case 'close':
-        await close()
-        break
-      case 'init':
-        await init(data.dbName, data.url)
+        await db.close()
         break
     }
   } catch (error) {

packages/dialect-wasqlite-worker/tsup.config.ts

@@ -11,7 +11,6 @@ export default defineConfig([
     shims: true,
     dts: true,
     external: ['kysely'],
-    noExternal: ['wa-sqlite'],
     treeshake: true,
   },
   {
@@ -21,17 +20,22 @@ export default defineConfig([
     clean: false,
     format: ['cjs', 'esm'],
     dts: true,
-    noExternal: ['wa-sqlite'],
     treeshake: true,
     minify: true,
     plugins: [
       {
         name: 'copy',
-        async buildEnd(this) {
-          this.format === 'esm' && await copyFile(
-            './node_modules/@subframe7536/wa-sqlite/dist/wa-sqlite-async.wasm',
-            './dist/wa-sqlite-async.wasm',
-          )
+        buildEnd(this) {
+          this.format === 'esm' && Promise.all([
+            copyFile(
+              './node_modules/@subframe7536/sqlite-wasm/dist/wa-sqlite.wasm',
+              './dist/wa-sqlite.wasm',
+            ),
+            copyFile(
+              './node_modules/@subframe7536/sqlite-wasm/dist/wa-sqlite-async.wasm',
+              './dist/wa-sqlite-async.wasm',
+            ),
+          ])
         },
       },
     ],

playground/package.json

@@ -1,9 +1,11 @@
 {
   "scripts": {
     "dev": "vite",
-    "build": "vite build"
+    "build": "vite build",
+    "preview": "pnpm build && vite preview"
   },
   "dependencies": {
+    "@subframe7536/sqlite-wasm": "^0.1.1",
     "kysely-sqlite-builder": "workspace:*",
     "kysely-wasm": "workspace:*",
     "kysely-wasqlite-worker": "workspace:*",