Skip to content

Commit 7bed48a

Browse files
Copilotsubframe7536
Copilot
and
subframe7536
authored
feat: add OPFSWriteAheadVFS storage wrapper (#14)
* chore: start OPFSWriteAheadVFS wrapper implementation Agent-Logs-Url: https://github.com/subframe7536/sqlite-wasm/sessions/b2a9d08b-c4cf-49d1-a058-06339288e482 Co-authored-by: subframe7536 <78338239+subframe7536@users.noreply.github.com> * feat: add OPFSWriteAheadVFS storage wrapper as opfs-wa Agent-Logs-Url: https://github.com/subframe7536/sqlite-wasm/sessions/b2a9d08b-c4cf-49d1-a058-06339288e482 Co-authored-by: subframe7536 <78338239+subframe7536@users.noreply.github.com> * feat: add isOpfsReadWriteUnsafeSupported util, exclude package-lock.json, update readme Agent-Logs-Url: https://github.com/subframe7536/sqlite-wasm/sessions/39e80190-e7fa-4a42-8312-4657d6dda0de Co-authored-by: subframe7536 <78338239+subframe7536@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: subframe7536 <78338239+subframe7536@users.noreply.github.com>
1 parent 09ab8e5 commit 7bed48a

File tree

9 files changed

+147
-4
lines changed

9 files changed

+147
-4
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
dist
22
node_modules
33
.DS_store
4-
*.tgz
4+
*.tgz
5+
package-lock.json

‎README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,38 @@ onmessage = async () => {
8282
}
8383
```
8484

85+
#### OPFS Write-Ahead
86+
87+
Store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) using write-ahead logging for improved performance and concurrency, use `OPFSWriteAheadVFS` with `wa-sqlite.wasm`, smaller and faster sync build.
88+
89+
See [discussion](https://github.com/rhashimoto/wa-sqlite/discussions/315) for detailed overview, advantages and restrictions.
90+
91+
> [!important]
92+
> **MUST RUN IN WEB WORKER!**
93+
94+
> [!warning]
95+
> Requires OPFS `readwrite-unsafe` locking mode — **Chromium browsers only**
96+
>
97+
> Only journal mode `DELETE` (default) or `OFF` should be used.
98+
> For multiple-statement write transactions, `BEGIN IMMEDIATE` or `BEGIN EXCLUSIVE` must be used.
99+
100+
[minimal OPFS readwrite-unsafe backend browser version](https://caniuse.com/mdn-api_filesystemsyncaccesshandle_mode_readwrite-unsafe)
101+
102+
```ts
103+
import { initSQLite, isOpfsReadWriteUnsafeSupported } from '@subframe7536/sqlite-wasm'
104+
import { useOpfsWriteAheadStorage } from '@subframe7536/sqlite-wasm/opfs-wa'
105+
106+
// optional url
107+
const url = 'https://cdn.jsdelivr.net/npm/@subframe7536/sqlite-wasm/dist/wa-sqlite.wasm'
108+
109+
// must run in web worker
110+
if (await isOpfsReadWriteUnsafeSupported()) {
111+
const { run, changes, lastInsertRowId, close } = await initSQLite(
112+
useOpfsWriteAheadStorage('test.db', { url }),
113+
)
114+
}
115+
```
116+
85117
#### File System Access API
86118

87119
Store data through `FileSystemFileHandle`, use modified `OPFSAnyContextVFS` with `wa-sqlite-async.wasm`, allow to directly read and write to device's local file or OPFS file entry in main or worker thread, but a little slower than [`useOpfsStorage`](#opfs)

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"./idb": "./dist/idb.js",
2323
"./idb-memory": "./dist/idb-memory.js",
2424
"./opfs": "./dist/opfs.js",
25+
"./opfs-wa": "./dist/opfs-wa.js",
2526
"./package.json": "./package.json",
2627
"./wasm": "./dist/wa-sqlite.wasm",
2728
"./wasm-async": "./dist/wa-sqlite-async.wasm",

‎src/io/import.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,15 @@ import { check, getHandle, ignoredDataView, isFsHandleVFS } from './common'
2020
const SQLITE_BINARY_HEADER = new Uint8Array([
2121
0x53,
2222
0x51,
23-
0x4c,
23+
0x4C,
2424
0x69,
2525
0x74,
2626
0x65,
2727
0x20,
2828
0x66, // SQLite f
29-
0x6f,
29+
0x6F,
3030
0x72,
31-
0x6d,
31+
0x6D,
3232
0x61,
3333
0x74,
3434
0x20,

‎src/types/index.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,21 @@ export interface FacadeVFS extends Base {
5353
jDeviceCharacteristics: (pFile: number) => number | Promise<number>
5454
}
5555

56+
export interface OPFSWriteAheadVFSOptions {
57+
/**
58+
* Number of temporary files to preallocate
59+
*/
60+
nTmpFiles?: number
61+
/**
62+
* Automatic checkpoint threshold in pages
63+
*/
64+
autoCheckpoint?: number
65+
/**
66+
* Interval in milliseconds for backstop checkpointing
67+
*/
68+
backstopInterval?: number
69+
}
70+
5671
export interface IDBBatchAtomicVFSOptions {
5772
/**
5873
* patched options for `navigator.locks.request()`

‎src/types/wa-sqlite.d.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ declare module 'wa-sqlite/src/examples/OPFSCoopSyncVFS.js' {
1616
}
1717
}
1818

19+
declare module 'wa-sqlite/src/examples/OPFSWriteAheadVFS.js' {
20+
export class OPFSWriteAheadVFS {
21+
static create(name: string, module: any, options?: any): Promise<any>
22+
}
23+
}
24+
1925
declare module 'wa-sqlite/src/examples/MemoryVFS.js' {
2026
export class MemoryVFS {
2127
static create(name: string, module: any): Promise<any>

‎src/utils.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,36 @@ export async function isOpfsSupported(): Promise<boolean> {
8181
}
8282
}
8383

84+
/**
85+
* check if OPFS [`readwrite-unsafe`](https://developer.mozilla.org/en-US/docs/Web/API/FileSystemFileHandle/createSyncAccessHandle#mode) locking mode is supported,
86+
* required by {@link useOpfsWriteAheadStorage}
87+
*
88+
* **MUST RUN IN WEB WORKER**
89+
*
90+
* [minimal browser version](https://caniuse.com/mdn-api_filesystemsyncaccesshandle_mode_readwrite-unsafe)
91+
*/
92+
export async function isOpfsReadWriteUnsafeSupported(): Promise<boolean> {
93+
if (typeof navigator?.storage?.getDirectory !== 'function') {
94+
return false
95+
}
96+
try {
97+
const root = await navigator.storage.getDirectory()
98+
const handle = await root.getFileHandle('_CHECK_RWU', { create: true })
99+
const access = await (handle as any).createSyncAccessHandle({ mode: 'readwrite-unsafe' })
100+
access.close()
101+
await root.removeEntry('_CHECK_RWU')
102+
return true
103+
} catch {
104+
try {
105+
const root = await navigator.storage.getDirectory()
106+
await root.removeEntry('_CHECK_RWU')
107+
} catch {
108+
// ignore cleanup errors
109+
}
110+
return false
111+
}
112+
}
113+
84114
/**
85115
* check `new Worker(url, { type: 'module' })` support
86116
*

‎src/vfs/opfs-write-ahead.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import SQLiteESMFactory from 'wa-sqlite-fts5/wa-sqlite.mjs'
2+
import { OPFSWriteAheadVFS } from 'wa-sqlite/src/examples/OPFSWriteAheadVFS.js'
3+
4+
import type { BaseStorageOptions, InitSQLiteOptions, OPFSWriteAheadVFSOptions } from '../types'
5+
6+
export { OPFSWriteAheadVFS } from 'wa-sqlite/src/examples/OPFSWriteAheadVFS.js'
7+
8+
export type OPFSWriteAheadStorageOptions = OPFSWriteAheadVFSOptions
9+
10+
/**
11+
* Store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system)
12+
* using write-ahead logging for improved performance and concurrency,
13+
* use `OPFSWriteAheadVFS` with `wa-sqlite.wasm`, smaller and faster sync build
14+
*
15+
* **MUST RUN IN WEB WORKER**
16+
*
17+
* Requires OPFS `readwrite-unsafe` locking mode — **Chromium browsers only**
18+
*
19+
* Only journal mode `DELETE` (default) or `OFF` should be used.
20+
* For multiple-statement write transactions, `BEGIN IMMEDIATE` or `BEGIN EXCLUSIVE` must be used.
21+
* @param path db file directory path
22+
* @param options options
23+
* @example
24+
* ```ts
25+
* import { initSQLite, isOpfsReadWriteUnsafeSupported } from '@subframe7536/sqlite-wasm'
26+
* import { useOpfsWriteAheadStorage } from '@subframe7536/sqlite-wasm/opfs-wa'
27+
*
28+
* // optional url
29+
* const url = 'https://cdn.jsdelivr.net/npm/@subframe7536/sqlite-wasm/dist/wa-sqlite.wasm'
30+
*
31+
* // must run in web worker
32+
* if (await isOpfsReadWriteUnsafeSupported()) {
33+
* const { run, changes, lastInsertRowId, close } = await initSQLite(
34+
* useOpfsWriteAheadStorage('test.db', { url })
35+
* )
36+
* }
37+
* ```
38+
*/
39+
export async function useOpfsWriteAheadStorage(
40+
path: string,
41+
options: OPFSWriteAheadStorageOptions & BaseStorageOptions = {},
42+
): Promise<InitSQLiteOptions> {
43+
const { url, nTmpFiles, autoCheckpoint, backstopInterval, ...rest } = options
44+
const sqliteModule = await SQLiteESMFactory(url ? { locateFile: () => url } : undefined)
45+
const vfsOptions: OPFSWriteAheadVFSOptions = {}
46+
if (nTmpFiles !== undefined) {vfsOptions.nTmpFiles = nTmpFiles}
47+
if (autoCheckpoint !== undefined) {vfsOptions.autoCheckpoint = autoCheckpoint}
48+
if (backstopInterval !== undefined) {vfsOptions.backstopInterval = backstopInterval}
49+
50+
return {
51+
path,
52+
sqliteModule,
53+
vfsFn: OPFSWriteAheadVFS.create,
54+
vfsOptions,
55+
...rest,
56+
}
57+
}

‎tsdown.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const shared: UserConfig = {
1010
'idb-memory': 'src/vfs/idb-memory.ts',
1111
opfs: 'src/vfs/opfs.ts',
1212
'fs-handle': 'src/vfs/fs-handle.ts',
13+
'opfs-wa': 'src/vfs/opfs-write-ahead.ts',
1314
constant: 'src/constant.ts',
1415
},
1516
platform: 'browser',

0 commit comments

Comments
 (0)