diff --git a/.changeset/sqlite-client-locking-defaults.md b/.changeset/sqlite-client-locking-defaults.md new file mode 100644 index 00000000000..ee01605b9fe --- /dev/null +++ b/.changeset/sqlite-client-locking-defaults.md @@ -0,0 +1,6 @@ +--- +"@effect/sql-sqlite-bun": patch +"@effect/sql-sqlite-node": patch +--- + +Use a configurable five-second busy timeout and immediate transactions by default to avoid SQLite lock failures under concurrent access. Busy waits can block the event loop, while immediate transactions serialize behind other writers. diff --git a/packages/sql/sqlite-bun/src/SqliteClient.ts b/packages/sql/sqlite-bun/src/SqliteClient.ts index 745f12a5dab..e54047a1122 100644 --- a/packages/sql/sqlite-bun/src/SqliteClient.ts +++ b/packages/sql/sqlite-bun/src/SqliteClient.ts @@ -3,14 +3,20 @@ * * This module opens a SQLite database and exposes it as both `SqliteClient` and * the generic Effect SQL client. It serializes access to the database, enables - * WAL mode unless disabled, and supports database export and extension loading. - * Streaming queries and `updateValues` are not supported by this driver. + * WAL mode unless disabled, and waits up to five seconds for busy databases by + * default. Explicit transactions on writable connections use `BEGIN IMMEDIATE` + * to avoid read-to-write lock upgrades, which serializes them behind other + * writers even when they only read. Clients opened with `readonly: true` are + * unaffected. Busy waits block the event loop because `bun:sqlite` is + * synchronous. Database export and extension loading are supported; streaming + * queries and `updateValues` are not. * * @since 4.0.0 */ import { Database } from "bun:sqlite" import * as Config from "effect/Config" import * as Context from "effect/Context" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Fiber from "effect/Fiber" import { identity } from "effect/Function" @@ -25,6 +31,7 @@ import { classifySqliteError, SqlError } from "effect/unstable/sql/SqlError" import * as Statement from "effect/unstable/sql/Statement" const ATTR_DB_SYSTEM_NAME = "db.system.name" +const MAX_BUSY_TIMEOUT = 2_147_483_647 const classifyError = (cause: unknown, message: string, operation: string) => classifySqliteError(cause, { message, operation }) @@ -74,7 +81,7 @@ export interface SqliteClient extends Client.SqlClient { export const SqliteClient = Context.Service("@effect/sql-sqlite-bun/Client") /** - * Configuration for a Bun SQLite client, including filename, open mode flags, WAL behavior, span attributes, and query/result name transforms. + * Configuration for a Bun SQLite client, including filename, open mode flags, WAL and busy timeout behavior, span attributes, and query/result name transforms. * * @category models * @since 4.0.0 @@ -85,6 +92,12 @@ export interface SqliteClientConfig { readonly create?: boolean | undefined readonly readwrite?: boolean | undefined readonly disableWAL?: boolean | undefined + /** + * How long SQLite waits when the database is busy. Defaults to 5 seconds. + * `Duration.infinity` is clamped to SQLite's maximum timeout. + * Waiting blocks the event loop because `bun:sqlite` is synchronous. + */ + readonly busyTimeout?: Duration.Input | undefined readonly spanAttributes?: Record | undefined @@ -98,7 +111,7 @@ interface SqliteConnection extends Connection { } /** - * Creates a scoped Bun SQLite client for a database file, enabling WAL by default and serializing access. Streaming queries are not implemented. + * Creates a scoped Bun SQLite client for a database file, enabling WAL and a 5-second busy timeout by default. Explicit transactions on writable connections take the write lock for their duration, even when they only read; clients opened with `readonly: true` are unaffected. Streaming queries are not implemented. * * @category constructors * @since 4.0.0 @@ -122,6 +135,11 @@ export const make = ( create: readonly ? false : options.create ?? true } as any) yield* Effect.addFinalizer(() => Effect.sync(() => db.close())) + const busyTimeout = Math.min( + MAX_BUSY_TIMEOUT, + Math.max(0, Math.round(Duration.toMillis(options.busyTimeout ?? Duration.seconds(5)))) + ) + db.run(`PRAGMA busy_timeout = ${busyTimeout};`) if (options.disableWAL !== true && !readonly) { db.run("PRAGMA journal_mode = WAL;") @@ -217,6 +235,7 @@ export const make = ( acquirer, compiler, transactionAcquirer, + beginTransaction: "BEGIN IMMEDIATE", spanAttributes: [ ...(options.spanAttributes ? Object.entries(options.spanAttributes) : []), [ATTR_DB_SYSTEM_NAME, "sqlite"] diff --git a/packages/sql/sqlite-bun/test/Client.test.ts b/packages/sql/sqlite-bun/test/Client.test.ts index 82259957333..905a4a5e69c 100644 --- a/packages/sql/sqlite-bun/test/Client.test.ts +++ b/packages/sql/sqlite-bun/test/Client.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect } from "effect" +import { Duration, Effect } from "effect" import { Reactivity } from "effect/unstable/reactivity" import { rm } from "node:fs/promises" @@ -8,6 +8,42 @@ const isBun = "bun" in process.versions describe("Client", () => { it.effect("should work", () => Effect.void) + it.effect.skipIf(!isBun)("uses a 5 second busy timeout", () => + Effect.gen(function*() { + const { SqliteClient } = yield* Effect.promise(() => import("@effect/sql-sqlite-bun")) + const sql = yield* SqliteClient.make({ filename: ":memory:" }) + assert.deepStrictEqual(yield* sql`PRAGMA busy_timeout`, [{ timeout: 5000 }]) + + const custom = yield* SqliteClient.make({ filename: ":memory:", busyTimeout: "1 second" }) + assert.deepStrictEqual(yield* custom`PRAGMA busy_timeout`, [{ timeout: 1000 }]) + + const infinite = yield* SqliteClient.make({ filename: ":memory:", busyTimeout: Duration.infinity }) + assert.deepStrictEqual(yield* infinite`PRAGMA busy_timeout`, [{ timeout: 2_147_483_647 }]) + }).pipe(Effect.provide(Reactivity.layer))) + + it.effect.skipIf(!isBun)("starts transactions immediately", () => + Effect.gen(function*() { + const { SqliteClient } = yield* Effect.promise(() => import("@effect/sql-sqlite-bun")) + const filename = `/tmp/effect-sqlite-bun-transaction-${crypto.randomUUID()}.db` + yield* Effect.acquireRelease( + Effect.void, + () => Effect.promise(() => rm(filename, { force: true })) + ) + + const client = yield* SqliteClient.make({ filename }) + const contender = yield* SqliteClient.make({ filename }) + yield* contender`PRAGMA busy_timeout = 1` + + yield* client.withTransaction( + Effect.gen(function*() { + const error = yield* Effect.flip(contender`BEGIN IMMEDIATE`) + assert.strictEqual(error._tag, "SqlError") + assert(error.reason.cause instanceof Error) + assert.match(error.reason.cause.message, /database is locked/i) + }) + ) + }).pipe(Effect.provide(Reactivity.layer))) + it.effect.skipIf(!isBun)("readonly clients reject writes", () => Effect.gen(function*() { const { SqliteClient } = yield* Effect.promise(() => import("@effect/sql-sqlite-bun")) @@ -26,6 +62,7 @@ describe("Client", () => { const sql = yield* SqliteClient.make({ filename, readonly: true }) assert.deepStrictEqual(yield* sql`SELECT * FROM test`, []) + assert.deepStrictEqual(yield* sql.withTransaction(sql`SELECT * FROM test`), []) const error = yield* Effect.flip(sql`INSERT INTO test DEFAULT VALUES`) assert.strictEqual(error._tag, "SqlError") diff --git a/packages/sql/sqlite-node/src/SqliteClient.ts b/packages/sql/sqlite-node/src/SqliteClient.ts index e67b48676ea..9124443ab71 100644 --- a/packages/sql/sqlite-node/src/SqliteClient.ts +++ b/packages/sql/sqlite-node/src/SqliteClient.ts @@ -3,9 +3,14 @@ * * This module opens a SQLite database and exposes it as both `SqliteClient` and * the generic Effect SQL client. It serializes access through one connection, - * caches prepared statements, enables WAL mode unless disabled, and supports - * database backup, and extension loading. Streaming queries and - * `updateValues` are not supported by this driver. + * caches prepared statements, enables WAL mode unless disabled, and waits up + * to five seconds for busy databases by default. Explicit transactions on + * writable connections use `BEGIN IMMEDIATE` to avoid read-to-write lock + * upgrades, which serializes them behind other writers even when they only + * read. Clients opened with `readonly: true` are unaffected. Busy waits block + * the Node.js event loop because `node:sqlite` is synchronous. Database backup + * and extension loading are supported; streaming queries and `updateValues` + * are not. * * @since 4.0.0 */ @@ -29,6 +34,7 @@ import { backup as backupDatabase, DatabaseSync } from "node:sqlite" import type { StatementSync } from "node:sqlite" const ATTR_DB_SYSTEM_NAME = "db.system.name" +const MAX_BUSY_TIMEOUT = 2_147_483_647 /** * Runtime type identifier used to mark Node `SqliteClient` values. @@ -82,7 +88,7 @@ export interface BackupMetadata { export const SqliteClient = Context.Service("@effect/sql-sqlite-node/SqliteClient") /** - * Configuration for a node SQLite client backed by `node:sqlite`, including the database filename, read-only mode, statement cache settings, WAL behavior, span attributes, and query/result name transforms. + * Configuration for a node SQLite client backed by `node:sqlite`, including the database filename, read-only mode, statement cache settings, WAL and busy timeout behavior, span attributes, and query/result name transforms. * * @category models * @since 4.0.0 @@ -93,6 +99,12 @@ export interface SqliteClientConfig { readonly prepareCacheSize?: number | undefined readonly prepareCacheTTL?: Duration.Input | undefined readonly disableWAL?: boolean | undefined + /** + * How long SQLite waits when the database is busy. Defaults to 5 seconds. + * `Duration.infinity` is clamped to SQLite's maximum timeout. + * Waiting blocks the Node.js event loop because `node:sqlite` is synchronous. + */ + readonly busyTimeout?: Duration.Input | undefined readonly spanAttributes?: Record | undefined readonly transformResultNames?: ((str: string) => string) | undefined @@ -105,7 +117,7 @@ interface SqliteConnection extends Connection { } /** - * Creates a scoped node SQLite client from the supplied configuration, using a single serialized connection with WAL enabled by default and exposing SQLite-specific `export`, `backup`, and `loadExtension` operations. + * Creates a scoped node SQLite client from the supplied configuration, using a single serialized connection with WAL and a 5-second busy timeout enabled by default. Explicit transactions on writable connections take the write lock for their duration, even when they only read; clients opened with `readonly: true` are unaffected. * * @category constructors * @since 4.0.0 @@ -129,6 +141,11 @@ export const make = ( }) yield* Scope.addFinalizer(scope, Effect.sync(() => db.close())) db.enableLoadExtension(false) + const busyTimeout = Math.min( + MAX_BUSY_TIMEOUT, + Math.max(0, Math.round(Duration.toMillis(options.busyTimeout ?? Duration.seconds(5)))) + ) + db.exec(`PRAGMA busy_timeout = ${busyTimeout}`) if (options.disableWAL !== true) { db.exec("PRAGMA journal_mode = WAL") @@ -303,6 +320,7 @@ export const make = ( acquirer, compiler, transactionAcquirer, + beginTransaction: "BEGIN IMMEDIATE", spanAttributes: [ ...(options.spanAttributes ? Object.entries(options.spanAttributes) : []), [ATTR_DB_SYSTEM_NAME, "sqlite"] diff --git a/packages/sql/sqlite-node/test/Client.test.ts b/packages/sql/sqlite-node/test/Client.test.ts index 8c35e0f11d9..3a20cec55e8 100644 --- a/packages/sql/sqlite-node/test/Client.test.ts +++ b/packages/sql/sqlite-node/test/Client.test.ts @@ -1,7 +1,7 @@ import { NodeFileSystem } from "@effect/platform-node" import { SqliteClient } from "@effect/sql-sqlite-node" import { assert, describe, it } from "@effect/vitest" -import { Effect, FileSystem } from "effect" +import { Duration, Effect, FileSystem } from "effect" import { Reactivity } from "effect/unstable/reactivity" const makeClient = Effect.gen(function*() { @@ -12,6 +12,16 @@ const makeClient = Effect.gen(function*() { }) }).pipe(Effect.provide([NodeFileSystem.layer, Reactivity.layer])) +const makeClients = Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const dir = yield* fs.makeTempDirectoryScoped() + const filename = dir + "/test.db" + return { + client: yield* SqliteClient.make({ filename }), + contender: yield* SqliteClient.make({ filename }) + } +}).pipe(Effect.provide([NodeFileSystem.layer, Reactivity.layer])) + describe("Client", () => { it.effect("should work", () => Effect.gen(function*() { @@ -75,6 +85,54 @@ describe("Client", () => { assert.deepStrictEqual(rows, []) })) + it.effect("uses a 5 second busy timeout", () => + Effect.gen(function*() { + const sql = yield* makeClient + assert.deepStrictEqual(yield* sql`PRAGMA busy_timeout`, [{ timeout: 5000 }]) + + const custom = yield* SqliteClient.make({ filename: ":memory:", busyTimeout: "1 second" }).pipe( + Effect.provide(Reactivity.layer) + ) + assert.deepStrictEqual(yield* custom`PRAGMA busy_timeout`, [{ timeout: 1000 }]) + + const infinite = yield* SqliteClient.make({ filename: ":memory:", busyTimeout: Duration.infinity }).pipe( + Effect.provide(Reactivity.layer) + ) + assert.deepStrictEqual(yield* infinite`PRAGMA busy_timeout`, [{ timeout: 2_147_483_647 }]) + })) + + it.effect("starts transactions immediately", () => + Effect.gen(function*() { + const { client, contender } = yield* makeClients + yield* contender`PRAGMA busy_timeout = 1` + + yield* client.withTransaction( + Effect.gen(function*() { + const error = yield* Effect.flip(contender`BEGIN IMMEDIATE`) + assert.strictEqual(error._tag, "SqlError") + assert(error.reason.cause instanceof Error) + assert.match(error.reason.cause.message, /database is locked/i) + }) + ) + })) + + it.effect("supports transactions on readonly clients", () => + Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const dir = yield* fs.makeTempDirectoryScoped() + const filename = dir + "/test.db" + + yield* Effect.scoped( + Effect.gen(function*() { + const sql = yield* SqliteClient.make({ filename }) + yield* sql`CREATE TABLE test (id INTEGER PRIMARY KEY)` + }) + ) + + const sql = yield* SqliteClient.make({ filename, readonly: true }) + assert.deepStrictEqual(yield* sql.withTransaction(sql`SELECT * FROM test`), []) + }).pipe(Effect.provide([NodeFileSystem.layer, Reactivity.layer]))) + it.effect("supports backup and export", () => Effect.gen(function*() { const sql = yield* makeClient