From a54e8b767c3ee5d28a6fa7f26d8465624b3bef74 Mon Sep 17 00:00:00 2001 From: bcoll Date: Mon, 23 Jan 2023 23:36:44 +0000 Subject: [PATCH] Implement `D1Database` using D1JS and D1 API Previously, Miniflare had its own implementations of `BetaDatabase` and `Statement`. These were subtly different to the implementations in the D1 Wrangler shim D1JS, causing behaviour mismatches. This change switches the implementation to use the shim, with Miniflare implementing the underlying D1 HTTP API instead. We'll need to do this anyway when adding D1 support to Miniflare 3 using `workerd`. Specific changes: - Throw when calling `D1PreparedStatement#run()` with statements that return data, closes #441 - Fix response envelope format, closes #442 and cloudflare/wrangler2#2504 - Fix binding/return of `BLOB`-typed values, closes wrangler2#2527 - Fix `D1Database#raw()` return, closes cloudflare/wrangler2#2238 (already fixed in #474) - Add support for `D1Database#dump()` - Run `D1Database#{batch,exec}()` statements in implicit transaction - Only run first statement when calling `D1PreparedStatement#{first,run,all,raw}()` --- packages/core/src/standards/http.ts | 2 +- packages/d1/src/api.ts | 204 +++++++++++++++++ packages/d1/src/d1js.ts | 274 ++++++++++++++++++++++ packages/d1/src/database.ts | 38 ---- packages/d1/src/index.ts | 4 +- packages/d1/src/plugin.ts | 12 +- packages/d1/src/splitter.ts | 167 ++++++++++++++ packages/d1/src/statement.ts | 135 ----------- packages/d1/test/d1js.spec.ts | 337 ++++++++++++++++++++++++++++ packages/d1/test/database.spec.ts | 46 ---- packages/web-sockets/src/fetch.ts | 2 +- packages/web-sockets/src/plugin.ts | 1 - 12 files changed, 993 insertions(+), 229 deletions(-) create mode 100644 packages/d1/src/api.ts create mode 100644 packages/d1/src/d1js.ts delete mode 100644 packages/d1/src/database.ts create mode 100644 packages/d1/src/splitter.ts delete mode 100644 packages/d1/src/statement.ts create mode 100644 packages/d1/test/d1js.spec.ts delete mode 100644 packages/d1/test/database.spec.ts diff --git a/packages/core/src/standards/http.ts b/packages/core/src/standards/http.ts index d8b40ed3c..a6d956161 100644 --- a/packages/core/src/standards/http.ts +++ b/packages/core/src/standards/http.ts @@ -752,7 +752,7 @@ class MiniflareDispatcher extends Dispatcher { } export async function fetch( - this: Dispatcher | void, + this: Dispatcher | unknown, input: RequestInfo, init?: RequestInit ): Promise { diff --git a/packages/d1/src/api.ts b/packages/d1/src/api.ts new file mode 100644 index 000000000..1854c66de --- /dev/null +++ b/packages/d1/src/api.ts @@ -0,0 +1,204 @@ +import crypto from "crypto"; +import fs from "fs/promises"; +import os from "os"; +import path from "path"; +import { performance } from "perf_hooks"; +import { Request, RequestInfo, RequestInit, Response } from "@miniflare/core"; +import type { SqliteDB } from "@miniflare/shared"; +import type { Statement as SqliteStatement } from "better-sqlite3"; +import splitSqlQuery from "./splitter"; + +// query +interface SingleQuery { + sql: string; + params?: any[] | null; +} + +// response +interface ErrorResponse { + error: string; + success: false; + served_by: string; +} +interface ResponseMeta { + duration: number; + last_row_id: number | null; + changes: number | null; + served_by: string; + internal_stats: null; +} +interface SuccessResponse { + results: any; + duration: number; + lastRowId: number | null; + changes: number | null; + success: true; + served_by: string; + meta: ResponseMeta | null; +} + +const served_by = "miniflare.db"; + +function ok(results: any, start: number): SuccessResponse { + const duration = performance.now() - start; + return { + results, + duration, + // These are all `null`ed out in D1 + lastRowId: null, + changes: null, + success: true, + served_by, + meta: { + duration, + last_row_id: null, + changes: null, + served_by, + internal_stats: null, + }, + }; +} +function err(error: any): ErrorResponse { + return { + error: String(error), + success: false, + served_by, + }; +} + +type QueryRunner = (query: SingleQuery) => SuccessResponse; + +function normaliseParams(params: SingleQuery["params"]): any[] { + return (params ?? []).map((param) => + // If `param` is an array, assume it's a byte array + Array.isArray(param) ? new Uint8Array(param) : param + ); +} +function normaliseResults(rows: any[]): any[] { + return rows.map((row) => + Object.fromEntries( + Object.entries(row).map(([key, value]) => [ + key, + // If `value` is an array, convert it to a regular numeric array + value instanceof Buffer ? Array.from(value) : value, + ]) + ) + ); +} + +const DOESNT_RETURN_DATA_MESSAGE = + "The columns() method is only for statements that return data"; +const EXECUTE_RETURNS_DATA_MESSAGE = + "SQL execute error: Execute returned results - did you mean to call query?"; +function returnsData(stmt: SqliteStatement): boolean { + try { + stmt.columns(); + return true; + } catch (e) { + // `columns()` fails on statements that don't return data + if (e instanceof TypeError && e.message === DOESNT_RETURN_DATA_MESSAGE) { + return false; + } + throw e; + } +} + +export class D1DatabaseAPI { + constructor(private readonly db: SqliteDB) {} + + #query: QueryRunner = (query) => { + const start = performance.now(); + // D1 only respects the first statement + const sql = splitSqlQuery(query.sql)[0]; + const stmt = this.db.prepare(sql); + const params = normaliseParams(query.params); + let results: any[]; + if (returnsData(stmt)) { + results = stmt.all(params); + } else { + // `/query` does support queries that don't return data, + // returning `[]` instead of `null` + stmt.run(params); + results = []; + } + return ok(normaliseResults(results), start); + }; + + #execute: QueryRunner = (query) => { + const start = performance.now(); + // D1 only respects the first statement + const sql = splitSqlQuery(query.sql)[0]; + const stmt = this.db.prepare(sql); + // `/execute` only supports queries that don't return data + if (returnsData(stmt)) throw new Error(EXECUTE_RETURNS_DATA_MESSAGE); + const params = normaliseParams(query.params); + stmt.run(params); + return ok(null, start); + }; + + async #handleQueryExecute( + request: Request, + runner: QueryRunner + ): Promise { + // `D1Database#batch()` will call `/query` with an array of queries + const query = await request.json(); + let results: SuccessResponse | SuccessResponse[]; + if (Array.isArray(query)) { + // Run batches in an implicit transaction. Note we have to use savepoints + // here as the SQLite transaction stack may not be empty if we're running + // inside the Miniflare testing environment, and nesting regular + // transactions is not permitted. + const savepointName = `MINIFLARE_D1_BATCH_${Date.now()}_${Math.floor( + Math.random() * Number.MAX_SAFE_INTEGER + )}`; + this.db.exec(`SAVEPOINT ${savepointName};`); // BEGIN TRANSACTION; + try { + results = query.map(runner); + this.db.exec(`RELEASE ${savepointName};`); // COMMIT; + } catch (e) { + this.db.exec(`ROLLBACK TO ${savepointName};`); // ROLLBACK; + this.db.exec(`RELEASE ${savepointName};`); + throw e; + } + } else { + results = runner(query); + } + return Response.json(results); + } + + async #handleDump(): Promise { + // `better-sqlite3` requires us to back up to a file, so create a temp one + const random = crypto.randomBytes(8).toString("hex"); + const tmpPath = path.join(os.tmpdir(), `miniflare-d1-dump-${random}.db`); + await this.db.backup(tmpPath); + const buffer = await fs.readFile(tmpPath); + // Delete file in the background, ignore errors as they don't really matter + void fs.unlink(tmpPath).catch(() => {}); + return new Response(buffer, { + headers: { "Content-Type": "application/octet-stream" }, + }); + } + + async fetch(input: RequestInfo, init?: RequestInit) { + // `D1Database` may call fetch with a relative URL, so resolve it, making + // sure to only construct a `new URL()` once. + if (typeof input === "string") input = new URL(input, "http://localhost"); + const request = new Request(input, init); + if (!(input instanceof URL)) input = new URL(request.url); + const pathname = input.pathname; + + if (request.method !== "POST") return new Response(null, { status: 405 }); + try { + if (pathname === "/query") { + return await this.#handleQueryExecute(request, this.#query); + } else if (pathname === "/execute") { + return await this.#handleQueryExecute(request, this.#execute); + } else if (pathname === "/dump") { + return await this.#handleDump(); + } + } catch (e) { + return Response.json(err(e)); + } + return new Response(null, { status: 404 }); + } +} diff --git a/packages/d1/src/d1js.ts b/packages/d1/src/d1js.ts new file mode 100644 index 000000000..f9e77cd90 --- /dev/null +++ b/packages/d1/src/d1js.ts @@ -0,0 +1,274 @@ +/* eslint-disable */ +// Vendored from internal D1JS repository, with some extra `@ts-expect-error`s + +import type { fetch } from "@miniflare/core"; + +export type DatabaseBinding = { + fetch: typeof fetch; +}; + +export type D1Result = { + results?: T[]; + success: boolean; + error?: string; + meta: any; +}; + +export type D1ExecResult = { + count: number; + duration: number; +}; + +type SQLError = { + error: string; +}; + +export class D1Database { + private readonly binding: DatabaseBinding; + + constructor(binding: DatabaseBinding) { + this.binding = binding; + } + + prepare(query: string): D1PreparedStatement { + return new D1PreparedStatement(this, query); + } + + async dump(): Promise { + const response = await this.binding.fetch("/dump", { + method: "POST", + headers: { + "content-type": "application/json", + }, + }); + if (response.status !== 200) { + try { + const err = (await response.json()) as SQLError; + // @ts-expect-error `cause` support was added in Node 16.9.0, + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_DUMP_ERROR", { + cause: new Error(err.error), + }); + } catch (e) { + // @ts-expect-error `cause` support was added in Node 16.9.0 + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_DUMP_ERROR", { + cause: new Error("Status " + response.status), + }); + } + } + return await response.arrayBuffer(); + } + + async batch( + statements: D1PreparedStatement[] + ): Promise[]> { + const exec = await this._send( + "/query", + statements.map((s: D1PreparedStatement) => s.statement), + statements.map((s: D1PreparedStatement) => s.params) + ); + return exec as D1Result[]; + } + + async exec(query: string): Promise { + const lines = query.trim().split("\n"); + const _exec = await this._send("/query", lines, [], false); + const exec = Array.isArray(_exec) ? _exec : [_exec]; + const error = exec + .map((r) => { + return r.error ? 1 : 0; + }) + .indexOf(1); + if (error !== -1) { + // @ts-expect-error `cause` support was added in Node 16.9.0, + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_EXEC_ERROR", { + cause: new Error( + "Error in line " + + (error + 1) + + ": " + + lines[error] + + ": " + + exec[error].error + ), + }); + } else { + return { + count: exec.length, + duration: exec.reduce((p, c) => { + return p + c.meta.duration; + }, 0), + }; + } + } + + async _send( + endpoint: string, + query: any, + params: any[], + dothrow: boolean = true + ): Promise[] | D1Result> { + /* this needs work - we currently only support ordered ?n params */ + const body = JSON.stringify( + typeof query == "object" + ? (query as any[]).map((s: string, index: number) => { + return { sql: s, params: params[index] }; + }) + : { + sql: query, + params: params, + } + ); + + const response = await this.binding.fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body, + }); + + try { + const answer = await response.json(); + + if ((answer as any).error && dothrow) { + const err = answer as SQLError; + // @ts-expect-error `cause` support was added in Node 16.9.0, + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_ERROR", { cause: new Error(err.error) }); + } else { + return Array.isArray(answer) + ? (answer.map((r) => mapD1Result(r)) as D1Result[]) + : (mapD1Result(answer) as D1Result); + } + } catch (e: any) { + // @ts-expect-error `cause` support was added in Node 16.9.0, + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_ERROR", { + cause: new Error(e.cause || "Something went wrong"), + }); + } + } +} + +export class D1PreparedStatement { + readonly statement: string; + private readonly database: D1Database; + params: any[]; + + constructor(database: D1Database, statement: string, values?: any) { + this.database = database; + this.statement = statement; + this.params = values || []; + } + + bind(...values: any[]) { + // Validate value types + for (var r in values) { + switch (typeof values[r]) { + case "number": + case "string": + break; + case "object": + // nulls are objects in javascript + if (values[r] == null) break; + // arrays with uint8's are good + if ( + Array.isArray(values[r]) && + values[r] + .map((b: any) => { + return typeof b == "number" && b >= 0 && b < 256 ? 1 : 0; + }) + .indexOf(0) == -1 + ) + break; + // convert ArrayBuffer to array + if (values[r] instanceof ArrayBuffer) { + values[r] = Array.from(new Uint8Array(values[r])); + break; + } + // convert view to array + if (ArrayBuffer.isView(values[r])) { + values[r] = Array.from(values[r]); + break; + } + default: + // @ts-expect-error `cause` support was added in Node 16.9.0, + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_TYPE_ERROR", { + cause: new Error( + "Type '" + + typeof values[r] + + "' not supported for value '" + + values[r] + + "'" + ), + }); + } + } + return new D1PreparedStatement(this.database, this.statement, values); + } + + async first(colName?: string): Promise { + const info = firstIfArray( + await this.database._send("/query", this.statement, this.params) + ); + const results = info.results ?? []; + if (colName !== undefined) { + // @ts-expect-error `T` here represents the value type, not the full row + if (results.length > 0 && results[0][colName] === undefined) { + // @ts-expect-error `cause` support was added in Node 16.9.0, + // and Miniflare's minimum supported version is 16.13.0 + throw new Error("D1_COLUMN_NOTFOUND", { + cause: new Error("Column not found"), + }); + } + // @ts-expect-error `T` here represents the value type, not the full row + return results.length < 1 ? null : results[0][colName]; + } else { + return results.length < 1 ? null : results[0]; + } + } + + async run(): Promise> { + return firstIfArray( + await this.database._send("/execute", this.statement, this.params) + ); + } + + async all(): Promise> { + return firstIfArray( + await this.database._send("/query", this.statement, this.params) + ); + } + + async raw(): Promise { + const s = firstIfArray( + await this.database._send("/query", this.statement, this.params) + ); + const raw = []; + for (const r in s.results) { + const entry = Object.keys(s.results[r as unknown as number]).map((k) => { + // @ts-expect-error `T` is raw row type, so we don't know column names + return s.results[r][k]; + }); + raw.push(entry); + } + return raw as unknown as T[]; + } +} + +function firstIfArray(results: T | T[]): T { + return Array.isArray(results) ? results[0] : results; +} + +function mapD1Result(result: any): D1Result { + let map: D1Result = { + results: result.results || [], + success: result.success === undefined ? true : result.success, + meta: result.meta || {}, + }; + result.error && (map.error = result.error); + return map; +} diff --git a/packages/d1/src/database.ts b/packages/d1/src/database.ts deleted file mode 100644 index 7bb3d4f68..000000000 --- a/packages/d1/src/database.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { performance } from "node:perf_hooks"; -import type { SqliteDB } from "@miniflare/shared"; -import { Statement } from "./statement"; - -export class BetaDatabase { - readonly #db: SqliteDB; - - constructor(db: SqliteDB) { - this.#db = db; - } - - prepare(source: string) { - return new Statement(this.#db, source); - } - - async batch(statements: Statement[]) { - return await Promise.all(statements.map((s) => s.all())); - } - - async exec(multiLineStatements: string) { - const statements = multiLineStatements - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0); - const start = performance.now(); - for (const statement of statements) { - await new Statement(this.#db, statement).all(); - } - return { - count: statements.length, - duration: performance.now() - start, - }; - } - - async dump() { - throw new Error("DB.dump() not implemented locally!"); - } -} diff --git a/packages/d1/src/index.ts b/packages/d1/src/index.ts index 0c3fb15ee..3067ccb3a 100644 --- a/packages/d1/src/index.ts +++ b/packages/d1/src/index.ts @@ -1,3 +1,3 @@ -export * from "./database"; +export * from "./api"; +export * from "./d1js"; export * from "./plugin"; -export * from "./statement"; diff --git a/packages/d1/src/plugin.ts b/packages/d1/src/plugin.ts index 50c15bb45..91eb96a85 100644 --- a/packages/d1/src/plugin.ts +++ b/packages/d1/src/plugin.ts @@ -8,7 +8,8 @@ import { StorageFactory, resolveStoragePersist, } from "@miniflare/shared"; -import { BetaDatabase } from "./database"; +import { D1DatabaseAPI } from "./api"; +import { D1Database } from "./d1js"; export interface D1Options { d1Databases?: string[]; @@ -42,19 +43,20 @@ export class D1Plugin extends Plugin implements D1Options { this.#persist = resolveStoragePersist(ctx.rootPath, this.d1Persist); } - async getBetaDatabase( + async getDatabase( storageFactory: StorageFactory, dbName: string - ): Promise { + ): Promise { const storage = await storageFactory.storage(dbName, this.#persist); - return new BetaDatabase(await storage.getSqliteDatabase()); + const db = await storage.getSqliteDatabase(); + return new D1Database(new D1DatabaseAPI(db)); } async setup(storageFactory: StorageFactory): Promise { const bindings: Context = {}; for (const dbName of this.d1Databases ?? []) { if (dbName.startsWith(D1_BETA_PREFIX)) { - bindings[dbName] = await this.getBetaDatabase( + bindings[dbName] = await this.getDatabase( storageFactory, // Store it locally without the prefix dbName.slice(D1_BETA_PREFIX.length) diff --git a/packages/d1/src/splitter.ts b/packages/d1/src/splitter.ts new file mode 100644 index 000000000..6ed1998dd --- /dev/null +++ b/packages/d1/src/splitter.ts @@ -0,0 +1,167 @@ +/** + * @module + * This code is inspired by that of https://www.atdatabases.org/docs/split-sql-query, which is published under MIT license, + * and is Copyright (c) 2019 Forbes Lindesay. + * + * See https://github.com/ForbesLindesay/atdatabases/blob/103c1e7/packages/split-sql-query/src/index.ts + * for the original code. + * + * ============================================================================= + * + * This updated code is lifted from https://github.com/cloudflare/wrangler2/blob/a0e5a4913621cffe757b2d14b6f3f466831f3d7f/packages/wrangler/src/d1/splitter.ts, + * with tests in https://github.com/cloudflare/wrangler2/blob/a0e5a4913621cffe757b2d14b6f3f466831f3d7f/packages/wrangler/src/__tests__/d1/splitter.test.ts. + * Thanks @petebacondarwin! + */ + +/** + * Is the given `sql` string likely to contain multiple statements. + * + * If `mayContainMultipleStatements()` returns `false` you can be confident that the sql + * does not contain multiple statements. Otherwise you have to check further. + */ +export function mayContainMultipleStatements(sql: string): boolean { + const trimmed = sql.trimEnd(); + const semiColonIndex = trimmed.indexOf(";"); + return semiColonIndex !== -1 && semiColonIndex !== trimmed.length - 1; +} + +/** + * Split an SQLQuery into an array of statements + */ +export default function splitSqlQuery(sql: string): string[] { + if (!mayContainMultipleStatements(sql)) return [sql]; + const split = splitSqlIntoStatements(sql); + if (split.length === 0) { + return [sql]; + } else { + return split; + } +} + +function splitSqlIntoStatements(sql: string): string[] { + const statements: string[] = []; + let str = ""; + const compoundStatementStack: ((s: string) => boolean)[] = []; + + const iterator = sql[Symbol.iterator](); + let next = iterator.next(); + while (!next.done) { + const char = next.value; + + if (compoundStatementStack[0]?.(str + char)) { + compoundStatementStack.shift(); + } + + switch (char) { + case `'`: + case `"`: + case "`": + str += char + consumeUntilMarker(iterator, char); + break; + case `$`: { + const dollarQuote = + "$" + consumeWhile(iterator, isDollarQuoteIdentifier); + str += dollarQuote; + if (dollarQuote.endsWith("$")) { + str += consumeUntilMarker(iterator, dollarQuote); + } + break; + } + case `-`: + str += char; + next = iterator.next(); + if (!next.done && next.value === "-") { + str += next.value + consumeUntilMarker(iterator, "\n"); + break; + } else { + continue; + } + case `/`: + str += char; + next = iterator.next(); + if (!next.done && next.value === "*") { + str += next.value + consumeUntilMarker(iterator, "*/"); + break; + } else { + continue; + } + case `;`: + if (compoundStatementStack.length === 0) { + statements.push(str); + str = ""; + } else { + str += char; + } + break; + default: + str += char; + break; + } + + if (isCompoundStatementStart(str)) { + compoundStatementStack.unshift(isCompoundStatementEnd); + } + + next = iterator.next(); + } + statements.push(str); + + return statements + .map((statement) => statement.trim()) + .filter((statement) => statement.length > 0); +} + +/** + * Pulls characters from the string iterator while the predicate remains true. + */ +function consumeWhile( + iterator: Iterator, + predicate: (str: string) => boolean +) { + let next = iterator.next(); + let str = ""; + while (!next.done) { + str += next.value; + if (!predicate(str)) { + break; + } + next = iterator.next(); + } + return str; +} + +/** + * Pulls characters from the string iterator until the `endMarker` is found. + */ +function consumeUntilMarker(iterator: Iterator, endMarker: string) { + return consumeWhile(iterator, (str) => !str.endsWith(endMarker)); +} + +/** + * Returns true if the `str` ends with a dollar-quoted string marker. + * See https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-DOLLAR-QUOTING. + */ +function isDollarQuoteIdentifier(str: string) { + const lastChar = str.slice(-1); + return ( + // The $ marks the end of the identifier + lastChar !== "$" && + // we allow numbers, underscore and letters with diacritical marks + (/[0-9_]/i.test(lastChar) || + lastChar.toLowerCase() !== lastChar.toUpperCase()) + ); +} + +/** + * Returns true if the `str` ends with a compound statement `BEGIN` marker. + */ +function isCompoundStatementStart(str: string) { + return /\sBEGIN\s$/.test(str); +} + +/** + * Returns true if the `str` ends with a compound statement `END` marker. + */ +function isCompoundStatementEnd(str: string) { + return /\sEND[;\s]$/.test(str); +} diff --git a/packages/d1/src/statement.ts b/packages/d1/src/statement.ts deleted file mode 100644 index a88c3151a..000000000 --- a/packages/d1/src/statement.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { performance } from "node:perf_hooks"; -import type { - Database as SqliteDB, - Statement as SqliteStatement, -} from "better-sqlite3"; - -export type BindParams = any[] | [Record]; - -function errorWithCause(message: string, e: unknown) { - // @ts-ignore Errors have causes now, why don't you know this Typescript? - return new Error(message, { cause: e }); -} - -export class Statement { - readonly #db: SqliteDB; - readonly #query: string; - readonly #bindings: BindParams | undefined; - - constructor(db: SqliteDB, query: string, bindings?: BindParams) { - this.#db = db; - this.#query = query; - this.#bindings = bindings; - } - - // Lazily accumulate binding instructions, because ".bind" in better-sqlite3 - // is a real action that means the query must be valid when it's written, - // not when it's about to be executed (i.e. in a batch). - bind(...params: BindParams) { - // Adopting better-sqlite3 behaviour—once bound, a statement cannot be bound again - if (this.#bindings !== undefined) { - throw new TypeError( - "The bind() method can only be invoked once per statement object" - ); - } - return new Statement(this.#db, this.#query, params); - } - - #prepareAndBind() { - const prepared = this.#db.prepare(this.#query); - if (this.#bindings === undefined) return prepared; - try { - return prepared.bind(this.#bindings); - } catch (e) { - // For statements using ?1 ?2, etc, we want to pass them as varargs but - // "better" sqlite3 wants them as an object of {1: params[0], 2: params[1], ...} - if (this.#bindings.length > 0 && typeof this.#bindings[0] !== "object") { - return prepared.bind( - Object.fromEntries(this.#bindings.map((v, i) => [i + 1, v])) - ); - } else { - throw e; - } - } - } - - async all() { - const start = performance.now(); - const statementWithBindings = this.#prepareAndBind(); - try { - const results = this.#all(statementWithBindings); - return { - results, - duration: performance.now() - start, - lastRowId: null, - changes: null, - success: true, - served_by: "x-miniflare.db3", - }; - } catch (e) { - throw errorWithCause("D1_ALL_ERROR", e); - } - } - - #all(statementWithBindings: SqliteStatement) { - try { - return statementWithBindings.all(); - } catch (e: unknown) { - // This is the quickest/simplest way I could find to return results by - // default, falling back to .run() - if ( - /This statement does not return data\. Use run\(\) instead/.exec( - (e as Error).message - ) - ) { - return this.#run(statementWithBindings); - } - throw e; - } - } - - async first(col?: string) { - const statementWithBindings = this.#prepareAndBind(); - try { - const data = this.#first(statementWithBindings); - return typeof col === "string" ? data[col] : data; - } catch (e) { - throw errorWithCause("D1_FIRST_ERROR", e); - } - } - - #first(statementWithBindings: SqliteStatement) { - return statementWithBindings.get(); - } - - async run() { - const start = performance.now(); - const statementWithBindings = this.#prepareAndBind(); - try { - const { changes, lastInsertRowid } = this.#run(statementWithBindings); - return { - results: null, - duration: performance.now() - start, - lastRowId: lastInsertRowid, - changes, - success: true, - served_by: "x-miniflare.db3", - }; - } catch (e) { - throw errorWithCause("D1_RUN_ERROR", e); - } - } - - #run(statementWithBindings: SqliteStatement) { - return statementWithBindings.run(); - } - - async raw() { - const statementWithBindings = this.#prepareAndBind(); - return this.#raw(statementWithBindings); - } - - #raw(statementWithBindings: SqliteStatement) { - return statementWithBindings.raw().all(); - } -} diff --git a/packages/d1/test/d1js.spec.ts b/packages/d1/test/d1js.spec.ts new file mode 100644 index 000000000..c502e0b56 --- /dev/null +++ b/packages/d1/test/d1js.spec.ts @@ -0,0 +1,337 @@ +import assert from "assert"; +import fs from "fs/promises"; +import path from "path"; +import { D1Database, D1DatabaseAPI } from "@miniflare/d1"; +import { Storage, createSQLiteDB } from "@miniflare/shared"; +import { testClock, useTmp, utf8Encode } from "@miniflare/shared-test"; +import { MemoryStorage } from "@miniflare/storage-memory"; +import anyTest, { TestInterface } from "ava"; + +interface Context { + storage: Storage; + db: D1Database; +} + +const test = anyTest as TestInterface; + +const COLOUR_SCHEMA = + "CREATE TABLE colours (id INTEGER PRIMARY KEY, name TEXT NOT NULL, rgb INTEGER NOT NULL);"; +interface ColourRow { + id: number; + name: string; + rgb: number; +} + +const KITCHEN_SINK_SCHEMA = + "CREATE TABLE kitchen_sink (id INTEGER PRIMARY KEY, int INTEGER, real REAL, text TEXT, blob BLOB);"; +interface KitchenSinkRow { + id: number; + int: number | null; + real: number | null; + text: string | null; + blob: number[] | null; +} + +test.beforeEach(async (t) => { + const storage = new MemoryStorage(undefined, testClock); + const sqliteDb = await storage.getSqliteDatabase(); + + // Seed data using `better-sqlite3` APIs + sqliteDb.exec(COLOUR_SCHEMA); + sqliteDb.exec(KITCHEN_SINK_SCHEMA); + const insertColour = sqliteDb.prepare( + "INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?)" + ); + insertColour.run(1, "red", 0xff0000); + insertColour.run(2, "green", 0x00ff00); + insertColour.run(3, "blue", 0x0000ff); + + const db = new D1Database(new D1DatabaseAPI(sqliteDb)); + t.context = { storage, db }; +}); + +function throwCause(promise: Promise): Promise { + return promise.catch((error) => { + assert.strictEqual(error.message, "D1_ERROR"); + assert.notStrictEqual(error.cause, undefined); + throw error.cause; + }); +} + +test("D1Database: dump", async (t) => { + const { db } = t.context; + const tmp = await useTmp(t); + const buffer = await db.dump(); + + // Load the dumped data as an SQLite database and try query it + const tmpPath = path.join(tmp, "db.sqlite3"); + await fs.writeFile(tmpPath, new Uint8Array(buffer)); + const sqliteDb = await createSQLiteDB(tmpPath); + const results = sqliteDb.prepare("SELECT name FROM colours").all(); + t.deepEqual(results, [{ name: "red" }, { name: "green" }, { name: "blue" }]); +}); +test("D1Database: batch", async (t) => { + const { db } = t.context; + + const insert = db.prepare( + "INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?)" + ); + const batchResults = await db.batch>([ + insert.bind(4, "yellow", 0xffff00), + db.prepare("SELECT name FROM colours"), + ]); + t.is(batchResults.length, 2); + t.true(batchResults[0].success); + t.deepEqual(batchResults[0].results, []); + t.true(batchResults[1].success); + const expectedResults = [ + { name: "red" }, + { name: "green" }, + { name: "blue" }, + { name: "yellow" }, + ]; + t.deepEqual(batchResults[1].results, expectedResults); + + // Check error mid-batch rolls-back entire batch + const badInsert = db.prepare( + "PUT IN colours (id, name, rgb) VALUES (?, ?, ?)" + ); + await t.throwsAsync( + throwCause( + db.batch([ + insert.bind(5, "purple", 0xff00ff), + badInsert.bind(6, "blurple", 0x5865f2), + insert.bind(7, "cyan", 0x00ffff), + ]) + ), + { message: /syntax error/ } + ); + const result = await db + .prepare("SELECT name FROM colours") + .all>(); + t.deepEqual(result.results, expectedResults); +}); +test("D1Database: exec", async (t) => { + const { db } = t.context; + + // Check with single statement + let execResult = await db.exec( + "UPDATE colours SET name = 'Red' WHERE name = 'red'" + ); + t.is(execResult.count, 1); + t.true(execResult.duration > 0); + let result = await db + .prepare("SELECT name FROM colours WHERE name = 'Red'") + .all>(); + t.deepEqual(result.results, [{ name: "Red" }]); + + // Check with multiple statements + const statements = [ + "UPDATE colours SET name = 'Green' WHERE name = 'green'", + "UPDATE colours SET name = 'Blue' WHERE name = 'blue'", + ].join("\n"); + execResult = await db.exec(statements); + t.is(execResult.count, 2); + t.true(execResult.duration > 0); + result = await db.prepare("SELECT name FROM colours").all(); + t.deepEqual(result.results, [ + { name: "Red" }, + { name: "Green" }, + { name: "Blue" }, + ]); +}); + +test("D1PreparedStatement: bind", async (t) => { + const { db } = t.context; + + // Check with all parameter types + const blob = utf8Encode("Walshy"); + const blobArray = Array.from(blob); + await db + .prepare( + "INSERT INTO kitchen_sink (id, int, real, text, blob) VALUES (?, ?, ?, ?, ?)" + ) + .bind(1, 42, 3.141, "🙈", blob) + .run(); + let result = await db + .prepare("SELECT * FROM kitchen_sink") + .all(); + t.deepEqual(result.results, [ + { id: 1, int: 42, real: 3.141, text: "🙈", blob: blobArray }, + ]); + + // Check with null values + await db.prepare("UPDATE kitchen_sink SET blob = ?").bind(null).run(); + result = await db.prepare("SELECT * FROM kitchen_sink").all(); + t.deepEqual(result.results, [ + { id: 1, int: 42, real: 3.141, text: "🙈", blob: null }, + ]); + + // Check with multiple statements (should only bind first) + const colourResults = await db + .prepare( + "SELECT * FROM colours WHERE name = ?; SELECT * FROM colours WHERE id = ?;" + ) + .bind("green") + .all(); + t.is(colourResults.results?.length, 1); +}); + +// Lots of strange edge cases here... + +test("D1PreparedStatement: first", async (t) => { + const { db } = t.context; + + // Check with read statement + const select = await db.prepare("SELECT * FROM colours"); + let result = await select.first(); + t.deepEqual(result, { id: 1, name: "red", rgb: 0xff0000 }); + let id = await select.first("id"); + t.is(id, 1); + + // Check with multiple statements (should only match on first statement) + result = await db + .prepare( + "SELECT * FROM colours WHERE name = 'none'; SELECT * FROM colours WHERE id = 1;" + ) + .first(); + t.is(result, null); + + // Check with write statement (should actually execute statement) + result = await db + .prepare("INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?)") + .bind(4, "yellow", 0xffff00) + .first(); + t.is(result, null); + id = await db + .prepare("SELECT id FROM colours WHERE name = ?") + .bind("yellow") + .first("id"); + t.is(id, 4); +}); +test("D1PreparedStatement: run", async (t) => { + const { db } = t.context; + + // Check with read statement + await t.throwsAsync(throwCause(db.prepare("SELECT * FROM colours").run()), { + message: /Execute returned results - did you mean to call query\?/, + }); + // Check with read/write statement + await t.throwsAsync( + throwCause( + db + .prepare( + "INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?) RETURNING *" + ) + .bind(4, "yellow", 0xffff00) + .run() + ), + { message: /Execute returned results - did you mean to call query\?/ } + ); + + // Check with multiple statements (should only execute first statement) + let result = await db + .prepare( + "INSERT INTO kitchen_sink (id) VALUES (1); INSERT INTO kitchen_sink (id) VALUES (2);" + ) + .run(); + t.true(result.success); + const results = await db + .prepare("SELECT id FROM kitchen_sink") + .all>(); + t.deepEqual(results.results, [{ id: 1 }]); + + // Check with write statement + result = await db + .prepare("INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?)") + .bind(4, "yellow", 0xffff00) + .run(); + t.true(result.meta.duration > 0); + t.deepEqual(result, { + results: [], + success: true, + meta: { + // Don't know duration, so just match on returned value asserted > 0 + duration: result.meta.duration, + last_row_id: null, + changes: null, + served_by: "miniflare.db", + internal_stats: null, + }, + }); +}); +test("D1PreparedStatement: all", async (t) => { + const { db } = t.context; + + // Check with read statement + let result = await db.prepare("SELECT * FROM colours").all(); + t.true(result.meta.duration > 0); + t.deepEqual(result, { + results: [ + { id: 1, name: "red", rgb: 0xff0000 }, + { id: 2, name: "green", rgb: 0x00ff00 }, + { id: 3, name: "blue", rgb: 0x0000ff }, + ], + success: true, + meta: { + // Don't know duration, so just match on returned value asserted > 0 + duration: result.meta.duration, + last_row_id: null, + changes: null, + served_by: "miniflare.db", + internal_stats: null, + }, + }); + + // Check with multiple statements (should only return first statement results) + result = await db + .prepare( + "SELECT * FROM colours WHERE id = 1; SELECT * FROM colours WHERE id = 3;" + ) + .all(); + t.deepEqual(result.results, [{ id: 1, name: "red", rgb: 0xff0000 }]); + + // Check with write statement (should actually execute, but return nothing) + result = await db + .prepare("INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?)") + .bind(4, "yellow", 0xffff00) + .all(); + t.deepEqual(result.results, []); + const id = await db + .prepare("SELECT id FROM colours WHERE name = ?") + .bind("yellow") + .first("id"); + t.is(id, 4); +}); +test("D1PreparedStatement: raw", async (t) => { + const { db } = t.context; + + // Check with read statement + type RawColourRow = [/* id */ number, /* name */ string, /* rgb*/ number]; + let results = await db.prepare("SELECT * FROM colours").raw(); + t.deepEqual(results, [ + [1, "red", 0xff0000], + [2, "green", 0x00ff00], + [3, "blue", 0x0000ff], + ]); + + // Check with multiple statements (should only return first statement results) + results = await db + .prepare( + "SELECT * FROM colours WHERE id = 1; SELECT * FROM colours WHERE id = 3;" + ) + .raw(); + t.deepEqual(results, [[1, "red", 0xff0000]]); + + // Check with write statement (should actually execute, but return nothing) + results = await db + .prepare("INSERT INTO colours (id, name, rgb) VALUES (?, ?, ?)") + .bind(4, "yellow", 0xffff00) + .raw(); + t.deepEqual(results, []); + const id = await db + .prepare("SELECT id FROM colours WHERE name = ?") + .bind("yellow") + .first("id"); + t.is(id, 4); +}); diff --git a/packages/d1/test/database.spec.ts b/packages/d1/test/database.spec.ts deleted file mode 100644 index cade96f8b..000000000 --- a/packages/d1/test/database.spec.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { BetaDatabase } from "@miniflare/d1"; -import { Storage } from "@miniflare/shared"; -import { testClock } from "@miniflare/shared-test"; -import { MemoryStorage } from "@miniflare/storage-memory"; -import anyTest, { TestInterface } from "ava"; - -interface Context { - storage: Storage; - db: BetaDatabase; -} - -const test = anyTest as TestInterface; - -test.beforeEach(async (t) => { - const storage = new MemoryStorage(undefined, testClock); - const db = new BetaDatabase(await storage.getSqliteDatabase()); - t.context = { storage, db }; -}); - -test("batch, prepare & all", async (t) => { - const { db } = t.context; - - await db.batch([ - db.prepare( - `CREATE TABLE my_table (cid INTEGER PRIMARY KEY, name TEXT NOT NULL);` - ), - ]); - const response = await db.prepare(`SELECT * FROM sqlite_schema`).all(); - t.deepEqual(Object.keys(response), [ - "results", - "duration", - "lastRowId", - "changes", - "success", - "served_by", - ]); - t.deepEqual(response.results, [ - { - type: "table", - name: "my_table", - tbl_name: "my_table", - rootpage: 2, - sql: "CREATE TABLE my_table (cid INTEGER PRIMARY KEY, name TEXT NOT NULL)", - }, - ]); -}); diff --git a/packages/web-sockets/src/fetch.ts b/packages/web-sockets/src/fetch.ts index 751bce3e8..0d23d377a 100644 --- a/packages/web-sockets/src/fetch.ts +++ b/packages/web-sockets/src/fetch.ts @@ -13,7 +13,7 @@ import StandardWebSocket from "ws"; import { WebSocketPair, coupleWebSocket } from "./websocket"; export async function upgradingFetch( - this: Dispatcher | void, + this: Dispatcher | unknown, input: RequestInfo, init?: RequestInit ): Promise { diff --git a/packages/web-sockets/src/plugin.ts b/packages/web-sockets/src/plugin.ts index 6c7d71050..44bc2d27f 100644 --- a/packages/web-sockets/src/plugin.ts +++ b/packages/web-sockets/src/plugin.ts @@ -41,7 +41,6 @@ export class WebSocketPlugin extends Plugin { } fetch = async (input: RequestInfo, init?: RequestInit): Promise => { - // @ts-expect-error `this` is correctly bound in the plugin constructor const response = await this.#upgradingFetch(input, init); if (response.webSocket) this.#webSockets.add(response.webSocket); return response;