diff --git a/public/client.js b/public/client.js index 78e8572d9..a27b5b039 100644 --- a/public/client.js +++ b/public/client.js @@ -8,6 +8,13 @@ const attachedFiles = new Map(); const resolveFile = (name) => attachedFiles.get(name); main.builtin("FileAttachment", runtime.fileAttachments(resolveFile)); +const databaseTokens = new Map(); +async function resolveDatabaseToken(name) { + const token = databaseTokens.get(name); + if (!token) throw new Error(`Database configuration for ${name} not found`); + return token; +} + const cellsById = new Map(); const Generators = library.Generators; @@ -51,6 +58,7 @@ function Mutable() { // loading the library twice). Also, it’s nice to avoid require! function recommendedLibraries() { return { + DatabaseClient: () => import("./database.js").then((db) => db.makeDatabaseClient(resolveDatabaseToken)), d3: () => import("npm:d3"), htl: () => import("npm:htl"), html: () => import("npm:htl").then((htl) => htl.html), @@ -71,7 +79,7 @@ function recommendedLibraries() { } export function define(cell) { - const {id, inline, inputs = [], outputs = [], files = [], body} = cell; + const {id, inline, inputs = [], outputs = [], files = [], databases = [], body} = cell; const variables = []; cellsById.get(id)?.variables.forEach((v) => v.delete()); cellsById.set(id, {cell, variables}); @@ -108,6 +116,7 @@ export function define(cell) { variables.push(v); for (const o of outputs) variables.push(main.define(o, [`cell ${id}`], (exports) => exports[o])); for (const f of files) attachedFiles.set(f.name, {url: String(new URL(`/_file/${f.name}`, location)), mimeType: f.mimeType}); // prettier-ignore + for (const d of databases) databaseTokens.set(d.name, d); } export function open({hash} = {}) { @@ -164,6 +173,7 @@ export function open({hash} = {}) { inline: item.inline, inputs: item.inputs, outputs: item.outputs, + databases: item.databases, files: item.files, body: (0, eval)(item.body) }); diff --git a/public/database.js b/public/database.js new file mode 100644 index 000000000..d7ceefde4 --- /dev/null +++ b/public/database.js @@ -0,0 +1,197 @@ +export function makeDatabaseClient(resolveToken) { + return function DatabaseClient(name) { + if (new.target !== undefined) throw new TypeError("DatabaseClient is not a constructor"); + return resolveToken((name += "")).then((token) => new DatabaseClientImpl(name, token)); + }; +} + +class DatabaseClientImpl { + #token; + + constructor(name, token) { + this.name = name; + this.#token = token; + } + async query(sql, params, {signal} = {}) { + const queryUrl = new URL("/query", this.#token.url).toString(); + const response = await fetch(queryUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${this.#token.token}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({sql, params}), + signal + }); + if (!response.ok) { + if (response.status === 401) { + throw new Error("Unauthorized: invalid or expired token. Try again?"); + } + const contentType = response.headers.get("content-type"); + throw new Error( + contentType && contentType.startsWith("application/json") + ? (await response.json()).message + : await response.text() + ); + } + const {data, schema: jsonSchema} = await response.json(); + + const schema = parseJsonSchema(jsonSchema); + if (schema) { + coerce(schema, data); + Object.defineProperty(data, "schema", { + value: schema, + writable: true + }); + } + + return data; + } + + queryTag(strings, ...args) { + switch (this.type) { + case "oracle": + case "databricks": + return [strings.reduce((prev, curr, i) => `${prev}:${i}${curr}`), args]; + case "mysql": + return [strings.reduce((prev, curr, i) => `${prev}@${i}${curr}`), args]; + case "postgres": + return [strings.reduce((prev, curr, i) => `${prev}$${i}${curr}`), args]; + } + return [strings.join("?"), args]; + } + + async sql() { + return this.query(...this.queryTag.apply(this, arguments)); + } +} + +function coerceBuffer(d) { + return Uint8Array.from(d.data).buffer; +} + +function coerceDate(d) { + return new Date(d); +} + +function coerceBigInt(d) { + return BigInt(d); +} + +function coercer(schema) { + const mappings = schema + .map(({name, type}) => { + switch (type) { + case "buffer": + return [name, coerceBuffer]; + case "date": + return [name, coerceDate]; + case "bigint": + return [name, coerceBigInt]; + } + }) + .filter((d) => d); + return (data) => { + for (const [column, coerce] of mappings) { + for (const row of data) { + if (row[column] != null) { + row[column] = coerce(row[column]); + } + } + } + return data; + }; +} + +function coerce(schema, data) { + return coercer(schema)(data); +} + +// The data connector returns certain types as "database types" that we want to +// treat as (JavaScript) types. +function jsType(type, typeFlag) { + if ( + (type === "string" && typeFlag === "date") || + (type === "object" && typeFlag === "buffer") + // (type === "string" && typeFlag === "bigint") // TODO coerce bigints + ) { + return typeFlag; + } + return type; +} + +function parseType(typeOptions) { + let type; + let nullable; + if (Array.isArray(typeOptions)) { + // type: ["string"] (not nullable) + // type: ["null", "string"] (nullable) + type = typeOptions.find((t) => t !== "null") ?? "other"; + nullable = typeOptions.some((t) => t === "null"); + } else { + // type: "string" (not nullable) + type = typeOptions; + nullable = false; + } + return {type, nullable}; +} + +/** + * This function parses a JSON schema object into an array of objects, matching + * the "column set schema" structure defined in the DatabaseClient + * specification. It does not support nested types (e.g. array element types, + * object property types), but may do so in the future. + * https://observablehq.com/@observablehq/database-client-specification + * + * For example, this JSON schema object: + * { + * type: "array", + * items: { + * type: "object", + * properties: { + * TrackId: { type: "integer" }, + * Name: { type: "string", varchar: true }, + * AlbumId: { type: "number", long: true }, + * GenreId: { type: "array" }, + * } + * } + * } + * + * will be parsed into this column set schema: + * + * [ + * {name: "TrackId", type: "integer"}, + * {name: "Name", type: "string", databaseType: "varchar"}, + * {name: "AlbumId", type: "number", databaseType: "long"}, + * {name: "GenreId", type: "array"} + * ] + */ +function parseJsonSchema(schema) { + if (schema?.type !== "array" || schema.items?.type !== "object" || schema.items.properties === undefined) { + return []; + } + return Object.entries(schema.items.properties).map(([name, {type: typeOptions, ...rest}]) => { + const {type, nullable} = parseType(typeOptions); + let typeFlag; + + // The JSON Schema representation used by the data connector includes some + // arbitrary additional boolean properties to indicate the database type, + // such as {type: ["null", "string"], date: true}. This code is a little + // bit dangerous because the first of ANY exactly true property will be + // considered the database type; for example, we must be careful to ignore + // {type: "object", properties: {…}} and {type: "array", items: {…}}. + for (const key in rest) { + if (rest[key] === true) { + typeFlag = key; + break; + } + } + + return { + name, + type: jsType(type, typeFlag), + nullable, + ...(typeFlag && {databaseType: typeFlag}) + }; + }); +} diff --git a/src/build.ts b/src/build.ts index a48ddd208..12093b15d 100644 --- a/src/build.ts +++ b/src/build.ts @@ -6,6 +6,7 @@ import {parseArgs} from "node:util"; import {visitFiles, visitMarkdownFiles} from "./files.js"; import {readPages} from "./navigation.js"; import {renderServerless} from "./render.js"; +import {makeCLIResolver} from "./resolver.js"; const EXTRA_FILES = new Map([["node_modules/@observablehq/runtime/dist/runtime.js", "_observablehq/runtime.js"]]); @@ -20,12 +21,18 @@ async function build(context: CommandContext) { // Render .md files, building a list of file attachments as we go. const pages = await readPages(sourceRoot); const files: string[] = []; + const resolver = await makeCLIResolver(); for await (const sourceFile of visitMarkdownFiles(sourceRoot)) { const sourcePath = join(sourceRoot, sourceFile); const outputPath = join(outputRoot, join(dirname(sourceFile), basename(sourceFile, ".md") + ".html")); console.log("render", sourcePath, "→", outputPath); const path = `/${join(dirname(sourceFile), basename(sourceFile, ".md"))}`; - const render = renderServerless(await readFile(sourcePath, "utf-8"), {root: sourceRoot, path, pages}); + const render = renderServerless(await readFile(sourcePath, "utf-8"), { + root: sourceRoot, + path, + pages, + resolver + }); files.push(...render.files.map((f) => join(sourceFile, "..", f.name))); await prepareOutput(outputPath); await writeFile(outputPath, render.html); diff --git a/src/error.ts b/src/error.ts index 7c89b9f07..33fbdc4ff 100644 --- a/src/error.ts +++ b/src/error.ts @@ -1,7 +1,7 @@ export class HttpError extends Error { public readonly statusCode: number; - constructor(message: string, statusCode: number, cause?: Error) { + constructor(message: string, statusCode: number, cause?: any) { super(message ?? `HTTP status ${statusCode}`, cause); this.statusCode = statusCode; Error.captureStackTrace(this, HttpError); diff --git a/src/javascript.ts b/src/javascript.ts index f205071b3..aaa17f5fa 100644 --- a/src/javascript.ts +++ b/src/javascript.ts @@ -11,6 +11,10 @@ import {findImports, rewriteImports} from "./javascript/imports.js"; import {findReferences} from "./javascript/references.js"; import {Sourcemap} from "./sourcemap.js"; +export interface DatabaseReference { + name: string; +} + export interface FileReference { name: string; mimeType: string | null; @@ -26,6 +30,7 @@ export interface Transpile { outputs?: string[]; inline?: boolean; body: string; + databases?: DatabaseReference[]; files?: FileReference[]; imports?: ImportReference[]; } @@ -42,6 +47,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans const {root, id} = options; try { const node = parseJavaScript(input, options); + const databases = node.features.filter((f) => f.type === "DatabaseClient").map((f) => ({name: f.name})); const files = node.features .filter((f) => f.type === "FileAttachment") .filter((f) => canReadSync(join(root, f.name))) @@ -61,6 +67,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans ...(inputs.length ? {inputs} : null), ...(options.inline ? {inline: true} : null), ...(node.declarations?.length ? {outputs: node.declarations.map(({name}) => name)} : null), + ...(databases.length ? {databases} : null), ...(files.length ? {files} : null), body: `${node.async ? "async " : ""}(${inputs}) => { ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""} diff --git a/src/preview.ts b/src/preview.ts index af00c778e..0c0079ff3 100644 --- a/src/preview.ts +++ b/src/preview.ts @@ -12,6 +12,8 @@ import type {ParseResult} from "./markdown.js"; import {diffMarkdown, readMarkdown} from "./markdown.js"; import {readPages} from "./navigation.js"; import {renderPreview} from "./render.js"; +import type {CellResolver} from "./resolver.js"; +import {makeCLIResolver} from "./resolver.js"; const publicRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "public"); @@ -21,6 +23,7 @@ class Server { readonly port: number; readonly hostname: string; readonly root: string; + private _resolver: CellResolver | undefined; constructor({port, hostname, root}: CommandContext) { this.port = port; @@ -33,6 +36,7 @@ class Server { } async start() { + this._resolver = await makeCLIResolver(); return new Promise((resolve) => { this._server.listen(this.port, this.hostname, resolve); }); @@ -85,7 +89,16 @@ class Server { // Anything else should 404; static files should be matched above. try { const pages = await readPages(this.root); // TODO cache? watcher? - res.end(renderPreview(await readFile(path + ".md", "utf-8"), {root: this.root, path: pathname, pages}).html); + res.end( + ( + await renderPreview(await readFile(path + ".md", "utf-8"), { + root: this.root, + path: pathname, + pages, + resolver: this._resolver! + }) + ).html + ); } catch (error) { if (!isNodeError(error) || error.code !== "ENOENT") throw error; // internal error throw new HttpError("Not found", 404); @@ -101,7 +114,7 @@ class Server { _handleConnection = (socket: WebSocket, req: IncomingMessage) => { if (req.url === "/_observablehq") { - handleWatch(socket, this.root); + handleWatch(socket, {root: this.root, resolver: this._resolver!}); } else { socket.close(); } @@ -122,7 +135,21 @@ class FileWatchers { } } -function handleWatch(socket: WebSocket, root: string) { +function resolveDiffs(diff: ReturnType, resolver: CellResolver): ReturnType { + for (const item of diff) { + if (item.type === "add") { + for (const addItem of item.items) { + if (addItem.type === "cell" && "databases" in addItem) { + Object.assign(addItem, resolver(addItem)); + } + } + } + } + return diff; +} + +function handleWatch(socket: WebSocket, options: {root: string; resolver: CellResolver}) { + const {root, resolver} = options; let markdownWatcher: FSWatcher | null = null; let attachmentWatcher: FileWatchers | null = null; console.log("socket open"); @@ -158,7 +185,7 @@ function handleWatch(socket: WebSocket, root: string) { case "change": { const updated = await readMarkdown(path, root); if (current.hash === updated.hash) break; - const diff = diffMarkdown(current, updated); + const diff = resolveDiffs(diffMarkdown(current, updated), resolver); send({type: "update", diff, previousHash: current.hash, updatedHash: updated.hash}); attachmentWatcher?.close(); attachmentWatcher = new FileWatchers(root, updated.parse.files, refreshAttachment(updated.parse)); diff --git a/src/render.ts b/src/render.ts index 521612ce2..c65c7f37a 100644 --- a/src/render.ts +++ b/src/render.ts @@ -1,5 +1,6 @@ import {computeHash} from "./hash.js"; import {type FileReference, type ImportReference} from "./javascript.js"; +import type {CellPiece} from "./markdown.js"; import {parseMarkdown, type ParseResult} from "./markdown.js"; export interface Render { @@ -12,6 +13,7 @@ export interface RenderOptions { root: string; path?: string; pages?: {path: string; name: string}[]; + resolver: (cell: CellPiece) => CellPiece; } export function renderPreview(source: string, options: RenderOptions): Render { @@ -33,8 +35,8 @@ export function renderServerless(source: string, options: RenderOptions): Render } export function renderDefineCell(cell) { - const {id, inline, inputs, outputs, files, body} = cell; - return `define({${Object.entries({id, inline, inputs, outputs, files}) + const {id, inline, inputs, outputs, files, body, databases} = cell; + return `define({${Object.entries({id, inline, inputs, outputs, files, databases}) .filter((arg) => arg[1] !== undefined) .map((arg) => `${arg[0]}: ${JSON.stringify(arg[1])}`) .join(", ")}, body: ${body}});\n`; @@ -44,7 +46,10 @@ type RenderInternalOptions = | {preview?: false; hash?: never} // serverless | {preview: true; hash: string}; // preview -function render(parseResult: ParseResult, {path, pages, preview, hash}: RenderOptions & RenderInternalOptions): string { +function render( + parseResult: ParseResult, + {path, pages, preview, hash, resolver}: RenderOptions & RenderInternalOptions +): string { const showSidebar = pages && pages.length > 1; const imports = getImportMap(parseResult); return ` @@ -61,11 +66,19 @@ ${Array.from(imports.values()) .concat(parseResult.imports.filter(({name}) => name.startsWith("./")).map(({name}) => `/_file/${name.slice(2)}`)) .map((href) => ``) .join("\n")} +${ + parseResult.cells.some((cell) => cell.databases?.length) + ? `` + : "" +} ${ parseResult.data ? ` diff --git a/src/resolver.ts b/src/resolver.ts new file mode 100644 index 000000000..62e490c0b --- /dev/null +++ b/src/resolver.ts @@ -0,0 +1,93 @@ +import {homedir} from "os"; +import {join} from "path"; +import {createHmac} from "node:crypto"; +import {readFile} from "node:fs/promises"; +import type {CellPiece} from "./markdown.js"; + +export type CellResolver = (cell: CellPiece) => CellPiece; + +export interface ResolvedDatabaseReference { + name: string; + origin: string; + token: string; + type: string; +} + +interface DatabaseProxyItem { + secret: string; +} + +type DatabaseProxyConfig = Record; + +interface ObservableConfig { + "database-proxy": DatabaseProxyConfig; +} + +interface DatabaseConfig { + host: string; + name: string; + origin?: string; + port: number; + secret: string; + ssl: "disabled" | "enabled"; + type: string; + url: string; +} + +const configFile = join(homedir(), ".observablehq"); +const key = `database-proxy`; + +export async function readDatabaseProxyConfig(): Promise { + const observableConfig = JSON.parse(await readFile(configFile, "utf-8")) as ObservableConfig | null; + return observableConfig && observableConfig[key]; +} + +function readDatabaseConfig(config: DatabaseProxyConfig | null, name): DatabaseConfig { + if (!config) throw new Error(`Missing database configuration file "${configFile}"`); + if (!name) throw new Error(`No database name specified`); + const raw = (config && config[name]) as DatabaseConfig | null; + if (!raw) throw new Error(`No configuration found for "${name}"`); + return { + ...decodeSecret(raw.secret), + url: raw.url + } as DatabaseConfig; +} + +function decodeSecret(secret: string): Record { + return JSON.parse(Buffer.from(secret, "base64").toString("utf8")); +} + +function encodeToken(payload: {name: string}, secret): string { + const data = JSON.stringify(payload); + const hmac = createHmac("sha256", Buffer.from(secret, "hex")).update(data).digest(); + return `${Buffer.from(data).toString("base64") + "." + Buffer.from(hmac).toString("base64")}`; +} + +export async function makeCLIResolver(): Promise { + const config = await readDatabaseProxyConfig(); + return (cell: CellPiece): CellPiece => { + if ("databases" in cell && cell.databases !== undefined) { + cell = { + ...cell, + databases: cell.databases.map((ref) => { + const db = readDatabaseConfig(config, ref.name); + if (db) { + const url = new URL("http://localhost"); + url.protocol = db.ssl !== "disabled" ? "https:" : "http:"; + url.host = db.host; + url.port = String(db.port); + url.toString(); + return { + ...ref, + token: encodeToken(ref, db.secret), + type: db.type, + url: url.toString() + }; + } + throw new Error(`Unable to resolve database "${ref.name}"`); + }) + }; + } + return cell; + }; +} diff --git a/test/input/databaseclient.md b/test/input/databaseclient.md new file mode 100644 index 000000000..0bf71354d --- /dev/null +++ b/test/input/databaseclient.md @@ -0,0 +1,3 @@ +```js +const db = DatabaseClient("dwh-local-proxy"); +``` diff --git a/test/output/databaseclient.html b/test/output/databaseclient.html new file mode 100644 index 000000000..74c94d305 --- /dev/null +++ b/test/output/databaseclient.html @@ -0,0 +1 @@ +
diff --git a/test/output/databaseclient.json b/test/output/databaseclient.json new file mode 100644 index 000000000..eaa94205b --- /dev/null +++ b/test/output/databaseclient.json @@ -0,0 +1,34 @@ +{ + "data": null, + "title": null, + "files": [], + "imports": [], + "pieces": [ + { + "type": "html", + "id": "", + "cellIds": [ + "1e6b99bd" + ], + "html": "
\n" + } + ], + "cells": [ + { + "type": "cell", + "id": "1e6b99bd", + "inputs": [ + "DatabaseClient" + ], + "outputs": [ + "db" + ], + "databases": [ + { + "name": "dwh-local-proxy" + } + ], + "body": "(DatabaseClient) => {\nconst db = DatabaseClient(\"dwh-local-proxy\");\nreturn {db};\n}" + } + ] +} \ No newline at end of file