From 0efed9d7deb82763917518b705ee408912f6595a Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Tue, 17 Oct 2023 17:06:00 -0700 Subject: [PATCH 1/8] Add DatabaseClient support --- public/client.js | 1 + public/database.js | 218 +++++++++++++++++++++++ src/database.ts | 76 ++++++++ src/error.ts | 2 +- src/javascript.ts | 7 + src/markdown.ts | 8 +- src/preview.ts | 3 + test/input/databaseclient.md | 3 + test/output/block-expression.json | 1 + test/output/databaseclient.html | 1 + test/output/databaseclient.json | 39 ++++ test/output/dollar-expression.json | 1 + test/output/double-quote-expression.json | 1 + test/output/embedded-expression.json | 1 + test/output/escaped-expression.json | 1 + test/output/fenced-code.json | 1 + test/output/heading-expression.json | 1 + test/output/hello-world.json | 1 + test/output/inline-expression.json | 1 + test/output/local-fetch.json | 1 + test/output/script-expression.json | 1 + test/output/single-quote-expression.json | 1 + test/output/template-expression.json | 1 + test/output/tex-expression.json | 1 + test/output/yaml-frontmatter.json | 1 + 25 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 public/database.js create mode 100644 src/database.ts create mode 100644 test/input/databaseclient.md create mode 100644 test/output/databaseclient.html create mode 100644 test/output/databaseclient.json diff --git a/public/client.js b/public/client.js index bd8b410ab..a2ec296b6 100644 --- a/public/client.js +++ b/public/client.js @@ -30,6 +30,7 @@ function width() { // loading the library twice). Also, it’s nice to avoid require! function recommendedLibraries() { return { + DatabaseClient: () => import("./database.js").then((m) => m.default), d3: () => import("npm:d3"), htl: () => import("npm:htl"), Plot: () => import("npm:@observablehq/plot"), diff --git a/public/database.js b/public/database.js new file mode 100644 index 000000000..718f5ec08 --- /dev/null +++ b/public/database.js @@ -0,0 +1,218 @@ +export default function DatabaseClient(name) { + if (new.target === undefined) { + return requestToken((name += "")).then(({type}) => { + return new DatabaseClientImpl(name, type); + }); + } + if (new.target === DatabaseClient) { + throw new TypeError("DatabaseClient is not a constructor"); + } + Object.defineProperties(this, { + name: {value: name, enumerable: true} + }); +} + +const tokenRequests = new Map(); + +async function requestToken(name) { + if (!name) throw new Error("Database name is not defined"); + if (tokenRequests.has(name)) { + return await tokenRequests.get(name); + } + const response = await fetch(`/_database/${name}/token`).then((response) => { + if (response.ok) { + return response.json().then((json) => json); + } + invalidateToken(name); + throw new Error(`Error ${response.statusText || response.status} creating database client '${name}':`); + }); + tokenRequests.set(name, response); + return response; +} + +function invalidateToken(name) { + tokenRequests.delete(name); +} + +class DatabaseClientImpl extends DatabaseClient { + async query(sql, params, {signal} = {}) { + const token = await requestToken(this.name); + const response = await fetch(`${token.origin}/query`, { + method: "POST", + headers: { + Authorization: `Bearer ${token.token}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({sql, params}), + signal + }); + if (!response.ok) { + if (response.status === 401) { + invalidateToken(this.name, token); // Force refresh on next query. + throw new Error("Unauthorized: invalid or expired token. Try again?"); + } + const contentType = response.headers.get("content-type"); + throw new Error( + contentType && contentType.startsWith("application/json") + ? (await response.json()).message + : await response.text() + ); + } + const {data, schema: jsonSchema} = await response.json(); + + const schema = parseJsonSchema(jsonSchema); + if (schema) { + coerce(schema, data); + Object.defineProperty(data, "schema", { + value: schema, + writable: true + }); + } + + return data; + } + + // TODO: Does this serve any purpose? + async queryRow(...params) { + return (await this.query(...params))[0] || null; + } + + queryTag(strings, ...args) { + // TODO: This is Database-dialect specific + return [strings.join("?"), args]; + } + + async sql() { + return this.query(...this.queryTag.apply(this, arguments)); + } +} + +function coerceBuffer(d) { + return Uint8Array.from(d.data).buffer; +} + +function coerceDate(d) { + return new Date(d); +} + +function coerceBigInt(d) { + return BigInt(d); +} + +function coercer(schema) { + const mappings = schema + .map(({name, type}) => { + switch (type) { + case "buffer": + return [name, coerceBuffer]; + case "date": + return [name, coerceDate]; + case "bigint": + return [name, coerceBigInt]; + } + }) + .filter((d) => d); + return (data) => { + for (const [column, coerce] of mappings) { + for (const row of data) { + if (row[column] != null) { + row[column] = coerce(row[column]); + } + } + } + return data; + }; +} + +function coerce(schema, data) { + return coercer(schema)(data); +} + +// The data connector returns certain types as "database types" that we want to +// treat as (JavaScript) types. +function jsType(type, typeFlag) { + if ( + (type === "string" && typeFlag === "date") || + (type === "object" && typeFlag === "buffer") + // (type === "string" && typeFlag === "bigint") // TODO coerce bigints + ) { + return typeFlag; + } + return type; +} + +function parseType(typeOptions) { + let type; + let nullable; + if (Array.isArray(typeOptions)) { + // type: ["string"] (not nullable) + // type: ["null", "string"] (nullable) + type = typeOptions.find((t) => t !== "null") ?? "other"; + nullable = typeOptions.some((t) => t === "null"); + } else { + // type: "string" (not nullable) + type = typeOptions; + nullable = false; + } + return {type, nullable}; +} + +/** + * This function parses a JSON schema object into an array of objects, matching + * the "column set schema" structure defined in the DatabaseClient + * specification. It does not support nested types (e.g. array element types, + * object property types), but may do so in the future. + * https://observablehq.com/@observablehq/database-client-specification + * + * For example, this JSON schema object: + * { + * type: "array", + * items: { + * type: "object", + * properties: { + * TrackId: { type: "integer" }, + * Name: { type: "string", varchar: true }, + * AlbumId: { type: "number", long: true }, + * GenreId: { type: "array" }, + * } + * } + * } + * + * will be parsed into this column set schema: + * + * [ + * {name: "TrackId", type: "integer"}, + * {name: "Name", type: "string", databaseType: "varchar"}, + * {name: "AlbumId", type: "number", databaseType: "long"}, + * {name: "GenreId", type: "array"} + * ] + */ +function parseJsonSchema(schema) { + if (schema?.type !== "array" || schema.items?.type !== "object" || schema.items.properties === undefined) { + return []; + } + return Object.entries(schema.items.properties).map(([name, {type: typeOptions, ...rest}]) => { + const {type, nullable} = parseType(typeOptions); + let typeFlag; + + // The JSON Schema representation used by the data connector includes some + // arbitrary additional boolean properties to indicate the database type, + // such as {type: ["null", "string"], date: true}. This code is a little + // bit dangerous because the first of ANY exactly true property will be + // considered the database type; for example, we must be careful to ignore + // {type: "object", properties: {…}} and {type: "array", items: {…}}. + for (const key in rest) { + if (rest[key] === true) { + typeFlag = key; + break; + } + } + + return { + name, + type: jsType(type, typeFlag), + nullable, + ...(typeFlag && {databaseType: typeFlag}) + }; + }); +} diff --git a/src/database.ts b/src/database.ts new file mode 100644 index 000000000..f50d3271b --- /dev/null +++ b/src/database.ts @@ -0,0 +1,76 @@ +import {homedir} from "os"; +import {join} from "path"; +import {createHmac} from "node:crypto"; +import {readFile} from "node:fs/promises"; +import {HttpError} from "./error.js"; + +// TODO: This should be configurable via flags. +const DATACONNECTOR_URL = "http://127.0.0.1:2899"; + +interface DatabaseConfig { + url: string; + secret: string; + type: string; +} + +const configFile = join(homedir(), ".observablehq"); +const key = `database-proxy`; + +async function readObservableConfig(): Promise { + const observableConfig = JSON.parse(await readFile(configFile, "utf-8")); + return observableConfig && observableConfig[key]; +} + +async function readDatabaseConfig(name): Promise { + try { + const config = await readObservableConfig(); + if (!name) throw new HttpError(`No database name specified`, 404); + const raw = (config && config[name]) as DatabaseConfig | null; + if (!raw) throw new HttpError(`No configuration found for "${name}"`, 404); + return { + ...decodeSecret(raw.secret), + url: raw.url + } as DatabaseConfig; + } catch (error) { + if (error instanceof HttpError) throw error; + console.log("error", error); + throw new HttpError(`Unable to read database configuration file "${configFile}"`, 404, error); + } +} + +function decodeSecret(secret: string): Record { + return JSON.parse(Buffer.from(secret, "base64").toString("utf8")); +} + +function encodeToken(payload: {name: string}, secret): string { + const data = JSON.stringify(payload); + const hmac = createHmac("sha256", Buffer.from(secret, "hex")).update(data).digest(); + return `${Buffer.from(data).toString("base64") + "." + Buffer.from(hmac).toString("base64")}`; +} + +// TODO: This would be better if it returned an input stream. +export async function handleDatabase(req, res, pathname) { + // Match /:name/:pathname + const match = /^\/([^/]+)(\/[^/]+)$/.exec(pathname); + if (match) { + const name = match[1]; + const pathname = match[2]; + const config = await readDatabaseConfig(name); + + if (pathname.startsWith("/token")) { + res.writeHead(200, {"Content-Type": "application/json"}); + res.end( + JSON.stringify({ + token: encodeToken({name}, config.secret), + type: config.type, + origin: DATACONNECTOR_URL + }) + ); + return; + } + return; + } + res.statusCode = 404; + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.end("Oops, an error occurred"); +} diff --git a/src/error.ts b/src/error.ts index 7c89b9f07..33fbdc4ff 100644 --- a/src/error.ts +++ b/src/error.ts @@ -1,7 +1,7 @@ export class HttpError extends Error { public readonly statusCode: number; - constructor(message: string, statusCode: number, cause?: Error) { + constructor(message: string, statusCode: number, cause?: any) { super(message ?? `HTTP status ${statusCode}`, cause); this.statusCode = statusCode; Error.captureStackTrace(this, HttpError); diff --git a/src/javascript.ts b/src/javascript.ts index ac9835c8d..d7b03e831 100644 --- a/src/javascript.ts +++ b/src/javascript.ts @@ -11,6 +11,10 @@ import {findImports, rewriteImports} from "./javascript/imports.js"; import {findReferences} from "./javascript/references.js"; import {Sourcemap} from "./sourcemap.js"; +export interface DatabaseReference { + name: string; +} + export interface FileReference { name: string; mimeType: string | null; @@ -26,6 +30,7 @@ export interface Transpile { outputs?: string[]; inline?: boolean; body: string; + databases?: DatabaseReference[]; files?: FileReference[]; imports?: ImportReference[]; } @@ -42,6 +47,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans const {root, id} = options; try { const node = parseJavaScript(input, options); + const databases = node.features.filter((f) => f.type === "DatabaseClient").map((f) => ({name: f.name})); const files = node.features .filter((f) => f.type === "FileAttachment") .filter((f) => canReadSync(join(root, f.name))) @@ -61,6 +67,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans ...(inputs.length ? {inputs} : null), ...(options.inline ? {inline: true} : null), ...(node.declarations?.length ? {outputs: node.declarations.map(({name}) => name)} : null), + ...(databases.length ? {databases} : null), ...(files.length ? {files} : null), body: `${node.async ? "async " : ""}(${inputs}) => { ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""} diff --git a/src/markdown.ts b/src/markdown.ts index 57138cf26..b6387090b 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -11,6 +11,7 @@ import {type default as Renderer, type RenderRule} from "markdown-it/lib/rendere import mime from "mime"; import {join} from "path"; import {canReadSync} from "./files.js"; +import type {DatabaseReference} from "./javascript.js"; import {transpileJavaScript, type FileReference, type ImportReference, type Transpile} from "./javascript.js"; import {computeHash} from "./hash.js"; @@ -31,6 +32,7 @@ export interface ParseResult { title: string | null; html: string; data: {[key: string]: any} | null; + databases: DatabaseReference[]; files: FileReference[]; imports: ImportReference[]; pieces: HtmlPiece[]; @@ -44,6 +46,7 @@ interface RenderPiece { interface ParseContext { pieces: RenderPiece[]; + databases: DatabaseReference[]; files: {name: string; mimeType: string | null}[]; imports: ImportReference[]; startLine: number; @@ -92,6 +95,7 @@ function makeFenceRenderer(root: string, baseRenderer: RenderRule): RenderRule { sourceLine: context.startLine + context.currentLine }); extendPiece(context, {code: [transpile]}); + if (transpile.databases) context.databases.push(...transpile.databases); if (transpile.files) context.files.push(...transpile.files); if (transpile.imports) context.imports.push(...transpile.imports); result += `
\n`; @@ -247,6 +251,7 @@ function makePlaceholderRenderer(root: string): RenderRule { sourceLine: context.startLine + context.currentLine }); extendPiece(context, {code: [transpile]}); + if (transpile.databases) context.databases.push(...transpile.databases); if (transpile.files) context.files.push(...transpile.files); return ``; }; @@ -344,13 +349,14 @@ export function parseMarkdown(source: string, root: string): ParseResult { md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!); md.renderer.rules.softbreak = makeSoftbreakRenderer(md.renderer.rules.softbreak!); md.renderer.render = renderIntoPieces(md.renderer); - const context: ParseContext = {files: [], imports: [], pieces: [], startLine: 0, currentLine: 0}; + const context: ParseContext = {databases: [], files: [], imports: [], pieces: [], startLine: 0, currentLine: 0}; const tokens = md.parse(parts.content, context); const html = md.renderer.render(tokens, md.options, context); return { html, data: isEmpty(parts.data) ? null : parts.data, title: parts.data?.title ?? findTitle(tokens) ?? null, + databases: context.databases, files: context.files, imports: context.imports, pieces: toParsePieces(context.pieces), diff --git a/src/preview.ts b/src/preview.ts index c3d87ac79..a8b0e514e 100644 --- a/src/preview.ts +++ b/src/preview.ts @@ -11,6 +11,7 @@ import {computeHash} from "./hash.js"; import {diffMarkdown, parseMarkdown} from "./markdown.js"; import {readPages} from "./navigation.js"; import {renderPreview} from "./render.js"; +import {handleDatabase} from "./database.js"; const publicRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "public"); @@ -48,6 +49,8 @@ class Server { send(req, pathname.slice("/_observablehq".length), {root: publicRoot}).pipe(res); } else if (pathname.startsWith("/_file/")) { send(req, pathname.slice("/_file".length), {root: this.root}).pipe(res); + } else if (pathname.startsWith("/_database/")) { + handleDatabase(req, res, pathname.slice("/_database".length)); } else { if (normalize(pathname).startsWith("..")) throw new Error("Invalid path: " + pathname); let path = join(this.root, pathname); diff --git a/test/input/databaseclient.md b/test/input/databaseclient.md new file mode 100644 index 000000000..0bf71354d --- /dev/null +++ b/test/input/databaseclient.md @@ -0,0 +1,3 @@ +```js +const db = DatabaseClient("dwh-local-proxy"); +``` diff --git a/test/output/block-expression.json b/test/output/block-expression.json index b6994baad..abe1ae629 100644 --- a/test/output/block-expression.json +++ b/test/output/block-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/databaseclient.html b/test/output/databaseclient.html new file mode 100644 index 000000000..74c94d305 --- /dev/null +++ b/test/output/databaseclient.html @@ -0,0 +1 @@ +
diff --git a/test/output/databaseclient.json b/test/output/databaseclient.json new file mode 100644 index 000000000..ab8a7bbb0 --- /dev/null +++ b/test/output/databaseclient.json @@ -0,0 +1,39 @@ +{ + "data": null, + "title": null, + "databases": [ + { + "name": "dwh-local-proxy" + } + ], + "files": [], + "imports": [], + "pieces": [ + { + "type": "html", + "id": "", + "cellIds": [ + "1e6b99bd" + ], + "html": "
\n" + } + ], + "cells": [ + { + "type": "cell", + "id": "1e6b99bd", + "inputs": [ + "DatabaseClient" + ], + "outputs": [ + "db" + ], + "databases": [ + { + "name": "dwh-local-proxy" + } + ], + "body": "(DatabaseClient) => {\nconst db = DatabaseClient(\"dwh-local-proxy\");\nreturn {db};\n}" + } + ] +} \ No newline at end of file diff --git a/test/output/dollar-expression.json b/test/output/dollar-expression.json index 4d2d6c123..42d47c221 100644 --- a/test/output/dollar-expression.json +++ b/test/output/dollar-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/double-quote-expression.json b/test/output/double-quote-expression.json index 4a79621f1..3f1e45d81 100644 --- a/test/output/double-quote-expression.json +++ b/test/output/double-quote-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/embedded-expression.json b/test/output/embedded-expression.json index 72317f4ce..8fea8ba5c 100644 --- a/test/output/embedded-expression.json +++ b/test/output/embedded-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": "Embedded expression", + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/escaped-expression.json b/test/output/escaped-expression.json index 0b9bd7976..e3ca16e55 100644 --- a/test/output/escaped-expression.json +++ b/test/output/escaped-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/fenced-code.json b/test/output/fenced-code.json index ea500a263..94069a8b6 100644 --- a/test/output/fenced-code.json +++ b/test/output/fenced-code.json @@ -1,6 +1,7 @@ { "data": null, "title": "Fenced code", + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/heading-expression.json b/test/output/heading-expression.json index d82812bc4..4ad59b8df 100644 --- a/test/output/heading-expression.json +++ b/test/output/heading-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/hello-world.json b/test/output/hello-world.json index 9b675fd5a..713e618e2 100644 --- a/test/output/hello-world.json +++ b/test/output/hello-world.json @@ -1,6 +1,7 @@ { "data": null, "title": "Hello, world!", + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/inline-expression.json b/test/output/inline-expression.json index 1401da533..415ef2bb0 100644 --- a/test/output/inline-expression.json +++ b/test/output/inline-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/local-fetch.json b/test/output/local-fetch.json index e362f25dc..2f7ec8748 100644 --- a/test/output/local-fetch.json +++ b/test/output/local-fetch.json @@ -1,6 +1,7 @@ { "data": null, "title": "Local fetch", + "databases": [], "files": [ { "name": "./local-fetch.md", diff --git a/test/output/script-expression.json b/test/output/script-expression.json index e9d3dfabb..4878a36af 100644 --- a/test/output/script-expression.json +++ b/test/output/script-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": "Script expression", + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/single-quote-expression.json b/test/output/single-quote-expression.json index 6f0fd6424..7f25166b3 100644 --- a/test/output/single-quote-expression.json +++ b/test/output/single-quote-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/template-expression.json b/test/output/template-expression.json index e0e55cca2..19b09ad90 100644 --- a/test/output/template-expression.json +++ b/test/output/template-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": null, + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/tex-expression.json b/test/output/tex-expression.json index 2111edd50..81c735f5e 100644 --- a/test/output/tex-expression.json +++ b/test/output/tex-expression.json @@ -1,6 +1,7 @@ { "data": null, "title": "Hello, ", + "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/yaml-frontmatter.json b/test/output/yaml-frontmatter.json index 75f092e4d..60acc1656 100644 --- a/test/output/yaml-frontmatter.json +++ b/test/output/yaml-frontmatter.json @@ -7,6 +7,7 @@ ] }, "title": "YAML", + "databases": [], "files": [], "imports": [], "pieces": [ From 7d6503a42dbd8e6c420f9f26128f665da51e5915 Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Tue, 24 Oct 2023 10:15:42 -0700 Subject: [PATCH 2/8] Implement a render-time resolver for databases --- public/client.js | 13 ++++++- public/database.js | 49 ++++++----------------- src/build.ts | 8 +++- src/database.ts | 76 ------------------------------------ src/markdown.ts | 10 +---- src/preview.ts | 38 ++++++++++++++---- src/render.ts | 16 ++++++-- src/resolver.ts | 97 ++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 172 insertions(+), 135 deletions(-) delete mode 100644 src/database.ts create mode 100644 src/resolver.ts diff --git a/public/client.js b/public/client.js index 01d4de109..c36217d5f 100644 --- a/public/client.js +++ b/public/client.js @@ -8,6 +8,13 @@ const attachedFiles = new Map(); const resolveFile = (name) => attachedFiles.get(name); main.builtin("FileAttachment", runtime.fileAttachments(resolveFile)); +const databaseTokens = new Map(); +const resolveDatabaseToken = (name) => { + const token = databaseTokens.get(name); + if (!token) Promise.reject(new Error(`Database configuration for ${name} not found`)); + return Promise.resolve(token); +}; + const cellsById = new Map(); const Generators = library.Generators; @@ -51,7 +58,7 @@ function Mutable() { // loading the library twice). Also, it’s nice to avoid require! function recommendedLibraries() { return { - DatabaseClient: () => import("./database.js").then((m) => m.default), + DatabaseClient: () => import("./database.js").then((db) => db.makeDatabaseClient(resolveDatabaseToken)), d3: () => import("npm:d3"), htl: () => import("npm:htl"), html: () => import("npm:htl").then((htl) => htl.html), @@ -72,7 +79,7 @@ function recommendedLibraries() { } export function define(cell) { - const {id, inline, inputs = [], outputs = [], files = [], body} = cell; + const {id, inline, inputs = [], outputs = [], files = [], databases = [], body} = cell; const variables = []; cellsById.get(id)?.variables.forEach((v) => v.delete()); cellsById.set(id, {cell, variables}); @@ -109,6 +116,7 @@ export function define(cell) { variables.push(v); for (const o of outputs) variables.push(main.define(o, [`cell ${id}`], (exports) => exports[o])); for (const f of files) attachedFiles.set(f.name, {url: String(new URL(`/_file/${f.name}`, location)), mimeType: f.mimeType}); // prettier-ignore + for (const d of databases) databaseTokens.set(d.name, d); } export function open({hash} = {}) { @@ -165,6 +173,7 @@ export function open({hash} = {}) { inline: item.inline, inputs: item.inputs, outputs: item.outputs, + databases: item.databases, files: item.files, body: (0, eval)(item.body) }); diff --git a/public/database.js b/public/database.js index 718f5ec08..4874f4666 100644 --- a/public/database.js +++ b/public/database.js @@ -1,46 +1,22 @@ -export default function DatabaseClient(name) { - if (new.target === undefined) { - return requestToken((name += "")).then(({type}) => { - return new DatabaseClientImpl(name, type); - }); - } - if (new.target === DatabaseClient) { - throw new TypeError("DatabaseClient is not a constructor"); - } - Object.defineProperties(this, { - name: {value: name, enumerable: true} - }); +export function makeDatabaseClient(resolveToken) { + return function DatabaseClient(name) { + if (new.target !== undefined) throw new TypeError("DatabaseClient is not a constructor"); + return resolveToken((name += "")).then((token) => new DatabaseClientImpl(name, token)); + }; } -const tokenRequests = new Map(); +class DatabaseClientImpl { + #token; -async function requestToken(name) { - if (!name) throw new Error("Database name is not defined"); - if (tokenRequests.has(name)) { - return await tokenRequests.get(name); + constructor(name, token) { + this.name = name; + this.#token = token; } - const response = await fetch(`/_database/${name}/token`).then((response) => { - if (response.ok) { - return response.json().then((json) => json); - } - invalidateToken(name); - throw new Error(`Error ${response.statusText || response.status} creating database client '${name}':`); - }); - tokenRequests.set(name, response); - return response; -} - -function invalidateToken(name) { - tokenRequests.delete(name); -} - -class DatabaseClientImpl extends DatabaseClient { async query(sql, params, {signal} = {}) { - const token = await requestToken(this.name); - const response = await fetch(`${token.origin}/query`, { + const response = await fetch(`${this.#token.url}query`, { method: "POST", headers: { - Authorization: `Bearer ${token.token}`, + Authorization: `Bearer ${this.#token.token}`, "Content-Type": "application/json" }, body: JSON.stringify({sql, params}), @@ -48,7 +24,6 @@ class DatabaseClientImpl extends DatabaseClient { }); if (!response.ok) { if (response.status === 401) { - invalidateToken(this.name, token); // Force refresh on next query. throw new Error("Unauthorized: invalid or expired token. Try again?"); } const contentType = response.headers.get("content-type"); diff --git a/src/build.ts b/src/build.ts index a48ddd208..a9a6a3d5d 100644 --- a/src/build.ts +++ b/src/build.ts @@ -6,6 +6,7 @@ import {parseArgs} from "node:util"; import {visitFiles, visitMarkdownFiles} from "./files.js"; import {readPages} from "./navigation.js"; import {renderServerless} from "./render.js"; +import {makeCLIResolver} from "./resolver.js"; const EXTRA_FILES = new Map([["node_modules/@observablehq/runtime/dist/runtime.js", "_observablehq/runtime.js"]]); @@ -25,7 +26,12 @@ async function build(context: CommandContext) { const outputPath = join(outputRoot, join(dirname(sourceFile), basename(sourceFile, ".md") + ".html")); console.log("render", sourcePath, "→", outputPath); const path = `/${join(dirname(sourceFile), basename(sourceFile, ".md"))}`; - const render = renderServerless(await readFile(sourcePath, "utf-8"), {root: sourceRoot, path, pages}); + const render = renderServerless(await readFile(sourcePath, "utf-8"), { + root: sourceRoot, + path, + pages, + resolver: await makeCLIResolver() + }); files.push(...render.files.map((f) => join(sourceFile, "..", f.name))); await prepareOutput(outputPath); await writeFile(outputPath, render.html); diff --git a/src/database.ts b/src/database.ts deleted file mode 100644 index f50d3271b..000000000 --- a/src/database.ts +++ /dev/null @@ -1,76 +0,0 @@ -import {homedir} from "os"; -import {join} from "path"; -import {createHmac} from "node:crypto"; -import {readFile} from "node:fs/promises"; -import {HttpError} from "./error.js"; - -// TODO: This should be configurable via flags. -const DATACONNECTOR_URL = "http://127.0.0.1:2899"; - -interface DatabaseConfig { - url: string; - secret: string; - type: string; -} - -const configFile = join(homedir(), ".observablehq"); -const key = `database-proxy`; - -async function readObservableConfig(): Promise { - const observableConfig = JSON.parse(await readFile(configFile, "utf-8")); - return observableConfig && observableConfig[key]; -} - -async function readDatabaseConfig(name): Promise { - try { - const config = await readObservableConfig(); - if (!name) throw new HttpError(`No database name specified`, 404); - const raw = (config && config[name]) as DatabaseConfig | null; - if (!raw) throw new HttpError(`No configuration found for "${name}"`, 404); - return { - ...decodeSecret(raw.secret), - url: raw.url - } as DatabaseConfig; - } catch (error) { - if (error instanceof HttpError) throw error; - console.log("error", error); - throw new HttpError(`Unable to read database configuration file "${configFile}"`, 404, error); - } -} - -function decodeSecret(secret: string): Record { - return JSON.parse(Buffer.from(secret, "base64").toString("utf8")); -} - -function encodeToken(payload: {name: string}, secret): string { - const data = JSON.stringify(payload); - const hmac = createHmac("sha256", Buffer.from(secret, "hex")).update(data).digest(); - return `${Buffer.from(data).toString("base64") + "." + Buffer.from(hmac).toString("base64")}`; -} - -// TODO: This would be better if it returned an input stream. -export async function handleDatabase(req, res, pathname) { - // Match /:name/:pathname - const match = /^\/([^/]+)(\/[^/]+)$/.exec(pathname); - if (match) { - const name = match[1]; - const pathname = match[2]; - const config = await readDatabaseConfig(name); - - if (pathname.startsWith("/token")) { - res.writeHead(200, {"Content-Type": "application/json"}); - res.end( - JSON.stringify({ - token: encodeToken({name}, config.secret), - type: config.type, - origin: DATACONNECTOR_URL - }) - ); - return; - } - return; - } - res.statusCode = 404; - res.setHeader("Content-Type", "text/plain; charset=utf-8"); - res.end("Oops, an error occurred"); -} diff --git a/src/markdown.ts b/src/markdown.ts index da95285f4..84c199c39 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -11,7 +11,6 @@ import {type default as Renderer, type RenderRule} from "markdown-it/lib/rendere import mime from "mime"; import {join} from "path"; import {canReadSync} from "./files.js"; -import type {DatabaseReference} from "./javascript.js"; import {transpileJavaScript, type FileReference, type ImportReference, type Transpile} from "./javascript.js"; import {computeHash} from "./hash.js"; import {readFile} from "fs/promises"; @@ -39,7 +38,6 @@ export interface ParseResult { title: string | null; html: string; data: {[key: string]: any} | null; - databases: DatabaseReference[]; files: FileReference[]; imports: ImportReference[]; pieces: HtmlPiece[]; @@ -53,7 +51,6 @@ interface RenderPiece { interface ParseContext { pieces: RenderPiece[]; - databases: DatabaseReference[]; files: {name: string; mimeType: string | null}[]; imports: ImportReference[]; startLine: number; @@ -102,7 +99,6 @@ function makeFenceRenderer(root: string, baseRenderer: RenderRule): RenderRule { sourceLine: context.startLine + context.currentLine }); extendPiece(context, {code: [transpile]}); - if (transpile.databases) context.databases.push(...transpile.databases); if (transpile.files) context.files.push(...transpile.files); if (transpile.imports) context.imports.push(...transpile.imports); result += `
\n`; @@ -258,7 +254,6 @@ function makePlaceholderRenderer(root: string): RenderRule { sourceLine: context.startLine + context.currentLine }); extendPiece(context, {code: [transpile]}); - if (transpile.databases) context.databases.push(...transpile.databases); if (transpile.files) context.files.push(...transpile.files); return ``; }; @@ -331,7 +326,7 @@ function toParseCells(pieces: RenderPiece[]): CellPiece[] { return cellPieces; } -export function parseMarkdown(source: string, root: string): ParseResult { +export function parseMarkdown(source: string, root: string /* options */): ParseResult { const parts = matter(source); // TODO: We need to know what line in the source the markdown starts on and pass that // as startLine in the parse context below. @@ -356,14 +351,13 @@ export function parseMarkdown(source: string, root: string): ParseResult { md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!); md.renderer.rules.softbreak = makeSoftbreakRenderer(md.renderer.rules.softbreak!); md.renderer.render = renderIntoPieces(md.renderer); - const context: ParseContext = {databases: [], files: [], imports: [], pieces: [], startLine: 0, currentLine: 0}; + const context: ParseContext = {files: [], imports: [], pieces: [], startLine: 0, currentLine: 0}; const tokens = md.parse(parts.content, context); const html = md.renderer.render(tokens, md.options, context); return { html, data: isEmpty(parts.data) ? null : parts.data, title: parts.data?.title ?? findTitle(tokens) ?? null, - databases: context.databases, files: context.files, imports: context.imports, pieces: toParsePieces(context.pieces), diff --git a/src/preview.ts b/src/preview.ts index cb69ddc0f..52228034f 100644 --- a/src/preview.ts +++ b/src/preview.ts @@ -12,7 +12,8 @@ import type {ParseResult} from "./markdown.js"; import {diffMarkdown, readMarkdown} from "./markdown.js"; import {readPages} from "./navigation.js"; import {renderPreview} from "./render.js"; -import {handleDatabase} from "./database.js"; +import type {CellResolver} from "./resolver.js"; +import {makeCLIResolver} from "./resolver.js"; const publicRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "public"); @@ -22,6 +23,7 @@ class Server { readonly port: number; readonly hostname: string; readonly root: string; + private _resolver: CellResolver | undefined; constructor({port, hostname, root}: CommandContext) { this.port = port; @@ -34,6 +36,7 @@ class Server { } async start() { + this._resolver = await makeCLIResolver(); return new Promise((resolve) => { this._server.listen(this.port, this.hostname, resolve); }); @@ -50,8 +53,6 @@ class Server { send(req, pathname.slice("/_observablehq".length), {root: publicRoot}).pipe(res); } else if (pathname.startsWith("/_file/")) { send(req, pathname.slice("/_file".length), {root: this.root}).pipe(res); - } else if (pathname.startsWith("/_database/")) { - handleDatabase(req, res, pathname.slice("/_database".length)); } else { if (normalize(pathname).startsWith("..")) throw new Error("Invalid path: " + pathname); let path = join(this.root, pathname); @@ -88,7 +89,16 @@ class Server { // Anything else should 404; static files should be matched above. try { const pages = await readPages(this.root); // TODO cache? watcher? - res.end(renderPreview(await readFile(path + ".md", "utf-8"), {root: this.root, path: pathname, pages}).html); + res.end( + ( + await renderPreview(await readFile(path + ".md", "utf-8"), { + root: this.root, + path: pathname, + pages, + resolver: this._resolver! + }) + ).html + ); } catch (error) { if (!isNodeError(error) || error.code !== "ENOENT") throw error; // internal error throw new HttpError("Not found", 404); @@ -104,7 +114,7 @@ class Server { _handleConnection = (socket: WebSocket, req: IncomingMessage) => { if (req.url === "/_observablehq") { - handleWatch(socket, this.root); + handleWatch(socket, {root: this.root, resolver: this._resolver!}); } else { socket.close(); } @@ -125,7 +135,21 @@ class FileWatchers { } } -function handleWatch(socket: WebSocket, root: string) { +function resolveDiffs(diff: ReturnType, resolver: CellResolver): ReturnType { + for (const item of diff) { + if (item.type === "add") { + for (const addItem of item.items) { + if (addItem.type === "cell" && "databases" in addItem) { + Object.assign(addItem, resolver(addItem)); + } + } + } + } + return diff; +} + +function handleWatch(socket: WebSocket, options: {root: string; resolver: CellResolver}) { + const {root, resolver} = options; let markdownWatcher: FSWatcher | null = null; let attachmentWatcher: FileWatchers | null = null; console.log("socket open"); @@ -147,7 +171,7 @@ function handleWatch(socket: WebSocket, root: string) { if (current.hash !== updated.hash) { send({ type: "update", - diff: diffMarkdown(current, updated), + diff: resolveDiffs(diffMarkdown(current, updated), resolver), previousHash: current.hash, updatedHash: updated.hash }); diff --git a/src/render.ts b/src/render.ts index 521612ce2..9821ed681 100644 --- a/src/render.ts +++ b/src/render.ts @@ -1,5 +1,6 @@ import {computeHash} from "./hash.js"; import {type FileReference, type ImportReference} from "./javascript.js"; +import type {CellPiece} from "./markdown.js"; import {parseMarkdown, type ParseResult} from "./markdown.js"; export interface Render { @@ -12,6 +13,7 @@ export interface RenderOptions { root: string; path?: string; pages?: {path: string; name: string}[]; + resolver: (cell: CellPiece) => CellPiece; } export function renderPreview(source: string, options: RenderOptions): Render { @@ -33,8 +35,8 @@ export function renderServerless(source: string, options: RenderOptions): Render } export function renderDefineCell(cell) { - const {id, inline, inputs, outputs, files, body} = cell; - return `define({${Object.entries({id, inline, inputs, outputs, files}) + const {id, inline, inputs, outputs, files, body, databases} = cell; + return `define({${Object.entries({id, inline, inputs, outputs, files, databases}) .filter((arg) => arg[1] !== undefined) .map((arg) => `${arg[0]}: ${JSON.stringify(arg[1])}`) .join(", ")}, body: ${body}});\n`; @@ -44,7 +46,10 @@ type RenderInternalOptions = | {preview?: false; hash?: never} // serverless | {preview: true; hash: string}; // preview -function render(parseResult: ParseResult, {path, pages, preview, hash}: RenderOptions & RenderInternalOptions): string { +function render( + parseResult: ParseResult, + {path, pages, preview, hash, resolver}: RenderOptions & RenderInternalOptions +): string { const showSidebar = pages && pages.length > 1; const imports = getImportMap(parseResult); return ` @@ -65,7 +70,10 @@ ${Array.from(imports.values()) import {${preview ? "open, " : ""}define} from "/_observablehq/client.js"; -${preview ? `open({hash: ${JSON.stringify(hash)}});\n` : ""}${parseResult.cells.map(renderDefineCell).join("")} +${preview ? `open({hash: ${JSON.stringify(hash)}});\n` : ""}${parseResult.cells + .map(resolver) + .map(renderDefineCell) + .join("")} ${ parseResult.data ? ` diff --git a/src/resolver.ts b/src/resolver.ts new file mode 100644 index 000000000..e32d3e819 --- /dev/null +++ b/src/resolver.ts @@ -0,0 +1,97 @@ +import {homedir} from "os"; +import {join} from "path"; +import {createHmac} from "node:crypto"; +import {readFile} from "node:fs/promises"; +import type {CellPiece} from "./markdown.js"; + +export type CellResolver = (cell: CellPiece) => CellPiece; + +export interface ResolvedDatabaseReference { + name: string; + origin: string; + token: string; + type: string; +} + +interface DatabaseProxyItem { + secret: string; +} + +type DatabaseProxyConfig = Record; + +interface ObservableConfig { + "database-proxy": DatabaseProxyConfig; +} + +interface DatabaseConfig { + host: string; + name: string; + origin?: string; + port: number; + secret: string; + ssl: "disabled" | "enabled"; + type: string; + url: string; +} + +const configFile = join(homedir(), ".observablehq"); +const key = `database-proxy`; + +export async function readDatabaseProxyConfig(): Promise { + const observableConfig = JSON.parse(await readFile(configFile, "utf-8")) as ObservableConfig | null; + return observableConfig && observableConfig[key]; +} + +function readDatabaseConfig(config: DatabaseProxyConfig | null, name): DatabaseConfig { + if (!config) throw new Error(`Missing database configuration file "${configFile}"`); + try { + if (!name) throw new Error(`No database name specified`); + const raw = (config && config[name]) as DatabaseConfig | null; + if (!raw) throw new Error(`No configuration found for "${name}"`); + return { + ...decodeSecret(raw.secret), + url: raw.url + } as DatabaseConfig; + } catch (error) { + throw new Error(`Unable to read database configuration file "${configFile}"`); + } +} + +function decodeSecret(secret: string): Record { + return JSON.parse(Buffer.from(secret, "base64").toString("utf8")); +} + +function encodeToken(payload: {name: string}, secret): string { + const data = JSON.stringify(payload); + const hmac = createHmac("sha256", Buffer.from(secret, "hex")).update(data).digest(); + return `${Buffer.from(data).toString("base64") + "." + Buffer.from(hmac).toString("base64")}`; +} + +export async function makeCLIResolver(): Promise { + const config = await readDatabaseProxyConfig(); + return (cell: CellPiece): CellPiece => { + if ("databases" in cell && cell.databases !== undefined) { + cell = { + ...cell, + databases: cell.databases.map((ref) => { + const db = readDatabaseConfig(config, ref.name); + if (db) { + const url = new URL("http://localhost"); + url.protocol = db.ssl !== "disabled" ? "https:" : "http:"; + url.host = db.host; + url.port = String(db.port); + url.toString(); + return { + ...ref, + token: encodeToken(ref, db.secret), + type: db.type, + url: url.toString() + }; + } + throw new Error(`Unable to resolve database "${ref.name}"`); + }) + }; + } + return cell; + }; +} From 2a28b394207a92d460d9f49f0b7a44d172a77006 Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Tue, 24 Oct 2023 10:17:38 -0700 Subject: [PATCH 3/8] Revert unneeded test changes --- test/output/block-expression.json | 1 - test/output/databaseclient.json | 5 ----- test/output/dollar-expression.json | 1 - test/output/double-quote-expression.json | 1 - test/output/embedded-expression.json | 1 - test/output/escaped-expression.json | 1 - test/output/fenced-code.json | 1 - test/output/heading-expression.json | 1 - test/output/hello-world.json | 1 - test/output/inline-expression.json | 1 - test/output/local-fetch.json | 1 - test/output/script-expression.json | 1 - test/output/single-quote-expression.json | 1 - test/output/template-expression.json | 1 - test/output/tex-expression.json | 1 - test/output/yaml-frontmatter.json | 1 - 16 files changed, 20 deletions(-) diff --git a/test/output/block-expression.json b/test/output/block-expression.json index abe1ae629..b6994baad 100644 --- a/test/output/block-expression.json +++ b/test/output/block-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/databaseclient.json b/test/output/databaseclient.json index ab8a7bbb0..eaa94205b 100644 --- a/test/output/databaseclient.json +++ b/test/output/databaseclient.json @@ -1,11 +1,6 @@ { "data": null, "title": null, - "databases": [ - { - "name": "dwh-local-proxy" - } - ], "files": [], "imports": [], "pieces": [ diff --git a/test/output/dollar-expression.json b/test/output/dollar-expression.json index 42d47c221..4d2d6c123 100644 --- a/test/output/dollar-expression.json +++ b/test/output/dollar-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/double-quote-expression.json b/test/output/double-quote-expression.json index 3f1e45d81..4a79621f1 100644 --- a/test/output/double-quote-expression.json +++ b/test/output/double-quote-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/embedded-expression.json b/test/output/embedded-expression.json index 8fea8ba5c..72317f4ce 100644 --- a/test/output/embedded-expression.json +++ b/test/output/embedded-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": "Embedded expression", - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/escaped-expression.json b/test/output/escaped-expression.json index e3ca16e55..0b9bd7976 100644 --- a/test/output/escaped-expression.json +++ b/test/output/escaped-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/fenced-code.json b/test/output/fenced-code.json index 94069a8b6..ea500a263 100644 --- a/test/output/fenced-code.json +++ b/test/output/fenced-code.json @@ -1,7 +1,6 @@ { "data": null, "title": "Fenced code", - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/heading-expression.json b/test/output/heading-expression.json index 4ad59b8df..d82812bc4 100644 --- a/test/output/heading-expression.json +++ b/test/output/heading-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/hello-world.json b/test/output/hello-world.json index 713e618e2..9b675fd5a 100644 --- a/test/output/hello-world.json +++ b/test/output/hello-world.json @@ -1,7 +1,6 @@ { "data": null, "title": "Hello, world!", - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/inline-expression.json b/test/output/inline-expression.json index 415ef2bb0..1401da533 100644 --- a/test/output/inline-expression.json +++ b/test/output/inline-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/local-fetch.json b/test/output/local-fetch.json index 2f7ec8748..e362f25dc 100644 --- a/test/output/local-fetch.json +++ b/test/output/local-fetch.json @@ -1,7 +1,6 @@ { "data": null, "title": "Local fetch", - "databases": [], "files": [ { "name": "./local-fetch.md", diff --git a/test/output/script-expression.json b/test/output/script-expression.json index 4878a36af..e9d3dfabb 100644 --- a/test/output/script-expression.json +++ b/test/output/script-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": "Script expression", - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/single-quote-expression.json b/test/output/single-quote-expression.json index 7f25166b3..6f0fd6424 100644 --- a/test/output/single-quote-expression.json +++ b/test/output/single-quote-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/template-expression.json b/test/output/template-expression.json index 19b09ad90..e0e55cca2 100644 --- a/test/output/template-expression.json +++ b/test/output/template-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": null, - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/tex-expression.json b/test/output/tex-expression.json index 81c735f5e..2111edd50 100644 --- a/test/output/tex-expression.json +++ b/test/output/tex-expression.json @@ -1,7 +1,6 @@ { "data": null, "title": "Hello, ", - "databases": [], "files": [], "imports": [], "pieces": [ diff --git a/test/output/yaml-frontmatter.json b/test/output/yaml-frontmatter.json index 60acc1656..75f092e4d 100644 --- a/test/output/yaml-frontmatter.json +++ b/test/output/yaml-frontmatter.json @@ -7,7 +7,6 @@ ] }, "title": "YAML", - "databases": [], "files": [], "imports": [], "pieces": [ From 200df4cec5cece0b6b8ecdf47d223e8f048f3e43 Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Tue, 24 Oct 2023 10:24:44 -0700 Subject: [PATCH 4/8] Fill out queryTag method --- public/database.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/public/database.js b/public/database.js index 4874f4666..c8dde4abc 100644 --- a/public/database.js +++ b/public/database.js @@ -53,7 +53,15 @@ class DatabaseClientImpl { } queryTag(strings, ...args) { - // TODO: This is Database-dialect specific + switch (this.type) { + case "oracle": + case "databricks": + return [strings.reduce((prev, curr, i) => `${prev}:${i}${curr}`), args]; + case "mysql": + return [strings.reduce((prev, curr, i) => `${prev}@${i}${curr}`), args]; + case "postgres": + return [strings.reduce((prev, curr, i) => `${prev}$${i}${curr}`), args]; + } return [strings.join("?"), args]; } From 00bc7ddd0f820c19c4952090e4531cc1c4dff944 Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Tue, 24 Oct 2023 12:17:26 -0700 Subject: [PATCH 5/8] Touch ups --- public/database.js | 5 ----- src/markdown.ts | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/public/database.js b/public/database.js index c8dde4abc..9b41ca29e 100644 --- a/public/database.js +++ b/public/database.js @@ -47,11 +47,6 @@ class DatabaseClientImpl { return data; } - // TODO: Does this serve any purpose? - async queryRow(...params) { - return (await this.query(...params))[0] || null; - } - queryTag(strings, ...args) { switch (this.type) { case "oracle": diff --git a/src/markdown.ts b/src/markdown.ts index 84c199c39..d0b7635a8 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -326,7 +326,7 @@ function toParseCells(pieces: RenderPiece[]): CellPiece[] { return cellPieces; } -export function parseMarkdown(source: string, root: string /* options */): ParseResult { +export function parseMarkdown(source: string, root: string): ParseResult { const parts = matter(source); // TODO: We need to know what line in the source the markdown starts on and pass that // as startLine in the parse context below. From 0f4014314f38c2adb6d1b701a23b3885b7eeed60 Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Tue, 24 Oct 2023 14:49:40 -0700 Subject: [PATCH 6/8] Review feedback and error handling fix --- public/client.js | 8 ++++---- src/resolver.ts | 18 +++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/public/client.js b/public/client.js index c36217d5f..1ebc894be 100644 --- a/public/client.js +++ b/public/client.js @@ -9,11 +9,11 @@ const resolveFile = (name) => attachedFiles.get(name); main.builtin("FileAttachment", runtime.fileAttachments(resolveFile)); const databaseTokens = new Map(); -const resolveDatabaseToken = (name) => { +async function resolveDatabaseToken(name) { const token = databaseTokens.get(name); - if (!token) Promise.reject(new Error(`Database configuration for ${name} not found`)); - return Promise.resolve(token); -}; + if (!token) throw new Error(`Database configuration for ${name} not found`); + return token; +} const cellsById = new Map(); const Generators = library.Generators; diff --git a/src/resolver.ts b/src/resolver.ts index e32d3e819..62e490c0b 100644 --- a/src/resolver.ts +++ b/src/resolver.ts @@ -44,17 +44,13 @@ export async function readDatabaseProxyConfig(): Promise { From 4343c4f283eace660b5716f02a22b53d6b8fa783 Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Mon, 30 Oct 2023 08:23:39 -0700 Subject: [PATCH 7/8] Review feedback --- public/database.js | 3 ++- src/build.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/public/database.js b/public/database.js index 9b41ca29e..d7ceefde4 100644 --- a/public/database.js +++ b/public/database.js @@ -13,7 +13,8 @@ class DatabaseClientImpl { this.#token = token; } async query(sql, params, {signal} = {}) { - const response = await fetch(`${this.#token.url}query`, { + const queryUrl = new URL("/query", this.#token.url).toString(); + const response = await fetch(queryUrl, { method: "POST", headers: { Authorization: `Bearer ${this.#token.token}`, diff --git a/src/build.ts b/src/build.ts index a9a6a3d5d..12093b15d 100644 --- a/src/build.ts +++ b/src/build.ts @@ -21,6 +21,7 @@ async function build(context: CommandContext) { // Render .md files, building a list of file attachments as we go. const pages = await readPages(sourceRoot); const files: string[] = []; + const resolver = await makeCLIResolver(); for await (const sourceFile of visitMarkdownFiles(sourceRoot)) { const sourcePath = join(sourceRoot, sourceFile); const outputPath = join(outputRoot, join(dirname(sourceFile), basename(sourceFile, ".md") + ".html")); @@ -30,7 +31,7 @@ async function build(context: CommandContext) { root: sourceRoot, path, pages, - resolver: await makeCLIResolver() + resolver }); files.push(...render.files.map((f) => join(sourceFile, "..", f.name))); await prepareOutput(outputPath); From 9c0c5a1da80a19e7549f7d4fe53e4389b45380be Mon Sep 17 00:00:00 2001 From: Wiltse Carpenter Date: Mon, 30 Oct 2023 15:52:25 -0700 Subject: [PATCH 8/8] Add database.js preload --- src/render.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/render.ts b/src/render.ts index 9821ed681..c65c7f37a 100644 --- a/src/render.ts +++ b/src/render.ts @@ -66,6 +66,11 @@ ${Array.from(imports.values()) .concat(parseResult.imports.filter(({name}) => name.startsWith("./")).map(({name}) => `/_file/${name.slice(2)}`)) .map((href) => ``) .join("\n")} +${ + parseResult.cells.some((cell) => cell.databases?.length) + ? `` + : "" +}