search

docs/config.md

@@ -180,3 +180,7 @@ The table of contents configuration can also be set in the page’s YAML front m
 toc: false
 ---
 ```
+
+## search
+
+Whether to enable [search](./search) on the project; defaults to false.

docs/index.md

@@ -1,5 +1,6 @@
 ---
 toc: false
+index: false
 ---
 
 <style>

docs/search.md

@@ -0,0 +1,33 @@
+---
+index: true
+---
+
+# Search
+
+Framework provides built-in full-text page search using [MiniSearch](https://lucaong.github.io/minisearch/). Search results are queried on the client, with fuzzy and prefix matching, using a static index computed during build.
+
+<div class="tip">Search is not enabled by default. It is intended for larger projects with lots of static text, such as reports and documentation. Search will not index dynamic content such as data or charts. To enable search, set the <a href="./config#search"><b>search</b> option</a> to true in your config.</div>
+
+Search works in two stages: when Framework builds the site, it creates an index of the contents. On the client, as soon as the user focuses the search input and starts typing, the index is retrieved and the matching pages are displayed in the sidebar. The user can then click on a result, or use the up ↑ and down ↓ arrow keys to navigate, then type return to open the page.
+
+Pages are indexed each time you build or deploy your project. When working in preview, they are reindexed every 10 minutes.
+
+By default, all the pages found in the project root (`docs` by default) or defined in the [**pages** config option](./config#pages) are indexed; you can however opt-out a page from the index by specifying an index: false property in its front matter:
+
+```yaml
+---
+title: This page won’t be indexed
+index: false
+---
+```
+
+Likewise, a page that is not referenced in **pages** can opt-in by having index: true in its front matter:
+
+```yaml
+---
+title: A page that is not in the sidebar, but gets indexed
+index: true
+---
+```
+
+Search is case-insensitive. The indexing script tries to avoid common pitfalls by ignoring HTML tags and non-word characters such as punctuation. It also ignores long words, as well as sequences that contain more than 6 digits (such as API keys, for example).

observablehq.config.ts

@@ -106,5 +106,6 @@ export default {
   </div>
 </div>`,
   footer: `© ${new Date().getUTCFullYear()} Observable, Inc.`,
-  style: "style.css"
+  style: "style.css",
+  search: true
 };

package.json

@@ -66,6 +66,7 @@
     "markdown-it": "^13.0.2",
     "markdown-it-anchor": "^8.6.7",
     "mime": "^3.0.0",
+    "minisearch": "^6.3.0",
     "open": "^9.1.0",
     "rollup": "^4.6.0",
     "rollup-plugin-esbuild": "^6.1.0",

src/build.ts

@@ -11,6 +11,7 @@ import {createImportResolver, rewriteModule} from "./javascript/imports.js";
 import type {Logger, Writer} from "./logger.js";
 import {renderServerless} from "./render.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
+import {searchIndex} from "./search.js";
 import {Telemetry} from "./telemetry.js";
 import {faint} from "./tty.js";
 import {resolvePath} from "./url.js";
@@ -22,24 +23,6 @@ const EXTRA_FILES = new Map([
   ]
 ]);
 
-// TODO Remove library helpers (e.g., duckdb) when they are published to npm.
-function clientBundles(clientPath: string): [entry: string, name: string][] {
-  return [
-    [clientPath, "client.js"],
-    ["./src/client/stdlib.js", "stdlib.js"],
-    ["./src/client/stdlib/dot.js", "stdlib/dot.js"],
-    ["./src/client/stdlib/duckdb.js", "stdlib/duckdb.js"],
-    ["./src/client/stdlib/inputs.css", "stdlib/inputs.css"],
-    ["./src/client/stdlib/inputs.js", "stdlib/inputs.js"],
-    ["./src/client/stdlib/mermaid.js", "stdlib/mermaid.js"],
-    ["./src/client/stdlib/sqlite.js", "stdlib/sqlite.js"],
-    ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
-    ["./src/client/stdlib/vega-lite.js", "stdlib/vega-lite.js"],
-    ["./src/client/stdlib/xlsx.js", "stdlib/xlsx.js"],
-    ["./src/client/stdlib/zip.js", "stdlib/zip.js"]
-  ];
-}
-
 export interface BuildOptions {
   config: Config;
   clientEntry?: string;
@@ -106,7 +89,23 @@ export async function build(
 
   // Generate the client bundles.
   if (addPublic) {
-    for (const [entry, name] of clientBundles(clientEntry)) {
+    for (const [entry, name] of [
+      [clientEntry, "client.js"],
+      ["./src/client/stdlib.js", "stdlib.js"],
+      // TODO Prune this list based on which libraries are actually used.
+      // TODO Remove library helpers (e.g., duckdb) when they are published to npm.
+      ["./src/client/stdlib/dot.js", "stdlib/dot.js"],
+      ["./src/client/stdlib/duckdb.js", "stdlib/duckdb.js"],
+      ["./src/client/stdlib/inputs.css", "stdlib/inputs.css"],
+      ["./src/client/stdlib/inputs.js", "stdlib/inputs.js"],
+      ["./src/client/stdlib/mermaid.js", "stdlib/mermaid.js"],
+      ["./src/client/stdlib/sqlite.js", "stdlib/sqlite.js"],
+      ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
+      ["./src/client/stdlib/vega-lite.js", "stdlib/vega-lite.js"],
+      ["./src/client/stdlib/xlsx.js", "stdlib/xlsx.js"],
+      ["./src/client/stdlib/zip.js", "stdlib/zip.js"],
+      ...(config.search ? [["./src/client/search.js", "search.js"]] : [])
+    ]) {
       const clientPath = getClientPath(entry);
       const outputPath = join("_observablehq", name);
       effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `);
@@ -115,6 +114,12 @@ export async function build(
         : rollupClient(clientPath, {minify: true}));
       await effects.writeFile(outputPath, code);
     }
+    if (config.search) {
+      const outputPath = join("_observablehq", "minisearch.json");
+      const code = await searchIndex(config, effects);
+      effects.output.write(`${faint("search")} ${faint("→")} `);
+      await effects.writeFile(outputPath, code);
+    }
     for (const style of styles) {
       if ("path" in style) {
         const outputPath = join("_import", style.path);

src/client/search-init.ts

@@ -0,0 +1,32 @@
+const container = document.querySelector("#observablehq-search")!;
+
+// Set the short dynamically based on the client’s platform.
+container.setAttribute("data-shortcut", `${/Mac|iPhone/.test(navigator.platform) ? "⌘" : "Alt-"}K`);
+
+// Load search.js on demand
+const input = container.querySelector<HTMLInputElement>("input")!;
+const base = container.getAttribute("data-root");
+const load = () => import(`${base}_observablehq/search.js`);
+input.addEventListener("focus", load, {once: true});
+input.addEventListener("keydown", load, {once: true});
+
+// Focus on meta-K and /
+const toggle = document.querySelector("#observablehq-sidebar-toggle")!;
+addEventListener("keydown", (event) => {
+  if (
+    (event.code === "KeyK" && event.metaKey && !event.altKey && !event.ctrlKey) ||
+    (event.key === "/" && !event.metaKey && !event.altKey && !event.ctrlKey && event.target === document.body)
+  ) {
+    // Force the sidebar to be temporarily open while the search input is
+    // focused. (We can’t use :focus-within because the sidebar isn’t focusable
+    // while it is invisible, and we don’t want to keep the sidebar open
+    // persistently after you blur the search input.)
+    toggle.classList.add("observablehq-sidebar-open");
+    input.focus();
+    input.select();
+    event.preventDefault();
+  }
+});
+
+// Allow the sidebar to close when the search input is blurred.
+input.addEventListener("blur", () => toggle.classList.remove("observablehq-sidebar-open"));

src/client/search.js

@@ -0,0 +1,78 @@
+import MiniSearch from "minisearch";
+
+const container = document.querySelector("#observablehq-search");
+const sidebar = document.querySelector("#observablehq-sidebar");
+const shortcut = container.getAttribute("data-shortcut");
+const input = container.querySelector("input");
+const resultsContainer = document.querySelector("#observablehq-search-results");
+const activeClass = "observablehq-link-active";
+let currentValue;
+
+const index = await fetch(import.meta.resolve("./minisearch.json"))
+  .then((response) => {
+    if (!response.ok) throw new Error(`unable to load minisearch.json: ${response.status}`);
+    return response.json();
+  })
+  .then((json) =>
+    MiniSearch.loadJS(json, {
+      ...json.options,
+      processTerm: (term) => term.slice(0, 15).toLowerCase() // see src/minisearch.json.ts
+    })
+  );
+
+input.addEventListener("input", () => {
+  if (currentValue === input.value) return;
+  currentValue = input.value;
+  if (!currentValue.length) {
+    container.setAttribute("data-shortcut", shortcut);
+    sidebar.classList.remove("observablehq-search-results");
+    resultsContainer.innerHTML = "";
+    return;
+  }
+  container.setAttribute("data-shortcut", ""); // prevent conflict with close button
+  sidebar.classList.add("observablehq-search-results"); // hide pages while showing search results
+  const results = index.search(currentValue, {boost: {title: 4}, fuzzy: 0.15, prefix: true});
+  resultsContainer.innerHTML =
+    results.length === 0
+      ? "<div>no results</div>"
+      : `<div>${results.length.toLocaleString("en-US")} result${results.length === 1 ? "" : "s"}</div><ol>${results
+          .map(renderResult)
+          .join("")}</ol>`;
+});
+
+function renderResult({id, score, title}, i) {
+  return `<li data-score="${Math.min(5, Math.round(0.6 * score))}" class="observablehq-link${
+    i === 0 ? ` ${activeClass}` : ""
+  }"><a href="${escapeDoubleQuote(import.meta.resolve(`../${id}`))}">${escapeText(title)}</a></li>`;
+}
+
+function escapeDoubleQuote(text) {
+  return text.replace(/["&]/g, entity);
+}
+
+function escapeText(text) {
+  return text.replace(/[<&]/g, entity);
+}
+
+function entity(character) {
+  return `&#${character.charCodeAt(0).toString()};`;
+}
+
+// Handle a race condition where an input event fires while awaiting the index fetch.
+input.dispatchEvent(new Event("input"));
+
+input.addEventListener("keydown", (event) => {
+  const {code} = event;
+  if (code === "Escape" && input.value === "") return input.blur();
+  if (code === "ArrowDown" || code === "ArrowUp" || code === "Enter") {
+    const results = resultsContainer.querySelector("ol");
+    if (!results) return;
+    let activeResult = results.querySelector(`.${activeClass}`);
+    if (code === "Enter") return activeResult.querySelector("a").click();
+    activeResult.classList.remove(activeClass);
+    if (code === "ArrowUp") activeResult = activeResult.previousElementSibling ?? results.lastElementChild;
+    else activeResult = activeResult.nextElementSibling ?? results.firstElementChild;
+    activeResult.classList.add(activeClass);
+    activeResult.scrollIntoView({block: "nearest"});
+  }
+});

src/config.ts

@@ -46,6 +46,7 @@ export interface Config {
   toc: TableOfContents;
   style: null | Style; // defaults to {theme: ["light", "dark"]}
   deploy: null | {workspace: string; project: string};
+  search: boolean; // default to false
 }
 
 export async function readConfig(configPath?: string, root?: string): Promise<Config> {
@@ -93,6 +94,7 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     sidebar,
     style,
     theme = "default",
+    search,
     deploy,
     scripts = [],
     head = "",
@@ -118,7 +120,8 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   footer = String(footer);
   toc = normalizeToc(toc);
   deploy = deploy ? {workspace: String(deploy.workspace).replace(/^@+/, ""), project: String(deploy.project)} : null;
-  return {root, output, base, title, sidebar, pages, pager, scripts, head, header, footer, toc, style, deploy};
+  search = Boolean(search);
+  return {root, output, base, title, sidebar, pages, pager, scripts, head, header, footer, toc, style, deploy, search};
 }
 
 function normalizeBase(base: any): string {

src/preview.ts

@@ -23,6 +23,7 @@ import {diffMarkdown, parseMarkdown} from "./markdown.js";
 import type {ParseResult} from "./markdown.js";
 import {renderPreview, resolveStylesheet} from "./render.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
+import {searchIndex} from "./search.js";
 import {Telemetry} from "./telemetry.js";
 import {bold, faint, green, link, red} from "./tty.js";
 import {relativeUrl} from "./url.js";
@@ -108,6 +109,10 @@ export class PreviewServer {
         }
       } else if (pathname === "/_observablehq/client.js") {
         end(req, res, await rollupClient(getClientPath("./src/client/preview.js")), "text/javascript");
+      } else if (pathname === "/_observablehq/search.js") {
+        end(req, res, await rollupClient(getClientPath("./src/client/search.js")), "text/javascript");
+      } else if (pathname === "/_observablehq/minisearch.json") {
+        end(req, res, await searchIndex(config), "application/json");
       } else if ((match = /^\/_observablehq\/theme-(?<theme>[\w-]+(,[\w-]+)*)?\.css$/.exec(pathname))) {
         end(req, res, await bundleStyles({theme: match.groups!.theme?.split(",") ?? []}), "text/css");
       } else if (pathname.startsWith("/_observablehq/")) {

src/render.ts

@@ -58,7 +58,7 @@ type RenderInternalOptions =
   | {preview: true}; // preview
 
 async function render(parseResult: ParseResult, options: RenderOptions & RenderInternalOptions): Promise<string> {
-  const {root, base, path, pages, title, preview} = options;
+  const {root, base, path, pages, title, preview, search} = options;
   const sidebar = parseResult.data?.sidebar !== undefined ? Boolean(parseResult.data.sidebar) : options.sidebar;
   const toc = mergeToc(parseResult.data?.toc, options.toc);
   return String(html`<!DOCTYPE html>
@@ -90,7 +90,7 @@ import ${preview || parseResult.cells.length > 0 ? `{${preview ? "open, " : ""}d
 ${
   preview ? `\nopen({hash: ${JSON.stringify(parseResult.hash)}, eval: (body) => (0, eval)(body)});\n` : ""
 }${parseResult.cells.map((cell) => `\n${renderDefineCell(cell)}`).join("")}`)}
-</script>${sidebar ? html`\n${await renderSidebar(title, pages, path)}` : ""}${
+</script>${sidebar ? html`\n${await renderSidebar(title, pages, path, search)}` : ""}${
     toc.show ? html`\n${renderToc(findHeaders(parseResult), toc.label)}` : ""
   }
 <div id="observablehq-center">${renderHeader(options, parseResult.data)}
@@ -100,7 +100,7 @@ ${html.unsafe(parseResult.html)}</main>${renderFooter(path, options, parseResult
 `);
 }
 
-async function renderSidebar(title = "Home", pages: (Page | Section)[], path: string): Promise<Html> {
+async function renderSidebar(title = "Home", pages: (Page | Section)[], path: string, search: boolean): Promise<Html> {
   return html`<input id="observablehq-sidebar-toggle" type="checkbox" title="Toggle sidebar">
 <label id="observablehq-sidebar-backdrop" for="observablehq-sidebar-toggle"></label>
 <nav id="observablehq-sidebar">
@@ -109,7 +109,18 @@ async function renderSidebar(title = "Home", pages: (Page | Section)[], path: st
     <li class="observablehq-link${
       normalizePath(path) === "/index" ? " observablehq-link-active" : ""
     }"><a href="${relativeUrl(path, "/")}">${title}</a></li>
-  </ol>
+  </ol>${
+    search
+      ? html`\n  <div id="observablehq-search" data-root="${relativeUrl(
+          path,
+          "/"
+        )}"><input type="search" placeholder="Search"></div>
+  <div id="observablehq-search-results"></div>
+  <script>{${html.unsafe(
+    (await rollupClient(getClientPath("./src/client/search-init.ts"), {minify: true})).trim()
+  )}}</script>`
+      : ""
+  }
   <ol>${pages.map((p, i) =>
     "pages" in p
       ? html`${i > 0 && "path" in pages[i - 1] ? html`</ol>` : ""}

src/rollup.ts

@@ -18,6 +18,9 @@ const STYLE_MODULES = {
   ...Object.fromEntries(THEMES.map(({name, path}) => [`observablehq:theme-${name}.css`, path]))
 };
 
+// These libraries are currently bundled in to a wrapper.
+const BUNDLED_MODULES = ["@observablehq/inputs", "minisearch"];
+
 function rewriteInputsNamespace(code: string) {
   return code.replace(/\b__ns__\b/g, "inputs-3a86ea");
 }
@@ -38,7 +41,7 @@ export async function rollupClient(clientPath: string, {minify = false} = {}): P
     input: clientPath,
     external: [/^https:/],
     plugins: [
-      nodeResolve({resolveOnly: ["@observablehq/inputs"]}),
+      nodeResolve({resolveOnly: BUNDLED_MODULES}),
       importResolve(clientPath),
       esbuild({
         target: "es2022",
@@ -109,7 +112,7 @@ async function resolveImport(source: string, specifier: string | AstNode): Promi
     ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/zip.js")), external: true} // TODO publish to npm
     : specifier.startsWith("npm:")
     ? {id: await resolveNpmImport(specifier.slice("npm:".length))}
-    : source !== specifier && !isPathImport(specifier) && specifier !== "@observablehq/inputs"
+    : source !== specifier && !isPathImport(specifier) && !BUNDLED_MODULES.includes(specifier)
     ? {id: await resolveNpmImport(specifier), external: true}
     : null;
 }