search

docs/config.md

@@ -180,3 +180,7 @@ The table of contents configuration can also be set in the page’s YAML front m
 toc: false
 ---
 ```
+
+## search
+
+Enable [search](search) on the project.

docs/index.md

@@ -1,5 +1,6 @@
 ---
 toc: false
+title: Introduction
 ---
 
 <style>

docs/search.md

@@ -0,0 +1,35 @@
+---
+index: true
+---
+
+# Search
+
+Search works in two stages: when Framework builds the site, it also creates an index of the contents. On the client, as soon as the user focuses the search input and starts typing a query, the index is retrieved and the matching pages are displayed in the sidebar. The user can then click on a result, or use the up ↑ and down ↓ arrow keys to navigate, then type return to open the page.
+
+To enable search on your project, add a **search**: true option to your site’s [configuration](config).
+
+Indexation and retrieval are using [MiniSearch](https://lucaong.github.io/minisearch/), a JavaScript library that enables full-text search with many useful features (like prefix search, fuzzy search, ranking, boosting of fields).
+
+The pages are indexed each time you build, or deploy, your project; when working in preview, they are reindexed every 10 minutes.
+
+By default, all the pages found in the docs/ folder or defined in the configuration’s **pages** are indexed; you can however opt-out a page from the index by specifying an index: false property in its front-matter:
+
+```yaml
+---
+title: This page won’t be indexed
+index: false
+---
+```
+
+Likewise, a page that is not referenced in the configuration’s **pages** can opt-in by having index: true in its front-matter:
+
+```yaml
+---
+title: A page that is not in the sidebar, but gets indexed
+index: true
+---
+```
+
+Search is case-insensitive. The indexing script tries to avoid common pitfalls by ignoring HTML tags and non-word characters (such as punctuation, backticks, etc.) found in the pages’ markdown. It also ignores long words as well as sequences that contain more than 6 digits (such as API keys, for example).
+
+The selected pages are sorted by descending relevance, represented by a number of dots that reflects the terms found (exactly or with a fuzzy match) in the page, with less points given to relatively frequent terms, and more points given to terms found in the title.

observablehq.config.ts

@@ -106,5 +106,6 @@ export default {
   </div>
 </div>`,
   footer: `© ${new Date().getUTCFullYear()} Observable, Inc.`,
-  style: "style.css"
+  style: "style.css",
+  search: true
 };

package.json

@@ -66,6 +66,7 @@
     "markdown-it": "^13.0.2",
     "markdown-it-anchor": "^8.6.7",
     "mime": "^3.0.0",
+    "minisearch": "^6.3.0",
     "open": "^9.1.0",
     "rollup": "^4.6.0",
     "rollup-plugin-esbuild": "^6.1.0",

src/build.ts

@@ -9,6 +9,7 @@ import {CliError, isEnoent} from "./error.js";
 import {getClientPath, prepareOutput, visitMarkdownFiles} from "./files.js";
 import {createImportResolver, rewriteModule} from "./javascript/imports.js";
 import type {Logger, Writer} from "./logger.js";
+import {searchIndex} from "./minisearch.json.js";
 import {renderServerless} from "./render.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
 import {Telemetry} from "./telemetry.js";
@@ -36,7 +37,8 @@ function clientBundles(clientPath: string): [entry: string, name: string][] {
     ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
     ["./src/client/stdlib/vega-lite.js", "stdlib/vega-lite.js"],
     ["./src/client/stdlib/xlsx.js", "stdlib/xlsx.js"],
-    ["./src/client/stdlib/zip.js", "stdlib/zip.js"]
+    ["./src/client/stdlib/zip.js", "stdlib/zip.js"],
+    ["./src/client/search.js", "search.js"]
   ];
 }
 
@@ -106,12 +108,22 @@ export async function build(
 
   // Generate the client bundles.
   if (addPublic) {
-    for (const [entry, name] of clientBundles(clientEntry)) {
+    for (const [entry, name] of [
+      ...clientBundles(clientEntry),
+      ...(config.search
+        ? [
+            ["./src/client/search.js", "search.js"],
+            ["./src/minisearch.json.ts", "minisearch.json"]
+          ]
+        : [])
+    ]) {
       const clientPath = getClientPath(entry);
       const outputPath = join("_observablehq", name);
       effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `);
       const code = await (entry.endsWith(".css")
         ? bundleStyles({path: clientPath})
+        : name === "minisearch.json"
+        ? searchIndex(config, effects)
         : rollupClient(clientPath, {minify: true}));
       await effects.writeFile(outputPath, code);
     }

src/client/search-init.ts

@@ -0,0 +1,35 @@
+const container = document.querySelector("#observablehq-search")!;
+
+const input = container.querySelector("input")! as HTMLInputElement;
+input.setAttribute("placeholder", "Search");
+container.setAttribute("data-shortcut", `${/Mac|iPhone/.test(navigator.platform) ? "⌘" : "Alt-"}K`);
+
+// fix links relative to the base
+const base = document.querySelector("#observablehq-search")?.getAttribute("data-root");
+for (const link of document.querySelectorAll("#observablehq-search-results a")) {
+  link.setAttribute("href", `${base}${link.parentElement?.getAttribute("data-reference")}`);
+}
+
+// load search.js on demand
+const load = () => {
+  input.removeEventListener("focus", load);
+  input.removeEventListener("keydown", load);
+  import(`${base}_observablehq/search.js`);
+};
+input.addEventListener("focus", load);
+input.addEventListener("keydown", load);
+
+// focus on meta-K and /
+const toggle = document.querySelector("#observablehq-sidebar-toggle")! as HTMLInputElement;
+addEventListener("keydown", (event) => {
+  if (
+    (event.code === "KeyK" && event.metaKey && !event.altKey && !event.ctrlKey) ||
+    (event.key === "/" && !event.metaKey && !event.altKey && !event.ctrlKey && event.target === document.body)
+  ) {
+    toggle.classList.add("observablehq-sidebar-on");
+    input.onblur = () => toggle.classList.remove("observablehq-sidebar-on");
+    input.focus();
+    input.select();
+    event.preventDefault();
+  }
+});

src/client/search.js

@@ -0,0 +1,72 @@
+// eslint-disable-next-line import/no-relative-packages
+import MiniSearch from "../../node_modules/minisearch/dist/es/index.js";
+
+const container = document.querySelector("#observablehq-search");
+const base = container.getAttribute("data-root");
+const input = container.querySelector("input");
+const r = document.querySelector("#observablehq-search-results");
+const c = "observablehq-link-active";
+let value;
+const index = await fetch(`${base}_observablehq/minisearch.json`)
+  .then((resp) => resp.json())
+  .then((json) =>
+    MiniSearch.loadJS(json, {
+      ...json.options,
+      processTerm: (term) => term.slice(0, 15).toLowerCase() // see src/minisearch.json.ts
+    })
+  );
+input.addEventListener("input", () => {
+  if (value === input.value) return;
+  value = input.value;
+  if (!value.length) {
+    container.parentElement.classList.remove("observablehq-search-results");
+    r.innerHTML = "";
+    return;
+  }
+  container.parentElement.classList.add("observablehq-search-results");
+  const results = index.search(value, {boost: {title: 4}, fuzzy: 0.15, prefix: true}).slice(0, 11);
+  r.innerHTML =
+    results.length === 0
+      ? "<div>no results</div>"
+      : `<div>${results.length === 11 ? "10+" : results.length} result${
+          results.length === 1 ? "" : "s"
+        }</div><ol>${"<li><a></a></li>".repeat(results.length)}</ol>`;
+  r.querySelectorAll("li").forEach((li, i) => {
+    const {id, score, title} = results[i];
+    li.setAttribute("class", `observablehq-link${i === 0 ? ` ${c}` : ""}`);
+    li.setAttribute("data-reference", id);
+    li.setAttribute("data-score", Math.min(5, Math.round(0.6 * score)));
+    const a = li.firstChild;
+    a.setAttribute("href", `${base}${id}`);
+    a.textContent = title;
+  });
+});
+
+// Handle a race condition where an input event fires while awaiting the index fetch.
+input.dispatchEvent(new Event("input"));
+
+addEventListener("keydown", (event) => {
+  const {code, target} = event;
+  if (target === input) {
+    if (code === "Escape" && input.value === "") {
+      input.blur();
+      return;
+    }
+    if (code === "ArrowDown" || code === "ArrowUp" || code === "Enter") {
+      const results = r.querySelector("ol");
+      const current = results?.querySelector(`.${c}`);
+      if (!current) return;
+      if (code === "Enter") {
+        current.querySelector("a")?.click();
+      } else {
+        if (code === "ArrowUp") {
+          current.classList.remove(c);
+          (current.previousElementSibling ?? results.querySelector("li:last-child"))?.classList.add(c);
+        } else if (code === "ArrowDown") {
+          current.classList.remove(c);
+          (current.nextElementSibling ?? results.querySelector("li:first-child")).classList.add(c);
+        }
+      }
+    }
+  }
+});

src/config.ts

@@ -46,6 +46,7 @@ export interface Config {
   toc: TableOfContents;
   style: null | Style; // defaults to {theme: ["light", "dark"]}
   deploy: null | {workspace: string; project: string};
+  search: boolean; // default to false
 }
 
 export async function readConfig(configPath?: string, root?: string): Promise<Config> {
@@ -93,6 +94,7 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     sidebar,
     style,
     theme = "default",
+    search,
     deploy,
     scripts = [],
     head = "",
@@ -118,7 +120,8 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   footer = String(footer);
   toc = normalizeToc(toc);
   deploy = deploy ? {workspace: String(deploy.workspace).replace(/^@+/, ""), project: String(deploy.project)} : null;
-  return {root, output, base, title, sidebar, pages, pager, scripts, head, header, footer, toc, style, deploy};
+  search = Boolean(search);
+  return {root, output, base, title, sidebar, pages, pager, scripts, head, header, footer, toc, style, deploy, search};
 }
 
 function normalizeBase(base: any): string {

src/minisearch.json.ts

@@ -0,0 +1,62 @@
+import {join} from "node:path";
+import MiniSearch from "minisearch";
+import {visitMarkdownFiles} from "../src/files.js";
+import type {BuildEffects} from "./build.js";
+import type {Config} from "./config.js";
+import {parseMarkdown} from "./markdown.js";
+import {faint} from "./tty.js";
+
+// Avoid reindexing too often in preview.
+const mem = new WeakMap();
+const reindexingDelay = 10 * 60 * 1000;
+
+export async function searchIndex(config: Config, effects?: BuildEffects): Promise<string> {
+  const {root, pages, search} = config;
+  const log = effects?.logger.log ?? console.info;
+  if (!search) return "{}";
+  if (mem.has(config) && mem.get(config).freshUntil > +new Date()) return mem.get(config).json;
+
+  const options = {
+    fields: ["title", "text"],
+    storeFields: ["title"],
+    processTerm: (term) => (term.match(/\d/g)?.length > 6 ? null : term.slice(0, 15).toLowerCase()) // fields to return with search results
+  };
+  const index = new MiniSearch(options);
+
+  const pagePaths = new Set();
+  for (const p of pages) {
+    if ("path" in p) pagePaths.add(p.path);
+    else for (const {path} of p.pages) pagePaths.add(path);
+  }
+
+  for await (const file of visitMarkdownFiles(root)) {
+    const {html, title, data} = await parseMarkdown(join(root, file), {root, path: "/" + file.slice(0, -3)});
+
+    // Skip page opted out of indexing, and non-pages unless opted-in.
+    // We only log the first case.
+    if (pagePaths.has(`/${file.slice(0, -3)}`)) {
+      if (data?.index === false) {
+        log(`${faint("skip")} ${file}`);
+        continue;
+      }
+    } else if (data?.index !== true) continue;
+
+    const id = file === "index.md" ? "" : "" + file.slice(0, -3);
+    const text = html
+      .replaceAll(/[\n\r]/g, " ")
+      .replaceAll(/<style\b.*<\/style\b[^>]*>/gi, " ")
+      .replaceAll(/<[^>]+>/g, " ")
+      .replaceAll(/\W+/g, " ");
+
+    log(`${faint("index")} ${id}: ${title}`);
+    index.add({id, title, text});
+  }
+
+  // One way of passing the options to the client; better than nothing, but note
+  // that the client can only read the options that are serializable. It's fine
+  // for field names, though, which is what we want to share.
+  const json = JSON.stringify(Object.assign({options}, index.toJSON()));
+
+  mem.set(config, {json, freshUntil: +new Date() + reindexingDelay});
+  return json;
+}

src/preview.ts

@@ -21,6 +21,7 @@ import {createImportResolver, rewriteModule} from "./javascript/imports.js";
 import {getImplicitSpecifiers, getImplicitStylesheets} from "./libraries.js";
 import {diffMarkdown, parseMarkdown} from "./markdown.js";
 import type {ParseResult} from "./markdown.js";
+import {searchIndex} from "./minisearch.json.js";
 import {renderPreview, resolveStylesheet} from "./render.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
 import {Telemetry} from "./telemetry.js";
@@ -108,6 +109,10 @@ export class PreviewServer {
         }
       } else if (pathname === "/_observablehq/client.js") {
         end(req, res, await rollupClient(getClientPath("./src/client/preview.js")), "text/javascript");
+      } else if (pathname === "/_observablehq/search.js") {
+        end(req, res, await rollupClient(getClientPath("./src/client/search.js")), "text/javascript");
+      } else if (pathname === "/_observablehq/minisearch.json") {
+        end(req, res, await searchIndex(config), "application/json");
       } else if ((match = /^\/_observablehq\/theme-(?<theme>[\w-]+(,[\w-]+)*)?\.css$/.exec(pathname))) {
         end(req, res, await bundleStyles({theme: match.groups!.theme?.split(",") ?? []}), "text/css");
       } else if (pathname.startsWith("/_observablehq/")) {

src/render.ts

@@ -58,7 +58,7 @@ type RenderInternalOptions =
   | {preview: true}; // preview
 
 async function render(parseResult: ParseResult, options: RenderOptions & RenderInternalOptions): Promise<string> {
-  const {root, base, path, pages, title, preview} = options;
+  const {root, base, path, pages, title, preview, search} = options;
   const sidebar = parseResult.data?.sidebar !== undefined ? Boolean(parseResult.data.sidebar) : options.sidebar;
   const toc = mergeToc(parseResult.data?.toc, options.toc);
   return String(html`<!DOCTYPE html>
@@ -90,7 +90,7 @@ import ${preview || parseResult.cells.length > 0 ? `{${preview ? "open, " : ""}d
 ${
   preview ? `\nopen({hash: ${JSON.stringify(parseResult.hash)}, eval: (body) => (0, eval)(body)});\n` : ""
 }${parseResult.cells.map((cell) => `\n${renderDefineCell(cell)}`).join("")}`)}
-</script>${sidebar ? html`\n${await renderSidebar(title, pages, path)}` : ""}${
+</script>${sidebar ? html`\n${await renderSidebar(title, pages, path, search)}` : ""}${
     toc.show ? html`\n${renderToc(findHeaders(parseResult), toc.label)}` : ""
   }
 <div id="observablehq-center">${renderHeader(options, parseResult.data)}
@@ -100,7 +100,7 @@ ${html.unsafe(parseResult.html)}</main>${renderFooter(path, options, parseResult
 `);
 }
 
-async function renderSidebar(title = "Home", pages: (Page | Section)[], path: string): Promise<Html> {
+async function renderSidebar(title = "Home", pages: (Page | Section)[], path: string, search: boolean): Promise<Html> {
   return html`<input id="observablehq-sidebar-toggle" type="checkbox" title="Toggle sidebar">
 <label id="observablehq-sidebar-backdrop" for="observablehq-sidebar-toggle"></label>
 <nav id="observablehq-sidebar">
@@ -109,7 +109,16 @@ async function renderSidebar(title = "Home", pages: (Page | Section)[], path: st
     <li class="observablehq-link${
       normalizePath(path) === "/index" ? " observablehq-link-active" : ""
     }"><a href="${relativeUrl(path, "/")}">${title}</a></li>
-  </ol>
+  </ol>${
+    search
+      ? html`<div id="observablehq-search" data-root="${relativeUrl(path, "/")}"><input type="search" />
+        </div>
+        <div id="observablehq-search-results"></div>
+        <script>{${html.unsafe(
+          (await rollupClient(getClientPath("./src/client/search-init.ts"), {minify: true})).trim()
+        )}}</script>`
+      : ""
+  }
   <ol>${pages.map((p, i) =>
     "pages" in p
       ? html`${i > 0 && "path" in pages[i - 1] ? html`</ol>` : ""}