diff --git a/.changeset/http-transport-cli-redirects.md b/.changeset/http-transport-cli-redirects.md new file mode 100644 index 0000000..ac1bc01 --- /dev/null +++ b/.changeset/http-transport-cli-redirects.md @@ -0,0 +1,19 @@ +--- +"prerender-crawler": minor +--- + +Prerender anything over HTTP, from the command line, with redirects the host can serve. + +**CLI.** `npx prerender-crawler --out ` prerenders a running server (`http://localhost:3000`) or a module exporting a request handler (`dist/server/server.js`) with no framework integration. Options cover seeds, mode, concurrency, interval, retries, link following, redirect handling, and `--continue` for skip-on-failure. + +**Transports.** `httpTransport(target, { headers?, fetch? })` re-addresses the crawl's requests to a running server and hands redirects back unfollowed; `moduleTransport(entry)` / `loadHandler(entry)` import a `handleRequest` / `fetch` / `default.fetch` module (what the Vite plugin now uses too). Both exported from `prerender-crawler`. + +**Redirects, reworked.** A 3xx path is now recorded in `result.redirects` (`{ from, to, status }`, one record per hop) and its same-origin target is crawled as a page in its own right, rendered once at its own URL. The redirected path gets a meta-refresh stub pointing at the chain's _final_ destination. Previously an internal redirect wrote the destination's full HTML under the old path — duplicate content with no canonical — and never rendered the destination at its own URL unless something linked to it. A redirect to another spelling of the same page (`/posts → /posts/`) is followed in place. `RenderedPage.redirect` marks stub pages so sitemap tooling can skip them. + +**`redirects()` integration.** Emits the crawl's redirects as a `_redirects` rules file (Netlify / Cloudflare Pages format; `filename`, `force`, and `format()` options for others) and declares the new `PrerenderIntegration.handlesRedirects`, which makes the engine skip its stubs — on Netlify a stub file would shadow the rule. The new `redirectStubs` option controls stubs directly. + +**Integration context.** `PrerenderContext` gains live, read-only `pages` and `redirects` views, complete by `teardown`. + +**Fixed:** `interval` now bounds the gap between _actual_ request starts. Previously it spaced claimed time slots, so a start delayed by a busy event loop could be followed by an on-time one less than `interval` later. + +**Removed:** `maxRedirects`. Chains are no longer followed in place, so there is nothing to bound; cycles terminate naturally because each path is crawled once. diff --git a/packages/crawler/README.md b/packages/crawler/README.md index a82d201..b06dfd7 100644 --- a/packages/crawler/README.md +++ b/packages/crawler/README.md @@ -2,7 +2,39 @@ Framework-agnostic build-time prerendering. Point it at anything fetch-shaped — `Request` in, `Response` out — and it crawls the site into static files: seed pages, link discovery, header hints, redirects, retries, throttling, and an integration seam for capturing build-time data alongside the pages. No browser, no subprocess, no framework knowledge. -Ships as an engine (`prerender-crawler`) and a Vite plugin built on it (`prerender-crawler/vite`). +Ships as an engine (`prerender-crawler`), a Vite plugin built on it (`prerender-crawler/vite`), and a CLI for everything else. + +## CLI + +Prerender any running server, or any module exporting a request handler, with no framework integration at all: + +```sh +# a running server — a framework's preview server, a container, a staging deploy +npx prerender-crawler http://localhost:3000 --out dist + +# a built server module (handleRequest, fetch, or default.fetch), in-process +npx prerender-crawler dist/server/server.js --out dist/client --redirects +``` + +``` +prerender-crawler --out [options] + -o, --out Output directory (required) + -p, --page Seed page; repeatable. Default: / + -m, --mode static (default) or hybrid + -c, --concurrency Pages in flight at once. Default: 8 + -i, --interval Minimum ms between request starts. Default: 0 + -r, --retries Re-fetch attempts for a failed page. Default: 2 + --origin Origin requests are minted under (module targets) + --hint-header Response header naming extra paths. Default: x-prerender + --redirects Write redirects as _redirects rules instead of stubs + --redirects-file Rules file name (implies --redirects). Default: _redirects + --no-links Do not follow links in rendered pages + --no-redirect-stubs Write no meta-refresh stubs at redirected paths + --continue Skip pages that fail instead of failing the run + --flat Write /about as about.html instead of about/index.html +``` + +For an HTTP target the crawl origin is the target's, so absolute links in the rendered HTML count as same-origin. ## Vite plugin @@ -49,42 +81,61 @@ The one distinction every downstream policy keys on: ## Engine -The plugin is a thin driver. The engine works with any transport: +The plugin and CLI are thin drivers. The engine works with any transport — anything with a `fetch(request: Request): Promise`: ```ts -import { runPrerender } from "prerender-crawler"; +import { runPrerender, httpTransport, moduleTransport } from "prerender-crawler"; const result = await runPrerender({ - transport: { fetch: request => app.handle(request) }, + transport: { fetch: request => app.handle(request) }, // or: + // transport: httpTransport("http://localhost:3000"), a running server + // transport: await moduleTransport("dist/server.js"), a handler module outDir: "dist", pages: ["/", "/about", { path: "/404", filename: "404.html" }], mode: "static" }); -result.pages; // RenderedPage[] — path, referrers, filename, emitted, html -result.files; // EmittedFile[] — what integrations emitted -result.skipped; // SkippedPage[] — failures left out (failOnError: false) +result.pages; // RenderedPage[] — path, referrers, filename, emitted, html, redirect? +result.redirects; // RedirectRecord[] — { from, to, status }, one per redirected path +result.files; // EmittedFile[] — what integrations emitted +result.skipped; // SkippedPage[] — failures left out (failOnError: false) ``` +`httpTransport` sends the crawl's requests to the target's origin (path and query kept) and hands redirects back as the 3xx responses the server sent. Pass `{ headers }` for an auth token or `{ fetch }` for a custom implementation. `moduleTransport` imports a module exporting `handleRequest`, `fetch`, or `default.fetch` and calls it directly. + +### Redirects + +A path that answers 3xx is recorded (`result.redirects`, one record per hop — `/a → /b → /c` is two records, the way host rules spell it) and its same-origin target is crawled as a page in its own right, so the destination renders once at its own URL. The redirected path itself gets a **meta-refresh stub** pointing at the chain's final destination, so the old URL keeps working on hosts with no redirect support. A redirect to another spelling of the same page (`/posts → /posts/`) is followed in place, not recorded. + +Hosts with real redirect rules do better than stubs: + +```ts +import { redirects } from "prerender-crawler"; + +runPrerender({ integrations: [redirects()] }); // or prerender({ integrations: [redirects()] }) +``` + +`redirects()` emits a `_redirects` file (`/from /to 301`, the format Netlify and Cloudflare Pages share) and declares `handlesRedirects`, which stops the engine writing stubs — necessary on Netlify, where an existing file shadows the rule. Options: `filename`, `force` (Netlify's `301!`), and `format(records)` for another host's syntax. + ### Engine options -| Option | Default | | -| ------------------------ | ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `mode` | `"static"` | See [Modes](#modes). Decides the `emitPages` default. | -| `pages` | `["/"]` | Seeds: strings, `{ path, filename?, emit? }` entries, or a (async) function returning them. Duplicates collapse to one render. | -| `crawlLinks` | `true` | Follow same-origin links in rendered HTML. The only way dynamic routes are discovered without explicit seeding. | -| `hintHeader` | `"x-prerender"` | Response header naming additional paths (comma-separated) — the route the data lives on announces the routes built from it. | -| `filter` | | `(path) => boolean`; drops a discovered path before it's fetched. | -| `concurrency` | `8` | Pages in flight at once. | -| `interval` | `0` | Minimum ms between the starts of consecutive requests across all workers — a throttle for renders hitting rate-limited APIs. | -| `retries` / `retryDelay` | `2` / `500` | Re-fetch attempts for a failed page, and the wait between them. | -| `failOnError` | `true` | Whether a page that still fails after retries fails the run. Otherwise it's reported in `skipped`, with the pages that linked to it. | -| `maxRedirects` | `5` | Internal redirect hops followed for one page. | -| `emitPages` | `true` static / `false` hybrid | Whether rendered pages are written: a boolean, or a per-path predicate. Per-entry `emit` overrides. Unemitted pages still render fully — links are still followed, data still captured. | -| `autoSubfolderIndex` | `true` | `/about` → `about/index.html` (true) or `about.html` (false). | -| `origin` | `"http://localhost"` | Origin requests are minted under. | -| `onRendered` | | Observes every rendered page — the seam for sitemaps and post-processing. | -| `integrations` | `[]` | See below. | +| Option | Default | | +| ------------------------ | ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `mode` | `"static"` | See [Modes](#modes). Decides the `emitPages` default. | +| `pages` | `["/"]` | Seeds: strings, `{ path, filename?, emit? }` entries, or a (async) function returning them. Duplicates collapse to one render. | +| `crawlLinks` | `true` | Follow same-origin links in rendered HTML. The only way dynamic routes are discovered without explicit seeding. | +| `hintHeader` | `"x-prerender"` | Response header naming additional paths (comma-separated) — the route the data lives on announces the routes built from it. | +| `filter` | | `(path) => boolean`; drops a discovered path before it's fetched. | +| `concurrency` | `8` | Pages in flight at once. | +| `interval` | `0` | Minimum ms between the starts of consecutive requests across all workers — a throttle for renders hitting rate-limited APIs. | +| `retries` / `retryDelay` | `2` / `500` | Re-fetch attempts for a failed page, and the wait between them. | +| `failOnError` | `true` | Whether a page that still fails after retries fails the run. Otherwise it's reported in `skipped`, with the pages that linked to it. | +| `redirectStubs` | `true` unless an integration `handlesRedirects` | Whether redirected paths get a meta-refresh stub file pointing at the chain's end. See [Redirects](#redirects). | +| `emitPages` | `true` static / `false` hybrid | Whether rendered pages are written: a boolean, or a per-path predicate. Per-entry `emit` overrides. Unemitted pages still render fully — links are still followed, data still captured. | +| `autoSubfolderIndex` | `true` | `/about` → `about/index.html` (true) or `about.html` (false). | +| `origin` | `"http://localhost"` | Origin requests are minted under. | +| `onRendered` | | Observes every rendered page — the seam for sitemaps and post-processing. | +| `integrations` | `[]` | See below. | ### Integrations @@ -95,6 +146,7 @@ interface PrerenderIntegration { name: string; setup?(context: PrerenderContext): void | Promise; // before the first render teardown?(context: PrerenderContext): void | Promise; // after the last render, before writes + handlesRedirects?: boolean; // "I write host redirect rules" — the engine skips its stubs client?: string; // module a bundler plugin imports into the client build (reserved) } @@ -102,14 +154,18 @@ interface PrerenderContext { mode: PrerenderMode; origin: string; outDir: string; + pages: readonly RenderedPage[]; // complete by teardown + redirects: readonly RedirectRecord[]; // complete by teardown emitFile(file: { filename: string; contents: string | Uint8Array }): void; } ``` -`emitFile` is the channel for artifacts produced during the crawl — captured server-function results, extracted payloads, sitemaps. Throwing from `teardown` fails the run: the place to verify the crawl produced everything the runtime half will need. [`@solidjs/prerender`](../solid) is the reference integration. +`emitFile` is the channel for artifacts produced during the crawl — captured server-function results, extracted payloads, sitemaps. Throwing from `teardown` fails the run: the place to verify the crawl produced everything the runtime half will need. `redirects()` above is the smallest example; [`@solidjs/prerender`](../solid) is the reference integration. ### Utilities +- `httpTransport(target, { headers?, fetch? })`, `moduleTransport(entry)`, `loadHandler(entry)` — the shipped transports. +- `redirects(options?)`, `formatRedirectsFile(records, force?)` — the redirects integration and its `_redirects` formatter. - `fileRoutePages({ root, dir, extensions })` / `staticRoutePaths(entries)` — the static page paths of a `filesystem-routing` manifest, as a `pages` source. - `extractLinks(html)`, `normalizeLink(href, from)`, `normalizePath(path)`, `outputFilename(path, autoSubfolderIndex)` — the crawl's own primitives. diff --git a/packages/crawler/package.json b/packages/crawler/package.json index b82f3e8..6ca4de6 100644 --- a/packages/crawler/package.json +++ b/packages/crawler/package.json @@ -32,6 +32,9 @@ "default": "./dist/vite.js" } }, + "bin": { + "prerender-crawler": "./dist/cli.js" + }, "files": [ "dist", "LICENSE", diff --git a/packages/crawler/src/cli-main.ts b/packages/crawler/src/cli-main.ts new file mode 100644 index 0000000..4fef73c --- /dev/null +++ b/packages/crawler/src/cli-main.ts @@ -0,0 +1,164 @@ +// The CLI's logic, separated from the executable entry (./cli.ts) so tests +// can drive it with an argv and capture its output. +import path from "node:path"; +import { parseArgs } from "node:util"; +import { runPrerender } from "./crawl.ts"; +import { redirects } from "./redirects.ts"; +import { httpTransport, moduleTransport } from "./transports.ts"; +import type { PrerenderMode, PrerenderIntegration, Transport } from "./types.ts"; + +export const USAGE = `Usage: prerender-crawler --out [options] + +Prerenders a site into static files by crawling it. + + An http(s) origin of a running server, or the path of a + module exporting a Request -> Response handler + (handleRequest, fetch, or default.fetch). + +Options: + -o, --out Output directory (required) + -p, --page Seed page; repeatable. Default: / + -m, --mode static (default) or hybrid — see docs for what each writes + -c, --concurrency Pages in flight at once. Default: 8 + -i, --interval Minimum ms between request starts. Default: 0 + -r, --retries Re-fetch attempts for a failed page. Default: 2 + --origin Origin requests are minted under (module targets). + Default: http://localhost + --hint-header Response header naming extra paths. Default: x-prerender + --redirects Write the redirects as host rules (_redirects format, + Netlify / Cloudflare Pages) instead of meta-refresh stubs + --redirects-file Rules file name (implies --redirects). Default: _redirects + --no-links Do not follow links in rendered pages + --no-redirect-stubs Write no meta-refresh stubs at redirected paths + --continue Skip pages that fail instead of failing the run + --flat Write /about as about.html instead of about/index.html + -h, --help Show this help +`; + +export interface CliIO { + stdout(line: string): void; + stderr(line: string): void; +} + +/** Runs the CLI for `argv` (without the node and script entries). Returns the exit code. */ +export async function main(argv: string[], io: CliIO): Promise { + const parsed = parse(argv); + if ("error" in parsed) { + io.stderr(`${parsed.error}\n\n${USAGE}`); + return 2; + } + const { values, positionals } = parsed; + if (values.help) { + io.stdout(USAGE); + return 0; + } + const target = positionals[0]; + if (!target || positionals.length > 1 || !values.out) { + io.stderr( + !target + ? "Missing ." + : !values.out + ? "Missing --out ." + : `Unexpected argument: ${positionals[1]}.` + ); + io.stderr(`\n${USAGE}`); + return 2; + } + const mode = values.mode ?? "static"; + if (mode !== "static" && mode !== "hybrid") { + io.stderr(`--mode must be static or hybrid, got ${mode}.`); + return 2; + } + + let transport: Transport; + let origin = values.origin; + try { + if (/^https?:\/\//.test(target)) { + transport = httpTransport(target); + origin ??= new URL(target).origin; + } else { + transport = await moduleTransport(path.resolve(target)); + } + } catch (error) { + io.stderr(describe(error)); + return 1; + } + + const integrations: PrerenderIntegration[] = []; + if (values.redirects || values["redirects-file"] !== undefined) { + integrations.push(redirects({ filename: values["redirects-file"] })); + } + + const outDir = path.resolve(values.out); + try { + const result = await runPrerender({ + transport, + outDir, + mode: mode as PrerenderMode, + origin, + pages: values.page?.length ? values.page : undefined, + concurrency: values.concurrency !== undefined ? integer(values.concurrency) : undefined, + interval: values.interval !== undefined ? integer(values.interval) : undefined, + retries: values.retries !== undefined ? integer(values.retries) : undefined, + hintHeader: values["hint-header"], + crawlLinks: !values["no-links"], + redirectStubs: values["no-redirect-stubs"] ? false : undefined, + failOnError: !values.continue, + autoSubfolderIndex: !values.flat, + integrations + }); + const written = result.pages.filter(page => page.emitted).length; + const parts = [`rendered ${result.pages.length} page(s) (${written} written)`]; + if (result.redirects.length) parts.push(`${result.redirects.length} redirect(s)`); + if (result.files.length) parts.push(`${result.files.length} file(s) emitted`); + const relative = path.relative(process.cwd(), outDir); + const shown = relative === "" ? "." : relative.startsWith("..") ? outDir : relative; + io.stdout(`[prerender] ${parts.join(", ")} -> ${shown}`); + for (const miss of result.skipped) { + io.stderr(`[prerender] skipped ${miss.path}: ${describe(miss.error)}`); + } + return 0; + } catch (error) { + io.stderr(`[prerender] ${describe(error)}`); + return 1; + } +} + +const spec = { + allowPositionals: true, + options: { + out: { type: "string", short: "o" }, + page: { type: "string", short: "p", multiple: true }, + mode: { type: "string", short: "m" }, + concurrency: { type: "string", short: "c" }, + interval: { type: "string", short: "i" }, + retries: { type: "string", short: "r" }, + origin: { type: "string" }, + "hint-header": { type: "string" }, + redirects: { type: "boolean" }, + "redirects-file": { type: "string" }, + "no-links": { type: "boolean" }, + "no-redirect-stubs": { type: "boolean" }, + continue: { type: "boolean" }, + flat: { type: "boolean" }, + help: { type: "boolean", short: "h" } + } +} as const; + +function parse(argv: string[]) { + try { + return parseArgs({ args: argv, ...spec }); + } catch (error) { + return { error: describe(error) }; + } +} + +function integer(value: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`Expected a non-negative integer, got ${JSON.stringify(value)}.`); + } + return parsed; +} + +const describe = (error: unknown) => (error instanceof Error ? error.message : String(error)); diff --git a/packages/crawler/src/cli.ts b/packages/crawler/src/cli.ts new file mode 100644 index 0000000..9223bf0 --- /dev/null +++ b/packages/crawler/src/cli.ts @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { main } from "./cli-main.ts"; + +process.exitCode = await main(process.argv.slice(2), { + stdout: line => console.log(line), + stderr: line => console.error(line) +}); diff --git a/packages/crawler/src/crawl.ts b/packages/crawler/src/crawl.ts index 9e79fdc..3546ccc 100644 --- a/packages/crawler/src/crawl.ts +++ b/packages/crawler/src/crawl.ts @@ -8,6 +8,7 @@ import type { PrerenderContext, PrerenderOptions, PrerenderResult, + RedirectRecord, RenderedPage, Transport } from "./types.ts"; @@ -53,25 +54,32 @@ export async function runPrerender(options: RunOptions): Promise integration.handlesRedirects) } = options; const originUrl = new URL(origin); const rendered: RenderedPage[] = []; + const redirects: RedirectRecord[] = []; const skipped: PrerenderResult["skipped"] = []; const emitted: EmittedFile[] = []; const context: PrerenderContext = { mode, origin, outDir, + pages: rendered, + redirects, emitFile: file => void emitted.push(file) }; + const shouldEmit = (entry: PageEntry) => + entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages); const seeds = await resolveSeeds(options.pages); const seen = new Set(seeds.map(page => page.path)); @@ -93,46 +101,73 @@ export async function runPrerender(options: RunOptions): Promise now) await wait(slot - now); + let started = performance.now(); + while (started - lastStart < interval) { + await wait(lastStart + interval - started); + started = performance.now(); + } + lastStart = started; + if (nextSlot < started + interval) nextSlot = started + interval; } - async function fetchFollowingRedirects(path: string): Promise { - let url = new URL(path, originUrl); - for (let hop = 0; ; hop++) { - await pace(); - const response = await transport.fetch( - new Request(url, { headers: { accept: "text/html,*/*", [hintHeader]: "1" } }) - ); - const location = response.headers.get("location"); - if (response.status < 300 || response.status >= 400 || !location) return response; - if (hop >= maxRedirects) { - throw new Error(`Redirect chain from ${path} exceeded ${maxRedirects} hops`); - } - const target = new URL(location, url); - if (target.origin !== originUrl.origin) { - // an external redirect terminates the chain; the page becomes a stub - return response; - } - url = target; + // Redirects are not followed in place: a 3xx makes the path a redirect + // record (and a stub, see finalizeRedirects), and its same-origin target + // enters the queue as a page in its own right — so the destination is + // rendered once, at its own URL, and a chain is one record per hop the + // way a host's redirect rules would spell it. Cycles are harmless: the + // seen-set admits each path once. + async function fetchUrl(url: URL): Promise { + await pace(); + return transport.fetch( + new Request(url, { headers: { accept: "text/html,*/*", [hintHeader]: "1" } }) + ); + } + + // A redirect to a spelling of the SAME page (`/posts` -> `/posts/`, the + // trailing-slash canonicalization static servers do) is not a redirect + // between pages: it is followed here, once, and the page renders as + // itself. Anything else is the caller's to record. + async function fetchPage(path: string): Promise { + const url = new URL(path, originUrl); + const response = await fetchUrl(url); + const location = response.headers.get("location"); + if (!location || response.status < 300 || response.status >= 400) return response; + const target = new URL(location, url); + if (target.origin !== originUrl.origin || normalizePath(target.pathname) !== path) { + return response; } + return fetchUrl(target); } + const pendingRedirects: Array<{ + entry: PageEntry; + filename: string; + response: Response; + redirect: RedirectRecord; + }> = []; + async function renderPage(entry: PageEntry): Promise { let response: Response | undefined; let error: unknown; for (let attempt = 0; attempt <= retries; attempt++) { if (attempt > 0) await wait(retryDelay); try { - response = await fetchFollowingRedirects(entry.path); + response = await fetchPage(entry.path); error = undefined; if (response.status < 500) break; // retry only what might heal } catch (thrown) { @@ -158,38 +193,39 @@ export async function runPrerender(options: RunOptions): Promise= 300 && response.status < 400) { - // external redirect (internal ones were followed): a meta-refresh stub - // keeps the path working on hosts without redirect support - html = redirectStub(location); - } else { - html = await response.text(); + const pageUrl = new URL(entry.path, originUrl); + const target = new URL(location, pageUrl); + const internal = target.origin === originUrl.origin; + const to = internal ? normalizePath(target.pathname) : target.href; + const redirect: RedirectRecord = { from: entry.path, to, status: response.status }; + redirects.push(redirect); + if (internal) discovered(to, entry.path); + // the stub needs the chain's end, known only once the crawl settles + pendingRedirects.push({ entry, filename, response, redirect }); + return; } + const html = await response.text(); // Emission is policy, rendering is not: an unemitted page has still // fully executed (integration capture happened server-side) and its // links still feed the crawl — it just leaves no HTML file behind to // shadow a live server's SSR of the route. - const emitted = - entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages); + const emitted = shouldEmit(entry); if (emitted) await writeOutput(outDir, filename, html); - if (isHTML) { + if (crawlLinks && (response.headers.get("content-type") ?? "").includes("text/html")) { const pageUrl = new URL(entry.path, originUrl); - if (crawlLinks) { - for (const path of extractLinks(html, pageUrl)) discovered(path, entry.path); - } - } - const hints = response.headers.get(hintHeader); - if (hints) { - for (const hint of hints.split(",")) { - const path = normalizeLink(hint.trim(), originUrl, originUrl.origin); - if (path !== undefined) discovered(path, entry.path); - } + for (const path of extractLinks(html, pageUrl)) discovered(path, entry.path); } const page: RenderedPage = { @@ -204,6 +240,41 @@ export async function runPrerender(options: RunOptions): Promise { + const hops = new Map(redirects.map(record => [record.from, record.to])); + // the chain's end — or, in a cycle (which has none), one hop on + const destination = (from: string) => { + const next = hops.get(from)!; + const visited = new Set([from]); + let at = next; + while (hops.has(at)) { + if (visited.has(at)) return next; + visited.add(at); + at = hops.get(at)!; + } + return at; + }; + for (const { entry, filename, response, redirect } of pendingRedirects) { + const html = redirectStub(destination(entry.path)); + const emitted = redirectStubs && shouldEmit(entry); + if (emitted) await writeOutput(outDir, filename, html); + const page: RenderedPage = { + path: entry.path, + referrers: referrersOf(entry.path), + filename, + emitted, + response, + html, + redirect + }; + rendered.push(page); + if (onRendered) await onRendered(page); + } + } + try { for (const integration of integrations) await integration.setup?.(context); @@ -235,6 +306,7 @@ export async function runPrerender(options: RunOptions): Promise (error instanceof Error ? error.message : String(error)); function redirectStub(location: string): string { - const target = String(location).replace(/"/g, """); + const target = location.replace(/&/g, "&").replace(/"/g, """); return ``; } diff --git a/packages/crawler/src/index.ts b/packages/crawler/src/index.ts index f11b7b1..c34ab06 100644 --- a/packages/crawler/src/index.ts +++ b/packages/crawler/src/index.ts @@ -2,6 +2,10 @@ export { runPrerender } from "./crawl.ts"; export type { RunOptions } from "./crawl.ts"; export { extractLinks, normalizeLink, normalizePath } from "./links.ts"; export { outputFilename } from "./output.ts"; +export { formatRedirectsFile, redirects } from "./redirects.ts"; +export type { RedirectsIntegrationOptions } from "./redirects.ts"; +export { httpTransport, loadHandler, moduleTransport } from "./transports.ts"; +export type { HttpTransportOptions, RequestHandler } from "./transports.ts"; export type { EmittedFile, PageEntry, @@ -11,6 +15,7 @@ export type { PrerenderMode, PrerenderOptions, PrerenderResult, + RedirectRecord, RenderedPage, SkippedPage, Transport diff --git a/packages/crawler/src/redirects.ts b/packages/crawler/src/redirects.ts new file mode 100644 index 0000000..47deca4 --- /dev/null +++ b/packages/crawler/src/redirects.ts @@ -0,0 +1,64 @@ +// The redirects integration: the redirects the crawl observed, written as +// the host's own rules so the deployed site answers them with real 3xx +// responses instead of the engine's meta-refresh stubs. +// +// The default output is the `_redirects` line format Netlify and Cloudflare +// Pages share: `/from /to 301`. One divergence between them shapes the +// design: Cloudflare applies a rule whether or not a file exists at the +// path, while Netlify lets an existing file shadow the rule unless it is +// forced (`301!`) — and Cloudflare rejects the forced form. So the +// integration declares `handlesRedirects`, which stops the engine writing +// stubs at redirected paths; with no file to shadow, the plain unforced +// rule works on both hosts. +import type { PrerenderIntegration, RedirectRecord } from "./types.ts"; + +export interface RedirectsIntegrationOptions { + /** Output file, relative to the output directory. @default "_redirects" */ + filename?: string; + /** + * Produces the file's contents from the run's redirects — for hosts with + * their own format. Defaults to the `_redirects` line format. + */ + format?(redirects: readonly RedirectRecord[]): string; + /** + * Netlify only: append `!` to force each rule past an existing file at + * its path. Unneeded when the engine writes no stubs (the default with + * this integration active), and Cloudflare Pages rejects the syntax. + * @default false + */ + force?: boolean; +} + +/** + * Emits the crawl's redirects as host rules — by default a `_redirects` file + * (Netlify, Cloudflare Pages). Declares `handlesRedirects`, so the engine + * writes no meta-refresh stubs at redirected paths. + * + * ```ts + * import { redirects } from "prerender-crawler"; + * prerender({ integrations: [redirects()] }) + * ``` + */ +export function redirects(options: RedirectsIntegrationOptions = {}): PrerenderIntegration { + const { filename = "_redirects", force = false } = options; + const format = options.format ?? (records => formatRedirectsFile(records, force)); + return { + name: "redirects", + handlesRedirects: true, + teardown(context) { + if (context.redirects.length === 0) return; + context.emitFile({ filename, contents: format(context.redirects) }); + } + }; +} + +/** + * The `_redirects` line format: `/from /to status`, one rule per line, + * sorted by source for a stable file across builds. + */ +export function formatRedirectsFile(redirects: readonly RedirectRecord[], force = false): string { + const lines = [...redirects] + .sort((a, b) => (a.from < b.from ? -1 : a.from > b.from ? 1 : 0)) + .map(({ from, to, status }) => `${from} ${to} ${status}${force ? "!" : ""}`); + return lines.join("\n") + "\n"; +} diff --git a/packages/crawler/src/transports.ts b/packages/crawler/src/transports.ts new file mode 100644 index 0000000..c9c0f23 --- /dev/null +++ b/packages/crawler/src/transports.ts @@ -0,0 +1,85 @@ +// The two transports the engine ships. Both are small because the engine's +// contract is small — `Request` in, `Response` out — and that is the point: +// anything answering that shape is prerenderable, whether it is a module +// in this process or a server on the other side of a socket. +import { pathToFileURL } from "node:url"; +import type { Transport } from "./types.ts"; + +export interface HttpTransportOptions { + /** Headers added to every request (an auth token for a preview deploy, say). */ + headers?: HeadersInit; + /** The fetch implementation to use. @default globalThis.fetch */ + fetch?: typeof fetch; +} + +/** + * Prerenders a RUNNING server over HTTP: every request the crawl mints is + * re-addressed to `target`'s origin (path and query kept) and sent with + * `fetch`. Works against anything that speaks HTTP — a framework's preview + * server, a container, a staging deploy — with no knowledge of what it is. + * + * Redirects are delivered to the engine as the 3xx responses the server + * sent (`redirect: "manual"`), not followed here: the engine records them, + * stubs them, and crawls their targets as pages in their own right. + * + * Set the run's `origin` to the same value so absolute links in the + * rendered HTML count as same-origin — the CLI does this for you. + */ +export function httpTransport(target: string | URL, options: HttpTransportOptions = {}): Transport { + const base = new URL(target); + const send = options.fetch ?? globalThis.fetch; + return { + async fetch(request) { + const url = new URL(request.url); + url.protocol = base.protocol; + url.host = base.host; + const headers = new Headers(request.headers); + new Headers(options.headers).forEach((value, key) => headers.set(key, value)); + return send(new Request(url, { method: request.method, headers, redirect: "manual" })); + } + }; +} + +/** A `Request -> Response` handler, as a built server entry exports it. */ +export type RequestHandler = (request: Request) => Response | Promise; + +/** + * Imports a built server module and returns its request handler: + * `handleRequest`, `fetch`, or `default.fetch` — the shapes SSR entries and + * WinterCG-style servers export. + */ +export async function loadHandler(entry: string | URL): Promise { + const href = entry instanceof URL ? entry.href : pathToFileURL(entry).href; + let serverModule: Record; + try { + serverModule = await import(href); + } catch (cause) { + throw new Error( + `prerender could not import the server entry at ${entry}. Prerendering renders pages ` + + `through a Request -> Response handler — point it at a module exporting ` + + `handleRequest, fetch, or default.fetch.`, + { cause } + ); + } + const handler = + serverModule.handleRequest ?? + serverModule.fetch ?? + (serverModule.default as { fetch?: unknown } | undefined)?.fetch; + if (typeof handler !== "function") { + throw new Error( + `The server entry at ${entry} exports none of handleRequest, fetch, or default.fetch — ` + + `prerender needs a Request -> Response handler to render pages through.` + ); + } + return handler as RequestHandler; +} + +/** + * Prerenders in-process against a built server module — no HTTP server, + * no subprocess: the crawl calls the handler directly. This is what the + * Vite plugin drives after the build. + */ +export async function moduleTransport(entry: string | URL): Promise { + const handler = await loadHandler(entry); + return { fetch: async request => handler(request) }; +} diff --git a/packages/crawler/src/types.ts b/packages/crawler/src/types.ts index c1817c8..cb532bf 100644 --- a/packages/crawler/src/types.ts +++ b/packages/crawler/src/types.ts @@ -41,6 +41,19 @@ export type PagesSource = | Array | (() => Array | Promise>); +/** + * One redirect the crawl observed: a request for `from` answered 3xx. `to` + * is a normalized same-origin path, or an absolute URL when the redirect + * leaves the origin. A chain (`/a` -> `/b` -> `/c`) is recorded hop by + * hop, one record per path, exactly as a host's redirect rules would + * express it. + */ +export interface RedirectRecord { + from: string; + to: string; + status: number; +} + /** A page the engine rendered (and, when `emitted`, wrote). */ export interface RenderedPage { /** The normalized route path (`/about`), origin and query stripped. */ @@ -57,8 +70,17 @@ export interface RenderedPage { emitted: boolean; /** The response the transport answered with (body consumed). */ response: Response; - /** The rendered HTML. */ + /** + * The rendered HTML — or, for a redirected path, the meta-refresh stub + * that stands in for it (see `redirect`). + */ html: string; + /** + * Set when the path answered a redirect instead of a page. The stub in + * `html` points at the chain's FINAL destination; `redirect` records this + * path's own hop. Sitemap tooling should skip these. + */ + redirect?: RedirectRecord; } /** An extra artifact an integration ships alongside the rendered pages. */ @@ -90,6 +112,10 @@ export interface PrerenderContext { mode: PrerenderMode; origin: string; outDir: string; + /** Every page rendered so far — complete by `teardown`. Live view; do not mutate. */ + pages: readonly RenderedPage[]; + /** Every redirect observed so far — complete by `teardown`. Live view; do not mutate. */ + redirects: readonly RedirectRecord[]; emitFile(file: EmittedFile): void; } @@ -104,6 +130,14 @@ export interface PrerenderIntegration { * produced everything its runtime half will need. */ teardown?(context: PrerenderContext): void | Promise; + /** + * Declares that this integration turns the run's redirects into the + * host's own rules (a `_redirects` file, say). The engine then skips its + * meta-refresh stubs at redirected paths: they would be redundant, and on + * hosts where an existing file shadows a rule (Netlify) they would + * defeat it. Equivalent to `redirectStubs: false` on the run. + */ + handlesRedirects?: boolean; /** * A module specifier the bundler integration imports for side effects * into the CLIENT build when this integration is active — how an @@ -152,8 +186,16 @@ export interface PrerenderOptions { * @default true */ failOnError?: boolean; - /** Internal redirect hops followed for one page. @default 5 */ - maxRedirects?: number; + /** + * Whether a redirected path gets a meta-refresh stub file pointing at the + * chain's final destination, so the old URL keeps working on hosts with + * no redirect support of their own. Turn it off when redirects are + * expressed as host rules instead (an integration declaring + * `handlesRedirects` does so implicitly): a stub file next to a rule is + * redundant at best and, on hosts where files shadow rules, defeats it. + * @default true unless an integration declares `handlesRedirects` + */ + redirectStubs?: boolean; /** * Whether rendered pages are written to disk: a blanket policy, or a * per-path predicate; per-entry `emit` flags override it either way. A @@ -185,6 +227,8 @@ export interface SkippedPage { /** What a finished run reports. */ export interface PrerenderResult { pages: RenderedPage[]; + /** Every redirect the crawl observed, one record per redirected path. */ + redirects: RedirectRecord[]; /** Extra files integrations emitted. */ files: EmittedFile[]; /** Paths that failed and were skipped (only with `failOnError: false`). */ diff --git a/packages/crawler/src/vite.ts b/packages/crawler/src/vite.ts index 4b49aee..5845f46 100644 --- a/packages/crawler/src/vite.ts +++ b/packages/crawler/src/vite.ts @@ -24,15 +24,17 @@ // seed the crawl automatically, so a page nothing links to still builds. import { existsSync } from "node:fs"; import path from "node:path"; -import { pathToFileURL } from "node:url"; import type { Plugin } from "vite"; import { runPrerender } from "./crawl.ts"; import { fileRoutePages, hasFileSystemRouting } from "./file-routes.ts"; import type { FileRoutePagesOptions } from "./file-routes.ts"; +import { moduleTransport } from "./transports.ts"; import type { PageEntry, PrerenderOptions } from "./types.ts"; export { fileRoutePages, staticRoutePaths } from "./file-routes.ts"; export type { FileRoutePagesOptions, RouteEntryLike } from "./file-routes.ts"; +export { redirects } from "./redirects.ts"; +export type { RedirectsIntegrationOptions } from "./redirects.ts"; export type * from "./types.ts"; /** The `import.meta.env` key the plugin defines with the build's `PrerenderMode`. */ @@ -110,7 +112,17 @@ export function prerender(options: PrerenderPluginOptions = {}): Plugin { ? path.resolve(root, options.serverEntry) : path.join(ssrOut, "server.js"); - const handleRequest = await loadHandler(entry); + let transport; + try { + transport = await moduleTransport(entry); + } catch (error) { + throw new Error( + `${describe(error)} Prerendering renders pages through the server build — make ` + + `sure an SSR build runs (the server build is a build-time tool here; it need not ` + + `be deployed) or point \`serverEntry\` at the module.`, + { cause: error } + ); + } const routeSeeds = await fileRouteSeeds(root, options.fileRoutes); const { serverEntry: _entry, fileRoutes: _fileRoutes, pages, ...crawl } = options; const result = await runPrerender({ @@ -120,15 +132,18 @@ export function prerender(options: PrerenderPluginOptions = {}): Plugin { ...(typeof pages === "function" ? await pages() : (pages ?? ["/"])), ...routeSeeds ], - transport: { fetch: request => handleRequest(request) }, + transport, outDir: clientOut }); const written = result.pages.filter(page => page.emitted).length; const seeded = routeSeeds.length ? `, ${routeSeeds.length} seeded from file routes` : ""; + const redirected = result.redirects.length + ? `, ${result.redirects.length} redirect(s)` + : ""; logger.info( - `[prerender] rendered ${result.pages.length} page(s) (${written} written${seeded}), ` + - `${result.files.length} file(s) emitted -> ${path.relative(root, clientOut)}` + `[prerender] rendered ${result.pages.length} page(s) (${written} written${seeded})` + + `${redirected}, ${result.files.length} file(s) emitted -> ${path.relative(root, clientOut)}` ); for (const miss of result.skipped) { logger.warn(`[prerender] skipped ${miss.path}: ${describe(miss.error)}`); @@ -138,34 +153,6 @@ export function prerender(options: PrerenderPluginOptions = {}): Plugin { }; } -type Handler = (request: Request) => Promise; - -async function loadHandler(entry: string): Promise { - let serverModule: Record; - try { - serverModule = await import(pathToFileURL(entry).href); - } catch (cause) { - throw new Error( - `prerender could not import the built server entry at ${entry}. Prerendering renders ` + - `pages through the server build — make sure an SSR build runs (the server build is a ` + - `build-time tool here; it need not be deployed) or point \`serverEntry\` at a module ` + - `exporting handleRequest/fetch.`, - { cause } - ); - } - const handler = - serverModule.handleRequest ?? - serverModule.fetch ?? - (serverModule.default as { fetch?: unknown } | undefined)?.fetch; - if (typeof handler !== "function") { - throw new Error( - `The server entry at ${entry} exports none of handleRequest, fetch, or default.fetch — ` + - `prerender needs a Request -> Response handler to render pages through.` - ); - } - return handler as Handler; -} - async function fileRouteSeeds( root: string, option: PrerenderPluginOptions["fileRoutes"] diff --git a/packages/crawler/test/cli.test.ts b/packages/crawler/test/cli.test.ts new file mode 100644 index 0000000..7aa8928 --- /dev/null +++ b/packages/crawler/test/cli.test.ts @@ -0,0 +1,131 @@ +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { Server } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { USAGE, main } from "../src/cli-main.ts"; + +function io() { + const out: string[] = []; + const err: string[] = []; + return { + out, + err, + io: { stdout: (l: string) => out.push(l), stderr: (l: string) => err.push(l) } + }; +} + +let dir: string; +beforeAll(async () => (dir = await mkdtemp(join(tmpdir(), "prerender-cli-")))); +afterAll(() => rm(dir, { recursive: true, force: true })); +let outDir: string; +afterEach(async () => outDir && (await rm(outDir, { recursive: true, force: true }))); + +describe("cli", () => { + it("prints usage and rejects missing arguments", async () => { + const help = io(); + expect(await main(["--help"], help.io)).toBe(0); + expect(help.out).toEqual([USAGE]); + + const noTarget = io(); + expect(await main(["--out", "x"], noTarget.io)).toBe(2); + expect(noTarget.err[0]).toMatch(/Missing /); + + const noOut = io(); + expect(await main(["./server.js"], noOut.io)).toBe(2); + expect(noOut.err[0]).toMatch(/Missing --out/); + + const badFlag = io(); + expect(await main(["./server.js", "--out", "x", "--bogus"], badFlag.io)).toBe(2); + expect(badFlag.err[0]).toMatch(/bogus/); + + const badMode = io(); + expect(await main(["./server.js", "--out", "x", "--mode", "fast"], badMode.io)).toBe(2); + expect(badMode.err[0]).toMatch(/--mode must be/); + }); + + it("prerenders a module target in-process with the given options", async () => { + const server = join(dir, "server.mjs"); + await writeFile( + server, + `export async function handleRequest(request) { + const path = new URL(request.url).pathname; + if (path === "/") return new Response('a o', { headers: { "content-type": "text/html" } }); + if (path === "/old") return new Response(null, { status: 301, headers: { location: "/about" } }); + if (path === "/about") return new Response("

about

", { headers: { "content-type": "text/html" } }); + return new Response("nope", { status: 404 }); + }` + ); + outDir = join(dir, "out-module"); + const run = io(); + const code = await main([server, "--out", outDir, "--redirects", "--flat"], run.io); + expect(run.err).toEqual([]); + expect(code).toBe(0); + expect(run.out[0]).toMatch( + /rendered 3 page\(s\) \(2 written\), 1 redirect\(s\), 1 file\(s\) emitted/ + ); + expect((await readdir(outDir)).sort()).toEqual(["_redirects", "about.html", "index.html"]); + expect(await readFile(join(outDir, "_redirects"), "utf8")).toBe("/old /about 301\n"); + }); + + it("prerenders a running server over HTTP, minting requests under its origin", async () => { + const hits: string[] = []; + const server: Server = createServer((req, res) => { + hits.push(req.url!); + res.writeHead(200, { "content-type": "text/html" }); + // an ABSOLUTE link to this server counts as same-origin only if the + // crawl origin is the target's — which the CLI arranges + res.end( + req.url === "/" + ? `d` + : "deep" + ); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as { port: number }; + try { + outDir = join(dir, "out-http"); + const run = io(); + const code = await main( + [`http://127.0.0.1:${port}`, "-o", outDir, "-p", "/", "--concurrency", "2"], + run.io + ); + expect(run.err).toEqual([]); + expect(code).toBe(0); + expect(hits.sort()).toEqual(["/", "/deep"]); + expect(await readFile(join(outDir, "deep/index.html"), "utf8")).toBe("deep"); + } finally { + await new Promise(resolve => server.close(() => resolve())); + } + }); + + it("fails with the engine's message, or skips with --continue", async () => { + const server = join(dir, "broken.mjs"); + await writeFile( + server, + `export const fetch = async r => new URL(r.url).pathname === "/" + ? new Response('m', { headers: { "content-type": "text/html" } }) + : new Response("nope", { status: 404 });` + ); + outDir = join(dir, "out-broken"); + const strict = io(); + expect(await main([server, "--out", outDir, "--retries", "0"], strict.io)).toBe(1); + expect(strict.err[0]).toMatch(/\/missing answered 404 \(linked from \/\)/); + + const lenient = io(); + expect(await main([server, "--out", outDir, "--retries", "0", "--continue"], lenient.io)).toBe( + 0 + ); + expect(lenient.out[0]).toMatch(/rendered 1 page/); + expect(lenient.err[0]).toMatch(/skipped \/missing/); + }); + + it("rejects a non-integer numeric option", async () => { + const run = io(); + const server = join(dir, "ok.mjs"); + await writeFile(server, `export const fetch = async () => new Response("x");`); + expect(await main([server, "--out", join(dir, "o"), "--concurrency", "two"], run.io)).toBe(1); + expect(run.err[0]).toMatch(/non-negative integer/); + }); +}); diff --git a/packages/crawler/test/crawl.test.ts b/packages/crawler/test/crawl.test.ts index 6fe8f57..57c6520 100644 --- a/packages/crawler/test/crawl.test.ts +++ b/packages/crawler/test/crawl.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { runPrerender } from "../src/crawl.ts"; +import { redirects } from "../src/redirects.ts"; import type { Transport } from "../src/types.ts"; type Answer = Response | (() => Response); @@ -137,43 +138,178 @@ describe("crawl", () => { expect(requests).not.toContain("/admin/secrets"); }); - it("follows internal redirects and stubs external ones", async () => { - const { transport } = site({ - "/moved": new Response(null, { status: 301, headers: { location: "/target" } }), - "/target": html("landed"), - "/gone": new Response(null, { - status: 302, - headers: { location: "https://elsewhere.example/x" } - }) + describe("redirects", () => { + const redirect = (location: string, status = 301) => + new Response(null, { status, headers: { location } }); + + it("records each hop, crawls the destination as its own page, and stubs the old path to the chain's end", async () => { + const { transport, requests } = site({ + "/oldest": redirect("/old", 301), + "/old": redirect("/new", 302), + "/new": html("landed") + }); + const result = await runPrerender({ + transport, + outDir: await makeOutDir(), + pages: ["/oldest"], + crawlLinks: false + }); + + expect(result.redirects).toEqual([ + { from: "/oldest", to: "/old", status: 301 }, + { from: "/old", to: "/new", status: 302 } + ]); + // the destination rendered once, at its own URL, discovered via the chain + expect(requests.sort()).toEqual(["/new", "/old", "/oldest"]); + expect(await readFile(join(outDir, "new/index.html"), "utf8")).toBe("landed"); + const landed = result.pages.find(p => p.path === "/new")!; + expect(landed.redirect).toBeUndefined(); + expect(landed.referrers).toEqual(["/old"]); + // both redirected paths stub straight to the FINAL destination + for (const path of ["oldest", "old"]) { + const stub = await readFile(join(outDir, `${path}/index.html`), "utf8"); + expect(stub).toContain('content="0; url=/new"'); + expect(stub).toContain('rel="canonical" href="/new"'); + } + const oldest = result.pages.find(p => p.path === "/oldest")!; + expect(oldest.redirect).toEqual({ from: "/oldest", to: "/old", status: 301 }); + expect(oldest.emitted).toBe(true); }); - await runPrerender({ - transport, - outDir: await makeOutDir(), - pages: ["/moved", "/gone"], - crawlLinks: false + + it("stubs external redirects with the absolute target and does not crawl it", async () => { + const { transport, requests } = site({ + "/gone": redirect("https://elsewhere.example/x?q=1&r=2", 302) + }); + const result = await runPrerender({ + transport, + outDir: await makeOutDir(), + pages: ["/gone"], + crawlLinks: false + }); + expect(result.redirects).toEqual([ + { from: "/gone", to: "https://elsewhere.example/x?q=1&r=2", status: 302 } + ]); + expect(requests).toEqual(["/gone"]); + const stub = await readFile(join(outDir, "gone/index.html"), "utf8"); + // attribute-escaped, so the query survives HTML parsing intact + expect(stub).toContain('url=https://elsewhere.example/x?q=1&r=2"'); }); - // the internal chain landed and the ORIGINAL path holds the content - expect(await readFile(join(outDir, "moved/index.html"), "utf8")).toBe("landed"); - // the external redirect became a meta-refresh stub - const stub = await readFile(join(outDir, "gone/index.html"), "utf8"); - expect(stub).toContain("url=https://elsewhere.example/x"); - }); - it("bounds redirect chains", async () => { - const { transport } = site({ - "/a": new Response(null, { status: 302, headers: { location: "/b" } }), - "/b": new Response(null, { status: 302, headers: { location: "/a" } }) + it("follows a redirect to another spelling of the same page in place", async () => { + // the trailing-slash canonicalization static file servers perform + const { transport, requests } = site({ + "/posts": redirect("/posts/", 301), + "/posts/": html("the posts") + }); + const result = await runPrerender({ + transport, + outDir: await makeOutDir(), + pages: ["/posts"], + crawlLinks: false + }); + expect(requests).toEqual(["/posts", "/posts/"]); + expect(result.redirects).toEqual([]); + const page = result.pages.find(p => p.path === "/posts")!; + expect(page.redirect).toBeUndefined(); + expect(await readFile(join(outDir, "posts/index.html"), "utf8")).toBe("the posts"); }); - await expect( - runPrerender({ + + it("terminates redirect cycles", async () => { + const { transport, requests } = site({ + "/a": redirect("/b", 302), + "/b": redirect("/a", 302) + }); + const result = await runPrerender({ transport, outDir: await makeOutDir(), pages: ["/a"], crawlLinks: false, - maxRedirects: 3, retries: 0 - }) - ).rejects.toThrow(/exceeded 3 hops/); + }); + expect(requests.sort()).toEqual(["/a", "/b"]); + expect(result.redirects).toHaveLength(2); + // a cycle has no end; the stub points one hop on rather than hanging + expect(await readFile(join(outDir, "a/index.html"), "utf8")).toContain("url=/b"); + }); + + it("writes no stubs when asked, or when an integration handles redirects", async () => { + const routes = { "/old": redirect("/new"), "/new": html("landed") }; + const explicit = await runPrerender({ + transport: site(routes).transport, + outDir: await makeOutDir(), + pages: ["/old"], + crawlLinks: false, + redirectStubs: false + }); + expect(explicit.pages.find(p => p.path === "/old")!.emitted).toBe(false); + expect(await readdir(outDir)).toEqual(["new"]); + + await rm(outDir, { recursive: true }); + const seen: string[] = []; + const declared = await runPrerender({ + transport: site(routes).transport, + outDir: await makeOutDir(), + pages: ["/old"], + crawlLinks: false, + integrations: [ + { + name: "rules", + handlesRedirects: true, + teardown(context) { + seen.push(...context.redirects.map(r => `${r.from}>${r.to}`)); + // pages are complete by teardown, redirect pages included + expect(context.pages.map(p => p.path).sort()).toEqual(["/new", "/old"]); + } + } + ] + }); + expect(seen).toEqual(["/old>/new"]); + expect(declared.pages.find(p => p.path === "/old")!.emitted).toBe(false); + expect(await readdir(outDir)).toEqual(["new"]); + }); + + it("the redirects() integration emits a _redirects rules file and suppresses stubs", async () => { + const { transport } = site({ + "/b-old": redirect("/b", 301), + "/a-old": redirect("https://elsewhere.example/", 302), + "/b": html("b") + }); + const result = await runPrerender({ + transport, + outDir: await makeOutDir(), + pages: ["/b-old", "/a-old"], + crawlLinks: false, + integrations: [redirects()] + }); + expect(result.files).toEqual([ + { + filename: "_redirects", + contents: "/a-old https://elsewhere.example/ 302\n/b-old /b 301\n" + } + ]); + expect((await readdir(outDir)).sort()).toEqual(["_redirects", "b"]); + + // Netlify's forced form and a custom filename + const { transport: again } = site({ "/x": redirect("/y"), "/y": html("y") }); + const forced = await runPrerender({ + transport: again, + outDir: await makeOutDir(), + pages: ["/x"], + crawlLinks: false, + integrations: [redirects({ filename: "rules.txt", force: true })] + }); + expect(forced.files[0]).toEqual({ filename: "rules.txt", contents: "/x /y 301!\n" }); + }); + + it("the redirects() integration emits nothing when nothing redirected", async () => { + const { transport } = site({ "/": html("home") }); + const result = await runPrerender({ + transport, + outDir: await makeOutDir(), + integrations: [redirects()] + }); + expect(result.files).toEqual([]); + }); }); it("retries 5xx answers and succeeds when the page heals", async () => { @@ -349,11 +485,43 @@ describe("crawl", () => { starts.sort((a, b) => a - b); for (let i = 1; i < starts.length; i++) { // timers may fire a hair early; the gap must be essentially the interval - expect(starts[i] - starts[i - 1]).toBeGreaterThanOrEqual(18); + expect(starts[i] - starts[i - 1]).toBeGreaterThanOrEqual(19); } expect(starts).toHaveLength(6); }); + it("keeps actual starts apart even when one runs late", async () => { + // A busy event loop fires a claimed start's timer late; the NEXT claim's + // on-time slot must not then land within the interval of that actual + // late start. The first request blocks the loop past the second's slot. + const starts: number[] = []; + const base = site({ "/a": html("a"), "/b": html("b"), "/c": html("c") }); + const transport: Transport = { + fetch(request) { + starts.push(performance.now()); + if (starts.length === 1) { + const until = performance.now() + 28; + while (performance.now() < until) { + /* the second start's timer (due at +20) fires ~8ms late */ + } + } + return base.transport.fetch(request); + } + }; + await runPrerender({ + transport, + outDir: await makeOutDir(), + pages: ["/a", "/b", "/c"], + crawlLinks: false, + concurrency: 3, + interval: 20 + }); + starts.sort((a, b) => a - b); + for (let i = 1; i < starts.length; i++) { + expect(starts[i] - starts[i - 1]).toBeGreaterThanOrEqual(19); + } + }); + it("respects the concurrency bound", async () => { let active = 0; let peak = 0; diff --git a/packages/crawler/test/transports.test.ts b/packages/crawler/test/transports.test.ts new file mode 100644 index 0000000..c994489 --- /dev/null +++ b/packages/crawler/test/transports.test.ts @@ -0,0 +1,104 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { Server } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { httpTransport, loadHandler, moduleTransport } from "../src/transports.ts"; + +/** A tiny HTTP site that echoes what it received, with one redirect. */ +let server: Server; +let origin: string; +const received: Array<{ url: string; method: string; headers: Record }> = []; + +beforeAll(async () => { + server = createServer((req, res) => { + received.push({ + url: req.url!, + method: req.method!, + headers: Object.fromEntries(Object.entries(req.headers).map(([k, v]) => [k, String(v)])) + }); + if (req.url === "/old") { + res.writeHead(301, { location: "/new" }).end(); + return; + } + res.writeHead(200, { "content-type": "text/html" }).end(`

${req.url}

`); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("no port"); + origin = `http://127.0.0.1:${address.port}`; +}); + +afterAll(() => new Promise(resolve => server.close(() => resolve()))); +afterEach(() => void received.splice(0)); + +describe("httpTransport", () => { + it("re-addresses requests to the target, keeping path and query, and forwards headers", async () => { + const transport = httpTransport(origin, { headers: { authorization: "Bearer t" } }); + const response = await transport.fetch( + new Request("http://localhost/about?x=1", { headers: { "x-prerender": "1" } }) + ); + expect(response.status).toBe(200); + expect(await response.text()).toBe("

/about?x=1

"); + expect(received).toHaveLength(1); + expect(received[0]!.url).toBe("/about?x=1"); + expect(received[0]!.method).toBe("GET"); + expect(received[0]!.headers["x-prerender"]).toBe("1"); + expect(received[0]!.headers.authorization).toBe("Bearer t"); + // the target's host, not the crawl origin's + expect(received[0]!.headers.host).toBe(new URL(origin).host); + }); + + it("hands redirects to the engine instead of following them", async () => { + const response = await httpTransport(origin).fetch(new Request("http://localhost/old")); + expect(response.status).toBe(301); + expect(response.headers.get("location")).toBe("/new"); + expect(received.map(r => r.url)).toEqual(["/old"]); + }); + + it("accepts a custom fetch", async () => { + const seen: string[] = []; + const transport = httpTransport("https://example.test", { + fetch: async request => { + seen.push(request.url); + return new Response("ok"); + } + }); + await transport.fetch(new Request("http://localhost/x")); + expect(seen).toEqual(["https://example.test/x"]); + }); +}); + +describe("moduleTransport", () => { + let dir: string; + beforeAll(async () => (dir = await mkdtemp(join(tmpdir(), "prerender-mod-")))); + afterAll(() => rm(dir, { recursive: true, force: true })); + + const write = async (name: string, source: string) => { + const file = join(dir, name); + await writeFile(file, source); + return file; + }; + + it("loads handleRequest, fetch, or default.fetch", async () => { + const shapes = { + "a.mjs": `export function handleRequest(r) { return new Response("handleRequest " + new URL(r.url).pathname); }`, + "b.mjs": `export const fetch = async r => new Response("fetch " + new URL(r.url).pathname);`, + "c.mjs": `export default { fetch: r => new Response("default " + new URL(r.url).pathname) };` + }; + for (const [name, source] of Object.entries(shapes)) { + const transport = await moduleTransport(await write(name, source)); + const response = await transport.fetch(new Request("http://localhost/p")); + expect(await response.text()).toBe( + `${name === "a.mjs" ? "handleRequest" : name === "b.mjs" ? "fetch" : "default"} /p` + ); + } + }); + + it("names the problem when the module is missing or exports no handler", async () => { + await expect(loadHandler(join(dir, "nope.mjs"))).rejects.toThrow(/could not import/); + const file = await write("empty.mjs", `export const x = 1;`); + await expect(loadHandler(file)).rejects.toThrow(/exports none of handleRequest/); + }); +});