diff --git a/package-lock.json b/package-lock.json index 9b93a59..a3bdefd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8516,7 +8516,7 @@ }, "packages/browser": { "name": "@harperfast/prerender-browser", - "version": "1.21.0", + "version": "1.22.0", "license": "Apache-2.0", "dependencies": { "mqtt": "^5.10.4", @@ -8556,7 +8556,7 @@ }, "packages/console": { "name": "@harperfast/prerender-console", - "version": "0.12.0", + "version": "0.13.0", "license": "Apache-2.0", "dependencies": { "undici": "^7.18.2" @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.64.1", + "version": "0.65.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/browser/README.md b/packages/browser/README.md index a8b3a71..96d5a6e 100644 --- a/packages/browser/README.md +++ b/packages/browser/README.md @@ -385,6 +385,32 @@ Two things worth checking before adding a rule: an attribute may be **load-beari `` on hydration, so stripping `ssr` would destroy the only marker distinguishing a healthy snapshot from an un-hydrated one. Strip what is inert, not what is merely non-visual. +### What the page asks of its origin — the `subrequests` tally (v1.22.0) + +Every result posts a count of the same-origin requests the page made beyond the document, judged by +whether a **shared cache** in front of the origin would have answered them (RFC 9111 rules): + +``` +subrequests: { sameOrigin, cacheable, uncacheable, unspecified, blocked } +scriptsStripped: +``` + +`uncacheable` is the per-page factor the plugin's offload figure needs — the XHR/API calls a +crawler that executes JavaScript would send to the origin when it runs the same page, which never +pass through the plugin and so cannot be counted anywhere else (prerender-plugin#153). It is +explicit-only: non-GET, an uncacheable status, `Set-Cookie`, `no-store` / `private` / `no-cache`, a +zero max-age, `Vary: *`, or an expired `Expires`. `cacheable` is explicit positive freshness; +`unspecified` had no freshness information at all and depends on the CDN's defaults, so it is +reported and counted on neither side. `blocked` is the same-origin requests this fleet's block list +aborted before any response — requests a crawler would make, of unknown class — the visible bound on +the undercount; blocked images, fonts and media are left out of it, since a CDN caches those as a +matter of course and they say nothing about k. `scriptsStripped` says whether the stored snapshot can make any of these calls when +a crawler runs it: with scripts stripped the factor is a **saving** at serve time, not a cost. + +Cost: header checks on a response hook that already fires per response; no body reads, nothing +awaited. The per-window stats line carries the totals as `subrequestsUncacheable` and +`subrequestsUnspecified`. + ## Custom renderer A renderer receives the Puppeteer `page` and the `RenderJob` and returns the serialized HTML (or diff --git a/packages/browser/package.json b/packages/browser/package.json index f4d667d..b6914b3 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-browser", - "version": "1.21.0", + "version": "1.22.0", "type": "module", "description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.", "keywords": [ diff --git a/packages/browser/src/RenderJob.ts b/packages/browser/src/RenderJob.ts index 2d173e7..c497537 100644 --- a/packages/browser/src/RenderJob.ts +++ b/packages/browser/src/RenderJob.ts @@ -2,6 +2,7 @@ import { setTimeout as sleep } from 'timers/promises'; import { request } from './external/http.js'; import logger from './util/Logger.js'; import { settings } from './settings.js'; +import type { SubrequestTally } from './subrequests.js'; import { encode } from './util/encoder.js'; import { getHostHealth, parseRetryAfter } from './HostHealth.js'; import { renderPhaseOf } from './util/renderPhase.js'; @@ -66,6 +67,13 @@ type RenderAttempt = { * request produces no response. */ subresourceErrors?: number; + /** + * Every same-origin response the page provoked beyond the document, by shared-cache verdict + * (src/subrequests.ts). `uncacheable` is the per-page factor the plugin applies at serve time + * to count the origin calls a script-running crawler's page-view makes — on both sides of the + * offload ledger (HarperFast/prerender-plugin#153). + */ + subrequests?: SubrequestTally; }; type OriginHttpResponse = { @@ -223,6 +231,14 @@ export default class RenderJob { redirectedTo: this.redirectedTo, isIndexable: this.isIndexable, structuredOffers: this.structuredOffers, + // The page's own origin traffic, and whether the snapshot being posted can still make it. + // Posted on EVERY result that had an attempt (a redirect or an error saw a partial tally + // and that is still what the page asked for), and `scriptsStripped` is this fleet's + // setting rather than a per-page fact — a snapshot stored with its scripts removed cannot + // hydrate anything when a crawler runs it, which is what turns the factor from a cost + // into a saving. An older plugin ignores both fields. + subrequests: this.latestAttempt?.subrequests, + scriptsStripped: settings.config.postProcess.stripScripts, outcome: this.outcome, // One slug for WHY there is no content (see the field doc). The redirect/error // fallbacks are derived here so every no-content result carries a reason without diff --git a/packages/browser/src/Worker.ts b/packages/browser/src/Worker.ts index 3a33e2b..0d35409 100644 --- a/packages/browser/src/Worker.ts +++ b/packages/browser/src/Worker.ts @@ -109,6 +109,12 @@ export default class RenderWorker { // cache is affected); `subresourceErrors` is the total refused assets (how badly). rendersDegraded: 0, subresourceErrors: 0, + // Same-origin requests the pages made that no shared cache would serve — the origin load a + // script-running crawler's page-view carries, summed over the window (RenderAttempt.subrequests). + // `subrequestsUnspecified` is the part with no freshness information at all, whose fate depends + // on the CDN's defaults — reported so the uncacheable figure can be read as the bound it is. + subrequestsUncacheable: 0, + subrequestsUnspecified: 0, renderTimes: [] as number[], // Per-phase wall-clock samples (ms), drained into percentiles by logStats. Attribute // the render time to network-wait (navTtfb/navTotal) vs in-browser work (settle/postProcess). @@ -269,6 +275,9 @@ export default class RenderWorker { // render "succeeded" — treat it like a failure count, not a curiosity. rendersDegraded: s.rendersDegraded, subresourceErrors: s.subresourceErrors, + // Per-window totals; divide by `completed` for the per-page factor the plugin applies. + subrequestsUncacheable: s.subrequestsUncacheable, + subrequestsUnspecified: s.subrequestsUnspecified, fromSitemap: s.fromSitemap, failures: failuresTotal, failuresByType: s.failures, @@ -499,6 +508,10 @@ export default class RenderWorker { this.stats.rendersDegraded++; this.stats.subresourceErrors += attempt.subresourceErrors; } + if (attempt?.subrequests) { + this.stats.subrequestsUncacheable += attempt.subrequests.uncacheable; + this.stats.subrequestsUnspecified += attempt.subrequests.unspecified; + } const t = attempt?.timings; if (t) { if (t.navTtfb !== undefined) this.stats.navTtfb.push(t.navTtfb); diff --git a/packages/browser/src/renderer.ts b/packages/browser/src/renderer.ts index 6477177..fe07c0e 100644 --- a/packages/browser/src/renderer.ts +++ b/packages/browser/src/renderer.ts @@ -2,6 +2,7 @@ import { Renderer } from './Worker.js'; import type { RenderTimings } from './RenderJob.js'; import { settings } from './settings.js'; import { CACHE_REPLAY_HEADER, getResourceCache } from './ResourceCache.js'; +import { countsAsBlocked, emptyTally, tallySubresponse } from './subrequests.js'; import type { PostProcessConfig } from './config.js'; import { canonicalizeUrl, canonicalVerdict } from './util/url.js'; import { markRenderPhase } from './util/renderPhase.js'; @@ -52,6 +53,12 @@ const renderer: Renderer = async (page, job) => { const timings: RenderTimings = {}; if (job.latestAttempt) job.latestAttempt.timings = timings; let navStart = 0; + // What the page asked of its own origin beyond the document, by shared-cache verdict — the + // per-page factor the offload figure downstream needs (src/subrequests.ts). Same in-place + // discipline as `timings`: the attempt holds the reference, so a partial tally survives an + // early return. + const subrequests = emptyTally(); + if (job.latestAttempt) job.latestAttempt.subrequests = subrequests; const blockedResourceTypes = new Set(config.block.resourceTypes); const blockedUrlPatterns = config.block.urlPatterns; @@ -117,10 +124,14 @@ const renderer: Renderer = async (page, job) => { return; } if (isBlockedUrl(req.url())) { + // A same-origin request this fleet refuses to make is one a crawler WOULD make, of a + // class nobody can judge without a response — counted so the undercount is visible. + if (isSameOrigin(req.url()) && countsAsBlocked(req.resourceType())) subrequests.blocked++; req.abort().catch(noop); return; } if (blockedResourceTypes.has(req.resourceType())) { + if (isSameOrigin(req.url()) && countsAsBlocked(req.resourceType())) subrequests.blocked++; // Stub blocked images (vs abort) so lazy-loaders keep their real src URLs. if (config.block.stubImages && req.resourceType() === 'image') { req.respond(STUB_IMAGE_RESPONSE).catch(noop); @@ -178,16 +189,27 @@ const renderer: Renderer = async (page, job) => { return; } + const sameOrigin = isSameOrigin(res.url()); // A same-origin asset the origin refused while the document succeeded. Recorded on the // attempt (mutated in place, like `timings`, so it survives an early return) and // aggregated by the worker — a render whose scripts all 403 otherwise reports as a // clean success. - if (res.status() >= 400 && isSameOrigin(res.url()) && job.latestAttempt) { + if (res.status() >= 400 && sameOrigin && job.latestAttempt) { job.latestAttempt.subresourceErrors = (job.latestAttempt.subresourceErrors ?? 0) + 1; } - if (!cache || !cache.isCacheableRequest(req)) return; const resHeaders = res.headers(); + // Every same-origin response the page provoked, judged by whether a shared cache in front + // of the origin would have answered it. Sub-frame documents count too: they are not the + // navigation, and a crawler's renderer loads them the same way. Header reads only — no + // body, no await — on a hook that already fires per response. + if (sameOrigin) { + tallySubresponse(subrequests, req.method(), res.status(), resHeaders, { + replayedFromOwnCache: Boolean(resHeaders[CACHE_REPLAY_HEADER]), + }); + } + + if (!cache || !cache.isCacheableRequest(req)) return; // Skip responses we just synthesized from our own cache. if (resHeaders[CACHE_REPLAY_HEADER]) return; const policy = cache.getCachePolicy(res); diff --git a/packages/browser/src/subrequests.ts b/packages/browser/src/subrequests.ts new file mode 100644 index 0000000..f27c602 --- /dev/null +++ b/packages/browser/src/subrequests.ts @@ -0,0 +1,133 @@ +/** + * What a page load asks of its own origin beyond the document — and how much of it a shared cache + * would have absorbed. + * + * WHY THIS IS MEASURED HERE. The prerender system's offload figure counts documents: bot requests + * the origin never saw, less the renders/probes/sitemap fetches this system made. It cannot see + * the other half of a page-view — the XHR/fetch calls the page's OWN scripts make once a crawler + * that executes JavaScript runs it — because those go crawler → CDN → origin and never pass + * through the plugin. But this process runs the same page in the same kind of browser, watches + * every request it makes, and reads every response's cache headers. So the per-page factor is + * measurable exactly once, here, and posted with the render for the plugin to apply at serve time + * (HarperFast/prerender-plugin#153). The arithmetic downstream needs one number per page — how + * many same-origin requests reach the origin whoever runs the page — which is the `uncacheable` + * count below. + * + * THE CLASSIFICATION IS ABOUT A SHARED CACHE, NOT OUR OWN. `ResourceCache.getCachePolicy` decides + * what THIS fleet may replay across renders and is deliberately narrow (scripts and stylesheets, + * private refused). The question here is what a CDN in front of the origin would serve without + * an origin round trip, for any request the page makes, so it follows RFC 9111's shared-cache + * rules and reports three verdicts rather than two: + * + * uncacheable — explicitly reaches the origin every time: a non-GET method, a status a cache + * may not store, `Set-Cookie`, `no-store` / `private` / `no-cache`, a zero max-age, + * `Vary: *`, or an `Expires` already in the past. Only this class is counted as + * origin load. + * cacheable — explicit positive freshness (`s-maxage`, `max-age`, or a future `Expires`). + * unspecified — no freshness information at all. A CDN may apply heuristic freshness or a + * configured default TTL, or may not; that is a deployment fact this process + * cannot see, so these are reported and counted on neither side. + * + * Nothing about the REQUEST's cookies or authorization is consulted: this fleet sends a bypass + * token the crawler would not, and a crawler's renderer starts with no cookies — the response is + * what decides shared cacheability, and `Set-Cookie` is the request-specific case it covers. + * + * Blocked requests are counted separately. `block.resourceTypes` / `block.urlPatterns` abort a + * request before any response exists, so a same-origin request this fleet refused to make is a + * request a crawler WOULD make whose class is unknown — the visible bound on the undercount. Static + * media (images, fonts, audio/video) is left out of that count: a fleet blocks those by the + * hundred, a CDN caches them as a matter of course, and counting them would make the bound read as + * an alarm about requests that say nothing about k. What remains — blocked scripts, XHR/fetch, + * documents, "other" — is exactly the class that might. + */ + +export type SubrequestClass = 'uncacheable' | 'cacheable' | 'unspecified'; + +export type SubrequestTally = { + /** Responses from the navigation origin, the document itself excluded. */ + sameOrigin: number; + cacheable: number; + uncacheable: number; + unspecified: number; + /** Same-origin requests this fleet's block list aborted before a response existed — static media excluded. */ + blocked: number; +}; + +// Resource types whose blocked requests are NOT counted as an unknown: static media a CDN caches. +const STATIC_MEDIA = new Set(['image', 'font', 'media']); + +/** Does a blocked same-origin request of this resource type count toward the `blocked` bound? */ +export const countsAsBlocked = (resourceType: string): boolean => !STATIC_MEDIA.has(resourceType); + +export const emptyTally = (): SubrequestTally => ({ + sameOrigin: 0, + cacheable: 0, + uncacheable: 0, + unspecified: 0, + blocked: 0, +}); + +// Status codes a cache may store without explicit freshness (RFC 9110 §15.1 "heuristically +// cacheable"), plus 308. Anything else — every 5xx, 401/403, 429 — reached the origin and will +// again. +const CACHEABLE_STATUSES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); + +// Lenient on purpose: the ABNF has no whitespace around `=` and no quotes on a delta-seconds value, +// but origins emit both (`max-age = 60`, `max-age="60"`) and the caches this classifier stands in +// for accept them. Strictness here would push a response a CDN happily caches into `unspecified`. +const directiveSeconds = (cc: string, name: string): number | null => { + const m = cc.match(new RegExp(`(?:^|[,\\s])${name}\\s*=\\s*"?(\\d+)"?`)); + return m ? parseInt(m[1], 10) : null; +}; + +/** + * Would a shared cache have answered this response without going to the origin? + * + * `headers` as puppeteer's `HTTPResponse.headers()` hands them: lower-cased keys, multi-valued + * headers joined. Pure, so the rules above are testable without a browser. + */ +export function classifySubresponse(method: string, status: number, headers: Record): SubrequestClass { + const verb = method.toUpperCase(); + if (verb !== 'GET' && verb !== 'HEAD') return 'uncacheable'; + if (!CACHEABLE_STATUSES.has(status)) return 'uncacheable'; + if (headers['set-cookie']) return 'uncacheable'; + + const cc = (headers['cache-control'] ?? '').toLowerCase(); + if (/(?:^|[,\s])(?:no-store|private|no-cache)(?:$|[,\s=])/.test(cc)) return 'uncacheable'; + if ((headers['vary'] ?? '').trim() === '*') return 'uncacheable'; + + // Shared caches honour s-maxage over max-age; a zero in either is "stale on arrival", i.e. a + // revalidation against the origin on every use — origin load, however the response is labelled. + const sMaxAge = directiveSeconds(cc, 's-maxage'); + if (sMaxAge !== null) return sMaxAge > 0 ? 'cacheable' : 'uncacheable'; + const maxAge = directiveSeconds(cc, 'max-age'); + if (maxAge !== null) return maxAge > 0 ? 'cacheable' : 'uncacheable'; + + if (headers['expires'] !== undefined) { + // Measured against the origin's own clock when it says what time it is; an unparseable + // Expires is "already expired" by specification. + const expires = Date.parse(headers['expires']); + if (Number.isNaN(expires)) return 'uncacheable'; + const dateHeader = headers['date'] !== undefined ? Date.parse(headers['date']) : NaN; + const now = Number.isNaN(dateHeader) ? Date.now() : dateHeader; + return expires > now ? 'cacheable' : 'uncacheable'; + } + + return 'unspecified'; +} + +/** Record one same-origin, non-navigation response in the tally. Mutates in place. */ +export function tallySubresponse( + tally: SubrequestTally, + method: string, + status: number, + headers: Record, + { replayedFromOwnCache = false } = {} +): void { + tally.sameOrigin++; + // A response this fleet replayed from its own resource cache passed getCachePolicy, which is + // stricter than the shared-cache rules here — so it is cacheable by construction, and its + // replayed headers (some stripped) are not the evidence to re-judge it on. + const verdict = replayedFromOwnCache ? 'cacheable' : classifySubresponse(method, status, headers); + tally[verdict]++; +} diff --git a/packages/browser/test/jobResult.test.ts b/packages/browser/test/jobResult.test.ts index 9eacede..7b6bc2b 100644 --- a/packages/browser/test/jobResult.test.ts +++ b/packages/browser/test/jobResult.test.ts @@ -124,3 +124,32 @@ test('a failed render posts outcome=error with the attempt error and derived rea { name: 'Error', message: 'Navigation timeout of 30000 ms exceeded' } ); }); + +// The per-page origin factor (HarperFast/prerender-plugin#153). The plugin reads an ABSENT +// `subrequests` as "this renderer predates the measurement", so a tally must ride every result +// that had an attempt — including one that never reached content — and `scriptsStripped` must +// say what this fleet does to the snapshot, since that is what turns the factor from a cost into +// a saving at serve time. +test('a render posts its same-origin subrequest tally and whether the snapshot keeps its scripts', async () => { + const job = makeJob(); + const attempt = job.attemptStarted(); + attempt.subrequests = { sameOrigin: 12, cacheable: 7, uncacheable: 4, unspecified: 1, blocked: 3 }; + job.httpResponse = { statusCode: 200, headers: {} }; + job.attemptEnded(undefined, 'ok'); + + const meta = await send(job); + assert.deepEqual(meta.subrequests, { sameOrigin: 12, cacheable: 7, uncacheable: 4, unspecified: 1, blocked: 3 }); + // The default config strips scripts; the field is the fleet's setting, posted as a boolean. + assert.equal(meta.scriptsStripped, true); +}); + +test('a result that never reached content still carries the partial tally', async () => { + const job = makeJob(); + const attempt = job.attemptStarted(); + attempt.subrequests = { sameOrigin: 2, cacheable: 1, uncacheable: 1, unspecified: 0, blocked: 0 }; + job.attemptEnded(new Error('Navigation timeout of 30000 ms exceeded'), undefined); + + const meta = await send(job); + assert.equal(meta.outcome, 'error'); + assert.deepEqual(meta.subrequests, { sameOrigin: 2, cacheable: 1, uncacheable: 1, unspecified: 0, blocked: 0 }); +}); diff --git a/packages/browser/test/subrequests.test.ts b/packages/browser/test/subrequests.test.ts new file mode 100644 index 0000000..dff078d --- /dev/null +++ b/packages/browser/test/subrequests.test.ts @@ -0,0 +1,96 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { classifySubresponse, countsAsBlocked, emptyTally, tallySubresponse } from '../dist/subrequests.js'; + +// The shared-cache verdict behind the per-page factor `k` (HarperFast/prerender-plugin#153). +// +// The rules are RFC 9111's for a SHARED cache, and the three-way split is the point: an explicit +// "reaches the origin" is counted as origin load, explicit freshness is not, and a response with +// no freshness information is neither — its fate is the CDN's default TTL, which this process +// cannot see, so guessing either way would put a made-up number on the offload figure. + +const ok = (headers: Record = {}) => classifySubresponse('GET', 200, headers); + +test('a non-GET is origin load whatever the response says — that is what an API call is', () => { + assert.equal(classifySubresponse('POST', 200, { 'cache-control': 'public, max-age=3600' }), 'uncacheable'); + assert.equal(classifySubresponse('PUT', 200, {}), 'uncacheable'); + assert.equal(classifySubresponse('HEAD', 200, { 'cache-control': 'max-age=60' }), 'cacheable'); +}); + +test('a status a cache may not store reached the origin and will again', () => { + for (const status of [500, 502, 503, 401, 403, 429, 302, 307]) { + assert.equal(classifySubresponse('GET', status, { 'cache-control': 'max-age=60' }), 'uncacheable', String(status)); + } + // The heuristically cacheable set is judged on its headers like a 200. + assert.equal(classifySubresponse('GET', 404, { 'cache-control': 'max-age=60' }), 'cacheable'); + assert.equal(classifySubresponse('GET', 301, {}), 'unspecified'); +}); + +test('Set-Cookie, no-store, private and no-cache each mean the origin answers every time', () => { + assert.equal(ok({ 'set-cookie': 'sid=1', 'cache-control': 'max-age=3600' }), 'uncacheable'); + assert.equal(ok({ 'cache-control': 'no-store' }), 'uncacheable'); + assert.equal(ok({ 'cache-control': 'private, max-age=600' }), 'uncacheable'); + // no-cache is "revalidate before use": a conditional request to the origin per use. + assert.equal(ok({ 'cache-control': 'no-cache' }), 'uncacheable'); + assert.equal(ok({ 'cache-control': 'public, no-cache="set-cookie"' }), 'uncacheable'); + assert.equal(ok({ vary: '*' }), 'uncacheable'); +}); + +test('a directive is matched as a token, never as a substring of another', () => { + // "private" inside a made-up extension must not trip the private rule; the real max-age wins. + assert.equal(ok({ 'cache-control': 'x-privateer=1, max-age=60' }), 'cacheable'); + // "no-cache-ish" is not "no-cache". + assert.equal(ok({ 'cache-control': 'no-cache-ish, max-age=60' }), 'cacheable'); +}); + +test('s-maxage wins over max-age for a shared cache, and zero is stale on arrival', () => { + assert.equal(ok({ 'cache-control': 'max-age=0, s-maxage=600' }), 'cacheable'); + assert.equal(ok({ 'cache-control': 'max-age=600, s-maxage=0' }), 'uncacheable'); + assert.equal(ok({ 'cache-control': 'max-age=0' }), 'uncacheable'); + assert.equal(ok({ 'cache-control': 'public, max-age=31536000, immutable' }), 'cacheable'); +}); + +test('whitespace around = and a quoted delta-seconds are read the way the caches they stand in for read them', () => { + assert.equal(ok({ 'cache-control': 'public, max-age = 600' }), 'cacheable'); + assert.equal(ok({ 'cache-control': 'max-age= 0' }), 'uncacheable'); + assert.equal(ok({ 'cache-control': 's-maxage ="300", max-age=0' }), 'cacheable'); + assert.equal(ok({ 'cache-control': 'max-age="0"' }), 'uncacheable'); +}); + +test('Expires is read against the origin clock, and an unparseable one is already expired', () => { + assert.equal(ok({ expires: 'Thu, 01 Jan 2099 00:00:00 GMT' }), 'cacheable'); + assert.equal(ok({ expires: 'Thu, 01 Jan 1970 00:00:00 GMT' }), 'uncacheable'); + assert.equal(ok({ expires: '0' }), 'uncacheable'); + assert.equal(ok({ expires: '-1' }), 'uncacheable'); + // A clock-skewed origin: Expires one hour after ITS Date header is fresh whatever our clock says. + assert.equal(ok({ date: 'Thu, 01 Jan 2015 00:00:00 GMT', expires: 'Thu, 01 Jan 2015 01:00:00 GMT' }), 'cacheable'); + // Cache-Control freshness outranks Expires. + assert.equal(ok({ 'cache-control': 'max-age=0', 'expires': 'Thu, 01 Jan 2099 00:00:00 GMT' }), 'uncacheable'); +}); + +test('no freshness information is UNSPECIFIED — not a guess in either direction', () => { + assert.equal(ok({}), 'unspecified'); + assert.equal( + ok({ 'content-type': 'application/json', 'last-modified': 'Thu, 01 Jan 2015 00:00:00 GMT' }), + 'unspecified' + ); + assert.equal(ok({ 'cache-control': 'public' }), 'unspecified'); + assert.equal(ok({ vary: 'accept-encoding, accept' }), 'unspecified'); +}); + +test('the tally counts every same-origin response once, and a replay from our own cache as cacheable', () => { + const tally = emptyTally(); + tallySubresponse(tally, 'GET', 200, { 'cache-control': 'max-age=60' }); + tallySubresponse(tally, 'POST', 200, {}); + tallySubresponse(tally, 'GET', 200, {}); + // Replayed headers have been through filterReplayHeaders; the verdict must not depend on them. + tallySubresponse(tally, 'GET', 200, {}, { replayedFromOwnCache: true }); + assert.deepEqual(tally, { sameOrigin: 4, cacheable: 2, uncacheable: 1, unspecified: 1, blocked: 0 }); +}); + +test('a blocked image, font or media request is not an unknown — a CDN caches those as a matter of course', () => { + for (const type of ['image', 'font', 'media']) assert.equal(countsAsBlocked(type), false, type); + // The classes that MIGHT have reached the origin stay in the bound. + for (const type of ['xhr', 'fetch', 'script', 'document', 'other', 'stylesheet']) + assert.equal(countsAsBlocked(type), true, type); +}); diff --git a/packages/console/README.md b/packages/console/README.md index 90d9e09..c51f50b 100644 --- a/packages/console/README.md +++ b/packages/console/README.md @@ -175,18 +175,22 @@ term over time. Two caveats are written on the panel rather than assumed: a rend origin request (the document — the page's own subresources reach the origin only if the CDN does not cache them for the renderer), and the probe and sitemap counters land in the bucket where a _pass finished_, so a short range reads either none of a running sweep or all of one that just -ended — quote the 24h figure. One term the plugin cannot see at all is stated rather than -omitted, and it is missing from _both_ sides: the requests a page's own scripts make when a -rendering crawler runs it. Without this system every Googlebot/Bingbot/Applebot page-view costs the -origin the document _plus_ the page's XHR/API calls (the ones no CDN caches), so the "crawlers asked -for" baseline understates what the origin was spared; with it, a snapshot served without scripts -triggers none of those calls (a saving the figure does not credit), while a snapshot that keeps its -scripts, a proxied origin page, and every one of our own renders still trigger them (a cost it does -not charge). None of it passes through the plugin, so the figure is documents-only on both sides, -the net tile says "before crawler follow-up requests", and the panel reports the exposure (every -page handed to a crawler) as a count, never multiplied by a guessed factor — where snapshots are -served with scripts stripped, the true net offload for rendering crawlers is _higher_ than shown. -The render fleet can measure the per-page factor; applied to both sides, that tile becomes a number. +ended — quote the 24h figure. The fifth term — the requests a page's own scripts make when a +rendering crawler runs it — is **counted on both sides from console v0.13.0**, given a plugin ≥ 0.65.0 +and a render fleet ≥ 1.22.0: the fleet classifies every same-origin response by whether a shared +cache would answer it, the plugin stores the uncacheable count per page (k) and, on each serve to a +crawler the registry flags as executing scripts, emits `hydration_calls` with the side this serve +earned — `saved` (a script-stripped snapshot: the origin is spared k), `incurred` (a snapshot that +kept its scripts, or an origin page: the origin takes k), or `unknown` (no k yet). The baseline +becomes crawler requests plus every k those page-views carry, the cost side adds the crawlers' +incurred calls and the fleet's own render calls (`render` `subrequests`), and the documents-only +figure stays on the tile beside it. The `unknown` count is printed as the size of the remaining blind +spot — it decays over one render cycle after the fleet upgrade, and while it outnumbers the known +serves the panel says the figure is still mostly documents-only. Two bounds on k ride along: +responses with no freshness headers (a CDN default decides them — counted on neither side) and +same-origin requests the fleet's block list aborted. Against an older plugin the panel falls back to +v0.12.0's reading — documents-only on both sides, the net tile saying "before crawler follow-up +requests", every page handed to a crawler reported as the exposure rather than multiplied by a guess. **Invalidations gained a third panel** for the same release: what the active rows are doing. Refused serves (`invalidated` — each an origin round trip) beside rescued ones (`verified`, plugin diff --git a/packages/console/package.json b/packages/console/package.json index 62f6268..5d2c05b 100644 --- a/packages/console/package.json +++ b/packages/console/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-console", - "version": "0.12.0", + "version": "0.13.0", "type": "module", "description": "Standalone Harper component serving the prerender management console UI, proxying to a prerender deployment's /prerender_admin API", "license": "Apache-2.0", diff --git a/packages/console/src/admin/charts.js b/packages/console/src/admin/charts.js index b3085e0..168f543 100644 --- a/packages/console/src/admin/charts.js +++ b/packages/console/src/admin/charts.js @@ -560,8 +560,10 @@ export const emptyNote = (what, data) => // Series names as constants: the route-contract scanner in adminAssets.test.js reads a quoted // name inside a Map lookup or a comparison as a fetch of a route by that name. const RENDER_OUTCOME = 'outcome'; +const RENDER_SUBREQUESTS = 'subrequests'; const PROBE_REQUESTS = 'probe_probed'; const SITEMAP_FETCHES = 'sitemap_sitemaps'; +const HYDRATION = 'hydration_calls'; /** * Every request the origin answered because this deployment exists, over the window, beside the @@ -597,31 +599,44 @@ const SITEMAP_FETCHES = 'sitemap_sitemaps'; * baseline is what ARRIVED. When the window has serves but no requests (it should not — both are * gated identically — but an older plugin could), the serve total stands in. * - * A FIFTH TERM IS MISSING FROM BOTH SIDES OF THE LEDGER, and it is stated rather than guessed: - * the requests a page's OWN SCRIPTS make. A rendering crawler (Google's WRS, Bingbot, Applebot) - * fetches a page and then runs it, and the page makes its XHR/fetch calls — inventory, pricing, - * personalisation — which are exactly the calls no CDN caches. Call that k per page load. Then: + * THE FIFTH TERM SITS ON BOTH SIDES OF THE LEDGER — the requests a page's OWN SCRIPTS make. A + * rendering crawler (Google's WRS, Bingbot, Applebot) fetches a page and then runs it, and the page + * makes its XHR/fetch calls — inventory, pricing, personalisation — which are exactly the calls no + * CDN caches. Call that k per page load. Then: * - * without us every rendering crawler's request costs the origin 1 + k, not 1 — the baseline - * above (`bot_request`) counts documents only, so it UNDERSTATES what the origin - * was spared. - * with us a snapshot served with its scripts stripped triggers NONE of the k (a saving not - * credited above); a snapshot that keeps its scripts, and every proxied origin page, - * trigger them as before (a cost not charged above); and our own renders run the - * page too, so each render costs 1 + k, not the 1 counted above. + * without us every rendering crawler's request costs the origin 1 + k, not 1 — a baseline that + * counts documents only UNDERSTATES what the origin was spared. + * with us a snapshot served with its scripts stripped triggers NONE of the k (a saving); a + * snapshot that keeps its scripts, and every proxied origin page, trigger them as + * before (a cost); and our own renders run the page too, so each render costs 1 + k. * - * None of these calls pass through this plugin — the CDN forwards the DOCUMENT to us and sends a - * crawler's subrequests straight to the origin — so no series here can count any of them, and the - * sign of the omission depends on facts this plugin cannot see (whether served snapshots carry - * scripts, which crawlers render). So the figure is documents-only on BOTH sides, says so, and - * reports the exposure: every page handed to a crawler — ALL of them, not a guessed subset of bots, - * because which crawler runs what is a fact about the crawler that this console should not assert. - * Where snapshots are served without scripts, the true net offload for rendering crawlers is - * HIGHER than shown. The measured version needs the render fleet and the registry: our headless - * Chrome already runs a cache policy on every same-origin response, so k ("uncacheable same-origin - * subrequests per page load") is measurable per render; stored on the cached page and applied to - * all four places above by what the registry says each crawler runs, it becomes a counted term on - * both sides. + * None of these calls pass through the plugin — the CDN forwards the DOCUMENT to it and sends a + * crawler's subrequests straight to the origin — so the plugin cannot count them on the serve path. + * What it CAN do (plugin ≥ 0.65.0 with a browser ≥ 1.22.0 fleet) is use the render fleet's + * measurement: the renderer runs the same page and classifies every same-origin response by + * whether a shared cache would have answered it, the plugin stores that per page, and at serve time + * it puts the page's k on the side this serve earned it — `hydration_calls` with side `saved` + * (script-stripped snapshot to a crawler that would have run the page), `incurred` (a snapshot + * that kept its scripts, or any origin serve), or `unknown` (k not known: a miss, or a page not + * yet re-rendered since the fleet upgrade). The fleet's own renders report their k as `render` + * series `subrequests` (class `uncacheable`). Every one of these is a VALUE — Σ is mean × count. + * + * So, when those rows exist (`measured`), the figure becomes + * baseline = arrived + Σsaved + Σincurred (documents + the calls those page-views cost) + * actual = proxied + renders + Σk_renders + probes + sitemaps + Σincurred + * and `unknown` is printed as the size of the remaining blind spot — it decays over one render + * cycle after the fleet upgrade, and until it is small the figure is still mostly documents-only. + * `netDocuments` keeps the documents-only reading beside it so the two can be compared. + * + * When they do not exist (an older plugin, or no rendering-crawler traffic), the figure is + * documents-only on BOTH sides, says so, and reports the exposure: every page handed to a crawler + * — ALL of them, not a guessed subset of bots, because which crawler runs what is a fact about the + * crawler that this console should not assert. Where snapshots are served without scripts, the + * true net offload for rendering crawlers is then HIGHER than shown. + * + * Two bounds on k are carried rather than hidden: `unspecified` responses had no freshness headers + * at all (a CDN's default TTL decides them — counted on neither side), and `blocked` is same-origin + * requests the fleet's block list aborted (a crawler would make them; class unknown). * * WHAT IS NOT COUNTED, so nobody assumes it is: crawler requests the CDN answered from its own * cache (they never reach this plugin), renders that crashed before posting a result (no row), and @@ -640,22 +655,47 @@ export function originLoad(data) { const sitemaps = sumValues(pick(data, 'prerender_ops', (s) => s.path === SITEMAP_FETCHES)); const requests = sumCount(pick(data, 'bot_request')); const arrived = requests > 0 ? requests : served; - const total = proxied + renders + probes + sitemaps; + const documents = proxied + renders + probes + sitemaps; + + // The fifth term, where a plugin ≥ 0.65.0 reports it. Values, so Σ = mean × count; the `unknown` + // side's value is always 0 and its COUNT is the blind spot. + const hydration = pick(data, HYDRATION); + const subrequests = pick(data, 'render', (s) => s.path === RENDER_SUBREQUESTS); + const bySide = (side) => hydration.filter((s) => s.path === side); + const saved = sumValues(bySide('saved')); + const incurred = sumValues(bySide('incurred')); + const rendersK = sumValues(subrequests.filter((s) => s.method === 'uncacheable')); + const unspecified = sumValues(subrequests.filter((s) => s.method === 'unspecified')); + const blocked = sumValues(subrequests.filter((s) => s.method === 'blocked')); + const knownServes = sumCount(bySide('saved')) + sumCount(bySide('incurred')); + const unknownServes = sumCount(bySide('unknown')); + // Measured once ANY hydration row exists: a window with only `unknown` rows is measured and + // blind, which is a different (and honestly reported) state from "this plugin cannot measure". + const measured = hydration.length > 0 || subrequests.length > 0; + + const baseline = arrived + saved + incurred; + const total = documents + rendersK + incurred; return { proxied, renders, + rendersK, probes, sitemaps, + // The documents-only pair, kept beside the full figure so the two can be compared. + documents, + netDocuments: ratioOf(arrived - documents, arrived), total, arrived, + baseline, // Null, never 0 or 100%: an empty window has no offload to report. Through ratioOf like every // other division in this module, so the zero-denominator guard cannot be forgotten here either. - net: ratioOf(arrived - total, arrived), + net: ratioOf(baseline - total, baseline), lumpy: probes > 0 || sitemaps > 0, // The fifth term's exposure: every page handed to a crawler, cache-served or proxied alike — // each one is a page whose scripts the crawler either ran against the origin or did not. handed: served, + scriptCalls: { measured, saved, incurred, knownServes, unknownServes, unspecified, blocked }, }; } @@ -692,6 +732,21 @@ export function originLoadBuckets(data) { false ), ], + // The fleet's own script calls, and the crawlers' — both values, both per-bucket mean × count. + [ + 'rendersK', + bucketsOf( + pick(data, 'render', (s) => s.path === RENDER_SUBREQUESTS && s.method === 'uncacheable'), + true + ), + ], + [ + 'incurred', + bucketsOf( + pick(data, HYDRATION, (s) => s.path === 'incurred'), + true + ), + ], [ 'probes', bucketsOf( diff --git a/packages/console/src/admin/views/overview.js b/packages/console/src/admin/views/overview.js index 83de21a..f5608c7 100644 --- a/packages/console/src/admin/views/overview.js +++ b/packages/console/src/admin/views/overview.js @@ -237,7 +237,9 @@ function traffic(ctx) { // Gross on the face, net underneath — the subtitle is what stops a 90% headline being // quoted for a deployment whose renders and probes hand most of it back. Number.isFinite(load.net) - ? `${fmtNet(load.net)} net of renders + probes · before crawler follow-up requests` + ? `${fmtNet(load.net)} net of renders + probes · ${ + load.scriptCalls.measured ? 'script calls counted' : 'before crawler follow-up requests' + }` : 'gross — crawler requests not proxied live', // Either figure under half is the flag; the net one is the one that can go negative. { warn: (total > 0 && originServes > total / 2) || (Number.isFinite(load.net) && load.net < 0.5) } diff --git a/packages/console/src/admin/views/traffic.js b/packages/console/src/admin/views/traffic.js index 2c42fbb..3ed294b 100644 --- a/packages/console/src/admin/views/traffic.js +++ b/packages/console/src/admin/views/traffic.js @@ -457,9 +457,9 @@ function kpis(data, scope) { // leaves out — and under a bot filter it says so, since this one cannot narrow. "Before // crawler follow-up requests" is the term that is NOT in the sum (see originLoad). load.arrived > 0 - ? `origin saw ${fmtCount(load.total)} of ${fmtCount(load.arrived)} asked · before crawler follow-up requests${ - filter ? ' · all bots' : '' - }` + ? `origin saw ${fmtCount(load.total)} of ${fmtCount(load.baseline)} it would have · ${ + load.scriptCalls.measured ? 'script calls counted' : 'before crawler follow-up requests' + }${filter ? ' · all bots' : ''}` : 'no crawler requests in the window', // Below half is a flag on either figure; below ZERO means this system is sending the origin // more requests than the crawlers would have on their own — the finding, not a display bug. @@ -1115,6 +1115,8 @@ function statusCodes(data, filter) { // same figure); this is the panel that shows its terms. /** What the origin saw, by cause, over time — the panel behind the net offload tile. */ +const PINK_SOFT = '#e08bb0'; + const LOAD_ROWS = [ { key: 'proxied', @@ -1128,6 +1130,20 @@ const LOAD_ROWS = [ color: SERIES[0], means: 'page loads by the render fleet, one per posted result — the document only', }, + { + key: 'rendersK', + label: 'render script calls', + color: '#7fb3ff', + means: + 'the origin calls the pages’ own scripts made while the fleet rendered them (uncacheable same-origin subrequests)', + }, + { + key: 'incurred', + label: 'crawler script calls', + color: PINK_SOFT, + means: + 'the origin calls a script-executing crawler’s page-view made — a snapshot that kept its scripts, or a proxied origin page', + }, { key: 'probes', label: 'change probes', @@ -1159,11 +1175,21 @@ function originSeen(data, { load, filter }) { ] : [ el('div', { cls: 'stat-grid tight' }, [ - stat('Crawlers asked for', fmtCount(load.arrived), 'requests at ingress, all bots'), + stat( + 'The origin would have seen', + fmtCount(load.baseline), + load.scriptCalls.measured + ? `${fmtCount(load.arrived)} crawler requests + ${fmtCount(load.scriptCalls.saved + load.scriptCalls.incurred)} script calls their page-views carry` + : 'crawler requests at ingress, all bots' + ), stat( 'The origin answered', fmtCount(load.total), - `${pct(load.total, load.arrived)} of that — ${fmtNet(load.net)} net offload`, + `${pct(load.total, load.baseline)} of that — ${fmtNet(load.net)} net offload${ + load.scriptCalls.measured && Number.isFinite(load.netDocuments) + ? ` · ${fmtNet(load.netDocuments)} documents-only` + : '' + }`, { warn: Number.isFinite(load.net) && load.net < 0.5 } ), stat( @@ -1171,13 +1197,26 @@ function originSeen(data, { load, filter }) { fmtCount(load.proxied), `${fmtCount(load.total - load.proxied)} more came from this system` ), - // The unmeasured term gets a tile so it sits in the same row as the measured ones, at - // the same size — not a footnote under a number that looks complete without it. - stat( - 'Crawler follow-up requests', - 'not measured', - `${fmtCount(load.handed)} pages handed to crawlers · counted on neither side of the ledger` - ), + // The fifth term gets a tile so it sits in the same row as the others, at the same size + // — measured, it is the saving and the cost side by side; unmeasured, it is the exposure + // rather than a footnote under a number that looks complete without it. + load.scriptCalls.measured + ? stat( + 'Script calls', + `${fmtCount(load.scriptCalls.saved)} saved`, + `${fmtCount(load.scriptCalls.incurred)} incurred${ + load.scriptCalls.unknownServes > 0 + ? ` · ${fmtCount(load.scriptCalls.unknownServes)} serves with k unknown` + : '' + }`, + // Blind on more serves than it can see: the figure is still mostly documents-only. + { warn: load.scriptCalls.unknownServes > load.scriptCalls.knownServes } + ) + : stat( + 'Crawler follow-up requests', + 'not measured', + `${fmtCount(load.handed)} pages handed to crawlers · counted on neither side of the ledger` + ), ]), stackedBars(data, keys, stacks, colorOf, { format: fmtCount }), barList( @@ -1191,11 +1230,16 @@ function originSeen(data, { load, filter }) { { format: fmtCount } ), el('p', { cls: 'muted chart-note' }, [ - 'Every request the origin answered because this deployment exists, against what crawlers asked ', - 'for. Gross offload counts only the first bar; net offload subtracts all of them. A render is ', - 'counted as ONE request — the document; the page’s own scripts and stylesheets reach the origin ', - 'too if the CDN does not cache them for the renderer, which nothing here can see. A probe is one ', - 'small endpoint call, not a page render — cheaper per request than the others, but a request.', + 'Every request the origin answered because this deployment exists, against what it would have ', + 'seen without it. Gross offload counts only the first bar; net offload subtracts all of them. ', + load.scriptCalls.measured + ? 'A render is the document plus the calls the page’s own scripts made while rendering (“render ' + + 'script calls”, as the fleet measured them); “crawler script calls” are the same calls made by a ' + + 'script-executing crawler that was handed a page it could run. ' + : 'A render is counted as ONE request — the document; the page’s own scripts and stylesheets reach ' + + 'the origin too if the CDN does not cache them for the renderer, which nothing here can see. ', + 'A probe is one small endpoint call, not a page render — cheaper per request than the others, but a ', + 'request.', load.lumpy ? ' Probe and sitemap counts land where a PASS FINISHED, not where the requests happened: over ' + 'a range shorter than a sweep this is either none of a running pass or all of one that just ' + @@ -1225,6 +1269,38 @@ function originSeen(data, { load, filter }) { */ function followUpNote(load) { if (!load.handed) return null; + const { measured, saved, incurred, knownServes, unknownServes, unspecified, blocked } = load.scriptCalls; + if (measured) { + const calls = saved + incurred; + return el('div', { cls: unknownServes > knownServes ? 'note warn' : 'note' }, [ + el('strong', { + text: + `${num(calls)} origin calls the pages’ own scripts would make were counted on both sides: ` + + `${num(saved)} spared, ${num(incurred)} taken. `, + }), + 'A crawler that executes scripts (Googlebot, Bingbot, Applebot…) runs the page it is handed, and the ', + 'page makes its own XHR/fetch calls to the origin — inventory, pricing, personalisation — which no CDN ', + 'caches. The render fleet measured how many each page makes (k), and each such page-view puts its k on ', + 'one side: SPARED when the crawler got a snapshot with its scripts stripped (nothing to run), TAKEN when ', + 'the snapshot kept them or the page came from the origin. Without this system every one of those ', + 'page-views would have cost the origin the document plus k, which is why the baseline above is larger ', + 'than the request count.', + unknownServes > 0 + ? ` ${num(unknownServes)} serves to script-executing crawlers had no k — a miss, or a page not yet ` + + 're-rendered since the fleet started measuring — and carry nothing on either side; ' + + (unknownServes > knownServes + ? 'that is MOST of them, so this figure is still mostly documents-only. It decays over one render cycle.' + : 'it decays over one render cycle.') + : '', + unspecified > 0 || blocked > 0 + ? ` k is a lower bound: ${num(unspecified)} same-origin responses during rendering carried no freshness ` + + 'headers at all (a CDN’s default TTL decides whether those reach the origin — counted on neither ' + + `side), and ${num(blocked)} same-origin requests were aborted by the fleet’s block list before any ` + + 'response (a crawler would make them; class unknown).' + : '', + ' Which crawlers run scripts is the registry’s claim (analytics.bots rendersJs), not an observation.', + ]); + } return el('div', { cls: 'note' }, [ el('strong', { text: `${num(load.handed)} pages were handed to crawlers, and the requests their scripts make are counted on neither side. `, @@ -1232,16 +1308,16 @@ function followUpNote(load) { 'A rendering crawler (Googlebot, Bingbot, Applebot) fetches a page and then runs it, and the page makes ', 'its own XHR/fetch calls to the origin — inventory, pricing, personalisation — which are exactly the ', 'calls no CDN caches. Without this system every such crawl costs the origin the document PLUS those ', - 'calls, so the “crawlers asked for” baseline above understates what the origin was spared. With it, a ', + 'calls, so the “would have seen” baseline above understates what the origin was spared. With it, a ', 'snapshot served without its scripts triggers none of them (a saving not credited above), a snapshot ', 'that keeps its scripts or a proxied origin page triggers them as before (a cost not charged), and our ', 'own renders run the page too (each render is really the document plus those calls). None of it passes ', 'through this plugin — the CDN sends a crawler’s subrequests straight to the origin — so the figure is ', 'documents-only on both sides and says so, for every crawler rather than a guessed subset. Where ', 'snapshots are served with scripts stripped, the true net offload for rendering crawlers is HIGHER than ', - 'shown. The render fleet can measure the per-page factor (it loads the same pages and already classifies ', - 'every same-origin response as cacheable or not); applied to both sides by what the registry says each ', - 'crawler runs, this becomes a counted term.', + 'shown. Plugin 0.65.0 with a browser 1.22.0 fleet measures the per-page factor and this note becomes a ', + 'count: the fleet classifies every same-origin response as cacheable or not, the plugin stores it per ', + 'page, and each serve to a crawler the registry flags as running scripts puts it on the right side.', ]); } diff --git a/packages/console/test/adminAssets.test.js b/packages/console/test/adminAssets.test.js index 0ca2a3f..f663823 100644 --- a/packages/console/test/adminAssets.test.js +++ b/packages/console/test/adminAssets.test.js @@ -230,6 +230,10 @@ test('a metric the plugin emits is charted by the console, or waived with a reas // the scan above, which is how three probe series once shipped with no panel and a green // suite on both sides. 'changeProbe', + // `hydration_calls` (plugin v0.65.0): its path slot is the SIDE (saved/incurred/unknown), a + // dimension, read by charts.js `originLoad`. This test is what said the console was blind + // to it — the plugin half of #153 landed first in the same PR and this line went red. + 'hydrationCalls', ]); const emitted = []; diff --git a/packages/console/test/overviewView.test.js b/packages/console/test/overviewView.test.js index e4467d7..0e6637d 100644 --- a/packages/console/test/overviewView.test.js +++ b/packages/console/test/overviewView.test.js @@ -371,3 +371,29 @@ test('the offload tile shows the gross figure with the net one underneath, from 'a 30% net offload should warn' ); }); + +test('with script calls measured the offload subtitle says so, instead of the documents-only caveat', async () => { + const analytics = { + ...ANALYTICS, + startMs: 0, + endMs: 3_600_000, + bucketMs: 900_000, + bucketCount: 4, + series: [ + combo('bot_serve', 'cache', 'hit', 'googlebot', 900), + combo('bot_serve', 'origin', 'miss', 'googlebot', 100), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 1000), + combo('render', 'outcome', 'rendered', null, 100), + // 900 script-stripped snapshots to Googlebot at k=4: 3,600 calls spared on the baseline. + combo('hydration_calls', 'saved', 'googlebot', 'cache', 900, 4), + ], + }; + const ctx = await ready({ analytics }); + const tile = find( + draw(ctx), + (n) => n.attributes?.class === 'stat' && n.children[0]?.textContent === 'Origin offload' + ); + // (4,600 − 200) ÷ 4,600 = 96% — the saving the documents-only figure (80%) could not credit. + assert.match(tile.textContent, /96% net of renders \+ probes · script calls counted/); + assert.doesNotMatch(tile.textContent, /before crawler follow-up requests/); +}); diff --git a/packages/console/test/trafficView.test.js b/packages/console/test/trafficView.test.js index f9110d0..10893f2 100644 --- a/packages/console/test/trafficView.test.js +++ b/packages/console/test/trafficView.test.js @@ -514,6 +514,97 @@ test('nothing reaching the origin is 100% offload, and the follow-up caveat stil assert.match(text, /500 pages were handed to crawlers/); }); +// ---- the fifth term, measured (plugin ≥ 0.65.0, browser ≥ 1.22.0) --------------- + +/** The fixture plus the script-call rows a 0.65.0 plugin emits, so the term is counted on both sides. */ +const MEASURED = { + ...ANALYTICS, + series: [ + ...ANALYTICS.series, + // hydration_calls: path=side, method=bot, type=source, VALUE = k per serve. Googlebot got 400 + // script-stripped snapshots at k=5 (2,000 calls spared) and 100 origin pages at k=5 (500 calls + // taken); 50 serves had no k yet. + combo('hydration_calls', 'saved', 'googlebot', 'cache', 400, 5), + combo('hydration_calls', 'incurred', 'googlebot', 'origin', 100, 5), + combo('hydration_calls', 'unknown', 'googlebot', 'origin', 50, 0), + // render subrequests: the fleet's own 120 renders each made 5 uncacheable calls, saw 2 responses + // with no freshness headers, and aborted 1 same-origin request per render. + combo('render', 'subrequests', 'uncacheable', null, 120, 5), + combo('render', 'subrequests', 'cacheable', null, 120, 30), + combo('render', 'subrequests', 'unspecified', null, 120, 2), + combo('render', 'subrequests', 'blocked', null, 120, 1), + ], +}; + +test('with script calls measured, k lands on both sides — spared on the baseline, taken and rendered on the cost', () => { + const load = originLoad(MEASURED); + assert.equal(load.scriptCalls.measured, true); + assert.equal(load.scriptCalls.saved, 2000); + assert.equal(load.scriptCalls.incurred, 500); + assert.equal(load.scriptCalls.knownServes, 500); + assert.equal(load.scriptCalls.unknownServes, 50); + // The fleet's own renders: 120 × 5 uncacheable calls; the bounds ride along. + assert.equal(load.rendersK, 600); + assert.equal(load.scriptCalls.unspecified, 240); + assert.equal(load.scriptCalls.blocked, 120); + // baseline = 1,010 documents + 2,500 calls those page-views carry; actual = 452 documents-only + // origin load + 600 render calls + 500 crawler calls. + assert.equal(load.baseline, 3510); + assert.equal(load.total, 452 + 600 + 500); + assert.ok(Math.abs(load.net - (3510 - 1552) / 3510) < 1e-9); + // The documents-only reading is kept beside it, unchanged. + assert.ok(Math.abs(load.netDocuments - (1010 - 452) / 1010) < 1e-9); +}); + +test('the measured panel shows saved beside taken, the documents-only figure for comparison, and the bounds', async () => { + const ctx = makeCtx({ analytics: MEASURED }); + await load(ctx); + const text = textOf(ctx); + assert.match(text, /Script calls/); + assert.match(text, /2\.0k saved/); + assert.match(text, /500 incurred/); + assert.match(text, /50 serves with k unknown/); + assert.match(text, /56% net offload · 55% documents-only/); + assert.match(text, /script calls counted/); + assert.match(text, /2,500 origin calls the pages’ own scripts would make were counted on both sides/); + assert.match(text, /240 same-origin responses during rendering carried no freshness/); + assert.match(text, /120 same-origin requests were aborted/); + // The two new causes appear as bars, and the exposure wording is gone. + assert.match(text, /render script calls/); + assert.match(text, /crawler script calls/); + assert.doesNotMatch(text, /not measured/); +}); + +test('a window that is mostly UNKNOWN says the figure is still documents-only, loudly', async () => { + const analytics = { + ...ANALYTICS, + series: [ + ...ANALYTICS.series, + combo('hydration_calls', 'saved', 'googlebot', 'cache', 10, 5), + combo('hydration_calls', 'unknown', 'googlebot', 'cache', 900, 0), + ], + }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const text = textOf(ctx); + assert.match(text, /900 serves to script-executing crawlers had no k/); + assert.match(text, /MOST of them, so this figure is still mostly documents-only/); + // And the Script calls tile warns. + const tile = find(draw(ctx), (n) => n.attributes?.class === 'stat' && n.children[0]?.textContent === 'Script calls'); + assert.ok( + find(tile, (n) => n.attributes?.class === 'value warn'), + 'a mostly-blind window should warn' + ); +}); + +test('an older plugin that emits no script-call rows is NOT measured, and keeps the exposure wording', () => { + const load = originLoad(ANALYTICS); + assert.equal(load.scriptCalls.measured, false); + assert.equal(load.baseline, load.arrived); + assert.equal(load.total, load.documents); + assert.equal(load.net, load.netDocuments); +}); + // ---- verified: a cache serve through an invalidation --------------------------- test('verified is a cache serve in the invalidation family — never "other", never outside cache-served', () => { diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 9dc10d2..dcdedf8 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -107,12 +107,12 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi ### The four questions dashboards actually ask -| Question | Read this | -| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Are we taking load off the origin? | `bot_serve` — share of rows with `path != 'origin'`, over all rows. Denominator sanity-check: `bot_request`. | -| Is the cache being hit, and is it fresh? | `bot_serve` grouped by `method`: cache-served is `hit + swr`; `hit` alone is "is the configured TTL being met". Then `page_age` p95 against the route's `renderInterval`. | -| Is the render queue keeping up? | `queue_health` (`overdue`, `lease_occupancy`, `claim_scan_ms`) plus `render` `time_ms` p95 — and `below_floor` / `floor_pin_age_ms` for the silent failures. | -| Which route should change its cadence? | `route_serve` (swr/stale share = cadence not delivered, miss share = corpus not covered) and `route_page_age` p95 per route. | +| Question | Read this | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Are we taking load off the origin? | GROSS: `bot_serve` share of rows with `path != 'origin'`. NET (what the origin actually saw): `1 − (bot_serve origin + render outcome count + Σ render subrequests uncacheable + Σ probe_probed + Σ sitemap_sitemaps + Σ hydration_calls incurred) ÷ (bot_request + Σ hydration_calls saved + Σ hydration_calls incurred)` — every Σ is mean × count. The console's Traffic view computes both. | +| Is the cache being hit, and is it fresh? | `bot_serve` grouped by `method`: cache-served is `hit + swr`; `hit` alone is "is the configured TTL being met". Then `page_age` p95 against the route's `renderInterval`. | +| Is the render queue keeping up? | `queue_health` (`overdue`, `lease_occupancy`, `claim_scan_ms`) plus `render` `time_ms` p95 — and `below_floor` / `floor_pin_age_ms` for the silent failures. | +| Which route should change its cadence? | `route_serve` (swr/stale share = cadence not delivered, miss share = corpus not covered) and `route_page_age` p95 per route. | --- @@ -121,17 +121,18 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*`, `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ----------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `hydration_calls` | value | side | botName | source | The offload term no serve-side counter can see: the origin calls a script-executing crawler's page-view makes AFTER the document, as the render fleet measured them for that page (k = `PrerenderedPage.uncacheableSubrequests`, browser ≥ 1.22.0). Emitted once per request from a crawler the registry flags `rendersJs`, value = k. `saved` = a cache serve of a snapshot stored without scripts (the origin is spared k); `incurred` = a snapshot that kept them, or any origin serve (the origin takes k); `unknown` = k not known (a miss, or a page rendered before the fleet reported the tally) — value 0, COUNT the rows. Baseline origin load = `bot_request + Σsaved + Σincurred`; actual = proxied + renders × (1 + k) + probes + sitemaps + `Σincurred`. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy — renders/hour = concurrency ÷ time_ms), `outcome` (counter by outcome × detail, exactly one per posted result — the render-failure alert), and `subrequests` (browser ≥ 1.22.0: one VALUE per class per result — uncacheable / cacheable / unspecified / blocked — of the same-origin requests the page made beyond the document, judged by whether a shared cache would answer them; `uncacheable` is k, the per-page origin factor, and mean × count is what the fleet's own renders cost the origin beyond the documents). | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*`, `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: @@ -209,6 +210,17 @@ demand_demoted + demand_held`. The other two decision counters are the paths whe any status arrived). - **`render` `outcome` emits exactly once per posted result**, so its outcomes sum to results processed and any single outcome reads as a share of render throughput. +- **`hydration_calls` and `render` `subrequests` are the two halves of one number, and both are + VALUES**: read Σ as mean × count, never `count`. `count` on `hydration_calls` is serves to + script-executing crawlers (useful for the `unknown` side, where it is the size of the blind spot); + `count` on `subrequests` is results. k is the render fleet's measurement of the ORIGIN page under + its own block list: it undercounts by whatever that list aborts (`blocked` is reported beside it) + and takes no position on responses with no freshness headers (`unspecified` — a CDN's default TTL + decides those, and this plugin cannot see it). Which crawlers execute scripts is the registry's + claim (`analytics.bots[].rendersJs`), not an observation — flag a bot only on its vendor's + documentation. `unknown` decays over one render cycle after the fleet upgrade to browser 1.22.0, + as pages re-render and acquire a k; a corpus whose serves are mostly `unknown` is one whose net + offload is still documents-only. - **Renamed in 0.39.0** (this plugin owns its only metric consumers, so the break was taken deliberately): `render_time` → `render`/`time_ms`; `demand_ladder` → `prerender_ops`/`demand_*`; `invalidation_error`, `invalidation_reenqueue`, `page_verification`, `page_age_negative` → `prerender_ops` series of diff --git a/packages/plugin/README.md b/packages/plugin/README.md index a0e7ec3..87e0d37 100644 --- a/packages/plugin/README.md +++ b/packages/plugin/README.md @@ -759,7 +759,11 @@ claim floor, schedule repair — are plugin behavior.) pass; a dashboard refresh never touches the index. - **Traffic** — the delivery half of [METRICS.md](METRICS.md)'s catalog, charted: origin - offload, cache-served and fresh-hit rates, serves by freshness state over time, the per-bot, + offload **gross and net** (net subtracts every origin request this system made — renders, probes, + sitemap fetches — and, from plugin 0.65.0 with a browser ≥ 1.22.0 fleet, counts the page's own + script-driven origin calls on both sides via `hydration_calls`: spared when a script-stripped + snapshot is served to a crawler that would have run the page, incurred when the page it got still + had scripts or came from the origin), cache-served and fresh-hit rates, serves by freshness state over time, the per-bot, per-device and status-code mix, origin-fetch cost and reasons, a per-route cadence table, and on-demand crawl breadth. Freshness is reported **relative to the cadence each route is configured for** (`page_age` ÷ that route's `renderInterval`, since a page expires one interval diff --git a/packages/plugin/package.json b/packages/plugin/package.json index b03926c..1afb54f 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.64.1", + "version": "0.65.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 69ed974..1a8ae81 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -1874,7 +1874,7 @@ export const configSchema = group('Prerender plugin configuration.', { // Search engines { name: 'Googlebot-Image', match: 'googlebot-image' }, { name: 'Googlebot-Video', match: 'googlebot-video' }, - { name: 'Google InspectionTool', match: 'google-inspectiontool' }, + { name: 'Google InspectionTool', match: 'google-inspectiontool', rendersJs: true }, // the -Image/-Video variants need their own entries: the matcher requires a // boundary after the match, so bare `googleother` can't cross the hyphen { name: 'GoogleOther-Image', match: 'googleother-image' }, @@ -1882,12 +1882,12 @@ export const configSchema = group('Prerender plugin configuration.', { { name: 'GoogleOther', match: 'googleother' }, { name: 'Storebot-Google', match: 'storebot-google' }, { name: 'AdsBot-Google', match: 'adsbot-google' }, - { name: 'Googlebot', match: 'googlebot' }, - { name: 'Bingbot', match: 'bingbot' }, + { name: 'Googlebot', match: 'googlebot', rendersJs: true }, + { name: 'Bingbot', match: 'bingbot', rendersJs: true }, { name: 'DuckDuckBot', match: 'duckduckbot-https' }, { name: 'DuckDuckBot', match: 'duckduckbot' }, - { name: 'Applebot', match: 'applebot' }, - { name: 'YandexBot', match: 'yandexbot' }, + { name: 'Applebot', match: 'applebot', rendersJs: true }, + { name: 'YandexBot', match: 'yandexbot', rendersJs: true }, { name: 'Baidu Spider', match: 'baiduspider' }, { name: 'SeznamBot', match: 'seznambot' }, { name: 'Naver Yeti', match: 'yeti' }, @@ -1923,9 +1923,16 @@ export const configSchema = group('Prerender plugin configuration.', { { name: 'OnCrawl', match: 'oncrawl' }, { name: 'Sitebulb', match: 'sitebulb' }, ], - 'Crawler registry: { name, match } entries, where `match` is a case-insensitive substring of ' + - 'the User-Agent; longer matches win over shorter ones (e.g. `googlebot-image` before ' + - '`googlebot`).', + 'Crawler registry: { name, match, rendersJs? } entries, where `match` is a case-insensitive substring ' + + 'of the User-Agent; longer matches win over shorter ones (e.g. `googlebot-image` before ' + + '`googlebot`). `rendersJs: true` marks a crawler that EXECUTES the pages it fetches — Google’s ' + + 'Web Rendering Service (Googlebot, the URL Inspection tool), Bingbot’s evergreen Chromium, ' + + 'Applebot, YandexBot — and is the gate on the `hydration_calls` metric: only such a crawler’s ' + + 'page-views carry the page’s own XHR/API calls to the origin, on either side of the offload ' + + 'ledger. It is a claim about the crawler, not something this plugin can observe: nothing here ' + + 'sees what a crawler does after it leaves with the document. Every AI crawler in the default ' + + 'registry fetches HTML and runs nothing, and is deliberately unflagged; flag a bot only on its ' + + 'vendor’s documentation.', { itemType: 'object' } ), } diff --git a/packages/plugin/src/http_handlers/bot_request.js b/packages/plugin/src/http_handlers/bot_request.js index c6d3b4d..2ba5081 100644 --- a/packages/plugin/src/http_handlers/bot_request.js +++ b/packages/plugin/src/http_handlers/bot_request.js @@ -1,6 +1,6 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { CacheKey } from '../util/cacheKey.js'; -import { getBotName, botMayDiscover, botCountsAsDemand } from '../util/userAgent.js'; +import { getBotName, botMayDiscover, botCountsAsDemand, botRendersJs } from '../util/userAgent.js'; import { isPrerenderCandidate } from '../util/indexSignals.js'; import { canonicalizeUrl } from '../util/url.js'; import { config } from '../config.js'; @@ -107,6 +107,7 @@ export function recordServeOutcome(resource, request, info, deviceType) { const route = info.route?.path ?? info.routeClass ?? 'unrouted'; metrics.botServe(info.source, info.cacheStatus, request.botName); metrics.routeServe(route, info.cacheStatus, deviceType); + recordHydrationCalls(resource, request.botName, info); if (info.source === 'cache' && resource.lastCached) { // lastCached is a schema Date — guard truthiness FIRST, then coerce, exactly like the // expiresAt read above: `new Date(null)` is epoch 0 (not NaN), so an unguarded null @@ -130,6 +131,42 @@ export function recordServeOutcome(resource, request, info, deviceType) { } } +// THE OFFLOAD TERM NO SERVE-SIDE COUNTER CAN SEE (metrics.js `hydration_calls`, prerender-plugin#153). +// +// A crawler that executes scripts runs the page it is handed, and the page makes its own XHR/API +// calls to the origin — calls that go crawler → CDN → origin and never pass through here. The +// render fleet measured how many such calls THIS page makes (`uncacheableSubrequests`, k) when it +// rendered it; this decides which side of the ledger those k calls land on for this serve: +// +// saved a cache serve of a snapshot stored WITHOUT its scripts — the crawler runs nothing, and +// the origin is spared the k calls a raw page-view would have cost it. +// incurred a cache serve of a snapshot that KEPT its scripts, or any origin serve (the crawler +// gets a page with scripts either way) — the origin takes the k calls. +// unknown k is not known for this serve: no page record (a miss), or one rendered by a fleet +// that did not report the tally. Value 0, counted so the blind spot has a size. +// +// Emitted only for crawlers the registry flags `rendersJs`: an unflagged crawler fetches HTML and +// runs nothing, so its page-view carries no k on either side. Same gate as bot_serve otherwise, and +// the same cost — one in-memory counter bump, no await. +// +// Exported for tests, which assert the side per source × stripped × k-known. +export function recordHydrationCalls(resource, botName, info) { + if (!botRendersJs(botName)) return; + // The record whose k applies: the served page when the bytes came from cache (a render-now hit is + // the fresh record), else the record the request was judged against before it was proxied. + const page = info.source === 'origin' ? info.page : (resource ?? info.page); + const k = page?.uncacheableSubrequests; + if (typeof k !== 'number' || !Number.isFinite(k) || k < 0) { + metrics.hydrationCalls(0, 'unknown', botName, info.source); + return; + } + // `scriptsStripped` decides the side ONLY for a cache serve: a proxied origin page carries its + // scripts whatever the snapshot did. Null (older row) reads as "kept" — the conservative side, + // since crediting a saving on a snapshot that might hydrate is the one error this must not make. + const saved = info.source !== 'origin' && page.scriptsStripped === true; + metrics.hydrationCalls(k, saved ? 'saved' : 'incurred', botName, info.source); +} + // Resolve the request into { url, cacheUrl, deviceType, routeClass, route }, dispatching on // ingress mode. In 'forwarded' mode isBotRequest already resolved + stashed the target; the // fallback resolve guards against direct calls. Returns null when a forwarded request @@ -193,6 +230,10 @@ async function resolveResource({ request, url, cacheUrl, deviceType, routeClass, const { skipCache, missMode, missModeExplicit } = resolveServingPolicy(routeClass, request.method, request.headers); const page = skipCache ? null : await PrerenderedPage.get(cacheKey); + // The record this request was judged against, kept for `recordHydrationCalls`: when the request + // ends up proxied, `resource` is the origin response and the page's own k is only here. Never + // surfaced — response.js reads named fields off `info`, not the object. + info.page = page ?? null; // expiresAt is a schema `Date` (stored from Date.now()); read it robustly so a Date, // number, or serialized string all compare correctly — cf. the Number() coercion in // util/renderNow.js. A bad/missing value yields NaN => not servable from cache. diff --git a/packages/plugin/src/metrics.js b/packages/plugin/src/metrics.js index 01b550c..0780b9d 100644 --- a/packages/plugin/src/metrics.js +++ b/packages/plugin/src/metrics.js @@ -223,19 +223,64 @@ export const METRICS = Object.freeze({ }, }), + hydration_calls: metric('hydration_calls', { + kind: 'value', + emittedBy: 'http_handlers/bot_request.js', + cadence: 'once per bot request from a crawler the registry flags `rendersJs`, beside bot_serve', + summary: + 'The origin calls a script-executing crawler’s page-view makes AFTER the document — the offload term ' + + 'no serve-side counter can see — as measured by the render fleet for that page (k).', + usefulFor: + 'The missing side of net offload (prerender-plugin#153). A crawler that runs the page (Googlebot, ' + + 'Bingbot, Applebot…) makes the page’s own XHR/API calls to the origin, and those never pass ' + + 'through this plugin. Without prerender every such page-view costs the origin 1 + k requests; ' + + 'with it, a snapshot served WITHOUT its scripts triggers none of the k (`saved`), while a snapshot ' + + 'that kept them or a proxied origin page triggers all of them (`incurred`). So: ' + + 'baseline origin load = bot_request + Σ(saved) + Σ(incurred); actual = proxied + renders(1 + k) + ' + + 'probes + sitemaps + Σ(incurred). Read Σ as mean × count — the VALUE is k per serve.', + caveats: + '`unknown` rows are serves where k could not be known: no page record (a miss), or a page rendered ' + + 'before the fleet reported subrequests (browser < 1.22.0) — their value is 0 and their COUNT is ' + + 'the population the figure is blind to; it decays over one render cycle after the fleet upgrade. ' + + 'k is the render fleet’s measurement of the ORIGIN page under its block list, so it undercounts by ' + + 'whatever that list aborts (the fleet reports `blocked` beside it) and says nothing about what a ' + + 'CDN’s default TTL does to responses with no freshness headers (`unspecified`). Which crawlers run ' + + 'scripts is the registry’s claim (`analytics.bots[].rendersJs`), not an observation.', + gatedBy: 'analytics.enabled, and the bot’s registry entry carrying rendersJs: true', + dimensions: { + path: { + name: 'side', + values: ['saved', 'incurred', 'unknown'], + description: + 'saved = a cache serve of a snapshot stored with its scripts stripped (the crawler runs nothing; ' + + 'the origin is spared k). incurred = a cache serve of a snapshot that kept its scripts, or an ' + + 'origin serve of any kind (the crawler runs the page; the origin takes k). unknown = k not ' + + 'known for this serve (value 0; count the rows).', + }, + method: { name: 'botName', description: 'As bot_request.method — only crawlers flagged rendersJs appear.' }, + type: { name: 'source', values: SERVE_SOURCES, description: 'As bot_serve.path.' }, + }, + }), + render: metric('render', { kind: 'value', emittedBy: 'resources/RenderQueue.js', cadence: 'per render result posted back by a browser worker: one `outcome` row always, one `time_ms` sample ' + - 'when the worker reported a duration', - summary: 'The render fleet, in one scan: how long each render took, and what became of it.', + 'when the worker reported a duration, one `subrequests` sample per class when the worker reported ' + + 'its tally (browser ≥ 1.22.0)', + summary: + 'The render fleet, in one scan: how long each render took, what became of it, and what the page asked of its origin.', usefulFor: '`time_ms` is fleet capacity (renders/hour/pod = concurrency ÷ time_ms) and what a settle-tuning ' + 'change has to move. `outcome` is the render-failure alert — "renders are failing", "the corpus is ' + 'being mass-suppressed", and "the renderer credential broke" were log-grep-only before it. One ' + '`outcome` emit per posted result, so outcomes sum to results processed and any share reads as a ' + - 'fraction of render throughput.', + 'fraction of render throughput. `subrequests` is the per-page origin factor k (prerender-plugin#153): ' + + 'the same-origin requests the page made beyond the document, by whether a SHARED cache would have ' + + 'answered them — `uncacheable` is what reaches the origin whoever runs the page, i.e. what a ' + + 'script-executing crawler’s page-view costs the origin and what each of OUR renders costs it too. ' + + 'A VALUE per result (mean × count = total; mean = k per render), never a count of emits.', caveats: 'auth-failure is special-cased on purpose: 401/403 never suppresses (it is almost never a statement ' + 'about the page), so a spike there with a steady `suppressed` is the signature of a broken bypass ' + @@ -245,17 +290,24 @@ export const METRICS = Object.freeze({ dimensions: { path: { name: 'series', - values: ['time_ms', 'outcome'], - description: 'time_ms = duration distribution (ms). outcome = counter of what became of the result.', + values: ['time_ms', 'outcome', 'subrequests'], + description: + 'time_ms = duration distribution (ms). outcome = counter of what became of the result. ' + + 'subrequests = same-origin subrequest counts per result, by shared-cache class (method).', }, method: { - name: 'statusCode (time_ms) / outcome (outcome)', + name: 'statusCode (time_ms) / outcome (outcome) / class (subrequests)', description: 'time_ms: HTTP status the render observed — a NUMBER at the emit site (for a redirect bail, ' + 'the FIRST hop’s 3xx). outcome: rendered | suppressed | auth-failure | transient | failed | ' + 'redirect — rendered = usable result, suppressed = genuine non-indexable verdict (target moves ' + 'to its recheck cadence), auth-failure = 401/403 kept and retried, transient = 408/429/5xx kept ' + - 'and retried, failed = the render itself broke, redirect = the page moved or bounced.', + 'and retried, failed = the render itself broke, redirect = the page moved or bounced. ' + + 'subrequests: uncacheable (explicitly reaches the origin: non-GET, uncacheable status, ' + + 'Set-Cookie, no-store/private/no-cache, zero max-age, Vary *, expired Expires) | cacheable ' + + '(explicit positive freshness) | unspecified (no freshness information — CDN-default ' + + 'dependent, counted on neither side of the offload figure) | blocked (same-origin requests the ' + + 'fleet’s block list aborted before a response — a crawler would make them; class unknown).', }, type: { name: 'candidacy (time_ms) / detail (outcome)', @@ -700,6 +752,18 @@ export const metrics = Object.freeze({ /** What became of one posted render result — exactly one call per result; the `render` outcome series. */ renderOutcome: (outcome, detail) => server.recordAnalytics(true, 'render', 'outcome', outcome, detail ?? null), + /** + * One class of one result's same-origin subrequest tally — the `render` subrequests series. A VALUE + * (the count for that class on that render), so mean × count is the fleet total and mean is k. + */ + renderSubrequests: (count, klass) => server.recordAnalytics(count, 'render', 'subrequests', klass, null), + + /** + * The page's origin factor k, on the side of the offload ledger this serve put it. Value = k (0 for + * 'unknown'); one call per bot request from a crawler the registry flags as running scripts. + */ + hydrationCalls: (k, side, botName, source) => server.recordAnalytics(k, 'hydration_calls', side, botName, source), + /** One claim pass's duration and how it ended — a queue_health series, so the queue reads in one scan. */ claimScan: (durationMs, result) => server.recordAnalytics(durationMs, 'queue_health', 'claim_scan_ms', result, null), diff --git a/packages/plugin/src/resources/RenderQueue.js b/packages/plugin/src/resources/RenderQueue.js index 76c1376..0e8bca9 100644 --- a/packages/plugin/src/resources/RenderQueue.js +++ b/packages/plugin/src/resources/RenderQueue.js @@ -110,6 +110,19 @@ async function syncQueueState(force = false, pending = null) { return { status, ...desired }; } +// The classes a browser ≥ 1.22.0 worker tallies per render (src/subrequests.ts over there). Read as +// finite non-negative numbers or not at all: a tally with a missing class is a worker this plugin +// does not know, and half a tally recorded as zeros would understate k rather than say nothing. +const SUBREQUEST_CLASSES = ['sameOrigin', 'cacheable', 'uncacheable', 'unspecified', 'blocked']; +const readSubrequests = (tally) => { + if (!tally || typeof tally !== 'object') return null; + for (const klass of SUBREQUEST_CLASSES) { + const n = tally[klass]; + if (typeof n !== 'number' || !Number.isFinite(n) || n < 0) return null; + } + return tally; +}; + export class RenderQueue extends Resource { static loadAsInstance = false; @@ -320,6 +333,15 @@ export class RenderQueue extends Resource { const hasContent = result.statusCode === 200 && result.content; + // The page's same-origin subrequest tally, by shared-cache class (browser ≥ 1.22.0). One VALUE + // per class per result, so the fleet total is mean × count and the mean is k per render. Every + // class the worker reported, not only `uncacheable`: `unspecified` and `blocked` bound what k + // leaves out, and are worth nothing unless they are recorded beside it. + const subrequests = readSubrequests(result.subrequests); + if (subrequests) { + for (const klass of SUBREQUEST_CLASSES) metrics.renderSubrequests(subrequests[klass], klass); + } + if (typeof result.renderTime === 'number') { metrics.renderTime( result.renderTime, @@ -386,6 +408,11 @@ export class RenderQueue extends Resource { headers: JSON.stringify(result.headers), expiresAt: nextRenderTime, isIndexable: typeof result.isIndexable === 'boolean' ? result.isIndexable : null, + // k, and which side of the offload ledger it lands on when this row is served (see the + // schema). Null — never 0 — when the worker predates the tally: the serve path reports + // null as `unknown`, where a 0 would read as "this page makes no origin calls". + uncacheableSubrequests: subrequests ? subrequests.uncacheable : null, + scriptsStripped: typeof result.scriptsStripped === 'boolean' ? result.scriptsStripped : null, }); } diff --git a/packages/plugin/src/schemas/schema.graphql b/packages/plugin/src/schemas/schema.graphql index e110e8a..080a719 100644 --- a/packages/plugin/src/schemas/schema.graphql +++ b/packages/plugin/src/schemas/schema.graphql @@ -338,6 +338,17 @@ type PrerenderedPage @table(database: "page_cache") @export { # Indexability verdict from the render, stored so the serving path can surface it # as a debug response header (x-harper-indexable). Optional — older rows lack it. isIndexable: Boolean + # The page's own origin factor k, as the render fleet measured it: same-origin requests the + # page made beyond the document that no shared cache would have answered (browser ≥ 1.22.0, + # `subrequests.uncacheable`). Read on the serve path for a crawler that executes scripts and + # emitted as `hydration_calls` — the origin load its page-view carries, which is either SPARED + # (snapshot stored without scripts) or INCURRED (scripts kept, or an origin serve). Optional: + # a row rendered by an older fleet has none, and the serve path reports it as `unknown`. + uncacheableSubrequests: Int + # Whether the stored body had its scripts stripped by the render fleet (its + # postProcess.stripScripts at render time). Decides which side of the ledger k lands on when + # this row is cache-served. Optional, same vintage as the field above. + scriptsStripped: Boolean } type SharedBuffer @table(database: "coordination", replicate: false) { diff --git a/packages/plugin/src/util/userAgent.js b/packages/plugin/src/util/userAgent.js index afb9544..29891f7 100644 --- a/packages/plugin/src/util/userAgent.js +++ b/packages/plugin/src/util/userAgent.js @@ -242,3 +242,34 @@ export const botCountsAsDemand = (botName) => { if (demandSet === null) return true; return typeof botName === 'string' && demandSet.has(botName.toLowerCase()); }; + +let rendersJsSet = null; // lowercase Set of registry names flagged rendersJs +let rendersJsFrom; // the registry array the current set was built from + +/** + * Does the crawler labeled `botName` EXECUTE the pages it fetches? + * + * Read off the registry: an `analytics.bots` entry with `rendersJs: true`. This is the gate on + * `hydration_calls` (metrics.js) — the origin calls a page's own scripts make are only made by a + * crawler that runs those scripts, so only such a crawler's page-views carry the per-page factor + * k on either side of the offload ledger. It is a CLAIM about the crawler, not an observation: + * nothing here can see what a crawler does after it leaves with the document. The defaults flag + * only the vendors who document rendering (see configSchema.js); an unflagged crawler is assumed + * to fetch HTML and run nothing, which is what every AI crawler in the registry does today. + * + * Names are compared case-insensitively, like the other two registry-derived allowlists, and a + * derived name (a self-identifying UA the registry does not list) is never flagged — it has no + * entry to carry the flag. + */ +export const botRendersJs = (botName) => { + if (config.analytics.bots !== rendersJsFrom) { + const entries = Array.isArray(config.analytics.bots) ? config.analytics.bots : []; + rendersJsSet = new Set( + entries + .filter((entry) => entry && typeof entry.name === 'string' && entry.rendersJs === true) + .map((entry) => entry.name.toLowerCase()) + ); + rendersJsFrom = config.analytics.bots; + } + return typeof botName === 'string' && rendersJsSet.has(botName.toLowerCase()); +}; diff --git a/packages/plugin/test/botServe.test.js b/packages/plugin/test/botServe.test.js index aac6747..f3c6e45 100644 --- a/packages/plugin/test/botServe.test.js +++ b/packages/plugin/test/botServe.test.js @@ -17,6 +17,10 @@ import assert from 'node:assert/strict'; * - lastCached may arrive as a Date, a number, or a serialized string — all must yield the * same age. A missing value (NaN) or a negative age (cross-node clock skew) records * nothing rather than poisoning the mean. + * - `hydration_calls` (v0.65.0) rides beside these for a crawler the registry flags as + * executing scripts — Googlebot, the bot every test here uses, is one — and its side is + * decided by source × scriptsStripped × whether k is known. The serve-outcome assertions + * above look at the rows WITHOUT it (`serveRows`), so each contract is pinned on its own. */ let analytics = []; @@ -56,11 +60,15 @@ beforeEach(() => { analytics = []; }); +/** The serve-outcome rows alone — hydration_calls has its own contract and its own tests below. */ +const serveRows = () => analytics.filter(([, metric]) => metric !== 'hydration_calls'); +const hydrationRows = () => analytics.filter(([, metric]) => metric === 'hydration_calls'); + const request = { botName: 'Googlebot' }; test('bot_serve records (source, cacheStatus, botName) and route_serve records (route, cacheStatus, deviceType)', () => { recordServeOutcome({}, request, { source: 'origin', cacheStatus: 'miss', route: { path: '/catalog/' } }, 'desktop'); - assert.deepEqual(analytics, [ + assert.deepEqual(serveRows(), [ [true, 'bot_serve', 'origin', 'miss', 'Googlebot'], [true, 'route_serve', '/catalog/', 'miss', 'desktop'], ]); @@ -69,8 +77,8 @@ test('bot_serve records (source, cacheStatus, botName) and route_serve records ( test('route label falls back route.path -> routeClass -> unrouted', () => { recordServeOutcome({}, request, { source: 'origin', cacheStatus: 'miss', routeClass: 'passthrough' }, 'desktop'); recordServeOutcome({}, request, { source: 'origin', cacheStatus: 'miss' }, 'desktop'); - assert.equal(analytics[1][2], 'passthrough'); - assert.equal(analytics[3][2], 'unrouted'); + assert.equal(serveRows()[1][2], 'passthrough'); + assert.equal(serveRows()[3][2], 'unrouted'); }); test('a cache serve also records page_age (botName, deviceType) and route_page_age (route, cacheStatus, deviceType)', () => { @@ -84,13 +92,13 @@ test('a cache serve also records page_age (botName, deviceType) and route_page_a { source: 'cache', cacheStatus: 'hit', route: { path: '/product/prd-' } }, 'mobile' ); - assert.equal(analytics.length, 4); - const [age, metric, bot, device] = analytics[2]; + assert.equal(serveRows().length, 4); + const [age, metric, bot, device] = serveRows()[2]; assert.equal(metric, 'page_age'); assert.equal(bot, 'Googlebot'); assert.equal(device, 'mobile'); assert.ok(age >= 4000 && age <= 7000, `expected age ~5000ms, got ${age}`); - const [rAge, rMetric, rRoute, rStatus, rDevice] = analytics[3]; + const [rAge, rMetric, rRoute, rStatus, rDevice] = serveRows()[3]; assert.equal(rMetric, 'route_page_age'); assert.equal(rRoute, '/product/prd-'); assert.equal(rStatus, 'hit'); @@ -106,7 +114,7 @@ test('an swr serve carries cacheStatus swr through both route metrics', () => { { source: 'cache', cacheStatus: 'swr', route: { path: '/catalog/' } }, 'desktop' ); - const statuses = analytics.map(([, metric, ...dims]) => [metric, dims]); + const statuses = serveRows().map(([, metric, ...dims]) => [metric, dims]); assert.deepEqual(statuses[0], ['bot_serve', ['cache', 'swr', 'Googlebot']]); assert.deepEqual(statuses[1], ['route_serve', ['/catalog/', 'swr', 'desktop']]); assert.equal(statuses[3][0], 'route_page_age'); @@ -115,9 +123,9 @@ test('an swr serve carries cacheStatus swr through both route metrics', () => { test('age metrics are skipped for a non-cache source, even with lastCached present', () => { recordServeOutcome({ lastCached: Date.now() }, request, { source: 'rendered', cacheStatus: 'miss' }, 'desktop'); - assert.equal(analytics.length, 2); + assert.equal(serveRows().length, 2); assert.deepEqual( - analytics.map(([, metric]) => metric), + serveRows().map(([, metric]) => metric), ['bot_serve', 'route_serve'] ); }); @@ -127,7 +135,7 @@ test('age metrics are skipped when lastCached is missing, null, or in the future // null is the trap case: new Date(null) is epoch 0, not NaN — unguarded, this would // record age ≈ Date.now() instead of nothing. recordServeOutcome({ lastCached: null }, request, { source: 'cache', cacheStatus: 'hit' }, 'desktop'); - const metrics = analytics.map(([, metric]) => metric); + const metrics = serveRows().map(([, metric]) => metric); assert.deepEqual(metrics, ['bot_serve', 'route_serve', 'bot_serve', 'route_serve'], 'no age sample either way'); }); @@ -138,10 +146,10 @@ test('a lastCached in the FUTURE records page_age_negative instead of silently v recordServeOutcome({ lastCached: Date.now() + 60_000 }, request, { source: 'cache', cacheStatus: 'hit' }, 'desktop'); // page_age_negative is a prerender_ops series: (true, 'prerender_ops', series, bot, device) assert.deepEqual( - analytics.map(([, metric, seriesOrDim]) => (metric === 'prerender_ops' ? seriesOrDim : metric)), + serveRows().map(([, metric, seriesOrDim]) => (metric === 'prerender_ops' ? seriesOrDim : metric)), ['bot_serve', 'route_serve', 'page_age_negative'] ); - const [value, , , bot, device] = analytics[2]; + const [value, , , bot, device] = serveRows()[2]; assert.equal(value, true, 'a counter, not a duration'); assert.equal(bot, 'Googlebot'); assert.equal(device, 'desktop'); @@ -195,3 +203,74 @@ test('an epoch demotes a page rendered before it, and only when it would otherwi // is false, so `<=` would have made a page with no usable timestamp serve straight through. assert.equal(serve(NOW + 1, { lastCachedMs: NaN, epoch }).status, 'invalidated'); }); + +// ---- hydration_calls: which side of the offload ledger the page's own origin calls land on ---- +// +// k is the render fleet's count of same-origin requests the page makes that no shared cache would +// answer (browser ≥ 1.22.0). A crawler that executes scripts makes those calls itself when it runs +// the page — unless the snapshot it was handed has no scripts left to run. + +test('a cache serve of a script-stripped snapshot SAVES k — the crawler runs nothing', () => { + const page = { uncacheableSubrequests: 7, scriptsStripped: true, lastCached: Date.now() - 1000 }; + recordServeOutcome(page, request, { source: 'cache', cacheStatus: 'hit', route: { path: '/p/' }, page }, 'desktop'); + assert.deepEqual(hydrationRows(), [[7, 'hydration_calls', 'saved', 'Googlebot', 'cache']]); +}); + +test('a cache serve of a snapshot that KEPT its scripts INCURS k', () => { + const page = { uncacheableSubrequests: 7, scriptsStripped: false }; + recordServeOutcome(page, request, { source: 'cache', cacheStatus: 'hit', page }, 'desktop'); + assert.deepEqual(hydrationRows(), [[7, 'hydration_calls', 'incurred', 'Googlebot', 'cache']]); +}); + +test('an origin serve INCURS k whatever the snapshot did — the proxied page carries its scripts', () => { + // The record the request was judged against (stale here) is on `info.page`; `resource` is the + // origin response and knows nothing about k. + const page = { uncacheableSubrequests: 4, scriptsStripped: true }; + recordServeOutcome({ statusCode: 200 }, request, { source: 'origin', cacheStatus: 'stale', page }, 'desktop'); + assert.deepEqual(hydrationRows(), [[4, 'hydration_calls', 'incurred', 'Googlebot', 'origin']]); +}); + +test('a render-now hit is a cache-shaped serve of the FRESH record', () => { + const fresh = { uncacheableSubrequests: 2, scriptsStripped: true }; + recordServeOutcome(fresh, request, { source: 'rendered', cacheStatus: 'skip', page: null }, 'desktop'); + assert.deepEqual(hydrationRows(), [[2, 'hydration_calls', 'saved', 'Googlebot', 'rendered']]); +}); + +test('an unknown k is reported as UNKNOWN with value 0 — never as "this page makes no calls"', () => { + // A miss: no record at all. + recordServeOutcome({ statusCode: 200 }, request, { source: 'origin', cacheStatus: 'miss', page: null }, 'desktop'); + // A row rendered by a fleet that predates the tally: the field is absent. + const old = { lastCached: Date.now() - 1000 }; + recordServeOutcome(old, request, { source: 'cache', cacheStatus: 'hit', page: old }, 'desktop'); + // Garbage in the field is not a number either. + const bad = { uncacheableSubrequests: -1, scriptsStripped: true }; + recordServeOutcome(bad, request, { source: 'cache', cacheStatus: 'hit', page: bad }, 'desktop'); + assert.deepEqual(hydrationRows(), [ + [0, 'hydration_calls', 'unknown', 'Googlebot', 'origin'], + [0, 'hydration_calls', 'unknown', 'Googlebot', 'cache'], + [0, 'hydration_calls', 'unknown', 'Googlebot', 'cache'], + ]); +}); + +test('a row that does not say whether its scripts were stripped is read as KEPT — the side that cannot over-credit', () => { + const page = { uncacheableSubrequests: 3 }; + recordServeOutcome(page, request, { source: 'cache', cacheStatus: 'hit', page }, 'desktop'); + assert.deepEqual(hydrationRows(), [[3, 'hydration_calls', 'incurred', 'Googlebot', 'cache']]); +}); + +test('a crawler the registry does not flag as executing scripts emits nothing — its page-view carries no k', () => { + const page = { uncacheableSubrequests: 7, scriptsStripped: true }; + for (const botName of ['GPTBot', 'ClaudeBot', 'other', 'SomeDerivedBot', undefined]) { + analytics = []; + recordServeOutcome(page, { botName }, { source: 'cache', cacheStatus: 'hit', page }, 'desktop'); + assert.deepEqual(hydrationRows(), [], String(botName)); + // bot_serve + route_serve (no lastCached on this record, so no age rows) — untouched either way. + assert.equal(serveRows().length, 2, 'the serve-outcome rows are untouched'); + } +}); + +test('the registry flag is matched case-insensitively, like the other registry-derived allowlists', () => { + const page = { uncacheableSubrequests: 1, scriptsStripped: true }; + recordServeOutcome(page, { botName: 'googlebot' }, { source: 'cache', cacheStatus: 'hit', page }, 'desktop'); + assert.equal(hydrationRows().length, 1); +}); diff --git a/packages/plugin/test/renderQueueRedirect.test.js b/packages/plugin/test/renderQueueRedirect.test.js index 84aea2a..281c98b 100644 --- a/packages/plugin/test/renderQueueRedirect.test.js +++ b/packages/plugin/test/renderQueueRedirect.test.js @@ -358,6 +358,72 @@ test('outcome=rendered stores the page and reschedules', async () => { assert.ok(stores.renderSchedule.get(key(A)).nextRenderTime > Date.now(), 'rescheduled one interval out'); }); +// ---- the page's origin factor k (browser ≥ 1.22.0) ---- + +const TALLY = { sameOrigin: 12, cacheable: 7, uncacheable: 4, unspecified: 1, blocked: 3 }; + +test('a result carrying the subrequest tally stores k and the scripts flag on the page, and emits every class', async () => { + seedSource({ renderInterval: 60_000 }); + await postResult( + { + id: key(A), + url: A, + statusCode: 200, + outcome: 'rendered', + isIndexable: true, + headers: {}, + subrequests: TALLY, + scriptsStripped: true, + }, + 'fresh' + ); + + const page = stores.prerenderedPage.get(key(A)); + assert.equal(page.uncacheableSubrequests, 4); + assert.equal(page.scriptsStripped, true); + // One VALUE per class — the console reads mean × count, so the value is the count for that class. + const rows = analytics.filter((a) => a[1] === 'render' && a[2] === 'subrequests').map((a) => [a[3], a[0]]); + assert.deepEqual(rows, [ + ['sameOrigin', 12], + ['cacheable', 7], + ['uncacheable', 4], + ['unspecified', 1], + ['blocked', 3], + ]); +}); + +test('a worker that posts no tally stores NULL, never zero — the serve path reports null as unknown', async () => { + seedSource({ renderInterval: 60_000 }); + await postResult( + { id: key(A), url: A, statusCode: 200, outcome: 'rendered', isIndexable: true, headers: {} }, + 'fresh' + ); + const page = stores.prerenderedPage.get(key(A)); + assert.equal(page.uncacheableSubrequests, null); + assert.equal(page.scriptsStripped, null); + assert.equal( + analytics.some((a) => a[1] === 'render' && a[2] === 'subrequests'), + false + ); +}); + +test('a half-shaped tally is not read at all — zeros for missing classes would understate k', async () => { + seedSource({ renderInterval: 60_000 }); + for (const bad of [{ uncacheable: 4 }, { ...TALLY, uncacheable: -1 }, { ...TALLY, blocked: 'many' }, 'nope', 7]) { + analytics = []; + await postResult( + { id: key(A), url: A, statusCode: 200, outcome: 'rendered', isIndexable: true, headers: {}, subrequests: bad }, + 'fresh' + ); + assert.equal(stores.prerenderedPage.get(key(A)).uncacheableSubrequests, null, JSON.stringify(bad)); + assert.equal( + analytics.some((a) => a[1] === 'render' && a[2] === 'subrequests'), + false, + JSON.stringify(bad) + ); + } +}); + // ---- route-typed render cadence ---- const HOUR_MS = 3_600_000; diff --git a/packages/plugin/test/userAgent.test.js b/packages/plugin/test/userAgent.test.js index 0786e13..4ea9ecb 100644 --- a/packages/plugin/test/userAgent.test.js +++ b/packages/plugin/test/userAgent.test.js @@ -1,7 +1,7 @@ import { test, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; import { applyOptions } from '../src/config.js'; -import { getBotName, botMayDiscover, botCountsAsDemand } from '../src/util/userAgent.js'; +import { getBotName, botMayDiscover, botCountsAsDemand, botRendersJs } from '../src/util/userAgent.js'; // Minimal stand-in for a WHATWG Headers object. const headers = (map) => ({ get: (k) => map[k.toLowerCase()] ?? null }); @@ -184,3 +184,62 @@ test('the discovery and demand allowlists are independent', () => { assert.equal(botCountsAsDemand('Googlebot'), false); assert.equal(botCountsAsDemand('Bingbot'), true); }); + +// ---- botRendersJs: the gate on hydration_calls ---- +// +// Same shape as the two allowlists above — a Set recompiled when the registry array's identity +// changes — and the same reason it is not a `??=` memo: applyOptions replaces `config.analytics.bots` +// with a fresh array on every change, and a set built once would keep honouring the registry the +// process booted with after an operator flagged or unflagged a crawler from the console. + +test('botRendersJs: the default registry flags only the documented renderers', () => { + for (const bot of ['Googlebot', 'Google InspectionTool', 'Bingbot', 'Applebot', 'YandexBot']) { + assert.equal(botRendersJs(bot), true, bot); + } + for (const bot of ['GPTBot', 'ClaudeBot', 'OAI-SearchBot', 'PerplexityBot', 'CCBot', 'AhrefsBot', 'other']) { + assert.equal(botRendersJs(bot), false, bot); + } + assert.equal(botRendersJs(undefined), false); +}); + +test('botRendersJs: matches case-insensitively, like the other registry-derived allowlists', () => { + assert.equal(botRendersJs('googlebot'), true); + assert.equal(botRendersJs('BINGBOT'), true); +}); + +test('botRendersJs: follows a live registry change in both directions', () => { + applyOptions({ + analytics: { + bots: [ + { name: 'MyBot', match: 'mybot', rendersJs: true }, + { name: 'Googlebot', match: 'googlebot', rendersJs: false }, + ], + }, + }); + assert.equal(botRendersJs('MyBot'), true, 'a deployment can flag a crawler the default does not'); + assert.equal(botRendersJs('Googlebot'), false, 'and unflag one the default does'); + // Back to the defaults: a fresh array, a fresh set. + applyOptions({}); + assert.equal(botRendersJs('MyBot'), false); + assert.equal(botRendersJs('Googlebot'), true); +}); + +test('botRendersJs: only a literal true flags; an entry without the field, or with junk in it, does not', () => { + applyOptions({ + analytics: { + bots: [ + { name: 'A', match: 'a' }, + { name: 'B', match: 'b', rendersJs: 'yes' }, + { name: 'C', match: 'c', rendersJs: 1 }, + ], + }, + }); + for (const bot of ['A', 'B', 'C']) assert.equal(botRendersJs(bot), false, bot); +}); + +test('botRendersJs: a derived name has no registry entry to carry the flag', () => { + applyOptions({ analytics: { bots: [], deriveUnknownBots: true } }); + const derived = getBotName(headers({ 'user-agent': 'RenderyBot/1.0 (+https://example.com/bot)' })); + assert.equal(derived, 'RenderyBot'); + assert.equal(botRendersJs(derived), false); +});