From 2935cf11aa14ea445dd0167589afbbf00a6b244c Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 15 May 2026 16:06:00 -0700 Subject: [PATCH 1/2] ci: pre-deploy smoke gate so a JS-dead render can't reach isamples.org MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deploy workflow runs `quarto render` and ships whatever docs/ it produces; nothing ever loads the rendered page in a browser, so a render that "succeeds" but yields a page where DuckDB-WASM never inits, Cesium never draws, or search returns nothing has historically deployed anyway (the failure class behind past "reviewed and still broke" incidents). Adds tests/test_smoke.py: single fresh context, one navigation, poll-for-readiness (no reload loop — rapid reloads exhaust the DuckDB-WASM worker and false-fail). Asserts four unambiguous liveness signals: facet query populated, Cesium canvas attached, a world search returns results, no uncaught JS exception / regression-fingerprint console error. Wires it into quarto-pages.yml between render and Deploy, serving the rendered docs/ locally. Fail-closed: smoke failure fails the job and the Deploy step is skipped. trap-reaps the static server under `bash -e`. Validated both directions: passes the known-good build in ~15s; raises TimeoutError on a rendered-but-JS-dead page. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/quarto-pages.yml | 26 +++++++ tests/test_smoke.py | 119 +++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 tests/test_smoke.py diff --git a/.github/workflows/quarto-pages.yml b/.github/workflows/quarto-pages.yml index c95cc14..e257928 100644 --- a/.github/workflows/quarto-pages.yml +++ b/.github/workflows/quarto-pages.yml @@ -59,6 +59,32 @@ jobs: # individual page content in the web site is defined by various xxx.qmd files. run: | scripts/generate_vocab_docs.sh; quarto render + + # Pre-deploy smoke gate (Option C). Loads the freshly-rendered + # docs/ in a headless browser and asserts the explorer is + # fundamentally alive (DuckDB-WASM inits, Cesium draws, a search + # returns results, no uncaught JS error). Fail-closed: if this + # step fails the job fails and the Deploy step below is skipped, + # so a JS-dead render never reaches isamples.org. + - name: Smoke test rendered site + run: | + pip install pytest playwright + playwright install --with-deps chromium + python -m http.server 8080 --directory docs & + SERVER_PID=$! + # Always reap the static server, even when pytest fails and + # `bash -e` aborts the script (GitHub's default shell). The + # non-zero exit still propagates -> step fails -> Deploy is + # skipped (fail-closed). + trap 'kill $SERVER_PID 2>/dev/null || true' EXIT + # Wait for the static server to accept connections. + for i in $(seq 1 30); do + curl -sf http://localhost:8080/explorer.html >/dev/null && break + sleep 1 + done + ISAMPLES_BASE_URL=http://localhost:8080 \ + pytest tests/test_smoke.py -s -q + - name: Deploy 🚀 # only deploy when push to main if: github.event_name != 'pull_request' diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..0449ca9 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,119 @@ +""" +Pre-deploy smoke test (Option C) — the gate that catches a JS-dead render. + +WHY THIS EXISTS +--------------- +The deploy workflow runs `quarto render` and ships whatever `docs/` it +produces. Neither Codex review nor `pytest --collect-only` ever *loads* +the rendered page in a browser, so a render that "succeeds" but yields a +page where DuckDB-WASM never inits, Cesium never draws, or search returns +nothing has historically deployed to isamples.org anyway. This test closes +exactly that gap: it is run in CI against the freshly-rendered `docs/` +(served locally) *before* the Deploy step. If it fails, the job fails and +the deploy never happens (fail-closed). + +DESIGN CONSTRAINTS (learned the hard way) +----------------------------------------- +- ONE fresh context, ONE navigation, poll-for-readiness. Hammering the + page with rapid reloads exhausts the DuckDB-WASM worker and produces + *false* failures — a test-harness artifact, not a real break. Never + add a reload loop here. +- Assert only on unambiguous "fundamentally alive" signals so a benign + console warning can't block a deploy: DuckDB-WASM inits, Cesium draws, + a search returns results, and no *uncaught* JS exception fired. +- Self-contained: does NOT import the slow CANONICAL_QUERIES benchmark + from test_search_perf.py. This must stay fast (well under a minute). + +Run locally against the rendered output: + + cd docs && python -m http.server 8080 & + ISAMPLES_BASE_URL=http://localhost:8080 pytest tests/test_smoke.py -s +""" +import re + +import pytest +from conftest import SITE_URL + +EXPLORER_URL = f"{SITE_URL}/explorer.html?perf=1" + +# DuckDB-WASM is "alive" once it has run the facet query and written a +# numeric count into the SESAR source facet. Same proxy the perf test +# uses for "ready to search". +_READY_JS = """() => { + const el = document.querySelector( + ".facet-count[data-facet='source'][data-value='SESAR']" + ); + return el && /\\(\\d/.test(el.textContent || ''); +}""" + +# High-signal regression fingerprints. We do NOT fail on every console +# error (benign third-party noise would block deploys); we DO fail on +# uncaught exceptions (pageerror) and on these specific "the JS broke" +# strings, which are what an OJS/scope/undefined-symbol regression emits. +_FATAL_CONSOLE = re.compile( + r"is not defined|is not a function|Cannot read propert|" + r"Uncaught|SyntaxError|ReferenceError", + re.IGNORECASE, +) + + +def test_explorer_smoke(browser): + """Fundamental-liveness gate for explorer.html. Fail-closed in CI.""" + context = browser.new_context(viewport={"width": 1280, "height": 900}) + page = context.new_page() + + page_errors: list[str] = [] + fatal_console: list[str] = [] + page.on("pageerror", lambda e: page_errors.append(str(e))) + + def _on_console(msg): + if msg.type == "error" and _FATAL_CONSOLE.search(msg.text or ""): + fatal_console.append(msg.text) + + page.on("console", _on_console) + + try: + # Single navigation. ?perf=1 matches what the perf test / users hit. + page.goto(EXPLORER_URL, wait_until="domcontentloaded", timeout=60_000) + + # 1. DuckDB-WASM initialized (facet query ran). Poll, do not reload. + page.wait_for_function(_READY_JS, timeout=90_000) + + # 2. Cesium actually drew a globe (canvas attached), not just a + # container div. + page.wait_for_selector( + ".cesium-viewer .cesium-widget canvas", + state="attached", + timeout=30_000, + ) + + # 3. A world search via the *visible* slim-overlay submit button + # returns results. "pottery" is a high-frequency term, so a + # healthy build always returns >=1; zero/blank means broken + # search wiring or a dead query path. + search = page.locator("#sampleSearch") + search.click() + search.fill("pottery") + page.locator("#searchSubmitBtn").click() + page.wait_for_function( + """() => { + const el = document.getElementById('searchResults'); + const t = (el && el.textContent || '').trim(); + return t && !/Searching/i.test(t) && /result/i.test(t); + }""", + timeout=60_000, + ) + results_text = page.locator("#searchResults").inner_text().strip() + + # 4. No uncaught JS exception and no regression-fingerprint + # console error fired during the whole flow. + assert not page_errors, f"Uncaught JS exception(s): {page_errors}" + assert not fatal_console, f"Fatal console error(s): {fatal_console}" + + # Sanity: the result line must actually carry a count. + assert re.search(r"\d", results_text), ( + f"Search returned no countable results: {results_text!r}" + ) + print(f"SMOKE OK — search result: {results_text!r}") + finally: + context.close() From 91e0af97781a448e628427022583f094b51788ed Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 15 May 2026 16:13:40 -0700 Subject: [PATCH 2/2] test(smoke): reduce false-fail surface per Codex review of #225 - Scope _FATAL_CONSOLE to same-origin scripts: a third-party console.error (Cesium CDN, injected extension) can no longer block a deploy; pageerror stays the unconditional hard signal for uncaught app exceptions. - Cesium check now also asserts non-zero canvas dimensions, catching the "widget mounted but globe never sized" case without flaky pixel readback. - Search-result wait 60s -> 90s, aligned with the perf test budget, so a slow CI cold DuckDB-WASM query + remote parquet fetch doesn't false-fail a healthy build. Re-validated: passes the known-good build (~20s). Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_smoke.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 0449ca9..ebe5630 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -67,8 +67,15 @@ def test_explorer_smoke(browser): page.on("pageerror", lambda e: page_errors.append(str(e))) def _on_console(msg): - if msg.type == "error" and _FATAL_CONSOLE.search(msg.text or ""): - fatal_console.append(msg.text) + # Only treat *same-origin* console errors as fatal. A third-party + # script (Cesium CDN, an injected extension) emitting a string that + # matches the regex must not block a deploy — pageerror remains the + # unambiguous hard signal for uncaught app exceptions. + if msg.type != "error" or not _FATAL_CONSOLE.search(msg.text or ""): + return + src = (msg.location or {}).get("url", "") or "" + if src.startswith(SITE_URL): + fatal_console.append(f"{msg.text} @{src}") page.on("console", _on_console) @@ -79,13 +86,27 @@ def _on_console(msg): # 1. DuckDB-WASM initialized (facet query ran). Poll, do not reload. page.wait_for_function(_READY_JS, timeout=90_000) - # 2. Cesium actually drew a globe (canvas attached), not just a - # container div. + # 2. Cesium actually drew a globe: canvas attached AND laid out + # with non-zero dimensions. A 0x0 canvas means the widget + # mounted but the globe never sized/rendered (the "container + # but no globe" failure). A full pixel-readback assertion is + # deliberately avoided — flaky timing, not worth the false-fail + # risk for a liveness gate. page.wait_for_selector( ".cesium-viewer .cesium-widget canvas", state="attached", timeout=30_000, ) + canvas_box = page.evaluate( + """() => { + const c = document.querySelector( + '.cesium-viewer .cesium-widget canvas'); + return c ? {w: c.clientWidth, h: c.clientHeight} : null; + }""" + ) + assert canvas_box and canvas_box["w"] > 0 and canvas_box["h"] > 0, ( + f"Cesium canvas has no dimensions: {canvas_box}" + ) # 3. A world search via the *visible* slim-overlay submit button # returns results. "pottery" is a high-frequency term, so a @@ -101,7 +122,10 @@ def _on_console(msg): const t = (el && el.textContent || '').trim(); return t && !/Searching/i.test(t) && /result/i.test(t); }""", - timeout=60_000, + # Aligned with the perf test's 90s search budget — a cold + # DuckDB-WASM query + remote parquet fetch on a slow CI + # runner can exceed 60s without the build being broken. + timeout=90_000, ) results_text = page.locator("#searchResults").inner_text().strip()