diff --git a/.github/workflows/quarto-pages.yml b/.github/workflows/quarto-pages.yml index c95cc14..e257928 100644 --- a/.github/workflows/quarto-pages.yml +++ b/.github/workflows/quarto-pages.yml @@ -59,6 +59,32 @@ jobs: # individual page content in the web site is defined by various xxx.qmd files. run: | scripts/generate_vocab_docs.sh; quarto render + + # Pre-deploy smoke gate (Option C). Loads the freshly-rendered + # docs/ in a headless browser and asserts the explorer is + # fundamentally alive (DuckDB-WASM inits, Cesium draws, a search + # returns results, no uncaught JS error). Fail-closed: if this + # step fails the job fails and the Deploy step below is skipped, + # so a JS-dead render never reaches isamples.org. + - name: Smoke test rendered site + run: | + pip install pytest playwright + playwright install --with-deps chromium + python -m http.server 8080 --directory docs & + SERVER_PID=$! + # Always reap the static server, even when pytest fails and + # `bash -e` aborts the script (GitHub's default shell). The + # non-zero exit still propagates -> step fails -> Deploy is + # skipped (fail-closed). + trap 'kill $SERVER_PID 2>/dev/null || true' EXIT + # Wait for the static server to accept connections. + for i in $(seq 1 30); do + curl -sf http://localhost:8080/explorer.html >/dev/null && break + sleep 1 + done + ISAMPLES_BASE_URL=http://localhost:8080 \ + pytest tests/test_smoke.py -s -q + - name: Deploy 🚀 # only deploy when push to main if: github.event_name != 'pull_request' diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..ebe5630 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,143 @@ +""" +Pre-deploy smoke test (Option C) — the gate that catches a JS-dead render. + +WHY THIS EXISTS +--------------- +The deploy workflow runs `quarto render` and ships whatever `docs/` it +produces. Neither Codex review nor `pytest --collect-only` ever *loads* +the rendered page in a browser, so a render that "succeeds" but yields a +page where DuckDB-WASM never inits, Cesium never draws, or search returns +nothing has historically deployed to isamples.org anyway. This test closes +exactly that gap: it is run in CI against the freshly-rendered `docs/` +(served locally) *before* the Deploy step. If it fails, the job fails and +the deploy never happens (fail-closed). + +DESIGN CONSTRAINTS (learned the hard way) +----------------------------------------- +- ONE fresh context, ONE navigation, poll-for-readiness. Hammering the + page with rapid reloads exhausts the DuckDB-WASM worker and produces + *false* failures — a test-harness artifact, not a real break. Never + add a reload loop here. +- Assert only on unambiguous "fundamentally alive" signals so a benign + console warning can't block a deploy: DuckDB-WASM inits, Cesium draws, + a search returns results, and no *uncaught* JS exception fired. +- Self-contained: does NOT import the slow CANONICAL_QUERIES benchmark + from test_search_perf.py. This must stay fast (well under a minute). + +Run locally against the rendered output: + + cd docs && python -m http.server 8080 & + ISAMPLES_BASE_URL=http://localhost:8080 pytest tests/test_smoke.py -s +""" +import re + +import pytest +from conftest import SITE_URL + +EXPLORER_URL = f"{SITE_URL}/explorer.html?perf=1" + +# DuckDB-WASM is "alive" once it has run the facet query and written a +# numeric count into the SESAR source facet. Same proxy the perf test +# uses for "ready to search". +_READY_JS = """() => { + const el = document.querySelector( + ".facet-count[data-facet='source'][data-value='SESAR']" + ); + return el && /\\(\\d/.test(el.textContent || ''); +}""" + +# High-signal regression fingerprints. We do NOT fail on every console +# error (benign third-party noise would block deploys); we DO fail on +# uncaught exceptions (pageerror) and on these specific "the JS broke" +# strings, which are what an OJS/scope/undefined-symbol regression emits. +_FATAL_CONSOLE = re.compile( + r"is not defined|is not a function|Cannot read propert|" + r"Uncaught|SyntaxError|ReferenceError", + re.IGNORECASE, +) + + +def test_explorer_smoke(browser): + """Fundamental-liveness gate for explorer.html. Fail-closed in CI.""" + context = browser.new_context(viewport={"width": 1280, "height": 900}) + page = context.new_page() + + page_errors: list[str] = [] + fatal_console: list[str] = [] + page.on("pageerror", lambda e: page_errors.append(str(e))) + + def _on_console(msg): + # Only treat *same-origin* console errors as fatal. A third-party + # script (Cesium CDN, an injected extension) emitting a string that + # matches the regex must not block a deploy — pageerror remains the + # unambiguous hard signal for uncaught app exceptions. + if msg.type != "error" or not _FATAL_CONSOLE.search(msg.text or ""): + return + src = (msg.location or {}).get("url", "") or "" + if src.startswith(SITE_URL): + fatal_console.append(f"{msg.text} @{src}") + + page.on("console", _on_console) + + try: + # Single navigation. ?perf=1 matches what the perf test / users hit. + page.goto(EXPLORER_URL, wait_until="domcontentloaded", timeout=60_000) + + # 1. DuckDB-WASM initialized (facet query ran). Poll, do not reload. + page.wait_for_function(_READY_JS, timeout=90_000) + + # 2. Cesium actually drew a globe: canvas attached AND laid out + # with non-zero dimensions. A 0x0 canvas means the widget + # mounted but the globe never sized/rendered (the "container + # but no globe" failure). A full pixel-readback assertion is + # deliberately avoided — flaky timing, not worth the false-fail + # risk for a liveness gate. + page.wait_for_selector( + ".cesium-viewer .cesium-widget canvas", + state="attached", + timeout=30_000, + ) + canvas_box = page.evaluate( + """() => { + const c = document.querySelector( + '.cesium-viewer .cesium-widget canvas'); + return c ? {w: c.clientWidth, h: c.clientHeight} : null; + }""" + ) + assert canvas_box and canvas_box["w"] > 0 and canvas_box["h"] > 0, ( + f"Cesium canvas has no dimensions: {canvas_box}" + ) + + # 3. A world search via the *visible* slim-overlay submit button + # returns results. "pottery" is a high-frequency term, so a + # healthy build always returns >=1; zero/blank means broken + # search wiring or a dead query path. + search = page.locator("#sampleSearch") + search.click() + search.fill("pottery") + page.locator("#searchSubmitBtn").click() + page.wait_for_function( + """() => { + const el = document.getElementById('searchResults'); + const t = (el && el.textContent || '').trim(); + return t && !/Searching/i.test(t) && /result/i.test(t); + }""", + # Aligned with the perf test's 90s search budget — a cold + # DuckDB-WASM query + remote parquet fetch on a slow CI + # runner can exceed 60s without the build being broken. + timeout=90_000, + ) + results_text = page.locator("#searchResults").inner_text().strip() + + # 4. No uncaught JS exception and no regression-fingerprint + # console error fired during the whole flow. + assert not page_errors, f"Uncaught JS exception(s): {page_errors}" + assert not fatal_console, f"Fatal console error(s): {fatal_console}" + + # Sanity: the result line must actually carry a count. + assert re.search(r"\d", results_text), ( + f"Search returned no countable results: {results_text!r}" + ) + print(f"SMOKE OK — search result: {results_text!r}") + finally: + context.close()