fix(scripts): measure-self-test-floor's probe reads NOT MEASURED when the baseline run already fails (#15391)

scripts/measure-self-test-floor.mjs

@@ -65,6 +65,15 @@
  *   ACCIDENT  exit != 0 AND it printed nothing -- a non-zero exit with no
  *             refusal behind it. NOT counted among HELD, ever.
  *
+ * A FOURTH reading sits UNDER all three, and it is a PRECONDITION rather than a
+ * verdict: if the UNMUTATED file already exits non-zero, this tree cannot run
+ * it at all, so the mutation had nothing to defeat and NOTHING WAS MEASURED.
+ * That case does not look like an absence -- both runs exit non-zero and both
+ * print a module-resolution stack, so the mutated run "speaks" and the verdict
+ * above reads HELD. A checkout that has not been `pnpm install`ed therefore
+ * reports the FLATTERING answer for every file it cannot load, and the same row
+ * reads ACCIDENT once the tree is installed (#15391).
+ *
  * The `mutatedBytes` / `mutatedHead` fields the row already carried are what this
  * reads; `mutatedSpoke` publishes the reading. Deliberately the verdict does NOT
  * match the refusal WORDING: the repair landed in three spellings and teaching
@@ -104,6 +113,9 @@ const DISPATCH = /(?:includes|has)\(\s*['"`]--self-test['"`]\s*\)/;
 /** Marker injected by the probe. Its presence on disk is the mutation's proof. */
 const PROBE_MARKER = 'OS_SELF_TEST_FLOOR_PROBE';
 
+/** "Did this run SAY anything": its first non-blank line, or '' if it said nothing. */
+const firstNonBlankLine = (out) => out.split('\n').find((l) => l.trim()) ?? '';
+
 // ---------------------------------------------------------------------------
 // Instrument 1 -- the static assertion-floor criterion
 // ---------------------------------------------------------------------------
@@ -212,6 +224,15 @@ export function injectEarlyReturn(src, name) {
  * resolution still answer the same, and the marker is re-read FROM DISK before
  * the run: an editor step that matched nothing exits 0 just as happily as one
  * that landed, and an unmutated file would report "held" for no reason at all.
+ *
+ * THE BASELINE IS A PRECONDITION, NOT A DATA POINT. It is read BEFORE any
+ * reading of the mutated run, and a non-zero one ends the probe as NOT MEASURED
+ * with the reason -- see the fourth reading in this file's header. The mutated
+ * run is then not spawned AT ALL: in the shape that motivates this (a checkout
+ * whose dependencies are absent) every row is baseline-red, so running each
+ * doomed mutation would double a whole sweep of spawns to learn nothing. The
+ * row still publishes what the baseline said, because `baselineHead` is usually
+ * the whole diagnosis (`Cannot find package ...` reads as "run pnpm install").
  */
 export function probeEarlyReturn(absFile, entry, { timeout = 120000 } = {}) {
   const src = readFileSync(absFile, 'utf8');
@@ -228,10 +249,28 @@ export function probeEarlyReturn(absFile, entry, { timeout = 120000 } = {}) {
     if (onDisk !== 1) return { verdict: 'NOT MEASURED', why: `mutation not on disk (marker x${onDisk})` };
 
     const base = spawnSync(cmd, [absFile, '--self-test'], { cwd: ROOT, timeout, encoding: 'utf8' });
-    const mut = spawnSync(cmd, [probePath, '--self-test'], { cwd: ROOT, timeout, encoding: 'utf8' });
     const baseOut = (base.stdout ?? '') + (base.stderr ?? '');
+    if (base.signal) return { verdict: 'NOT MEASURED', why: `killed by ${base.signal}` };
+    // PRECONDITION. A file the tree cannot run offered the mutation nothing to
+    // defeat, so no verdict below is available -- however loudly the mutated run
+    // would have exited and spoken. Read before the mutated run is spawned.
+    if (base.status !== 0) {
+      return {
+        verdict: 'NOT MEASURED',
+        why:
+          base.status === null
+            ? `baseline run could not start (${base.error?.code ?? 'no exit status'})`
+            : `baseline run failed (exit ${base.status})`,
+        entry,
+        baselineExit: base.status,
+        baselineBytes: baseOut.length,
+        baselineHead: firstNonBlankLine(baseOut),
+      };
+    }
+
+    const mut = spawnSync(cmd, [probePath, '--self-test'], { cwd: ROOT, timeout, encoding: 'utf8' });
     const mutOut = (mut.stdout ?? '') + (mut.stderr ?? '');
-    if (mut.signal || base.signal) return { verdict: 'NOT MEASURED', why: `killed by ${mut.signal ?? base.signal}` };
+    if (mut.signal) return { verdict: 'NOT MEASURED', why: `killed by ${mut.signal}` };
     // A mutation that changed nothing observable did not reach the executed
     // path, whatever its exit code says.
     if (baseOut === mutOut && base.status === mut.status) {
@@ -240,7 +279,7 @@ export function probeEarlyReturn(absFile, entry, { timeout = 120000 } = {}) {
     // Did the mutated run SAY anything? Read as "printed a non-blank line", the
     // same reading `mutatedHead` already publishes and quotes -- a run whose whole
     // output is a newline has non-zero bytes and still refused nothing.
-    const mutatedHead = mutOut.split('\n').find((l) => l.trim()) ?? '';
+    const mutatedHead = firstNonBlankLine(mutOut);
     const mutatedSpoke = mutatedHead !== '';
     return {
       // Exit code alone cannot tell a refusal from an accident -- see the header.
@@ -323,6 +362,34 @@ const ACCIDENT_GATE = [
   '',
 ].join('\n');
 
+/**
+ * The measured BASELINE-RED shape, reduced: a file this tree cannot run at all,
+ * because one of its imports does not resolve. Its self-test is otherwise
+ * perfectly ordinary and injectable -- the point is that neither run ever
+ * reaches it. Both runs die in module resolution, both exit non-zero, and both
+ * PRINT a stack trace, so `mutatedSpoke` is true and an exit-code-and-speech
+ * verdict scores it HELD: a hold awarded for the tree being broken. This is the
+ * shape `scripts/audits/14744-before-update-per-row-value-census.mjs` takes in a
+ * checkout whose `node_modules` lacks its `typescript` dependency, where it read
+ * HELD while reading ACCIDENT in an installed one (#15391).
+ *
+ * The unresolvable import is a name no registry can supply, and it is the
+ * FIRST statement, so the failure is the module loader's and cannot be confused
+ * with anything the self-test did.
+ */
+const UNRUNNABLE_GATE = [
+  '#!/usr/bin/env node',
+  "import 'os-self-test-floor-control-no-such-package';",
+  'function selfTest() {',
+  '  const failures = [];',
+  "  if (1 !== 1) failures.push('x');",
+  "  if (failures.length) { console.error('nope'); process.exit(1); }",
+  "  console.log('fixture self-test: 1 case passes');",
+  '}',
+  "if (process.argv.includes('--self-test')) selfTest();",
+  '',
+].join('\n');
+
 /**
  * The ternary exit, reduced: a roster floor whose ONLY failure production is
  * `process.exit(<cond> ? 0 : 1)`. It carries none of the NAMED spellings -- no
@@ -404,12 +471,15 @@ export function runControls() {
     const holed = join(dir, 'holed-gate.mjs');
     const sound = join(dir, 'sound-gate.mjs');
     const accident = join(dir, 'accident-gate.mjs');
+    const unrunnable = join(dir, 'unrunnable-gate.mjs');
     writeFileSync(holed, HOLED_GATE);
     writeFileSync(sound, SOUND_GATE);
     writeFileSync(accident, ACCIDENT_GATE);
+    writeFileSync(unrunnable, UNRUNNABLE_GATE);
     const h = probeEarlyReturn(holed, 'selfTest');
     const s = probeEarlyReturn(sound, 'selfTest');
     const a = probeEarlyReturn(accident, 'runSelfTest');
+    const u = probeEarlyReturn(unrunnable, 'selfTest');
     say(h.verdict === 'DEFEATED',
       `POSITIVE CONTROL FAILED: the probe read a known-holed gate as ${h.verdict} (${h.why ?? ''})`);
     say(h.mutatedBytes === 0,
@@ -425,6 +495,16 @@ export function runControls() {
       `POSITIVE CONTROL FAILED: the probe read a silent non-zero exit as ${a.verdict} (${a.why ?? ''}) -- an exit code is not a handshake`);
     say(a.mutatedExit !== 0 && a.mutatedBytes === 0 && a.mutatedSpoke === false,
       `POSITIVE CONTROL FAILED: the accident fixture no longer produces the measured shape (exit ${a.mutatedExit}, ${a.mutatedBytes} byte(s)); the ACCIDENT verdict above would then be passing for the wrong reason`);
+    // The precondition, in the direction that matters: a baseline that already
+    // failed ends the probe, and it must end it as NOT MEASURED -- never as the
+    // HELD an exit-code-and-speech reading would award it (#15391).
+    say(u.verdict === 'NOT MEASURED' && /^baseline run failed \(exit /.test(u.why ?? ''),
+      `POSITIVE CONTROL FAILED: a file whose BASELINE run already exits non-zero was read as ${u.verdict} (${u.why ?? ''}); the mutation had nothing to defeat, so nothing was measured`);
+    // ... and for the RIGHT reason: this fixture has to be the flattering shape,
+    // a red baseline that SPEAKS. A fixture that fell silent, or that stopped
+    // being red, would satisfy the verdict above while testing nothing.
+    say(u.baselineExit !== 0 && u.baselineBytes > 0 && u.baselineHead !== '',
+      `POSITIVE CONTROL FAILED: the unrunnable fixture no longer produces the measured shape (baseline exit ${u.baselineExit}, ${u.baselineBytes} byte(s)); the NOT MEASURED verdict above would then be passing for the wrong reason`);
   } finally {
     rmSync(dir, { recursive: true, force: true });
   }