feat(web): the Evidence Matrix, read rather than computed (#708 view 2)

keel/commands/evidence_matrix.py

@@ -0,0 +1,202 @@
+"""The Evidence Matrix: every recorded CSCV run, read rather than computed (#708 view 2).
+
+── WHY THIS IS A READ AND NOT A COMPUTATION ─────────────────────────────────────────────────────
+
+The obvious implementation is to build the matrix on request. Measured on the real ledger, that
+costs 11.9 / 12.9 / 14.3 seconds per session -- roughly 39 seconds of CPU for three sessions, on a
+page the console re-polls every 15 seconds. And over the ledger as a WHOLE it does not run at all:
+
+    ValueError: columns are not synchronous: found lengths [1819, 1828];
+    §78.6 requires a true matrix with the same rows for every column
+
+`matrix.build_matrix` requires synchronous columns, so a PBO is only ever defined WITHIN a session
+whose trials share a bar count. A page cannot pick that scope for the operator without inventing
+their decision.
+
+So #726 made `trials pbo` record every field of its `PBOResult`, and this reads them. The
+distinction is the whole design: **the console displays results an operator ran, and never runs
+one on their behalf.**
+
+── AN UNRUN MATRIX IS NOT AN EMPTY ONE ──────────────────────────────────────────────────────────
+
+Three states, and the middle one is the reason this module has a `candidate_sessions` field at
+all:
+
+* **no ledger** -- a deployment without the research repository beside it.
+* **a ledger with columns and no recorded run** -- the gauntlet has simply not been run here yet,
+  and the page can name the exact command that would change that.
+* **a ledger with recorded runs** -- the matrix.
+
+The guidance names a session that ACTUALLY HAS COLUMNS. `keel trials pbo --session all` looks like
+the obvious thing to suggest and would filter to a session literally named "all", find nothing, and
+print a refusal -- teaching an operator that the page does not know what it is talking about.
+
+⛔ THE STRATHERN RAIL. Every figure here is a diagnostic and none of them is sortable, on the route
+or in the view. A matrix ordered by PBO is a leaderboard of overfitting scores, and PBO's own
+module carries the warning: it "evaluates the quality of a selection process and must never become
+the objective that selection relies on".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+from keel.research.ledger import read_trials
+
+#: The kind #726 writes a recorded CSCV run under.
+CSCV_KIND = "cscv"
+
+#: The command that would populate this page. Composed in Python and placed by the client (the
+#: rule #707's cancel modal follows), and it names a REAL session: `--session all` filters to a
+#: session literally called "all", finds nothing, and prints a refusal.
+MATRIX_INVOCATION = "keel trials pbo --session {session}"
+
+#: How many synchronous columns a session needs before `trials pbo` can say anything about it.
+#: `matrix.build_matrix` warns below `MIN_RECOMMENDED_COLUMNS` and refuses at zero; two is the
+#: floor at which a combinatorial split exists at all.
+MIN_COLUMNS_FOR_A_RUN = 2
+
+
+@dataclass(frozen=True)
+class MatrixRow:
+    """One recorded CSCV run -- every field `PBOResult` carries, as it was recorded.
+
+    Absent figures are `None`, never zero. A `pbo` of `0` is the strongest possible statement
+    about a selection process and a missing one is no statement at all; the six pre-#726 gauntlet
+    rows carry neither, and this page says so rather than rendering them as perfect.
+    """
+
+    trial_id: str
+    timestamp: int
+    session: str
+    pbo: Decimal | None
+    degradation_slope: Decimal | None
+    degradation_intercept: Decimal | None
+    prob_loss: Decimal | None
+    dominance_1st: bool | None
+    dominance_2nd: bool | None
+    n_columns: int | None
+    n_blocks: int | None
+    n_combinations: int | None
+    rows_used: int | None
+    rows_dropped: int | None
+    columns_refused: int | None
+
+
+@dataclass(frozen=True)
+class MatrixReport:
+    now_ts: int
+    ledger_present: bool
+    rows: tuple[MatrixRow, ...]
+    #: Sessions whose trials could form a matrix, whether or not one has been run over them.
+    #: What the empty state names, so its command is one that would actually work.
+    candidate_sessions: tuple[str, ...]
+
+    @property
+    def recorded_count(self) -> int:
+        """Held on the report because `keel/web/payload.py` may not call `len()` (Rule 6e)."""
+        return len(self.rows)
+
+    @property
+    def any_recorded(self) -> bool:
+        return bool(self.rows)
+
+    @property
+    def suggested_session(self) -> str:
+        """The session the empty state tells an operator to run against, or `""` when none could.
+
+        The FIRST candidate rather than a chosen one: choosing would be this page ranking sessions
+        by something, and there is nothing here it may rank by.
+        """
+        return self.candidate_sessions[0] if self.candidate_sessions else ""
+
+
+def _decimal_or_none(summary: dict[str, Any], key: str) -> Decimal | None:
+    value = summary.get(key)
+    return value if isinstance(value, Decimal) else None
+
+
+def _int_or_none(summary: dict[str, Any], key: str) -> int | None:
+    value = summary.get(key)
+    # `bool` is an `int` in Python and is never one of these counts. Checked first, because
+    # `isinstance(True, int)` would otherwise render a dominance flag as a column count.
+    if isinstance(value, bool):
+        return None
+    return value if isinstance(value, int) else None
+
+
+def _flag_or_none(summary: dict[str, Any], key: str) -> bool | None:
+    """THREE-VALUED. `bool(None)` is `False`, and `False` on a dominance flag is a positive claim
+    -- "the in-sample distribution did not dominate" -- which is not what an absent field says."""
+    value = summary.get(key)
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, int):
+        return bool(value)
+    return None
+
+
+def _candidate_sessions(trials: list[Any]) -> tuple[str, ...]:
+    """Sessions holding enough usable columns for `trials pbo` to run over them.
+
+    Counts the trials `matrix.build_matrix` would ACCEPT -- a per-bar series, not `series_missing`
+    -- rather than every trial with the session label, because a session of six backfilled rows
+    would otherwise be suggested and the suggested command would refuse.
+
+    It does NOT check synchronicity. Doing so means reading every series, which is most of the
+    cost this module exists to avoid, and a session whose columns turn out to be ragged gets a
+    clear refusal from the command itself. Suggesting a session that might not work is a much
+    smaller harm than a page that costs 12 seconds to render.
+    """
+    usable: dict[str, int] = {}
+    for trial in trials:
+        if trial.series_missing or not trial.per_bar_pnl:
+            continue
+        usable[trial.session] = usable.get(trial.session, 0) + 1
+    return tuple(
+        session for session, count in usable.items() if count >= MIN_COLUMNS_FOR_A_RUN
+    )
+
+
+def gather_matrix(path: Path | str, *, now_ts: int) -> MatrixReport:
+    """Every recorded CSCV run in the ledger at `path`, oldest first. No computation."""
+    ledger = Path(path)
+    if not ledger.exists():
+        return MatrixReport(
+            now_ts=now_ts, ledger_present=False, rows=(), candidate_sessions=()
+        )
+
+    trials = list(read_trials(ledger))
+    rows = tuple(
+        MatrixRow(
+            trial_id=trial.trial_id,
+            timestamp=trial.timestamp,
+            session=trial.session,
+            pbo=_decimal_or_none(trial.summary, "pbo"),
+            degradation_slope=_decimal_or_none(trial.summary, "degradation_slope"),
+            degradation_intercept=_decimal_or_none(trial.summary, "degradation_intercept"),
+            prob_loss=_decimal_or_none(trial.summary, "prob_loss"),
+            dominance_1st=_flag_or_none(trial.summary, "dominance_1st"),
+            dominance_2nd=_flag_or_none(trial.summary, "dominance_2nd"),
+            n_columns=_int_or_none(trial.summary, "n_columns"),
+            n_blocks=_int_or_none(trial.summary, "n_blocks"),
+            n_combinations=_int_or_none(trial.summary, "n_combinations"),
+            rows_used=_int_or_none(trial.summary, "rows_used"),
+            rows_dropped=_int_or_none(trial.summary, "rows_dropped"),
+            columns_refused=_int_or_none(trial.summary, "columns_refused"),
+        )
+        for trial in trials
+        # The KIND, not the presence of a `pbo` key: the six pre-#726 gauntlet rows carry a `pbo`
+        # and are not CSCV runs -- they are per-trial gauntlet outcomes, which #708's view 3 shows.
+        # Reading them here would put two different measurements in one table under one heading.
+        if trial.kind == CSCV_KIND
+    )
+    return MatrixReport(
+        now_ts=now_ts,
+        ledger_present=True,
+        rows=rows,
+        candidate_sessions=_candidate_sessions(trials),
+    )

keel/web/api.py

@@ -465,6 +465,19 @@ def read_gauntlet(cfg: ServeConfig, _query: Query, _state: Any, now_ts: int) ->
     return payload.gauntlet_payload(gather_gauntlet(_ledger_path(cfg), now_ts=now_ts))
 
 
+def read_matrix(cfg: ServeConfig, _query: Query, _state: Any, now_ts: int) -> dict[str, Any]:
+    """The Evidence Matrix (#708 view 2) -- recorded CSCV runs, READ.
+
+    No database and no computation. `matrix.build_matrix` costs 11.9-14.3 s per session on the
+    real ledger and raises over the ledger as a whole (columns are only synchronous within a
+    session), and this route answers a page that polls every 15 s. #726 made `trials pbo` record
+    its whole result; this reads it.
+    """
+    from keel.commands.evidence_matrix import gather_matrix
+
+    return payload.matrix_payload(gather_matrix(_ledger_path(cfg), now_ts=now_ts))
+
+
 def read_slippage(cfg: ServeConfig, _query: Query, _state: Any, now_ts: int) -> dict[str, Any]:
     """What a fill is assumed to cost, per product (#708, view 4).
 
@@ -1000,6 +1013,16 @@ class ApiRoute:
         collection="",
         sortable=(),
     ),
+    # #708 view 2. The rail again: no `collection`, no `sortable`. A matrix ordered by PBO is a
+    # leaderboard of overfitting scores, and `cscv.py` forbids PBO as a ranking key in its own
+    # source.
+    "/api/research/matrix": ApiRoute(
+        html_route="/research",
+        read=read_matrix,
+        needs_database=False,
+        collection="",
+        sortable=(),
+    ),
     "/api/research/gauntlet": ApiRoute(
         html_route="/research",
         read=read_gauntlet,

keel/web/payload.py

@@ -121,6 +121,7 @@
 if TYPE_CHECKING:  # pragma: no cover - typing only
     from keel.commands.activity import ActivityCycle, ActivityEvent, ActivityFeed
     from keel.commands.balances import AssetBalanceRow, BalancesReport
+    from keel.commands.evidence_matrix import MatrixReport, MatrixRow
     from keel.commands.gauntlet import GauntletReport, GauntletRow
     from keel.commands.insights import (
         AccountSummary,
@@ -1394,6 +1395,133 @@ def _rules_followed_state(value: bool | None) -> str:
     return NEUTRAL if value else WARN
 
 
+# -- the evidence matrix (#708 view 2) -------------------------------------------------------------
+#
+# READ, never computed. `build_matrix` costs 11.9-14.3 s per session on the real ledger and raises
+# over the ledger as a whole; this page polls every 15 s. #726 made `trials pbo` record its whole
+# `PBOResult` and this serialises what it recorded.
+#
+# ⛔ THE STRATHERN RAIL, on the wire. No sortable column on the route and no sort key in the view.
+# A matrix ordered by PBO is a leaderboard of overfitting scores, and `cscv.py` carries the
+# warning: PBO "evaluates the quality of a selection process and must never become the objective
+# that selection relies on".
+
+
+def _matrix_row_payload(row: MatrixRow) -> dict[str, Any]:
+    """One recorded CSCV run.
+
+    **`pbo` carries NO state.** It is the one figure a reader will want graded, and grading it is
+    exactly what the rail refuses: a high PBO beside a flat, positive OOS scatter is the GOOD
+    outcome -- a broad plateau of near-identical configurations produces high PBO by construction
+    -- so a colour here would be a verdict the number does not support. `trials pbo`'s own closing
+    sentence says to read it alongside the degradation slope, never alone, and both cross plainly
+    so a reader does exactly that.
+
+    The dominance flags DO carry a state, because they are already verdicts: stochastic dominance
+    either held or it did not. Three-valued, so an unrecorded flag is not read as a denial.
+    """
+    return {
+        "at": moment(row.timestamp),
+        "trial_id": row.trial_id,
+        "session": row.session,
+        "pbo": ratio(row.pbo, places=4),
+        "degradation_slope": ratio(row.degradation_slope, places=4),
+        "degradation_intercept": ratio(row.degradation_intercept, places=4),
+        "prob_loss": ratio(row.prob_loss, places=4),
+        "dominance_1st": _dominance_payload(row.dominance_1st),
+        "dominance_2nd": _dominance_payload(row.dominance_2nd),
+        "n_columns": count(row.n_columns),
+        "n_blocks": count(row.n_blocks),
+        "n_combinations": count(row.n_combinations),
+        "rows_used": count(row.rows_used),
+        "rows_dropped": count(row.rows_dropped),
+        "columns_refused": count(row.columns_refused),
+    }
+
+
+def _dominance_payload(value: bool | None) -> Field:
+    """A verdict that already happened, in three readings.
+
+    `flag()` would collapse the third: `False` says the in-sample distribution did NOT dominate,
+    and `None` says nobody recorded whether it did.
+    """
+    if value is None:
+        return label("", display="not recorded", state=UNKNOWN)
+    return label(
+        "yes" if value else "no",
+        display="dominated" if value else "did not dominate",
+        state=NEUTRAL,
+    )
+
+
+def _matrix_state_payload(report: MatrixReport) -> Field:
+    """Which of the three states this deployment is in, as the one sentence the page leads with.
+
+    An unrun matrix is not an empty one, and the middle state is why this is not a `flag`:
+
+    * **no ledger** -- a deployment without the research repository beside it. Nothing to run.
+    * **columns, no run** -- the honest common case, and the one that can be acted on.
+    * **recorded runs** -- the matrix.
+    """
+    if not report.ledger_present:
+        return label(
+            "no-ledger",
+            display="No research ledger beside this deployment — there is nothing to compile.",
+            state=UNKNOWN,
+        )
+    if report.any_recorded:
+        return label(
+            "recorded",
+            display=(
+                "Compiled from recorded gauntlet runs — nothing here was computed "
+                "for this page."
+            ),
+            state=NEUTRAL,
+        )
+    if report.suggested_session:
+        return label(
+            "unrun",
+            display=(
+                "No recorded evidence matrix. Matrix data is compiled from combinatorial gauntlet "
+                "runs, which are never computed for this page — run one in your terminal."
+            ),
+            state=UNKNOWN,
+        )
+    return label(
+        "no-columns",
+        display=(
+            "No recorded evidence matrix, and no session holds enough usable columns to run one: "
+            "a matrix needs trials with a per-bar P&L series, and every recorded trial is "
+            "series_missing."
+        ),
+        state=UNKNOWN,
+    )
+
+
+def matrix_payload(report: MatrixReport) -> dict[str, Any]:
+    """The Evidence Matrix (#708 view 2).
+
+    `invocation` is composed HERE and placed by the client, the same rule #707's cancel modal
+    follows: a client concatenating `keel trials pbo --session ` and a name could print a command
+    that does not exist, and the session it names has to be one that ACTUALLY HAS COLUMNS --
+    `--session all` would filter to a session literally named "all" and refuse.
+    """
+    from keel.commands.evidence_matrix import MATRIX_INVOCATION
+
+    return {
+        "as_of": iso(report.now_ts),
+        "generated_at": moment(report.now_ts),
+        "state": _matrix_state_payload(report),
+        "recorded_count": count(report.recorded_count),
+        "invocation": (
+            MATRIX_INVOCATION.format(session=report.suggested_session)
+            if report.suggested_session and not report.any_recorded
+            else ""
+        ),
+        "rows": [_matrix_row_payload(row) for row in report.rows],
+    }
+
+
 # -- plans, inverted (#706) ------------------------------------------------------------------------
 #
 # THE ONE PAGE IN THIS APPLICATION WHOSE SUBJECT IS THE PROJECT RATHER THAN THE DEPLOYMENT, and