feat(web): one chronology over four stores, with an injection-safe au…

keel/web/api.py

@@ -37,6 +37,7 @@
 import time
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any
 
 from keel.web import payload
@@ -358,6 +359,68 @@ def read_balances(cfg: ServeConfig, _query: Query, _state: Any, now_ts: int) ->
     return payload.balances_payload(report)
 
 
+def _timeline_report(cfg: ServeConfig, query: Query, now_ts: int) -> Any:
+    """The merged timeline for one request. Shared by the JSON route and the CSV export so the
+    file an operator downloads is the same chronology the page showed them -- two builders would
+    be two answers to "what happened", and the export is the one that goes to an auditor."""
+    from keel.commands.activity import (
+        feed_from_lines,
+        read_log_window,
+        resolve_log_path,
+    )
+    from keel.commands.timeline import (
+        DEFAULT_TIMELINE_LIMIT,
+        MAX_TIMELINE_LIMIT,
+        gather_timeline,
+    )
+
+    config = load_config(cfg.config_path)
+    raw_limit = _first(query, "limit")
+    limit = (
+        DEFAULT_TIMELINE_LIMIT if not raw_limit else _whole_number(raw_limit, MAX_TIMELINE_LIMIT)
+    )
+
+    # The engine log is read through `activity`'s own bounded window rather than re-parsed here:
+    # that module owns finding the file, reading a bounded tail of it and turning it into cycles,
+    # and a second implementation would be a second answer to "what did the agent do".
+    cycles: tuple[Any, ...] = ()
+    try:
+        log_path = resolve_log_path(config)
+        window = read_log_window(log_path)
+        # `LogWindow` carries the lines and a read status, not the path -- `source` is the
+        # feed's own label for where the lines came from, so it is passed the path we resolved.
+        cycles = feed_from_lines(
+            window.lines, source=str(log_path), truncated=window.truncated
+        ).cycles
+    except OSError:
+        # No log yet, or an unreadable one. The timeline still has three other sources, and a
+        # missing log is not a reason to fail the whole page -- the `system` rows are simply
+        # absent, which is what an unread log honestly means.
+        cycles = ()
+
+    repo = open_repo(cfg.db_path)
+    try:
+        return gather_timeline(
+            repo,
+            now_ts=now_ts,
+            scope=_first(query, "scope") or "all",
+            kind=_first(query, "kind") or "",
+            limit=limit,
+            cycles=cycles,
+        )
+    finally:
+        close_repo(repo)
+
+
+def read_timeline(cfg: ServeConfig, query: Query, _state: Any, now_ts: int) -> dict[str, Any]:
+    """One chronology over the engine log, the orders book, the ledger and the attestations.
+
+    READ ONLY, no broker, no network -- the same posture as every route here. `?kind=` is applied
+    rather than refused, `?scope=`'s own normalisation is reused, and both are echoed back.
+    """
+    return payload.timeline_payload(_timeline_report(cfg, query, now_ts))
+
+
 def read_insights(cfg: ServeConfig, _query: Query, _state: Any, now_ts: int) -> dict[str, Any]:
     """The per-rule track records, the promotion-gate distances, and the account-equity series.
 
@@ -635,6 +698,12 @@ class ApiRoute:
         collection="assets",
         sortable=("product_id", "qty", "mark", "market_value"),
     ),
+    "/api/timeline": ApiRoute(
+        html_route="/timeline",
+        read=read_timeline,
+        collection="rows",
+        sortable=("ts", "kind", "provenance", "source", "product_id"),
+    ),
     "/api/rules": ApiRoute(
         html_route="/rules",
         read=read_rules,
@@ -893,3 +962,31 @@ def action_document(result: Any) -> dict[str, Any]:
 def sortable_columns() -> Mapping[str, Sequence[str]]:
     """The declared sort surface, for a test to read rather than restate."""
     return {path: route.sortable for path, route in API_ROUTES.items() if route.sortable}
+
+
+#: The one path on this server that does not answer JSON (#703).
+#:
+#: Deliberately NOT an `ApiRoute`: every entry in `API_ROUTES` is wrapped in the JSON envelope by
+#: `respond`, and `tests/web/test_api.py` parametrises the envelope, the no-JSON-number walk and
+#: the JSON MIME assertions over that table. A CSV route in it would either break those or force
+#: each of them to grow an exception -- and an exception inside a security pin is how the pin
+#: stops meaning anything. It gets its own handler branch and its own header suite instead.
+CSV_EXPORT_PATH = "/api/timeline/export.csv"
+
+
+def export_timeline_csv(cfg: ServeConfig, query: Query) -> tuple[str, str]:
+    """`(csv_text, filename)` for the timeline export.
+
+    Built from the SAME `_timeline_report` the JSON route uses, so the file an operator hands an
+    auditor is the chronology the page showed them.
+
+    Every text cell goes through `csv_safe` (see `keel/commands/timeline.py`): the file is meant
+    to be opened in Excel or Sheets, both of which execute a cell beginning `=`, `+`, `-` or `@`,
+    and several columns carry text keel did not write.
+    """
+    from keel.commands.timeline import to_csv
+
+    now_ts = int(time.time())
+    report = _timeline_report(cfg, query, now_ts)
+    stamp = datetime.fromtimestamp(now_ts, tz=UTC).strftime("%Y%m%d-%H%M%S")
+    return to_csv(report), f"keel-activity-{stamp}.csv"

keel/web/payload.py

@@ -145,6 +145,7 @@
         SubscriptionStatusRow,
         WithdrawalAttestationStatus,
     )
+    from keel.commands.timeline import TimelineReport, TimelineRow
     from keel.venue_readiness import VenueReadinessRow
 
 
@@ -1716,6 +1717,84 @@ def balances_payload(report: BalancesReport) -> dict[str, Any]:
     }
 
 
+#: How each provenance is styled (#703). NOT a judgement about quality -- an imported ledger line
+#: is not "worse" evidence than a venue report, it is DIFFERENT evidence -- so nothing here is
+#: `bad`. `simulated` is the one that warns, because a synthetic fill sitting in a chronology
+#: beside real ones is the single thing a reader must not skim past.
+_PROVENANCE_STATES: Mapping[str, str] = {
+    "venue-reported": NEUTRAL,
+    "simulated": WARN,
+    "imported-ledger": NEUTRAL,
+    "human-attested": NEUTRAL,
+    "engine-log": NEUTRAL,
+}
+
+#: What each provenance MEANS, spelled out. The word is a term of art; the sentence is what a
+#: reader who has not read `timeline.py` can act on -- and on an audit surface, "how do we know
+#: this happened" is the question the whole page exists to answer.
+_PROVENANCE_NOTES: Mapping[str, str] = {
+    "venue-reported": "the venue reported this fill",
+    "simulated": "the paper trader wrote this -- no venue was involved",
+    "imported-ledger": "imported from a venue CSV; nothing verified it on the way in",
+    "human-attested": "a person typed this and signed their name to it",
+    "engine-log": "the agent's own log of what it did",
+}
+
+
+def _timeline_row_payload(row: TimelineRow) -> dict[str, Any]:
+    """One event, placed.
+
+    `provenance` is a `label` and not a bare string BECAUSE it carries a judgement -- `simulated`
+    warns -- and Rule 3 keeps that judgement here rather than letting a client infer it from the
+    word. `kind`, `source` and `reference` are bare: enum words and identifiers with nothing to
+    decide.
+
+    `amount` rides with `amount_kind` for the reason the report holds them together: a fill price
+    and a cash-flow total in one column, with nothing saying which is which, is a column that
+    will be summed by someone.
+    """
+    return {
+        "at": moment(row.ts),
+        "kind": row.kind,
+        "provenance": label(
+            row.provenance,
+            display=_PROVENANCE_NOTES.get(row.provenance, row.provenance),
+            state=_PROVENANCE_STATES.get(row.provenance, UNKNOWN),
+        ),
+        "source": row.source,
+        "reference": row.reference,
+        "product_id": row.product_id,
+        "amount": money(row.amount),
+        "amount_kind": row.amount_kind,
+        "summary": row.summary,
+        # A `label`, so the "we did not check" reading carries a state a client can style rather
+        # than a bare string it might render as though it were a hash.
+        "row_hash": label(row.row_hash, state=UNKNOWN),
+    }
+
+
+def timeline_payload(report: TimelineReport) -> dict[str, Any]:
+    """`gather_timeline`'s `TimelineReport`, as JSON (#703).
+
+    `kinds_present` is the chip bar and comes off the report, built from the SCOPED set -- a bar
+    built from the rows on screen would delete its own alternatives the moment one was chosen.
+    Every count comes off the report too (Rule 6e bans `len()` here).
+    """
+    return {
+        "as_of": iso(report.now_ts),
+        "generated_at": moment(report.now_ts),
+        "scope": report.scope,
+        "scope_start_at": moment(report.scope_start_ts),
+        "kind": report.kind,
+        "kinds": [str(kind) for kind in report.kinds_present],
+        "limit": count(report.limit),
+        "scoped_count": count(report.scoped_count),
+        "filtered_count": count(report.filtered_count),
+        "shown_count": count(report.shown_count),
+        "rows": [_timeline_row_payload(row) for row in report.rows],
+    }
+
+
 # -- the envelope (#534) -------------------------------------------------------------------------
 #
 # Every `GET /api/*` success is wrapped in the same four keys, so #536's single `fetch` wrapper

keel/web/server.py

@@ -390,6 +390,36 @@ def _send(
         if self.command != "HEAD":
             self.wfile.write(payload)
 
+    def _send_csv(self, text: str, filename: str) -> None:
+        """One CSV export, with its own headers (#703).
+
+        `nosniff` matters MORE here than on the JSON routes, not less: this body is a file a
+        browser is being told to save, and a sniffing browser that decided some other type for it
+        would be deciding what a downloaded file IS.
+
+        `Content-Disposition: attachment` is the second half of that. Without it a browser may
+        render the CSV inline, and an inline-rendered document from this origin is a different
+        security question from a saved file -- `attachment` keeps it a download, and names it so
+        the operator has a dated artefact rather than `export.csv` among ten others.
+
+        `no-store` for the same reason every API response carries it: this is the operator's
+        whole audit trail, and a copy of it in a shared cache is a copy nobody chose to make.
+
+        The filename is server-generated and never echoed from the query string -- a
+        caller-supplied one would put attacker-controlled text into a response header, which is
+        the header-injection version of the formula injection `csv_safe` already defends the body
+        against.
+        """
+        body = text.encode("utf-8")
+        self.send_response(200)
+        self.send_header("Content-Type", "text/csv; charset=utf-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Content-Disposition", f'attachment; filename="{filename}"')
+        for name, value in _API_HEADERS:
+            self.send_header(name, value)
+        self.end_headers()
+        self.wfile.write(body)
+
     def _send_json(self, code: int, document: dict[str, Any]) -> None:
         """One JSON response, with its own headers.
 
@@ -839,6 +869,14 @@ def do_GET(self) -> None:  # noqa: N802 - stdlib's naming, not ours
             # not exempt from the loopback-plus-session model for being machine-readable. What it
             # does NOT additionally require is `X-Keel-Client` -- that header gates POSTs, and its
             # docstring explains why a GET is not the gap it closes.
+            # #703's CSV export is the one path under `/api/` that does not answer JSON. It is
+            # checked HERE, inside the same admission the JSON routes passed, so it inherits the
+            # loopback-plus-session model unchanged -- an export of the whole audit trail is the
+            # last thing that should be reachable more easily than the page it came from.
+            if parsed.path == api.CSV_EXPORT_PATH:
+                text, filename = api.export_timeline_csv(self.cfg, query)
+                self._send_csv(text, filename)
+                return
             code, document = api.respond(self.cfg, parsed.path, query)
             self._send_json(code, document)
             return

keel/web/static/index.html

@@ -120,6 +120,7 @@
       <li><a href="/orders">Orders</a></li>
       <li><a href="/positions">Positions</a></li>
       <li><a href="/balances">Balances</a></li>
+      <li><a href="/timeline">Timeline</a></li>
       <li><a href="/insights">Insights</a></li>
       <li><a href="/rules">Rules</a></li>
       <li><a href="/venues">Venues</a></li>

keel/web/static/js/main.js

@@ -46,6 +46,7 @@ import {
   modeBadge,
   ordersView,
   balancesView,
+  timelineView,
   positionsView,
   refusedView,
   rulesView,
@@ -101,6 +102,7 @@ const ROUTES = [
   { name: "orders", label: "Orders", endpoints: ["orders"] },
   { name: "positions", label: "Positions", endpoints: ["positions"] },
   { name: "balances", label: "Balances", endpoints: ["balances"] },
+  { name: "timeline", label: "Timeline", endpoints: ["timeline"] },
   { name: "insights", label: "Insights", endpoints: ["insights", "journal"] },
   { name: "rules", label: "Rules", endpoints: ["rules"] },
   { name: "venues", label: "Venues", endpoints: ["venues"] },
@@ -430,6 +432,15 @@ function mount(route, readings) {
   }
   if (route.name === "positions") return positionsView(data, primary.sort, onSort);
   if (route.name === "balances") return balancesView(data, primary.sort, onSort);
+  if (route.name === "timeline") {
+    return timelineView(data, primary.sort, onSort, (kind) => {
+      // #703: the chip re-asks the SERVER, like the Orders status tabs. Filtering the
+      // capped page here would filter the rows that happened to arrive and present the
+      // result as "every flow this month".
+      paramsFor(route.endpoints[0]).kind = kind;
+      void paint(route, true, true);
+    });
+  }
   if (route.name === "rules") return rulesView(data, primary.sort, onSort);
   if (route.name === "venues") return venuesView(data, primary.sort, onSort);
   if (route.name === "gates") return gatesView(data);