perf(prover): per-table scheduler with VRAM admission for multi_prove

.github/workflows/pr_main.yaml

@@ -406,6 +406,15 @@ jobs:
           name: prover-tests
 
       - name: Run prover tests (shard ${{ matrix.partition }}/4)
+        # Shard 1 only: force k > 1 so the per-table admission scheduler really
+        # runs several table closures concurrently. ubuntu-latest has 2-4 vCPU
+        # and table_parallelism() defaults to (cores / 3).max(1), so every
+        # other shard proves with a single driver thread and never exercises
+        # the concurrent path or VramGate's blocking path on a PR. The other
+        # three shards keep the default-k coverage. An empty value on those
+        # fails to parse and falls back to the default, so this is inert there.
+        env:
+          TABLE_PARALLELISM: ${{ matrix.partition == 1 && '6' || '' }}
         run: |
           cargo nextest run \
             --archive-file prover-tests.tar.zst \

crypto/math-cuda/src/device.rs

@@ -509,6 +509,12 @@ impl Backend {
 
     /// Map `rayon::current_thread_index()` to a slot index, with a defensive
     /// clamp in case the rayon pool grew past the Vec we sized at init.
+    ///
+    /// The per-table scheduler's driver threads are not rayon workers: they
+    /// all resolve to slot 0 and deliberately share one slab. Spreading them
+    /// over per-driver slots costs more in repeated pinned allocation than
+    /// the shared mutex does — the staged transfers are already hidden by
+    /// cross-table overlap.
     fn worker_slot(&self, len: usize) -> usize {
         let idx = rayon::current_thread_index().unwrap_or(0);
         // Should be unreachable with rayon's fixed default pool, but if a

crypto/stark/src/instruments.rs

@@ -4,14 +4,26 @@ use std::sync::OnceLock;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
-// Wall clock span timeline: the trustworthy per step latency breakdown.
+// Wall clock span timeline: the per step latency breakdown.
 //
-// Spans open and close on the main thread at phase boundaries. They do not
-// overlap and sum to their parent, so the tree is a true latency breakdown
-// (unlike the accum_* thread local sub timers below, which sum per worker CPU
-// time across rayon threads and can exceed 100%). A parallel region is one span
-// around the blocking call; its internal split is reported separately as CPU
-// time, never mixed into the wall tree.
+// Phase spans open and close on the thread that drives the phase, at phase
+// boundaries. Those are a true latency breakdown: they do not overlap and they
+// sum to their parent, unlike the accum_* thread local sub timers below, which
+// sum per worker CPU time across rayon threads and can exceed 100%. A parallel
+// region is one span around the blocking call; its internal split is reported
+// separately as CPU time, never mixed into the wall tree.
+//
+// Two properties of the recorded data are easy to misread:
+//
+//  - Spans are ALSO opened on worker threads, not only on the main thread —
+//    the per table drivers in `multi_prove` (`*_table` labels) and the
+//    per stage workers in `continuation.rs`. `SPAN_DEPTH` is thread local and
+//    a fresh thread starts at 0, so those records carry depth 0 and their
+//    siblings overlap in wall time. Read them as per instance wall time.
+//  - `scripts/profiling/phase_table.py` SUMS spans that share a label, so a
+//    label used once per table reports the sum over all tables, which can
+//    exceed the enclosing phase's wall clock by up to `table_parallelism()`.
+//    Give a per instance span its own label; never reuse a phase label for it.
 //
 //     let _s = instruments::span("trace_build");   // RAII, stops on drop
 //
@@ -262,9 +274,13 @@ pub struct Round1SubOps {
 /// Timing data collected inside `multi_prove`.
 pub struct MultiProveTiming {
     pub prepass: Duration,
+    /// Round 1 main trace commits. The last phase-wide barrier — every main
+    /// root must be absorbed before the shared LogUp challenges are sampled.
     pub main_commits: Duration,
-    pub aux_build: Duration,
-    pub aux_commit: Duration,
+    /// Wall clock of the fused per-table region: aux build, aux commit and
+    /// rounds 2-4, which run as one task per table across `table_parallelism()`
+    /// drivers. There is no phase-level wall for the aux stages on their own
+    /// any more; their CPU time shows up in `round1_sub`.
     pub rounds_2_4: Duration,
     /// Sub-op breakdown for Round 1 (main + aux LDE vs Merkle).
     pub round1_sub: Round1SubOps,