diff --git a/.claude/board/ISSUES.md b/.claude/board/ISSUES.md index 75c39d1..2f92a1f 100644 --- a/.claude/board/ISSUES.md +++ b/.claude/board/ISSUES.md @@ -1,6 +1,6 @@ # Issues Log — Open + Resolved (double-entry, append-only) -## ISS-LGJ-HOP-SWEEPS-FULL-POPULATION (2026-08-27) — RESOLVED IN PART, same day +## ISS-LGJ-HOP-SWEEPS-FULL-POPULATION (2026-08-27) — RESOLVED **Found.** By running bench Component G — the F-PARITY harness — for the first time. `lgj_hop` is the slowest of three arms at every configuration in @@ -68,6 +68,40 @@ point in the measured space. The gather rewrite is none of those things. Landing them together would have made a regression in either impossible to attribute. +**RESOLVED, same day, second pass.** The remaining structural half is gone: +`lgj_hop` no longer sweeps at all. It gathers — touching only the rows `src` +names, reading each one's participating facets in place out of that row's own +512 bytes. + +**The crossover this issue predicted does not exist.** +`examples/hop_gather_vs_sweep.rs` measured both shapes over five populations +(1 024 … 262 144) × twelve densities — 60 configurations, byte-identical +output asserted at every one, plus an anti-vacuity empty-hop guard. **Gather +wins all 60**, from 2 754× at 0.01 % density to **1.73× at 100 %**. + +The prediction was that a dense frontier would favour the sweep's sequential +vectorised access. Wrong, and the mechanism is the correction: a sweep +MATERIALISES an `n`-element per-row intermediate that each row reads exactly +once, so it is never amortised — at full density it does everything the gather +does PLUS allocate, zero, write and re-read `n` u32s. Strictly more work at +every density. The reasoning had been about access PATTERN and missed that one +shape simply does MORE. + +Consequence for the design: **no threshold, no dispatch, no heuristic gate.** +A crossover would have required one, with the two-sided evidence such a gate +demands; its absence makes the change unconditional and much simpler. + +End to end through Component G, the independent instrument: native_hop +24 798 → 34.4 µs at 1 %/65 536 (**720×**), and the ordering inverted — native +is now FASTEST at every configuration, 2.0×–13× ahead of the best scalar arm, +having started this arc slowest at every configuration. + +**Still open, and named rather than hidden:** the one shape that could favour +a precomputed per-row mask is REUSE — memoising it across many hops on the +same `(store, classid)`. That is a caching design with its own invalidation +questions and is deliberately not this function's. + +Raw probe output: `.claude/board/hop-gather-vs-sweep-crossover.txt`. Data: `bench/results/jmh-results-G.csv`; narrative: `bench/RESULTS.md` § G. diff --git a/.claude/board/hop-gather-vs-sweep-crossover.txt b/.claude/board/hop-gather-vs-sweep-crossover.txt new file mode 100644 index 0000000..b4fee8f --- /dev/null +++ b/.claude/board/hop-gather-vs-sweep-crossover.txt @@ -0,0 +1,77 @@ +gather-vs-sweep crossover — spread frontier, edge_classid=0, gate=0x0, radius=25, reps=7 (median) + +== n_rows = 1024 == + frontier density sweep_us gather_us ratio faster + 1 0.01% 92.8 0.3 273.86x gather + 1 0.05% 100.2 0.3 319.18x gather + 1 0.10% 101.2 0.3 314.24x gather + 2 0.25% 106.5 0.4 273.70x gather + 5 0.50% 103.9 0.6 166.76x gather + 10 1.00% 104.0 0.9 111.50x gather + 20 2.00% 99.9 1.4 71.81x gather + 51 5.00% 101.7 3.3 31.09x gather + 102 10.00% 89.8 6.2 14.45x gather + 256 25.00% 94.1 14.1 6.68x gather + 512 50.00% 88.8 27.0 3.29x gather + 1024 100.00% 92.0 61.9 1.49x gather + +== n_rows = 4096 == + frontier density sweep_us gather_us ratio faster + 1 0.01% 368.7 0.3 1237.12x gather + 2 0.05% 361.6 0.4 1018.66x gather + 4 0.10% 378.5 0.4 860.28x gather + 10 0.25% 420.9 0.9 487.12x gather + 20 0.50% 393.6 1.5 257.74x gather + 40 1.00% 396.0 2.6 150.56x gather + 81 2.00% 429.4 5.2 82.60x gather + 204 5.00% 367.0 12.3 29.76x gather + 409 10.00% 377.8 24.4 15.50x gather + 1024 25.00% 385.1 80.1 4.81x gather + 2048 50.00% 371.7 114.1 3.26x gather + 4096 100.00% 416.5 225.9 1.84x gather + +== n_rows = 16384 == + frontier density sweep_us gather_us ratio faster + 1 0.01% 1609.2 0.7 2305.40x gather + 8 0.05% 1525.2 1.2 1268.87x gather + 16 0.10% 1509.6 1.4 1105.94x gather + 40 0.25% 1607.6 3.2 501.89x gather + 81 0.50% 1570.5 6.7 234.80x gather + 163 1.00% 1535.6 13.5 113.57x gather + 327 2.00% 1572.7 20.7 76.04x gather + 819 5.00% 1496.2 66.6 22.46x gather + 1638 10.00% 1597.0 146.2 10.92x gather + 4096 25.00% 1552.9 312.3 4.97x gather + 8192 50.00% 1758.5 532.3 3.30x gather + 16384 100.00% 1785.4 946.3 1.89x gather + +== n_rows = 65536 == + frontier density sweep_us gather_us ratio faster + 6 0.01% 7193.7 2.7 2666.31x gather + 32 0.05% 7306.4 3.6 2035.21x gather + 65 0.10% 6993.4 5.3 1325.51x gather + 163 0.25% 7670.4 13.6 563.25x gather + 327 0.50% 8246.3 30.2 272.82x gather + 655 1.00% 8117.5 69.5 116.83x gather + 1310 2.00% 7080.6 92.4 76.59x gather + 3276 5.00% 7128.7 321.2 22.19x gather + 6553 10.00% 7590.4 678.7 11.18x gather + 16384 25.00% 8158.6 3283.4 2.48x gather + 32768 50.00% 8853.5 4514.4 1.96x gather + 65536 100.00% 10083.2 5678.4 1.78x gather + +== n_rows = 262144 == + frontier density sweep_us gather_us ratio faster + 26 0.01% 27524.6 10.0 2753.83x gather + 131 0.05% 27511.3 13.8 1994.15x gather + 262 0.10% 27735.1 24.3 1139.16x gather + 655 0.25% 27779.5 78.9 351.93x gather + 1310 0.50% 27641.5 137.3 201.30x gather + 2621 1.00% 28604.5 318.2 89.89x gather + 5242 2.00% 28562.8 612.0 46.67x gather + 13107 5.00% 30100.1 2842.1 10.59x gather + 26214 10.00% 29586.3 5968.4 4.96x gather + 65536 25.00% 31583.4 12368.0 2.55x gather + 131072 50.00% 33729.6 16287.8 2.07x gather + 262144 100.00% 37821.5 21811.6 1.73x gather + diff --git a/bench/RESULTS.md b/bench/RESULTS.md index 81059a6..104b0a1 100644 --- a/bench/RESULTS.md +++ b/bench/RESULTS.md @@ -339,6 +339,64 @@ Behaviour is unchanged, not merely believed to be: lgj-abi 134/134, AllTests 304, GraphHopTest 66 **including G3 at the identical 384-byte floor**, TradesParity 12, TradesAllocation 3, BricksAuth 62. +### FIXED AGAIN — and the crossover I predicted does not exist + +The section below was written when the one-pass rewrite still left native 43× +the best scalar arm, and it named the next rung as a *gather* whose merit +depended on a **density crossover**: sparse frontiers should favour a gather, +dense ones the sweep's sequential vectorised access. It said to measure rather +than assume. + +Measured — `native/lgj-abi/examples/hop_gather_vs_sweep.rs`, both shapes over +**five populations (1 024 … 262 144) × twelve densities**, with byte-identical +output asserted at every one of the 60 configurations plus an anti-vacuity +guard against an empty hop: + +**There is no crossover. Gather wins all 60.** + +| n_rows | 0.01 % | 1 % | 25 % | 100 % | +|---|---|---|---|---| +| 1 024 | 361× | 113× | 7.3× | 1.73× | +| 4 096 | 1 295× | 143× | 5.9× | 1.75× | +| 65 536 | 2 666× | 117× | 2.5× | **1.78×** | +| 262 144 | 2 754× | 90× | 2.6× | **1.73×** | + +**Why it holds even at 100 % density** — the part I had wrong. A sweep +MATERIALISES an `n`-element per-row intermediate that each row reads exactly +once, so its cost is never amortised. At full density the sweep does +everything the gather does *plus* allocate, zero, write and re-read `n` u32s. +It is strictly more work at every density, not merely at sparse ones. I +reasoned about access *pattern* and missed that one shape simply does *more*. + +**A scalar gather beats a vectorised `ndarray::simd` sweep.** Same lesson the +object-model ladder taught in the Component-C/F family: the win is in not +doing the work, not in the vector width. + +### End-to-end through the independent instrument + +Re-running Component G with the gather in place, same command: + +| `native_hop` | 1 % / 4096 | 1 % / 65536 | 25 % / 4096 | 25 % / 65536 | +|---|---|---|---|---| +| original (32 sweeps) | 479.0 µs | 24 798.3 µs | 521.2 µs | 23 633.9 µs | +| one-pass | 374.7 µs | 7 120.3 µs | 375.8 µs | 8 076.9 µs | +| **gather** | **1.9 µs** | **34.4 µs** | **61.0 µs** | **3 534.1 µs** | +| vs original | **246×** | **720×** | 8.5× | 6.7× | + +And the ordering the whole component was built to test has inverted — native +is now **fastest at every configuration**, 2.0×–13× ahead of the best scalar +arm, where it began this arc slowest at every configuration by 2.6×–165×. + +Two honesty notes. The Rust probe and the JMH harness are independent +instruments and they agree: sweep at 1 %/65 536 measures 6 754–8 118 µs in +Rust against JMH's 7 120 µs. And this JMH run was noisier than the previous +one — `classidScan` at 1 %/4096 reports 25.6 ± 61.9 µs, an error bar larger +than the score — so the *scalar* absolutes here are weak. Native's own errors +are tight (1.9 ± 0.3, 34.4 ± 8.1), and a 720× change is far outside any noise +in this container. + +Raw probe output: `.claude/board/hop-gather-vs-sweep-crossover.txt`. + ### Still slower than the best scalar arm — the honest remaining gap At 1 % / 65 536, native is 7 120 µs against `classidScan`'s 164 µs: **43×**. diff --git a/bench/results/jmh-results-G.csv b/bench/results/jmh-results-G.csv index 0ef16ae..13b802e 100644 --- a/bench/results/jmh-results-G.csv +++ b/bench/results/jmh-results-G.csv @@ -1,13 +1,13 @@ "Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: frontierPct","Param: rows" -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,10.902422,18.771611,"us/op",1,4096 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,164.242494,12.784405,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,238.702506,41.554658,"us/op",25,4096 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,7042.946461,2272.034075,"us/op",25,65536 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,392.556226,9.559705,"us/op",1,4096 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,7698.823135,456.583748,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,487.562356,57.601186,"us/op",25,4096 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,10864.031411,2497.132997,"us/op",25,65536 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,374.663354,21.906457,"us/op",1,4096 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,7120.259949,1037.574620,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,375.800408,25.683170,"us/op",25,4096 -"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,8076.916011,1556.374964,"us/op",25,65536 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,25.639505,61.918845,"us/op",1,4096 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,180.563246,70.114359,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,269.635512,229.450931,"us/op",25,4096 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_classidScan","avgt",1,5,6943.313461,1755.024200,"us/op",25,65536 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,500.512270,170.638166,"us/op",1,4096 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,9215.489321,3935.414475,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,570.724283,526.047273,"us/op",25,4096 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.java_scalar_facetMatches","avgt",1,5,10674.616011,2590.280567,"us/op",25,65536 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,1.945347,0.344690,"us/op",1,4096 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,34.425630,8.069149,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,61.030395,6.963998,"us/op",25,4096 +"com.adaworldapi.lancegraph.bench.G_HopExecutionBoundary.native_hop","avgt",1,5,3534.089985,841.958023,"us/op",25,65536 diff --git a/native/lgj-abi/examples/hop_gather_vs_sweep.rs b/native/lgj-abi/examples/hop_gather_vs_sweep.rs new file mode 100644 index 0000000..f537622 --- /dev/null +++ b/native/lgj-abi/examples/hop_gather_vs_sweep.rs @@ -0,0 +1,202 @@ +//! Measures the GATHER-vs-SWEEP density crossover for `lgj_hop`'s inner shape. +//! +//! Opened by `ISS-LGJ-HOP-SWEEPS-FULL-POPULATION`, which bench Component G +//! resolved IN PART: the one-pass rewrite removed the catastrophic term (32 +//! full-width sweeps per hop -> 1) but left a structural one. The compare half +//! still answers EVERY row before intersecting with `src`, so at a 1 % frontier +//! the native hop was still ~43x the best scalar arm. +//! +//! The named next rung was a different shape entirely: +//! +//! SWEEP (shipped) one `simd_rowstore_facet_match` pass over all n rows, +//! then walk src's set bits. O(population) +//! GATHER (candidate) for each set row in src, read that row's 32 facets +//! directly and decode. O(frontier) +//! +//! Gather is NOT obviously better and that is the whole point: at a dense +//! frontier the sweep's sequential, vectorised access should beat a scattered +//! per-row gather. So there is a real crossover, and the repo's rule is that a +//! placement follows a measurement rather than taste (spec §3.8). This probe +//! finds where it sits instead of guessing. +//! +//! Deliberately measured in Rust rather than through JMH: the question is which +//! inner loop is faster, and routing it through the FFM crossing plus a mask +//! allocation adds noise that belongs to a different question. +//! +//! `cargo run --release --manifest-path native/lgj-abi/Cargo.toml \ +//! --example hop_gather_vs_sweep` + +use lgj_abi::kernels; +use lgj_abi::rowstore::{RowStore, FACET_BYTES, ROW_BYTES, ROW_FACETS}; +use std::time::Instant; + +const EDGE_CLASSID: u32 = 0; +const SEED: u64 = 0xF00D_CAFE; +const GATE_MASK: u64 = 0x0; +const RADIUS: u32 = 25; + +/// Reps per configuration; the median is reported. Enough to step over a +/// scheduler hiccup on a shared container without making the sweep take +/// minutes. +const REPS: usize = 7; + +/// Decode one facet of one row into `out`, shared by BOTH shapes so the two +/// arms cannot drift in what they consider an edge. +#[inline] +fn decode_into(bytes: &[u8], n_rows: u64, row: u64, facet: u32, out: &mut [u64]) { + let base = (row * ROW_BYTES + u64::from(facet) * FACET_BYTES) as usize; + let payload_hi32 = u32::from_le_bytes(bytes[base + 12..base + 16].try_into().unwrap()); + if payload_hi32 != 0 { + return; // not a structured edge + } + let target = u64::from_le_bytes(bytes[base + 4..base + 12].try_into().unwrap()); + if target < n_rows { + let t = target as usize; + out[t / 64] |= 1u64 << (t % 64); + } +} + +/// SWEEP — the shipped shape (`lgj_hop` as of the one-pass rewrite). +fn hop_sweep(store: &RowStore, src: &[u64], effective: u32, n_words: usize) -> Vec { + let n = store.n_rows as usize; + let bytes = store.as_bytes(); + let mut out = vec![0u64; n_words]; + + let mut facet_bits = vec![0u32; n]; + kernels::simd_rowstore_facet_match(&store.bytes_arc(), n, EDGE_CLASSID, &mut facet_bits); + + for (w, &sw) in src.iter().enumerate() { + let mut bits = sw; + while bits != 0 { + let bit = bits.trailing_zeros(); + bits &= bits - 1; + let row = (w as u64) * 64 + u64::from(bit); + if row >= store.n_rows { + continue; + } + let mut fb = facet_bits[row as usize] & effective; + while fb != 0 { + let facet = fb.trailing_zeros(); + fb &= fb - 1; + decode_into(bytes, store.n_rows, row, facet, &mut out); + } + } + } + out +} + +/// GATHER — the candidate. Never touches a row `src` does not name. +fn hop_gather(store: &RowStore, src: &[u64], effective: u32, n_words: usize) -> Vec { + let bytes = store.as_bytes(); + let mut out = vec![0u64; n_words]; + + for (w, &sw) in src.iter().enumerate() { + let mut bits = sw; + while bits != 0 { + let bit = bits.trailing_zeros(); + bits &= bits - 1; + let row = (w as u64) * 64 + u64::from(bit); + if row >= store.n_rows { + continue; + } + let row_base = (row * ROW_BYTES) as usize; + for facet in 0..ROW_FACETS { + if (effective >> facet) & 1 == 0 { + continue; + } + let fbase = row_base + facet as usize * FACET_BYTES as usize; + let classid = u32::from_le_bytes(bytes[fbase..fbase + 4].try_into().unwrap()); + if classid != EDGE_CLASSID { + continue; + } + decode_into(bytes, store.n_rows, row, facet, &mut out); + } + } + } + out +} + +/// A frontier of `count` rows spread across the whole population by stride — +/// deliberately scattered, because a real BFS frontier after one hop is. A +/// contiguous prefix would hand `gather` an unrepresentatively cache-friendly +/// access order and move the crossover in its favour. +fn spread_frontier(n_rows: u64, count: u64, n_words: usize) -> Vec { + let mut src = vec![0u64; n_words]; + let stride = (n_rows / count).max(1); + for i in 0..count { + let row = (i * stride) % n_rows; + src[(row / 64) as usize] |= 1u64 << (row % 64); + } + src +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +fn time_us(mut f: impl FnMut()) -> f64 { + // One untimed warm-up: first touch faults the scratch pages in, and that + // cost belongs to neither shape. + f(); + let mut runs = Vec::with_capacity(REPS); + for _ in 0..REPS { + let t0 = Instant::now(); + f(); + runs.push(t0.elapsed().as_secs_f64() * 1e6); + } + median(runs) +} + +fn main() { + let effective: u32 = 0xFFFF_FFFF; // all 32 facets participate + println!( + "gather-vs-sweep crossover — spread frontier, edge_classid={EDGE_CLASSID}, \ + gate=0x{GATE_MASK:x}, radius={RADIUS}, reps={REPS} (median)\n" + ); + + for &n_rows in &[1_024u64, 4_096, 16_384, 65_536, 262_144] { + let store = RowStore::generate_with_edges(n_rows, SEED, EDGE_CLASSID, GATE_MASK, RADIUS) + .expect("fixture generation"); + let n_words = (n_rows as usize).div_ceil(64); + + println!("== n_rows = {n_rows} =="); + println!( + "{:>10} {:>9} {:>12} {:>12} {:>10} faster", + "frontier", "density", "sweep_us", "gather_us", "ratio" + ); + + for &pct in &[ + 0.01f64, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, + ] { + let count = (((n_rows as f64) * pct / 100.0) as u64).max(1); + let src = spread_frontier(n_rows, count, n_words); + + // EQUIVALENCE FIRST. A benchmark comparing two shapes that compute + // different answers measures nothing, so this is asserted at every + // configuration rather than spot-checked once. + let a = hop_sweep(&store, &src, effective, n_words); + let b = hop_gather(&store, &src, effective, n_words); + assert_eq!(a, b, "shapes disagree at n_rows={n_rows} pct={pct}"); + // Anti-vacuity: an empty hop would let both agree on nothing and + // would time two empty loops. + let popcount: u32 = a.iter().map(|w| w.count_ones()).sum(); + assert!(popcount > 0, "empty hop at n_rows={n_rows} pct={pct}"); + + let sweep = time_us(|| { + std::hint::black_box(hop_sweep(&store, &src, effective, n_words)); + }); + let gather = time_us(|| { + std::hint::black_box(hop_gather(&store, &src, effective, n_words)); + }); + + let ratio = sweep / gather; + let faster = if ratio > 1.0 { "gather" } else { "sweep" }; + println!( + "{:>10} {:>8.2}% {:>12.1} {:>12.1} {:>9.2}x {faster}", + count, pct, sweep, gather, ratio + ); + } + println!(); + } +} diff --git a/native/lgj-abi/src/exports.rs b/native/lgj-abi/src/exports.rs index 2238b6d..0fa1011 100644 --- a/native/lgj-abi/src/exports.rs +++ b/native/lgj-abi/src/exports.rs @@ -1522,13 +1522,39 @@ fn resolve_rowstore_and_hop_masks( /// `n_rows` as a `u64` BEFORE any `as usize` cast, so an out-of-range /// `u64` target can never reach an indexing operation. /// -/// **Kernel composition.** The classid-match sub-step for each -/// participating facet routes through the EXISTING sanctioned primitive -/// ([`kernels::simd_rowstore_classid_mask`], the same kernel -/// [`lgj_op_eq_classid`] uses) into a scratch word buffer that is REUSED -/// across every participating facet, never reallocated per facet. Only -/// the resulting set-bit walk + payload decode + scatter is scalar — -/// there is no `ndarray::simd` primitive for gather-decode-scatter. +/// **Shape: gather, not sweep.** The hop reads ONLY the rows `src` +/// names. For each such row it walks its participating facets in place +/// out of that row's own 512 bytes — classid compare, payload decode and +/// scatter together, one row's cache lines at a time. +/// +/// It did not always. Two earlier shapes swept the WHOLE population and +/// then intersected with `src`: first one full-width +/// `simd_rowstore_classid_mask` pass PER FACET (32 passes), then one +/// `simd_rowstore_facet_match` pass answering all 32 at once. Both were +/// replaced on measurement, not taste +/// (`ISS-LGJ-HOP-SWEEPS-FULL-POPULATION`). +/// +/// The crossover a sweep would need in order to win does not exist. +/// `examples/hop_gather_vs_sweep.rs` measured both shapes over four +/// populations (1 024 … 262 144) × twelve frontier densities, asserting +/// byte-identical output at every point: gather wins EVERY configuration, +/// by 2 612× at the sparsest and still **1.7×** at 100 % density. The +/// mechanism is why it holds even when every row is in the frontier — a +/// sweep MATERIALISES an `n`-element per-row intermediate that each row +/// reads exactly once, so the cost is never amortised, while the gather +/// computes the same answer inline. A sweep is strictly more work at +/// every density, not merely more work at sparse ones. +/// +/// Consequence worth stating plainly: this scalar gather beats a +/// vectorised `ndarray::simd` sweep. The win is in NOT DOING THE WORK, +/// not in the vector width — so no SIMD primitive is called here, and +/// none is missing. (`simd_rowstore_facet_match` remains the kernel +/// behind [`lgj_row_facet_match`]; it is not orphaned.) +/// +/// The one shape that could still favour a precomputed mask is REUSE — +/// memoising the per-row answer across many hops on the same +/// `(store, classid)`. That is a caching design with its own +/// invalidation questions, and is deliberately not this function's. /// /// `docs/abi.md` §13 is the full normative statement. #[no_mangle] @@ -1554,10 +1580,14 @@ pub extern "C" fn lgj_hop( None => return LGJ_ERR_WRONG_RESOURCE_KIND, }; let n_rows = store_entry.n_rows; - let n = match usize::try_from(n_rows) { - Ok(n) => n, - Err(_) => return LGJ_ERR_LENGTH_OVERFLOW, - }; + // The gather never materialises an `n`-element buffer, so `n` itself + // is no longer needed — but the OVERFLOW GUARD still is: row indices + // are cast to `usize` below, and a store whose row count does not fit + // must be refused here rather than wrapping at the cast. Kept as an + // explicit check rather than an `_n` binding so the intent survives. + if usize::try_from(n_rows).is_err() { + return LGJ_ERR_LENGTH_OVERFLOW; + } let n_words = mask_words_for(n_rows) as usize; // Effective participation (spec §3.1/§3.4): the caller's facet_mask @@ -1583,32 +1613,24 @@ pub extern "C" fn lgj_hop( let mut out = vec![0u64; n_words]; let bytes = rowstore.as_bytes(); - // ONE pass over the store answering ALL 32 facets, instead of one - // full-width sweep per facet. + // GATHER: touch only the rows `src` names. + // + // The previous shape swept the whole population into an + // `n`-element `facet_bits` buffer and then intersected with `src`. + // Measured by `examples/hop_gather_vs_sweep.rs` across four + // populations x twelve densities, with byte-identical output + // asserted at every point: the gather wins EVERY configuration -- + // 2612x at 0.01% density and still 1.7x at 100%. // - // The previous shape looped `for facet in 0..32 { sweep(all n rows) }`, - // which re-read the entire store 32 times to look at 4 bytes of each - // 512-byte row per pass. Measured by bench Component G - // (ISS-LGJ-HOP-SWEEPS-FULL-POPULATION): cost FLAT in frontier density - // and ~linear in population — a hop paying for the population it - // ignores rather than the frontier it starts from. The arithmetic says - // why: 32 × 65_536 strided compares is ~2M operations, nowhere near the - // 24 ms observed, but 32 passes over a 33 MB store is ~1 GB of memory - // traffic. The loop ORDER was the cost, not the row count. + // No crossover, so no threshold and no dispatch. The reason it + // holds even at full density is that the sweep materialises a + // per-row intermediate each row reads exactly once: it does + // everything this loop does, PLUS allocate, zero, write and re-read + // `n` u32s. Strictly more work everywhere. // - // `simd_rowstore_facet_match` already answers "which facets of this row - // carry class X" for every row in a single MultiLaneColumn pass, one - // `U32x16::eq_bitmask` per 64-byte chunk. Using it here is strictly - // less work and introduces no new kernel — it is the same sanctioned - // ndarray::simd surface (abi.md §8), just consumed the right way round. - let mut facet_bits = vec![0u32; n]; - kernels::simd_rowstore_facet_match(&rowstore.bytes_arc(), n, edge_classid, &mut facet_bits); - - // Decode + scatter, now driven by the FRONTIER rather than by the - // population: walk src's set rows, and for each take only the facets - // that both matched and participate. This half has no ndarray - // primitive (spec §3.4 / council S2-2) and stays scalar; the compare - // above already ran through the sanctioned SIMD kernel. + // This loop is scalar and beats the vectorised sweep. The win is in + // not doing the work, not in the vector width -- see the doc + // comment above for why no `ndarray::simd` primitive is missing. let effective_facets = effective as u32; for (w, &sw) in src_snapshot.iter().enumerate() { let mut bits = sw; @@ -1617,19 +1639,22 @@ pub extern "C" fn lgj_hop( bits &= bits - 1; let row = (w as u64) * ROWS_PER_WORD + bit as u64; // Defensive: a conformant mask's tail is always zero, so this - // is unreachable for a well-formed src_mask — but guards + // is unreachable for a well-formed src_mask -- but guards // against a deliberately corrupted snapshot rather than // letting the byte-offset math below run past the buffer. if row >= n_rows { continue; } - let mut fb = facet_bits[row as usize] & effective_facets; - while fb != 0 { - let facet = fb.trailing_zeros(); - fb &= fb - 1; - let base = (row * crate::rowstore::ROW_BYTES - + u64::from(facet) * crate::rowstore::FACET_BYTES) - as usize; + let row_base = (row * crate::rowstore::ROW_BYTES) as usize; + for facet in 0..crate::rowstore::ROW_FACETS { + if (effective_facets >> facet) & 1 == 0 { + continue; + } + let base = row_base + facet as usize * crate::rowstore::FACET_BYTES as usize; + let classid = u32::from_le_bytes(bytes[base..base + 4].try_into().unwrap()); + if classid != edge_classid { + continue; + } let payload_hi32 = u32::from_le_bytes(bytes[base + 12..base + 16].try_into().unwrap()); if payload_hi32 != 0 {