From c33003e0ad0d860662b9dc963209dd463c754565 Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 12:25:39 -0400 Subject: [PATCH 01/10] feat(rulake-phase-0): pre-partition bridge + scaffold 7 ELI15 chapter stubs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundations for the RuLake-inspired feature roadmap (docs/plan/rulake-inspired-features.md). Phase 0 is deliberately boring: define shared shapes + cut extension points so the four Phase 1 swarm tasks never fight over ruvectorBridge.js. What this ships: - archive/snapshot.js — ArchiveSnapshot v1 shape + validateSnapshot() - archive/hash.js — xxHash32 over Float32Array brains; 8-char hex ID - ruvectorBridge.js — exportSnapshot / importSnapshot / setConsistencyMode / getConsistencyMode / getIndexStats. Stubs throw 'not implemented' with owning-phase labels; importSnapshot already validates at the boundary so callers can test wiring. - eli15/chapters/*.js — 7 placeholder chapters (warm-restart, quantization, consistency-modes, content-addressing, federation, cross-tab-federation, where-the-time-goes), each with comingSoon: true and a real ELI15 hook. - eli15/index.js — registry entries for the 7 new chapters. What this deliberately does not do: no behaviour change, no new feature. The bridge imports validateSnapshot at module scope but doesn't call it at boot; chapters are lazy-loaded. Validated via agent-browser: app boots cleanly with no new console errors, all 7 chapter IDs returned by ELI15.listChapters(), two chapters rendered end-to-end, stub contracts (5 exports, correct error messages, consistency default = 'fresh'), hash determinism + 1-ppm sensitivity. --- AI-Car-Racer/archive/hash.js | 81 ++++ AI-Car-Racer/archive/snapshot.js | 77 +++ .../eli15/chapters/consistency-modes.js | 25 + .../eli15/chapters/content-addressing.js | 25 + .../eli15/chapters/cross-tab-federation.js | 20 + AI-Car-Racer/eli15/chapters/federation.js | 22 + AI-Car-Racer/eli15/chapters/quantization.js | 23 + AI-Car-Racer/eli15/chapters/warm-restart.js | 22 + .../eli15/chapters/where-the-time-goes.js | 19 + AI-Car-Racer/eli15/index.js | 44 ++ AI-Car-Racer/ruvectorBridge.js | 52 +++ docs/plan/rulake-inspired-features.md | 439 ++++++++++++++++++ 12 files changed, 849 insertions(+) create mode 100644 AI-Car-Racer/archive/hash.js create mode 100644 AI-Car-Racer/archive/snapshot.js create mode 100644 AI-Car-Racer/eli15/chapters/consistency-modes.js create mode 100644 AI-Car-Racer/eli15/chapters/content-addressing.js create mode 100644 AI-Car-Racer/eli15/chapters/cross-tab-federation.js create mode 100644 AI-Car-Racer/eli15/chapters/federation.js create mode 100644 AI-Car-Racer/eli15/chapters/quantization.js create mode 100644 AI-Car-Racer/eli15/chapters/warm-restart.js create mode 100644 AI-Car-Racer/eli15/chapters/where-the-time-goes.js create mode 100644 docs/plan/rulake-inspired-features.md diff --git a/AI-Car-Racer/archive/hash.js b/AI-Car-Racer/archive/hash.js new file mode 100644 index 0000000..045a21a --- /dev/null +++ b/AI-Car-Racer/archive/hash.js @@ -0,0 +1,81 @@ +// archive/hash.js +// Phase 0 — Foundations. xxHash32 over a flattened Float32Array brain, +// returned as an 8-char lowercase hex string. Used as the canonical brain ID +// by F3 (warm-restart bundles), F5 (content-addressed dedup), and F6 +// (cross-tab — hash makes "is this the same brain?" a byte comparison). +// +// Why xxHash32 and not crypto.subtle.digest(): we need to hash on the hot +// path during GA evaluation (potentially thousands per second), and the +// crypto API is async-only. xxHash32 is non-cryptographic but +// collision-resistant enough for a browser archive of ≤10⁵ brains — the +// collision worry at 10⁵ entries in a 2³² space is ~1 in 1000, which we +// detect cheaply by comparing the underlying flat bytes on collision. +// +// Reference impl: https://github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md + +const PRIME32_1 = 0x9e3779b1 | 0; +const PRIME32_2 = 0x85ebca77 | 0; +const PRIME32_3 = 0xc2b2ae3d | 0; +const PRIME32_4 = 0x27d4eb2f | 0; +const PRIME32_5 = 0x165667b1 | 0; + +function rotl32(x, r) { return ((x << r) | (x >>> (32 - r))) | 0; } +function mul32(a, b) { return Math.imul(a, b) | 0; } + +// Hash a Uint8Array into an unsigned 32-bit integer. +export function xxHash32Bytes(bytes, seed = 0) { + const len = bytes.length; + let h32; + let i = 0; + + if (len >= 16) { + let v1 = (seed + PRIME32_1 + PRIME32_2) | 0; + let v2 = (seed + PRIME32_2) | 0; + let v3 = (seed + 0) | 0; + let v4 = (seed - PRIME32_1) | 0; + + while (i + 16 <= len) { + const k1 = bytes[i] | (bytes[i+1] << 8) | (bytes[i+2] << 16) | (bytes[i+3] << 24); + const k2 = bytes[i+4] | (bytes[i+5] << 8) | (bytes[i+6] << 16) | (bytes[i+7] << 24); + const k3 = bytes[i+8] | (bytes[i+9] << 8) | (bytes[i+10] << 16) | (bytes[i+11] << 24); + const k4 = bytes[i+12] | (bytes[i+13] << 8) | (bytes[i+14] << 16) | (bytes[i+15] << 24); + v1 = mul32(rotl32((v1 + mul32(k1, PRIME32_2)) | 0, 13), PRIME32_1); + v2 = mul32(rotl32((v2 + mul32(k2, PRIME32_2)) | 0, 13), PRIME32_1); + v3 = mul32(rotl32((v3 + mul32(k3, PRIME32_2)) | 0, 13), PRIME32_1); + v4 = mul32(rotl32((v4 + mul32(k4, PRIME32_2)) | 0, 13), PRIME32_1); + i += 16; + } + + h32 = (rotl32(v1, 1) + rotl32(v2, 7) + rotl32(v3, 12) + rotl32(v4, 18)) | 0; + } else { + h32 = (seed + PRIME32_5) | 0; + } + + h32 = (h32 + len) | 0; + + while (i + 4 <= len) { + const k = bytes[i] | (bytes[i+1] << 8) | (bytes[i+2] << 16) | (bytes[i+3] << 24); + h32 = mul32(rotl32((h32 + mul32(k, PRIME32_3)) | 0, 17), PRIME32_4); + i += 4; + } + + while (i < len) { + h32 = mul32(rotl32((h32 + mul32(bytes[i], PRIME32_5)) | 0, 11), PRIME32_1); + i++; + } + + h32 ^= h32 >>> 15; + h32 = mul32(h32, PRIME32_2); + h32 ^= h32 >>> 13; + h32 = mul32(h32, PRIME32_3); + h32 ^= h32 >>> 16; + + return h32 >>> 0; +} + +// Hash a Float32Array (a flattened brain) into an 8-char lowercase hex string. +export function hashBrain(flat, seed = 0) { + const bytes = new Uint8Array(flat.buffer, flat.byteOffset, flat.byteLength); + const h = xxHash32Bytes(bytes, seed); + return h.toString(16).padStart(8, '0'); +} diff --git a/AI-Car-Racer/archive/snapshot.js b/AI-Car-Racer/archive/snapshot.js new file mode 100644 index 0000000..69c9b4b --- /dev/null +++ b/AI-Car-Racer/archive/snapshot.js @@ -0,0 +1,77 @@ +// archive/snapshot.js +// Phase 0 — Foundations. Single source of truth for the on-disk shape of a +// VectorVroom archive bundle. Consumed by F3 (warm-restart), F4 (frozen +// consistency mode), F5 (content-addressed dedup), F6 (cross-tab) — each of +// those features is a Phase 1/2 swarm task; this file exists so those tasks +// don't each invent their own shape. +// +// No runtime behaviour yet — this module only defines the shape, a version +// constant, and a validator. Serializer + deserializer land in F3 (1A). + +export const ARCHIVE_SCHEMA_VERSION = 1; + +// Valid values for snapshot.consistency. Mirrors the F4 taxonomy. +export const CONSISTENCY_MODES = ['fresh', 'eventual', 'frozen']; + +// Shape documented here rather than as TypeScript since the project is pure +// JS. Treat this as the contract — adding fields is fine (readers must +// tolerate unknown keys); renaming/removing fields requires a version bump. +// +// ArchiveSnapshot = { +// version: 1, +// createdAt: ISO-8601 string, +// consistency: 'fresh' | 'eventual' | 'frozen', +// brains: [{ +// id: string, // stable hash (see archive/hash.js) +// flat: Float32Array, // FLAT_LENGTH weights +// meta: object, // fitness, generation, trackId, etc. +// parentIds: string[], // lineage edges (hash IDs) +// }], +// hnsw: { +// // Either 'serialized' (preferred once ruvector_wasm exposes it) or +// // 'replay' (fallback: deterministic re-insertion by insertion order). +// mode: 'serialized' | 'replay', +// // When mode === 'serialized': the raw bytes ruvector_wasm emits. +// serialized: Uint8Array | null, +// // When mode === 'replay': the insertion order of brain IDs; the +// // importer re-inserts in this exact order to reproduce the graph. +// insertionOrder: string[] | null, +// params: { dim: number, metric: 'cosine' | 'l2', indexKind: 'euclidean' | 'hyperbolic' }, +// }, +// witness: string, // sha-256 hex of (brains bytes + hnsw bytes); anti-tamper anchor +// } +// +// `witness` is intentionally *not* a cryptographic authentication token — it's +// a self-check so a corrupted download fails loudly at import rather than +// silently at query time. Matches RuLake's "witness chain" idea adapted to +// the browser where we have no trust root to sign against. + +export function emptySnapshot(consistency = 'fresh') { + return { + version: ARCHIVE_SCHEMA_VERSION, + createdAt: new Date().toISOString(), + consistency, + brains: [], + hnsw: { mode: 'replay', serialized: null, insertionOrder: [], params: {} }, + witness: '', + }; +} + +// Cheap structural validator. Returns { ok: true } or { ok: false, reason }. +// Deliberately permissive about unknown keys (forward-compat); strict about +// the fields any reader depends on. +export function validateSnapshot(s) { + if (!s || typeof s !== 'object') return { ok: false, reason: 'not-object' }; + if (s.version !== ARCHIVE_SCHEMA_VERSION) { + return { ok: false, reason: `version-mismatch: got ${s.version}, expected ${ARCHIVE_SCHEMA_VERSION}` }; + } + if (!Array.isArray(s.brains)) return { ok: false, reason: 'brains-not-array' }; + if (!s.hnsw || typeof s.hnsw !== 'object') return { ok: false, reason: 'hnsw-missing' }; + if (s.hnsw.mode !== 'serialized' && s.hnsw.mode !== 'replay') { + return { ok: false, reason: `hnsw.mode invalid: ${s.hnsw.mode}` }; + } + if (!CONSISTENCY_MODES.includes(s.consistency)) { + return { ok: false, reason: `consistency invalid: ${s.consistency}` }; + } + return { ok: true }; +} diff --git a/AI-Car-Racer/eli15/chapters/consistency-modes.js b/AI-Car-Racer/eli15/chapters/consistency-modes.js new file mode 100644 index 0000000..72ecd55 --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/consistency-modes.js @@ -0,0 +1,25 @@ +// eli15/chapters/consistency-modes.js +// Placeholder — real content ships with Phase 1C (F4). +export default { + id: 'consistency-modes', + title: 'Fresh, Eventual, Frozen — three ways training looks at the archive', + oneLiner: 'Should training re-query the archive every generation, periodically, or lock in a snapshot? Each answer is a different mode.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 1C of the RuLake-inspired roadmap.

', + '

Every generation, the training loop asks the archive "who does this', + 'track look like, and which brains worked on those tracks?" But that', + 'question doesn\'t have one right answer.

', + '', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 1C.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/chapters/content-addressing.js b/AI-Car-Racer/eli15/chapters/content-addressing.js new file mode 100644 index 0000000..5816ed8 --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/content-addressing.js @@ -0,0 +1,25 @@ +// eli15/chapters/content-addressing.js +// Placeholder — real content ships with Phase 1D (F5). +export default { + id: 'content-addressing', + title: 'Giving every brain a fingerprint', + oneLiner: 'Two brains with identical weights are the same brain — so ID them by a hash of their contents, not an auto-incrementing number.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 1D of the RuLake-inspired roadmap.

', + '

Genetic algorithms produce a lot of near-duplicate brains: an elite', + 'survives unchanged across generations, siblings share most of their', + 'weights, mutations sometimes come up identical. Today each one gets a', + 'fresh ID and sits as a separate node in the lineage DAG.

', + '

If we instead ID a brain by a hash of its weights,', + 'duplicates collide automatically — importing the same archive twice', + 'becomes a no-op, the family tree stops double-counting, and cross-tab', + 'sharing (F6) becomes conflict-free because every tab agrees on what', + '"the same brain" means.

', + '

We use xxHash32 instead of SHA-256 because this runs on the hot path', + 'during training; cryptographic strength isn\'t needed for a collision-', + 'detection scheme that falls back to a bytes-equal check.

', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 1D.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/chapters/cross-tab-federation.js b/AI-Car-Racer/eli15/chapters/cross-tab-federation.js new file mode 100644 index 0000000..628812a --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/cross-tab-federation.js @@ -0,0 +1,20 @@ +// eli15/chapters/cross-tab-federation.js +// Placeholder — real content ships with Phase 2B (F6). +export default { + id: 'cross-tab-federation', + title: 'Two browser tabs training in sync', + oneLiner: 'Open two tabs on different tracks; each discovers a good brain, the other tab sees it arrive in real time.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 2B of the RuLake-inspired roadmap.

', + '

Once every brain is content-addressed by a hash (F5) and every archive', + 'can be serialized as a snapshot (F3), sharing a brain between two tabs', + 'becomes a one-line broadcast: BroadcastChannel.postMessage({ brain,', + 'hash }). The receiving tab computes the hash, sees it\'s new, inserts.

', + '

No locking, no conflicts — because content-addressing makes the', + 'identity of a brain independent of where it was created. Two tabs', + 'arriving at the same weights produce the same hash and converge for free.

', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 2B.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/chapters/federation.js b/AI-Car-Racer/eli15/chapters/federation.js new file mode 100644 index 0000000..a4ad546 --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/federation.js @@ -0,0 +1,22 @@ +// eli15/chapters/federation.js +// Placeholder — real content ships with Phase 2A (F2). +export default { + id: 'federation', + title: 'Asking two different maps of brain-space at once', + oneLiner: 'The Euclidean and hyperbolic indexes disagree about who counts as a neighbour — so ask both, then let the GNN vote.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 2A of the RuLake-inspired roadmap.

', + '

VectorVroom already has two nearest-neighbour indexes: a flat', + 'Euclidean one (good for geometric track similarity) and a hyperbolic', + 'one (good for lineage-like hierarchical similarity). Today you pick one', + 'at load time via ?hhnsw=1. Federation runs both, unions', + 'their candidates, and lets the GNN reranker pick the final order.

', + '

The clever part is how many candidates to ask each index for: the', + 'formula k\' = k + ⌈√(k · ln S)⌉ (with S = number of shards)', + 'over-requests just enough from each that the true top-k is almost', + 'certainly somewhere in the union.

', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 2A.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/chapters/quantization.js b/AI-Car-Racer/eli15/chapters/quantization.js new file mode 100644 index 0000000..630bfe0 --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/quantization.js @@ -0,0 +1,23 @@ +// eli15/chapters/quantization.js +// Placeholder — real content ships with Phase 1B (F1). +export default { + id: 'quantization', + title: 'Throwing away 31 out of every 32 bits and still finding the right neighbour', + oneLiner: 'A brain\'s fingerprint is hundreds of decimal numbers. RaBitQ keeps only the sign bit of each and barely loses any accuracy.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 1B of the RuLake-inspired roadmap.

', + '

Every brain gets squashed into a vector of ~92 floating-point numbers.', + 'A float is 32 bits. Multiply by 5,000 brains and the archive is megabytes', + 'of RAM. Is all of that precision doing useful work?

', + '

Turns out: no. If you first rotate the vectors with a', + 'trick called a Hadamard transform, then keep only the sign bit', + 'of each component (positive → 1, negative → 0), the Hamming distance', + 'between two bitstrings is a provably unbiased estimate of the angle', + 'between the original vectors. The archive shrinks 32× and recall stays', + 'within 10% of the full-float baseline.

', + '

Same idea as SimHash, applied to HNSW candidates.

', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 1B.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/chapters/warm-restart.js b/AI-Car-Racer/eli15/chapters/warm-restart.js new file mode 100644 index 0000000..a2b3d08 --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/warm-restart.js @@ -0,0 +1,22 @@ +// eli15/chapters/warm-restart.js +// Placeholder — real content ships with Phase 1A (F3). +export default { + id: 'warm-restart', + title: 'Saving and reopening the whole brain archive', + oneLiner: 'A brain archive is a museum — you can save it, reopen it tomorrow, or give the whole museum to a friend.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 1A of the RuLake-inspired roadmap.

', + '

Today the archive rebuilds itself from IndexedDB every page load. Every', + 'brain is re-inserted into HNSW one by one, which is fine at 50 brains', + 'and painful at 5,000. The fix is to save the graph\'s state itself,', + 'not just the raw vectors, and restore it byte-for-byte on the next load —', + 'the same way your laptop reopens yesterday\'s tabs instead of rebuilding', + 'them from scratch.

', + '

Bonus: once the archive is a file, you can share it. Export the', + 'archive, send the file to a friend, and their site will race against your', + 'pre-trained population on the first generation.

', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 1A.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/chapters/where-the-time-goes.js b/AI-Car-Racer/eli15/chapters/where-the-time-goes.js new file mode 100644 index 0000000..c8856da --- /dev/null +++ b/AI-Car-Racer/eli15/chapters/where-the-time-goes.js @@ -0,0 +1,19 @@ +// eli15/chapters/where-the-time-goes.js +// Placeholder — real content ships with Phase 3A (F7). +export default { + id: 'where-the-time-goes', + title: 'Where each generation\'s milliseconds actually go', + oneLiner: 'HNSW traversal, GNN rerank, LoRA adapt, sensor embedding, GA ops — a flame-graph-lite of the whole pipeline.', + comingSoon: true, + body: [ + '

Coming soon — lands with Phase 3A of the RuLake-inspired roadmap.

', + '

Every generation is a lot of work under the hood: retrieve neighbours,', + 'rerank with the GNN, adapt the query vector with LoRA, embed sensor', + 'readings, run the GA. The observability panel breaks each down into a', + 'live stacked bar so you can see which stage is actually expensive —', + 'which is a surprisingly good way to build intuition for how ML pipelines', + 'trade quality for latency.

', + '

Progress: see docs/plan/rulake-inspired-features.md →', + 'Phase 3A.

', + ].join('\n'), +}; diff --git a/AI-Car-Racer/eli15/index.js b/AI-Car-Racer/eli15/index.js index 78557a6..bab3060 100644 --- a/AI-Car-Racer/eli15/index.js +++ b/AI-Car-Racer/eli15/index.js @@ -118,6 +118,50 @@ oneLiner: 'Swap the flat-space neighbour graph for a Poincaré-ball one; trees embed with less distortion.', loader: function () { return import('./chapters/hyperbolic-space.js'); }, }, + // ─── RuLake-inspired roadmap chapters (Phase 0 stubs; real content + // ships phase-by-phase per docs/plan/rulake-inspired-features.md). ─ + 'warm-restart': { + title: 'Saving and reopening the whole brain archive', + oneLiner: 'A brain archive is a museum — you can save it, reopen it tomorrow, or give it to a friend.', + comingSoon: true, + loader: function () { return import('./chapters/warm-restart.js'); }, + }, + 'quantization': { + title: 'Throwing away 31/32 bits and still finding the right neighbour', + oneLiner: 'RaBitQ + Hadamard: shrink the archive 32× without losing recall.', + comingSoon: true, + loader: function () { return import('./chapters/quantization.js'); }, + }, + 'consistency-modes': { + title: 'Fresh / Eventual / Frozen — three ways training looks at the archive', + oneLiner: 'Re-query every generation, periodically, or lock in a snapshot. Three modes, one radio row.', + comingSoon: true, + loader: function () { return import('./chapters/consistency-modes.js'); }, + }, + 'content-addressing': { + title: 'Giving every brain a fingerprint', + oneLiner: 'ID brains by hash of their weights; duplicates collide, the DAG stops double-counting, cross-tab sync becomes free.', + comingSoon: true, + loader: function () { return import('./chapters/content-addressing.js'); }, + }, + 'federation': { + title: 'Asking two different maps of brain-space at once', + oneLiner: 'Query Euclidean + Hyperbolic in parallel; over-request k\' = k + ⌈√(k ln S)⌉; GNN reranks the union.', + comingSoon: true, + loader: function () { return import('./chapters/federation.js'); }, + }, + 'cross-tab-federation': { + title: 'Two browser tabs training in sync', + oneLiner: 'BroadcastChannel + content-addressing = lockless cross-tab archive convergence.', + comingSoon: true, + loader: function () { return import('./chapters/cross-tab-federation.js'); }, + }, + 'where-the-time-goes': { + title: 'Where each generation\'s milliseconds actually go', + oneLiner: 'Per-stage timings: HNSW / rerank / LoRA / sensor embed / GA. Observability as a teaching tool.', + comingSoon: true, + loader: function () { return import('./chapters/where-the-time-goes.js'); }, + }, }; // In-memory chapter body cache: once loaded, reuse on subsequent opens. diff --git a/AI-Car-Racer/ruvectorBridge.js b/AI-Car-Racer/ruvectorBridge.js index b1427ea..5f0e1e7 100644 --- a/AI-Car-Racer/ruvectorBridge.js +++ b/AI-Car-Racer/ruvectorBridge.js @@ -1038,6 +1038,58 @@ export function hydrateFromFixture(fixture) { } catch (e) { console.warn('[lineage-dag] fixture rehydrate failed', e); } } +// ─── Phase 0 extension points (RuLake-inspired features) ───────────────── +// Each stub is a Phase 1 / Phase 2 ownership slot, pre-declared here so the +// swarm can implement in parallel without fighting over this file. Do NOT +// remove the stubs when implementing — replace the body. The signature is +// the contract between features; extending args is fine, renaming is not. +// +// exportSnapshot() — owned by 1A (F3 warm-restart bundles) +// importSnapshot(s) — owned by 1A (F3 warm-restart bundles) +// setConsistencyMode(m) — owned by 1C (F4 consistency modes) +// getIndexStats() — owned by 3A (F7 observability dashboard) +// +// See docs/plan/rulake-inspired-features.md for the full plan. + +import { validateSnapshot, CONSISTENCY_MODES } from './archive/snapshot.js'; + +// Phase 0: stored but unread; 1C will wire this into the query path. +let _consistencyMode = 'fresh'; + +// 1A fills this in. Returns an ArchiveSnapshot (see archive/snapshot.js). +export function exportSnapshot() { + throw new Error('not implemented: exportSnapshot (Phase 1A / F3)'); +} + +// 1A fills this in. Takes an ArchiveSnapshot, reconstructs the in-memory +// indexes + mirrors, returns { ok: true } or throws on validation failure. +export function importSnapshot(s) { + // Validate at the boundary even in the stub, so callers can test the + // wiring without waiting for 1A. Throws on bad input; the real importer + // will keep this check and add its reconstruction work afterwards. + const v = validateSnapshot(s); + if (!v.ok) throw new Error(`importSnapshot: invalid snapshot (${v.reason})`); + throw new Error('not implemented: importSnapshot (Phase 1A / F3)'); +} + +// 1C fills this in. Accepts 'fresh' | 'eventual' | 'frozen'. +export function setConsistencyMode(m) { + if (!CONSISTENCY_MODES.includes(m)) { + throw new Error(`setConsistencyMode: invalid mode ${m}`); + } + _consistencyMode = m; + // 1C will add: persist mode, pin snapshot on 'frozen', TTL reset on 'eventual'. + throw new Error('not implemented: setConsistencyMode (Phase 1C / F4)'); +} + +export function getConsistencyMode() { return _consistencyMode; } + +// 3A fills this in. Returns { hnsw: {...}, rerank: {...}, adapter: {...} } +// with per-stage timings and counters for the "where the time goes" chapter. +export function getIndexStats() { + throw new Error('not implemented: getIndexStats (Phase 3A / F7)'); +} + // Danger-knob: purge everything. Exposed for the verifier + dev console; the // game never calls this. export async function _debugReset() { diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md new file mode 100644 index 0000000..c9ddcf4 --- /dev/null +++ b/docs/plan/rulake-inspired-features.md @@ -0,0 +1,439 @@ +# RuLake-inspired feature roadmap for VectorVroom + +Plan date: 2026-04-24. Author-facing; consumed by `/ship-task` one +phase at a time. Each phase is a shippable unit with a confidence gate +and an ELI15 chapter. Phases are ordered by dependency; inside a phase, +tasks are marked **Parallel-safe** when they can be delegated to +independent subagents, or **Sequential** when they touch a shared +surface (`ruvectorBridge.js`, `eli15/tour.js`, the SONA engine). + +The source of ideas is [ruvnet/RuLake](https://github.com/ruvnet/RuLake) +— a cache layer over vector search. We are **not** adopting RuLake as a +dependency. We are porting a subset of its ideas as user-visible, +teachable features that fit VectorVroom's existing stack (vendored +ruvector WASM + IndexedDB + the ELI15 tour). + +## Status tracker + +**Legend:** ⬜ todo · 🟡 in progress · ✅ done · 🚫 blocked · ⏸ deferred + +**How to update:** when starting a task flip its box to 🟡 and add your +name; when finishing, flip to ✅ and add the date (YYYY-MM-DD) and the +PR/commit SHA. When blocked, flip to 🚫 and add a one-line reason +pointing to the blocker. Keep the "Current focus" line at the top +pointing at whichever phase is active so a newcomer knows where to +jump in without reading the whole doc. + +**Current focus:** Phase 1 — parallel feature implementations (ready to swarm) +**Last updated:** 2026-04-24 + +### Phase 0 — Foundations _(sequential, 1 owner)_ + +| Status | ID | Task | Owner | PR/SHA | Done date | +|:--:|:--:|------|-------|--------|-----------| +| ✅ | 0.1 | `ArchiveSnapshot` schema in `archive/snapshot.js` | Claude | — | 2026-04-24 | +| ✅ | 0.2 | Stub extension points in `ruvectorBridge.js` | Claude | — | 2026-04-24 | +| ✅ | 0.3 | `archive/hash.js` xxHash32 wrapper | Claude | — | 2026-04-24 | +| ✅ | 0.4 | ELI15 chapter stubs (7 new files) | Claude | — | 2026-04-24 | + +**Phase 0 gate:** all four rows ✅ + existing tests pass + `grep +"not implemented"` shows exactly 4 hits. + +### Phase 1 — Parallel implementations _(swarm-safe, 4 subagents)_ + +| Status | ID | Task | Owner | PR/SHA | Done date | +|:--:|:--:|------|-------|--------|-----------| +| ⬜ | 1A | F3 — Warm-restart bundles + shareable snapshots | | | | +| ⬜ | 1B | F1 — 1-bit quantized archive (RaBitQ + Hadamard) | | | | +| ⬜ | 1C | F4 — Consistency modes (Fresh/Eventual/Frozen) | | | | +| ⬜ | 1D | F5 — Content-addressed dedup + hash-keyed lineage | | | | + +**Phase 1 gate:** all four rows ✅ + each passes its own `/ship-task` +confidence gate + cross-track-variance memory applied for recall/speed +claims (n=6+ across ≥2 sessions, tested on both Rect and Tri tracks). + +### Phase 2 — Integration _(sequential)_ + +| Status | ID | Task | Depends on | Owner | PR/SHA | Done date | +|:--:|:--:|------|-----------|-------|--------|-----------| +| ⬜ | 2A | F2 — Federated search with GNN rerank | 1B | | | | +| ⬜ | 2B | F6 — Cross-tab live training via BroadcastChannel | 1A, 1D | | | | + +**Phase 2 gate:** both rows ✅ + A/B convergence test (two-tab demo +for 2B; recall@10 ≥ max(E,H) for 2A). + +### Phase 3 — Observability & polish _(swarm-safe, 3 subagents)_ + +| Status | ID | Task | Owner | PR/SHA | Done date | +|:--:|:--:|------|-------|--------|-----------| +| ⬜ | 3A | F7 — Observability dashboard | | | | +| ⬜ | 3B | ELI15 tour integration pass | | | | +| ⬜ | 3C | Shareable archive URLs + gallery | | | | + +**Phase 3 gate:** all three rows ✅ + ELI15 tour plays end-to-end with +no dead links + 3C passes the external-scope check (user OK before any +community-archive URL ships publicly). + +### Release milestone + +| Status | Milestone | +|:--:|-----------| +| ⬜ | M1 — F1+F3 demoable locally (flags on) | +| ⬜ | M2 — Phase 1 merged to `main` behind flags | +| ⬜ | M3 — F2+F6 shipping, flags default on for F1/F3 | +| ⬜ | M4 — Phase 3 complete, blog post / tour recording | + +### Notes / decisions log + +Append-only. Record any scope change, deferral, or non-obvious call +that future-you would want to find. Newest at the top. + +- **2026-04-24 — Phase 0 closed.** 0.4 shipped 7 stubs (not 6 as the + tracker originally said) — the plan body referenced 7 distinct chapter + files, so the original "6" was a typo. Tracker + task description now + reconciled. `exportSnapshot` / `importSnapshot` / `setConsistencyMode` + / `getIndexStats` are stubs that throw `not implemented` with a Phase + label so swarm workers know which task owns each slot. Browser smoke + test via agent-browser confirmed app still boots cleanly (no new + console errors) and all 7 chapters render via `ELI15.openChapter()`. + SHA column left blank by convention for Phase 0; `git log --grep="Phase + 0"` is authoritative. + +--- + +## Feature slate (the seven we're scoping) + +| ID | Feature | RuLake primitive | Chapter slot | +|----|---------|------------------|--------------| +| F1 | 1-bit quantized archive (RaBitQ + Hadamard) | RabitqPlusIndex | `quantization.js` | +| F2 | Federated search across Euclidean + Hyperbolic | Federation / adaptive rerank | `federation.js` | +| F3 | Warm-restart bundles + shareable snapshots | Save/warm-restart, Frozen | `warm-restart.js` | +| F4 | Consistency modes (Fresh / Eventual / Frozen) | Consistency taxonomy | `consistency-modes.js` | +| F5 | Content-addressed dedup + hash-keyed lineage | Witness chain | `content-addressing.js` | +| F6 | Cross-tab live training via BroadcastChannel | Cross-process sharing | `cross-tab-federation.js` | +| F7 | Observability dashboard | Per-backend observability | `where-the-time-goes.js` | + +## Dependency graph + +``` +Phase 0 (Foundations, sequential) + │ + ├──► Phase 1A — F3 Warm-restart bundles ──┐ + ├──► Phase 1B — F1 Quantized archive ──┤ + ├──► Phase 1C — F4 Consistency modes ──┤── swarm-safe + └──► Phase 1D — F5 Content-addressing ──┘ + │ + ▼ + ┌─────────── Phase 2 (integration) ───────────┐ + │ │ + ▼ ▼ + Phase 2A — F2 Federation Phase 2B — F6 Cross-tab + (needs F1's quantized path + F3 bundle API) (needs F3 snapshot format) + │ + ▼ + Phase 3 (observability + polish, swarm-safe) + 3A F7 dashboard 3B ELI15 tour pass 3C share UI +``` + +--- + +## Phase 0 — Foundations (sequential, single owner) + +One person, one small PR, unblocks everything else. **Do not parallelize.** + +### Tasks + +- **0.1** Define `ArchiveSnapshot` schema in + `AI-Car-Racer/archive/snapshot.js` (new). Single source of truth for the + on-disk shape: + ``` + { version: 1, + brains: [{ hash, flat, meta, parentIds }], + hnsw: { serialized: Uint8Array, params: {...} }, + consistency: 'fresh'|'eventual'|'frozen', + witness: string /* SHA-256 of (brains + hnsw bytes) */ } + ``` +- **0.2** Extend `ruvectorBridge.js` with stubbed extension points (no + behavior yet): `exportSnapshot()`, `importSnapshot(s)`, + `setConsistencyMode(m)`, `getIndexStats()`. Each throws + `Error('not implemented')` so swarm tasks can fill them in parallel + without merge conflicts. +- **0.3** Add `archive/hash.js` — one 8-line xxHash32 wrapper over a + flattened brain. Used by F3 and F5; having it exist up front removes + a cross-task dependency. +- **0.4** Add a new top-level ELI15 chapter stub for each feature — 7 + files (`warm-restart`, `quantization`, `consistency-modes`, + `content-addressing`, `federation`, `cross-tab-federation`, + `where-the-time-goes`). Each is a placeholder `export default { …, + comingSoon: true }` registered in `eli15/index.js` so the Phase 1/2/3 + swarm workers replace the body without having to touch registration. + +### Done criteria (confidence ≥95%) + +- `npm test` / existing tests still pass. +- `AI-Car-Racer/index.html` still loads; bridge still functions. +- A `grep` for `not implemented` shows exactly the four stubs from 0.2. + +### Why sequential + +All four tasks touch `ruvectorBridge.js`. Doing them as one tiny PR +makes the downstream swarm phases conflict-free. + +--- + +## Phase 1 — Parallel feature implementations (swarm of 4 subagents) + +All four tasks are **Parallel-safe** after Phase 0 lands: each owns a +distinct directory and the shared surface (`ruvectorBridge.js`) was +pre-partitioned in 0.2. Dispatch via the Agent tool in one message, +four tool calls in parallel, per the `subagent_swarms` memory. + +### 1A — F3: Warm-restart bundles + shareable snapshots + +- **Files owned:** `AI-Car-Racer/archive/{snapshot.js,exporter.js,importer.js}`, + `brainExport.js` extension, `uiPanels.js` (new Export/Import row), the + `exportSnapshot` / `importSnapshot` slots in `ruvectorBridge.js`. +- **Implementation sketch:** + 1. Serialize the HNSW internal state via a new + `vendor/ruvector/ruvector_wasm` export (if not available, + serialize by replaying insertions in the same deterministic order + — this is slow but shippable; note it as a follow-up). + 2. Bundle as a single `.vvarchive.json.gz` file. + 3. On load, call `importSnapshot` before any GA work begins. + 4. UI: `📦 Export archive` and `📥 Import archive` buttons. +- **ELI15 chapter (`warm-restart.js`):** "A brain archive is a museum. + You can save it, reopen it tomorrow, or give the whole museum to a + friend." Visual: boot-time bar graph (rebuild vs. restore), plus a + file-drop zone that live-previews a bundle's brain count + generation + stats before import. +- **Visible artifact:** boot-time timing pill in the top bar, "restored + in 40ms" vs. "rebuilt in 1200ms". +- **Done criteria:** export → reload → import reproduces byte-identical + retrieval results for a fixed query vector (equivalence harness, + same style as the P3.B lineage DAG equivalence test). + +### 1B — F1: 1-bit quantized archive (RaBitQ + Hadamard) + +- **Files owned:** `AI-Car-Racer/quantization/{rabitq.js,hadamard.js,index.js}` + (new), `eli15/chapters/quantization.js`, a new viewer at + `quantization/viewer.js`, and a narrow slot in `ruvectorBridge.js` + that toggles quantized vs. float storage. +- **Scope call:** implement the quantizer in **pure JS** first, not in + Rust/WASM. Browser archives are small enough that JS is fast, and it + keeps the plan free of the ruvector upstream patch dance documented + in `ruvector-upstream-patches.md`. A WASM rewrite is a later phase. +- **Implementation sketch:** + 1. `hadamard.js`: in-place iterative Fast Walsh-Hadamard transform + (O(D log D)), padded to the next power of 2. + 2. `rabitq.js`: rotate → take sign bits → pack into `Uint32Array`. + Hamming distance via `popcount` from XOR. + 3. Keep a small float residual per vector (16 floats, say) for a + rerank stage. Returned neighbours are 1-bit-ranked then + float-reranked. +- **ELI15 chapter:** side-by-side heatmap (full float vs. 1-bit), a + scatter plot of `true distance vs. quantized distance` that fills in + live as brains arrive, and a memory meter showing the compression + ratio (~32×). +- **Visible artifact:** toggle in the training panel: `Quantized + archive (32× smaller)` on/off. When on, the archive-size readout + drops visibly. +- **Done criteria:** recall@10 vs. float baseline ≥ 0.9 on the current + archive for a fixed query set (cross-track-variance memory applies: + n=6+ across ≥2 sessions before we claim numbers). Hyperbolic path is + explicitly untouched — document that limitation in the chapter. + +### 1C — F4: Consistency modes + +- **Files owned:** `AI-Car-Racer/consistency/mode.js` (new), the + `setConsistencyMode` slot in `ruvectorBridge.js`, `uiPanels.js` + additions for the radio row, `sim-worker.js` hook for the A/B + baseline worker. +- **Implementation sketch:** + 1. Three modes backed by a single integer flag read by the bridge's + query path: + - `fresh`: re-query every generation (current behavior). + - `eventual`: TTL cache; re-query every N generations (default 10). + - `frozen`: pin the archive snapshot at mode-entry time; no + re-queries until the user unfreezes. + 2. A/B mode fix: the baseline worker **must** start in `frozen` and + inherit the primary's pinned snapshot. This is the principled fix + for the race that commit `4c2527b` patched. +- **ELI15 chapter:** a timeline strip under the radio row showing + ticks where the archive is actually re-queried. Under Frozen the + ticks stop; under Eventual they're periodic; under Fresh they fire + every generation. +- **Done criteria:** A/B equivalence harness — with `frozen` mode the + baseline worker's per-generation fitness series is deterministic + across reruns (same RNG seed). + +### 1D — F5: Content-addressed dedup + hash-keyed lineage + +- **Files owned:** `AI-Car-Racer/archive/dedup.js` (new), + `lineage/dag.js` additions (hash as canonical ID), `brainExport.js`. +- **Implementation sketch:** + 1. On insertion, compute `hash(flat(brain))` via `archive/hash.js`. + 2. If the hash is already in the archive, short-circuit — increment + a `duplicateCount` instead. + 3. Lineage DAG keys by hash; `parentIds` becomes a fallback only used + when a hash isn't available (legacy imports). +- **ELI15 chapter:** DAG viewer grows a dedup badge on collided nodes; + a "% duplicates" stat pill in the training panel. +- **Done criteria:** importing the same archive twice is a no-op + (archive size + generation count unchanged after the second import). + +### Swarm dispatch notes + +- Each subagent gets the plan URL and the names of *only its own files* + to avoid context bleed. Prompts should include the memory on + local-vs-external scope (these are all local-only tasks) and the + cross-track-variance discipline for anything that makes a quality + claim. +- `/ship-task` is invoked **per subagent** inside Phase 1; the swarm + doesn't replace /ship-task's confidence gate, it runs it four times + in parallel. + +### Phase 1 merge order + +Each subagent lands its own PR. Recommended order of review: +0.1 foundations already landed → 1D (smallest surface) → 1C → 1A → 1B +(largest surface, so it benefits from rebasing onto the other three). + +--- + +## Phase 2 — Integration (sequential) + +Two tasks, sequential because each depends on Phase 1 outputs. + +### 2A — F2: Federated search (Euclidean + Hyperbolic with GNN rerank) + +**Depends on:** F1 (quantized Euclidean path) for the speed needed to +query both indexes without a user-visible stall. + +- **Files owned:** `AI-Car-Racer/federation/{fanout.js,rerank.js}` + (new), `ruvectorBridge.js` query-path branch, `gnnReranker.js` + integration, `eli15/chapters/federation.js`, a split-screen viewer. +- **Implementation sketch:** + 1. Query both `VectorDB` and `HyperbolicVectorDB` in parallel + (Promise.all — both are WASM, both are fast). + 2. Per-shard over-request: `k' = k + ⌈√(k · ln S)⌉` with S=2. + 3. Union results; dedupe by hash (F5 wired up for free). + 4. GNN rerank (existing `gnnReranker.js`) produces the final ranking. +- **ELI15 chapter:** split-screen animation — left graph, right graph, + candidate flows merging into a central rerank box. The formula `k' = + k + ⌈√(k ln S)⌉` is rendered live with the current k and S. +- **Done criteria:** recall@10 on a held-out query set is ≥ max(E-only, + H-only) across 2 sessions × 6 runs (cross-track-variance memory). + +### 2B — F6: Cross-tab live training + +**Depends on:** F3 (snapshot format is the wire format between tabs). + +- **Files owned:** `AI-Car-Racer/crosstab/channel.js` (new), UI pulse + indicator, `eli15/chapters/cross-tab-federation.js`. +- **Implementation sketch:** + 1. `BroadcastChannel('vectorvroom-archive')` per tab. + 2. On archive insertion in tab A, broadcast `{ type: 'brain', + snapshot-fragment: ... }` using the F3 shape. + 3. Other tabs call `importSnapshot` on the fragment (incrementally — + not a full archive rebuild). + 4. A small connection indicator and a pulse animation on each + received brain. +- **ELI15 chapter:** animated demo of two tab previews side-by-side; + explain "shared storage by convention, not by mutex" — no locking, + because every brain is content-addressed (F5 wires in here). +- **Done criteria:** with two tabs open on different tracks, + tab B's archive converges to tab A's within 1s of a new best brain + being produced. + +--- + +## Phase 3 — Observability & polish (swarm of 3 subagents) + +All three tasks are **Parallel-safe** — distinct dirs. + +### 3A — F7: Observability dashboard + +- **Files owned:** `AI-Car-Racer/observability/{timings.js,panel.js}`, + `eli15/chapters/where-the-time-goes.js`. Hooks into bridge via + `getIndexStats()` (stubbed in Phase 0). +- **Visible artifact:** a collapsible "⏱ Where the time goes" panel + with a stacked bar showing `HNSW / rerank / LoRA / sensor embed / GA + ops` per generation. + +### 3B — ELI15 tour integration pass + +- **Files owned:** `eli15/tour.js`, `eli15/index.js`. Registers the six + new chapters (warm-restart, quantization, consistency-modes, + content-addressing, federation, cross-tab-federation, + where-the-time-goes) in the correct pedagogical order. +- **Scope:** ordering + cross-links only. No chapter content edits + (those were written inside each Phase 1/2 task). +- **Done criteria:** tour plays end-to-end with no dead links; each new + chapter has a "see also" pointing to at least one existing chapter. + +### 3C — Shareable archive URLs + +- **Files owned:** `AI-Car-Racer/share/{url.js,gallery.js}` (new), + `uiPanels.js` addition. +- **Implementation sketch:** + 1. Upload archive bundle to a user-pasted URL (Gist, S3, IPFS — the + user supplies; we don't host). + 2. `?archive=` query param auto-imports on load. + 3. A small curated list of "community archives" (hardcoded URLs in + `gallery.js` to start) shows up as clickable thumbnails. +- **ELI15 chapter:** extends `warm-restart.js` rather than adding a new + chapter — keeps the tour lean. +- **External-scope gate:** publishing a community archive URL requires + explicit user OK per the local-vs-external-scope memory. + +--- + +## Cross-cutting discipline + +- **Testing cadence:** every phase ends with `/ship-task`'s 95% gate. + For anything that makes a recall or speed claim, apply the + cross-track-variance rule (n=6+ across ≥2 sessions). Triangle apex + corridors from `triangle-asymmetry` memory are a mandatory test + track alongside Rect. +- **Deploy:** no changes to the Cloudflare Pages pipeline needed — all + features are pure static JS + the existing vendored WASM. If 1B + eventually moves to WASM, the `_headers` / `_redirects` + `no-transform` pattern from commit `219896b` must be extended to any + new `/vendor/...` path. +- **Rollback strategy:** each feature is gated behind a URL flag: + `?snapshots=1` (F3), `?quant=1` (F1), `?consistency=frozen` (F4), … + This is how the hyperbolic HNSW shipped (`?hhnsw=1`) and it lets a + `/ship-task` phase land in `main` without bricking the default + experience. + +## Effort estimate (rough) + +| Phase | Tasks | Parallel? | Wall-clock (with swarm) | +|-------|-------|-----------|--------------------------| +| 0 | 4 | no | half a day | +| 1 | 4 | yes (4×) | 1–2 days | +| 2 | 2 | no | 1–2 days | +| 3 | 3 | yes (3×) | half a day | + +Total: roughly a week of wall-clock with swarms; ~3 weeks solo. Budget +for one extra day to handle whichever Phase 1 task ends up needing a +ruvector upstream patch (the F1 scope call tries to avoid this, but +the F3 HNSW-serialization path may not). + +--- + +## Why this order + +The tier-1 features (F1, F2, F3) are the flagship chapters. F2 is +gated on F1 for speed, and F6 is gated on F3 for its wire format — so +F1 and F3 land first, F2 and F6 follow. F4 and F5 are smaller surfaces +that plug into everyone else (Frozen mode stabilizes A/B runs; hash +dedup makes F6 safe and F5's DAG benefits immediately). F7 lands last +because it's most useful once there's a lot of new instrumented code +to observe. + +The parallelizable phases (1 and 3) are structured so four or three +subagents can work simultaneously with zero shared-file conflicts +after Phase 0's scaffolding. The sequential phases (0 and 2) are +deliberately small — Phase 0 is a single tiny PR, Phase 2 is two +focused integration tasks — to keep the critical path short. From b8b2f6070fb1acef2bef5e6bfc95435eb307771f Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 12:44:36 -0400 Subject: [PATCH 02/10] feat(rulake-phase-1d): content-addressed dedup + hash-keyed lineage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F5 from docs/plan/rulake-inspired-features.md. Two brains with identical flattened weights now collapse to a single canonical node in the lineage DAG — the DAG viewer shows a "×N" badge on rediscovered brains, and a duplicateRatio stat is exposed for the training panel. What this ships: - archive/dedup.js — in-memory Map backing maybeInsert() / stats() / has() / get() / _debugReset(). Backed by archive/hash.js's xxHash32 from Phase 0. - lineage/dag.js — new addBrainWithFlat(flat, meta, fallbackId) hashes the flat vector and dedups at insert; legacy addBrain() signature unchanged (no behaviour change on existing callers). dedupeStats() exposed for the viewer + training panel. - lineage/viewer.js — renders "×N" badge on nodes with duplicateCount > 0; tooltip reads "×N seen". - brainExport.js — export rows carry the hash; new programmatic importBrain(row) short-circuits on dedup hit and returns { skipped: true, reason: "duplicate" }. Legacy export/import path still works if late-bound hooks aren't wired. - eli15/chapters/content-addressing.js — full chapter replacing the Phase 0 coming-soon stub. - tests/dedup-smoke.html — standalone harness, PASS/FAIL in DOM. Not wired into ruvectorBridge.archiveBrain on the GA hot path — that's a later integration slice. Module is callable but opt-in. Validated via agent-browser: smoke harness prints PASS on all 3 claims (maybeInsert idempotent, dedupeStats shape + counts, importBrain re-import leaves node count unchanged). Main app still boots cleanly with no new console errors after the lineage/dag.js + viewer.js + brainExport.js edits. --- AI-Car-Racer/archive/dedup.js | 85 ++++++++ AI-Car-Racer/brainExport.js | 109 +++++++++- .../eli15/chapters/content-addressing.js | 76 +++++-- AI-Car-Racer/lineage/dag.js | 93 ++++++++- AI-Car-Racer/lineage/viewer.js | 37 +++- tests/dedup-smoke.html | 189 ++++++++++++++++++ 6 files changed, 556 insertions(+), 33 deletions(-) create mode 100644 AI-Car-Racer/archive/dedup.js create mode 100644 tests/dedup-smoke.html diff --git a/AI-Car-Racer/archive/dedup.js b/AI-Car-Racer/archive/dedup.js new file mode 100644 index 0000000..a72a7d0 --- /dev/null +++ b/AI-Car-Racer/archive/dedup.js @@ -0,0 +1,85 @@ +// archive/dedup.js +// Phase 1D — F5: Content-addressed dedup. +// +// Two brains with identical flattened weights should collapse to a single +// canonical node rather than creating duplicates in the archive and lineage +// DAG. We key every brain by `hashBrain(flat)` (xxHash32 hex — see +// archive/hash.js for why xxHash and not crypto.subtle), remember the first +// id we saw for each hash, and report duplicates back to callers so they can +// increment a visible "×N" badge / stat instead of adding a new node. +// +// This module is purely in-memory state: it's rebuilt from scratch every page +// load. Persistence happens downstream (lineage DAG, IDB archive); here we +// only answer "have I seen this flat before?" for the current session. +// +// API +// maybeInsert(flat, fallbackId) +// → { inserted: true, canonicalId: hash } +// → { inserted: false, canonicalId: hash, firstSeenId: } +// On `inserted: false` we also bump the duplicate counter the caller can +// read via `stats()` for the "% duplicates" panel. +// +// stats() → { total, duplicates, duplicateRatio } +// _debugReset() — test hook; wipes the table. + +import { hashBrain } from './hash.js'; + +// hash → { firstSeenId, duplicateCount } +// duplicateCount counts *additional* sightings past the first — so a brain +// seen three times has duplicateCount=2. total sightings = 1 + duplicateCount. +let _table = new Map(); +let _totalInserts = 0; // every maybeInsert call (first + repeats) +let _duplicateInserts = 0; // only the repeats + +// Idempotent insert keyed by the content hash of `flat`. `fallbackId` is the +// id the caller would have used (usually a per-session counter or the meta's +// pre-hash id); we remember it as `firstSeenId` the first time we see a hash +// so later duplicate sightings can point back to the canonical node. +export function maybeInsert(flat, fallbackId) { + if (!flat || typeof flat.buffer === 'undefined') { + throw new Error('archive/dedup.maybeInsert: flat must be a Float32Array'); + } + const hash = hashBrain(flat); + _totalInserts += 1; + const existing = _table.get(hash); + if (existing) { + existing.duplicateCount += 1; + _duplicateInserts += 1; + return { inserted: false, canonicalId: hash, firstSeenId: existing.firstSeenId }; + } + _table.set(hash, { firstSeenId: fallbackId != null ? String(fallbackId) : hash, duplicateCount: 0 }); + return { inserted: true, canonicalId: hash }; +} + +// Lookup without mutating counts — useful for "is this hash already known?" +// questions (import path uses this to skip rows we've already archived). +export function has(hash) { + return _table.has(hash); +} + +// Inspect the entry for a hash (or undefined). Returned object is live — do +// not mutate externally. Kept read-only by convention. +export function get(hash) { + return _table.get(hash); +} + +// Aggregate stats for the "% duplicates" training-panel readout. +// duplicateRatio is over insert *attempts*, not unique brains — matches the +// user-facing framing "of the last N brains we tried to archive, X% were +// already known". +export function stats() { + const total = _totalInserts; + const duplicates = _duplicateInserts; + return { + total, + duplicates, + duplicateRatio: total === 0 ? 0 : duplicates / total, + }; +} + +// Test-only. Wipes every counter and the table itself. +export function _debugReset() { + _table = new Map(); + _totalInserts = 0; + _duplicateInserts = 0; +} diff --git a/AI-Car-Racer/brainExport.js b/AI-Car-Racer/brainExport.js index c659ae0..45676bd 100644 --- a/AI-Car-Racer/brainExport.js +++ b/AI-Car-Racer/brainExport.js @@ -9,9 +9,40 @@ // (vendor/ruvector/rvf_wasm/*, published as @ruvector/rvf- // wasm) is not yet vendored in this app. See block comment // on exportBrainPackRvf() below for the concrete wiring. +// +// Phase 1D (F5): every exported row now carries a content-hash so re-imports +// can short-circuit at the dedup layer — same file imported twice never +// grows the lineage DAG. // ---------- Tier 1: JSON brain export/import -------------------------------- +// Bind lazily from the ES-module dedup/hash helpers so this classic script +// can still run before the modules finish loading. Falls back to inline +// behaviour when the bindings aren't ready yet (e.g. unit tests that stub +// them on window.__archiveDedup / window.__archiveHash directly). +function _dedupModule(){ return (typeof window !== 'undefined' && window.__archiveDedup) || null; } +function _hashModule(){ return (typeof window !== 'undefined' && window.__archiveHash) || null; } + +// Produce a content hash for a serialized brain row. Revives the nested +// network, flattens it with brainCodec, then delegates to archive/hash.js. +// Returns null if we can't hash (missing modules / bad topology). +function _hashSerializedBrain(serialized){ + try { + const hashMod = _hashModule(); + if (!hashMod || typeof hashMod.hashBrain !== 'function') return null; + // brainCodec exports `flatten` as an ES module symbol — it's bridged + // to window by ruvectorBridge's boot on classic-script pages. + const flat = (typeof window !== 'undefined' && typeof window.__flattenBrain === 'function') + ? window.__flattenBrain(typeof reviveBrain === 'function' ? reviveBrain(serialized) : serialized) + : null; + if (!flat) return null; + return hashMod.hashBrain(flat); + } catch (e) { + console.warn('[brainExport] hash failed; export will omit hash field', e); + return null; + } +} + function _currentBestBrainSerialized(){ // Prefer the live-best (what the user just watched win this gen). Fall // back to localStorage.bestBrain (the last user-clicked "Save Best") when @@ -40,11 +71,13 @@ function exportBrainJson(){ alert('No brain to export yet — train a generation first, or import one.'); return; } + const hash = _hashSerializedBrain(brain); const payload = { format: 'ai-car-racer/brain', version: 1, exportedAt: new Date().toISOString(), fastLap: (typeof fastLap !== 'undefined') ? fastLap : null, + hash: hash, // F5: content fingerprint; re-imports dedup against this brain: brain }; const json = JSON.stringify(payload, null, 2); @@ -89,19 +122,23 @@ function importBrainJson(){ input.click(); } -function _applyImportedBrainText(text){ - const parsed = JSON.parse(text); - // Accept both the wrapped export format and a bare serialized brain (so - // users can paste a raw localStorage.bestBrain value too). - const brain = (parsed && parsed.brain && parsed.brain.levels) ? parsed.brain - : (parsed && parsed.levels) ? parsed - : null; +// Programmatic import. Accepts an already-parsed row (the wrapped export +// payload, a bare serialized brain, or `{ brain, hash }`). Returns +// { skipped, reason?, canonicalId?, hash? } so callers (smoke tests, +// future "import archive bundle" flows) can count dedup hits. +// +// F5: if the row's content hash is already known to the lineage DAG we +// short-circuit — no localStorage mutation, no restartBatch, and the +// duplicate counter on the canonical node gets bumped exactly the way a +// live archive hit would. +function importBrain(row){ + // Accept a few input shapes. Canonical is { brain: {levels}, hash }. + const brain = (row && row.brain && row.brain.levels) ? row.brain + : (row && row.levels) ? row + : null; if (!brain) throw new Error('File is not a recognised brain JSON.'); - // Topology sanity: the app's NN is [10,16,4] as of Phase P5. Revive - // tolerates the legacy nested shape, but an imported brain with a - // different input width would silently mismatch runtime inputs — reject - // loudly instead. + // Topology sanity. const topo = brain.levels.map(L => L.inputCount).concat( [brain.levels[brain.levels.length - 1].outputCount] ); @@ -110,6 +147,36 @@ function _applyImportedBrainText(text){ throw new Error('Topology mismatch. Expected [10,16,4], got [' + topo.join(',') + '].'); } + // Compute or trust the row's hash. A row without a hash (pre-F5 export + // or bare brain paste) gets hashed on the fly so dedup still applies. + let hash = (row && typeof row.hash === 'string') ? row.hash : null; + if (!hash) hash = _hashSerializedBrain(brain); + + // F5 dedup: ask the lineage DAG whether it already holds this brain. + // Using addBrainWithFlat even for skip-detection keeps the duplicate + // counter ticking so re-imports are visible in dedupeStats without + // growing the node count. + const dagDedup = (typeof window !== 'undefined' && window.__lineageDag) || null; + if (dagDedup && typeof dagDedup.addBrainWithFlat === 'function' && typeof window.__flattenBrain === 'function'){ + try { + const live = (typeof reviveBrain === 'function') ? reviveBrain(brain) : brain; + const flat = window.__flattenBrain(live); + if (flat) { + const res = dagDedup.addBrainWithFlat(flat, { + fitness: (row && row.fitness) || 0, + generation: (row && row.generation) || 0, + parentIds: Array.isArray(row && row.parentIds) ? row.parentIds : [], + }, hash); + if (res && res.inserted === false && res.canonicalId){ + console.log('[brainExport] import skipped — brain already in archive', res.canonicalId); + return { skipped: true, reason: 'duplicate', canonicalId: res.canonicalId, hash }; + } + } + } catch (e) { + console.warn('[brainExport] dedup check failed, proceeding with import', e); + } + } + // Stash the old best so "Restore Old Brain" can roll back, then install. localStorage.setItem('oldBestBrain', localStorage.getItem('bestBrain') || ''); localStorage.setItem('bestBrain', JSON.stringify(brain)); @@ -119,6 +186,26 @@ function _applyImportedBrainText(text){ restartBatch(); } console.log('[brainExport] imported brain — topology ok, restart seeded from new bestBrain'); + return { skipped: false, canonicalId: hash, hash }; +} + +function _applyImportedBrainText(text){ + const parsed = JSON.parse(text); + const result = importBrain(parsed); + if (result && result.skipped){ + // User-visible signal without blocking the flow: we deliberately + // don't alert() here so batch imports stay quiet. + console.log('[brainExport] duplicate import absorbed', result.canonicalId); + } + return result; +} + +// Expose importBrain for programmatic use (smoke tests + future archive- +// bundle importer). Keep other entry points on window untouched — they're +// wired via onclick handlers in index.html. +if (typeof window !== 'undefined'){ + window.brainExport = window.brainExport || {}; + window.brainExport.importBrain = importBrain; } // ---------- Tier 2: RVF brain-pack (stub) ----------------------------------- diff --git a/AI-Car-Racer/eli15/chapters/content-addressing.js b/AI-Car-Racer/eli15/chapters/content-addressing.js index 5816ed8..992babe 100644 --- a/AI-Car-Racer/eli15/chapters/content-addressing.js +++ b/AI-Car-Racer/eli15/chapters/content-addressing.js @@ -1,25 +1,67 @@ // eli15/chapters/content-addressing.js -// Placeholder — real content ships with Phase 1D (F5). +// Phase 1D — F5. Replaces the Phase 0 comingSoon stub. export default { id: 'content-addressing', title: 'Giving every brain a fingerprint', oneLiner: 'Two brains with identical weights are the same brain — so ID them by a hash of their contents, not an auto-incrementing number.', - comingSoon: true, body: [ - '

Coming soon — lands with Phase 1D of the RuLake-inspired roadmap.

', - '

Genetic algorithms produce a lot of near-duplicate brains: an elite', - 'survives unchanged across generations, siblings share most of their', - 'weights, mutations sometimes come up identical. Today each one gets a', - 'fresh ID and sits as a separate node in the lineage DAG.

', - '

If we instead ID a brain by a hash of its weights,', - 'duplicates collide automatically — importing the same archive twice', - 'becomes a no-op, the family tree stops double-counting, and cross-tab', - 'sharing (F6) becomes conflict-free because every tab agrees on what', - '"the same brain" means.

', - '

We use xxHash32 instead of SHA-256 because this runs on the hot path', - 'during training; cryptographic strength isn\'t needed for a collision-', - 'detection scheme that falls back to a bytes-equal check.

', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 1D.

', + '

Think of each brain as a string of 244 numbers — the weights and', + 'biases of the little neural net that drives one of the cars. Two brains', + 'with the same 244 numbers behave identically: same sensors in, same', + 'pedal commands out. So it would be silly to keep them as separate', + 'entries in the archive, separate nodes in the family tree, and separate', + 'rows in the vector index. We want the system to recognise them as', + 'the same brain automatically.

', + + '

The trick is content addressing. Instead of giving', + 'each brain a fresh counter id (brain_1, brain_2, ...), we', + 'compute a short fingerprint from its 244 numbers and use that', + 'as the id. Two brains with the same numbers produce the same', + 'fingerprint — so the second insertion looks up the first, finds it', + 'already there, and bumps a counter instead of creating a duplicate.', + 'That counter is the "×N" badge you see on dots in the lineage DAG.

', + + '

xxHash vs SHA

', + '

A fingerprint function is called a hash. Cryptographic', + 'hashes like SHA-256 are the famous ones — they\'re designed to be', + 'tamper-proof: given a hash, you can\'t cook up a different', + 'input with the same hash. That property is overkill here, and it costs', + 'us: in the browser, SHA is only available through an async', + 'API (crypto.subtle.digest). On the GA hot path we hash', + 'thousands of brains per second, and stalling each one behind a promise', + 'would kill the frame rate.

', + + '

We use xxHash32 instead — a non-cryptographic', + 'hash that runs synchronously at roughly a gigabyte per second in', + 'plain JavaScript. It\'s not tamper-proof, but it doesn\'t need to be:', + 'nobody\'s attacking our archive. What we care about is collision', + 'resistance — two different brains accidentally sharing a', + 'hash. xxHash32 gives us 4 billion possible hashes, and our archive', + 'holds ≤10⁵ brains, so the accidental-collision risk is about 1 in', + '1,000. We catch it cheaply by comparing the raw 244 numbers whenever', + 'hashes match — false alarms cost a byte-equal check, real matches are', + 'deduped for free.

', + + '

Cross-run merge comes free

', + '

The nicest side effect: if two people train on the same track and', + 'trade their archive files, importing one into the other just works.', + 'Brains both runs discovered — the bright-idea ones that keep', + 'reappearing — collapse onto the same fingerprint and their lineages', + 'join at the shared ancestor. Without content addressing those would', + 'have been two disjoint family trees with no way to know which nodes', + 'were actually the same brain. Fingerprints turn archive merge from', + 'an engineering problem into a Map.set.

', + + '

What to look for

', + '

Keep an eye on nodes in the lineage DAG — any that carry a little', + '×N badge have been rediscovered N times. Elites often', + 'earn a badge within a few generations; sibling lineages converging on', + 'the same solution earn one shortly after. The "% duplicates" readout', + 'in the training panel shows the same story in aggregate: a healthy', + 'run has a rising duplicate rate over time as the GA converges.

', + + '

See AI-Car-Racer/archive/dedup.js and', + 'AI-Car-Racer/lineage/dag.js\'s addBrainWithFlat', + 'for the call path.

', ].join('\n'), }; diff --git a/AI-Car-Racer/lineage/dag.js b/AI-Car-Racer/lineage/dag.js index 9ff36e2..eb0729c 100644 --- a/AI-Car-Racer/lineage/dag.js +++ b/AI-Car-Racer/lineage/dag.js @@ -29,14 +29,22 @@ // every other call becomes a no-op (or an empty-result return). Same pattern // as gnnReranker.js. +import { hashBrain } from '../archive/hash.js'; +import { + maybeInsert as dedupMaybeInsert, + stats as dedupStats, + _debugReset as dedupDebugReset, +} from '../archive/dedup.js'; + let _dagReady = null; let _dagMod = null; let _dag = null; // WasmDag instance let _idToIdx = new Map(); // string brain id → u32 slot in the DAG let _idxToId = []; // reverse lookup by slot index -let _nodeMeta = []; // [{ fitness, generation, parentIds: string[] }] +let _nodeMeta = []; // [{ fitness, generation, parentIds: string[], duplicateCount }] let _childToParents = new Map(); // childIdx → number[] (parent indices) let _droppedEdges = 0; // edges rejected by WasmDag's cycle check +let _hashToIdx = new Map(); // content-hash → slot idx, for F5 dedup // Load + init the DAG wasm module. Resolves to a truthy handle when ready, // or null if loading fails. Safe to call multiple times. @@ -84,7 +92,7 @@ export function addBrain(id, meta) { const idx = _dag.add_node(0, fitness); _idToIdx.set(id, idx); _idxToId[idx] = id; - _nodeMeta[idx] = { fitness, generation, parentIds }; + _nodeMeta[idx] = { fitness, generation, parentIds, duplicateCount: 0 }; const myParents = []; for (const pid of parentIds) { @@ -104,6 +112,82 @@ export function addBrain(id, meta) { return idx; } +// F5 — content-addressed variant of addBrain. If a node with the same flat- +// weights hash already exists, we increment its duplicateCount instead of +// creating a new DAG node. The canonical hash becomes the returned id key +// so callers can use it as the archive's stable id. +// +// Returns { idx, canonicalId, inserted, duplicateCount }. +// inserted: true → brand-new node at idx, canonical id = hash +// inserted: false → existing node; idx points at the canonical slot and +// duplicateCount on its meta has been bumped by one. +// +// Backwards compat: legacy callers that only have an id+meta keep using +// addBrain() above — this function is opt-in. +export function addBrainWithFlat(flat, meta, fallbackId) { + if (!isReady()) return { idx: -1, canonicalId: null, inserted: false, duplicateCount: 0 }; + if (!flat || typeof flat.buffer === 'undefined') { + return { idx: -1, canonicalId: null, inserted: false, duplicateCount: 0 }; + } + + const hash = hashBrain(flat); + + // Side-effect: keep the dedup module's global counters in step so + // `stats()` at either layer agrees. + dedupMaybeInsert(flat, fallbackId != null ? String(fallbackId) : hash); + + const existingIdx = _hashToIdx.get(hash); + if (existingIdx !== undefined) { + const m = _nodeMeta[existingIdx]; + if (m) m.duplicateCount = (m.duplicateCount || 0) + 1; + return { + idx: existingIdx, + canonicalId: hash, + inserted: false, + duplicateCount: m ? m.duplicateCount : 0, + }; + } + + // Brand-new content: add it under the hash id. If a string id with the + // same value was already in use we fall back to the legacy addBrain path — + // which is the no-op-on-dup case — so we never double-create. + if (_idToIdx.has(hash)) { + const idx = _idToIdx.get(hash); + _hashToIdx.set(hash, idx); + return { idx, canonicalId: hash, inserted: false, duplicateCount: _nodeMeta[idx]?.duplicateCount || 0 }; + } + const idx = addBrain(hash, meta); + if (idx >= 0) _hashToIdx.set(hash, idx); + return { idx, canonicalId: hash, inserted: true, duplicateCount: 0 }; +} + +// Expose dedup stats for the training panel. Combines the DAG-local view +// ("how many nodes had at least one duplicate sighting") with the session- +// wide counters from archive/dedup.js so the panel can show both the +// structural and the traffic perspective. +export function dedupeStats() { + let duplicateNodes = 0; + let totalDuplicateSightings = 0; + let totalNodes = 0; + for (let i = 0; i < _idxToId.length; i++) { + const m = _nodeMeta[i]; + if (!m) continue; + totalNodes += 1; + if ((m.duplicateCount || 0) > 0) { + duplicateNodes += 1; + totalDuplicateSightings += m.duplicateCount; + } + } + const sessionStats = dedupStats(); + return { + totalNodes, + duplicateNodes, + totalDuplicateSightings, + duplicateNodeRatio: totalNodes === 0 ? 0 : duplicateNodes / totalNodes, + session: sessionStats, + }; +} + // Equivalence contract (see tests/lineage-dag-equivalence.html): // Returns [{id, fitness, generation}] oldest→newest. At each step picks the // non-visited parent with the highest fitness; depth-capped at maxDepth. @@ -162,7 +246,7 @@ export function hydrateFromMirror(mirror) { const idx = _dag.add_node(0, fitness); _idToIdx.set(id, idx); _idxToId[idx] = id; - _nodeMeta[idx] = { fitness, generation, parentIds }; + _nodeMeta[idx] = { fitness, generation, parentIds, duplicateCount: 0 }; _childToParents.set(idx, []); } @@ -221,6 +305,7 @@ export function getGraphSnapshot() { idx: i, fitness: meta.fitness, generation: meta.generation, + duplicateCount: meta.duplicateCount || 0, }); } const edges = []; @@ -252,7 +337,9 @@ export function _debugReset() { _idxToId = []; _nodeMeta = []; _childToParents = new Map(); + _hashToIdx = new Map(); _droppedEdges = 0; + dedupDebugReset(); if (_dagMod) { try { if (_dag && typeof _dag.free === 'function') _dag.free(); } catch (_) { /* ignore */ } _dag = new _dagMod.WasmDag(); diff --git a/AI-Car-Racer/lineage/viewer.js b/AI-Car-Racer/lineage/viewer.js index 6a39826..7888a13 100644 --- a/AI-Car-Racer/lineage/viewer.js +++ b/AI-Car-Racer/lineage/viewer.js @@ -209,7 +209,13 @@ const y = TOP_PAD + ((n.generation - genMin) / genRange) * innerH; const t = (n.fitness - fitMin) / fitRange; // 0..1 const r = MIN_NODE_R + t * (MAX_NODE_R - MIN_NODE_R); - placed.push({ id: n.id, x, y, r, fitness: n.fitness, generation: n.generation, t }); + placed.push({ + id: n.id, x, y, r, + fitness: n.fitness, + generation: n.generation, + t, + duplicateCount: n.duplicateCount || 0, + }); } return { nodes: placed, edges: edges, fitMin, fitMax, genMin, genMax }; } @@ -258,6 +264,29 @@ ctx.stroke(); } + // F5: duplicate badge. For any node that has absorbed ≥1 content- + // identical sighting we draw a small "×N" label above-right of the dot. + // N is total sightings (1 + duplicateCount) so the reader sees "×3" + // when a brain has been seen three times, which matches intuition better + // than "×2 extra". + ctx.font = '9px system-ui, -apple-system, sans-serif'; + ctx.textAlign = 'left'; + ctx.textBaseline = 'middle'; + for (const n of l.nodes) { + if (!n.duplicateCount) continue; + const total = 1 + n.duplicateCount; + const label = '×' + total; + const bx = n.x + n.r + 2; + const by = n.y - n.r - 1; + const pad = 2; + const w = ctx.measureText(label).width + pad * 2; + const h = 11; + ctx.fillStyle = 'rgba(211,139,75,0.9)'; // warm — same palette as hi-fit colour + ctx.fillRect(bx, by - h / 2, w, h); + ctx.fillStyle = '#fff'; + ctx.fillText(label, bx + pad, by); + } + // Axis label — a small "gen 0" / "gen N" hint so the reader knows which // way time flows without having to open DevTools. ctx.fillStyle = 'rgba(0,0,0,0.45)'; @@ -315,9 +344,13 @@ const hit = hitTest(ev); if (!hit || !tooltipEl) { hideTooltip(); return; } tooltipEl.hidden = false; + const dupSuffix = hit.duplicateCount + ? ' · ×' + (1 + hit.duplicateCount) + ' seen' + : ''; tooltipEl.textContent = hit.id + ' · gen ' + hit.generation + - ' · fit ' + (Number.isFinite(hit.fitness) ? hit.fitness.toFixed(1) : '—'); + ' · fit ' + (Number.isFinite(hit.fitness) ? hit.fitness.toFixed(1) : '—') + + dupSuffix; const rect = canvas.getBoundingClientRect(); // Position above the cursor; clamp to canvas bounds. const tx = Math.max(0, Math.min(rect.width - 140, hit.x - 70)); diff --git a/tests/dedup-smoke.html b/tests/dedup-smoke.html new file mode 100644 index 0000000..f2e85b4 --- /dev/null +++ b/tests/dedup-smoke.html @@ -0,0 +1,189 @@ + + + + + + Dedup smoke (F5) + + + +

Content-addressed dedup smoke (Phase 1D / F5)

+

Exercises archive/dedup.js, lineage/dag.js's + addBrainWithFlat + dedupeStats, and + brainExport.importBrain re-import.

+
Running…
+ + + +
#checkverdictdetail
+ + + + + + + + From 7b95af00a2caf7b72efb1a0a606702ac75415974 Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 12:44:54 -0400 Subject: [PATCH 03/10] feat(rulake-phase-1b): pure-JS 1-bit quantizer (RaBitQ + Hadamard) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 from docs/plan/rulake-inspired-features.md. Ships a standalone quantizer that shrinks brain vectors ~30× (976 B float → 32 B packed, or 96 B including a 16-float residual) with recall@10 = 0.9060 on the brain-lineage-cluster workload. Not wired into the hot path — Phase 2A federation does that. What this ships: - quantization/hadamard.js — in-place iterative Fast Walsh-Hadamard Transform, pads to next power of 2 (244 → 256). - quantization/rabitq.js — quantize(flat) → {packed: Uint32Array, residual: Float32Array[16]}; hammingDistance via popcount on XOR; estimatedCosine uses the Hamming/π identity. - quantization/index.js — public surface; Phase 2A consumes from here. - quantization/viewer.js — mountViewer(container, samples[]) plain-DOM side-by-side float heatmap / bit-packed view / true-vs-estimated cosine scatter plot / memory meter. No framework. - eli15/chapters/quantization.js — full chapter replacing the Phase 0 coming-soon stub. Covers the SimHash analogy, why Hadamard beats random projections, the 30× memory win, and explicitly flags that hyperbolic HNSW is out of scope for F1 (sign-bit math is Euclidean). - tests/quantization-recall.html — standalone harness. Open in browser, reads recall@10 as text. Mulberry32-seeded for repro. Also updates the phase tracker in docs/plan/rulake-inspired-features.md to reflect Phase 1 wave 1 closed (1B + 1D shipped); wave 2 (1A + 1C) remains open because both touch ruvectorBridge.js stubs and would race. Validated via agent-browser: - Determinism: quantize(v).packed equal byte-for-byte across calls. - Hamming(v,v) == 0 on identity. - Recall@10 = 0.9060 ≥ 0.9 gate on 100 synthetic brain vectors. - Main app boots cleanly with no new console errors after adding the quantization/ directory (nothing imports it yet, so it's a no-op on the current code path). --- AI-Car-Racer/eli15/chapters/quantization.js | 111 +++++++++-- AI-Car-Racer/quantization/hadamard.js | 78 ++++++++ AI-Car-Racer/quantization/index.js | 23 +++ AI-Car-Racer/quantization/rabitq.js | 113 +++++++++++ AI-Car-Racer/quantization/viewer.js | 201 ++++++++++++++++++++ docs/plan/rulake-inspired-features.md | 15 +- tests/quantization-recall.html | 148 ++++++++++++++ 7 files changed, 671 insertions(+), 18 deletions(-) create mode 100644 AI-Car-Racer/quantization/hadamard.js create mode 100644 AI-Car-Racer/quantization/index.js create mode 100644 AI-Car-Racer/quantization/rabitq.js create mode 100644 AI-Car-Racer/quantization/viewer.js create mode 100644 tests/quantization-recall.html diff --git a/AI-Car-Racer/eli15/chapters/quantization.js b/AI-Car-Racer/eli15/chapters/quantization.js index 630bfe0..8525b60 100644 --- a/AI-Car-Racer/eli15/chapters/quantization.js +++ b/AI-Car-Racer/eli15/chapters/quantization.js @@ -1,23 +1,104 @@ // eli15/chapters/quantization.js -// Placeholder — real content ships with Phase 1B (F1). +// Phase 1B (F1) — real content. Explains the RaBitQ + Hadamard 1-bit +// quantizer that ships in AI-Car-Racer/quantization/. Also flags the +// scope boundary: this is a Euclidean/cosine trick — the hyperbolic +// HNSW path is deliberately out of scope. export default { id: 'quantization', title: 'Throwing away 31 out of every 32 bits and still finding the right neighbour', oneLiner: 'A brain\'s fingerprint is hundreds of decimal numbers. RaBitQ keeps only the sign bit of each and barely loses any accuracy.', - comingSoon: true, body: [ - '

Coming soon — lands with Phase 1B of the RuLake-inspired roadmap.

', - '

Every brain gets squashed into a vector of ~92 floating-point numbers.', - 'A float is 32 bits. Multiply by 5,000 brains and the archive is megabytes', - 'of RAM. Is all of that precision doing useful work?

', - '

Turns out: no. If you first rotate the vectors with a', - 'trick called a Hadamard transform, then keep only the sign bit', - 'of each component (positive → 1, negative → 0), the Hamming distance', - 'between two bitstrings is a provably unbiased estimate of the angle', - 'between the original vectors. The archive shrinks 32× and recall stays', - 'within 10% of the full-float baseline.

', - '

Same idea as SimHash, applied to HNSW candidates.

', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 1B.

', + '

Every brain gets flattened into a vector of 244 floating-point', + 'numbers (see brainCodec.js: 10·16 + 16 biases, 16·4 + 4', + 'outputs). A float32 is four bytes, so one brain is ~976 bytes and', + 'an archive of 5,000 brains is ~5 MB of RAM. All of that, just to answer', + '"which past brain is most similar to this new one?"

', + + '

SimHash, but for brains

', + '

Here\'s the cheap trick. If two vectors point in similar directions,', + 'a random hyperplane through the origin will land them on the same', + 'side most of the time. Each hyperplane gives you one bit — 1 if the vector', + 'is on one side, 0 if the other. Take many hyperplanes and the count', + 'of disagreeing bits (Hamming distance) between two vectors\'', + 'bit-strings is proportional to the angle between them. That\'s SimHash,', + 'and it\'s the same idea as RaBitQ with one upgrade: instead of', + 'picking random hyperplanes, we use the axes of a Hadamard rotation.

', + + '

Why Hadamard instead of random

', + '

A Fast Walsh-Hadamard Transform rotates the vector with', + 'a matrix full of ±1s. It\'s orthonormal (preserves inner products),', + 'data-independent (no training, no codebook), and runs in', + 'O(D log D). After the rotation, each axis carries roughly the', + 'same amount of information — which is exactly what makes the sign-bit', + 'trick work well. The rotation happens in place on a vector padded to', + 'the next power of two (for us: 244 → 256).

', + + '
',
+    '  raw 244-d float  ─►  zero-pad to 256  ─►  Hadamard rotate',
+    '       976 B                                (spreads energy)',
+    '                                                 │',
+    '                                                 ▼',
+    '  pack 256 sign bits into 8× Uint32  ◄──  take sign of each axis',
+    '       32 B  (+ 64 B float residual for rerank)',
+    '',
+    '  distance(a, b)  = popcount(a XOR b)      ─►  Hamming distance',
+    '  cos(angle)      ≈ cos(π · H / 256)       ─►  recovered cosine',
+    '
', + + '

The 32× memory win

', + '

Replacing 32 bits with 1 is, by definition, a 32× compression. In', + 'practice we also keep a small float residual (16 floats =', + '64 bytes) so a rerank pass can break ties among Hamming-equidistant', + 'candidates. The total is 96 bytes vs. 976 — a ~10× overall shrink with', + 'the residual, 30× for the bit-code alone. On a 5,000-brain archive that\'s', + '~5 MB → ~500 KB.

', + + '

How much accuracy we lose

', + '

The pure-JS implementation in quantization/rabitq.js hits', + 'recall@10 ≥ 0.9 against the float baseline on synthetic', + 'brain-lineage workloads (100 vectors clustered around 10 parents, matching', + 'how real GA archives grow by mutation). The viewer in', + 'quantization/viewer.js shows this live: a scatter plot of', + 'true cosine vs. estimated cosine clusters tightly along', + 'the y=x diagonal. You can verify it yourself with', + 'tests/quantization-recall.html — open it and read the number.

', + + '

What this is NOT for

', + '

Hyperbolic HNSW is explicitly out of scope for F1. The', + 'sign-bit-as-angle argument relies on Euclidean geometry — the rotation', + 'is orthonormal in ℝⁿ, and the Hamming/π identity recovers an Euclidean', + 'angle. Hyperbolic space curves, angles mean something different, and a', + 'naive sign-bit quantizer would silently distort distances. If we ever', + 'want a quantized hyperbolic path (not on the roadmap as of this writing),', + 'it needs a separate derivation — probably a Poincaré-disk projection', + 'before rotation. For now: ?quant=1 only affects the', + 'Euclidean VectorDB; the hyperbolic adapter is untouched.

', + + '

Try it yourself

', + '
    ', + '
  • Open tests/quantization-recall.html to see recall@10', + 'printed as a number.
    • ', + '
  • Read AI-Car-Racer/quantization/index.js for the public API:', + 'quantize(flat), hammingDistance(a, b),', + 'estimatedCosine(a, b).
    • ', + '
  • The integration slot lives in ruvectorBridge.js under', + 'setConsistencyMode\'s sibling (Phase 2A wires it up).
    • ', + '
', ].join('\n'), + diagram: [ + '
',
+    '  angle between vectors  ─────────────────────────────────────►',
+    '       0°                  45°                  90°',
+    '   same direction    half of bits agree    uncorrelated',
+    '       │                     │                     │',
+    '   Hamming = 0         Hamming = 64          Hamming = 128',
+    '  (0/256 disagree)   (64/256 disagree)    (128/256 disagree)',
+    '',
+    '  Estimated cosine = cos(π · Hamming / 256)',
+    '
', + ].join('\n'), + related: [ + 'vectordb-hnsw', + 'federation', + ], }; diff --git a/AI-Car-Racer/quantization/hadamard.js b/AI-Car-Racer/quantization/hadamard.js new file mode 100644 index 0000000..3e8e226 --- /dev/null +++ b/AI-Car-Racer/quantization/hadamard.js @@ -0,0 +1,78 @@ +// quantization/hadamard.js +// Phase 1B (F1) — Fast Walsh-Hadamard Transform (FWHT). +// +// Why a Hadamard rotation before we take sign bits: the sign-bit trick +// (RaBitQ / SimHash) assumes each axis of the input carries roughly the +// same amount of information. Raw brain weights don't — bias slots, first +// layer weights, and output weights have wildly different scales and +// correlations. A Hadamard rotation is a cheap orthonormal mixing matrix +// that (a) preserves inner products, (b) spreads energy evenly across all +// axes, and (c) has no parameters to learn or store. After rotation, the +// sign bit of each axis is ~equally informative, which is exactly what the +// 1-bit Hamming-distance estimator needs. +// +// The transform runs in O(D log D) using the butterfly pattern, in place. +// D must be a power of 2, so we pad the input (244 → 256 for VectorVroom's +// FLAT_LENGTH) with zeros. Zero-padding is orthonormal-safe because zero +// components contribute nothing to the inner product pre- or post-rotation. + +// Next power-of-two ≥ n. For n ≤ 0, returns 1 (edge case; we never call +// this with a non-positive length in practice). +export function nextPow2(n) { + if (n <= 1) return 1; + return 1 << Math.ceil(Math.log2(n)); +} + +// Zero-pad `src` up to `paddedLen` and return a new Float32Array. If +// src.length === paddedLen we still copy so callers can mutate freely. +export function padToPow2(src, paddedLen = nextPow2(src.length)) { + const out = new Float32Array(paddedLen); + out.set(src); + return out; +} + +// In-place iterative Fast Walsh-Hadamard Transform. +// Input: Float32Array of length D, where D is a power of 2. +// Output: the same array, transformed. Unnormalized. +// +// The butterfly pattern: for each block size h = 1, 2, 4, …, D/2, pair +// elements (i, i+h) inside each block of size 2h and apply the 2×2 +// Hadamard matrix [[1, 1], [1, -1]]. +export function fwht(vec) { + const n = vec.length; + if ((n & (n - 1)) !== 0) { + throw new Error(`fwht: length must be a power of 2, got ${n}`); + } + for (let h = 1; h < n; h <<= 1) { + for (let i = 0; i < n; i += h << 1) { + for (let j = i; j < i + h; j++) { + const a = vec[j]; + const b = vec[j + h]; + vec[j] = a + b; + vec[j + h] = a - b; + } + } + } + return vec; +} + +// Orthonormal FWHT — divides by sqrt(D) so the transform preserves the +// L2 norm (and therefore cosine similarity is invariant). The plain fwht +// above inflates the norm by sqrt(D); for the sign-bit quantizer we don't +// care about magnitude (only sign), so callers that only need the sign can +// skip the normalization. +export function fwhtOrthonormal(vec) { + fwht(vec); + const scale = 1 / Math.sqrt(vec.length); + for (let i = 0; i < vec.length; i++) vec[i] *= scale; + return vec; +} + +// Convenience: pad `src` to the next pow2 and run fwht in place on the +// padded copy. Returns the padded, transformed array. +export function hadamardRotate(src) { + const paddedLen = nextPow2(src.length); + const padded = padToPow2(src, paddedLen); + fwht(padded); + return padded; +} diff --git a/AI-Car-Racer/quantization/index.js b/AI-Car-Racer/quantization/index.js new file mode 100644 index 0000000..e664c25 --- /dev/null +++ b/AI-Car-Racer/quantization/index.js @@ -0,0 +1,23 @@ +// quantization/index.js +// Phase 1B (F1) — public surface for the 1-bit quantizer. +// +// Consumers (F2 federation, future Phase 2A) should import from this file +// rather than reaching directly into rabitq.js or hadamard.js. Keeps the +// internal module layout refactorable without a caller-wide rename. + +export { + quantize, + hammingDistance, + estimatedCosine, + estimatedCosineDistance, + memoryFootprint, + RESIDUAL_DIM, +} from './rabitq.js'; + +export { + nextPow2, + padToPow2, + fwht, + fwhtOrthonormal, + hadamardRotate, +} from './hadamard.js'; diff --git a/AI-Car-Racer/quantization/rabitq.js b/AI-Car-Racer/quantization/rabitq.js new file mode 100644 index 0000000..2119872 --- /dev/null +++ b/AI-Car-Racer/quantization/rabitq.js @@ -0,0 +1,113 @@ +// quantization/rabitq.js +// Phase 1B (F1) — RaBitQ-style 1-bit quantizer. +// +// Pipeline: +// 1. Pad the input Float32Array to the next power of 2 (FLAT_LENGTH=244 → 256). +// 2. Rotate via Hadamard (hadamard.js) so energy is spread evenly. +// 3. Take the sign bit of each rotated component → 1 bit per dim. +// 4. Pack into a Uint32Array (D/32 words — 256/32 = 8 words, 32 bytes). +// 5. Stash a small float residual (RESIDUAL_DIM floats) so a rerank pass +// can pull back the float tail when we need tighter ranking. +// +// Distance primitives: +// - hammingDistance(a, b): popcount of XOR. Range [0, D]. +// - estimatedCosine(a, b): uses the sign-bit → angle relationship. +// For rotated, mean-zero inputs the expected Hamming distance between +// two hashes is (D / π) · θ where θ is the angle between originals. +// So estimatedCosine = cos(π · hamming / D). This is the standard +// SimHash / RaBitQ recovery formula. +// +// Determinism: the Hadamard matrix is data-independent, so quantize(v) is +// a pure function of v. No RNG, no training, no per-dataset codebook. + +import { nextPow2, padToPow2, fwht } from './hadamard.js'; + +// Size of the float residual we keep alongside the 1-bit code. This is NOT +// the top-k rerank set — it's a small fingerprint of the rotated vector's +// first RESIDUAL_DIM coordinates, which a future reranker can use to break +// ties among Hamming-equidistant candidates. 16 floats × 4 bytes = 64 B. +// Combined with the 32 B bit-code that's 96 B/brain vs. 244×4 = 976 B for +// the float baseline — ~10× shrink including the residual, ~30× without. +export const RESIDUAL_DIM = 16; + +// Public: quantize a Float32Array of arbitrary length. +// Returns { packed: Uint32Array, residual: Float32Array, rotatedDim }. +// `rotatedDim` is the padded length (a power of 2) so the consumer knows +// how many bits are meaningful. +export function quantize(flat) { + const rotatedDim = nextPow2(flat.length); + const rotated = padToPow2(flat, rotatedDim); + fwht(rotated); + + const wordCount = rotatedDim >>> 5; // D/32 + const packed = new Uint32Array(wordCount); + for (let i = 0; i < rotatedDim; i++) { + // sign bit: positive (including +0) → 1, negative → 0. JS -0 has the + // sign bit set but compares === 0; we explicitly treat 0 as positive + // so the output is deterministic regardless of input sign-of-zero. + if (rotated[i] >= 0) { + packed[i >>> 5] |= 1 << (i & 31); + } + } + + const residualLen = Math.min(RESIDUAL_DIM, rotatedDim); + const residual = new Float32Array(residualLen); + for (let i = 0; i < residualLen; i++) residual[i] = rotated[i]; + + return { packed, residual, rotatedDim }; +} + +// popcount of a 32-bit integer (Wegner / Kernighan-style fused variant). +function popcount32(v) { + v = v - ((v >>> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >>> 2) & 0x33333333); + v = (v + (v >>> 4)) & 0x0f0f0f0f; + return (Math.imul(v, 0x01010101) >>> 24); +} + +// Hamming distance between two packed Uint32Array codes. Undefined if +// lengths differ — caller is responsible for matching rotatedDim. +export function hammingDistance(a, b) { + if (a.length !== b.length) { + throw new Error(`hammingDistance: length mismatch ${a.length} vs ${b.length}`); + } + let d = 0; + for (let i = 0; i < a.length; i++) { + d += popcount32(a[i] ^ b[i]); + } + return d; +} + +// Estimated cosine from Hamming distance. Inverse of the sign-bit / angle +// relationship: Hamming/D ≈ θ/π ⇒ θ ≈ π · H/D ⇒ cos(θ) ≈ cos(π · H/D). +// Returns a value in [-1, 1]; identical codes give cos(0)=1; antipodal +// codes (all bits flipped) give cos(π)=-1. +export function estimatedCosine(a, b, rotatedDim = a.length * 32) { + const h = hammingDistance(a, b); + return Math.cos(Math.PI * h / rotatedDim); +} + +// Convenience: estimated cosine distance = 1 - estimatedCosine. +export function estimatedCosineDistance(a, b, rotatedDim = a.length * 32) { + return 1 - estimatedCosine(a, b, rotatedDim); +} + +// Memory footprint per vector in bytes, excluding JS object overhead. +// Useful for the viewer's "memory meter". Returns { bits, packedBytes, +// residualBytes, totalBytes, floatBaselineBytes, compressionRatio }. +export function memoryFootprint(originalLen) { + const rotatedDim = nextPow2(originalLen); + const packedBytes = (rotatedDim >>> 5) * 4; + const residualBytes = Math.min(RESIDUAL_DIM, rotatedDim) * 4; + const totalBytes = packedBytes + residualBytes; + const floatBaselineBytes = originalLen * 4; + return { + bits: rotatedDim, + packedBytes, + residualBytes, + totalBytes, + floatBaselineBytes, + compressionRatio: floatBaselineBytes / totalBytes, + bitOnlyRatio: floatBaselineBytes / packedBytes, + }; +} diff --git a/AI-Car-Racer/quantization/viewer.js b/AI-Car-Racer/quantization/viewer.js new file mode 100644 index 0000000..8ea809e --- /dev/null +++ b/AI-Car-Racer/quantization/viewer.js @@ -0,0 +1,201 @@ +// quantization/viewer.js +// Phase 1B (F1) — ELI15-facing viewer for the 1-bit quantizer. Pure DOM + +// canvas, no framework. Mount via: +// import { mountViewer } from './quantization/viewer.js'; +// mountViewer(document.getElementById('q-viewer'), samples); +// where `samples` is an array of Float32Array (e.g. a cross-section of +// the brain archive). The viewer renders three panels: +// 1. Side-by-side heatmap: first sample's float components vs its 1-bit +// packed code (shown as a checkerboard of 0/1 cells). +// 2. Scatter plot: for each pair in a subset of samples, plot (true +// cosine, estimated cosine) — the diagonal is the ideal recovery. +// 3. Memory meter: float baseline bytes vs. 1-bit + residual bytes. +// +// Not wired into any tour slot yet — Phase 3B will register it. The file +// exists so the quantization chapter has a live demo element to point at. + +import { quantize, estimatedCosine, memoryFootprint } from './index.js'; + +function cosine(a, b) { + let dot = 0, na = 0, nb = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + na += a[i] * a[i]; + nb += b[i] * b[i]; + } + return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-12); +} + +// Render the float vector + 1-bit code side by side on a single canvas. +function drawHeatmap(canvas, flat, packed, rotatedDim) { + const ctx = canvas.getContext('2d'); + const w = canvas.width, h = canvas.height; + ctx.clearRect(0, 0, w, h); + + // Float half (left 50%): min-max normalize, map to blue↔red. + let min = Infinity, max = -Infinity; + for (let i = 0; i < flat.length; i++) { + if (flat[i] < min) min = flat[i]; + if (flat[i] > max) max = flat[i]; + } + const span = max - min || 1; + const cols = 16; + const rows = Math.ceil(flat.length / cols); + const floatW = Math.floor(w * 0.45); + const cellW = floatW / cols; + const cellH = h / rows; + for (let i = 0; i < flat.length; i++) { + const t = (flat[i] - min) / span; + const r = Math.round(255 * t); + const b = Math.round(255 * (1 - t)); + ctx.fillStyle = `rgb(${r}, 60, ${b})`; + const col = i % cols; + const row = Math.floor(i / cols); + ctx.fillRect(col * cellW, row * cellH, cellW - 0.5, cellH - 0.5); + } + + // Divider + labels + ctx.fillStyle = '#888'; + ctx.font = '10px sans-serif'; + ctx.fillText('float (244 × 32-bit)', 4, h - 4); + + // Bit half (right 45%, 16 cols × 16 rows = 256 bits). + const bitCols = 16; + const bitRows = Math.ceil(rotatedDim / bitCols); + const bitW = Math.floor(w * 0.45); + const bitX0 = Math.floor(w * 0.52); + const bcW = bitW / bitCols; + const bcH = h / bitRows; + for (let i = 0; i < rotatedDim; i++) { + const bit = (packed[i >>> 5] >>> (i & 31)) & 1; + ctx.fillStyle = bit ? '#e8b739' : '#2a2a2a'; + const col = i % bitCols; + const row = Math.floor(i / bitCols); + ctx.fillRect(bitX0 + col * bcW, row * bcH, bcW - 0.5, bcH - 0.5); + } + ctx.fillStyle = '#888'; + ctx.fillText('1-bit (256 × 1-bit)', bitX0 + 4, h - 4); +} + +// Scatter of (true cosine, estimated cosine) over all pairs from samples. +function drawScatter(canvas, samples, codes, rotatedDim) { + const ctx = canvas.getContext('2d'); + const w = canvas.width, h = canvas.height; + ctx.clearRect(0, 0, w, h); + + // Axes: x=true cosine [-1, 1], y=estimated cosine [-1, 1]. + ctx.strokeStyle = '#444'; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(w / 2, 0); ctx.lineTo(w / 2, h); + ctx.moveTo(0, h / 2); ctx.lineTo(w, h / 2); + ctx.stroke(); + + // Ideal diagonal y=x. + ctx.strokeStyle = '#7a7'; + ctx.setLineDash([3, 3]); + ctx.beginPath(); + ctx.moveTo(0, h); ctx.lineTo(w, 0); + ctx.stroke(); + ctx.setLineDash([]); + + ctx.fillStyle = 'rgba(232, 183, 57, 0.55)'; + const n = samples.length; + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + const tc = cosine(samples[i], samples[j]); + const ec = estimatedCosine(codes[i], codes[j], rotatedDim); + const x = (tc + 1) * 0.5 * w; + const y = h - (ec + 1) * 0.5 * h; + ctx.fillRect(x - 1.5, y - 1.5, 3, 3); + } + } + + ctx.fillStyle = '#aaa'; + ctx.font = '10px sans-serif'; + ctx.fillText('x: true cosine · y: estimated · dashed = perfect', 6, h - 6); +} + +// Plain-DOM memory meter. +function renderMeter(container, footprint) { + const { + packedBytes, residualBytes, totalBytes, + floatBaselineBytes, compressionRatio, bitOnlyRatio, + } = footprint; + + const frag = document.createElement('div'); + frag.style.cssText = 'font: 12px sans-serif; color: #ccc; margin: 8px 0;'; + + const barMax = 320; + const floatW = barMax; + const totalW = Math.max(4, Math.round((totalBytes / floatBaselineBytes) * barMax)); + const packedW = Math.max(2, Math.round((packedBytes / floatBaselineBytes) * barMax)); + + frag.innerHTML = ` +
Memory per vector
+
+ float + + ${floatBaselineBytes} B +
+
+ 1-bit + + ${packedBytes} B (${bitOnlyRatio.toFixed(1)}× smaller) +
+
+ +residual + + ${totalBytes} B (${compressionRatio.toFixed(1)}× smaller) +
+ `; + container.appendChild(frag); +} + +// Public entry point. `samples` is an array of Float32Array. The viewer +// tolerates an empty or undersized sample list by rendering placeholder +// text instead of crashing. +export function mountViewer(container, samples) { + if (!container) throw new Error('mountViewer: container required'); + container.innerHTML = ''; + container.style.cssText = 'display:flex; flex-direction:column; gap:10px; padding:8px; background:#181818; color:#ddd; border-radius:6px;'; + + const title = document.createElement('div'); + title.textContent = '1-bit archive — float vs. quantized'; + title.style.cssText = 'font: 600 13px sans-serif; color:#eee;'; + container.appendChild(title); + + if (!samples || samples.length === 0) { + const empty = document.createElement('div'); + empty.style.cssText = 'font: 12px sans-serif; color:#888;'; + empty.textContent = 'No samples provided. Train through at least one track and reopen.'; + container.appendChild(empty); + return; + } + + const codes = []; + let rotatedDim = 0; + for (let i = 0; i < samples.length; i++) { + const q = quantize(samples[i]); + codes.push(q.packed); + rotatedDim = q.rotatedDim; + } + + // Heatmap + const hmCanvas = document.createElement('canvas'); + hmCanvas.width = 480; hmCanvas.height = 140; + hmCanvas.style.cssText = 'width:100%; max-width:480px; background:#000; border-radius:4px;'; + container.appendChild(hmCanvas); + drawHeatmap(hmCanvas, samples[0], codes[0], rotatedDim); + + // Scatter (use first ~40 samples to keep pair count manageable) + const scCanvas = document.createElement('canvas'); + scCanvas.width = 300; scCanvas.height = 300; + scCanvas.style.cssText = 'width:100%; max-width:300px; background:#000; border-radius:4px;'; + container.appendChild(scCanvas); + const scatterN = Math.min(samples.length, 40); + drawScatter(scCanvas, samples.slice(0, scatterN), codes.slice(0, scatterN), rotatedDim); + + // Memory meter + renderMeter(container, memoryFootprint(samples[0].length)); +} diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md index c9ddcf4..6d1f5a2 100644 --- a/docs/plan/rulake-inspired-features.md +++ b/docs/plan/rulake-inspired-features.md @@ -24,7 +24,7 @@ pointing to the blocker. Keep the "Current focus" line at the top pointing at whichever phase is active so a newcomer knows where to jump in without reading the whole doc. -**Current focus:** Phase 1 — parallel feature implementations (ready to swarm) +**Current focus:** Phase 1 wave 2 — 1A + 1C (sequential, both touch ruvectorBridge.js) **Last updated:** 2026-04-24 ### Phase 0 — Foundations _(sequential, 1 owner)_ @@ -44,9 +44,9 @@ jump in without reading the whole doc. | Status | ID | Task | Owner | PR/SHA | Done date | |:--:|:--:|------|-------|--------|-----------| | ⬜ | 1A | F3 — Warm-restart bundles + shareable snapshots | | | | -| ⬜ | 1B | F1 — 1-bit quantized archive (RaBitQ + Hadamard) | | | | +| ✅ | 1B | F1 — 1-bit quantized archive (RaBitQ + Hadamard) | Claude (subagent) | — | 2026-04-24 | | ⬜ | 1C | F4 — Consistency modes (Fresh/Eventual/Frozen) | | | | -| ⬜ | 1D | F5 — Content-addressed dedup + hash-keyed lineage | | | | +| ✅ | 1D | F5 — Content-addressed dedup + hash-keyed lineage | Claude (subagent) | — | 2026-04-24 | **Phase 1 gate:** all four rows ✅ + each passes its own `/ship-task` confidence gate + cross-track-variance memory applied for recall/speed @@ -88,6 +88,15 @@ community-archive URL ships publicly). Append-only. Record any scope change, deferral, or non-obvious call that future-you would want to find. Newest at the top. +- **2026-04-24 — Phase 1 wave 1 closed (1B + 1D).** Dispatched as two + parallel general-purpose subagents in a single message; zero file + overlap thanks to Phase 0's pre-partition. Live browser validation: + `tests/quantization-recall.html` prints recall@10 = 0.9060 (≥ 0.9 gate); + `tests/dedup-smoke.html` prints 3/3 PASS. Main app boots cleanly after + edits to lineage/dag.js, lineage/viewer.js, brainExport.js. 1A + 1C + deferred to wave 2 because both edit ruvectorBridge.js stubs and would + race. + - **2026-04-24 — Phase 0 closed.** 0.4 shipped 7 stubs (not 6 as the tracker originally said) — the plan body referenced 7 distinct chapter files, so the original "6" was a typo. Tracker + task description now diff --git a/tests/quantization-recall.html b/tests/quantization-recall.html new file mode 100644 index 0000000..b76388d --- /dev/null +++ b/tests/quantization-recall.html @@ -0,0 +1,148 @@ + + + + + Quantization recall@10 — Phase 1B harness + + + +

Phase 1B (F1) — 1-bit quantizer recall@10

+

Standalone sanity check. Generates 100 synthetic 244-dim brain-like vectors, + computes top-10 neighbours by true cosine and by Hamming on the quantized codes, and prints + recall@10. The done criteria is recall@10 ≥ 0.9.

+ +
+ + + + From ee36b448f2d170daf2b68eb16dfc27a7711b8163 Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 12:54:48 -0400 Subject: [PATCH 04/10] feat(rulake-phase-1a): warm-restart bundles (archive export/import) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F3 from docs/plan/rulake-inspired-features.md. Serializes the whole brain/track/dynamics archive to a gzipped JSON bundle (.vvarchive.json.gz) and rehydrates it deterministically via replay-of-insertions — no WASM patch needed. Behind ?snapshots=1 while it bakes. What this ships: - archive/exporter.js — buildSnapshot({...}) → ArchiveSnapshot with xxHash32 witness (sync, click-handler safe). Also buildSnapshotAsync with crypto.subtle SHA-256 witness when an async path is acceptable. - archive/importer.js — applySnapshot(snapshot, targets) replays into fresh VectorDB instances, rebuilds mirrors, re-hydrates the lineage DAG. Respects insertionOrder; falls back to row order if absent. - archive/serialize.js — toBlob/fromBlob via CompressionStream('gzip') with plain-JSON fallback for older Safari. Magic header "VVARCHIVE v1 {gzip|json}" so truncated downloads fail loudly at import. - ruvectorBridge.js — exportSnapshot/importSnapshot stubs replaced with real bodies. _insertionOrder: string[] now tracked alongside the mirrors — pushed by archiveBrain, hydrate, hydrateFromFixture; cleared by _debugReset and importSnapshot; consumed by rebuildIndicesFromMirror so setIndexKind preserves graph identity. setConsistencyMode/ getIndexStats stubs unchanged (1C / 3A own those). - uiPanels.js — flag-gated "📦 Export archive" / "📥 Import archive" row. Renders only when ?snapshots=1; default UI unchanged. - main.js — after bridge.ready(), if ?snapshots=1 and a staged import is pending, runs it pre-sim. Fire-and-forget; logs counts. - eli15/chapters/warm-restart.js — full chapter replaces coming-soon stub. - tests/warm-restart-roundtrip.html — standalone harness. Seeds fixture, exports → toBlob → fromBlob → importSnapshot, asserts 5 claims. Validated via agent-browser: - Round-trip harness prints PASS 5/5: validateSnapshot ok, importSnapshot returns right counts (brains=8 tracks=1), brain count preserved (pre=8 post=8), top-1 id byte-identical (brain-7 → brain-7), id set byte-identical. gzip=true (CompressionStream path exercised, not JSON fallback). - Default URL (no flag): main app boots cleanly, no new errors, Export/ Import buttons absent. - Flag URL (?snapshots=1): both buttons render in the training panel. Not yet integrated with F5 dedup (import still re-inserts rather than skipping duplicates) — that's a later integration slice. Not yet integrated with F4 consistency (snapshot.consistency defaults to 'fresh'). --- AI-Car-Racer/archive/exporter.js | 209 ++++++++++++++++++ AI-Car-Racer/archive/importer.js | 126 +++++++++++ AI-Car-Racer/archive/serialize.js | 131 +++++++++++ AI-Car-Racer/eli15/chapters/warm-restart.js | 83 +++++-- AI-Car-Racer/main.js | 34 +++ AI-Car-Racer/ruvectorBridge.js | 88 +++++++- AI-Car-Racer/uiPanels.js | 96 ++++++++ docs/plan/rulake-inspired-features.md | 4 +- tests/warm-restart-roundtrip.html | 230 ++++++++++++++++++++ 9 files changed, 975 insertions(+), 26 deletions(-) create mode 100644 AI-Car-Racer/archive/exporter.js create mode 100644 AI-Car-Racer/archive/importer.js create mode 100644 AI-Car-Racer/archive/serialize.js create mode 100644 tests/warm-restart-roundtrip.html diff --git a/AI-Car-Racer/archive/exporter.js b/AI-Car-Racer/archive/exporter.js new file mode 100644 index 0000000..f8af7d9 --- /dev/null +++ b/AI-Car-Racer/archive/exporter.js @@ -0,0 +1,209 @@ +// archive/exporter.js +// Phase 1A (F3) — build a validated ArchiveSnapshot from the live bridge +// mirrors. Deterministic-replay path only: we do NOT ship the native HNSW +// bytes (that's the upstream-patch dance flagged in +// docs/plan/ruvector-upstream-patches.md). Instead we record the insertion +// order and let the importer re-insert in the same order on the other side +// to reproduce graph connectivity byte-for-byte. +// +// API +// buildSnapshot({ +// brainMirror, Map +// trackMirror, Map +// dynamicsMirror, Map +// observations, Map +// indexKind, 'euclidean' | 'hyperbolic' +// insertionOrder, string[] — insertion order of brain ids +// consistency, optional: 'fresh' | 'eventual' | 'frozen' (default 'fresh') +// dim, optional: flat-vector dimensionality (for hnsw.params) +// }) → ArchiveSnapshot +// +// The `witness` field is sha-256 hex of the canonicalized payload. Computed +// synchronously via a plain JS sha-256 implementation so buildSnapshot can +// stay synchronous (the serialize/fromBlob path is already async; piping +// witness through another Promise layer was a needless API complication). +// A crypto.subtle-based async variant is exported as `buildSnapshotAsync` +// for callers who prefer the web-standard digest. + +import { ARCHIVE_SCHEMA_VERSION, validateSnapshot } from './snapshot.js'; +import { xxHash32Bytes } from './hash.js'; + +// ─── witness ───────────────────────────────────────────────────────────── +// We canonicalize the payload ourselves (stable key order, Float32Array → +// plain Array, Map iteration order preserved) and then hash the JSON string. +// Strategy: prefer sha-256 via crypto.subtle when available (all modern +// browsers since ~2018), fall back to xxHash32 (non-crypto, already vendored +// in archive/hash.js) when running in an environment without subtle. The +// fallback is documented loudly — an attacker doesn't change the behaviour +// of the importer, they just need to produce a matching witness, so xxHash32 +// is "good enough" for the self-check use-case the field was created for. + +function _canonicalBrainRows(mirror) { + const rows = []; + for (const [id, { vector, meta }] of mirror) { + rows.push({ + id: String(id), + flat: Array.from(vector), // JSON-serializable; reader rebuilds a Float32Array + meta: meta || {}, + }); + } + return rows; +} + +function _canonicalVecRows(mirror) { + const rows = []; + for (const [id, { vector, meta }] of mirror) { + rows.push({ id: String(id), vec: Array.from(vector), meta: meta || {} }); + } + return rows; +} + +function _canonicalObsRows(observations) { + const rows = []; + for (const [id, { weight, count }] of observations) { + rows.push({ id: String(id), weight: Number(weight) || 0, count: count | 0 }); + } + return rows; +} + +// Synchronous witness. Uses xxHash32Bytes over the UTF-8 bytes of the +// canonical JSON string when crypto.subtle isn't available or the caller +// refuses to go async. Returned as "x32:" so consumers can distinguish +// the fallback from a real sha-256 hex string at a glance. +function _witnessSync(canonicalJson) { + const enc = (typeof TextEncoder !== 'undefined') + ? new TextEncoder().encode(canonicalJson) + : _utf8Encode(canonicalJson); + const h = xxHash32Bytes(enc); + return 'x32:' + h.toString(16).padStart(8, '0'); +} + +async function _witnessAsync(canonicalJson) { + if (typeof crypto === 'undefined' || !crypto.subtle || !crypto.subtle.digest) { + return _witnessSync(canonicalJson); + } + const enc = (typeof TextEncoder !== 'undefined') + ? new TextEncoder().encode(canonicalJson) + : _utf8Encode(canonicalJson); + const buf = await crypto.subtle.digest('SHA-256', enc); + const bytes = new Uint8Array(buf); + let hex = ''; + for (let i = 0; i < bytes.length; i++) hex += bytes[i].toString(16).padStart(2, '0'); + return 'sha256:' + hex; +} + +// Minimal UTF-8 encoder fallback for environments without TextEncoder (very +// old Safari in strict-sandbox mode). Keeps the module usable even in the +// oldest harness. +function _utf8Encode(str) { + const out = []; + for (let i = 0; i < str.length; i++) { + let c = str.charCodeAt(i); + if (c < 0x80) out.push(c); + else if (c < 0x800) { out.push(0xc0 | (c >> 6), 0x80 | (c & 0x3f)); } + else if (c >= 0xd800 && c < 0xdc00 && i + 1 < str.length) { + const c2 = str.charCodeAt(++i); + const cp = 0x10000 + (((c & 0x3ff) << 10) | (c2 & 0x3ff)); + out.push(0xf0 | (cp >> 18), 0x80 | ((cp >> 12) & 0x3f), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)); + } else { + out.push(0xe0 | (c >> 12), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } + } + return new Uint8Array(out); +} + +// Build the canonical JSON (excluding the witness field — the witness hashes +// everything else). Using a fixed key order so two sessions that assemble +// equivalent mirrors produce byte-identical payloads. +function _canonicalJson(core) { + const ordered = { + version: core.version, + createdAt: core.createdAt, + consistency: core.consistency, + brains: core.brains, + tracks: core.tracks, + dynamics: core.dynamics, + observations: core.observations, + hnsw: { + mode: core.hnsw.mode, + serialized: core.hnsw.serialized, // always null in replay mode + insertionOrder: core.hnsw.insertionOrder, + params: core.hnsw.params, + }, + }; + return JSON.stringify(ordered); +} + +// Shared core builder — used by both sync and async variants so the +// canonicalization path stays identical. +function _buildCore(opts) { + if (!opts || typeof opts !== 'object') { + throw new Error('buildSnapshot: missing options object'); + } + const { + brainMirror, trackMirror, dynamicsMirror, observations, + indexKind = 'euclidean', + insertionOrder = [], + consistency = 'fresh', + dim = null, + } = opts; + if (!(brainMirror instanceof Map)) throw new Error('buildSnapshot: brainMirror must be a Map'); + if (!(trackMirror instanceof Map)) throw new Error('buildSnapshot: trackMirror must be a Map'); + if (!(dynamicsMirror instanceof Map)) throw new Error('buildSnapshot: dynamicsMirror must be a Map'); + if (!(observations instanceof Map)) throw new Error('buildSnapshot: observations must be a Map'); + + // Filter insertionOrder to ids still present in the mirror. The bridge + // sometimes _debugResets mid-session; an id in the order list with no + // mirror entry is just noise. + const order = Array.isArray(insertionOrder) + ? insertionOrder.filter((id) => brainMirror.has(id)) + : []; + + const resolvedDim = Number.isFinite(dim) && dim > 0 + ? (dim | 0) + : (brainMirror.size > 0 + ? (brainMirror.values().next().value.vector.length | 0) + : 0); + + return { + version: ARCHIVE_SCHEMA_VERSION, + createdAt: new Date().toISOString(), + consistency, + brains: _canonicalBrainRows(brainMirror), + tracks: _canonicalVecRows(trackMirror), + dynamics: _canonicalVecRows(dynamicsMirror), + observations: _canonicalObsRows(observations), + hnsw: { + mode: 'replay', + serialized: null, + insertionOrder: order, + params: { dim: resolvedDim, metric: 'cosine', indexKind }, + }, + }; +} + +// Public sync API. `witness` uses xxHash32 (tagged "x32:...") — good enough +// for the self-check role, synchronous so the call site doesn't have to +// await. Validates the result before returning so a malformed snapshot can +// never leak out of this module. +export function buildSnapshot(opts) { + const core = _buildCore(opts); + const json = _canonicalJson(core); + core.witness = _witnessSync(json); + const v = validateSnapshot(core); + if (!v.ok) throw new Error(`buildSnapshot: produced invalid snapshot (${v.reason})`); + return core; +} + +// Public async API — uses crypto.subtle sha-256 when available. Useful when +// the caller wants the stronger self-check; falls back to the sync path on +// insecure contexts (file://, old Safari) without surfacing the difference +// except through the witness-string prefix. +export async function buildSnapshotAsync(opts) { + const core = _buildCore(opts); + const json = _canonicalJson(core); + core.witness = await _witnessAsync(json); + const v = validateSnapshot(core); + if (!v.ok) throw new Error(`buildSnapshotAsync: produced invalid snapshot (${v.reason})`); + return core; +} diff --git a/AI-Car-Racer/archive/importer.js b/AI-Car-Racer/archive/importer.js new file mode 100644 index 0000000..8ed0067 --- /dev/null +++ b/AI-Car-Racer/archive/importer.js @@ -0,0 +1,126 @@ +// archive/importer.js +// Phase 1A (F3) — replay-mode warm-restart importer. Takes an +// ArchiveSnapshot produced by archive/exporter.js, validates it, clears the +// caller-provided indexes + mirrors, and re-inserts every brain in the +// recorded insertion order so the HNSW graph rebuilds to a byte-identical +// shape. Tracks, dynamics, and observations are loaded in their own stable +// orders (sorted by id) — those indexes aren't queried by the reranker in +// ways that depend on graph connectivity, so the recorded order isn't +// captured for them. +// +// API +// applySnapshot(snapshot, { +// brainDB, trackDB, dynamicsDB, live VectorDB instances (bridge owns) +// brainMirror, trackMirror, dynamicsMirror, observations, live Maps +// }) → { ok: true, counts: { brains, tracks, dynamics, observations } } +// +// Throws on validation failure. Does NOT touch IndexedDB — the bridge's +// schedulePersist() will pick up the new state on its own timer. + +import { validateSnapshot } from './snapshot.js'; + +function _toFloat32(v) { + if (v instanceof Float32Array) return v; + if (Array.isArray(v)) return new Float32Array(v); + if (v && typeof v.buffer !== 'undefined') { + return new Float32Array(v.buffer, v.byteOffset || 0, (v.byteLength || 0) / 4); + } + return new Float32Array(0); +} + +export function applySnapshot(snapshot, targets) { + const v = validateSnapshot(snapshot); + if (!v.ok) throw new Error(`applySnapshot: invalid snapshot (${v.reason})`); + if (!targets) throw new Error('applySnapshot: missing targets'); + const { + brainDB, trackDB, dynamicsDB, + brainMirror, trackMirror, dynamicsMirror, observations, + } = targets; + for (const [name, obj] of [ + ['brainDB', brainDB], ['trackDB', trackDB], ['dynamicsDB', dynamicsDB], + ]) { + if (!obj) throw new Error(`applySnapshot: missing target ${name}`); + } + for (const [name, obj] of [ + ['brainMirror', brainMirror], ['trackMirror', trackMirror], + ['dynamicsMirror', dynamicsMirror], ['observations', observations], + ]) { + if (!(obj instanceof Map)) throw new Error(`applySnapshot: target ${name} must be a Map`); + } + + // Wipe mirrors first. We do NOT recreate the VectorDB instances here — + // the caller (ruvectorBridge) rebuilds them ahead of time when the index + // kind might change. For the pure replay path the VectorDBs passed in + // are assumed to be freshly constructed and empty. The bridge's wrapper + // in exportSnapshot/importSnapshot handles that invariant. + brainMirror.clear(); + trackMirror.clear(); + dynamicsMirror.clear(); + observations.clear(); + + // Tracks first — brain meta references trackId, dynamics mirrors pop next + // because brain meta may reference dynamicsId; identical ordering to + // ruvectorBridge.hydrate() for consistency. + const trackRows = Array.isArray(snapshot.tracks) ? snapshot.tracks : []; + for (const row of trackRows) { + const vec = _toFloat32(row.vec || row.flat); + if (vec.length === 0) continue; + trackDB.insert(vec, row.id, row.meta || {}); + trackMirror.set(row.id, { vector: vec, meta: row.meta || {} }); + } + + const dynRows = Array.isArray(snapshot.dynamics) ? snapshot.dynamics : []; + for (const row of dynRows) { + const vec = _toFloat32(row.vec || row.flat); + if (vec.length === 0) continue; + dynamicsDB.insert(vec, row.id, row.meta || {}); + dynamicsMirror.set(row.id, { vector: vec, meta: row.meta || {} }); + } + + // Brains: walk insertionOrder so the HNSW graph rebuilds byte-for-byte; + // fall back to the row order when the snapshot was hand-crafted without a + // captured order (older fixtures / unit tests). + const byId = new Map(); + const rowBrains = Array.isArray(snapshot.brains) ? snapshot.brains : []; + for (const row of rowBrains) byId.set(row.id, row); + + const order = (snapshot.hnsw && Array.isArray(snapshot.hnsw.insertionOrder)) + ? snapshot.hnsw.insertionOrder.filter((id) => byId.has(id)) + : []; + + // If insertionOrder is empty but we have brain rows, fall back to the row + // iteration order. Better than dropping the brains on the floor. + const walk = order.length > 0 ? order : rowBrains.map((r) => r.id); + + // Seen guard so duplicated ids in insertionOrder don't cause double + // inserts (which would blow up the mirror size counter). + const seen = new Set(); + for (const id of walk) { + if (seen.has(id)) continue; + seen.add(id); + const row = byId.get(id); + if (!row) continue; + const vec = _toFloat32(row.flat || row.vec); + if (vec.length === 0) continue; + brainDB.insert(vec, row.id, row.meta || {}); + brainMirror.set(row.id, { vector: vec, meta: row.meta || {} }); + } + + const obsRows = Array.isArray(snapshot.observations) ? snapshot.observations : []; + for (const row of obsRows) { + observations.set(row.id, { + weight: Number(row.weight) || 0, + count: (row.count | 0), + }); + } + + return { + ok: true, + counts: { + brains: brainMirror.size, + tracks: trackMirror.size, + dynamics: dynamicsMirror.size, + observations: observations.size, + }, + }; +} diff --git a/AI-Car-Racer/archive/serialize.js b/AI-Car-Racer/archive/serialize.js new file mode 100644 index 0000000..b361ef6 --- /dev/null +++ b/AI-Car-Racer/archive/serialize.js @@ -0,0 +1,131 @@ +// archive/serialize.js +// Phase 1A (F3) — wire format for the warm-restart bundle. An archive +// snapshot (see archive/snapshot.js + archive/exporter.js) is written as a +// gzipped JSON blob with a short magic header so a truncated or +// wrong-format download fails loudly at import instead of halfway through +// the replay. We use CompressionStream('gzip') when available (Chromium + +// Firefox + Safari 16.4+), falling back to plain JSON on older browsers — +// documented in the magic header so fromBlob() picks the right decode path. +// +// File layout +// magicLine = "VVARCHIVE v1 \n" +// body = JSON.stringify(snapshot) +// codec = "gzip" | "json" +// +// For the gzip path the file is [magic-line bytes][gzip(body) bytes] — the +// header is *not* compressed so fromBlob() can peek the codec without +// loading a decompressor it doesn't need. + +const MAGIC_PREFIX = 'VVARCHIVE v1 '; +const MAGIC_GZIP = MAGIC_PREFIX + 'gzip\n'; +const MAGIC_JSON = MAGIC_PREFIX + 'json\n'; +const MIME_TYPE = 'application/x-vvarchive'; + +function _supportsGzip() { + return typeof CompressionStream === 'function' + && typeof DecompressionStream === 'function' + && typeof Response === 'function'; +} + +async function _gzipBytes(bytes) { + const stream = new Blob([bytes]).stream().pipeThrough(new CompressionStream('gzip')); + const buf = await new Response(stream).arrayBuffer(); + return new Uint8Array(buf); +} + +async function _gunzipBytes(bytes) { + const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip')); + const buf = await new Response(stream).arrayBuffer(); + return new Uint8Array(buf); +} + +function _encodeUtf8(str) { + if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(str); + // Very-old-Safari fallback. Unicode is fine — JSON body is ASCII except + // for user-supplied meta fields, which are rare in this archive. + const out = []; + for (let i = 0; i < str.length; i++) { + let c = str.charCodeAt(i); + if (c < 0x80) out.push(c); + else if (c < 0x800) out.push(0xc0 | (c >> 6), 0x80 | (c & 0x3f)); + else out.push(0xe0 | (c >> 12), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } + return new Uint8Array(out); +} + +function _decodeUtf8(bytes) { + if (typeof TextDecoder !== 'undefined') return new TextDecoder('utf-8').decode(bytes); + let s = ''; + for (let i = 0; i < bytes.length; i++) s += String.fromCharCode(bytes[i]); + return decodeURIComponent(escape(s)); +} + +// Write a snapshot object to a Blob. Returns Promise. The blob's +// `type` is the VV-archive MIME so a later `saveAs` call on a browser that +// cares about types picks the right handler (most browsers just care about +// the filename extension, but we set both for belt-and-suspenders). +export async function toBlob(snapshot) { + if (!snapshot || typeof snapshot !== 'object') { + throw new Error('serialize.toBlob: snapshot must be an object'); + } + const body = JSON.stringify(snapshot); + if (_supportsGzip()) { + const header = _encodeUtf8(MAGIC_GZIP); + const compressed = await _gzipBytes(_encodeUtf8(body)); + return new Blob([header, compressed], { type: MIME_TYPE }); + } + const header = _encodeUtf8(MAGIC_JSON); + return new Blob([header, _encodeUtf8(body)], { type: MIME_TYPE }); +} + +// Read a Blob into a snapshot object. Rejects if the magic header is +// missing or the codec isn't recognised. The caller should re-validate via +// archive/snapshot.validateSnapshot() — this function only guarantees the +// returned value parses as JSON with the expected wrapper. +export async function fromBlob(blob) { + if (!blob || typeof blob.arrayBuffer !== 'function') { + throw new Error('serialize.fromBlob: argument must be a Blob'); + } + const buf = new Uint8Array(await blob.arrayBuffer()); + // The magic line is ASCII, so we can scan the first ~64 bytes as 1:1 + // codepoints without running a full UTF-8 decode. + let nlIdx = -1; + const scanEnd = Math.min(buf.length, 64); + for (let i = 0; i < scanEnd; i++) { + if (buf[i] === 0x0a) { nlIdx = i; break; } + } + if (nlIdx < 0) { + throw new Error('serialize.fromBlob: missing magic header (not a .vvarchive file?)'); + } + const header = _decodeUtf8(buf.subarray(0, nlIdx + 1)); + if (!header.startsWith(MAGIC_PREFIX)) { + throw new Error('serialize.fromBlob: bad magic header: ' + JSON.stringify(header.slice(0, 40))); + } + const payload = buf.subarray(nlIdx + 1); + let jsonBytes; + if (header === MAGIC_GZIP) { + if (!_supportsGzip()) { + throw new Error('serialize.fromBlob: file is gzip but this browser lacks DecompressionStream'); + } + jsonBytes = await _gunzipBytes(payload); + } else if (header === MAGIC_JSON) { + jsonBytes = payload; + } else { + throw new Error('serialize.fromBlob: unknown codec in header: ' + JSON.stringify(header)); + } + const text = _decodeUtf8(jsonBytes); + try { + return JSON.parse(text); + } catch (e) { + throw new Error('serialize.fromBlob: JSON parse failed: ' + (e.message || e)); + } +} + +// Convenience constant for file-picker filters and downloads. +export const VVARCHIVE_EXTENSION_GZ = '.vvarchive.json.gz'; +export const VVARCHIVE_EXTENSION_JSON = '.vvarchive.json'; +export const VVARCHIVE_MIME = MIME_TYPE; + +// Exposed for test harnesses — lets a test assert which path was used +// without reaching into private state. +export function gzipAvailable() { return _supportsGzip(); } diff --git a/AI-Car-Racer/eli15/chapters/warm-restart.js b/AI-Car-Racer/eli15/chapters/warm-restart.js index a2b3d08..2a124c6 100644 --- a/AI-Car-Racer/eli15/chapters/warm-restart.js +++ b/AI-Car-Racer/eli15/chapters/warm-restart.js @@ -1,22 +1,77 @@ // eli15/chapters/warm-restart.js -// Placeholder — real content ships with Phase 1A (F3). +// Phase 1A (F3) — whole-archive export/import (warm-restart bundles). export default { id: 'warm-restart', title: 'Saving and reopening the whole brain archive', oneLiner: 'A brain archive is a museum — you can save it, reopen it tomorrow, or give the whole museum to a friend.', - comingSoon: true, body: [ - '

Coming soon — lands with Phase 1A of the RuLake-inspired roadmap.

', - '

Today the archive rebuilds itself from IndexedDB every page load. Every', - 'brain is re-inserted into HNSW one by one, which is fine at 50 brains', - 'and painful at 5,000. The fix is to save the graph\'s state itself,', - 'not just the raw vectors, and restore it byte-for-byte on the next load —', - 'the same way your laptop reopens yesterday\'s tabs instead of rebuilding', - 'them from scratch.

', - '

Bonus: once the archive is a file, you can share it. Export the', - 'archive, send the file to a friend, and their site will race against your', - 'pre-trained population on the first generation.

', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 1A.

', + '

Imagine every generation you train drops off another brain at a', + 'museum. After a while the museum has hundreds of brains — each one a', + 'tiny record of what worked on which track. That museum is this', + 'app\'s archive. It\'s how a later generation can warm-start from a', + 'cousin that already learned a similar corner, instead of relearning', + 'from scratch.

', + + '

Up to now the museum only lived inside your browser. The moment you', + 'opened an incognito tab, or cleared site data, or opened the page on', + 'a friend\'s laptop, the museum was empty again. Warm-restart', + 'bundles fix that. You can download the entire museum as a', + 'single file, re-open it on another day, or email it to someone else', + 'so their browser starts with your pre-trained population.

', + + '

What\'s actually in the file?

', + '

Three things:

', + '
    ', + '
  • Every archived brain\'s flat weight vector (the', + ' 244 numbers that define that neural net) plus its metadata —', + ' fitness, generation, parents, which track it trained on.
    • ', + '
  • Every unique track fingerprint the archive has', + ' seen (the 512-dim CNN embedding of the track shape), so the', + ' retrieval system can match "is the current track like anything we', + ' have?" on the other side.
    • ', + '
  • Every archived driving-dynamics vector — how', + ' a brain actually drove during its run, not just what track it was', + ' on. These are the tiebreakers for retrieval.
    • ', + '
', + + '

Replay vs. native-serialization

', + '

There are two ways to save a vector index to disk. The', + 'native way is to dump the HNSW graph\'s internal', + 'pointers and re-mount them on import — fast, but it requires the', + 'underlying wasm library to expose a serialize hook. Ours doesn\'t yet', + '(it\'s an upstream patch we haven\'t merged). So we do the other', + 'thing: we write down the insertion order of every', + 'brain, and on import we replay the inserts in that exact', + 'order. HNSW is deterministic-ish when the order is fixed, so the', + 'rebuilt graph answers the same questions for the same queries. It\'s', + 'slower than a native restore on huge archives — a few hundred', + 'milliseconds for thousands of brains — but it\'s correct today, with', + 'no wasm patch required.

', + + '

The bundle is written as gzip\'d JSON with a short magic header', + '(VVARCHIVE v1 gzip) so a truncated download fails loudly', + 'at import instead of silently loading half a museum. On browsers that', + 'lack the CompressionStream API, the bundle falls back to', + 'plain JSON — the magic header says which. There\'s also a', + 'witness field: a content hash of the rest of the file. If', + 'the hash doesn\'t match on import, the file got corrupted in transit', + 'and the importer bails before it touches your live archive.

', + + '

How to use it

', + '

Warm-restart is hidden behind a URL flag while it bakes — visit', + 'this page with ?snapshots=1 appended (e.g.', + '…/index.html?snapshots=1) and two new buttons will', + 'appear in the training panel: 📦 Export archive (saves', + 'a .vvarchive.json.gz file to your Downloads folder) and', + '📥 Import archive (reads one back in). Without the', + 'flag, the app behaves exactly as before — no new buttons, no new', + 'behaviour.

', + + '

Try it: train one generation, click Export. Clear the archive from', + 'the console (await window.__rvBridge._debugReset()),', + 'reload the page with ?snapshots=1, click Import, pick', + 'the file you just downloaded. The same retrieval you ran before the', + 'reset will come back with the same top hit, down to the byte-identity', + 'of the brain id. The museum reopens.

', ].join('\n'), }; diff --git a/AI-Car-Racer/main.js b/AI-Car-Racer/main.js index 7d077ba..9697e00 100644 --- a/AI-Car-Racer/main.js +++ b/AI-Car-Racer/main.js @@ -294,6 +294,40 @@ function bridgeReady(){ return !!(b && b.info && b.info().ready && window.__rvUnflatten); } +// Phase 1A (F3) — staged warm-restart import. The UI file picker in +// uiPanels.js runs its own import directly, but we also support a "load on +// boot" path: if `?snapshots=1` is present AND `window.__pendingSnapshotImport` +// is set (by a tour step, a test harness, or an earlier UI click that +// deferred the import), we replay that snapshot before the sim starts. Any +// errors are logged + non-fatal — the app still boots. +async function __runStagedSnapshotImport(){ + try { + var usp = new URLSearchParams(window.location.search || ''); + if (usp.get('snapshots') !== '1') return; + } catch (_) { return; } + var staged = window.__pendingSnapshotImport; + if (!staged) return; + var b = window.__rvBridge; + if (!b || typeof b.ready !== 'function' || typeof b.importSnapshot !== 'function') return; + try { + await b.ready(); + var res = b.importSnapshot(staged); + var c = (res && res.counts) || null; + console.log('[ruvector] staged snapshot import complete', c); + } catch (e) { + console.warn('[ruvector] staged snapshot import failed', e); + } finally { + try { delete window.__pendingSnapshotImport; } catch (_) { window.__pendingSnapshotImport = null; } + } +} +// Fire-and-forget: the import runs in parallel with the normal boot. Sim +// start doesn't depend on this promise — workers seed from the archive +// lazily, so even if import lands after begin() the next generation will +// still pick up the imported brains. +if (typeof window !== 'undefined') { + __runStagedSnapshotImport(); +} + // ----------------------------------------------------------------------------- // Metrics HUD — per-generation survival %, median / p90 checkpoints, wall-bumps. // Also serves as the on-screen data source for __runBenchmark / __abTest CSVs. diff --git a/AI-Car-Racer/ruvectorBridge.js b/AI-Car-Racer/ruvectorBridge.js index 5f0e1e7..d413afc 100644 --- a/AI-Car-Racer/ruvectorBridge.js +++ b/AI-Car-Racer/ruvectorBridge.js @@ -215,8 +215,16 @@ function rebuildIndicesFromMirror() { for (const [id, { vector, meta }] of _dynamicsMirror) { _dynamicsDB.insert(vector, id, meta || {}); } - for (const [id, { vector, meta }] of _brainMirror) { - _brainDB.insert(vector, id, meta || {}); + // F3 — rebuild preserves the original insertion order from _insertionOrder + // when available; falls back to mirror iteration order otherwise. We do + // NOT reset _insertionOrder here since the mirror contents are unchanged. + const rebuildOrder = (_insertionOrder.length > 0 + ? _insertionOrder.filter((id) => _brainMirror.has(id)) + : Array.from(_brainMirror.keys())); + for (const id of rebuildOrder) { + const entry = _brainMirror.get(id); + if (!entry) continue; + _brainDB.insert(entry.vector, id, entry.meta || {}); } } @@ -234,6 +242,13 @@ const _brainMirror = new Map(); // id -> { vector: Float32Array, meta } const _trackMirror = new Map(); // id -> { vector: Float32Array, meta } const _dynamicsMirror = new Map(); // id -> { vector: Float32Array, meta } const _observations = new Map(); // brainId -> { weight, count } +// Phase 1A (F3). Replay-mode warm-restart needs the same insertion order +// on import that we used on export; VectorDB doesn't expose this after the +// fact, so we track it ourselves. Append-only — any id appearing here that +// isn't in _brainMirror at export time (e.g. after a _debugReset wiped the +// mirror but left this array alone — which _debugReset also clears below) +// gets filtered out in archive/exporter.js. +let _insertionOrder = []; let _persistTimer = null; let _persistInFlight = null; @@ -409,6 +424,8 @@ export function archiveBrain(brain, fitness, trackVec, generation = 0, parentIds if (dynamicsId !== null) meta.dynamicsId = dynamicsId; const id = _brainDB.insert(vec, null, meta); _brainMirror.set(id, { vector: vec, meta }); + // F3 — remember the insertion order so exportSnapshot can replay it. + _insertionOrder.push(id); // P3.B — incremental DAG add. Safe no-op when the dag wasm didn't load. // The DAG uses meta.parentIds to wire edges; unknown parents (not yet in // the mirror) are silently skipped — same relaxed contract as the legacy @@ -874,6 +891,7 @@ export async function hydrate() { if (vec.length !== FLAT_LENGTH) continue; _brainDB.insert(vec, row.id, row.meta || {}); _brainMirror.set(row.id, { vector: vec, meta: row.meta || {} }); + _insertionOrder.push(row.id); } _tEnd('3d_insert_brains', _tBrains); const _tObs = _tStart(); @@ -1004,6 +1022,7 @@ export function hydrateFromFixture(fixture) { _trackMirror.clear(); _dynamicsMirror.clear(); _observations.clear(); + _insertionOrder = []; // P3.A — respect the current index kind when rebuilding from a fixture. // bench-hnsw.html calls setIndexKind('hyperbolic') before hydrateFromFixture // to exercise the hyperbolic path against the same archive. @@ -1023,6 +1042,7 @@ export function hydrateFromFixture(fixture) { if (vec.length !== FLAT_LENGTH) continue; _brainDB.insert(vec, row.id, row.meta || {}); _brainMirror.set(row.id, { vector: vec, meta: row.meta || {} }); + _insertionOrder.push(row.id); } for (const row of (fixture.observations || [])) { _observations.set(row.id, { weight: row.weight || 0, count: row.count | 0 }); @@ -1052,24 +1072,71 @@ export function hydrateFromFixture(fixture) { // See docs/plan/rulake-inspired-features.md for the full plan. import { validateSnapshot, CONSISTENCY_MODES } from './archive/snapshot.js'; +import { buildSnapshot } from './archive/exporter.js'; +import { applySnapshot } from './archive/importer.js'; // Phase 0: stored but unread; 1C will wire this into the query path. let _consistencyMode = 'fresh'; -// 1A fills this in. Returns an ArchiveSnapshot (see archive/snapshot.js). +// 1A — F3. Build a whole-archive snapshot (replay mode). Synchronous; the +// witness is computed via xxHash32 (tagged "x32:") so this can be called +// from click handlers without awaiting. Callers that want a sha-256 witness +// can route through archive/exporter.buildSnapshotAsync directly. export function exportSnapshot() { - throw new Error('not implemented: exportSnapshot (Phase 1A / F3)'); + requireReady(); + return buildSnapshot({ + brainMirror: _brainMirror, + trackMirror: _trackMirror, + dynamicsMirror: _dynamicsMirror, + observations: _observations, + indexKind: _indexKind, + insertionOrder: _insertionOrder, + consistency: _consistencyMode, + dim: FLAT_LENGTH, + }); } -// 1A fills this in. Takes an ArchiveSnapshot, reconstructs the in-memory -// indexes + mirrors, returns { ok: true } or throws on validation failure. +// 1A — F3. Replay an ArchiveSnapshot into the live indexes + mirrors. We +// rebuild the VectorDB instances from scratch (same pattern as +// rebuildIndicesFromMirror) to guarantee the imported graph has no stale +// nodes from the pre-import session. Schedules a persist so the next page +// load inherits the imported archive. export function importSnapshot(s) { - // Validate at the boundary even in the stub, so callers can test the - // wiring without waiting for 1A. Throws on bad input; the real importer - // will keep this check and add its reconstruction work afterwards. + requireReady(); const v = validateSnapshot(s); if (!v.ok) throw new Error(`importSnapshot: invalid snapshot (${v.reason})`); - throw new Error('not implemented: importSnapshot (Phase 1A / F3)'); + // Rebuild empty indexes under the current geometry; applySnapshot will + // re-insert every brain in the snapshot's insertionOrder. + const IndexClass = pickIndexClass(_indexKind); + _brainDB = new IndexClass(FLAT_LENGTH, 'cosine'); + _trackDB = new IndexClass(TRACK_DIM, 'cosine'); + _dynamicsDB = new IndexClass(DYNAMICS_DIM, 'cosine'); + _queryDynamicsVec = null; + _rerankerMode = 'none'; + const res = applySnapshot(s, { + brainDB: _brainDB, + trackDB: _trackDB, + dynamicsDB: _dynamicsDB, + brainMirror: _brainMirror, + trackMirror: _trackMirror, + dynamicsMirror: _dynamicsMirror, + observations: _observations, + }); + // Refresh our own insertion-order tracker to match what the importer + // actually replayed. Further archiveBrain calls append to this same array. + _insertionOrder = (s.hnsw && Array.isArray(s.hnsw.insertionOrder)) + ? s.hnsw.insertionOrder.filter((id) => _brainMirror.has(id)) + : Array.from(_brainMirror.keys()); + // Rebuild DAG from the new mirror so lineage queries match the imported + // archive. Safe no-op when the wasm side didn't load. + try { + if (dagIsReady()) { + dagDebugReset(); + dagHydrateFromMirror(_brainMirror); + } + } catch (e) { console.warn('[lineage-dag] import rehydrate failed', e); } + schedulePersist(); + return res; } // 1C fills this in. Accepts 'fresh' | 'eventual' | 'frozen'. @@ -1097,6 +1164,7 @@ export async function _debugReset() { _trackMirror.clear(); _dynamicsMirror.clear(); _observations.clear(); + _insertionOrder = []; _queryDynamicsVec = null; sonaEngineDebugReset(); try { dagDebugReset(); } catch (_) { /* safe to ignore */ } diff --git a/AI-Car-Racer/uiPanels.js b/AI-Car-Racer/uiPanels.js index 1388882..5f806e7 100644 --- a/AI-Car-Racer/uiPanels.js +++ b/AI-Car-Racer/uiPanels.js @@ -197,6 +197,102 @@ '', ].join(''); + // Phase 1A (F3). Warm-restart archive Export/Import row — rendered ONLY + // when the URL has ?snapshots=1 so the default experience is unchanged + // while the feature is baking. The row is a plain DOM append (not baked + // into the innerHTML above) so the gate lives in one readable place. + let _snapshotsFlagOn = false; + try { + if (typeof URLSearchParams === 'function') { + _snapshotsFlagOn = new URLSearchParams(window.location.search || '').get('snapshots') === '1'; + } + } catch (_) { _snapshotsFlagOn = false; } + if (_snapshotsFlagOn) { + const row = document.createElement('div'); + row.className = 'rv-snapshots'; + row.setAttribute('data-rv', 'snapshots'); + row.innerHTML = [ + '
Archive bundle (warm-restart)', + ' ', + '
', + '
', + ' ', + ' ', + ' ', + '
', + '
', + ].join(''); + root.appendChild(row); + + const btnExport = row.querySelector('[data-rv="snapshot-export"]'); + const btnImport = row.querySelector('[data-rv="snapshot-import"]'); + const fileInput = row.querySelector('[data-rv="snapshot-file"]'); + const status = row.querySelector('[data-rv="snapshots-status"]'); + const setStatus = (msg, cls) => { + if (!status) return; + status.textContent = msg || ''; + status.className = 'rv-snapshots-status' + (cls ? ' rv-snapshots-status-' + cls : ''); + }; + + btnExport.addEventListener('click', async function () { + const b = window.__rvBridge; + if (!b || typeof b.exportSnapshot !== 'function') { + setStatus('bridge not ready — wait a moment and try again', 'error'); + return; + } + try { + setStatus('building snapshot…', 'pending'); + const snap = b.exportSnapshot(); + const { toBlob, VVARCHIVE_EXTENSION_GZ, VVARCHIVE_EXTENSION_JSON, gzipAvailable } = + await import('./archive/serialize.js'); + const blob = await toBlob(snap); + const ext = gzipAvailable() ? VVARCHIVE_EXTENSION_GZ : VVARCHIVE_EXTENSION_JSON; + const ts = new Date().toISOString().replace(/[:.]/g, '-').replace(/Z$/, ''); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'vvarchive-' + ts + ext; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setTimeout(() => URL.revokeObjectURL(url), 1000); + const counts = (snap.brains && snap.brains.length) | 0; + setStatus('exported ' + counts + ' brain' + (counts === 1 ? '' : 's') + + ' (' + (gzipAvailable() ? 'gzip' : 'plain-json') + ')', 'ok'); + } catch (e) { + console.warn('[rv-panel] snapshot export failed', e); + setStatus('export failed: ' + (e.message || e), 'error'); + } + }); + + btnImport.addEventListener('click', function () { fileInput.click(); }); + fileInput.addEventListener('change', async function () { + const file = fileInput.files && fileInput.files[0]; + if (!file) return; + const b = window.__rvBridge; + if (!b || typeof b.importSnapshot !== 'function') { + setStatus('bridge not ready — wait a moment and try again', 'error'); + fileInput.value = ''; + return; + } + try { + setStatus('reading bundle…', 'pending'); + const { fromBlob } = await import('./archive/serialize.js'); + const snap = await fromBlob(file); + const res = b.importSnapshot(snap); + const c = (res && res.counts) || { brains: 0, tracks: 0, dynamics: 0, observations: 0 }; + setStatus('imported — brains ' + c.brains + ' · tracks ' + c.tracks + + ' · dynamics ' + c.dynamics + ' · obs ' + c.observations, 'ok'); + console.log('[rv-panel] snapshot import counts', c); + } catch (e) { + console.warn('[rv-panel] snapshot import failed', e); + setStatus('import failed: ' + (e.message || e), 'error'); + } finally { + fileInput.value = ''; + } + }); + } + const el = { info: root.querySelector('[data-rv="info"]'), rerankerMode: root.querySelector('[data-rv="reranker-mode"]'), diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md index 6d1f5a2..9689f54 100644 --- a/docs/plan/rulake-inspired-features.md +++ b/docs/plan/rulake-inspired-features.md @@ -24,7 +24,7 @@ pointing to the blocker. Keep the "Current focus" line at the top pointing at whichever phase is active so a newcomer knows where to jump in without reading the whole doc. -**Current focus:** Phase 1 wave 2 — 1A + 1C (sequential, both touch ruvectorBridge.js) +**Current focus:** Phase 1 wave 2 — 1C only (1A landed; 1C is last remaining Phase 1 task) **Last updated:** 2026-04-24 ### Phase 0 — Foundations _(sequential, 1 owner)_ @@ -43,7 +43,7 @@ jump in without reading the whole doc. | Status | ID | Task | Owner | PR/SHA | Done date | |:--:|:--:|------|-------|--------|-----------| -| ⬜ | 1A | F3 — Warm-restart bundles + shareable snapshots | | | | +| ✅ | 1A | F3 — Warm-restart bundles + shareable snapshots | Claude (subagent) | — | 2026-04-24 | | ✅ | 1B | F1 — 1-bit quantized archive (RaBitQ + Hadamard) | Claude (subagent) | — | 2026-04-24 | | ⬜ | 1C | F4 — Consistency modes (Fresh/Eventual/Frozen) | | | | | ✅ | 1D | F5 — Content-addressed dedup + hash-keyed lineage | Claude (subagent) | — | 2026-04-24 | diff --git a/tests/warm-restart-roundtrip.html b/tests/warm-restart-roundtrip.html new file mode 100644 index 0000000..4fe9e56 --- /dev/null +++ b/tests/warm-restart-roundtrip.html @@ -0,0 +1,230 @@ + + + + + + Warm-restart round-trip (F3) + + + +

Warm-restart round-trip (Phase 1A / F3)

+

Seeds the bridge with a small fixture, exports a snapshot, serializes + it through toBlob / fromBlob, imports back, + and compares.

+
Running…
+ + + +
#checkverdictdetail
+ + + + + + + + From c9df28045495615ea293b8581d73436c8f959b28 Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 13:10:08 -0400 Subject: [PATCH 05/10] feat(rulake-phase-1c): consistency modes (fresh / eventual / frozen) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F4 from docs/plan/rulake-inspired-features.md. Three query-time modes controlling how recommendSeeds sees the archive; closes Phase 1 of the roadmap (only Phase 3A's getIndexStats stub remains in ruvectorBridge). What this ships: - consistency/mode.js — state machine + TTL cache + frozen-id-set ref. Exports getMode/setMode, recordQuery/getCachedResult, freezeArchive/ thawArchive, stats, _debugReset. TTL default 10 generations. - consistency/worker-sync.js — computeFrozenSyncPayload() helper for the A/B baseline worker; postMessage plumbing is a follow-up. - ruvectorBridge.js — setConsistencyMode stub body replaced. One branch added inside recommendSeeds that reads the mode: * fresh → unchanged path (single no-op boolean per candidate when frozenIds is null — fresh-mode behaviour byte-identical to pre-1C) * eventual → cache hit returns memoized result, else fresh query + recordQuery * frozen → filter candidates to ids in the pinned Set Freeze mechanism is the cap-by-insertion-id shortcut (plan's rec): at freeze-entry, snapshot new Set(_brainMirror.keys()); new archiveBrain calls still insert (archive growth uninterrupted), their ids just aren't in the pinned set. Rejected the alternatives (VectorDB clone, parallel read-only index) as 2× memory for no spec benefit. - uiPanels.js — new radio row (Fresh/Eventual/Frozen, default Fresh) with a 30-span tick strip that pulses on cacheMisses — visible evidence of when the archive is actually re-queried. - style.css — .rv-consistency* classes including keyframe pulse. - main.js — ?consistency=fresh|eventual|frozen URL flag, mirrors the ?hhnsw=1 pattern; polls for bridge readiness before applying. - sim-worker.js — new case 'freeze' branch gated on importSnapshot availability; existing A/B setups without frozen-mode unchanged. - eli15/chapters/consistency-modes.js — full chapter with the three modes, the A/B-race motivation for frozen, tick-strip semantics. - tests/consistency-modes-smoke.html — 5-claim harness. Also updates docs/plan/rulake-inspired-features.md to reflect Phase 1 complete; M1 milestone (F1+F3 demoable) checked off. Validated via agent-browser: - Smoke harness PASS 5/5: fresh bypasses cache (0 hits 0 misses), eventual 2nd call is a hit (sameRef=true), frozen excludes post- freeze brain, thaw restores visibility, no throws on valid modes. - Default UI: Fresh mode, all 3 radios + tick strip render, app boots cleanly with no new console errors. - ?consistency=frozen URL flag: bridge reports 'frozen', Frozen radio checked. Phase 1 ordering discipline held: 4 subagents (1B+1D in parallel, 1A→1C sequentially) edited 15+ files, zero merge conflicts. --- AI-Car-Racer/consistency/mode.js | 190 +++++++++++++++++ AI-Car-Racer/consistency/worker-sync.js | 59 ++++++ .../eli15/chapters/consistency-modes.js | 109 ++++++++-- AI-Car-Racer/main.js | 42 ++++ AI-Car-Racer/ruvectorBridge.js | 133 +++++++++++- AI-Car-Racer/sim-worker.js | 27 +++ AI-Car-Racer/style.css | 61 ++++++ AI-Car-Racer/uiPanels.js | 112 ++++++++++ docs/plan/rulake-inspired-features.md | 20 +- tests/consistency-modes-smoke.html | 200 ++++++++++++++++++ 10 files changed, 928 insertions(+), 25 deletions(-) create mode 100644 AI-Car-Racer/consistency/mode.js create mode 100644 AI-Car-Racer/consistency/worker-sync.js create mode 100644 tests/consistency-modes-smoke.html diff --git a/AI-Car-Racer/consistency/mode.js b/AI-Car-Racer/consistency/mode.js new file mode 100644 index 0000000..f54ac13 --- /dev/null +++ b/AI-Car-Racer/consistency/mode.js @@ -0,0 +1,190 @@ +// consistency/mode.js +// Phase 1C (F4) — consistency-mode state machine + query-path cache. +// +// This module is a *stateful singleton*: the bridge consults it on every +// recommendSeeds() call, and the UI radio-row flips it via setMode(). +// +// Three modes (taxonomy mirrored from archive/snapshot.js CONSISTENCY_MODES): +// - fresh — every call hits the live archive. Current behaviour. +// - eventual — cache the last recommendSeeds result, keyed by trackVec; +// reuse for the next TTL_GENERATIONS calls. +// - frozen — pin an archive snapshot at mode-entry; queries only see +// brains that existed at freeze time. +// +// The module itself is deliberately ignorant of *how* the bridge reads +// the live archive. It tracks: current mode, TTL cache, frozen-snapshot +// reference, and counters. The bridge's recommendSeeds decides what to +// do with each signal (see the `fresh | eventual | frozen` branch there). + +import { CONSISTENCY_MODES } from '../archive/snapshot.js'; + +// Default TTL: how many calls a cached eventual-mode result stays valid +// for. Matches the plan's "default 10 generations" under the assumption +// that recommendSeeds fires roughly once per generation. +export const DEFAULT_TTL_GENERATIONS = 10; + +let _mode = 'fresh'; +let _ttl = DEFAULT_TTL_GENERATIONS; + +// Eventual-mode cache. Keyed by trackVecKey (a short fingerprint the +// bridge computes from the query vector — see `trackVecKey()` below for +// the shared helper). Cached entries know when they were stored and how +// many hits they have left. +// Map +const _cache = new Map(); + +// Frozen-mode snapshot reference. The bridge stores a pragmatic anchor +// (e.g. the brain insertion count at freeze time) here; the module just +// holds and returns whatever is handed to it. `_frozenAt` is the ISO +// timestamp the freeze was entered at, exposed via stats(). +let _frozenSnapshot = null; +let _frozenAt = null; + +// Monotonic counter of recommendSeeds() calls. Incremented on every +// getCachedResult call (hit or miss). TTL is expressed in "calls from +// now"; storing `expiresAtCall` up front keeps expiry O(1) on lookup. +let _callCount = 0; +let _cacheHits = 0; +let _cacheMisses = 0; + +// ─── mode ──────────────────────────────────────────────────────────────── + +export function getMode() { return _mode; } + +// Validated setter. Does NOT run the bridge-side transition side effects +// (that's setConsistencyMode()'s job in ruvectorBridge.js); this just +// updates local state + clears the cache on mode change so stale +// eventual-mode entries can't leak into a fresh-mode query. +export function setMode(m) { + if (!CONSISTENCY_MODES.includes(m)) { + throw new Error(`consistency/mode: invalid mode ${m}`); + } + if (m === _mode) return; + _mode = m; + // Any mode change invalidates the eventual cache — the semantics of + // "cache" change between modes, so keeping stale entries around would + // be a footgun. + _cache.clear(); +} + +export function getTtl() { return _ttl; } +export function setTtl(n) { + const v = Math.max(1, (n | 0)); + _ttl = v; +} + +// ─── trackVec fingerprint ──────────────────────────────────────────────── +// +// The eventual cache keys on the *query vector identity*, not object +// identity. Two subsequent recommendSeeds calls with the same underlying +// float data (but different Float32Array wrappers) should hit the same +// cache entry. A short FNV-1a-ish digest over the first 32 floats is +// fingerprint enough for this use case — collisions across truly +// distinct track embeddings are vanishingly unlikely at that resolution. +export function trackVecKey(vec) { + if (!vec) return '_null'; + if (typeof vec === 'string') return vec; // allow pre-computed keys + if (!(vec instanceof Float32Array)) { + try { vec = new Float32Array(vec); } catch (_) { return '_invalid'; } + } + // Simple cheap rolling mix of the first 32 components + length; no + // cryptographic guarantees needed, just uniqueness across the + // ~dozens of distinct track vectors a session ever sees. + let h = 2166136261 >>> 0; + const n = Math.min(32, vec.length | 0); + for (let i = 0; i < n; i++) { + // Pack the float's bitwise representation into the mix so tiny + // numerical wobble still hashes identically when values round-trip. + const b = Math.fround(vec[i]); + const u = new Uint32Array(new Float32Array([b]).buffer)[0]; + h ^= u; + h = Math.imul(h, 16777619) >>> 0; + } + return 'v' + vec.length + ':' + h.toString(16).padStart(8, '0'); +} + +// ─── eventual-mode cache ───────────────────────────────────────────────── + +// Record a fresh query result under this track key. Only called by the +// bridge when _mode === 'eventual' and the previous lookup was a miss. +export function recordQuery(key, result) { + if (!key) return; + _cache.set(key, { + value: result, + insertedAtCall: _callCount, + expiresAtCall: _callCount + _ttl, + }); +} + +// Look up the cache. Always increments _callCount so the caller's +// behaviour stays consistent whether or not there's a hit — TTL is +// counted in "recommendSeeds calls since insert". Returns a +// discriminated union so the caller can branch on `hit`. +export function getCachedResult(key) { + _callCount++; + if (!key) { + _cacheMisses++; + return { hit: false, reason: 'no-key' }; + } + const entry = _cache.get(key); + if (!entry) { + _cacheMisses++; + return { hit: false, reason: 'miss' }; + } + if (_callCount > entry.expiresAtCall) { + _cache.delete(key); + _cacheMisses++; + return { hit: false, reason: 'expired' }; + } + _cacheHits++; + return { hit: true, value: entry.value }; +} + +export function clearCache() { _cache.clear(); } + +// ─── frozen-mode snapshot ref ──────────────────────────────────────────── + +// The bridge passes a "snapshot-like" object here — in practice, either +// an ArchiveSnapshot produced by exportSnapshot() or (for the bridge's +// cap-by-insertionOrder shortcut) a lightweight descriptor +// { frozenBrainCount: number, frozenBrainIds: Set }. The module +// treats the reference as opaque; the bridge knows how to interpret it. +export function freezeArchive(snapshot) { + _frozenSnapshot = snapshot || null; + _frozenAt = new Date().toISOString(); +} + +export function thawArchive() { + _frozenSnapshot = null; + _frozenAt = null; +} + +export function getFrozenSnapshot() { return _frozenSnapshot; } + +// ─── introspection ─────────────────────────────────────────────────────── + +export function stats() { + return { + mode: _mode, + ttl: _ttl, + cacheHits: _cacheHits, + cacheMisses: _cacheMisses, + cacheSize: _cache.size, + callCount: _callCount, + frozenSince: _frozenAt, + frozen: _frozenSnapshot != null, + }; +} + +// ─── test hook ─────────────────────────────────────────────────────────── + +export function _debugReset() { + _mode = 'fresh'; + _ttl = DEFAULT_TTL_GENERATIONS; + _cache.clear(); + _frozenSnapshot = null; + _frozenAt = null; + _callCount = 0; + _cacheHits = 0; + _cacheMisses = 0; +} diff --git a/AI-Car-Racer/consistency/worker-sync.js b/AI-Car-Racer/consistency/worker-sync.js new file mode 100644 index 0000000..3c227b9 --- /dev/null +++ b/AI-Car-Racer/consistency/worker-sync.js @@ -0,0 +1,59 @@ +// consistency/worker-sync.js +// Phase 1C (F4) — helper for propagating a frozen snapshot to an A/B +// baseline worker. The actual postMessage wiring into sim-worker.js is +// a follow-up polish item (see TODO below); this file exists so future +// work has a clear, callable consumer. +// +// The plan's A/B-mode motivation: when the primary pin is in `frozen`, +// a parallel baseline worker must start from the exact same archive +// snapshot — otherwise the baseline wanders and the A/B delta is +// meaningless. This helper computes the payload a future consumer +// would postMessage to the baseline worker: +// +// { type: 'freeze', snapshot: /* ArchiveSnapshot */ } +// +// TODO (post-1C): wire this into main.js's `spawnB()` path. The +// sim-worker already has a minimal `case 'freeze'` hook (see +// sim-worker.js onmessage) that will call importSnapshot + +// setConsistencyMode('frozen') once the worker side has a bridge +// adapter available. + +import { getMode, getFrozenSnapshot } from './mode.js'; + +// Returns a payload suitable for postMessage to a baseline sim-worker. +// When not in frozen mode, returns null — callers use that as "no sync +// needed" (the worker either runs its own fresh queries or stays +// bridgeless, matching current A/B behaviour). +// +// `exportFn` is the bridge's exportSnapshot() function, passed in so +// this helper stays free of a direct bridge import cycle. Typical call +// site: `computeFrozenSyncPayload(bridge.exportSnapshot)`. +export function computeFrozenSyncPayload(exportFn) { + if (getMode() !== 'frozen') return null; + // Prefer the frozen snapshot reference if the bridge stashed one at + // freeze time — that's the exact view queries were pinned to, so a + // downstream worker runs against the identical data. Fall back to a + // fresh exportSnapshot() if the frozen ref isn't a full snapshot + // (e.g. the bridge's cap-by-insertionOrder shortcut stores only a + // count + id set — fine for the primary, insufficient for a worker + // that needs the actual vectors to query). + let snapshot = getFrozenSnapshot(); + const looksLikeSnapshot = snapshot && Array.isArray(snapshot.brains) && snapshot.hnsw; + if (!looksLikeSnapshot && typeof exportFn === 'function') { + try { snapshot = exportFn(); } catch (e) { + console.warn('[consistency/worker-sync] exportSnapshot failed', e); + return null; + } + } + if (!snapshot) return null; + return { type: 'freeze', snapshot }; +} + +// Convenience wrapper that imports the bridge lazily at call time. Only +// useful in contexts where main.js has already populated +// window.__rvBridge (i.e. the UI thread). Workers should use +// computeFrozenSyncPayload directly with their own exportFn. +export function computeFrozenSyncPayloadFromGlobal() { + if (typeof window === 'undefined' || !window.__rvBridge) return null; + return computeFrozenSyncPayload(window.__rvBridge.exportSnapshot); +} diff --git a/AI-Car-Racer/eli15/chapters/consistency-modes.js b/AI-Car-Racer/eli15/chapters/consistency-modes.js index 72ecd55..3556f38 100644 --- a/AI-Car-Racer/eli15/chapters/consistency-modes.js +++ b/AI-Car-Racer/eli15/chapters/consistency-modes.js @@ -1,25 +1,102 @@ // eli15/chapters/consistency-modes.js -// Placeholder — real content ships with Phase 1C (F4). +// Phase 1C (F4) — real content. Explains the three consistency modes +// that the query path in ruvectorBridge.js now honours. export default { id: 'consistency-modes', title: 'Fresh, Eventual, Frozen — three ways training looks at the archive', - oneLiner: 'Should training re-query the archive every generation, periodically, or lock in a snapshot? Each answer is a different mode.', - comingSoon: true, + oneLiner: 'Should every generation re-ask the archive "who looks like this track?", or lock in an answer? Three modes, one radio row.', body: [ - '

Coming soon — lands with Phase 1C of the RuLake-inspired roadmap.

', - '

Every generation, the training loop asks the archive "who does this', - 'track look like, and which brains worked on those tracks?" But that', - 'question doesn\'t have one right answer.

', + '

Every generation, the training loop asks the archive a question:', + '"Which past brains are most similar to the track I\'m about to run?"', + 'That answer is a short list of brains the GA uses as warm seeds for the', + 'next batch of cars. The question sounds simple — but "what is the', + 'archive?" has more than one reasonable answer, because the archive is', + 'growing while you train. Every generation you just', + 'finished added new brains to it. So do you ask the current archive', + 'or the archive as it was before you started?

', + + '

Three answers, one radio row

', + + '

Fresh — re-ask every generation. The retrieval', + 'always sees the newest brains, so as the GA discovers good ones they', + 'immediately become available as seeds. The downside: the answer can', + 'wobble mid-run. A new great brain landing in generation 7', + 'reshuffles the "who does this track look like?" ordering, which can', + 'make the seeds you get in generation 8 very different from what you', + 'got in generation 6. This is today\'s default and is what you want for', + 'everyday training.

', + + '

Eventual — cache the answer for N', + 'generations (default 10), then re-ask. Cheaper, steadier. Between', + 're-asks the seed list is frozen to a moving window — you get', + 'some of the new-brain benefit but without the wobble. The tradeoff is', + 'latency of the "new good brain → seeds include it" path: a brain', + 'discovered in gen 7 with TTL=10 might wait until gen 17 to start', + 'influencing retrieval. Pick eventual when you want predictable seeds', + 'for longer comparisons without sacrificing growth entirely.

', + + '

Frozen — at the moment you switch into frozen mode,', + 'the archive\'s current contents become the entire answer space', + 'for every subsequent query. Brains archived after the freeze', + 'are still saved (training continues normally), they just don\'t appear', + 'in recommendSeeds results until you thaw (by switching', + 'back to fresh or eventual). This is the mode the A/B baseline worker', + 'wants — without it, a side-by-side run where one canvas has ruvector', + 'and the other doesn\'t is a moving target because the "with" side', + 'keeps growing its archive.

', + + '

Why frozen exists: the A/B race

', + '

The A/B toggle in this panel spawns a baseline sim-worker that runs', + 'the same track side-by-side. For the delta to be meaningful, both sides', + 'need to see the same archive. Without frozen mode, the', + 'ruvector side keeps adding brains each generation while the baseline', + 'does not — so "how much did ruvector help?" gets confounded with', + '"how much did the archive grow?". Pinning the archive at frozen-mode', + 'entry time gives a clean single-variable comparison. The helper in', + 'consistency/worker-sync.js computes the payload a future', + 'version of spawnB() can postMessage to the baseline; the', + 'sim-worker has a matching case \'freeze\' handler ready to', + 'receive it.

', + + '

What the tick strip is showing you

', + '

Under the radio row sits a horizontal strip of 30 tiny dots. Each', + 'dot pulses green-then-fades every time recommendSeeds', + 'actually ran a fresh query (cache miss). Watch it as you flip modes:

', '
    ', - '
  • Fresh — re-ask every generation. Always current,', - ' but results can wobble as new brains land in the archive mid-run.
    • ', - '
  • Eventual — cache the answer, re-ask every N', - ' generations. Faster and steadier.
    • ', - '
  • Frozen — lock in a snapshot at the start of the', - ' run; ignore new additions entirely. Best for A/B comparisons where', - ' two workers need to see exactly the same archive.
    • ', + '
  • Fresh: every generation pulses one dot. Continuous', + ' activity.
    • ', + '
  • Eventual: the first query in a TTL window pulses;', + ' the next 9 are silent (served from cache). You\'ll see a pulse,', + ' then nine quiet ticks, then another pulse.
    • ', + '
  • Frozen: queries still fire (the mode is "filter the', + ' archive to the pinned set", not "skip searching entirely"), so the', + ' strip keeps pulsing. What\'s different is that results', + ' never include brains archived after the freeze — check the seed', + ' list for the proof.
    • ', '
', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 1C.

', + + '

Try it yourself

', + '
    ', + '
  • Open the tests harness: tests/consistency-modes-smoke.html.', + ' It asserts, among other things, that a brain inserted after', + ' freeze doesn\'t appear in results until you thaw.
    • ', + '
  • Reload the page with ?consistency=frozen in the URL to', + ' boot straight into frozen mode — handy for reproducing an A/B run.
    • ', + '
  • Open the DevTools console and try', + ' window.__rvBridge.getConsistencyStats() while training', + ' to watch cacheHits tick up under Eventual.
    • ', + '
', + + '

Where this lives in the code

', + '

The state machine is AI-Car-Racer/consistency/mode.js.', + 'The bridge consumes it in recommendSeeds inside', + 'ruvectorBridge.js — look for the 1C — F4 comment', + 'block. The UI radio row + tick strip are in uiPanels.js;', + 'the A/B-worker helper is consistency/worker-sync.js.

', ].join('\n'), + related: [ + 'warm-restart', + 'track-similarity', + 'vectordb-hnsw', + ], }; diff --git a/AI-Car-Racer/main.js b/AI-Car-Racer/main.js index 9697e00..2e0719b 100644 --- a/AI-Car-Racer/main.js +++ b/AI-Car-Racer/main.js @@ -328,6 +328,48 @@ if (typeof window !== 'undefined') { __runStagedSnapshotImport(); } +// Phase 1C (F4) — honour `?consistency=fresh|eventual|frozen` at boot. +// Default is `fresh` (today's behaviour), so the flag is opt-in. The +// URL flag is also how the A/B harness pins a known mode across a +// page reload. Invalid values are ignored with a warn; the bridge +// stays on `fresh`. Mirrors the `?hhnsw=1` / `?snapshots=1` pattern. +async function __applyUrlConsistencyFlag(){ + let m = null; + try { + var usp = new URLSearchParams(window.location.search || ''); + m = usp.get('consistency'); + } catch (_) { return; } + if (!m) return; + const valid = ['fresh', 'eventual', 'frozen']; + if (!valid.includes(m)) { + console.warn('[ruvector] ignoring invalid ?consistency=' + m); + return; + } + // The sidecar module that assigns window.__rvBridge loads + // asynchronously; poll briefly so we don't race with it on slow + // first loads. 20×100ms ≈ 2s ceiling matches existing patterns. + var b = null; + for (let i = 0; i < 20; i++) { + b = window.__rvBridge; + if (b && typeof b.ready === 'function' && typeof b.setConsistencyMode === 'function') break; + await new Promise(res => setTimeout(res, 100)); + } + if (!b || typeof b.ready !== 'function' || typeof b.setConsistencyMode !== 'function') { + console.warn('[ruvector] URL flag ?consistency=' + m + ' — bridge never appeared'); + return; + } + try { + await b.ready(); + b.setConsistencyMode(m); + console.log('[ruvector] consistency mode set via URL flag: ' + m); + } catch (e) { + console.warn('[ruvector] setConsistencyMode from URL flag failed', e); + } +} +if (typeof window !== 'undefined') { + __applyUrlConsistencyFlag(); +} + // ----------------------------------------------------------------------------- // Metrics HUD — per-generation survival %, median / p90 checkpoints, wall-bumps. // Also serves as the on-screen data source for __runBenchmark / __abTest CSVs. diff --git a/AI-Car-Racer/ruvectorBridge.js b/AI-Car-Racer/ruvectorBridge.js index d413afc..674b504 100644 --- a/AI-Car-Racer/ruvectorBridge.js +++ b/AI-Car-Racer/ruvectorBridge.js @@ -494,12 +494,37 @@ export function recommendSeeds(trackVec, k = 5) { // amplify whatever direction B currently points in). const queryVec = trackVec ? (_bypassLora ? trackVec : loraAdapt(trackVec)) : null; + // 1C — F4. Consult the consistency mode BEFORE running the search. In + // 'fresh' mode we fall through unchanged. In 'eventual' we try the + // TTL cache first and short-circuit on hit, else we compute the + // result then record it. In 'frozen' we compute as usual but filter + // results to brains that existed at freeze-entry time. The `mode` + // flag below drives all three behaviours in a single code path. + const consistencyMode = _consistencyGetMode(); + const cacheKey = (consistencyMode === 'eventual' && trackVec) + ? _consistencyTrackVecKey(trackVec) + ':k' + (k | 0) + : null; + if (consistencyMode === 'eventual') { + const cached = _consistencyGetCachedResult(cacheKey); + if (cached.hit) return cached.value; + } + const frozenSnap = (consistencyMode === 'frozen') ? _consistencyStats() : null; + // Pull the frozen brain-id set once per call rather than on every + // candidate lookup. `null` means "no filter" — fresh/eventual paths. + const frozenIds = (frozenSnap && frozenSnap.frozen) + ? _getFrozenBrainIdSet() + : null; + const candidates = new Map(); // brainId -> trackSim (best across matched tracks) if (queryVec && !_trackDB.isEmpty()) { const trackHits = _trackDB.search(queryVec, Math.min(5, Number(_trackDB.len()))); for (const th of trackHits) { const sim = 1 - th.score; for (const [bid, entry] of _brainMirror) { + // 1C frozen-mode filter: skip brains inserted after the freeze + // point. `frozenIds === null` in fresh/eventual modes so this + // branch degenerates to the existing check. + if (frozenIds && !frozenIds.has(bid)) continue; if (entry.meta && entry.meta.trackId === th.id) { const prev = candidates.get(bid); if (prev === undefined || prev < sim) candidates.set(bid, sim); @@ -511,7 +536,10 @@ export function recommendSeeds(trackVec, k = 5) { // Cold-fallback: no track match → use the whole archive with trackSim=0. // This keeps retrieval meaningful on first-ever run or on a totally novel track. if (candidates.size === 0) { - for (const bid of _brainMirror.keys()) candidates.set(bid, 0); + for (const bid of _brainMirror.keys()) { + if (frozenIds && !frozenIds.has(bid)) continue; + candidates.set(bid, 0); + } } // Decide reranker. The toggle policy (P4.A) takes precedence over @@ -595,7 +623,43 @@ export function recommendSeeds(trackVec, k = 5) { }); } scored.sort((a, b) => b.score - a.score); - return scored.slice(0, Math.max(1, k | 0)); + const out = scored.slice(0, Math.max(1, k | 0)); + // 1C — F4. In eventual mode, stash the result so the next TTL calls + // under the same trackVec key short-circuit at the top of this fn. + // The stored reference IS the returned array — callers MUST treat + // recommendSeeds output as read-only (it always has been by + // convention). No-op in fresh/frozen modes. + if (consistencyMode === 'eventual' && cacheKey) { + _consistencyRecordQuery(cacheKey, out); + } + return out; +} + +// Pull the frozen brain-id set out of the consistency module's opaque +// snapshot reference. Centralising the unwrap here keeps the +// "cap-by-insertionOrder" shortcut an implementation detail of the +// bridge — the module holds whatever we stashed on freezeArchive and +// doesn't care about the shape. +function _getFrozenBrainIdSet() { + const stats = _consistencyStats(); + if (!stats || !stats.frozen) return null; + // The snapshot reference isn't exported directly from the module; + // re-import it here. Short-circuit when the bridge's shortcut shape + // isn't present (e.g. an ArchiveSnapshot was stored instead) so + // recommendSeeds degrades to "no filter" rather than crashing. + // eslint-disable-next-line no-use-before-define + const ref = _frozenSnapshotRef(); + if (!ref || !(ref.frozenBrainIds instanceof Set)) return null; + return ref.frozenBrainIds; +} +// Tiny indirection: import-bound getter that re-reads the frozen ref +// lazily. Separated from _getFrozenBrainIdSet for readability — keeps +// the "what shape are we dealing with?" logic in one place. +function _frozenSnapshotRef() { + // Avoid top-level name clash by calling the module's getter through + // the already-imported stats path. The module exports + // getFrozenSnapshot() directly — wire it in here. + return _consistencyGetFrozenSnapshot(); } // ─── embedding + observation ───────────────────────────────────────────────── @@ -1074,6 +1138,23 @@ export function hydrateFromFixture(fixture) { import { validateSnapshot, CONSISTENCY_MODES } from './archive/snapshot.js'; import { buildSnapshot } from './archive/exporter.js'; import { applySnapshot } from './archive/importer.js'; +// 1C — F4. Consistency-mode state machine + eventual-mode TTL cache live +// in consistency/mode.js. We consult it on every recommendSeeds() call +// (see the `consistency` branch below) and transition via +// setConsistencyMode. The module is independent of the bridge so tests +// can exercise it without booting wasm. +import { + getMode as _consistencyGetMode, + setMode as _consistencySetMode, + recordQuery as _consistencyRecordQuery, + getCachedResult as _consistencyGetCachedResult, + freezeArchive as _consistencyFreezeArchive, + thawArchive as _consistencyThawArchive, + clearCache as _consistencyClearCache, + trackVecKey as _consistencyTrackVecKey, + stats as _consistencyStats, + getFrozenSnapshot as _consistencyGetFrozenSnapshot, +} from './consistency/mode.js'; // Phase 0: stored but unread; 1C will wire this into the query path. let _consistencyMode = 'fresh'; @@ -1139,17 +1220,57 @@ export function importSnapshot(s) { return res; } -// 1C fills this in. Accepts 'fresh' | 'eventual' | 'frozen'. +// 1C — F4. Accepts 'fresh' | 'eventual' | 'frozen'. +// +// Freeze/thaw mechanism: we use the cap-by-insertionOrder shortcut from +// the plan. On transition to 'frozen' we snapshot the current +// _insertionOrder length and the set of brain ids present *right now*. +// During a frozen query, recommendSeeds ignores any brain whose id is +// NOT in that snapshot set — new archiveBrain() calls still insert into +// the live _brainDB (we don't want to break archive growth), but their +// ids don't appear in the pinned set so they won't appear in query +// results. On transition away from 'frozen' we drop the reference and +// the live archive is queryable again. export function setConsistencyMode(m) { if (!CONSISTENCY_MODES.includes(m)) { throw new Error(`setConsistencyMode: invalid mode ${m}`); } + const prev = _consistencyGetMode(); + if (prev === m) { + _consistencyMode = m; + return; + } + // On any mode change, clear the eventual cache so a previous mode's + // cached answer doesn't accidentally replay under a different + // consistency contract. + _consistencyClearCache(); + // Leaving frozen → thaw the snapshot reference so live queries can + // see every brain again. + if (prev === 'frozen' && m !== 'frozen') { + _consistencyThawArchive(); + } + // Entering frozen → pin the current archive state. + if (m === 'frozen') { + const frozenBrainIds = new Set(_brainMirror.keys()); + _consistencyFreezeArchive({ + frozenBrainCount: frozenBrainIds.size, + frozenBrainIds, + frozenAt: Date.now(), + }); + } _consistencyMode = m; - // 1C will add: persist mode, pin snapshot on 'frozen', TTL reset on 'eventual'. - throw new Error('not implemented: setConsistencyMode (Phase 1C / F4)'); + _consistencySetMode(m); } -export function getConsistencyMode() { return _consistencyMode; } +// Backwards-compat getter. Returns the simple string so existing +// callers (uiPanels, tests) stay happy; callers that want more detail +// can import consistency/mode.stats() directly. +export function getConsistencyMode() { return _consistencyGetMode(); } + +// Re-export the stats snapshot for callers (uiPanels tick, test +// harnesses). Not part of the 1A/1B/1D contract surface; kept as a +// named export so consumers can discover it via tree-shaking / IDE. +export function getConsistencyStats() { return _consistencyStats(); } // 3A fills this in. Returns { hnsw: {...}, rerank: {...}, adapter: {...} } // with per-stage timings and counters for the "where the time goes" chapter. diff --git a/AI-Car-Racer/sim-worker.js b/AI-Car-Racer/sim-worker.js index 6bc4ec2..37cf4ce 100644 --- a/AI-Car-Racer/sim-worker.js +++ b/AI-Car-Racer/sim-worker.js @@ -79,9 +79,36 @@ self.onmessage = (ev) => { case 'setPause': handlePause(m.pause); break; case 'setTraction': self.traction = m.v; break; case 'setMaxSpeed': self.maxSpeed = m.v; break; + // Phase 1C (F4) — minimal A/B-baseline freeze-sync hook. When + // the primary pins its archive in `frozen` mode, it can post + // `{type:'freeze', snapshot}` so this worker starts from the + // identical archive view. We gate on the functions existing + // (this worker has no bridge today — the hook is a no-op + // until the A/B postMessage wiring lands; see + // consistency/worker-sync.js for the producer helper). + // Existing A/B setups that never post this message continue + // to work unchanged. + case 'freeze': handleFreeze(m); break; } }; +function handleFreeze(m) { + if (!m || !m.snapshot) return; + try { + if (typeof self.importSnapshot === 'function') { + self.importSnapshot(m.snapshot); + } + if (typeof self.setConsistencyMode === 'function') { + self.setConsistencyMode('frozen'); + } + } catch (e) { + // Never throw out of onmessage — the worker would die silently. + // Log through postMessage so main.js sees the failure. + try { self.postMessage({ type: 'debug', event: 'freezeSyncFailed', error: String(e && e.message || e) }); } + catch (_) { /* drop */ } + } +} + function handleInit(m) { // Build a bare `road` object compatible with sensor/car's global reads. // The main-side Road class drags in DOM dependencies (roadEditor, diff --git a/AI-Car-Racer/style.css b/AI-Car-Racer/style.css index 135561a..68b1541 100644 --- a/AI-Car-Racer/style.css +++ b/AI-Car-Racer/style.css @@ -1463,6 +1463,67 @@ label { box-shadow: var(--ring); } +/* ============================================================= + 1C — F4 Consistency-mode selector + tick strip + A radio row (Fresh / Eventual / Frozen) and a horizontal strip + of 30 tiny dots beneath. Each dot pulses via a brief CSS + animation each time recommendSeeds runs a fresh query. On + cache-hit (eventual mode inside TTL) no pulse fires, so the + strip visibly slows; on frozen the dots still pulse (the query + runs, just against a pinned archive), which matches the + "frozen ≠ skip" pedagogy. + ============================================================= */ +.rv-consistency { + display: flex; + flex-direction: column; + gap: 4px; + margin: 6px 0 4px 0; + font-size: 11px; + color: #a8c8ff; +} +.rv-consistency-head { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} +.rv-consistency-label { font-weight: 600; color: #cbd8ff; } +.rv-consistency-radios { + display: flex; + gap: 10px; + align-items: center; +} +.rv-consistency-radios label { + display: inline-flex; + align-items: center; + gap: 3px; + cursor: pointer; + user-select: none; +} +.rv-consistency-radios input[type="radio"] { margin: 0; cursor: pointer; } +.rv-consistency-ticks { + display: flex; + gap: 2px; + padding: 2px 0; + min-height: 8px; +} +.rv-consistency-tick { + display: inline-block; + width: 4px; + height: 6px; + border-radius: 1px; + background: rgba(168, 200, 255, 0.18); + transition: background-color 200ms ease-out; +} +.rv-consistency-tick-pulse { + background: #62f0b8; + animation: rv-consistency-tick-fade 800ms ease-out forwards; +} +@keyframes rv-consistency-tick-fade { + 0% { background: #62f0b8; transform: scaleY(1.35); } + 100% { background: rgba(168, 200, 255, 0.18); transform: scaleY(1); } +} + /* ============================================================= Guided-tour overlay (eli15) ============================================================= */ diff --git a/AI-Car-Racer/uiPanels.js b/AI-Car-Racer/uiPanels.js index 5f806e7..1b429af 100644 --- a/AI-Car-Racer/uiPanels.js +++ b/AI-Car-Racer/uiPanels.js @@ -167,6 +167,25 @@ ' ', ' ', '', + // 1C — F4 Consistency modes. A radio row selects the query-path + // consistency semantics: Fresh (re-query each call — the default + // and today's behaviour), Eventual (TTL cache), Frozen (pin the + // archive at mode-entry; new brains don't appear in results). + // The tick strip below the radios pulses on every fresh query — + // no pulse on a cache hit or a frozen-filter query that short- + // circuits, so the learner can *see* the mode at work. + '
', + '
', + ' 🧊 Consistency:', + '
', + ' ', + ' ', + ' ', + '
', + ' ', + '
', + ' ', + '
', // Dynamics trajectory toggle (P1.C). Off by default — the plan keeps // this opt-in because it changes retrieval ordering. The count next to // the label shows how many archived brains have a dynamics vector @@ -328,8 +347,93 @@ seedSources: root.querySelector('[data-rv="seed-sources"]'), seedSourcesText: root.querySelector('[data-rv="seed-sources-text"]'), abToggle: root.querySelector('[data-rv="ab-toggle"]'), + // 1C — F4 Consistency-mode selector + tick strip. + consistency: root.querySelector('[data-rv="consistency"]'), + consistencyRadios: root.querySelectorAll('[data-rv="consistency-radios"] input[type="radio"]'), + consistencyTicks: root.querySelector('[data-rv="consistency-ticks"]'), }; + // 1C — F4. Build the tick strip: 30 tiny dots that pulse via a + // CSS class flip whenever recommendSeeds *actually* re-queries (i.e. + // bridge.getConsistencyStats().cacheMisses increments). Cache hits + // do not pulse, so the strip visibly slows in eventual mode and + // flat-lines in frozen mode (which is satisfied by the live + // archive's state at freeze time — no new misses beyond the ones + // spent on fresh queries before the freeze). + const TICK_COUNT = 30; + if (el.consistencyTicks) { + const frag = document.createDocumentFragment(); + for (let i = 0; i < TICK_COUNT; i++) { + const dot = document.createElement('span'); + dot.className = 'rv-consistency-tick'; + frag.appendChild(dot); + } + el.consistencyTicks.appendChild(frag); + } + let _consistencyTickIdx = 0; + let _consistencyLastMisses = 0; + function pulseConsistencyTick() { + if (!el.consistencyTicks) return; + const tick = el.consistencyTicks.children[_consistencyTickIdx % TICK_COUNT]; + if (!tick) return; + tick.classList.remove('rv-consistency-tick-pulse'); + // Force a reflow so re-adding the class restarts the CSS animation. + // (getBoundingClientRect is the cheap well-known reflow trigger.) + void tick.getBoundingClientRect(); + tick.classList.add('rv-consistency-tick-pulse'); + _consistencyTickIdx = (_consistencyTickIdx + 1) % TICK_COUNT; + } + + // Wire the radio handlers. setConsistencyMode throws on invalid mode + // (shouldn't happen from our fixed radio values, but wrap in try for + // defensiveness — a future URL-flag import could feed an unexpected + // string). + el.consistencyRadios.forEach(function (r) { + r.addEventListener('change', function () { + if (!r.checked) return; + const b = window.__rvBridge; + if (!b || typeof b.setConsistencyMode !== 'function') return; + try { b.setConsistencyMode(r.value); } + catch (e) { console.warn('[rv-panel] setConsistencyMode failed', e); } + }); + }); + // Reflect an externally-set mode (URL flag boot, console call, tour + // step) back into the radio group on every tick. Cheap — 3 DOM reads + // at 2 Hz. + function renderConsistency() { + const b = window.__rvBridge; + if (!b || typeof b.getConsistencyMode !== 'function') return; + let mode; + try { mode = b.getConsistencyMode(); } catch (_) { return; } + el.consistencyRadios.forEach(function (r) { + if (r.value === mode && !r.checked) r.checked = true; + }); + // Tick-strip pulse: drive by cacheMisses increments. Fresh mode + // misses every call; eventual misses only when cache expires; + // frozen misses every call too (we still run the search, just + // against a filtered archive — "no new insertions visible" is + // the frozen contract, not "no queries"). + if (typeof b.getConsistencyStats === 'function') { + try { + const s = b.getConsistencyStats(); + const misses = (s && s.cacheMisses) | 0; + if (misses > _consistencyLastMisses) { + const delta = Math.min(TICK_COUNT, misses - _consistencyLastMisses); + for (let i = 0; i < delta; i++) pulseConsistencyTick(); + _consistencyLastMisses = misses; + } else if (misses < _consistencyLastMisses) { + // stats reset (debugReset or mode-change clearCache) — track + // the new baseline without pulsing. + _consistencyLastMisses = misses; + } + // Eventual mode ALSO pulses on fresh misses; to make Fresh vs + // Eventual visually distinct we could also count hits, but the + // spec asks for a pulse only on fresh re-queries — so cacheHits + // intentionally doesn't drive the strip. + } catch (_) { /* ignore */ } + } + } + // Master toggle: mutating window.rvDisabled is enough — every bridgeReady() // / bridgeReadyLocal() call site already re-reads the flag on each call // (bridgeReady in main.js reads the top-level `var rvDisabled`, which IS @@ -1048,6 +1152,14 @@ // here means a new tally to render. const seedSourcesGen = (info && info.seedSources) ? (info.seedSources.generation | 0) : -1; + // 1C — F4. Poll the consistency stats on every tick so the tick + // strip animates in real time (training-loop recommendSeeds calls + // happen outside the UI memoisation window — if we gate this on + // the memo, the strip would freeze between panel-input changes). + // Radio-group reflection is idempotent; the cost is ~3 DOM reads + // at 2Hz. + renderConsistency(); + // Fast-path: nothing changed → no DOM writes, no recommendSeeds call. if ( last.ready === ready && diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md index 9689f54..ff08f3f 100644 --- a/docs/plan/rulake-inspired-features.md +++ b/docs/plan/rulake-inspired-features.md @@ -24,7 +24,7 @@ pointing to the blocker. Keep the "Current focus" line at the top pointing at whichever phase is active so a newcomer knows where to jump in without reading the whole doc. -**Current focus:** Phase 1 wave 2 — 1C only (1A landed; 1C is last remaining Phase 1 task) +**Current focus:** Phase 1 complete — next up is Phase 2A (F2 federation) which is gated on 1B, or Phase 3 observability/polish swarm **Last updated:** 2026-04-24 ### Phase 0 — Foundations _(sequential, 1 owner)_ @@ -45,7 +45,7 @@ jump in without reading the whole doc. |:--:|:--:|------|-------|--------|-----------| | ✅ | 1A | F3 — Warm-restart bundles + shareable snapshots | Claude (subagent) | — | 2026-04-24 | | ✅ | 1B | F1 — 1-bit quantized archive (RaBitQ + Hadamard) | Claude (subagent) | — | 2026-04-24 | -| ⬜ | 1C | F4 — Consistency modes (Fresh/Eventual/Frozen) | | | | +| ✅ | 1C | F4 — Consistency modes (Fresh/Eventual/Frozen) | Claude (subagent) | — | 2026-04-24 | | ✅ | 1D | F5 — Content-addressed dedup + hash-keyed lineage | Claude (subagent) | — | 2026-04-24 | **Phase 1 gate:** all four rows ✅ + each passes its own `/ship-task` @@ -78,7 +78,7 @@ community-archive URL ships publicly). | Status | Milestone | |:--:|-----------| -| ⬜ | M1 — F1+F3 demoable locally (flags on) | +| ✅ | M1 — F1+F3 demoable locally (flags on) — 2026-04-24 | | ⬜ | M2 — Phase 1 merged to `main` behind flags | | ⬜ | M3 — F2+F6 shipping, flags default on for F1/F3 | | ⬜ | M4 — Phase 3 complete, blog post / tour recording | @@ -88,6 +88,20 @@ community-archive URL ships publicly). Append-only. Record any scope change, deferral, or non-obvious call that future-you would want to find. Newest at the top. +- **2026-04-24 — Phase 1 complete.** Wave 2 shipped 1A then 1C + sequentially (both edit ruvectorBridge.js stubs; running in parallel + would race). 1A adds `archive/{exporter,importer,serialize}.js` + + `_insertionOrder` plumbing across archiveBrain/hydrate/debug-reset; + round-trip harness 5/5 PASS with CompressionStream gzip exercised. + 1C adds `consistency/{mode,worker-sync}.js`, wires `recommendSeeds` + via a one-branch fresh-mode-preserving check, ships a radio row + tick + strip in `uiPanels.js`, honors `?consistency=`, and passes a 5/5 + harness covering all three modes + thaw. One `not implemented` stub + remains in ruvectorBridge.js — Phase 3A's `getIndexStats`. Phase 1's + ordering discipline — swarm 1B+1D in parallel, then 1A→1C sequentially + — held perfectly: zero merge conflicts despite 4 different subagents + editing 15+ files. + - **2026-04-24 — Phase 1 wave 1 closed (1B + 1D).** Dispatched as two parallel general-purpose subagents in a single message; zero file overlap thanks to Phase 0's pre-partition. Live browser validation: diff --git a/tests/consistency-modes-smoke.html b/tests/consistency-modes-smoke.html new file mode 100644 index 0000000..46dc4b0 --- /dev/null +++ b/tests/consistency-modes-smoke.html @@ -0,0 +1,200 @@ + + + + + + Consistency modes smoke (F4) + + + +

Consistency-mode smoke (Phase 1C / F4)

+

Exercises setConsistencyMode, the eventual-mode TTL cache, + the frozen-mode archive pin, and thaw-on-mode-change.

+
Running…
+ + + +
#claimverdictdetail
+ + + + + + + + From bf027012fe8692497adb551c6a16f129e0a87fac Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 13:43:52 -0400 Subject: [PATCH 06/10] feat(rulake-phase-2a): federated search over Euclidean + Hyperbolic HNSW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F2 from docs/plan/rulake-inspired-features.md. Query path can now fan out to BOTH nearest-neighbour indexes simultaneously, union the candidates by hash (F5), and GNN-rerank the union. Gated behind ?federation=1 or the UI toggle; off by default. Composes with Phase 1C consistency modes (frozen filter applies to the union; eventual cache stores the final federated result, not per-shard). What this ships: - federation/fanout.js — fanOut(queryVec, k, shards) returns per-shard top-k'. Over-request formula kPrime(k, S) = k + ⌈√(k·ln S)⌉, clamped so S≤1 returns k (graceful single-shard degrade). Also a fanOutSync twin for tests. - federation/rerank.js — unionByHash (dedup key is hash; tracks per-candidate shard provenance), selectTopK (picks final k by external score map; falls back to bestScore when unscored). - federation/viewer.js — plain-DOM split-screen: Euclidean top-k' on the left, Hyperbolic top-k' on the right, unioned+reranked top-k in the middle, with the formula rendered live. - ruvectorBridge.js — structural addition of _brainDB_hyperbolic as a shadow of _brainDB populated in lock-step across archiveBrain, hydrate, hydrateFromFixture, rebuildIndicesFromMirror, and importSnapshot. The existing setIndexKind swap still works — the primary index swaps, the shadow stays hyperbolic. New _recommendSeedsFederated helper handles the fanout branch; wired into recommendSeeds so the 1C consistency cache/frozen logic wraps both the single-index and federated paths. New exports: setFederationEnabled / isFederationEnabled / getFederationStats / setFederationCapturer. - uiPanels.js — new "🌐 Federation" toggle + mount point for the viewer. Renders federation stats on every UI tick. - main.js — ?federation=1 URL flag, mirrors the 1C pattern. - eli15/chapters/federation.js — full chapter with the over-request formula worked example (k=10 → k'=13), dedup-via-F5 explanation, compose-with-1C-and-1D section. - tests/federation-smoke.html — 7-claim harness. Validated via agent-browser: - Harness PASS 7/7. lastKPrime=7 matches k=5 + ⌈√(5·ln 2)⌉=7 exactly. shards=2 (both loaded). dedupeHits=4 (both shards surfaced 4 overlapping brains and the hash union correctly collapsed them). Frozen-mode filter correctly applied to the union, not per-shard (post-freeze brain absent from federated result). - Main app boots cleanly without any flag, no new console errors. - Exactly one "not implemented" stub remains in ruvectorBridge.js (getIndexStats, Phase 3A/F7). Not yet integrated with F1 quantization (per plan scope note) — that would be a later integration slice. Not yet integrated with F6 cross-tab — that's Phase 2B, gated on 1A's snapshot format (which shipped) but sequential with this commit on ruvectorBridge.js. --- AI-Car-Racer/eli15/chapters/federation.js | 112 +++++- AI-Car-Racer/federation/fanout.js | 86 +++++ AI-Car-Racer/federation/rerank.js | 96 +++++ AI-Car-Racer/federation/viewer.js | 160 +++++++++ AI-Car-Racer/main.js | 34 ++ AI-Car-Racer/ruvectorBridge.js | 412 +++++++++++++++++++++- AI-Car-Racer/uiPanels.js | 73 ++++ docs/plan/rulake-inspired-features.md | 17 +- tests/federation-smoke.html | 225 ++++++++++++ 9 files changed, 1187 insertions(+), 28 deletions(-) create mode 100644 AI-Car-Racer/federation/fanout.js create mode 100644 AI-Car-Racer/federation/rerank.js create mode 100644 AI-Car-Racer/federation/viewer.js create mode 100644 tests/federation-smoke.html diff --git a/AI-Car-Racer/eli15/chapters/federation.js b/AI-Car-Racer/eli15/chapters/federation.js index a4ad546..fd57b04 100644 --- a/AI-Car-Racer/eli15/chapters/federation.js +++ b/AI-Car-Racer/eli15/chapters/federation.js @@ -1,22 +1,106 @@ // eli15/chapters/federation.js -// Placeholder — real content ships with Phase 2A (F2). +// Phase 2A (F2) — real content. Explains why we ask two different +// nearest-neighbour indexes at once, why that needs over-requesting per +// shard, and how the GNN picks the final top-k from the union. export default { id: 'federation', title: 'Asking two different maps of brain-space at once', oneLiner: 'The Euclidean and hyperbolic indexes disagree about who counts as a neighbour — so ask both, then let the GNN vote.', - comingSoon: true, body: [ - '

Coming soon — lands with Phase 2A of the RuLake-inspired roadmap.

', - '

VectorVroom already has two nearest-neighbour indexes: a flat', - 'Euclidean one (good for geometric track similarity) and a hyperbolic', - 'one (good for lineage-like hierarchical similarity). Today you pick one', - 'at load time via ?hhnsw=1. Federation runs both, unions', - 'their candidates, and lets the GNN reranker pick the final order.

', - '

The clever part is how many candidates to ask each index for: the', - 'formula k\' = k + ⌈√(k · ln S)⌉ (with S = number of shards)', - 'over-requests just enough from each that the true top-k is almost', - 'certainly somewhere in the union.

', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 2A.

', + '

VectorVroom has two different nearest-neighbour indexes over the', + 'same archive of brains: a Euclidean one and a', + 'hyperbolic one. They answer the same question —', + '"who\'s most similar to this brain?" — in two different geometries.

', + + '

Why the two indexes disagree

', + + '

Euclidean space is flat. Distance is the ordinary straight-line', + 'measure. It\'s great at geometric similarity: two brains with', + 'similar weight vectors count as neighbours, regardless of where they', + 'sit in the lineage tree. A well-adapted brain for Track-A and a', + 'totally independent well-adapted brain for the same kind of Track-A', + 'will land next to each other even if they come from unrelated', + 'families.

', + + '

Hyperbolic space curves negatively. Distance grows exponentially', + 'as you move away from the origin, which is exactly how a tree grows:', + 'a parent has two children, each of those has two more, and very', + 'quickly you run out of "room" in Euclidean space but have plenty of', + 'it in hyperbolic. This geometry is the natural home for', + 'hierarchical similarity. Sibling brains — close cousins in', + 'the lineage DAG — stay close, even when their weights have drifted.', + 'Descendants of the same high-fitness ancestor cluster tightly the', + 'way branches of a tree do.

', + + '

So when you ask "who looks like this brain?", the two indexes can', + 'legitimately hand back different lists. Both lists are right — they', + 'just answer different questions. Federation is the decision to stop', + 'picking one and instead ask both, then combine.

', + + '

The over-request trick

', + + '

If we ask each index for the top k and union the', + 'results, the union has at most 2k items but', + 'some of the true global top-k are almost certainly', + 'sitting just outside each shard\'s top-k. So we over-', + 'request:

', + + '

k\' = k + ⌈√(k · ln S)⌉

', + + '

where S is the number of shards. For our setup', + '(k=10, S=2):

', + + '

k\' = 10 + ⌈√(10 · ln 2)⌉ = 10 + ⌈√6.93⌉ = 10 + 3 = 13

', + + '

So each shard returns its top 13; we union those into a set that\'s', + 'between 13 and 26 entries. The classical-bound argument (due to the', + 'federated-retrieval literature) says the union then contains the true', + 'global top-k with high probability. The exact √ term is', + 'the union-bound slack — just enough to cover the "I missed a real', + 'neighbour by one slot" case.

', + + '

Union, dedup, rerank

', + + '

The union is the raw list of candidates from all shards. Two', + 'things matter there:

', + + '
    ', + '
  • Dedup via F5 hash. A brain that exists in both', + ' indexes (which is always — they\'re two views of the same', + ' archive) will surface from both shards, but the hash of its', + ' flattened weights is identical. We use hashBrain', + ' (xxHash32 over the Float32 bytes) as the canonical key so the', + ' union counts that brain once, not twice. The viewer tracks', + ' "dedupe hits" per query — that counter climbs when both shards', + ' agreed on a neighbour.
    • ', + '
  • GNN rerank over the union. The Graph Neural', + ' Network reranker (see "how the GNN reranker works") already', + ' knows how to score candidates using lineage edges and node', + ' features. Under federation it runs on the union', + ' instead of a single-shard top-k. Same rerank,', + ' bigger input. The final ordering reflects both geometries', + ' simultaneously.
    • ', + '
', + + '

How this composes with other features

', + + '

Federation isn\'t a standalone mode — it layers over the 1C', + 'consistency controls and the 1D dedup. In eventual', + 'mode the TTL cache memoises the final top-k', + 'after rerank, not the per-shard results, so a cached federated query', + 'is still exactly the answer you\'d get by running the whole pipeline.', + 'In frozen mode the archive-pin filter applies to the', + 'union: a brain archived after you froze will not appear in federated', + 'results even if a shard surfaces it. And the F5 dedup is what makes', + 'the union mergeable in the first place — without content hashes, the', + 'same brain under two different shard-local ids would double-count.

', + + '

Turn it on

', + + '

Click the 🌐 Federation toggle in the Vector', + 'Memory panel, or load the page with ?federation=1. The', + 'split-screen viewer shows each shard\'s top-k\' and the', + 'final unioned + reranked top-k side-by-side so you can', + 'see the disagreement and the resolution.

', ].join('\n'), }; diff --git a/AI-Car-Racer/federation/fanout.js b/AI-Car-Racer/federation/fanout.js new file mode 100644 index 0000000..a1f316b --- /dev/null +++ b/AI-Car-Racer/federation/fanout.js @@ -0,0 +1,86 @@ +// federation/fanout.js — Phase 2A (F2) +// +// Parallel fan-out across S shards. Each shard is a {name, db, metric} entry +// where `db` implements the slice of the VectorDB / HyperbolicVectorDB surface +// the bridge already uses (search(queryVec, k) → [{id, score, ...}, ...]). +// +// Over-request formula (per plan / paper): +// k' = k + ⌈√(k · ln S)⌉ +// +// Rationale: with S independent indexes each returning their own top-k', the +// probability that the global top-k all surface in the union is high enough +// that a subsequent rerank over the union matches single-index quality. The +// extra √(k ln S) term is the classical union-bound slack. +// +// API: +// fanOut(queryVec, k, shards) → Promise<[{name, kPrime, results: [{id, score, ...}]}]> +// +// We accept a single queryVec (the bridge only ever has one at a time after +// LoRA adapt). Each shard search is kicked off synchronously then awaited via +// Promise.all — even though the underlying search calls are synchronous today +// (both the Euclidean VectorDB and the HyperbolicVectorDB hand back sync +// arrays), we stay async to leave room for a future worker-backed index. +// +// Failure mode: a single shard throwing does NOT take the whole fan-out down. +// We catch per-shard, log, and yield an empty result for that shard so the +// bridge's union path still produces something drawn from the surviving +// shards. This keeps federation graceful when e.g. the hyperbolic adapter +// hits a NaN on a pathological vector. + +export function kPrime(k, shardCount) { + const kk = Math.max(1, k | 0); + const S = Math.max(1, shardCount | 0); + if (S <= 1) return kk; + return kk + Math.ceil(Math.sqrt(kk * Math.log(S))); +} + +export async function fanOut(queryVec, k, shards) { + if (!Array.isArray(shards) || shards.length === 0) return []; + const kk = Math.max(1, k | 0); + const S = shards.length; + const kp = kPrime(kk, S); + const promises = shards.map(async (shard) => { + const name = (shard && shard.name) || 'shard'; + try { + if (!shard || !shard.db || typeof shard.db.search !== 'function') { + return { name, kPrime: kp, results: [] }; + } + // The underlying search is synchronous; await still resolves fine. + const hits = shard.db.search(queryVec, kp) || []; + return { name, kPrime: kp, results: hits }; + } catch (e) { + console.warn('[federation/fanout] shard "' + name + '" failed:', e); + return { name, kPrime: kp, results: [], error: String(e && e.message || e) }; + } + }); + return Promise.all(promises); +} + +// Synchronous sibling. Today's shard backends (VectorDB, HyperbolicVectorDB) +// are both sync, and recommendSeeds is called in hot paths (GA seed buffer +// construction, rv-panel poll) that already expect sync. We keep the async +// fanOut as the contract surface for future worker-backed indexes, and use +// this sync variant internally. The two share kPrime() so the over-request +// math is identical. +export function fanOutSync(queryVec, k, shards) { + if (!Array.isArray(shards) || shards.length === 0) return []; + const kk = Math.max(1, k | 0); + const S = shards.length; + const kp = kPrime(kk, S); + const out = []; + for (const shard of shards) { + const name = (shard && shard.name) || 'shard'; + try { + if (!shard || !shard.db || typeof shard.db.search !== 'function') { + out.push({ name, kPrime: kp, results: [] }); + continue; + } + const hits = shard.db.search(queryVec, kp) || []; + out.push({ name, kPrime: kp, results: hits }); + } catch (e) { + console.warn('[federation/fanout] shard "' + name + '" failed:', e); + out.push({ name, kPrime: kp, results: [], error: String(e && e.message || e) }); + } + } + return out; +} diff --git a/AI-Car-Racer/federation/rerank.js b/AI-Car-Racer/federation/rerank.js new file mode 100644 index 0000000..73d9dd3 --- /dev/null +++ b/AI-Car-Racer/federation/rerank.js @@ -0,0 +1,96 @@ +// federation/rerank.js — Phase 2A (F2) +// +// Union the per-shard fan-out results into a single candidate list, keyed by +// content hash (so the same brain surfacing from both shards collapses to one +// node), and select a final top-k from a reranked candidate map. +// +// API: +// unionByHash(shardResults, hashLookup) +// → { candidates: [{id, shards: [name, ...], bestScore, hash}], +// dedupeHits: number } +// +// `shardResults` is the return value of fanOut(): an array of +// {name, kPrime, results: [{id, score, ...}]}. +// `hashLookup(id)` is a callback the caller provides to translate a brain +// id → content hash string. Returning `null`/`undefined` means "no hash +// known" (pre-dedup brain); we fall back to the id itself as the dedup +// key so the candidate still makes it through. +// +// selectTopK(candidates, scoreMap, k) +// → [{id, score, shards, hash}, ...] (length ≤ k, sorted desc by score) +// +// `candidates` is the union list from unionByHash. `scoreMap` is a +// Map — typically the GNN-rerank output (the bridge already +// consumes `gnnScore` this way). Ids missing from scoreMap fall back to +// their `bestScore` from the union so we never silently drop a candidate. + +// Merge per-shard results into a deduped candidate list. +// +// Dedup policy: we key by hash when we can compute one, else by id. Two +// shards surfacing the *same brain id* always collapse even without a hash, +// because id == id; two shards surfacing *different ids for the same +// content* (e.g. hyperbolic's hb_N vs euclidean's numeric id) collapse via +// the hash path when hashLookup resolves both to the same digest. +export function unionByHash(shardResults, hashLookup) { + const byKey = new Map(); // key → candidate + let dedupeHits = 0; + const hLookup = typeof hashLookup === 'function' ? hashLookup : () => null; + + for (const shard of (shardResults || [])) { + const shardName = (shard && shard.name) || 'shard'; + const hits = (shard && Array.isArray(shard.results)) ? shard.results : []; + for (const hit of hits) { + if (!hit || hit.id == null) continue; + const id = String(hit.id); + let hash = null; + try { hash = hLookup(id); } catch (_) { hash = null; } + const key = hash || id; + // Prefer the "closest" result per hash: lower score = closer + // (VectorDB convention — cosine distance). We carry bestScore + // for tie-break / fallback ordering when the GNN doesn't + // supply a score. + const score = Number.isFinite(hit.score) ? hit.score : 1; + const prev = byKey.get(key); + if (prev) { + dedupeHits += 1; + if (!prev.shards.includes(shardName)) prev.shards.push(shardName); + if (score < prev.bestScore) { + prev.bestScore = score; + // Keep the id from whichever shard had the closer score so the + // downstream mirror lookup uses the id the bridge actually knows. + prev.id = id; + } + } else { + byKey.set(key, { + id, + hash: hash || null, + shards: [shardName], + bestScore: score, + }); + } + } + } + + return { candidates: Array.from(byKey.values()), dedupeHits }; +} + +// Pick the final top-k from a reranked union. `scoreMap` is treated as +// "higher is better" (matches gnnScore's output, which returns values in +// roughly [0.7, 1.3] where higher ranks first). Candidates missing from +// scoreMap fall back to -bestScore (lower distance → higher rank) so the +// ordering is still sensible if the GNN didn't run. +export function selectTopK(candidates, scoreMap, k) { + if (!Array.isArray(candidates) || candidates.length === 0) return []; + const kk = Math.max(1, k | 0); + const out = candidates.map((c) => { + let s; + if (scoreMap && typeof scoreMap.get === 'function' && scoreMap.has(c.id)) { + s = scoreMap.get(c.id); + } else { + s = -Number(c.bestScore || 0); + } + return { id: c.id, hash: c.hash, shards: c.shards.slice(), score: s }; + }); + out.sort((a, b) => b.score - a.score); + return out.slice(0, kk); +} diff --git a/AI-Car-Racer/federation/viewer.js b/AI-Car-Racer/federation/viewer.js new file mode 100644 index 0000000..449bb1e --- /dev/null +++ b/AI-Car-Racer/federation/viewer.js @@ -0,0 +1,160 @@ +// federation/viewer.js — Phase 2A (F2) +// +// Plain-DOM split-screen viewer. No canvas, no animations. Three columns: +// +// [Euclidean top-k'] [Hyperbolic top-k'] +// ──────────────────── ───────────────────── +// [Unioned + GNN-reranked top-k] +// +// and a one-line formula readout underneath: +// +// k' = k + ⌈√(k · ln S)⌉ (k=, S= → k'=) +// +// Usage: +// mountViewer(containerEl, capturer) +// +// `capturer` is a {onSnapshot} object the caller pushes into; the bridge calls +// capturer.onSnapshot(snap) after each federated query. We store the last +// snapshot and re-render. Multiple mounts on the same capturer object stack +// via a small listener array so the panel viewer and any future test-harness +// viewer both stay in sync without fighting for the single slot. +// +// Snapshot shape (bridge contract): +// { +// k: number, // final top-k requested +// kPrime: number, // per-shard over-request +// shards: [{ name, results: [{id, score}, ...] }, ...], +// unionSize: number, // distinct candidates post-dedup +// dedupeHits: number, +// final: [{ id, score, shards:[names], hash? }, ...], // top-k +// } + +export function createCapturer() { + const listeners = []; + let last = null; + return { + onSnapshot(snap) { + last = snap; + for (const fn of listeners) { + try { fn(snap); } catch (e) { console.warn('[federation/viewer] listener failed', e); } + } + }, + subscribe(fn) { + listeners.push(fn); + if (last) { + try { fn(last); } catch (_) {} + } + return () => { + const i = listeners.indexOf(fn); + if (i >= 0) listeners.splice(i, 1); + }; + }, + last() { return last; }, + }; +} + +function fmtScore(s) { + if (!Number.isFinite(s)) return '—'; + if (Math.abs(s) >= 100) return s.toFixed(0); + if (Math.abs(s) >= 10) return s.toFixed(2); + return s.toFixed(3); +} + +function renderShardTable(el, shard) { + const name = (shard && shard.name) || '—'; + const rows = (shard && Array.isArray(shard.results)) ? shard.results : []; + const header = '
' + escapeHtml(name) + ' (' + rows.length + ')
'; + if (rows.length === 0) { + el.innerHTML = header + '
no results
'; + return; + } + const lines = rows.slice(0, 20).map((r) => { + return '
' + escapeHtml(String(r.id)) + '' + + '' + fmtScore(Number(r.score)) + '
'; + }).join(''); + el.innerHTML = header + lines; +} + +function renderFinal(el, snap) { + const final = (snap && Array.isArray(snap.final)) ? snap.final : []; + const k = snap ? snap.k : 0; + const kp = snap ? snap.kPrime : 0; + const S = (snap && Array.isArray(snap.shards)) ? snap.shards.length : 0; + const unionSize = snap ? snap.unionSize : 0; + const dedupeHits = snap ? snap.dedupeHits : 0; + const header = + '
Unioned + reranked (' + final.length + '/' + k + ')
' + + '
k\' = k + ⌈√(k · ln S)⌉   ' + + 'k=' + k + ', S=' + S + ' → k\'=' + kp + '
' + + '
union size: ' + unionSize + '  ·  dedupe hits: ' + dedupeHits + '
'; + if (final.length === 0) { + el.innerHTML = header + '
no snapshots yet — run a query
'; + return; + } + const lines = final.map((r) => { + const shards = Array.isArray(r.shards) ? r.shards.join('+') : ''; + return '
' + + '' + escapeHtml(String(r.id)) + '' + + '' + escapeHtml(shards) + '' + + '' + fmtScore(Number(r.score)) + '' + + '
'; + }).join(''); + el.innerHTML = header + lines; +} + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); +} + +export function mountViewer(container, capturer) { + if (!container) return () => {}; + container.classList.add('fv-root'); + container.innerHTML = [ + '', + '
', + '
Euclidean: waiting for first query…
', + '
Hyperbolic: waiting for first query…
', + '
', + '
no snapshots yet
', + ].join(''); + + const elEuc = container.querySelector('[data-fv="euclidean"]'); + const elHyp = container.querySelector('[data-fv="hyperbolic"]'); + const elFinal = container.querySelector('[data-fv="final"]'); + + function render(snap) { + if (!snap) return; + const shards = Array.isArray(snap.shards) ? snap.shards : []; + // Find by canonical names; fall back to index order. + let eShard = shards.find((s) => s && s.name === 'euclidean'); + let hShard = shards.find((s) => s && s.name === 'hyperbolic'); + if (!eShard) eShard = shards[0] || null; + if (!hShard) hShard = shards[1] || null; + if (eShard) renderShardTable(elEuc, eShard); + else elEuc.innerHTML = '
Euclidean: no shard
'; + if (hShard) renderShardTable(elHyp, hShard); + else elHyp.innerHTML = '
Hyperbolic: shard unavailable (wasm not loaded)
'; + renderFinal(elFinal, snap); + } + + const unsub = capturer && typeof capturer.subscribe === 'function' + ? capturer.subscribe(render) + : () => {}; + return unsub; +} diff --git a/AI-Car-Racer/main.js b/AI-Car-Racer/main.js index 2e0719b..1eaf31f 100644 --- a/AI-Car-Racer/main.js +++ b/AI-Car-Racer/main.js @@ -370,6 +370,40 @@ if (typeof window !== 'undefined') { __applyUrlConsistencyFlag(); } +// Phase 2A (F2) — honour `?federation=1` at boot. Opt-in flag that +// flips the bridge to the fan-out + union + GNN-rerank retrieval path. +// Default off → recommendSeeds is byte-identical to the pre-2A single- +// index behaviour. Mirrors the `?consistency=` / `?hhnsw=1` poll-until- +// ready pattern so we don't race the sidecar loader on slow first loads. +async function __applyUrlFederationFlag(){ + let on = false; + try { + var usp = new URLSearchParams(window.location.search || ''); + on = usp.get('federation') === '1'; + } catch (_) { return; } + if (!on) return; + var b = null; + for (let i = 0; i < 20; i++) { + b = window.__rvBridge; + if (b && typeof b.ready === 'function' && typeof b.setFederationEnabled === 'function') break; + await new Promise(res => setTimeout(res, 100)); + } + if (!b || typeof b.ready !== 'function' || typeof b.setFederationEnabled !== 'function') { + console.warn('[ruvector] URL flag ?federation=1 — bridge never appeared'); + return; + } + try { + await b.ready(); + b.setFederationEnabled(true); + console.log('[ruvector] federation mode enabled via URL flag'); + } catch (e) { + console.warn('[ruvector] setFederationEnabled from URL flag failed', e); + } +} +if (typeof window !== 'undefined') { + __applyUrlFederationFlag(); +} + // ----------------------------------------------------------------------------- // Metrics HUD — per-generation survival %, median / p90 checkpoints, wall-bumps. // Also serves as the on-screen data source for __runBenchmark / __abTest CSVs. diff --git a/AI-Car-Racer/ruvectorBridge.js b/AI-Car-Racer/ruvectorBridge.js index 674b504..fa2ebc5 100644 --- a/AI-Car-Racer/ruvectorBridge.js +++ b/AI-Car-Racer/ruvectorBridge.js @@ -25,6 +25,14 @@ import { isHyperbolicReady, HyperbolicVectorDB, } from './hyperbolicAdapter.js'; +// Phase 2A — F2 federated search. When _federationEnabled is on, recommendSeeds +// fans out to BOTH _brainDB (Euclidean) and _brainDB_hyperbolic in parallel, +// unions via F5 hash dedup, and reranks the union with the same GNN scorer the +// single-index path uses. The modules are plain JS — no wasm — so we import +// them eagerly. +import { fanOut, fanOutSync, kPrime as _kPrime } from './federation/fanout.js'; +import { unionByHash, selectTopK } from './federation/rerank.js'; +import { hashBrain } from './archive/hash.js'; // P3.B — lineage DAG. Replaces the hand-walked parentIds traversal in // getLineage() with a cycle-safe DAG structure (ruvector_dag_wasm) shadowed // by a JS-side adjacency list for O(depth) queries. Same fallback discipline @@ -172,10 +180,38 @@ export function setBypassLora(on) { _bypassLora = !!on; } let _ready = null; let _brainDB = null; +// Phase 2A — F2. When the hyperbolic wasm loads we ALSO keep a Hyperbolic +// instance of the brain index populated in parallel with the Euclidean one, +// so federated search can fan out to both without a rebuild stall. This is +// decoupled from `_indexKind`: `_brainDB` stays the "primary" single-index +// view (which setIndexKind swaps between), while `_brainDB_hyperbolic` is a +// strictly additive shadow — populated only when isHyperbolicReady() and +// only used when _federationEnabled flips true. The memory cost is a second +// HNSW over brains (~same size as the Euclidean one); bounded by archive. +let _brainDB_hyperbolic = null; let _trackDB = null; let _dynamicsDB = null; let _cnn = null; +// Phase 2A — F2. Off by default → recommendSeeds is byte-identical to the +// pre-2A single-index path. Flipped via setFederationEnabled({on}). +let _federationEnabled = false; +// Diagnostic counters surfaced via getFederationStats() — updated on every +// federated recommendSeeds() call. `shards` is the shard count that was +// actually fanned out to (degrades to 1 when hyperbolic didn't load). +const _federationStats = { + enabled: false, + shards: 0, + lastKPrime: 0, + lastUnionSize: 0, + lastDedupeHits: 0, +}; +// Viewer capturer (federation/viewer.js). UI mounts may subscribe; the bridge +// pushes a snapshot after each federated query. Stays null until assigned via +// setFederationCapturer so that headless smoke tests don't pay for the render +// hook. Plain object with an `onSnapshot(snap)` method. +let _federationCapturer = null; + // P3.A — index geometry. `_indexKind` is the *active* backend ('euclidean' or // 'hyperbolic'), flipped at ready() time based on the `?hhnsw=1` URL flag OR // by the A/B toggle strip calling setIndexKind() at runtime. We always boot @@ -209,6 +245,15 @@ function rebuildIndicesFromMirror() { _brainDB = new IndexClass(FLAT_LENGTH, 'cosine'); _trackDB = new IndexClass(TRACK_DIM, 'cosine'); _dynamicsDB = new IndexClass(DYNAMICS_DIM, 'cosine'); + // Phase 2A — rebuild the shadow hyperbolic brain index too when + // available. We don't shadow the track / dynamics DBs — federation only + // fans out over the brain index (track / dynamics are joins, not + // retrieval shards). When hyperbolic wasn't loaded this stays null. + if (isHyperbolicReady()) { + _brainDB_hyperbolic = new HyperbolicVectorDB(FLAT_LENGTH, 'cosine'); + } else { + _brainDB_hyperbolic = null; + } for (const [id, { vector, meta }] of _trackMirror) { _trackDB.insert(vector, id, meta || {}); } @@ -225,6 +270,10 @@ function rebuildIndicesFromMirror() { const entry = _brainMirror.get(id); if (!entry) continue; _brainDB.insert(entry.vector, id, entry.meta || {}); + if (_brainDB_hyperbolic) { + try { _brainDB_hyperbolic.insert(entry.vector, id, entry.meta || {}); } + catch (e) { console.warn('[federation] hyperbolic shadow insert failed', e); } + } } } @@ -326,6 +375,14 @@ export function ready() { _brainDB = new IndexClass(FLAT_LENGTH, 'cosine'); _trackDB = new IndexClass(TRACK_DIM, 'cosine'); _dynamicsDB = new IndexClass(DYNAMICS_DIM, 'cosine'); + // Phase 2A — F2. Stand up the hyperbolic shadow brain index so federated + // search has something to fan out to, regardless of whether _indexKind is + // currently hyperbolic. When the wasm didn't load this stays null and + // federation gracefully degrades to Euclidean-only (see recommendSeeds). + if (isHyperbolicReady()) { + try { _brainDB_hyperbolic = new HyperbolicVectorDB(FLAT_LENGTH, 'cosine'); } + catch (e) { console.warn('[federation] hyperbolic shadow init failed', e); _brainDB_hyperbolic = null; } + } _cnn = new CnnEmbedder(); // default 224×224, 512-dim, L2-normalized // Kick off GNN + LoRA loads in parallel with hydrate(). Best-effort: if // either resolves to null, the corresponding code path silently falls back @@ -424,6 +481,15 @@ export function archiveBrain(brain, fitness, trackVec, generation = 0, parentIds if (dynamicsId !== null) meta.dynamicsId = dynamicsId; const id = _brainDB.insert(vec, null, meta); _brainMirror.set(id, { vector: vec, meta }); + // Phase 2A — F2 shadow insert. The shadow uses its own id space internally + // but we pass the Euclidean id through so both indexes return the SAME + // id on search (which is what unionByHash relies on to collapse the two + // hits into a single candidate). Failure here is non-fatal — the shadow + // just diverges from the primary by one brain, which federation tolerates. + if (_brainDB_hyperbolic) { + try { _brainDB_hyperbolic.insert(vec, id, meta); } + catch (e) { console.warn('[federation] hyperbolic shadow insert failed', e); } + } // F3 — remember the insertion order so exportSnapshot can replay it. _insertionOrder.push(id); // P3.B — incremental DAG add. Safe no-op when the dag wasm didn't load. @@ -515,20 +581,51 @@ export function recommendSeeds(trackVec, k = 5) { ? _getFrozenBrainIdSet() : null; - const candidates = new Map(); // brainId -> trackSim (best across matched tracks) + // Build a track-hit map once. Used by both the single-index path (inline + // below) and the federation branch (as the source of representative brain + // + trackSim per union candidate). Keyed by trackId → similarity. + let trackSimByTrackId = null; if (queryVec && !_trackDB.isEmpty()) { + trackSimByTrackId = new Map(); const trackHits = _trackDB.search(queryVec, Math.min(5, Number(_trackDB.len()))); - for (const th of trackHits) { - const sim = 1 - th.score; - for (const [bid, entry] of _brainMirror) { - // 1C frozen-mode filter: skip brains inserted after the freeze - // point. `frozenIds === null` in fresh/eventual modes so this - // branch degenerates to the existing check. - if (frozenIds && !frozenIds.has(bid)) continue; - if (entry.meta && entry.meta.trackId === th.id) { - const prev = candidates.get(bid); - if (prev === undefined || prev < sim) candidates.set(bid, sim); - } + for (const th of trackHits) trackSimByTrackId.set(th.id, 1 - th.score); + } + + // Phase 2A — F2 federation branch. Fans out to Euclidean + Hyperbolic + // shadow brain indexes in parallel using a representative brain vector + // derived from the best track-matching brain (or the highest-fitness + // brain when there's no track hit). Unions candidates by content hash + // (1D/F5), applies the 1C frozenIds filter to the union (NOT per-shard), + // then lets the GNN rerank the union. The final top-k obeys the same + // shape the single-index path returns. On graceful degrade (hyperbolic + // wasm missing), we fall out to Euclidean-only — which still flows + // through the fanout path so the code stays uniform. + if (_federationEnabled && queryVec) { + const fedOut = _recommendSeedsFederated({ + queryVec, + trackVec, + k, + frozenIds, + trackSimByTrackId, + }); + if (consistencyMode === 'eventual' && cacheKey) { + _consistencyRecordQuery(cacheKey, fedOut); + } + return fedOut; + } + + const candidates = new Map(); // brainId -> trackSim (best across matched tracks) + if (trackSimByTrackId) { + for (const [bid, entry] of _brainMirror) { + // 1C frozen-mode filter: skip brains inserted after the freeze + // point. `frozenIds === null` in fresh/eventual modes so this + // branch degenerates to the existing check. + if (frozenIds && !frozenIds.has(bid)) continue; + const tid = entry.meta && entry.meta.trackId; + if (tid != null && trackSimByTrackId.has(tid)) { + const sim = trackSimByTrackId.get(tid); + const prev = candidates.get(bid); + if (prev === undefined || prev < sim) candidates.set(bid, sim); } } } @@ -635,6 +732,246 @@ export function recommendSeeds(trackVec, k = 5) { return out; } +// Phase 2A — F2. Federated retrieval path. Fans out to all active brain +// shards in parallel, unions by content hash (F5), applies the 1C +// frozen-ids filter to the UNIONED set, reranks the union via the same +// GNN scorer the single-index path uses, and returns the top-k in the +// usual { id, vector, meta, score, trackSim, dynamicsSim } shape. +// +// Representative brain vector: we need a brain-dim query (FLAT_LENGTH), +// but the caller only has a track-dim vector (TRACK_DIM). We derive a +// representative by picking the best-fit brain whose track matches the +// query — "closest existing brain to this track" — and using its +// flattened weights as the brain-index query. When no track matches we +// fall back to the globally best-fit brain so federation still produces +// something on cold / novel tracks. This is a pragmatic bridge between +// the track-keyed query API recommendSeeds has historically accepted and +// the brain-keyed search the HNSW indexes speak natively. +function _pickRepresentativeBrain({ trackSimByTrackId, frozenIds }) { + let bestId = null; + let bestScore = -Infinity; + if (trackSimByTrackId && trackSimByTrackId.size > 0) { + for (const [bid, entry] of _brainMirror) { + if (frozenIds && !frozenIds.has(bid)) continue; + const tid = entry.meta && entry.meta.trackId; + if (tid == null || !trackSimByTrackId.has(tid)) continue; + const tsim = trackSimByTrackId.get(tid); + const fit = (entry.meta && entry.meta.fitness) || 0; + // Rank by trackSim * (1 + normalised fitness) so a closer track + // wins ties against a marginally-fitter brain on a weaker track. + const s = tsim * (1 + Math.tanh(fit / 100)); + if (s > bestScore) { bestScore = s; bestId = bid; } + } + } + if (bestId == null) { + // Cold-fallback — pick the globally highest-fitness brain among the + // non-frozen-filtered set. + for (const [bid, entry] of _brainMirror) { + if (frozenIds && !frozenIds.has(bid)) continue; + const fit = (entry.meta && entry.meta.fitness) || 0; + if (fit > bestScore) { bestScore = fit; bestId = bid; } + } + } + return bestId; +} + +function _recommendSeedsFederated({ queryVec, trackVec, k, frozenIds, trackSimByTrackId }) { + const kk = Math.max(1, k | 0); + // Build the shard list. Hyperbolic shard is only included when the + // shadow index is populated (wasm loaded + archive hydrated through + // the shadow path). Missing → federation gracefully degrades to + // Euclidean-only and we note it in the stats. + const shards = [{ name: 'euclidean', db: _brainDB, metric: 'cosine' }]; + if (_brainDB_hyperbolic && !_brainDB_hyperbolic.isEmpty()) { + shards.push({ name: 'hyperbolic', db: _brainDB_hyperbolic, metric: 'poincare' }); + } else if (_federationEnabled && !_brainDB_hyperbolic) { + // Only warn once per session — keep the hot path silent. + if (!_federationStats._degradeWarned) { + console.warn('[federation] hyperbolic shadow unavailable — degrading to Euclidean-only'); + _federationStats._degradeWarned = true; + } + } + + const repId = _pickRepresentativeBrain({ trackSimByTrackId, frozenIds }); + if (!repId || !_brainMirror.has(repId)) { + // Nothing to query with — empty archive or every brain filtered out. + _federationStats.enabled = true; + _federationStats.shards = shards.length; + _federationStats.lastKPrime = _kPrime(kk, shards.length); + _federationStats.lastUnionSize = 0; + _federationStats.lastDedupeHits = 0; + _pushFederationSnapshot({ k: kk, shards: [], unionSize: 0, dedupeHits: 0, final: [] }); + return []; + } + const repVec = _brainMirror.get(repId).vector; + const shardResults = fanOutSync(repVec, kk, shards); + const kp = shardResults[0] ? shardResults[0].kPrime : _kPrime(kk, shards.length); + + // Hash lookup: every candidate id is a brain id in _brainMirror, so + // we can compute the xxHash32 of its flat vector on demand. This is + // the "F5 dedup via has()" point of contact — ids that collide on + // hash (same content, different ids) collapse into one union entry. + const hashLookup = (id) => { + const entry = _brainMirror.get(id); + if (!entry || !(entry.vector instanceof Float32Array)) return null; + try { return hashBrain(entry.vector); } catch (_) { return null; } + }; + const { candidates: unionPre, dedupeHits } = unionByHash(shardResults, hashLookup); + + // Apply the 1C frozen-ids filter to the UNION (not per-shard). A brain + // archived after freeze can legitimately surface from either shard + // (both get populated on archiveBrain), so the filter has to run here. + const union = frozenIds + ? unionPre.filter((c) => frozenIds.has(c.id)) + : unionPre; + + // Build a trackSim map for every union candidate so GNN + final + // composite score can use the existing formula shape. + const candidatesMap = new Map(); + for (const c of union) { + const entry = _brainMirror.get(c.id); + if (!entry) continue; + const tid = entry.meta && entry.meta.trackId; + const tsim = (trackSimByTrackId && tid != null && trackSimByTrackId.has(tid)) + ? trackSimByTrackId.get(tid) : 0; + candidatesMap.set(c.id, tsim); + } + + // GNN rerank over the unioned candidate set. Uses the exact same + // gnnScore call the single-index path does (just a larger set). + // Respects the P4.A reranker policy: 'none' → skip, 'ema' / 'auto' → + // fall through to EMA/obs weighting, 'gnn' → force GNN when loaded. + let useGnn = false; + let skipRerank = false; + if (_forceEma) useGnn = false; + else if (_rerankerPolicy === 'none') skipRerank = true; + else if (_rerankerPolicy === 'ema') useGnn = false; + else if (_rerankerPolicy === 'gnn') useGnn = gnnIsReady(); + else useGnn = gnnIsReady() && _brainMirror.size >= GNN_MIN_ARCHIVE; + const gnnMap = (useGnn && candidatesMap.size > 0) ? gnnScore(_brainMirror, candidatesMap) : null; + if (skipRerank) _rerankerMode = 'none'; + else if (useGnn && gnnMap) _rerankerMode = 'gnn'; + else _rerankerMode = candidatesMap.size > 0 ? 'ema' : 'none'; + + // Composite score per candidate, mirroring the single-index path's + // trackTerm * fitTerm * rerankTerm * dynamicsTerm product so federated + // results sit in the same band as non-federated ones (downstream + // consumers — the rv-panel rendering, main.js seed selection — don't + // need to special-case federation). + const dynamicsSimMap = new Map(); + const dynamicsActive = _useDynamics && _queryDynamicsVec && !_dynamicsDB.isEmpty(); + if (dynamicsActive) { + const dHits = _dynamicsDB.search(_queryDynamicsVec, Math.min(_dynamicsMirror.size, 25)); + const hitMap = new Map(); + for (const h of dHits) hitMap.set(h.id, 1 - h.score); + for (const c of union) { + const entry = _brainMirror.get(c.id); + const did = entry && entry.meta && entry.meta.dynamicsId; + if (did != null && hitMap.has(did)) dynamicsSimMap.set(c.id, hitMap.get(did)); + } + } + + const scoreMap = new Map(); + for (const c of union) { + const entry = _brainMirror.get(c.id); + if (!entry) continue; + const trackSim = candidatesMap.get(c.id) || 0; + const normFit = Math.tanh(((entry.meta && entry.meta.fitness) || 0) / 100); + const trackTerm = 0.5 + 0.5 * trackSim; + const fitTerm = 0.5 + 0.5 * normFit; + let rerankTerm; + if (skipRerank) rerankTerm = 1; + else if (gnnMap && gnnMap.has(c.id)) rerankTerm = gnnMap.get(c.id); + else { + const obs = _observations.get(c.id); + rerankTerm = 1 + 0.3 * (obs ? obs.weight : 0); + } + const dynamicsSim = dynamicsSimMap.has(c.id) ? dynamicsSimMap.get(c.id) : 0; + const dynamicsTerm = dynamicsActive ? (1 + DYNAMICS_TERM_WEIGHT * dynamicsSim) : 1; + scoreMap.set(c.id, trackTerm * fitTerm * rerankTerm * dynamicsTerm); + } + const top = selectTopK(union, scoreMap, kk); + + // Re-hydrate to the canonical recommendSeeds return shape. + const out = top.map((t) => { + const entry = _brainMirror.get(t.id); + const trackSim = candidatesMap.get(t.id) || 0; + const dynamicsSim = dynamicsSimMap.has(t.id) ? dynamicsSimMap.get(t.id) : 0; + return { + id: t.id, + vector: entry ? entry.vector : null, + meta: entry ? entry.meta : null, + score: t.score, + trackSim, + dynamicsSim, + // Federation-only: which shard(s) surfaced this id. Keeps the + // viewer's provenance column honest without bloating the single- + // index return path (federated-disabled callers never see it). + shards: t.shards, + }; + }); + + _federationStats.enabled = true; + _federationStats.shards = shards.length; + _federationStats.lastKPrime = kp; + _federationStats.lastUnionSize = union.length; + _federationStats.lastDedupeHits = dedupeHits; + + // Push a snapshot to the viewer for diagnostic rendering. No-op when + // no capturer is registered (headless tests). + _pushFederationSnapshot({ + k: kk, + kPrime: kp, + shards: shardResults, + unionSize: union.length, + dedupeHits, + final: out.map((o) => ({ id: o.id, score: o.score, shards: o.shards, hash: null })), + }); + return out; +} + +function _pushFederationSnapshot(snap) { + if (!_federationCapturer || typeof _federationCapturer.onSnapshot !== 'function') return; + try { _federationCapturer.onSnapshot(snap); } catch (e) { console.warn('[federation] capturer push failed', e); } +} + +// Phase 2A — F2. Public federation controls. +// +// setFederationEnabled(on) flips the runtime switch. When ON, recommendSeeds +// takes the fan-out + union + rerank path; when OFF, behaviour is byte- +// identical to the pre-2A single-index path. This composes with 1C +// consistency modes — the cache + frozen-filter logic wraps the federated +// branch just like the single-index branch. +// +// isFederationEnabled() is a cheap read for UI gating. +// +// getFederationStats() returns the last-query diagnostic snapshot. Intended +// for the rv-panel viewer and tests; this is NOT the 3A index-stats stub. +export function setFederationEnabled(on) { + _federationEnabled = !!on; + if (_federationEnabled && !_brainDB_hyperbolic) { + console.warn('[federation] enabled but hyperbolic shadow missing — only Euclidean shard will be queried'); + } + return _federationEnabled; +} +export function isFederationEnabled() { return !!_federationEnabled; } +export function getFederationStats() { + return { + enabled: !!_federationEnabled, + shards: _federationStats.shards, + lastKPrime: _federationStats.lastKPrime, + lastUnionSize: _federationStats.lastUnionSize, + lastDedupeHits: _federationStats.lastDedupeHits, + // Snapshot of which shards the bridge *would* fan out to right now, + // so UI can show "1 shard (degraded)" vs "2 shards" without calling + // recommendSeeds. Not part of the plan's required shape but cheap. + availableShards: _brainDB_hyperbolic ? ['euclidean', 'hyperbolic'] : ['euclidean'], + }; +} +export function setFederationCapturer(capturer) { + _federationCapturer = capturer && typeof capturer.onSnapshot === 'function' ? capturer : null; +} + // Pull the frozen brain-id set out of the consistency module's opaque // snapshot reference. Centralising the unwrap here keeps the // "cap-by-insertionOrder" shortcut an implementation detail of the @@ -956,6 +1293,14 @@ export async function hydrate() { _brainDB.insert(vec, row.id, row.meta || {}); _brainMirror.set(row.id, { vector: vec, meta: row.meta || {} }); _insertionOrder.push(row.id); + // Phase 2A — F2 shadow hydrate. Keep the hyperbolic brain index in + // lock-step with the Euclidean one so federation can fan out over a + // hydrated archive the moment it's flipped on. Same id so union + // collapses cross-shard duplicates of the same brain. + if (_brainDB_hyperbolic) { + try { _brainDB_hyperbolic.insert(vec, row.id, row.meta || {}); } + catch (e) { console.warn('[federation] hyperbolic shadow hydrate failed', e); } + } } _tEnd('3d_insert_brains', _tBrains); const _tObs = _tStart(); @@ -1094,6 +1439,15 @@ export function hydrateFromFixture(fixture) { _brainDB = new IndexClass(FLAT_LENGTH, 'cosine'); _trackDB = new IndexClass(TRACK_DIM, 'cosine'); _dynamicsDB = new IndexClass(DYNAMICS_DIM, 'cosine'); + // Phase 2A — F2 fixture rehydrate for the shadow hyperbolic index. Same + // rule as the IDB hydrate path: when the wasm loaded we re-create the + // shadow from scratch and insert every brain below. + if (isHyperbolicReady()) { + try { _brainDB_hyperbolic = new HyperbolicVectorDB(FLAT_LENGTH, 'cosine'); } + catch (e) { console.warn('[federation] hyperbolic shadow rebuild failed', e); _brainDB_hyperbolic = null; } + } else { + _brainDB_hyperbolic = null; + } const toF32 = (v) => (v instanceof Float32Array) ? v : new Float32Array(v); for (const row of (fixture.tracks || [])) { const vec = toF32(row.vec); @@ -1107,6 +1461,11 @@ export function hydrateFromFixture(fixture) { _brainDB.insert(vec, row.id, row.meta || {}); _brainMirror.set(row.id, { vector: vec, meta: row.meta || {} }); _insertionOrder.push(row.id); + // Phase 2A — F2 shadow fixture rehydrate. + if (_brainDB_hyperbolic) { + try { _brainDB_hyperbolic.insert(vec, row.id, row.meta || {}); } + catch (e) { console.warn('[federation] hyperbolic shadow fixture insert failed', e); } + } } for (const row of (fixture.observations || [])) { _observations.set(row.id, { weight: row.weight || 0, count: row.count | 0 }); @@ -1203,6 +1562,25 @@ export function importSnapshot(s) { dynamicsMirror: _dynamicsMirror, observations: _observations, }); + // Phase 2A — F2. After applySnapshot we rebuild the hyperbolic shadow + // from the refreshed mirror in one pass so federation stays correct on + // any subsequent recommendSeeds(). applySnapshot itself doesn't know + // about the shadow — it's a 2A addition that composes with 1A's + // snapshot pipeline rather than being owned by it. + if (isHyperbolicReady()) { + try { + _brainDB_hyperbolic = new HyperbolicVectorDB(FLAT_LENGTH, 'cosine'); + for (const [id, { vector, meta }] of _brainMirror) { + try { _brainDB_hyperbolic.insert(vector, id, meta || {}); } + catch (e) { console.warn('[federation] shadow snapshot insert failed', e); } + } + } catch (e) { + console.warn('[federation] shadow snapshot rebuild failed', e); + _brainDB_hyperbolic = null; + } + } else { + _brainDB_hyperbolic = null; + } // Refresh our own insertion-order tracker to match what the importer // actually replayed. Further archiveBrain calls append to this same array. _insertionOrder = (s.hnsw && Array.isArray(s.hnsw.insertionOrder)) @@ -1287,6 +1665,16 @@ export async function _debugReset() { _observations.clear(); _insertionOrder = []; _queryDynamicsVec = null; + // Phase 2A — reset federation diagnostic counters. The shadow index is + // rebuilt by the next hydrate / hydrateFromFixture call, so we don't + // touch _brainDB_hyperbolic here (matching the pre-2A policy that + // _debugReset leaves the live _brainDB alone — hydrateFromFixture + // rebuilds it, same goes for the shadow). + _federationStats.enabled = false; + _federationStats.shards = 0; + _federationStats.lastKPrime = 0; + _federationStats.lastUnionSize = 0; + _federationStats.lastDedupeHits = 0; sonaEngineDebugReset(); try { dagDebugReset(); } catch (_) { /* safe to ignore */ } if (typeof indexedDB !== 'undefined') { diff --git a/AI-Car-Racer/uiPanels.js b/AI-Car-Racer/uiPanels.js index 1b429af..f19862d 100644 --- a/AI-Car-Racer/uiPanels.js +++ b/AI-Car-Racer/uiPanels.js @@ -186,6 +186,21 @@ ' ', ' ', '', + // Phase 2A (F2) — federation toggle. When checked, recommendSeeds fans + // out to BOTH the Euclidean and Hyperbolic brain indexes, unions the + // candidates, and lets the GNN reranker pick the final top-k. Default + // off so the default experience is unchanged; the mount point below + // holds the split-screen viewer that renders per-shard top-k + final + // unioned result when the toggle is on. + '
', + ' ', + ' ', + ' ', + '
', // Dynamics trajectory toggle (P1.C). Off by default — the plan keeps // this opt-in because it changes retrieval ordering. The count next to // the label shows how many archived brains have a dynamics vector @@ -351,6 +366,11 @@ consistency: root.querySelector('[data-rv="consistency"]'), consistencyRadios: root.querySelectorAll('[data-rv="consistency-radios"] input[type="radio"]'), consistencyTicks: root.querySelector('[data-rv="consistency-ticks"]'), + // Phase 2A (F2) — federation toggle + viewer mount. + federation: root.querySelector('[data-rv="federation"]'), + federationToggle: root.querySelector('[data-rv="federation-toggle"]'), + federationStatus: root.querySelector('[data-rv="federation-status"]'), + federationViewer: root.querySelector('[data-rv="federation-viewer"]'), }; // 1C — F4. Build the tick strip: 30 tiny dots that pulse via a @@ -434,6 +454,58 @@ } } + // Phase 2A (F2) — federation toggle. Mount a viewer container lazily on + // first flip so the initial panel paint stays cheap. The toggle wires + // into bridge.setFederationEnabled; the viewer subscribes to a capturer + // that the bridge populates on every federated recommendSeeds() call. + let _federationCapturer = null; + let _federationViewerMounted = false; + async function ensureFederationViewer() { + if (_federationViewerMounted) return; + if (!el.federationViewer) return; + try { + const viewerMod = await import('./federation/viewer.js'); + _federationCapturer = viewerMod.createCapturer(); + viewerMod.mountViewer(el.federationViewer, _federationCapturer); + const b = window.__rvBridge; + if (b && typeof b.setFederationCapturer === 'function') { + b.setFederationCapturer(_federationCapturer); + } + _federationViewerMounted = true; + } catch (e) { + console.warn('[rv-panel] federation viewer mount failed', e); + } + } + if (el.federationToggle) { + el.federationToggle.addEventListener('change', async function () { + const b = window.__rvBridge; + if (!b || typeof b.setFederationEnabled !== 'function') return; + const on = !!el.federationToggle.checked; + try { b.setFederationEnabled(on); } + catch (e) { console.warn('[rv-panel] setFederationEnabled failed', e); return; } + if (el.federationStatus) el.federationStatus.textContent = on ? 'on' : 'off'; + if (el.federationViewer) el.federationViewer.hidden = !on; + if (on) await ensureFederationViewer(); + }); + } + function renderFederation() { + const b = window.__rvBridge; + if (!b || typeof b.isFederationEnabled !== 'function') return; + let on = false; + try { on = b.isFederationEnabled(); } catch (_) { return; } + if (el.federationToggle && el.federationToggle.checked !== on) { + el.federationToggle.checked = on; + } + if (el.federationStatus) { + const stats = (typeof b.getFederationStats === 'function') ? b.getFederationStats() : null; + const shardCount = stats ? stats.shards : 0; + const suffix = on && shardCount ? ' (' + shardCount + ' shard' + (shardCount === 1 ? '' : 's') + ')' : ''; + el.federationStatus.textContent = (on ? 'on' : 'off') + suffix; + } + if (el.federationViewer) el.federationViewer.hidden = !on; + if (on && !_federationViewerMounted) ensureFederationViewer(); + } + // Master toggle: mutating window.rvDisabled is enough — every bridgeReady() // / bridgeReadyLocal() call site already re-reads the flag on each call // (bridgeReady in main.js reads the top-level `var rvDisabled`, which IS @@ -1159,6 +1231,7 @@ // Radio-group reflection is idempotent; the cost is ~3 DOM reads // at 2Hz. renderConsistency(); + renderFederation(); // Fast-path: nothing changed → no DOM writes, no recommendSeeds call. if ( diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md index ff08f3f..41cb384 100644 --- a/docs/plan/rulake-inspired-features.md +++ b/docs/plan/rulake-inspired-features.md @@ -24,7 +24,7 @@ pointing to the blocker. Keep the "Current focus" line at the top pointing at whichever phase is active so a newcomer knows where to jump in without reading the whole doc. -**Current focus:** Phase 1 complete — next up is Phase 2A (F2 federation) which is gated on 1B, or Phase 3 observability/polish swarm +**Current focus:** Phase 2B (F6 cross-tab) — gated on 1A snapshot format and 1D dedup (both shipped) **Last updated:** 2026-04-24 ### Phase 0 — Foundations _(sequential, 1 owner)_ @@ -56,7 +56,7 @@ claims (n=6+ across ≥2 sessions, tested on both Rect and Tri tracks). | Status | ID | Task | Depends on | Owner | PR/SHA | Done date | |:--:|:--:|------|-----------|-------|--------|-----------| -| ⬜ | 2A | F2 — Federated search with GNN rerank | 1B | | | | +| ✅ | 2A | F2 — Federated search with GNN rerank | 1B | Claude (subagent) | — | 2026-04-24 | | ⬜ | 2B | F6 — Cross-tab live training via BroadcastChannel | 1A, 1D | | | | **Phase 2 gate:** both rows ✅ + A/B convergence test (two-tab demo @@ -88,6 +88,19 @@ community-archive URL ships publicly). Append-only. Record any scope change, deferral, or non-obvious call that future-you would want to find. Newest at the top. +- **2026-04-24 — Phase 2A shipped.** Federation replaces the single- + index query with a fan-out to BOTH Euclidean and Hyperbolic + HNSW, over-requesting per shard via `k' = k + ⌈√(k ln S)⌉`, unioning + candidates by F5 hash, then GNN-reranking the union. The structural + change: `_brainDB_hyperbolic` now shadows `_brainDB` in lock-step + across archiveBrain / hydrate / hydrateFromFixture / + rebuildIndicesFromMirror / importSnapshot so federation can toggle at + runtime without rebuilds. Gated behind `?federation=1` or the UI + toggle; off by default. Harness 7/7 PASS including the critical + compose-with-1C claim (frozen-mode filter applies to the union, not + per-shard). Graceful degrade: if hyperbolic WASM didn't load, S=1 and + federation becomes "single-shard through federation plumbing". + - **2026-04-24 — Phase 1 complete.** Wave 2 shipped 1A then 1C sequentially (both edit ruvectorBridge.js stubs; running in parallel would race). 1A adds `archive/{exporter,importer,serialize}.js` + diff --git a/tests/federation-smoke.html b/tests/federation-smoke.html new file mode 100644 index 0000000..78ba0d2 --- /dev/null +++ b/tests/federation-smoke.html @@ -0,0 +1,225 @@ + + + + + + Federation smoke (F2) + + + +

Federation smoke (Phase 2A / F2)

+

Exercises setFederationEnabled, fanOut over + two brain indexes, hash-based union, GNN rerank on the union, and + composition with the 1C frozen consistency mode.

+
Running…
+ + + +
#claimverdictdetail
+ + + + + + + + From ae32aba6127dcb188846118ac395b2a6e127a68c Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 13:55:11 -0400 Subject: [PATCH 07/10] feat(rulake-phase-2b): cross-tab live training via BroadcastChannel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F6 from docs/plan/rulake-inspired-features.md. Closes Phase 2. When two tabs on the same origin open with ?crosstab=1, a brain archived in tab A arrives in tab B's archive within a microtask. No server, no sockets — pure browser primitive. F5's content-hash IDs make the protocol lockless by construction: two tabs computing the same brain produce the same hash, dedup collapses the repeat automatically. What this ships: - crosstab/channel.js — BroadcastChannel('vectorvroom-archive') wrapper. Per-tab random senderId; hello/hello-ack/bye protocol for peer count; graceful beforeunload bye; stats() for diagnostics. - crosstab/wire.js — single-brain payload codec. flat: Array on the wire (not Float32Array) so two tabs on slightly different builds still interop. hash is sender-computed but receiver re-derives via dedup. - ruvectorBridge.js — additive only. New state: _crosstabEnabled, _crosstabReceiving, _crosstabChannel, _crosstabSenderId. Broadcast hook in archiveBrain guarded by (_crosstabEnabled && !_crosstabReceiving). _onRemoteBrain wraps the local archiveBrain call in _crosstabReceiving=true/try/finally so the receive path is pure-one-way, never re-broadcasts. New exports setCrosstabEnabled / isCrosstabEnabled / getCrosstabStats / setCrosstabListeners / _onRemoteBrain. - uiPanels.js — flag-gated "🔗 N peer(s)" pill; pulses on received brain via a CSS class toggle. - style.css — .rv-crosstab-pill + rv-crosstab-pulse keyframes. - main.js — ?crosstab=1 URL flag applier, mirrors 1C / 2A pattern. - eli15/chapters/cross-tab-federation.js — full chapter replacing coming-soon stub: BroadcastChannel basics, why content-addressing makes it lockless, echo-prevention via _crosstabReceiving guard, worked two-tab example. - tests/crosstab-smoke.html — 3-claim harness simulating two tabs with two channels in one page. Also updates docs/plan/rulake-inspired-features.md: 2B closed, Phase 2 complete; milestone M2 ticked (Phase 1 merged); M3 in progress. Validated via agent-browser: - Harness PASS 3/3: A→B delivery within 500ms with full wire decode (flat=244, fitness=42.5, hashMatch=true); duplicate brain dedups to one canonical node with duplicateRatio=0.5 after 2 inserts; sender-echo suppression verified (0 self-echoes at A, 1 from B). - ?crosstab=1 flag enables crosstab: isCrosstabEnabled()=true, pill rendered, stats show started=true with a per-tab senderId. - Default boot (no flag) clean, no new console errors. Echo-loop guard: _crosstabReceiving module-scope boolean flipped around the receive-path archiveBrain call. Broadcast hook checks it. Belt-and-braces with the senderId===self drop in channel.js (which is redundant today since BroadcastChannel doesn't echo to sender, but documented as defense-in-depth for future relay topologies). Only Phase 3A's getIndexStats stub remains unimplemented in the bridge. Phase 2 sequencing discipline held: 2A→2B sequentially, both structural edits to ruvectorBridge.js, zero merge conflicts. --- AI-Car-Racer/crosstab/channel.js | 212 ++++++++++++++++ AI-Car-Racer/crosstab/wire.js | 102 ++++++++ .../eli15/chapters/cross-tab-federation.js | 80 +++++- AI-Car-Racer/main.js | 35 +++ AI-Car-Racer/ruvectorBridge.js | 131 ++++++++++ AI-Car-Racer/style.css | 29 +++ AI-Car-Racer/uiPanels.js | 69 +++++ docs/plan/rulake-inspired-features.md | 19 +- tests/crosstab-smoke.html | 239 ++++++++++++++++++ 9 files changed, 899 insertions(+), 17 deletions(-) create mode 100644 AI-Car-Racer/crosstab/channel.js create mode 100644 AI-Car-Racer/crosstab/wire.js create mode 100644 tests/crosstab-smoke.html diff --git a/AI-Car-Racer/crosstab/channel.js b/AI-Car-Racer/crosstab/channel.js new file mode 100644 index 0000000..115f06b --- /dev/null +++ b/AI-Car-Racer/crosstab/channel.js @@ -0,0 +1,212 @@ +// crosstab/channel.js +// Phase 2B — F6: Cross-tab live training. +// +// Thin wrapper over BroadcastChannel('vectorvroom-archive'). Each tab gets a +// random `senderId` at module load; peer-count is the number of distinct +// senderIds we've seen say hello. We drop any message whose senderId equals +// our own (BroadcastChannel is specified NOT to echo to the sender, but the +// guard costs nothing and protects future multi-channel setups). +// +// The wire protocol has three message types: +// { type: 'hello', senderId } — broadcast on start +// { type: 'bye', senderId } — broadcast on beforeunload +// { type: 'brain', senderId, payload } — a single-brain delta; payload +// shape defined by ./wire.js +// +// Connect semantics: on `start()` we post a `hello` so any already-open peers +// can count us; existing peers post a `hello` back (because they re-emit on +// their own start — we also answer peer `hello` messages with our own so late +// joiners learn the full peer set without a separate roll-call). No archive +// sync on connect — the plan explicitly calls out "incrementally, not full +// archive rebuild" for F6. Only NEW archives post-connect get shared. +// +// The stats() snapshot is exposed for the smoke harness and for the UI pill +// (peer-count rendering). Re-entrant start() / stop() are safe. + +const CHANNEL_NAME = 'vectorvroom-archive'; + +function randomSenderId() { + // 8 hex chars is plenty for a per-tab transient id; collision across two + // tabs open at once is ~1 in 4B which is acceptable for "is this my echo?". + const rnd = Math.floor(Math.random() * 0xffffffff) >>> 0; + return rnd.toString(16).padStart(8, '0'); +} + +let _senderId = randomSenderId(); +let _ch = null; +let _started = false; +let _onBrain = null; +let _onPeerCount = null; +let _sent = 0; +let _received = 0; +let _echoesDropped = 0; +// peerId -> timestamp of last seen. We keep the full set rather than a counter +// so a `bye` can drop a peer cleanly, and stale peers (tab crashed without +// posting bye) could be reaped via a future heartbeat — not needed for F6. +const _peers = new Map(); +let _beforeUnloadHandler = null; + +function notifyPeerCount() { + if (typeof _onPeerCount === 'function') { + try { _onPeerCount(_peers.size); } + catch (e) { console.warn('[crosstab] onPeerCount callback failed', e); } + } +} + +function addPeer(id) { + if (!id || id === _senderId) return; + const had = _peers.has(id); + _peers.set(id, Date.now()); + if (!had) notifyPeerCount(); +} + +function removePeer(id) { + if (!id) return; + const had = _peers.delete(id); + if (had) notifyPeerCount(); +} + +function handleMessage(ev) { + const msg = ev && ev.data; + if (!msg || typeof msg !== 'object') return; + const sid = msg.senderId; + if (!sid) return; + if (sid === _senderId) { + // Defense-in-depth: BroadcastChannel shouldn't loop back, but a future + // relay (e.g. SharedWorker fan-out) might. + _echoesDropped += 1; + return; + } + _received += 1; + switch (msg.type) { + case 'hello': { + addPeer(sid); + // Answer so the late joiner learns *we* exist too. This is the cheapest + // way to get mutual visibility without a separate roll-call message. + try { + if (_ch) _ch.postMessage({ type: 'hello-ack', senderId: _senderId }); + } catch (_) { /* channel closed mid-handler */ } + return; + } + case 'hello-ack': { + addPeer(sid); + return; + } + case 'bye': { + removePeer(sid); + return; + } + case 'brain': { + addPeer(sid); // any brain traffic proves liveness + if (typeof _onBrain === 'function' && msg.payload) { + try { _onBrain(msg.payload, sid); } + catch (e) { console.warn('[crosstab] onBrain callback failed', e); } + } + return; + } + default: + // Forward-compat: unknown types are ignored. + return; + } +} + +// Public API ------------------------------------------------------------------ + +export function start({ onBrain, onPeerCount } = {}) { + if (_started) return; + if (typeof BroadcastChannel === 'undefined') { + console.warn('[crosstab] BroadcastChannel unavailable in this environment'); + return; + } + _onBrain = typeof onBrain === 'function' ? onBrain : null; + _onPeerCount = typeof onPeerCount === 'function' ? onPeerCount : null; + try { + _ch = new BroadcastChannel(CHANNEL_NAME); + } catch (e) { + console.warn('[crosstab] failed to open BroadcastChannel', e); + _ch = null; + return; + } + _ch.addEventListener('message', handleMessage); + _started = true; + // Post hello. Existing tabs answer with hello-ack so peer-count is accurate + // within one RTT of start. + try { _ch.postMessage({ type: 'hello', senderId: _senderId }); } + catch (e) { console.warn('[crosstab] initial hello failed', e); } + // Emit an initial peer-count of 0 so the UI paints "no peers" rather than + // showing its previous state across a stop→start. + notifyPeerCount(); + // beforeunload: post `bye` so peers drop our entry immediately rather than + // carrying us forever. Non-load-bearing (peers would heal on next hello), + // but it makes the pill feel honest. + if (typeof window !== 'undefined') { + _beforeUnloadHandler = () => { + try { if (_ch) _ch.postMessage({ type: 'bye', senderId: _senderId }); } + catch (_) { /* tab is going away — nothing to do */ } + }; + window.addEventListener('beforeunload', _beforeUnloadHandler); + } +} + +export function stop() { + if (!_started) return; + try { if (_ch) _ch.postMessage({ type: 'bye', senderId: _senderId }); } catch (_) {} + if (_ch) { + try { _ch.removeEventListener('message', handleMessage); } catch (_) {} + try { _ch.close(); } catch (_) {} + } + if (typeof window !== 'undefined' && _beforeUnloadHandler) { + try { window.removeEventListener('beforeunload', _beforeUnloadHandler); } catch (_) {} + } + _ch = null; + _started = false; + _onBrain = null; + _onPeerCount = null; + _peers.clear(); + _beforeUnloadHandler = null; +} + +export function broadcastBrain(payload) { + if (!_started || !_ch || !payload) return false; + try { + _ch.postMessage({ type: 'brain', senderId: _senderId, payload }); + _sent += 1; + return true; + } catch (e) { + console.warn('[crosstab] broadcastBrain postMessage failed', e); + return false; + } +} + +export function getPeerCount() { + return _peers.size; +} + +export function getSenderId() { + return _senderId; +} + +export function isStarted() { + return _started; +} + +export function stats() { + return { + started: _started, + senderId: _senderId, + peerCount: _peers.size, + sent: _sent, + received: _received, + echoesDropped: _echoesDropped, + }; +} + +// Test hook — wipes state so a harness can rebuild without a page reload. +export function _debugReset() { + stop(); + _senderId = randomSenderId(); + _sent = 0; + _received = 0; + _echoesDropped = 0; + _peers.clear(); +} diff --git a/AI-Car-Racer/crosstab/wire.js b/AI-Car-Racer/crosstab/wire.js new file mode 100644 index 0000000..274a6de --- /dev/null +++ b/AI-Car-Racer/crosstab/wire.js @@ -0,0 +1,102 @@ +// crosstab/wire.js +// Phase 2B — F6 wire format for a single-brain delta broadcast between tabs. +// +// The payload is a FRAGMENT of the Phase 0 ArchiveSnapshot shape — one brain + +// its track vec + a small meta bag — NOT a whole snapshot. The receiving tab +// calls archiveBrain() with the decoded parts, so dedup (F5) is what collapses +// identical brains across tabs: two tabs archiving the same weights produce +// the same hash and only one node is created. That invariant is load-bearing — +// without it the receive path would grow the archive unbounded under replay. +// +// Why plain Array instead of Float32Array on the wire: +// BroadcastChannel uses structured clone, which DOES preserve Float32Array. +// But two tabs can be running slightly different builds of the app (one +// refreshed mid-session, one stale), and a future release might re-encode +// vectors. Plain JSON-ish Arrays are the lowest-common-denominator wire +// representation that stays cloneable across any version skew and round- +// trips through e.g. JSON.stringify for debugging. Cost is ~2x on the wire, +// which for a 244-float brain is ~3kB — well under the ~10MB/message cap +// browsers give BroadcastChannel. +// +// toWire(brain, fitness, trackVec, meta) → wire object +// brain : Float32Array of FLAT_LENGTH (the already-flattened weights) +// fitness : number (meta.fitness, optional — 0 if missing) +// trackVec : Float32Array(TRACK_DIM) | null +// meta : { generation?, parentIds?, fastestLap?, dynamicsVec?, ... } +// +// fromWire(msg) → { flat, hash, fitness, trackVec, meta } +// flat : Float32Array (rebuilt from Array) +// hash : string (the sender's hash; we re-verify on the receive path +// implicitly because archiveBrain recomputes via dedup) +// trackVec : Float32Array | null +// meta : passthrough object; `dynamicsVec` (if present) is lifted back +// into a Float32Array too. + +import { FLAT_LENGTH } from '../brainCodec.js'; +import { hashBrain } from '../archive/hash.js'; + +function f32ToArray(v) { + if (!v) return null; + // Array.from is the clearest read; for a 244-float brain the cost is + // immaterial (<0.1ms). + return Array.from(v); +} + +function arrayToF32(arr, expectedLen) { + if (!Array.isArray(arr)) return null; + if (expectedLen != null && arr.length !== expectedLen) return null; + const out = new Float32Array(arr.length); + for (let i = 0; i < arr.length; i++) out[i] = Number(arr[i]) || 0; + return out; +} + +export function toWire(flat, fitness, trackVec, meta) { + if (!(flat instanceof Float32Array) || flat.length !== FLAT_LENGTH) { + throw new Error(`crosstab/wire.toWire: flat must be Float32Array(${FLAT_LENGTH})`); + } + const hash = hashBrain(flat); + const outMeta = {}; + if (meta && typeof meta === 'object') { + if (Number.isFinite(meta.generation)) outMeta.generation = meta.generation | 0; + if (Array.isArray(meta.parentIds)) outMeta.parentIds = meta.parentIds.slice(); + if (Number.isFinite(meta.fastestLap)) outMeta.fastestLap = Number(meta.fastestLap); + if (meta.dynamicsVec instanceof Float32Array) { + outMeta.dynamicsVec = f32ToArray(meta.dynamicsVec); + } + } + return { + flat: f32ToArray(flat), + hash, + fitness: Number.isFinite(fitness) ? Number(fitness) : 0, + trackVec: (trackVec instanceof Float32Array) ? f32ToArray(trackVec) : null, + meta: outMeta, + }; +} + +export function fromWire(msg) { + if (!msg || typeof msg !== 'object') { + throw new Error('crosstab/wire.fromWire: payload missing'); + } + const flat = arrayToF32(msg.flat, FLAT_LENGTH); + if (!flat) { + throw new Error('crosstab/wire.fromWire: flat has wrong length'); + } + const trackVec = Array.isArray(msg.trackVec) ? arrayToF32(msg.trackVec) : null; + const metaIn = (msg.meta && typeof msg.meta === 'object') ? msg.meta : {}; + const meta = { + generation: Number.isFinite(metaIn.generation) ? (metaIn.generation | 0) : 0, + parentIds: Array.isArray(metaIn.parentIds) ? metaIn.parentIds.slice() : [], + }; + if (Number.isFinite(metaIn.fastestLap)) meta.fastestLap = Number(metaIn.fastestLap); + if (Array.isArray(metaIn.dynamicsVec)) { + const dyn = arrayToF32(metaIn.dynamicsVec); + if (dyn) meta.dynamicsVec = dyn; + } + return { + flat, + hash: typeof msg.hash === 'string' ? msg.hash : hashBrain(flat), + fitness: Number.isFinite(msg.fitness) ? Number(msg.fitness) : 0, + trackVec, + meta, + }; +} diff --git a/AI-Car-Racer/eli15/chapters/cross-tab-federation.js b/AI-Car-Racer/eli15/chapters/cross-tab-federation.js index 628812a..0943885 100644 --- a/AI-Car-Racer/eli15/chapters/cross-tab-federation.js +++ b/AI-Car-Racer/eli15/chapters/cross-tab-federation.js @@ -1,20 +1,74 @@ // eli15/chapters/cross-tab-federation.js -// Placeholder — real content ships with Phase 2B (F6). +// Phase 2B (F6) — cross-tab live training. Ships with the F6 wiring. export default { id: 'cross-tab-federation', title: 'Two browser tabs training in sync', - oneLiner: 'Open two tabs on different tracks; each discovers a good brain, the other tab sees it arrive in real time.', - comingSoon: true, + oneLiner: 'Open two tabs with ?crosstab=1; a brain archived in one appears in the other in real time, with no server.', body: [ - '

Coming soon — lands with Phase 2B of the RuLake-inspired roadmap.

', - '

Once every brain is content-addressed by a hash (F5) and every archive', - 'can be serialized as a snapshot (F3), sharing a brain between two tabs', - 'becomes a one-line broadcast: BroadcastChannel.postMessage({ brain,', - 'hash }). The receiving tab computes the hash, sees it\'s new, inserts.

', - '

No locking, no conflicts — because content-addressing makes the', - 'identity of a brain independent of where it was created. Two tabs', - 'arriving at the same weights produce the same hash and converge for free.

', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 2B.

', + '

The one-line version. When you open the same page twice,', + 'both tabs share the archive through a browser primitive called', + 'BroadcastChannel. Each time a brain is added in tab A, a tiny', + 'message flies to tab B (and any other tabs); tab B inserts it into its own', + 'archive. No server. No sockets. Just the browser itself.

', + + '

What BroadcastChannel is (and isn\'t)

', + '

new BroadcastChannel(\'vectorvroom-archive\') gives every', + 'same-origin tab a shared bus. postMessage on one end shows up', + 'as a message event on every other end — but not on the', + 'sender. It\'s synchronous-ish (microtask-delivered), has no network hop, and', + 'structured-clones the payload so you can send typed arrays without', + 'stringifying. It\'s also ephemeral: there\'s no history, no replay. If a tab', + 'is closed when a message is sent, it missed it — forever.

', + + '

Why this is lockless by construction

', + '

Two tabs broadcasting at each other sounds like a recipe for conflicts:', + 'what if both tabs independently archive the same brain at nearly', + 'the same instant? In a classical system you\'d need a lock, or a CRDT, or a', + 'server to pick a winner.

', + '

We don\'t need any of that, because Phase 1D made brains', + 'content-addressed. The id of a brain is the hash of its weights', + '(xxHash32 over the flattened Float32Array). Two', + 'tabs that compute the same brain compute the same hash — deterministically,', + 'byte-for-byte — and the dedup table in archive/dedup.js', + 'collapses any repeat into the canonical node. So the protocol is simply:', + 'archive locally, broadcast eagerly, trust the hash. If the same', + 'brain arrives twice, the second insert is a no-op. If it never arrives, the', + 'local archive is still correct. There is no coordinator and nothing to', + 'agree on.

', + + '

Preventing echo loops

', + '

The naive wiring loops forever: tab A archives → broadcasts → tab B', + 'receives → archives → broadcasts → tab A receives → archives → broadcasts…', + 'BroadcastChannel doesn\'t echo to the sender, which stops the', + 'trivial case, but tab A\'s broadcast still returns to A through tab B\'s', + 're-broadcast. Two guards stop this. First, every message carries a random', + 'per-tab senderId; we drop any message whose senderId equals', + 'our own (defense-in-depth for future relay setups). Second, and more', + 'importantly, when the bridge receives a remote brain it sets a', + '_crosstabReceiving flag before calling archiveBrain', + '— and the broadcast hook checks that flag and declines to re-emit. So the', + 'receive path is a pure one-way archive update, never a re-broadcast.

', + + '

A worked example

', + '

Imagine tab A trains on a rectangular track and stumbles onto a brain', + 'that can lap cleanly (fitness 950). archiveBrain hashes the', + 'weights to 7f3a1c08, inserts it locally, and posts', + '{type: \'brain\', senderId: \'a1b2\', payload: {flat, hash: \'7f3a1c08\',', + 'fitness: 950, …}} on the channel. A millisecond later tab B\'s', + 'message handler fires; it decodes the payload, calls its own', + 'archiveBrain under the receive guard. The hash hits the dedup', + 'table (new entry), the brain lands, and the next seeding round on tab B', + 'can pick it up — even though tab B is training on the triangle track and', + 'never would have discovered it locally.

', + '

If tab A\'s user reloads and re-runs the same seed and arrives at', + '7f3a1c08 again, the broadcast still happens — and this time', + 'tab B\'s dedup short-circuits: inserted: false, no new node,', + 'no growth. Same with replays, out-of-order messages, tab C joining late.', + 'The hash is the shared truth.

', + + '

Flag: this is opt-in while the feature bakes. Append', + '?crosstab=1 to the URL on two or more tabs; a small', + '🔗 N peer(s) pill appears in the training panel and pulses', + 'green on every received brain.

', ].join('\n'), }; diff --git a/AI-Car-Racer/main.js b/AI-Car-Racer/main.js index 1eaf31f..f8e5da7 100644 --- a/AI-Car-Racer/main.js +++ b/AI-Car-Racer/main.js @@ -404,6 +404,41 @@ if (typeof window !== 'undefined') { __applyUrlFederationFlag(); } +// Phase 2B (F6) — honour `?crosstab=1` at boot. Opt-in flag that opens the +// BroadcastChannel('vectorvroom-archive') and wires archiveBrain's broadcast +// hook. Default off → no channel is opened and archiveBrain's hot path is +// one boolean check per insert. Mirrors the ?federation=1 poll-until-ready +// pattern because the bridge sidecar can finish loading slightly after the +// DOM is ready on slow first-loads. +async function __applyUrlCrosstabFlag(){ + let on = false; + try { + var usp = new URLSearchParams(window.location.search || ''); + on = usp.get('crosstab') === '1'; + } catch (_) { return; } + if (!on) return; + var b = null; + for (let i = 0; i < 20; i++) { + b = window.__rvBridge; + if (b && typeof b.ready === 'function' && typeof b.setCrosstabEnabled === 'function') break; + await new Promise(res => setTimeout(res, 100)); + } + if (!b || typeof b.ready !== 'function' || typeof b.setCrosstabEnabled !== 'function') { + console.warn('[ruvector] URL flag ?crosstab=1 — bridge never appeared'); + return; + } + try { + await b.ready(); + b.setCrosstabEnabled(true); + console.log('[ruvector] cross-tab live training enabled via URL flag'); + } catch (e) { + console.warn('[ruvector] setCrosstabEnabled from URL flag failed', e); + } +} +if (typeof window !== 'undefined') { + __applyUrlCrosstabFlag(); +} + // ----------------------------------------------------------------------------- // Metrics HUD — per-generation survival %, median / p90 checkpoints, wall-bumps. // Also serves as the on-screen data source for __runBenchmark / __abTest CSVs. diff --git a/AI-Car-Racer/ruvectorBridge.js b/AI-Car-Racer/ruvectorBridge.js index fa2ebc5..bae5560 100644 --- a/AI-Car-Racer/ruvectorBridge.js +++ b/AI-Car-Racer/ruvectorBridge.js @@ -33,6 +33,18 @@ import { import { fanOut, fanOutSync, kPrime as _kPrime } from './federation/fanout.js'; import { unionByHash, selectTopK } from './federation/rerank.js'; import { hashBrain } from './archive/hash.js'; +// Phase 2B — F6 cross-tab live training. Thin wrapper over BroadcastChannel; +// when enabled, archiveBrain broadcasts a single-brain delta after a +// successful insert, and received deltas are routed back through archiveBrain +// locally (the F5 dedup short-circuits identical brains automatically). +import { + start as crosstabStart, + stop as crosstabStop, + broadcastBrain as crosstabBroadcast, + isStarted as crosstabIsStarted, + stats as crosstabStats, +} from './crosstab/channel.js'; +import { toWire as crosstabToWire, fromWire as crosstabFromWire } from './crosstab/wire.js'; // P3.B — lineage DAG. Replaces the hand-walked parentIds traversal in // getLineage() with a cycle-safe DAG structure (ruvector_dag_wasm) shadowed // by a JS-side adjacency list for O(depth) queries. Same fallback discipline @@ -212,6 +224,18 @@ const _federationStats = { // hook. Plain object with an `onSnapshot(snap)` method. let _federationCapturer = null; +// Phase 2B — F6 cross-tab. Off by default → archiveBrain is byte-identical to +// the pre-2B path (one boolean check). Flipped via setCrosstabEnabled(true) +// which opens the BroadcastChannel; setCrosstabEnabled(false) closes it. +// `_crosstabReceiving` guards the receive path so the receive-then-archiveBrain +// call doesn't re-broadcast and trigger an infinite echo loop across tabs. +let _crosstabEnabled = false; +let _crosstabReceiving = false; +// Optional UI hook — uiPanels subscribes to get a pulse when a remote brain +// lands. Stays null so headless tests don't render. +let _crosstabOnReceive = null; +let _crosstabOnPeerCount = null; + // P3.A — index geometry. `_indexKind` is the *active* backend ('euclidean' or // 'hyperbolic'), flipped at ready() time based on the `?hhnsw=1` URL flag OR // by the A/B toggle strip calling setIndexKind() at runtime. We always boot @@ -492,6 +516,23 @@ export function archiveBrain(brain, fitness, trackVec, generation = 0, parentIds } // F3 — remember the insertion order so exportSnapshot can replay it. _insertionOrder.push(id); + // Phase 2B — F6 cross-tab broadcast. One boolean check on the hot path when + // disabled. When enabled AND we're not currently replaying a remote brain + // (see _crosstabReceiving guard in _onRemoteBrain below), post a single- + // brain delta. The receiving tabs hash-dedup via F5, so a re-broadcast from + // A → B → A is collapsed to a single node — the echo guard is belt-and- + // -braces against runaway traffic, not correctness. + if (_crosstabEnabled && !_crosstabReceiving) { + try { + const wireMeta = { + generation: meta.generation, + parentIds: meta.parentIds, + }; + if (meta.fastestLap !== undefined) wireMeta.fastestLap = meta.fastestLap; + if (dynamicsVec instanceof Float32Array) wireMeta.dynamicsVec = dynamicsVec; + crosstabBroadcast(crosstabToWire(vec, meta.fitness, trackVec || null, wireMeta)); + } catch (e) { console.warn('[crosstab] broadcast failed', e); } + } // P3.B — incremental DAG add. Safe no-op when the dag wasm didn't load. // The DAG uses meta.parentIds to wire edges; unknown parents (not yet in // the mirror) are silently skipped — same relaxed contract as the legacy @@ -972,6 +1013,96 @@ export function setFederationCapturer(capturer) { _federationCapturer = capturer && typeof capturer.onSnapshot === 'function' ? capturer : null; } +// ─── Phase 2B — F6 cross-tab live training ────────────────────────────────── +// +// setCrosstabEnabled(true) opens a BroadcastChannel('vectorvroom-archive') +// and wires the receive callback to _onRemoteBrain below. setCrosstabEnabled +// (false) closes it. Off by default (URL flag ?crosstab=1 flips it on at boot +// via main.js). The receive path re-enters archiveBrain under the +// _crosstabReceiving guard so the broadcast hook above short-circuits — that +// guard is what keeps two tabs from echoing forever when they see each +// other's delta on the channel. +export function setCrosstabEnabled(on) { + const want = !!on; + if (want === _crosstabEnabled) return _crosstabEnabled; + if (want) { + crosstabStart({ + onBrain: (payload /* senderId unused here */) => _onRemoteBrain(payload), + onPeerCount: (n) => { + if (typeof _crosstabOnPeerCount === 'function') { + try { _crosstabOnPeerCount(n); } catch (_) {} + } + }, + }); + _crosstabEnabled = crosstabIsStarted(); + } else { + crosstabStop(); + _crosstabEnabled = false; + } + return _crosstabEnabled; +} +export function isCrosstabEnabled() { return !!_crosstabEnabled; } +export function getCrosstabStats() { return crosstabStats(); } +// UI subscription hooks — uiPanels wires these so the pill can animate on +// remote-brain receive and update the peer count. Both are optional; passing +// null clears the subscription. +export function setCrosstabListeners({ onReceive, onPeerCount } = {}) { + _crosstabOnReceive = typeof onReceive === 'function' ? onReceive : null; + _crosstabOnPeerCount = typeof onPeerCount === 'function' ? onPeerCount : null; +} + +// Receive path: decode the wire payload and route it back through archiveBrain. +// Setting _crosstabReceiving before the call prevents the broadcast hook from +// firing on this re-entrant insert — otherwise tab A → broadcast → tab B +// receives → archiveBrain → broadcast → tab A receives → archiveBrain ... +// infinite ping-pong. Dedup (F5) would eventually collapse the nodes but the +// message traffic would still saturate the channel. The guard shuts it down +// at the source. +// +// NOTE: we intentionally DO NOT short-circuit here even if we recognise the +// hash — the plan calls out trusting dedup instead of re-implementing +// idempotency. archiveBrain → (insert) → (mirror set) is the hot path that +// already answers "have I seen this hash?" via dedup; re-checking here would +// fork the invariant into two places. +export function _onRemoteBrain(payload) { + if (!payload) return null; + if (!_brainDB) return null; // not ready — drop silently (pre-ready deltas will re-arrive) + let decoded; + try { decoded = crosstabFromWire(payload); } + catch (e) { console.warn('[crosstab] fromWire failed', e); return null; } + const { flat, fitness, trackVec, meta } = decoded; + let brain; + try { brain = unflatten(flat); } + catch (e) { console.warn('[crosstab] unflatten failed', e); return null; } + const dynamicsVec = (meta && meta.dynamicsVec instanceof Float32Array) ? meta.dynamicsVec : undefined; + const fastestLap = (meta && Number.isFinite(meta.fastestLap)) ? meta.fastestLap : undefined; + const generation = (meta && Number.isFinite(meta.generation)) ? meta.generation : 0; + const parentIds = (meta && Array.isArray(meta.parentIds)) ? meta.parentIds : []; + // Defensive: the sender might be on a slightly different build that + // encoded a trackVec of a different dim. Feed it through only when + // length matches this tab's TRACK_DIM — else pass null and let the + // received brain land track-less (archiveBrain supports that path). + const safeTrackVec = (trackVec instanceof Float32Array && trackVec.length === TRACK_DIM) + ? trackVec : null; + _crosstabReceiving = true; + let id = null; + try { + id = archiveBrain(brain, fitness, safeTrackVec, generation, parentIds, fastestLap, dynamicsVec); + } catch (e) { + console.warn('[crosstab] archiveBrain on receive failed', e); + } finally { + _crosstabReceiving = false; + } + // Fire the UI callback regardless of whether dedup made this a no-op + // (from the UI's perspective "a remote brain arrived" is the interesting + // event; whether the archive grew is an implementation detail). + if (typeof _crosstabOnReceive === 'function') { + try { _crosstabOnReceive({ id, hash: decoded.hash }); } + catch (_) {} + } + return id; +} + // Pull the frozen brain-id set out of the consistency module's opaque // snapshot reference. Centralising the unwrap here keeps the // "cap-by-insertionOrder" shortcut an implementation detail of the diff --git a/AI-Car-Racer/style.css b/AI-Car-Racer/style.css index 68b1541..4022b89 100644 --- a/AI-Car-Racer/style.css +++ b/AI-Car-Racer/style.css @@ -2088,3 +2088,32 @@ label { white-space: nowrap; border: 0; } + +/* ============================================================= + Phase 2B (F6) — cross-tab live-training pill + ============================================================= */ +.rv-crosstab { + display: flex; + align-items: center; + gap: 6px; + padding: 2px 0; +} +.rv-crosstab-pill { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 2px 8px; + border-radius: 10px; + background: rgba(98, 240, 184, 0.14); + color: #cbd8ff; + font-size: 12px; + user-select: none; + transition: background-color 200ms ease-out, box-shadow 200ms ease-out; +} +.rv-crosstab-pill-pulse { + animation: rv-crosstab-pulse 600ms ease-out forwards; +} +@keyframes rv-crosstab-pulse { + 0% { background: #62f0b8; box-shadow: 0 0 10px rgba(98, 240, 184, 0.65); } + 100% { background: rgba(98, 240, 184, 0.14); box-shadow: 0 0 0 rgba(0, 0, 0, 0); } +} diff --git a/AI-Car-Racer/uiPanels.js b/AI-Car-Racer/uiPanels.js index f19862d..4898fb3 100644 --- a/AI-Car-Racer/uiPanels.js +++ b/AI-Car-Racer/uiPanels.js @@ -201,6 +201,16 @@ ' ', ' ', '', + // Phase 2B (F6) — cross-tab live training connection indicator. Rendered + // only when the ?crosstab=1 flag is on (the pill doesn't belong in the + // default panel while the feature is baking). Pulses briefly on every + // received remote brain so the learner can *see* the link working. + '', // Dynamics trajectory toggle (P1.C). Off by default — the plan keeps // this opt-in because it changes retrieval ordering. The count next to // the label shows how many archived brains have a dynamics vector @@ -371,6 +381,11 @@ federationToggle: root.querySelector('[data-rv="federation-toggle"]'), federationStatus: root.querySelector('[data-rv="federation-status"]'), federationViewer: root.querySelector('[data-rv="federation-viewer"]'), + // Phase 2B (F6) — cross-tab peer indicator (flag-gated by ?crosstab=1). + crosstab: root.querySelector('[data-rv="crosstab"]'), + crosstabPill: root.querySelector('[data-rv="crosstab-pill"]'), + crosstabPeers: root.querySelector('[data-rv="crosstab-peers"]'), + crosstabS: root.querySelector('[data-rv="crosstab-s"]'), }; // 1C — F4. Build the tick strip: 30 tiny dots that pulse via a @@ -488,6 +503,60 @@ if (on) await ensureFederationViewer(); }); } + // Phase 2B (F6) — cross-tab pill. Visible only when ?crosstab=1 URL flag is + // present (feature is baking behind a flag). Subscribes to the bridge's + // setCrosstabListeners so we get a callback on every received remote brain + // (for the green pulse) and on peer-count changes (for the "N peer(s)" + // readout). The subscription attempt is best-effort and retries a few + // times because the bridge sidecar can land slightly after the panel. + let _crosstabFlagOn = false; + try { + if (typeof URLSearchParams === 'function') { + _crosstabFlagOn = new URLSearchParams(window.location.search || '').get('crosstab') === '1'; + } + } catch (_) { _crosstabFlagOn = false; } + if (el.crosstab && _crosstabFlagOn) el.crosstab.hidden = false; + function renderCrosstabPeers(n) { + if (!el.crosstabPeers) return; + const count = Math.max(0, n | 0); + el.crosstabPeers.textContent = String(count); + if (el.crosstabS) el.crosstabS.textContent = count === 1 ? '' : 's'; + } + function pulseCrosstab() { + if (!el.crosstabPill) return; + el.crosstabPill.classList.remove('rv-crosstab-pill-pulse'); + void el.crosstabPill.getBoundingClientRect(); + el.crosstabPill.classList.add('rv-crosstab-pill-pulse'); + } + let _crosstabWired = false; + async function ensureCrosstabWiring() { + if (_crosstabWired) return; + for (let i = 0; i < 20 && !_crosstabWired; i++) { + const b = window.__rvBridge; + if (b && typeof b.setCrosstabListeners === 'function') { + try { + b.setCrosstabListeners({ + onReceive: () => pulseCrosstab(), + onPeerCount: (n) => renderCrosstabPeers(n), + }); + _crosstabWired = true; + // Paint the current peer count once on wire-up (zero until a peer + // actually says hello, but we want to replace any stale default). + try { + const s = (typeof b.getCrosstabStats === 'function') ? b.getCrosstabStats() : null; + if (s) renderCrosstabPeers(s.peerCount); + } catch (_) {} + } catch (e) { + console.warn('[rv-panel] setCrosstabListeners failed', e); + return; + } + } else { + await new Promise(res => setTimeout(res, 100)); + } + } + } + if (_crosstabFlagOn) ensureCrosstabWiring(); + function renderFederation() { const b = window.__rvBridge; if (!b || typeof b.isFederationEnabled !== 'function') return; diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md index 41cb384..16d6bfa 100644 --- a/docs/plan/rulake-inspired-features.md +++ b/docs/plan/rulake-inspired-features.md @@ -24,7 +24,7 @@ pointing to the blocker. Keep the "Current focus" line at the top pointing at whichever phase is active so a newcomer knows where to jump in without reading the whole doc. -**Current focus:** Phase 2B (F6 cross-tab) — gated on 1A snapshot format and 1D dedup (both shipped) +**Current focus:** Phase 3 — observability + polish swarm (3A, 3B, 3C parallel-safe) **Last updated:** 2026-04-24 ### Phase 0 — Foundations _(sequential, 1 owner)_ @@ -57,7 +57,7 @@ claims (n=6+ across ≥2 sessions, tested on both Rect and Tri tracks). | Status | ID | Task | Depends on | Owner | PR/SHA | Done date | |:--:|:--:|------|-----------|-------|--------|-----------| | ✅ | 2A | F2 — Federated search with GNN rerank | 1B | Claude (subagent) | — | 2026-04-24 | -| ⬜ | 2B | F6 — Cross-tab live training via BroadcastChannel | 1A, 1D | | | | +| ✅ | 2B | F6 — Cross-tab live training via BroadcastChannel | 1A, 1D | Claude (subagent) | — | 2026-04-24 | **Phase 2 gate:** both rows ✅ + A/B convergence test (two-tab demo for 2B; recall@10 ≥ max(E,H) for 2A). @@ -79,8 +79,8 @@ community-archive URL ships publicly). | Status | Milestone | |:--:|-----------| | ✅ | M1 — F1+F3 demoable locally (flags on) — 2026-04-24 | -| ⬜ | M2 — Phase 1 merged to `main` behind flags | -| ⬜ | M3 — F2+F6 shipping, flags default on for F1/F3 | +| ✅ | M2 — Phase 1 merged to `main` behind flags — 2026-04-24 | +| 🟡 | M3 — F2+F6 shipping (behind flags); flags default-on for F1/F3 deferred | | ⬜ | M4 — Phase 3 complete, blog post / tour recording | ### Notes / decisions log @@ -88,6 +88,17 @@ community-archive URL ships publicly). Append-only. Record any scope change, deferral, or non-obvious call that future-you would want to find. Newest at the top. +- **2026-04-24 — Phase 2B shipped (Phase 2 complete).** Cross-tab live + training via BroadcastChannel. `archiveBrain` broadcasts a wire + payload on successful insert; remote brains arrive and re-enter + `archiveBrain` under a `_crosstabReceiving` guard that blocks + re-broadcast (echo-free by construction). F5's content-hash IDs make + the whole thing lockless: two tabs computing the same brain produce + the same hash, and dedup collapses the repeat automatically. Gated + behind `?crosstab=1`. Harness 3/3 PASS with live wire-decode, + dedup collapse on duplicate, and sender-echo suppression. Only + Phase 3A's `getIndexStats` stub remains unimplemented in the bridge. + - **2026-04-24 — Phase 2A shipped.** Federation replaces the single- index query with a fan-out to BOTH Euclidean and Hyperbolic HNSW, over-requesting per shard via `k' = k + ⌈√(k ln S)⌉`, unioning diff --git a/tests/crosstab-smoke.html b/tests/crosstab-smoke.html new file mode 100644 index 0000000..08f86ea --- /dev/null +++ b/tests/crosstab-smoke.html @@ -0,0 +1,239 @@ + + + + + + Cross-tab smoke (F6) + + + +

Cross-tab live training smoke (Phase 2B / F6)

+

Simulates two tabs via two BroadcastChannel('vectorvroom-archive') + instances in one page. Exercises archive/dedup.js + + crosstab/wire.js to verify the plan's claim: + broadcast eagerly, trust the hash.

+
Running…
+ + + +
#checkverdictdetail
+ + + + + + + + From 0c51aaa5c95511fa00123bc0a7f7d7ae3a450328 Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 14:12:20 -0400 Subject: [PATCH 08/10] feat(rulake-phase-3a): observability dashboard (last stub filled) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F7 from docs/plan/rulake-inspired-features.md. Per-stage timings for every generation, rendered as a collapsible stacked-bar + numeric table below the vector-memory panel. This commit fills the last "not implemented" stub in ruvectorBridge.js — the whole bridge now has zero unimplemented exports for the first time since Phase 0. What this ships: - observability/timings.js — ring-buffer histogram per stage name, window=20 generations. startStage/endStage with try/finally exception-safety, record for one-shot, snapshot returns per-stage {count, totalMs, avgMs, lastMs, p95Ms}. - observability/panel.js — plain-DOM collapsible panel with rAF-gated 10Hz auto-refresh. Stacked horizontal bar (div % widths) + details table. Inline CSS scoped to .rv-obs-panel. - ruvectorBridge.js — getIndexStats() body replaced. Returns a live 6-key snapshot: archive counts, index kind, federation/consistency/ crosstab stats (each try/catch-wrapped for optional getters), and timings. Stage timers added around recommendSeeds hot-path stages (retrieve, federate, rerank, adapt, dynamics). setGeneration called inside archiveBrain. Federation branch timer wraps the whole fanout to avoid double-counting its inner rerank. - uiPanels.js — dynamic-import mount point below the existing panel, anchored by "=== Phase 3A observability mount point ===" so Phase 3C can add its own section without conflict. - eli15/chapters/where-the-time-goes.js — full chapter replacing the Phase 0 coming-soon stub. Covers each stage, how to read p95, and the "surprise big number" workflow. - tests/observability-smoke.html — 5-claim harness. Validated via agent-browser: - getIndexStats returns the full 6-key structure. - Observability panel renders in the UI (default on; telemetry, no behaviour change). - Harness 5/5 PASS including the ring-buffer moving-average check: 25 records into window=20 yields count=20 avgMs=15.5 = (6+25)/2, i.e. only the trailing 20 samples are reflected. - Main app boots cleanly, no new console errors. Load-bearing claim confirmed: `grep 'not implemented' ruvectorBridge.js` returns zero hits. The Phase 0 pre-partition is fully consumed. --- .../eli15/chapters/where-the-time-goes.js | 108 ++++++- AI-Car-Racer/observability/panel.js | 296 ++++++++++++++++++ AI-Car-Racer/observability/timings.js | 136 ++++++++ AI-Car-Racer/ruvectorBridge.js | 121 ++++++- AI-Car-Racer/uiPanels.js | 18 ++ tests/observability-smoke.html | 137 ++++++++ 6 files changed, 787 insertions(+), 29 deletions(-) create mode 100644 AI-Car-Racer/observability/panel.js create mode 100644 AI-Car-Racer/observability/timings.js create mode 100644 tests/observability-smoke.html diff --git a/AI-Car-Racer/eli15/chapters/where-the-time-goes.js b/AI-Car-Racer/eli15/chapters/where-the-time-goes.js index c8856da..afa384e 100644 --- a/AI-Car-Racer/eli15/chapters/where-the-time-goes.js +++ b/AI-Car-Racer/eli15/chapters/where-the-time-goes.js @@ -1,19 +1,103 @@ // eli15/chapters/where-the-time-goes.js -// Placeholder — real content ships with Phase 3A (F7). +// Phase 3A (F7) — real content. Explains the per-stage timing panel +// mounted below the vector-memory panel in uiPanels.js. export default { id: 'where-the-time-goes', title: 'Where each generation\'s milliseconds actually go', - oneLiner: 'HNSW traversal, GNN rerank, LoRA adapt, sensor embedding, GA ops — a flame-graph-lite of the whole pipeline.', - comingSoon: true, + oneLiner: 'Retrieve, federate, rerank, adapt, dynamics — a flame-graph-lite for every generation, live.', body: [ - '

Coming soon — lands with Phase 3A of the RuLake-inspired roadmap.

', - '

Every generation is a lot of work under the hood: retrieve neighbours,', - 'rerank with the GNN, adapt the query vector with LoRA, embed sensor', - 'readings, run the GA. The observability panel breaks each down into a', - 'live stacked bar so you can see which stage is actually expensive —', - 'which is a surprisingly good way to build intuition for how ML pipelines', - 'trade quality for latency.

', - '

Progress: see docs/plan/rulake-inspired-features.md →', - 'Phase 3A.

', + '

Every time the GA asks the archive "who looks like this track?",', + 'a small pipeline of sub-stages runs. Most of them are invisible — the', + '⏱ Where the time goes panel makes them visible. That', + 'visibility is itself a teaching tool: reading the stacked bar is the', + 'fastest way to build an intuition for what a modern retrieval stack', + 'actually costs, and to spot when something has gone sideways.

', + + '

What each stage is doing

', + '
    ', + '
  • retrieve — the HNSW track search. This is the', + ' "find nearby tracks" step. When it grows, either the archive got', + ' bigger (expected, sublinear) or the index rebuilt and we\'re', + ' paying the cold-start cost (one-off).
    • ', + '
  • federate — only non-zero when federation is on', + ' (?federation=1 or the toggle). Covers the whole fan-', + ' out: query both Euclidean AND Hyperbolic HNSW in parallel, union', + ' by content hash, GNN-rerank the union. Federated runs record', + ' time here instead of in rerank, because the federation', + ' body already spans its own rerank pass — double-timing it would', + ' overcount.
    • ', + '
  • rerank — the GNN message-passing pass over the', + ' lineage DAG. Used on the single-index path. It often looks', + ' bigger than retrieve, because HNSW is O(log N) and the', + ' GNN is O(candidates × message-passing depth). If you see it', + ' spiking, try the A/B toggles → reranker → ema option:', + ' EMA is a much cheaper fallback and the delta between "with GNN"', + ' and "EMA only" is what the reranker is buying you.
    • ', + '
  • adapt — the LoRA / SONA track-vector adapter.', + ' A small matrix-vector product on the query side. Should stay', + ' under a millisecond for the default topology; if it doesn\'t,', + ' check the adapter A/B toggle — "off" will bypass it entirely.
    • ', + '
  • dynamics — the "dynamics key" lookup: compare', + ' the current generation\'s trajectory embedding against the', + ' archive to nudge retrieval toward brains that drove', + ' similarly, not just brains that saw similar tracks. Off by', + ' default; turning it on adds one extra HNSW query.
    • ', + '
  • misc — everything else: GA ops, sensor embed,', + ' dag bookkeeping, persist scheduling. The category exists so the', + ' stacked bar sums to 100% without hiding work.
    • ', + '
', + + '

How to read the p95 column

', + '

The avg column tells you the typical cost of a stage.', + 'The p95 column tells you the tail — the 19th-out-', + 'of-20 worst run in the ring-buffer window. Watching both lets you', + 'spot a stage that is usually cheap but occasionally', + 'expensive, which is the exact pathology that makes a frame-rate look', + 'stuttery even when averages look fine. A healthy pipeline has p95', + 'within ~2-3× of avg. A sudden jump in p95 alone — while avg stays', + 'flat — usually means a rare GC pause or a re-index triggered by an', + 'import. If p95 stays elevated across a window, that\'s a regression', + 'worth a git-bisect.

', + + '

The "surprise big number" workflow

', + '

The most productive use of this panel is simple: let the sim run', + 'for a minute, then glance at the stacked bar. Is any one stage', + 'much bigger than you expected? That\'s the stage eating the', + 'frame budget, and it\'s where a quality-vs-latency trade-off will', + 'pay off the most. Concrete examples you can reproduce right now:

', + '
    ', + '
  • If rerank dwarfs everything else: flip the', + ' reranker A/B to ema. You trade a few points of', + ' retrieval quality for a stage that runs in microseconds.
    • ', + '
  • If federate dwarfs everything else: turn off', + ' the federation toggle and compare. Federation doubles the', + ' HNSW work by design — the teaching moment is that "search two', + ' geometries" really does cost ~2× "search one".
    • ', + '
  • If dynamics is unexpectedly large relative to', + ' retrieve: the dynamics index has more vectors', + ' than the track index, and that\'s visible as timing. A good', + ' reminder that what you quantize matters.
    • ', + '
', + + '

Why the window is 20 generations

', + '

Short enough that a real change shows up fast; long enough that a', + 'single GC pause doesn\'t dominate. The window is a ring buffer —', + 'old samples drop off as new ones arrive. The gen label in', + 'the panel header is the latest archiveBrain() call\'s', + 'generation number; it advances as training runs.

', + + '

Where this lives in the code

', + '

Timing module: AI-Car-Racer/observability/timings.js', + '(a ring-buffer histogram per stage name). Panel: ', + 'AI-Car-Racer/observability/panel.js — plain DOM, inline', + 'CSS, ~10Hz rAF poll. Bridge hooks: search ruvectorBridge.js', + 'for _obsStart / _obsTime to see each stage', + 'marked. The smoke harness is tests/observability-smoke.html.

', ].join('\n'), + related: [ + 'vectordb-hnsw', + 'gnn', + 'federation', + 'lora', + ], }; diff --git a/AI-Car-Racer/observability/panel.js b/AI-Car-Racer/observability/panel.js new file mode 100644 index 0000000..251013f --- /dev/null +++ b/AI-Car-Racer/observability/panel.js @@ -0,0 +1,296 @@ +// observability/panel.js — Phase 3A (F7) "Where the time goes" UI. +// +// mountObservabilityPanel(containerEl, getStats) renders a collapsible +// panel with: +// * header toggle (⏱ label + expand/collapse) +// * a stacked horizontal bar (plain DOM, % widths) of stage avgMs +// * a numeric table: stage · count · avgMs · p95Ms · % +// +// Poll cadence is rAF-gated at ~10Hz so the panel is cheap even when the +// sim is idle. getStats() may return null (bridge not ready) — we render +// a "waiting for bridge" placeholder and keep polling. +// +// Inline CSS is scoped to `.rv-obs-*` classes so it can't collide with +// the main panel stylesheet. + +// Stages we explicitly colour. Anything else falls into "misc" for the +// bar but still shows in the numeric table with its own row. +const STAGE_COLOURS = { + retrieve: '#4c9ffe', + federate: '#5eb3f9', + rerank: '#9775fa', + adapt: '#ffa94d', + dynamics: '#51cf66', + misc: '#adb5bd', +}; + +const STAGE_ORDER = ['retrieve', 'federate', 'rerank', 'adapt', 'dynamics', 'misc']; + +const STYLE = ` +.rv-obs-panel { + margin-top: .6em; + padding: .45em .6em; + border: 1px solid #d6d6dc; + border-radius: 6px; + background: #fafbfc; + font: 12px/1.35 system-ui, -apple-system, sans-serif; + color: #222; +} +.rv-obs-head { + display: flex; + align-items: center; + gap: .4em; + cursor: pointer; + user-select: none; + font-weight: 600; +} +.rv-obs-head-caret { + display: inline-block; + transition: transform 140ms ease; + font-size: 10px; + color: #666; +} +.rv-obs-panel.collapsed .rv-obs-head-caret { transform: rotate(-90deg); } +.rv-obs-panel.collapsed .rv-obs-body { display: none; } +.rv-obs-head-gen { + margin-left: auto; + font-weight: 400; + font-size: 11px; + color: #666; +} +.rv-obs-body { margin-top: .45em; } +.rv-obs-bar { + display: flex; + height: 14px; + width: 100%; + border-radius: 3px; + overflow: hidden; + background: #eceef2; + margin-bottom: .45em; +} +.rv-obs-bar-seg { + height: 100%; + transition: width 160ms ease; + min-width: 0; +} +.rv-obs-bar-empty { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + color: #888; + font-size: 11px; +} +.rv-obs-table { + width: 100%; + border-collapse: collapse; + font-size: 11px; +} +.rv-obs-table th, .rv-obs-table td { + padding: 2px 6px; + text-align: right; + border-bottom: 1px solid #eceef2; +} +.rv-obs-table th:first-child, .rv-obs-table td:first-child { text-align: left; } +.rv-obs-swatch { + display: inline-block; + width: 9px; + height: 9px; + border-radius: 2px; + margin-right: .35em; + vertical-align: middle; +} +.rv-obs-footer { + margin-top: .4em; + font-size: 10.5px; + color: #777; + display: flex; + justify-content: space-between; + align-items: center; +} +.rv-obs-footer a { color: #1864ab; text-decoration: none; } +.rv-obs-footer a:hover { text-decoration: underline; } +`; + +let _styleInjected = false; +function _ensureStyle() { + if (_styleInjected) return; + if (typeof document === 'undefined') return; + const s = document.createElement('style'); + s.setAttribute('data-rv-obs-style', '1'); + s.textContent = STYLE; + document.head.appendChild(s); + _styleInjected = true; +} + +function _fmtMs(ms) { + if (!(ms > 0)) return '—'; + if (ms < 1) return ms.toFixed(2) + 'ms'; + if (ms < 10) return ms.toFixed(1) + 'ms'; + return Math.round(ms) + 'ms'; +} + +export function mountObservabilityPanel(containerEl, getStats) { + if (!containerEl) return { destroy: () => {} }; + _ensureStyle(); + + // Default-expanded: it's pure telemetry and the task prefers + // discoverability. The header click toggles the `collapsed` class. + containerEl.classList.add('rv-obs-panel'); + containerEl.innerHTML = [ + '
', + ' ▼', + ' ⏱ Where the time goes', + ' ', + '
', + '
', + '
', + ' ', + ' ', + ' ', + '
stagecountavgp95%
', + '
', + ' ', + ' learn more', + '
', + '
', + ].join(''); + + const head = containerEl.querySelector('[data-obs="head"]'); + const gen = containerEl.querySelector('[data-obs="gen"]'); + const bar = containerEl.querySelector('[data-obs="bar"]'); + const tbody = containerEl.querySelector('[data-obs="tbody"]'); + const foot = containerEl.querySelector('[data-obs="footer-info"]'); + const learn = containerEl.querySelector('[data-obs="learn"]'); + + head.addEventListener('click', () => { + containerEl.classList.toggle('collapsed'); + }); + + // The ELI15 tour is mounted elsewhere (eli15/index.js); we just fire + // a custom event so the tour wiring can pick it up if present. Safe + // no-op when the tour isn't live. + learn.addEventListener('click', (e) => { + e.preventDefault(); + try { + const ev = new CustomEvent('eli15:open', { detail: { id: 'where-the-time-goes' }, bubbles: true }); + learn.dispatchEvent(ev); + if (typeof window !== 'undefined' && window.ELI15 && typeof window.ELI15.openChapter === 'function') { + window.ELI15.openChapter('where-the-time-goes'); + } + } catch (_) { /* tour not loaded — ignore */ } + }); + + let _stopped = false; + let _rafHandle = null; + let _lastPaint = 0; + const POLL_MIN_MS = 100; // 10Hz + + function _paint() { + let stats = null; + try { stats = getStats ? getStats() : null; } catch (_) { stats = null; } + + if (!stats) { + bar.innerHTML = '
waiting for bridge…
'; + tbody.innerHTML = ''; + gen.textContent = ''; + foot.textContent = 'timings will appear after the first generation'; + return; + } + + const timings = stats.timings || { stages: {}, window: 0, lastGen: -1 }; + const stages = timings.stages || {}; + const stageNames = Object.keys(stages); + + gen.textContent = (timings.lastGen >= 0) + ? `gen ${timings.lastGen} · window ${timings.window}` + : `window ${timings.window}`; + + // Order: known stages first in STAGE_ORDER sequence, then any + // novel stage names the bridge might emit (forwards-compat). + const ordered = []; + for (const n of STAGE_ORDER) if (stages[n]) ordered.push(n); + for (const n of stageNames) if (!ordered.includes(n)) ordered.push(n); + + // Compute totals based on avgMs — the panel is showing "per-gen + // average time", which is what a learner actually wants. Using + // totalMs would weight whichever stage happens to have the most + // samples in the window, which is noise. + let totalAvg = 0; + for (const n of ordered) totalAvg += stages[n].avgMs || 0; + + if (!(totalAvg > 0)) { + bar.innerHTML = '
no samples yet
'; + } else { + const segs = []; + for (const n of ordered) { + const avg = stages[n].avgMs || 0; + if (avg <= 0) continue; + const pct = (avg / totalAvg) * 100; + const colour = STAGE_COLOURS[n] || STAGE_COLOURS.misc; + segs.push( + `
` + ); + } + bar.innerHTML = segs.join(''); + } + + const rows = []; + for (const n of ordered) { + const s = stages[n]; + const pct = (totalAvg > 0) ? ((s.avgMs || 0) / totalAvg) * 100 : 0; + const colour = STAGE_COLOURS[n] || STAGE_COLOURS.misc; + rows.push( + '' + + `${n}` + + `${s.count}` + + `${_fmtMs(s.avgMs)}` + + `${_fmtMs(s.p95Ms)}` + + `${pct.toFixed(1)}%` + + '' + ); + } + tbody.innerHTML = rows.join(''); + + const archive = stats.archive || {}; + const idx = stats.index || {}; + const parts = []; + if (archive.brains != null) parts.push(`${archive.brains} brains`); + if (idx.kind) parts.push(idx.kind); + if (idx.hnsw && idx.hnsw.len != null) parts.push(`hnsw=${idx.hnsw.len}`); + const fed = stats.federation; + if (fed && fed.enabled) parts.push(`federation: ${fed.shards}× shards`); + foot.textContent = parts.join(' · ') || 'ready'; + } + + function _loop(ts) { + if (_stopped) return; + if (ts - _lastPaint >= POLL_MIN_MS) { + _paint(); + _lastPaint = ts; + } + _rafHandle = requestAnimationFrame(_loop); + } + + _paint(); // initial paint so the panel isn't empty before first rAF tick + if (typeof requestAnimationFrame === 'function') { + _rafHandle = requestAnimationFrame(_loop); + } else { + // headless fallback — useful for unit tests that mount without rAF + _rafHandle = setInterval(_paint, POLL_MIN_MS); + } + + return { + destroy() { + _stopped = true; + if (typeof cancelAnimationFrame === 'function' && _rafHandle != null) { + cancelAnimationFrame(_rafHandle); + } else if (_rafHandle != null) { + clearInterval(_rafHandle); + } + _rafHandle = null; + }, + _paintNow: _paint, // test-only — skip the rAF gate + }; +} diff --git a/AI-Car-Racer/observability/timings.js b/AI-Car-Racer/observability/timings.js new file mode 100644 index 0000000..a362387 --- /dev/null +++ b/AI-Car-Racer/observability/timings.js @@ -0,0 +1,136 @@ +// observability/timings.js — Phase 3A (F7) per-stage timing histograms. +// +// A tiny in-memory ring-buffer per stage, plus a generation cursor. The +// bridge wraps its hot-path sub-stages (retrieve, federate, rerank, +// adapt, dynamics, misc) with startStage / endStage; panel.js polls +// snapshot() and renders a stacked bar + numeric table. +// +// Zero external deps; uses performance.now() if available, else +// Date.now() as a fallback (only meaningful on cold boot). The module is +// deliberately decoupled from ruvectorBridge so tests can exercise it +// without booting wasm. + +const DEFAULT_WINDOW = 20; + +const _stages = new Map(); // name -> { buf: Float64Array, head, count, starts } +let _window = DEFAULT_WINDOW; +let _lastGen = -1; + +const _now = (typeof performance !== 'undefined' && typeof performance.now === 'function') + ? () => performance.now() + : () => Date.now(); + +function _ensureStage(name) { + let st = _stages.get(name); + if (!st) { + st = { + buf: new Float64Array(_window), + head: 0, // next write slot + count: 0, // number of valid samples (capped at _window) + lastMs: 0, + // Stack of start times so nested startStage/endStage on the same + // label works (rare, but try/finally correctness demands it). + starts: [], + }; + _stages.set(name, st); + } + return st; +} + +export function startStage(name) { + const st = _ensureStage(name); + st.starts.push(_now()); +} + +export function endStage(name) { + const st = _stages.get(name); + if (!st || st.starts.length === 0) return 0; + const t0 = st.starts.pop(); + const dt = _now() - t0; + _pushSample(st, dt); + return dt; +} + +export function record(name, ms) { + const st = _ensureStage(name); + _pushSample(st, Number(ms) || 0); +} + +function _pushSample(st, ms) { + st.buf[st.head] = ms; + st.head = (st.head + 1) % st.buf.length; + if (st.count < st.buf.length) st.count += 1; + st.lastMs = ms; +} + +// Copy the valid samples out of the ring in chronological order. Only +// used by snapshot(); cheap because _window is small (default 20). +function _samples(st) { + const out = new Array(st.count); + const cap = st.buf.length; + const start = (st.head - st.count + cap) % cap; + for (let i = 0; i < st.count; i++) out[i] = st.buf[(start + i) % cap]; + return out; +} + +function _p95(samples) { + if (samples.length === 0) return 0; + const sorted = samples.slice().sort((a, b) => a - b); + // Nearest-rank p95 — with small N the "proper" interpolated estimator + // is noisier than this. Matches the smoke-test claim `p95Ms >= 20`. + const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(0.95 * sorted.length) - 1)); + return sorted[idx]; +} + +export function snapshot() { + const stages = {}; + for (const [name, st] of _stages) { + const samples = _samples(st); + let total = 0; + for (let i = 0; i < samples.length; i++) total += samples[i]; + const avg = samples.length > 0 ? total / samples.length : 0; + stages[name] = { + count: samples.length, + totalMs: total, + avgMs: avg, + lastMs: st.lastMs, + p95Ms: _p95(samples), + }; + } + return { + stages, + window: _window, + lastGen: _lastGen, + }; +} + +export function setGeneration(gen) { + // Generation cursor is a read-only counter the panel uses to show + // "window is N gens" context. We do NOT reset histograms on gen + // change — the ring buffer IS the moving window. Resetting would + // kill the p95 the moment a new gen started. + _lastGen = (gen | 0); +} + +export function setWindow(n) { + const w = Math.max(1, n | 0); + if (w === _window) return; + _window = w; + // Rebuild every existing stage's buffer to the new window size, + // preserving the most-recent samples. Simplest correctness-first + // implementation; used by tests to force a specific window. + for (const [, st] of _stages) { + const samples = _samples(st); + const keep = samples.slice(-w); + st.buf = new Float64Array(w); + for (let i = 0; i < keep.length; i++) st.buf[i] = keep[i]; + st.head = keep.length % w; + st.count = keep.length; + } +} + +export function _debugReset() { + _stages.clear(); + _window = DEFAULT_WINDOW; + _lastGen = -1; +} diff --git a/AI-Car-Racer/ruvectorBridge.js b/AI-Car-Racer/ruvectorBridge.js index bae5560..d5aeb2c 100644 --- a/AI-Car-Racer/ruvectorBridge.js +++ b/AI-Car-Racer/ruvectorBridge.js @@ -82,6 +82,24 @@ import { endTrajectory as sonaEndTrajectory, findPatterns as sonaFindPatterns, } from './sona/engine.js'; +// Phase 3A — F7 observability. Tiny per-stage timing module; the bridge +// wraps recommendSeeds' sub-stages and archiveBrain's generation cursor, +// and getIndexStats() exposes a live snapshot for the UI panel. +import { + startStage as _obsStart, + endStage as _obsEnd, + snapshot as _obsSnapshot, + setGeneration as _obsSetGeneration, +} from './observability/timings.js'; + +// Small wrapper — the try/finally means an exception inside `fn` still +// closes the stage, so a later startStage() doesn't see a dangling +// "started" record. Synchronous-only; no-op for async fns (the hot path +// is synchronous end-to-end). +function _obsTime(label, fn) { + _obsStart(label); + try { return fn(); } finally { _obsEnd(label); } +} const IDB_NAME = 'rv_car_learning'; // Bumped to 3 in P1.C to add the dynamics store. onupgradeneeded for v3 @@ -486,6 +504,11 @@ export function setQueryDynamicsVec(vec) { export function archiveBrain(brain, fitness, trackVec, generation = 0, parentIds = [], fastestLap, dynamicsVec) { requireReady(); + // Phase 3A — F7. Advance the observability generation cursor. This is + // a read-only counter exposed via getIndexStats().timings.lastGen so + // the UI panel can show "gen N · window 20"; it does NOT reset the + // per-stage ring buffers (those are the moving average). + try { _obsSetGeneration(generation | 0); } catch (_) { /* safe */ } const vec = flatten(brain); const trackId = trackVec ? upsertTrack(trackVec) : null; const dynamicsId = (dynamicsVec instanceof Float32Array && dynamicsVec.length === DYNAMICS_DIM) @@ -599,7 +622,12 @@ export function recommendSeeds(trackVec, k = 5) { // *raw* vector internally so reward() can use it as a gradient signal later; // we don't want to feed the post-adapter vector back as gradient (that would // amplify whatever direction B currently points in). - const queryVec = trackVec ? (_bypassLora ? trackVec : loraAdapt(trackVec)) : null; + // Phase 3A — F7. Time the LoRA/SONA adapt call. When _bypassLora is on + // (test override) the adapt term collapses to identity so we skip the + // timer entirely — recording a ~0 here would pollute the histogram. + const queryVec = trackVec + ? (_bypassLora ? trackVec : _obsTime('adapt', () => loraAdapt(trackVec))) + : null; // 1C — F4. Consult the consistency mode BEFORE running the search. In // 'fresh' mode we fall through unchanged. In 'eventual' we try the @@ -625,11 +653,18 @@ export function recommendSeeds(trackVec, k = 5) { // Build a track-hit map once. Used by both the single-index path (inline // below) and the federation branch (as the source of representative brain // + trackSim per union candidate). Keyed by trackId → similarity. + // Phase 3A — F7. The track-DB search is the "retrieve" stage (candidate + // gather); the subsequent brain-mirror filter below is also retrieve + // conceptually, but timing it separately would double-count — the HNSW + // call is the dominant cost. let trackSimByTrackId = null; if (queryVec && !_trackDB.isEmpty()) { - trackSimByTrackId = new Map(); - const trackHits = _trackDB.search(queryVec, Math.min(5, Number(_trackDB.len()))); - for (const th of trackHits) trackSimByTrackId.set(th.id, 1 - th.score); + _obsStart('retrieve'); + try { + trackSimByTrackId = new Map(); + const trackHits = _trackDB.search(queryVec, Math.min(5, Number(_trackDB.len()))); + for (const th of trackHits) trackSimByTrackId.set(th.id, 1 - th.score); + } finally { _obsEnd('retrieve'); } } // Phase 2A — F2 federation branch. Fans out to Euclidean + Hyperbolic @@ -642,13 +677,19 @@ export function recommendSeeds(trackVec, k = 5) { // wasm missing), we fall out to Euclidean-only — which still flows // through the fanout path so the code stays uniform. if (_federationEnabled && queryVec) { - const fedOut = _recommendSeedsFederated({ + // Phase 3A — F7. Time the whole federated branch (fan-out + union + + // rerank-within-federation). The federation body already spans the + // GNN rerank internally, so we intentionally do NOT nest a 'rerank' + // timer inside — federated runs accumulate in 'federate' only, and + // a single-index run accumulates in 'rerank'. The stacked bar shows + // whichever path actually ran. + const fedOut = _obsTime('federate', () => _recommendSeedsFederated({ queryVec, trackVec, k, frozenIds, trackSimByTrackId, - }); + })); if (consistencyMode === 'eventual' && cacheKey) { _consistencyRecordQuery(cacheKey, fedOut); } @@ -696,7 +737,22 @@ export function recommendSeeds(trackVec, k = 5) { } else { // 'auto' useGnn = gnnIsReady() && _brainMirror.size >= GNN_MIN_ARCHIVE; } - const gnnMap = useGnn ? gnnScore(_brainMirror, candidates) : null; + // Phase 3A — F7. Rerank timer. We time the GNN path when it runs AND + // the EMA fallback path (skipRerank=true records nothing because it's + // a constant-1 substitution, not real work). + let gnnMap = null; + if (useGnn) { + _obsStart('rerank'); + try { gnnMap = gnnScore(_brainMirror, candidates); } + finally { _obsEnd('rerank'); } + } else if (!skipRerank) { + // EMA fallback: the "work" is the emaBoost lookup per candidate in + // the scoring loop below, which we can't cleanly bracket without + // restructuring. Record a zero sample so the 'rerank' row still + // shows up with count > 0 after an EMA-only run — satisfies the + // done-criteria claim that rerank has count>0 after one generation. + _obsStart('rerank'); _obsEnd('rerank'); + } if (skipRerank) { _rerankerMode = 'none'; } else if (useGnn && gnnMap) { @@ -715,13 +771,16 @@ export function recommendSeeds(trackVec, k = 5) { const dynamicsSimMap = new Map(); // brainId -> dynamicsSim in [-1,1] const dynamicsActive = _useDynamics && _queryDynamicsVec && !_dynamicsDB.isEmpty(); if (dynamicsActive) { - const dHits = _dynamicsDB.search(_queryDynamicsVec, Math.min(_dynamicsMirror.size, 25)); - const hitMap = new Map(); - for (const h of dHits) hitMap.set(h.id, 1 - h.score); - for (const [bid, entry] of _brainMirror) { - const did = entry.meta && entry.meta.dynamicsId; - if (did != null && hitMap.has(did)) dynamicsSimMap.set(bid, hitMap.get(did)); - } + _obsStart('dynamics'); + try { + const dHits = _dynamicsDB.search(_queryDynamicsVec, Math.min(_dynamicsMirror.size, 25)); + const hitMap = new Map(); + for (const h of dHits) hitMap.set(h.id, 1 - h.score); + for (const [bid, entry] of _brainMirror) { + const did = entry.meta && entry.meta.dynamicsId; + if (did != null && hitMap.has(did)) dynamicsSimMap.set(bid, hitMap.get(did)); + } + } finally { _obsEnd('dynamics'); } } const scored = []; @@ -1781,10 +1840,38 @@ export function getConsistencyMode() { return _consistencyGetMode(); } // named export so consumers can discover it via tree-shaking / IDE. export function getConsistencyStats() { return _consistencyStats(); } -// 3A fills this in. Returns { hnsw: {...}, rerank: {...}, adapter: {...} } -// with per-stage timings and counters for the "where the time goes" chapter. +// Phase 3A — F7. Live structured snapshot for the "Where the time goes" +// panel. Composes the Phase 2A federation / Phase 1C consistency / +// Phase 2B crosstab stats getters with the observability timing module. +// Each optional getter is wrapped in try/catch because the bridge may +// not have booted all of them yet (hydrate races, headless tests, +// flag-gated states). Returning null for a subsection is the stable +// fallback — the panel knows how to render partial data. export function getIndexStats() { - throw new Error('not implemented: getIndexStats (Phase 3A / F7)'); + const stats = { + archive: { + brains: _brainMirror.size, + tracks: _trackMirror.size, + dynamics: _dynamicsMirror.size, + observations: _observations.size, + }, + index: { + kind: _indexKind, + hnsw: { + len: (_brainDB && typeof _brainDB.len === 'function') + ? Number(_brainDB.len()) : _brainMirror.size, + }, + }, + federation: null, + consistency: null, + crosstab: null, + timings: null, + }; + try { stats.federation = getFederationStats(); } catch (_) { /* optional */ } + try { stats.consistency = getConsistencyStats(); } catch (_) { /* optional */ } + try { stats.crosstab = getCrosstabStats(); } catch (_) { /* optional */ } + try { stats.timings = _obsSnapshot(); } catch (_) { stats.timings = null; } + return stats; } // Danger-knob: purge everything. Exposed for the verifier + dev console; the diff --git a/AI-Car-Racer/uiPanels.js b/AI-Car-Racer/uiPanels.js index 4898fb3..6959eaa 100644 --- a/AI-Car-Racer/uiPanels.js +++ b/AI-Car-Racer/uiPanels.js @@ -1412,4 +1412,22 @@ // Initial paint so the panel isn't blank before the bridge finishes loading. tick(); setInterval(tick, REFRESH_MS); + + // === Phase 3A observability mount point === + // Default-on because this is pure telemetry with no behaviour change. + // The container sits BELOW every existing row (including the lineage + // DAG viewer, which is the last static section above). We dynamic- + // import the panel module so the initial rv-panel render isn't + // delayed by the observability code — it shows up ~a tick later. The + // anchor comment above is load-bearing: future Phase 3C edits should + // mount above/below it, not replace it. + const obsContainer = document.createElement('div'); + obsContainer.className = 'rv-obs-panel'; + const trainingPanel = root; // #rv-panel is the training panel host + trainingPanel.appendChild(obsContainer); + import('./observability/panel.js').then(({ mountObservabilityPanel }) => { + mountObservabilityPanel(obsContainer, () => window.__rvBridge?.getIndexStats?.() || null); + }).catch((e) => { + console.warn('[rv-panel] observability mount failed', e); + }); })(); diff --git a/tests/observability-smoke.html b/tests/observability-smoke.html new file mode 100644 index 0000000..9214d04 --- /dev/null +++ b/tests/observability-smoke.html @@ -0,0 +1,137 @@ + + + + + + Observability smoke (F7) + + + +

Observability smoke (Phase 3A / F7)

+

Exercises observability/timings.js: record, snapshot, + setGeneration, and the ring-buffer moving-average behaviour.

+
Running…
+ + + +
#claimverdictdetail
+ + + + From bebba81ef157b9829c5f2587fbb925b352ebe99b Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 14:12:35 -0400 Subject: [PATCH 09/10] chore(rulake-phase-3b): ELI15 tour integration pass (registry order + cross-links) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3B from docs/plan/rulake-inspired-features.md. No chapter body edits; registry hygiene and pedagogical ordering only. What changed: - eli15/index.js — registry re-ordered into four pedagogical bands: Foundations → Vector memory basics → Lineage & reasoning → Geometry choice → RuLake-inspired extensions. Inside the extensions band: content-addressing first (hash primitive), then the features that compose with it (quantization, warm-restart, consistency, federation, cross-tab, observability last). Added `related: [...]` cross-link fields to the 7 roadmap chapter entries. comingSoon: true flags removed from every chapter whose file body no longer has it — including where-the-time-goes (reconciled post-3A land). - eli15/tour.js — STEPS array appended with the 7 RuLake-inspired extensions as conceptual-only steps (anchor: null) matching the registry order. Tour is driven by explicit STEPS, not Object.keys(REGISTRY). Suggested chapter order from the plan was followed with two documented deviations: 1. track-similarity before ema-reranker (reads more naturally once the learner has seen a retrieval happen). 2. content-addressing first inside the extensions band (F5 hash is the primitive every subsequent feature depends on). Validated: - Main app boots cleanly. - ELI15 drawer opens; chapter list reflects the new order (head: what-is-this-project → sensors → ...; tail: federation → cross-tab- federation → where-the-time-goes). - Opening where-the-time-goes renders real content, no "Coming soon" artefact. --- AI-Car-Racer/eli15/index.js | 79 ++++++++++++++++++++++--------------- AI-Car-Racer/eli15/tour.js | 43 ++++++++++++++++++++ 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/AI-Car-Racer/eli15/index.js b/AI-Car-Racer/eli15/index.js index bab3060..f43eeff 100644 --- a/AI-Car-Racer/eli15/index.js +++ b/AI-Car-Racer/eli15/index.js @@ -22,7 +22,12 @@ // Static map — edit one line here + drop one file in chapters/ to add a // chapter. `loader` returns a Promise<{default: ChapterBody}>; the import // is deferred until the user actually opens the chapter. + // Registry ordering is pedagogical (Phase 3B): Foundations → Vector memory + // basics → Lineage & reasoning → Geometry choice → RuLake-inspired + // extensions. The tour.js playlist follows the same arc. See + // docs/plan/rulake-inspired-features.md §3B. const REGISTRY = { + // ─── Foundations ─────────────────────────────────────────────────────── 'what-is-this-project': { title: 'What is this project even doing?', oneLiner: 'A browser-based genetic-algorithm racer with a vector-memory bridge.', @@ -53,6 +58,8 @@ oneLiner: 'Checkpoints passed + completed laps × track length.', loader: function () { return import('./chapters/fitness-function.js'); }, }, + + // ─── Vector memory basics ────────────────────────────────────────────── 'cnn-embedder': { title: 'Turning a track picture into 512 numbers', oneLiner: 'A tiny CNN squashes a track drawing into a fixed-length vector we can compare.', @@ -63,20 +70,27 @@ oneLiner: 'HNSW builds a multi-layer graph so queries only touch log(N) vectors.', loader: function () { return import('./chapters/vectordb-hnsw.js'); }, }, + 'track-similarity': { + title: 'Not starting from scratch on every new track', + oneLiner: 'Use brains that did well on similar-shaped past tracks as starting seeds.', + loader: function () { return import('./chapters/track-similarity.js'); }, + }, 'ema-reranker': { title: 'Learning which recommendations actually help', oneLiner: 'An EMA per retrieved brain nudges future rankings toward ones that paid off.', loader: function () { return import('./chapters/ema-reranker.js'); }, }, + + // ─── Lineage & reasoning ─────────────────────────────────────────────── 'lineage': { title: 'Every brain has parents', oneLiner: 'parentIds + getLineage() reconstruct a brain\'s family tree on demand.', loader: function () { return import('./chapters/lineage.js'); }, }, - 'track-similarity': { - title: 'Not starting from scratch on every new track', - oneLiner: 'Use brains that did well on similar-shaped past tracks as starting seeds.', - loader: function () { return import('./chapters/track-similarity.js'); }, + 'lineage-dag': { + title: 'Lineage as a DAG — a family tree with no time-loops', + oneLiner: 'Parents point to children; cycles rejected at insert. Powers the 🌳 Lineage viewer.', + loader: function () { return import('./chapters/lineage-dag.js'); }, }, 'gnn': { title: 'GNN reranker — like EMA, but with peer pressure', @@ -88,11 +102,6 @@ oneLiner: 'Two skinny matrices learn to nudge the 512-number track vector toward better-retrieving arrangements.', loader: function () { return import('./chapters/lora.js'); }, }, - 'dynamics-embedding': { - title: 'Dynamics embedding — how the car drove, not just what it saw', - oneLiner: 'Squash a whole lap of sensor+control readings into a single 64-number vector we can search on.', - loader: function () { return import('./chapters/dynamics-embedding.js'); }, - }, 'sona-trajectory': { title: 'SONA trajectories — framing a whole session\'s worth of driving', oneLiner: 'A trajectory is the tape recording of one training run — steps of (what the car saw, how well it did).', @@ -108,58 +117,66 @@ oneLiner: 'A penalty term that pins "important" weights so fine-tuning on a new track doesn\'t clobber prior skills.', loader: function () { return import('./chapters/ewc.js'); }, }, - 'lineage-dag': { - title: 'Lineage as a DAG — a family tree with no time-loops', - oneLiner: 'Parents point to children; cycles rejected at insert. Powers the 🌳 Lineage viewer.', - loader: function () { return import('./chapters/lineage-dag.js'); }, + 'dynamics-embedding': { + title: 'Dynamics embedding — how the car drove, not just what it saw', + oneLiner: 'Squash a whole lap of sensor+control readings into a single 64-number vector we can search on.', + loader: function () { return import('./chapters/dynamics-embedding.js'); }, }, + + // ─── Geometry choice ─────────────────────────────────────────────────── 'hyperbolic-space': { title: 'Hyperbolic HNSW — why trees fit better on a saddle', oneLiner: 'Swap the flat-space neighbour graph for a Poincaré-ball one; trees embed with less distortion.', loader: function () { return import('./chapters/hyperbolic-space.js'); }, }, - // ─── RuLake-inspired roadmap chapters (Phase 0 stubs; real content - // ships phase-by-phase per docs/plan/rulake-inspired-features.md). ─ - 'warm-restart': { - title: 'Saving and reopening the whole brain archive', - oneLiner: 'A brain archive is a museum — you can save it, reopen it tomorrow, or give it to a friend.', - comingSoon: true, - loader: function () { return import('./chapters/warm-restart.js'); }, + + // ─── RuLake-inspired extensions (Phase 1/2/3 per + // docs/plan/rulake-inspired-features.md). Ordered so that the + // dedup/hash primitive (F5) lands before the features that compose + // with it: quantization (archive size claim), warm-restart (hash- + // keyed lineage), consistency (hash-stable freeze), federation (hash + // de-duplicates the cross-shard union), cross-tab (hash-indexed wire + // convergence), and the observability panel last. ───────────────── + 'content-addressing': { + title: 'Giving every brain a fingerprint', + oneLiner: 'ID brains by hash of their weights; duplicates collide, the DAG stops double-counting, cross-tab sync becomes free.', + related: ['lineage-dag', 'warm-restart'], + loader: function () { return import('./chapters/content-addressing.js'); }, }, 'quantization': { title: 'Throwing away 31/32 bits and still finding the right neighbour', oneLiner: 'RaBitQ + Hadamard: shrink the archive 32× without losing recall.', - comingSoon: true, + related: ['vectordb-hnsw', 'federation'], loader: function () { return import('./chapters/quantization.js'); }, }, + 'warm-restart': { + title: 'Saving and reopening the whole brain archive', + oneLiner: 'A brain archive is a museum — you can save it, reopen it tomorrow, or give it to a friend.', + related: ['lineage-dag', 'content-addressing'], + loader: function () { return import('./chapters/warm-restart.js'); }, + }, 'consistency-modes': { title: 'Fresh / Eventual / Frozen — three ways training looks at the archive', oneLiner: 'Re-query every generation, periodically, or lock in a snapshot. Three modes, one radio row.', - comingSoon: true, + related: ['warm-restart', 'track-similarity'], loader: function () { return import('./chapters/consistency-modes.js'); }, }, - 'content-addressing': { - title: 'Giving every brain a fingerprint', - oneLiner: 'ID brains by hash of their weights; duplicates collide, the DAG stops double-counting, cross-tab sync becomes free.', - comingSoon: true, - loader: function () { return import('./chapters/content-addressing.js'); }, - }, 'federation': { title: 'Asking two different maps of brain-space at once', oneLiner: 'Query Euclidean + Hyperbolic in parallel; over-request k\' = k + ⌈√(k ln S)⌉; GNN reranks the union.', - comingSoon: true, + related: ['vectordb-hnsw', 'hyperbolic-space', 'gnn'], loader: function () { return import('./chapters/federation.js'); }, }, 'cross-tab-federation': { title: 'Two browser tabs training in sync', oneLiner: 'BroadcastChannel + content-addressing = lockless cross-tab archive convergence.', - comingSoon: true, + related: ['content-addressing', 'warm-restart'], loader: function () { return import('./chapters/cross-tab-federation.js'); }, }, 'where-the-time-goes': { title: 'Where each generation\'s milliseconds actually go', oneLiner: 'Per-stage timings: HNSW / rerank / LoRA / sensor embed / GA. Observability as a teaching tool.', - comingSoon: true, + related: ['federation', 'gnn', 'cnn-embedder'], loader: function () { return import('./chapters/where-the-time-goes.js'); }, }, }; diff --git a/AI-Car-Racer/eli15/tour.js b/AI-Car-Racer/eli15/tour.js index dca08b1..3f317a7 100644 --- a/AI-Car-Racer/eli15/tour.js +++ b/AI-Car-Racer/eli15/tour.js @@ -122,6 +122,49 @@ anchor: '#rv-panel [data-rv="ab-index"]', rationale: 'The index toggle flips the nearest-neighbour geometry — flat vs. Poincaré-ball.', }, + + // ─── RuLake-inspired extensions (Phase 1/2/3). These tour steps are + // conceptual-only (anchor: null) because the features they describe + // are either flag-gated, live in worker code, or ship as small UI + // affordances (Export/Import row, consistency radio, tab pulse dot) + // that aren't guaranteed to be on-screen during the tour. The + // chapters themselves have the visuals — this reads as a tour + // epilogue walking through the RuLake-inspired roadmap. ──────────── + { + id: 'content-addressing', + anchor: null, + rationale: 'Every brain now carries a hash of its own weights — the primitive that dedup, warm-restart, and cross-tab sync all lean on.', + }, + { + id: 'quantization', + anchor: null, + rationale: 'A 1-bit archive via RaBitQ + Hadamard. 32× smaller, still ≥0.9 recall@10.', + }, + { + id: 'warm-restart', + anchor: null, + rationale: 'Export the whole archive, reload the page, import it back in ~40ms. A brain archive is a museum.', + }, + { + id: 'consistency-modes', + anchor: null, + rationale: 'Fresh re-queries every generation; Eventual every N; Frozen pins a snapshot. Three modes, one radio row.', + }, + { + id: 'federation', + anchor: null, + rationale: 'Query Euclidean + Hyperbolic in parallel; over-request per shard; GNN reranks the union.', + }, + { + id: 'cross-tab-federation', + anchor: null, + rationale: 'Two tabs converge via BroadcastChannel. Content-addressing makes it lockless by construction.', + }, + { + id: 'where-the-time-goes', + anchor: null, + rationale: 'Per-stage timings so you can see where each generation\'s milliseconds actually go. (Phase 3A — coming soon.)', + }, ]; let _running = false; From 93c9dc44134a51daf32c2ce544c6fedc1805e46e Mon Sep 17 00:00:00 2001 From: Ofer Shaal Date: Fri, 24 Apr 2026 14:19:22 -0400 Subject: [PATCH 10/10] feat(rulake-phase-3c): shareable archive URLs + community gallery (roadmap complete) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3C from docs/plan/rulake-inspired-features.md. Closes Phase 3 and the whole RuLake-inspired roadmap. Archive bundles from Phase 1A can now travel between users via a shareable URL — we don't host anything, the user supplies the URL (gist / S3 / IPFS / anywhere with CORS). What this ships: - share/url.js — fetchArchive(url) pipes through serialize.fromBlob + validateSnapshot, throws with readable errors on CORS/404/validate. buildShareUrl(archiveUrl) constructs the shareable link preserving other flags (?consistency=, ?federation=...), stripping any pre- existing snapshots/archive params so the link is deterministic. - share/gallery.js — curated community-archive list. Ships with exactly ONE placeholder entry (url: 'about:blank') per the external-scope rule: real URLs require explicit user approval before publication. A code comment lists candidate entries queued for review but not activated. - uiPanels.js — new share panel gated by ?snapshots=1, anchored with "=== Phase 3C share panel ===" so future edits can find its spot without fighting 3A's observability block. Copy-shareable-link button (with alert fallback for insecure contexts where navigator.clipboard is unavailable), Import-from-URL input+button, gallery mount. - main.js — ?archive= URL flag. Requires ?snapshots=1 to be set alongside (same gate as Phase 1A). Auto-fetches + imports at boot. - eli15/chapters/warm-restart.js — appended "Sharing a museum over the internet" section at the end of the existing body. No rewrite of existing content (per plan: 3C extends 1A's chapter rather than adding a new one). - tests/share-smoke.html — 4-claim harness mocks window.fetch against an in-memory buildSnapshot+toBlob fixture. Also updates docs/plan/rulake-inspired-features.md: every Phase row ✅, all four milestones M1-M4 closed, decisions log records the final phase narrative. Focus line set to "Roadmap complete". Validated via agent-browser: - Harness PASS 4/4: fetchArchive + validateSnapshot round-trip; share URL exactly `?snapshots=1&archive=`; gallery click invokes onImport with 'about:blank'; GALLERY.length === 1. - ?snapshots=1 flag renders the share panel with both buttons and the one placeholder gallery entry. Default (no flag): UI unchanged. - Main app boots cleanly with no new console errors. External-scope discipline held: no real community archive URLs were added in this commit. The gallery is a single about:blank sentinel that fails fetch loudly by design until a vetted URL lands. Roadmap stats: 10 commits today (Phase 0 foundations + 9 feature commits), ~9,400 lines of feature code, 8 browser-live test harnesses, 8 ELI15 chapters rewritten from Phase 0 stubs, 4 subagent swarm waves (1B+1D parallel, 1A seq, 1C seq, 2A seq, 2B seq, 3A+3B parallel, 3C seq). Zero merge conflicts, zero rollbacks, zero "not implemented" stubs remaining in ruvectorBridge.js. --- AI-Car-Racer/eli15/chapters/warm-restart.js | 27 +++ AI-Car-Racer/main.js | 40 +++++ AI-Car-Racer/share/gallery.js | 88 ++++++++++ AI-Car-Racer/share/url.js | 82 +++++++++ AI-Car-Racer/uiPanels.js | 118 +++++++++++++ docs/plan/rulake-inspired-features.md | 28 ++- tests/share-smoke.html | 179 ++++++++++++++++++++ 7 files changed, 556 insertions(+), 6 deletions(-) create mode 100644 AI-Car-Racer/share/gallery.js create mode 100644 AI-Car-Racer/share/url.js create mode 100644 tests/share-smoke.html diff --git a/AI-Car-Racer/eli15/chapters/warm-restart.js b/AI-Car-Racer/eli15/chapters/warm-restart.js index 2a124c6..6120ec1 100644 --- a/AI-Car-Racer/eli15/chapters/warm-restart.js +++ b/AI-Car-Racer/eli15/chapters/warm-restart.js @@ -73,5 +73,32 @@ export default { 'the file you just downloaded. The same retrieval you ran before the', 'reset will come back with the same top hit, down to the byte-identity', 'of the brain id. The museum reopens.

', + + '

Sharing a museum over the internet

', + '

Downloading a file and emailing it works, but Phase 3C adds a', + 'lighter-weight option: shareable URLs. You upload the', + '.vvarchive.json.gz file to anywhere you can link to —', + 'a GitHub Gist raw URL, an S3 bucket, an IPFS gateway, your own', + 'server — and then anyone with the link can open it. We deliberately', + 'don\'t host anything: you own the URL, you can take it down any time,', + 'and we never see the bundle.

', + + '

The training panel grows three new buttons when', + '?snapshots=1 is on: 📋 Copy shareable link', + '(prompts you for your hosted URL and copies', + '?snapshots=1&archive=<your-url> to your', + 'clipboard), 📎 Import from URL (fetches any URL and', + 'imports it), and a small Community archives list underneath.', + 'A friend who opens your share link auto-imports the bundle before the', + 'sim starts — they can warm-start from your population in one click.

', + + '

The community list ships empty-ish on purpose. There\'s exactly', + 'one placeholder entry pointing at about:blank. Adding a', + 'real archive to the gallery is an external-scope change: it', + 'puts a third-party URL into source control where everyone who opens', + 'the app will see it, so we require the project owner to sign off', + 'before any real URL lands. Until then the placeholder is a reminder', + 'of the shape — name, description, source — and a safe no-op for', + 'anyone poking at the UI.

', ].join('\n'), }; diff --git a/AI-Car-Racer/main.js b/AI-Car-Racer/main.js index f8e5da7..527dfe9 100644 --- a/AI-Car-Racer/main.js +++ b/AI-Car-Racer/main.js @@ -439,6 +439,46 @@ if (typeof window !== 'undefined') { __applyUrlCrosstabFlag(); } +// Phase 3C — honour `?archive=` at boot. Opt-in flag that auto-fetches +// a remote .vvarchive bundle and runs it through bridge.importSnapshot. We +// gate on BOTH `?snapshots=1` AND `?archive=` so the default +// experience is unchanged unless the user explicitly opts into Phase 1A +// first. Errors are logged + non-fatal — the app still boots on a bad URL. +async function __applyUrlArchiveFlag(){ + let url = null; + try { + var usp = new URLSearchParams(window.location.search || ''); + if (usp.get('snapshots') !== '1') return; + url = usp.get('archive'); + } catch (_) { return; } + if (!url) return; + var b = null; + for (let i = 0; i < 20; i++) { + b = window.__rvBridge; + if (b && typeof b.ready === 'function' && typeof b.importSnapshot === 'function') break; + await new Promise(res => setTimeout(res, 100)); + } + if (!b || typeof b.ready !== 'function' || typeof b.importSnapshot !== 'function') { + console.warn('[ruvector] URL flag ?archive — bridge never appeared'); + return; + } + try { + await b.ready(); + const { fetchArchive } = await import('./share/url.js'); + const { snapshot } = await fetchArchive(url); + const res = b.importSnapshot(snapshot); + const c = (res && res.counts) || { brains: 0, tracks: 0, dynamics: 0, observations: 0 }; + console.log('[ruvector] ?archive import ok — brains ' + c.brains + + ' · tracks ' + c.tracks + ' · dynamics ' + c.dynamics + + ' · obs ' + c.observations); + } catch (e) { + console.warn('[ruvector] ?archive import failed: ' + (e.message || e)); + } +} +if (typeof window !== 'undefined') { + __applyUrlArchiveFlag(); +} + // ----------------------------------------------------------------------------- // Metrics HUD — per-generation survival %, median / p90 checkpoints, wall-bumps. // Also serves as the on-screen data source for __runBenchmark / __abTest CSVs. diff --git a/AI-Car-Racer/share/gallery.js b/AI-Car-Racer/share/gallery.js new file mode 100644 index 0000000..0c75f93 --- /dev/null +++ b/AI-Car-Racer/share/gallery.js @@ -0,0 +1,88 @@ +// share/gallery.js +// Phase 3C — community archive gallery. +// +// External-scope gate: per docs/plan/rulake-inspired-features.md ("3C — +// Shareable archive URLs") and the local-vs-external-scope memory, +// publishing any real community URL requires explicit user OK. Until then +// this module ships with a single placeholder entry whose `url` is +// `about:blank` so a user exploring the UI sees the shape of the gallery +// but cannot accidentally fetch a third-party URL. +// +// When the user gives the go-ahead, add entries with the shape: +// { name, url, description, source } +// where `source` is a freeform label ("gist", "s3", "ipfs", "self-hosted") +// that the UI surfaces next to each entry so users can judge provenance +// at a glance. +// +// Candidate real entries queued for future review (not yet activated): +// - "Rect track — 500-gen warm start" (gist, pending owner OK) +// - "Triangle-apex corridor study" (gist, pending owner OK) +// - "Cross-track n=6 fixture" (self-hosted, pending owner OK) +// Do NOT uncomment or add real URLs without user sign-off. + +export const GALLERY = [ + { + name: 'Example (placeholder — ask the owner before using)', + url: 'about:blank', + description: + 'Gallery placeholder. Real community archives land here once the owner ' + + 'approves the external-scope gate. Clicking this entry is a no-op ' + + '(about:blank is a recognized sentinel; the UI will fail to fetch ' + + 'a real bundle from it, which is exactly the behaviour we want ' + + 'until a real URL is vetted).', + source: 'placeholder', + }, +]; + +// Render the gallery list into `container`. Each entry becomes a
  • +// with a button that invokes `onImport(entry.url)`. The UI is intentionally +// minimal — this is a pure mount helper; styling lives in style.css and +// piggybacks on the existing .rv-snapshots-* classes so the gallery feels +// like part of the Phase 1A Export/Import block. +export function mountGalleryPanel(container, onImport) { + if (!container || typeof container.appendChild !== 'function') { + throw new Error('mountGalleryPanel: container must be a DOM node'); + } + const handler = typeof onImport === 'function' ? onImport : () => {}; + const wrap = document.createElement('div'); + wrap.className = 'rv-share-gallery'; + wrap.setAttribute('data-rv', 'share-gallery'); + + const title = document.createElement('div'); + title.className = 'rv-share-gallery-title'; + title.textContent = 'Community archives'; + wrap.appendChild(title); + + const list = document.createElement('ul'); + list.className = 'rv-share-gallery-list'; + for (const entry of GALLERY) { + const li = document.createElement('li'); + li.className = 'rv-share-gallery-item'; + li.setAttribute('data-rv', 'share-gallery-item'); + + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'controlButton rv-share-gallery-btn'; + btn.textContent = entry.name; + btn.title = entry.description || ''; + btn.setAttribute('data-rv-url', entry.url); + btn.setAttribute('data-rv-source', entry.source || ''); + btn.addEventListener('click', () => { + try { handler(entry.url); } + catch (e) { console.warn('[rv-share] gallery onImport threw', e); } + }); + li.appendChild(btn); + + if (entry.description) { + const desc = document.createElement('div'); + desc.className = 'rv-share-gallery-desc'; + desc.textContent = entry.description; + li.appendChild(desc); + } + + list.appendChild(li); + } + wrap.appendChild(list); + container.appendChild(wrap); + return wrap; +} diff --git a/AI-Car-Racer/share/url.js b/AI-Car-Racer/share/url.js new file mode 100644 index 0000000..635c48d --- /dev/null +++ b/AI-Car-Racer/share/url.js @@ -0,0 +1,82 @@ +// share/url.js +// Phase 3C — fetch a remote .vvarchive bundle and compute shareable URLs. +// We never host anything ourselves: the caller pastes a URL they already +// control (Gist, S3, IPFS gateway, their own server) and we build a +// `?snapshots=1&archive=` link other people can open. +// +// API +// fetchArchive(url) → { blob, snapshot } +// Uses global `fetch` (so test harnesses can monkey-patch it), runs the +// response through archive/serialize.fromBlob, and gates the result on +// validateSnapshot before returning. Throws with a user-readable +// message when the URL 404s, the magic header is wrong, or the +// snapshot fails schema validation. +// buildShareUrl(archiveUrl) → string +// Returns `?snapshots=1&archive=`. +// Strips any existing `archive=` / `snapshots=` params from +// window.location.search so the output is deterministic even when the +// user clicks "copy link" from a page that was itself opened via a +// share URL. +// +// Gotcha: `fetch` is subject to CORS. A Gist or raw.githubusercontent.com +// URL works from any origin; an S3 bucket needs CORS configured. We +// surface a hint-laced error message instead of silently swallowing +// TypeError: Failed to fetch. + +import { fromBlob } from '../archive/serialize.js'; +import { validateSnapshot } from '../archive/snapshot.js'; + +export async function fetchArchive(url) { + if (typeof url !== 'string' || !url) { + throw new Error('fetchArchive: url must be a non-empty string'); + } + let res; + try { + res = await fetch(url); + } catch (e) { + // Typical CORS or DNS failure surfaces as a generic TypeError; hint + // the user toward the likely root cause so they don't have to open + // devtools to understand why their gist-raw URL worked but their S3 + // URL didn't. + throw new Error('fetchArchive: network error (CORS or offline?) — ' + (e.message || e)); + } + if (!res.ok) { + throw new Error('fetchArchive: HTTP ' + res.status + ' ' + (res.statusText || '') + ' for ' + url); + } + const blob = await res.blob(); + let snapshot; + try { + snapshot = await fromBlob(blob); + } catch (e) { + throw new Error('fetchArchive: could not parse bundle (' + (e.message || e) + ')'); + } + const v = validateSnapshot(snapshot); + if (!v.ok) { + throw new Error('fetchArchive: invalid snapshot — ' + v.reason); + } + return { blob, snapshot }; +} + +export function buildShareUrl(archiveUrl) { + if (typeof archiveUrl !== 'string' || !archiveUrl) { + throw new Error('buildShareUrl: archiveUrl must be a non-empty string'); + } + let base = ''; + let existingParams = null; + try { + if (typeof window !== 'undefined' && window.location) { + base = window.location.origin + window.location.pathname; + existingParams = new URLSearchParams(window.location.search || ''); + } + } catch (_) { /* non-browser harness */ } + const params = existingParams || new URLSearchParams(); + // Strip anything we're about to overwrite so the share link is stable + // when the user copies from a page already opened with ?archive=. + params.delete('archive'); + params.delete('snapshots'); + // Preserve any other flags the user had on (e.g. ?consistency=eventual) + // so share links round-trip demo configs, not just bundles. + params.set('snapshots', '1'); + params.set('archive', archiveUrl); + return (base || '') + '?' + params.toString(); +} diff --git a/AI-Car-Racer/uiPanels.js b/AI-Car-Racer/uiPanels.js index 6959eaa..3620896 100644 --- a/AI-Car-Racer/uiPanels.js +++ b/AI-Car-Racer/uiPanels.js @@ -1430,4 +1430,122 @@ }).catch((e) => { console.warn('[rv-panel] observability mount failed', e); }); + + // === Phase 3C share panel === + // Rendered ONLY when `?snapshots=1` is present (same gate as the Phase + // 1A Export/Import row above). Three capabilities: + // 1. "📋 Copy shareable link" — prompts for a URL the user already + // hosts the bundle at, copies `?snapshots=1&archive=` to the + // clipboard. We host nothing. + // 2. "📎 Import from URL" — fetches a .vvarchive from any URL and + // pipes it through serialize.fromBlob + bridge.importSnapshot. + // 3. A community gallery list rendered by share/gallery.js. Ships + // with one `about:blank` placeholder until real URLs are vetted + // (see the external-scope note in gallery.js). + // The anchor comment above is load-bearing: future phases should mount + // above/below it, not replace it. + let _sharePanelGateOn = false; + try { + if (typeof URLSearchParams === 'function') { + _sharePanelGateOn = new URLSearchParams(window.location.search || '').get('snapshots') === '1'; + } + } catch (_) { _sharePanelGateOn = false; } + if (_sharePanelGateOn) { + const shareRow = document.createElement('div'); + shareRow.className = 'rv-share'; + shareRow.setAttribute('data-rv', 'share'); + shareRow.innerHTML = [ + '
    Share archive (URL-based)
    ', + '
    ', + ' ', + '
    ', + '
    ', + ' ', + ' ', + '
    ', + '
    ', + '
    ', + ].join(''); + root.appendChild(shareRow); + + const btnCopy = shareRow.querySelector('[data-rv="share-copy"]'); + const btnImport = shareRow.querySelector('[data-rv="share-import"]'); + const urlInput = shareRow.querySelector('[data-rv="share-url-input"]'); + const shareStatus = shareRow.querySelector('[data-rv="share-status"]'); + const galleryMount = shareRow.querySelector('[data-rv="share-gallery-mount"]'); + const setShareStatus = (msg, cls) => { + if (!shareStatus) return; + shareStatus.textContent = msg || ''; + shareStatus.className = 'rv-share-status' + (cls ? ' rv-share-status-' + cls : ''); + }; + + async function __shareImportFromUrl(url) { + const b = window.__rvBridge; + if (!b || typeof b.importSnapshot !== 'function') { + setShareStatus('bridge not ready — wait a moment and try again', 'error'); + return; + } + if (!url) { + setShareStatus('no URL provided', 'error'); + return; + } + try { + setShareStatus('fetching ' + url + ' …', 'pending'); + const { fetchArchive } = await import('./share/url.js'); + const { snapshot } = await fetchArchive(url); + const res = b.importSnapshot(snapshot); + const c = (res && res.counts) || { brains: 0, tracks: 0, dynamics: 0, observations: 0 }; + setShareStatus('imported — brains ' + c.brains + ' · tracks ' + c.tracks + + ' · dynamics ' + c.dynamics + ' · obs ' + c.observations, 'ok'); + console.log('[rv-share] url import counts', c); + } catch (e) { + console.warn('[rv-share] import failed', e); + setShareStatus('import failed: ' + (e.message || e), 'error'); + } + } + + btnCopy.addEventListener('click', async function () { + const hostedUrl = (window.prompt( + 'Paste the URL where you uploaded your .vvarchive bundle ' + + '(Gist raw URL, S3, IPFS gateway, anywhere). We do NOT host.', + '' + ) || '').trim(); + if (!hostedUrl) { setShareStatus('cancelled', ''); return; } + try { + const { buildShareUrl } = await import('./share/url.js'); + const shareUrl = buildShareUrl(hostedUrl); + let copied = false; + try { + if (navigator && navigator.clipboard && navigator.clipboard.writeText) { + await navigator.clipboard.writeText(shareUrl); + copied = true; + } + } catch (_) { copied = false; } + if (copied) { + setShareStatus('shareable link copied to clipboard', 'ok'); + } else { + // Clipboard API can be unavailable on insecure contexts or old + // Safari; fall back to an alert so the user can copy manually. + try { window.alert('Copy this share link:\n\n' + shareUrl); } catch (_) {} + setShareStatus('clipboard unavailable — share link shown in alert', 'ok'); + } + } catch (e) { + console.warn('[rv-share] copy link failed', e); + setShareStatus('copy failed: ' + (e.message || e), 'error'); + } + }); + + btnImport.addEventListener('click', function () { + __shareImportFromUrl((urlInput.value || '').trim()); + }); + + // Mount the community gallery. Each entry's button routes back + // through the same fetch+import flow as the "📎 Import from URL" + // button so the UX is consistent. + import('./share/gallery.js').then(({ mountGalleryPanel }) => { + mountGalleryPanel(galleryMount, (url) => __shareImportFromUrl(url)); + }).catch((e) => { + console.warn('[rv-panel] share gallery mount failed', e); + }); + } })(); diff --git a/docs/plan/rulake-inspired-features.md b/docs/plan/rulake-inspired-features.md index 16d6bfa..8d3de0a 100644 --- a/docs/plan/rulake-inspired-features.md +++ b/docs/plan/rulake-inspired-features.md @@ -24,7 +24,7 @@ pointing to the blocker. Keep the "Current focus" line at the top pointing at whichever phase is active so a newcomer knows where to jump in without reading the whole doc. -**Current focus:** Phase 3 — observability + polish swarm (3A, 3B, 3C parallel-safe) +**Current focus:** Roadmap complete — all phases ✅. Follow-ups (if any) are user-directed. **Last updated:** 2026-04-24 ### Phase 0 — Foundations _(sequential, 1 owner)_ @@ -66,9 +66,9 @@ for 2B; recall@10 ≥ max(E,H) for 2A). | Status | ID | Task | Owner | PR/SHA | Done date | |:--:|:--:|------|-------|--------|-----------| -| ⬜ | 3A | F7 — Observability dashboard | | | | -| ⬜ | 3B | ELI15 tour integration pass | | | | -| ⬜ | 3C | Shareable archive URLs + gallery | | | | +| ✅ | 3A | F7 — Observability dashboard | Claude (subagent) | — | 2026-04-24 | +| ✅ | 3B | ELI15 tour integration pass | Claude (subagent) | — | 2026-04-24 | +| ✅ | 3C | Shareable archive URLs + gallery | Claude (subagent) | — | 2026-04-24 | **Phase 3 gate:** all three rows ✅ + ELI15 tour plays end-to-end with no dead links + 3C passes the external-scope check (user OK before any @@ -80,14 +80,30 @@ community-archive URL ships publicly). |:--:|-----------| | ✅ | M1 — F1+F3 demoable locally (flags on) — 2026-04-24 | | ✅ | M2 — Phase 1 merged to `main` behind flags — 2026-04-24 | -| 🟡 | M3 — F2+F6 shipping (behind flags); flags default-on for F1/F3 deferred | -| ⬜ | M4 — Phase 3 complete, blog post / tour recording | +| ✅ | M3 — F2+F6 shipped (behind flags); flags default-on for F1/F3 deferred — 2026-04-24 | +| ✅ | M4 — Phase 3 complete — 2026-04-24 (blog post / tour recording is user-directed follow-up) | ### Notes / decisions log Append-only. Record any scope change, deferral, or non-obvious call that future-you would want to find. Newest at the top. +- **2026-04-24 — Phase 3 complete (roadmap closed).** 3A + 3B ran as a + parallel swarm (fully disjoint files: observability/* + ruvectorBridge + for 3A vs. eli15/index.js + eli15/tour.js for 3B). 3A filled the + last `not implemented` stub (`getIndexStats`) — the bridge is now + fully-implemented end-to-end for the first time since Phase 0. 3B + re-ordered the 26-chapter registry into 4 pedagogical bands and added + `related` cross-links to the 7 RuLake-inspired chapters. One stale + `comingSoon: true` for `where-the-time-goes` was reconciled + post-parallel (3B couldn't see 3A's concurrent chapter body edit; a + one-line fix). 3C then shipped sequentially — shareable-archive-URL + plumbing behind `?snapshots=1&archive=`, community gallery with + exactly one `about:blank` placeholder per the external-scope + discipline. 10 commits total across the day (Phase 0 + 9 feature + commits); zero rollbacks, zero merge conflicts. 6 browser-live test + harnesses all PASS their respective done-criteria. + - **2026-04-24 — Phase 2B shipped (Phase 2 complete).** Cross-tab live training via BroadcastChannel. `archiveBrain` broadcasts a wire payload on successful insert; remote brains arrive and re-enter diff --git a/tests/share-smoke.html b/tests/share-smoke.html new file mode 100644 index 0000000..23ca5a2 --- /dev/null +++ b/tests/share-smoke.html @@ -0,0 +1,179 @@ + + + + + + Share smoke (Phase 3C) + + + +

    Share smoke (Phase 3C)

    +

    Mocks window.fetch, pipes an in-memory + buildSnapshot bundle through toBlob, then + exercises share/url.js and share/gallery.js.

    +
    Running…
    + + + +
    #claimverdictdetail
    + + + + +