From 130e242f35917c065b7830ab57f498e01211a4b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 20:25:15 +0000 Subject: [PATCH 1/2] fix(dig-node-core): bind admit root to ChunkPool content (#2246 #2240) The capsule-admit trust gate (ChainAnchoredModuleVerifier::rejection_reason) recomputed the merkle root from the capsule's own MerkleNodes section and compared it to the committed CurrentRoot. That is hollow: rule 4 already forces CurrentRoot == chain_root, a single untagged leaf's root IS that leaf, and decode_merkle_leaves accepts arbitrary bytes -- so MerkleNodes=[chain_root] with an empty/garbage ChunkPool passed for free, admitting a contentless capsule that was then cached and DHT-announced (phantom-holder / holder-set poisoning). Bind the SERVED content instead: recompute each resource leaf as resource_leaf(concat(ChunkPool ciphertexts)) per KeyTable entry, sort by static_key (the producer's order), fold, and require the root == committed root (== chain_root). Also require the recomputed leaves equal decode_merkle_leaves (MerkleNodes) so served inclusion proofs stay consistent with content. MerkleNodes is no longer trusted for the admit decision. Fail-closed on absent/undecodable KeyTable/ChunkPool or any out-of-range chunk index; an empty store folds to sha256([]). Covers both reshare-admit and cache.pushCapsule land (shared gate; full ChunkPool is materialized before the gate on both paths). No digstore change. 0.100.2->0.100.3. Closes #2246, Closes #2240. Co-Authored-By: Claude --- Cargo.lock | 2 +- Cargo.toml | 2 +- DEVELOPMENT_LOG.md | 41 ++ SPEC.md | 35 +- crates/dig-node-core/src/lib.rs | 133 ++-- .../src/seams/capsule/push_capsule.rs | 129 +++- .../src/seams/dig_peer/module_anchor.rs | 642 +++++++++++++++++- .../src/seams/dig_peer/module_reshare.rs | 83 ++- 8 files changed, 962 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a0b771..ad5358f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2295,7 +2295,7 @@ dependencies = [ [[package]] name = "dig-node-service" -version = "0.102.0" +version = "0.102.1" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index 6123e64..5ddd531 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ edition = "2021" # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet) # keep their own independent versions — only the released binary tracks the workspace version. -version = "0.102.0" +version = "0.102.1" # Release hardening, matching digstore: keep integer-overflow checks ON in release. # The node parses untrusted serialized input and does offset/length arithmetic over diff --git a/DEVELOPMENT_LOG.md b/DEVELOPMENT_LOG.md index eafd15e..dbb848f 100644 --- a/DEVELOPMENT_LOG.md +++ b/DEVELOPMENT_LOG.md @@ -25,6 +25,47 @@ dig-updater's feedsign resolves dig-node by native-package file names and fails ENTIRE manifest, that froze — then expired — stable auto-update for all five components, dig-app included. A dig-node release without its `.msi`/`.pkg`/`.deb` is not a partial dig-node release, it is an ecosystem-wide auto-update outage. +## Admit gate must recompute from CONTENT, not from the attacker's MerkleNodes digests (#2246/#2240) + +`ChainAnchoredModuleVerifier` (the capsule-admit gate shared by the reshare-admit pull AND the +`cache.pushCapsule` land via `verify_capsule_integrity`) once only byte-compared the capsule's committed +`CurrentRoot` header against the chain-anchored root. A first fix RECOMPUTED +`MerkleTree::from_leaves(decode_merkle_leaves(MerkleNodes)).root()` — but that was HOLLOW and adversarial +verification refuted it. Rule 4 already forces `committed_root == chain_root` (a public value); a +one-leaf tree's root IS that leaf (`from_leaves` does NOT re-tag leaves — `LEAF_TAG` is applied only in +`build`, and there is no fold for a single leaf); and `decode_merkle_leaves` accepts arbitrary bytes. So +`MerkleNodes = [chain_root]` plus an empty or garbage `ChunkPool` recomputed to the committed root FOR +FREE — admitting, caching, serving, and DHT-announcing a contentless phantom-holder capsule. Trusting +attacker-supplied digests for an admit decision proves nothing. + +The gate now recomputes the root from the SERVED CONTENT: for each `KeyTable` (id 8) entry, gather its +chunk ciphertexts from the `ChunkPool` (id 9) via `datasection::read_chunk`, `leaf = +merkle::resource_leaf(serving::concat_output(cts))`, SORT the `(static_key, leaf)` pairs ASCENDING by +`static_key`, then fold `MerkleTree::from_leaves`. The sort is load-bearing: the producer +(`digstore-store` `store.rs`) sorts `resource_leaves.sort_by_key(|r| r.0)` before folding, but KeyTable +storage order is NOT guaranteed sorted, so recomputing in storage order yields the wrong root for ≥2 +resources. `MerkleNodes` is retained ONLY as a defense-in-depth cross-check (its leaves must equal the +sorted content leaves, since the served inclusion proofs are generated from it) — never as the trust +anchor. Fail-closed on an absent `KeyTable`/`ChunkPool`, an out-of-range chunk index, an undecodable +section, or a `MerkleNodes`↔content mismatch. A legitimately EMPTY store (no entries) folds to +`from_leaves(vec![]).root() == sha256(&[])` and MUST be admitted, not errored (§5.1). Lesson: an +integrity gate must bind the BYTES it will serve, never a sibling digest field the same attacker chose. + +The content recompute is CURRENT-GENERATION-SCOPED (#2246 gen-scoping fix): the embedded `KeyTable` is +MULTI-generation (`digstore-compiler` `key_table.rs` pushes one entry per (generation, resource), each +stamped `entry.generation = gen.root()`), but the committed `CurrentRoot` is folded over the CURRENT +generation ONLY (`pipeline.rs` → `current_generation_leaves(generations.last())`, whose `gen.root()` == +`CurrentRoot`). Folding EVERY entry over-counts leaves, so ANY store published then updated even once +(≥2 generations — the normal lifecycle) was false-rejected `NotAnchored`, breaking admit/cache/announce. +Fix: fold only entries where `entry.generation.0 == committed_root`. The single-generation fixtures +missed this — a faithful multi-gen fixture is now the regression. Also DoS-bounded (#2246): `chunk_indices` +is attacker-controlled and legitimately permits REPEATED indices (the producer dedups chunks — identical +or shared chunks yield repeated/non-increasing global indices, so repeats can't be banned), so a ~1 MB +module with `chunk_indices=[0;N]` over one large chunk would `concat_output` gigabytes and abort the +allocator. Fix: STREAM each ciphertext into an incremental SHA-256 (`resource_leaf(concat)==sha256(ct0++ct1++…)` +since `resource_leaf` is plain sha256 and `concat_output` is plain concat → O(1) memory) AND cap total +referenced bytes at `MAX_STORE_BYTES` (bounds CPU). Lesson: recompute-from-content must scope to the +producer's committed generation AND bound attacker-controlled index fan-out. ## Local-RPC authz — holder-REVEALING reads gate too, not just mutators (#2108) diff --git a/SPEC.md b/SPEC.md index 0feabd7..1e666e6 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1174,6 +1174,28 @@ OPENED push MUST additionally prove the caller is the store's **§21.6/§21.9 au the target store: the pushed module commits a publisher public key whose `SHA-256` DERIVES `store_id` (`store_id = sha256(publisher_pubkey)`, the DIG store-identity derivation), AND the request carries a BLS signature over `SHA-256(root || store_id)` that verifies under that key. The merkle-integrity +check RECOMPUTES the merkle root from the capsule's own SERVED CONTENT — for each **current-generation** +`KeyTable` entry, `leaf = resource_leaf(concat_output(its ChunkPool ciphertexts))`, the leaves sorted +ASCENDING by `static_key`, folded with `MerkleTree::from_leaves` — and refuses the push unless it +reproduces the committed `CurrentRoot`. The recompute is SCOPED TO THE CURRENT GENERATION: the embedded +`KeyTable` is multi-generation (the producer stores one entry per (generation, resource), each stamped +with THAT generation's root), but the committed `CurrentRoot` is folded over the CURRENT generation only +(`current_generation_leaves(generations.last())`), whose `gen.root()` equals `CurrentRoot`. So the +recompute folds ONLY entries whose `entry.generation == CurrentRoot`; folding every generation's entries +over-counts and would false-reject the genuine current content of any store published then updated even +once (#2246). The recompute is also BOUNDED against a remote pre-auth OOM/CPU-DoS: `chunk_indices` is +attacker-controlled and permits repeated indices (the producer dedups chunks), so the TOTAL referenced +ciphertext bytes across the module is capped at `MAX_STORE_BYTES` and each resource's leaf is hashed by +STREAMING its ciphertexts into an incremental SHA-256 (O(1) memory) rather than materializing their +concatenation — without which a ~1 MB module addressing one chunk N times could reference gigabytes and +abort the allocator (#2246). The attacker-supplied `MerkleNodes` digests are NEVER trusted for this +decision (retained only as a defense-in-depth cross-check that the served inclusion proofs match the +content): a single-leaf `from_leaves(vec![x]).root() == x` meant a `MerkleNodes = [chain_root]` plus an +empty/garbage `ChunkPool` recomputed to the committed root for free, admitting a contentless +phantom-holder capsule (#2246/#2240). An absent `KeyTable`/`ChunkPool`, a chunk index the pool cannot +satisfy, an undecodable section, a `MerkleNodes`↔content mismatch, or referenced content exceeding +`MAX_STORE_BYTES` fails closed; a legitimately EMPTY store folds to `from_leaves(vec![]).root() == sha256(&[])` and passes. A header naming the chain root is +not proof the bytes hash to it. This check gives INTEGRITY, never AUTHORITY — without the writer check an opened node would be an unauthenticated cache-poison + DHT-announce-amplification surface (the #179/#1576 class). A push that arrives on the peer surface with no signature, a signature under a key that does not derive @@ -1912,7 +1934,18 @@ NOT make naming a near key cost an on-chain mint: a peer may name any `(store, r near our `peer_id` and, on an opted-in node, drive a CHEAP DHT provider-lookup for it (a key that names no real store simply finds no providers and the pull fails there — the low cost is "no providers", not a per-key mint). The on-chain-mint + merkle cost binds a LATER step — actually becoming a cached -HOLDER: a pulled module is bound to its `root` by merkle verification and is never SERVED as current +HOLDER: a pulled module is bound to its `root` by merkle verification — the admit gate +(`ChainAnchoredModuleVerifier`, shared by the reshare-admit pull AND the `cache.pushCapsule` land via +`verify_capsule_integrity`) RECOMPUTES the merkle root from the capsule's own SERVED CONTENT (per +`KeyTable` entry, `leaf = resource_leaf(concat_output(its ChunkPool ciphertexts))`, leaves sorted +ASCENDING by `static_key`, folded via `MerkleTree::from_leaves`) and refuses (`NotAnchored`) unless it +equals the committed `CurrentRoot`. The attacker-supplied `MerkleNodes` digests are NEVER trusted for +the admit decision (only cross-checked for served-proof consistency): trusting them let a single-leaf +`MerkleNodes = [chain_root]` plus an empty/garbage `ChunkPool` recompute to the committed root for free +and admit a contentless phantom-holder capsule (#2246/#2240). So a header-matching but +tampered/incomplete `.dig` (or one with an absent `KeyTable`/`ChunkPool`, an out-of-range chunk index, +or an undecodable section) is never admitted; a legitimately EMPTY store folds to `sha256(&[])` and is. +It is never SERVED as current unless `root` equals the chain-anchored tip (the serve-time read-path pin, §7.10d(a) / §14.4). So the worst a near-key attacker extracts from an opted-in node is a bounded, single-flighted, byte-capped pull of REAL near-neighbourhood content of a possibly-old generation — never caching of fabricated, diff --git a/crates/dig-node-core/src/lib.rs b/crates/dig-node-core/src/lib.rs index ef6b8f9..4021fd0 100644 --- a/crates/dig-node-core/src/lib.rs +++ b/crates/dig-node-core/src/lib.rs @@ -4917,21 +4917,25 @@ mod tests { } /// Spawn the REAL §21 `RemoteServer` (auth REQUIRED by default) over an - /// in-memory backend seeded with one store whose module is `module` at root - /// 0x10. Returns `(base_url, store_id_hex)`. Unlike the header-recording mock - /// below, this exercises the actual §21.9 auth middleware end-to-end. + /// in-memory backend seeded with one store serving `module` at the generation + /// root the module itself commits (parsed from its `CurrentRoot`) — so a faithful + /// capsule is served under the same root its content folds to (#2246). Returns + /// `(base_url, store_id_hex)`. Unlike the header-recording mock below, this + /// exercises the actual §21.9 auth middleware end-to-end. async fn spawn_authed_remote(module: Vec) -> (String, String) { + use digstore_core::datasection::{DataView, SectionId}; use digstore_core::Bytes48; use digstore_remote::{InMemoryBackend, RemoteServer}; + let served_root = { + let view = DataView::parse(&module).expect("module is a valid data section"); + let body = view + .section(SectionId::CurrentRoot) + .expect("module commits a CurrentRoot"); + Bytes32(<[u8; 32]>::try_from(body).expect("CurrentRoot is 32 bytes")) + }; let be = Arc::new(InMemoryBackend::new()); let store_id = Bytes32([1u8; 32]); - be.add_store( - store_id, - Bytes48([2u8; 48]), - Bytes32([0x10; 32]), - module, - None, - ); + be.add_store(store_id, Bytes48([2u8; 48]), served_root, module, None); let app = RemoteServer::new(be).router(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -4947,7 +4951,7 @@ mod tests { // whole module is synced, and it lands in the on-disk cache for local-first. let store = Bytes32([1u8; 32]); // the id spawn_authed_remote seeds let root = Bytes32([0x10; 32]); // its served genesis root - let module = chain_anchored_module(store.0, root.0); + let (module, root) = chain_anchored_module(store.0, root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let (node, _td) = test_node_with_resolver(Some([5u8; 32]), MockResolver::one(&store_hex, root)); @@ -4971,7 +4975,7 @@ mod tests { async fn gap_fill_pulls_a_missing_generation_from_a_remote() { // The remote's served genesis root. let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let store_id: [u8; 32] = Bytes32::from_hex(&store_hex).unwrap().0; // A node with a §21 identity whose UPSTREAM is the authed remote (gap-fill pulls via upstream). @@ -5023,7 +5027,7 @@ mod tests { #[tokio::test] async fn chain_watch_tick_gap_fills_a_subscribed_store_end_to_end() { let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let td = tempfile::tempdir().unwrap(); let node = Arc::new(Node { @@ -5102,7 +5106,7 @@ mod tests { .unwrap(); rt.block_on(async { let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let td = tempfile::tempdir().unwrap(); @@ -5182,7 +5186,7 @@ mod tests { .unwrap(); rt.block_on(async { let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let td = tempfile::tempdir().unwrap(); @@ -5465,7 +5469,7 @@ mod tests { let store_a = Bytes32([0x2au8; 32]); let root = Bytes32([0x2bu8; 32]); let store_b = "bb".repeat(32); - let capsule = chain_anchored_module(store_a.0, root.0); + let (capsule, root) = chain_anchored_module(store_a.0, root.0); assert!( (capsule.len() as u64) < 100_000, "the synced capsule must fit UNDER the cap so ONLY the tier-0 module is the eviction victim" @@ -5534,7 +5538,7 @@ mod tests { let served = Bytes32([0xAAu8; 32]); let requested = Bytes32([0xBBu8; 32]); let store_b = "cc".repeat(32); - let module = chain_anchored_module(store.0, served.0); + let (module, served) = chain_anchored_module(store.0, served.0); assert!( (module.len() as u64) < 100_000, "the synced capsule must fit UNDER the cap so ONLY the tier-0 module is the eviction victim" @@ -5617,11 +5621,13 @@ mod tests { // Store A: landed by a real reshare warm (chain-anchored, verified, promoted). Store B: a // PRE-EXISTING tier-0 precache module, oversized so the cache is over the cap the instant A lands. let store_a = [0x5au8; 32]; - let root = [0x5bu8; 32]; let store_a_hex = hex::encode(store_a); - let root_hex = hex::encode(root); let store_b = "b5".repeat(32); - let module = chain_anchored_module(store_a, root); + // The served generation is DERIVED from the faithful capsule's content (rule 5, #2246); the + // seed only distinguishes this fixture's generation. + let (module, root) = chain_anchored_module(store_a, [0x5bu8; 32]); + let root = root.0; + let root_hex = hex::encode(root); assert!( (module.len() as u64) < 100_000, "the warmed capsule must fit UNDER the cap so ONLY the tier-0 module is the eviction victim" @@ -6027,7 +6033,7 @@ mod tests { let window = 4096; // A REAL chain-anchored capsule (so the verify-before-land gate admits it) padded past three // windows by a large filler section, so a single-response implementation still cannot pass. - let capsule = chain_anchored_module_with_filler(store.0, root.0, window * 2 + 101); + let (capsule, root) = chain_anchored_module_with_filler(store.0, root.0, window * 2 + 101); assert!( capsule.len() > window * 2, "the fixture must span >2 windows" @@ -6111,7 +6117,7 @@ mod tests { let seed = [7u8; 32]; let store = Bytes32([3u8; 32]); let root = Bytes32([9u8; 32]); - let module = chain_anchored_module(store.0, root.0); + let (module, root) = chain_anchored_module(store.0, root.0); let captured = Arc::new(std::sync::Mutex::new(None)); let url = spawn_mock_module_server(captured.clone(), root, module).await; @@ -6164,7 +6170,7 @@ mod tests { let store = Bytes32([2u8; 32]); let served = Bytes32([0xAA; 32]); // the chain-anchored generation the upstream serves let requested = Bytes32([0xBB; 32]); // differs from served - let module = chain_anchored_module(store.0, served.0); + let (module, served) = chain_anchored_module(store.0, served.0); let captured = Arc::new(std::sync::Mutex::new(None)); let url = spawn_mock_module_server(captured, served, module.clone()).await; @@ -6187,31 +6193,62 @@ mod tests { // -- Chain-anchored verify before announce (#1623) ------------------------------------------ - /// A minimal chain-anchored `.dig` data section committing `(store, root)` — the shape the reshare - /// leg's [`ChainAnchoredModuleVerifier`] admits, so a test upstream can serve a capsule that is - /// genuinely the store's generation. - fn chain_anchored_module(store: [u8; 32], root: [u8; 32]) -> Vec { - use digstore_core::datasection::{encode_blob, SectionId}; - encode_blob(&[ - (SectionId::StoreId as u16, store.to_vec()), - (SectionId::CurrentRoot as u16, root.to_vec()), - ]) + /// A FAITHFUL chain-anchored `.dig` data section — the shape the reshare leg's + /// [`ChainAnchoredModuleVerifier`] admits under the hardened admit gate (rule 5, #2246): its + /// `ChunkPool`/`KeyTable`/`MerkleNodes` reproduce the committed root. The root is DERIVED from the + /// content (a preimage of an arbitrary root cannot be chosen), so `seed` merely distinguishes one + /// fixture's generation from another; the returned [`Bytes32`] is the generation the module commits. + /// Returns `(module, root)` — callers use the returned root as this capsule's generation. + fn chain_anchored_module(store: [u8; 32], seed: [u8; 32]) -> (Vec, Bytes32) { + chain_anchored_module_with_filler(store, seed, 0) } - /// A chain-anchored module (as above) padded past `min_len` bytes with a filler section, for the - /// multi-window capsule fixture — the padding lives in an extra section the verifier ignores, so the - /// module still commits exactly `(store, root)`. + /// Like [`chain_anchored_module`] but padded past `min_len` bytes with a filler section (an extra + /// section the verifier ignores) for the multi-window capsule fixture. Returns `(module, root)`. fn chain_anchored_module_with_filler( store: [u8; 32], - root: [u8; 32], + seed: [u8; 32], min_len: usize, - ) -> Vec { - use digstore_core::datasection::{encode_blob, SectionId}; - encode_blob(&[ + ) -> (Vec, Bytes32) { + use digstore_core::datasection::{ + encode_blob, encode_chunk_pool, encode_key_table, encode_merkle_nodes, SectionId, + }; + use digstore_core::merkle::{resource_leaf, MerkleTree}; + use digstore_core::serving::concat_output; + use digstore_core::KeyTableEntry; + + // One resource: a `seed`-distinguished static_key and a `seed`-derived content chunk. Its leaf + // is the producer's `resource_leaf(concat_output(cts))`, and the one-leaf tree's root IS the + // committed root — so the served content folds to exactly the root the header commits. + let chunk = { + let mut c = b"chain-anchored capsule content:".to_vec(); + c.extend_from_slice(&seed); + c + }; + let leaf = resource_leaf(&concat_output(&[chunk.as_slice()])); + let leaves = vec![leaf]; + let root = MerkleTree::from_leaves(leaves.clone()).root(); + let entries = vec![KeyTableEntry { + static_key: Bytes32(seed), + generation: root, + chunk_indices: vec![0], + total_size: chunk.len() as u64, + }]; + + let mut sections = vec![ (SectionId::StoreId as u16, store.to_vec()), - (SectionId::CurrentRoot as u16, root.to_vec()), - (SectionId::Filler as u16, vec![0x5a; min_len]), - ]) + (SectionId::CurrentRoot as u16, root.0.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&entries)), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[chunk.as_slice()]), + ), + (SectionId::MerkleNodes as u16, encode_merkle_nodes(&leaves)), + ]; + if min_len > 0 { + sections.push((SectionId::Filler as u16, vec![0x5a; min_len])); + } + (encode_blob(§ions), root) } /// Install an inventory-refresher spy that records whether the node announced itself a holder, so a @@ -6236,7 +6273,7 @@ mod tests { async fn a_capsule_whose_root_is_not_chain_confirmed_is_never_announced() { let store = Bytes32([1u8; 32]); // the id spawn_authed_remote seeds let served_root = Bytes32([0x10; 32]); // the genesis root it serves at - let module = chain_anchored_module(store.0, served_root.0); + let (module, served_root) = chain_anchored_module(store.0, served_root.0); let (base, store_hex) = spawn_authed_remote(module).await; // The chain has NO confirmed generation for this store, so the served root is unverifiable. @@ -6271,7 +6308,7 @@ mod tests { async fn a_chain_confirmed_capsule_lands_and_is_announced() { let store = Bytes32([1u8; 32]); let root = Bytes32([0x10; 32]); - let module = chain_anchored_module(store.0, root.0); + let (module, root) = chain_anchored_module(store.0, root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let (mut node, _td) = @@ -7243,7 +7280,7 @@ mod tests { // (its own backend, isolated per test on an ephemeral port) — matched here exactly as // `gap_fill_pulls_a_missing_generation_from_a_remote` does. let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let store_id: [u8; 32] = Bytes32::from_hex(&store_hex).unwrap().0; let td = tempfile::tempdir().unwrap(); @@ -7544,7 +7581,7 @@ mod tests { // cache.fetchAndCache pulls a whole store over the §21 authed sync path and // lands it in the cache, reporting the served root + size. let root = Bytes32([0x10; 32]); // the served genesis root - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let (mut node, _td) = test_node_with_resolver(Some([5u8; 32]), MockResolver::one(&store_hex, root)); @@ -7602,7 +7639,7 @@ mod tests { // NOT re-announce (unchanged inventory). use std::sync::atomic::{AtomicUsize, Ordering}; let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let (mut node, _td) = test_node_with_resolver(Some([7u8; 32]), MockResolver::one(&store_hex, root)); @@ -7646,7 +7683,7 @@ mod tests { // removed, so a gap-fill announces EXACTLY once — not twice. use std::sync::atomic::{AtomicUsize, Ordering}; let root = Bytes32([0x10; 32]); - let module = chain_anchored_module([1u8; 32], root.0); + let (module, root) = chain_anchored_module([1u8; 32], root.0); let (base, store_hex) = spawn_authed_remote(module.clone()).await; let (mut node, _td) = test_node_with_resolver(Some([8u8; 32]), MockResolver::one(&store_hex, root)); diff --git a/crates/dig-node-core/src/seams/capsule/push_capsule.rs b/crates/dig-node-core/src/seams/capsule/push_capsule.rs index c787e06..fa602ae 100644 --- a/crates/dig-node-core/src/seams/capsule/push_capsule.rs +++ b/crates/dig-node-core/src/seams/capsule/push_capsule.rs @@ -506,19 +506,82 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; - use digstore_core::datasection::encode_blob; + use digstore_core::datasection::{ + encode_blob, encode_chunk_pool, encode_key_table, encode_merkle_nodes, + }; + use digstore_core::merkle::{resource_leaf, MerkleTree}; + use digstore_core::serving::concat_output; + use digstore_core::KeyTableEntry; use crate::seams::dig_peer::peer_network::PeerNetwork; - /// A minimal but genuine `.dig` data-section blob committing `(store_id, root, publisher_pk)` plus - /// filler to reach `min_len` bytes — enough for the anchor verifier AND the authority check, and - /// (with a large `min_len`) a multi-window fixture. - fn push_module(store: [u8; 32], root: [u8; 32], pk: &Bytes48, min_len: usize) -> Vec { + /// A FAITHFUL `.dig` data-section blob committing `(store_id, root, publisher_pk)` whose `ChunkPool` + /// content actually recomputes to the committed `root`, plus filler to reach `min_len` bytes — + /// enough for the anchor verifier AND the authority check, and (with a large `min_len`) a + /// multi-window fixture. Returns `(blob, root)`: the root is DERIVED from the content (a preimage + /// of an arbitrary root cannot be chosen), so callers use the returned root as the capsule's root. + /// + /// Mirrors the producer recipe (`module_anchor.rs::honest_capsule_blob`): one resource with a + /// `seed`-derived `static_key` and a single content chunk; its leaf is + /// `resource_leaf(concat_output(cts))`, the one-leaf tree's root IS the committed root, and the + /// `KeyTable`/`ChunkPool`/`MerkleNodes` are mutually consistent — the shape a genuine capsule has, + /// so the admit gate's rule-5 recompute admits it. A capsule whose ChunkPool does NOT recompute is + /// refused; `push_module_bad_merkle` builds that shape. + fn push_module(store: [u8; 32], pk: &Bytes48, seed: u8, min_len: usize) -> (Vec, [u8; 32]) { + let chunk = format!("faithful capsule content for seed {seed:#04x}").into_bytes(); + let leaf = resource_leaf(&concat_output(&[chunk.as_slice()])); + let leaves = vec![leaf]; + let root = MerkleTree::from_leaves(leaves.clone()).root().0; + let entries = vec![KeyTableEntry { + static_key: Bytes32([seed; 32]), + generation: Bytes32(root), + chunk_indices: vec![0], + total_size: chunk.len() as u64, + }]; + + let mut sections = vec![ + (SectionId::StoreId as u16, store.to_vec()), + (SectionId::CurrentRoot as u16, root.to_vec()), + (SectionId::PublicKey as u16, pk.0.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&entries)), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[chunk.as_slice()]), + ), + (SectionId::MerkleNodes as u16, encode_merkle_nodes(&leaves)), + ]; + if min_len > 0 { + sections.push((SectionId::Filler as u16, vec![0x5a; min_len])); + } + (encode_blob(§ions), root) + } + + /// Like [`push_module`] but the `ChunkPool` content recomputes to a DIFFERENT root than the + /// committed `CurrentRoot` — a header-matching-but-tampered capsule (#2246/#2240). The committed + /// header commits the caller's `root` (so a byte-compare on `CurrentRoot` passes), while the served + /// `ChunkPool`/`KeyTable` fold to the content's own root, which cannot equal an arbitrary `root`. + /// The integrity gate's rule-5 recompute must refuse it before it lands. + fn push_module_bad_merkle(store: [u8; 32], root: [u8; 32], pk: &Bytes48) -> Vec { + let chunk = b"tampered capsule content".to_vec(); + let leaf = resource_leaf(&concat_output(&[chunk.as_slice()])); + let leaves = vec![leaf]; + let entries = vec![KeyTableEntry { + static_key: Bytes32([0xab; 32]), + generation: Bytes32(root), + chunk_indices: vec![0], + total_size: chunk.len() as u64, + }]; encode_blob(&[ (SectionId::StoreId as u16, store.to_vec()), + // Header commits the caller's `root`, but the ChunkPool below folds to the content's root. (SectionId::CurrentRoot as u16, root.to_vec()), (SectionId::PublicKey as u16, pk.0.to_vec()), - (SectionId::Filler as u16, vec![0x5a; min_len]), + (SectionId::KeyTable as u16, encode_key_table(&entries)), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[chunk.as_slice()]), + ), + (SectionId::MerkleNodes as u16, encode_merkle_nodes(&leaves)), ]) } @@ -614,9 +677,8 @@ mod tests { #[tokio::test] async fn a_local_push_lands_and_announces_exactly_once() { let (_sk, pk, store) = store_keypair(0x11); - let root = [0x22; 32]; + let (module, root) = push_module(store, &pk, 0x22, 0); let (store_hex, root_hex) = (hex::encode(store), hex::encode(root)); - let module = push_module(store, root, &pk, 0); let (node, _td) = crate::test_support::test_node_for_peer_surface(); let node = node.as_ref(); @@ -649,9 +711,8 @@ mod tests { #[tokio::test] async fn a_repushed_capsule_does_not_announce_twice() { let (_sk, pk, store) = store_keypair(0x33); - let root = [0x44; 32]; + let (module, root) = push_module(store, &pk, 0x44, 0); let (store_hex, root_hex) = (hex::encode(store), hex::encode(root)); - let module = push_module(store, root, &pk, 0); let (node, _td) = crate::test_support::test_node_for_peer_surface(); let node = node.as_ref(); @@ -679,10 +740,9 @@ mod tests { #[tokio::test] async fn chunked_reassembly_lands_and_root_mismatch_is_rejected() { let (_sk, pk, store) = store_keypair(0x55); - let root = [0x66; 32]; - let (store_hex, root_hex) = (hex::encode(store), hex::encode(root)); // A ~200 KiB module so a small window forces multiple chunks. - let module = push_module(store, root, &pk, 200 * 1024); + let (module, root) = push_module(store, &pk, 0x66, 200 * 1024); + let (store_hex, root_hex) = (hex::encode(store), hex::encode(root)); let total = module.len(); let win = 64 * 1024; @@ -734,7 +794,9 @@ mod tests { // Root-mismatch: bytes commit a DIFFERENT root than requested → rejected, nothing lands. let wrong_root = [0x67; 32]; let wrong_root_hex = hex::encode(wrong_root); - let mismatched = push_module(store, [0x99; 32], &pk, 0); // commits neither the requested root + // A faithful capsule committing its OWN content-derived root, pushed under `wrong_root_hex` — + // its committed root is neither the requested root nor the chain root, so it is rejected. + let (mismatched, _mroot) = push_module(store, &pk, 0x99, 0); let resp = push_one_shot( node, &store_hex, @@ -755,6 +817,42 @@ mod tests { )); } + /// (#2246/#2240) A push whose bytes commit the requested `(store_id, root)` in their HEADER but + /// whose `MerkleNodes` leaves recompute to a different root is refused BEFORE landing. The integrity + /// gate reuses `ChainAnchoredModuleVerifier`, so the admit-gate recompute (rule 5) guards the push + /// land too: a header-matching-but-tampered capsule never reaches disk or the announce. + /// **Catches:** an integrity gate that trusts the committed `CurrentRoot` header without recomputing + /// the merkle root from the capsule's own data. + #[tokio::test] + async fn a_push_whose_data_does_not_recompute_to_its_root_is_refused_before_landing() { + let (_sk, pk, store) = store_keypair(0x5b); + let root = [0x6c; 32]; + let (store_hex, root_hex) = (hex::encode(store), hex::encode(root)); + // Commits the requested root in its header (clears the byte-compare) but its leaves recompute to + // a different root — the tampered/incomplete capsule. + let tampered = push_module_bad_merkle(store, root, &pk); + + let (node, _td) = crate::test_support::test_node_for_peer_surface(); + let node = node.as_ref(); + let announces = Arc::new(AtomicUsize::new(0)); + install_announce_counter(node, announces.clone()); + + let resp = push_one_shot(node, &store_hex, &root_hex, &tampered, None, local()).await; + assert!( + resp.get("error").is_some(), + "a push whose data does not recompute to its committed root must be refused: {resp}" + ); + assert!( + !crate::module_exists(&node.cache_dir, &store_hex, &root_hex), + "the tampered capsule must never reach disk" + ); + assert_eq!( + announces.load(Ordering::SeqCst), + 0, + "a refused push must not announce" + ); + } + /// (c) With `DIG_NODE_PUSH_OPEN` unset, `cache.pushCapsule` is not peer-reachable — the peer /// allowlist answers `-32601` before dispatch. #[test] @@ -790,9 +888,8 @@ mod tests { #[tokio::test] async fn open_push_requires_the_authorized_writer_signature() { let (sk, pk, store) = store_keypair(0x77); - let root = [0x88; 32]; + let (module, root) = push_module(store, &pk, 0x88, 0); let (store_hex, root_hex) = (hex::encode(store), hex::encode(root)); - let module = push_module(store, root, &pk, 0); // A DIFFERENT keypair — an attacker who owns a key, but not THIS store's key. let (attacker_sk, _apk, _astore) = store_keypair(0xEE); diff --git a/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs b/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs index 1f5c437..aa18577 100644 --- a/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs +++ b/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs @@ -34,11 +34,32 @@ //! own `StoreId` and `CurrentRoot` in its data section. A module whose committed ids do not match //! the generation being pulled is a real capsule for the WRONG generation — a rollback primitive if //! admitted (serve yesterday's content under today's root), so it fails closed too. +//! 5. **The committed root must be the MERKLE ROOT of the SERVED CONTENT.** Rule 4 only proves the +//! module *names* the chain-anchored root in its `CurrentRoot` section — a header a fabricator +//! writes freely. It says nothing about whether the bytes hash to that root. So the verifier +//! RECOMPUTES the merkle root **from the capsule's own `ChunkPool` ciphertexts** — the same +//! content→leaf recipe the producer (`digstore-store`) commits and the browser verifier +//! (`dig-client-wasm`) checks: for each `KeyTable` entry `leaf = resource_leaf(concat_output(its +//! chunk ciphertexts))`, the leaves sorted ASCENDING by `static_key`, folded via +//! [`MerkleTree::from_leaves`] — and refuses unless it reproduces the committed root. +//! +//! The `MerkleNodes` section is **never trusted for the admit decision.** An earlier fix +//! recomputed from `MerkleNodes` leaves instead, which was HOLLOW: rule 4 pins `committed_root == +//! chain_root`, a single-leaf tree has `from_leaves(vec![x]).root() == x` (no fold, no tag), and +//! `decode_merkle_leaves` accepts arbitrary bytes — so `MerkleNodes = [chain_root]` plus an empty +//! or garbage `ChunkPool` passed for free, admitting a contentless phantom-holder capsule. The +//! trust anchor is therefore fold-of-CONTENT == chain_root. `MerkleNodes` is retained only as a +//! defense-in-depth cross-check: because the served inclusion proofs are generated from it, its +//! leaves must equal the content leaves in producer order, or the served proofs would disagree +//! with the bytes. An absent `KeyTable`/`ChunkPool`, a chunk index the pool cannot satisfy, an +//! undecodable section, or any mismatch fails closed. A legitimately EMPTY store (no entries) +//! folds to `from_leaves(vec![]).root() == sha256(&[])` and is admitted, not errored. use std::sync::{Arc, Mutex}; -use digstore_core::datasection::{DataView, SectionId}; -use digstore_core::Bytes32; +use digstore_core::datasection::{decode_merkle_leaves, read_chunk, DataView, SectionId}; +use digstore_core::merkle::MerkleTree; +use digstore_core::{Bytes32, Decode, Decoder, KeyTableEntry}; use sha2::{Digest, Sha256}; use dig_download::{ModuleAnchor, ModuleAnchorVerifier, ModuleReader}; @@ -210,10 +231,139 @@ impl ChainAnchoredModuleVerifier { "the module's committed root is not the store's chain-anchored root", )); } + + // Rule 5: the committed root must be the merkle root of the SERVED CONTENT, not merely a + // header that happens to name the chain root. Recompute from the capsule's own `ChunkPool` + // ciphertexts (NEVER from the attacker-supplied `MerkleNodes` digests — see the module docs + // for why trusting those was hollow) and refuse unless the content reproduces `committed_root` + // (which rule 4 has already pinned == the chain root). + let content_leaves = match content_leaves(&view, committed_root) { + Ok(leaves) => leaves, + Err(reason) => return Some(Rejection::NotAnchored(reason)), + }; + let content_root = MerkleTree::from_leaves(content_leaves.clone()).root().0; + if content_root != committed_root { + return Some(Rejection::NotAnchored( + "the module's ChunkPool content does not recompute to its committed root", + )); + } + + // Defense-in-depth: the served inclusion proofs are generated from `MerkleNodes`, so its + // leaves MUST equal the content leaves in producer (static-key-ascending) order — else a + // capsule whose content folds correctly could still ship proofs that disagree with the bytes + // it serves. This is a consistency cross-check, NOT the trust anchor (that is the content + // fold above); absence or an undecodable/disagreeing section fails closed. + let Some(merkle_body) = view.section(SectionId::MerkleNodes) else { + return Some(Rejection::NotAnchored( + "the module commits no MerkleNodes leaves alongside its content", + )); + }; + let Ok(declared_leaves) = decode_merkle_leaves(merkle_body) else { + return Some(Rejection::NotAnchored( + "the module's MerkleNodes section does not decode into merkle leaves", + )); + }; + if declared_leaves != content_leaves { + return Some(Rejection::NotAnchored( + "the module's MerkleNodes leaves disagree with its served ChunkPool content", + )); + } None } } +/// The per-resource merkle leaves recomputed from a module's SERVED CONTENT for the CURRENT +/// GENERATION, sorted into producer order — the trustworthy input to rule 5's root fold. +/// +/// **Scoped to the current generation (`committed_root`).** The embedded `KeyTable` is +/// MULTI-generation: the producer pushes one entry per (generation, resource), each stamped with its +/// generation's root (`digstore-compiler` `key_table.rs`). But the committed root the producer folds +/// (`pipeline.rs` → `current_generation_leaves(generations.last())`) is over the CURRENT generation +/// ONLY — and the last generation's root IS `committed_root` (rule 4 pins it == the chain root, and +/// §9.4's `state.root == tree.root()` invariant makes the last generation's `gen.root()` equal that). +/// So this folds ONLY the entries whose `generation == committed_root`; folding every generation's +/// entries would over-count for any store published then updated even once (the normal lifecycle), +/// false-rejecting its genuine current content as `NotAnchored`. +/// +/// For each such `KeyTable` (id 8) entry, this gathers the resource's chunk ciphertexts from the +/// `ChunkPool` (id 9) by their global indices and hashes their concatenation with the ONE shared +/// content→leaf recipe (`resource_leaf(concat_output(cts))`) the producer commits and the browser +/// verifier checks — never trusting the attacker-supplied `MerkleNodes` digests. The pairs are then +/// sorted ASCENDING by `static_key`, because the producer folds `resource_leaves.sort_by_key(|r| r.0)` +/// and KeyTable storage order is not guaranteed sorted, so an independent sort is what makes the root +/// reproducible. +/// +/// Fail-closed on every gap: an absent `KeyTable`/`ChunkPool`, an undecodable entry, or a chunk index +/// the pool cannot satisfy is an `Err`, never a silent empty result. A genuinely empty store (no +/// current-generation entries) returns an empty leaf list, which folds to `sha256(&[])` — the +/// legitimate empty edge. +/// +/// Processed resource-by-resource (each resource's leaf computed then its ciphertext borrows dropped) +/// so no second whole-module copy is held in RAM. +fn content_leaves( + view: &DataView<'_>, + committed_root: [u8; 32], +) -> Result, &'static str> { + let Some(key_table_body) = view.section(SectionId::KeyTable) else { + return Err("the module commits no KeyTable to recompute its content root from"); + }; + let Some(chunk_pool_body) = view.section(SectionId::ChunkPool) else { + return Err("the module commits no ChunkPool to recompute its content root from"); + }; + + let mut decoder = Decoder::new(key_table_body); + let Ok(entry_count) = u32::decode(&mut decoder) else { + return Err("the module's KeyTable count does not decode"); + }; + + // Do NOT pre-size from `entry_count`: it is attacker-supplied (up to `u32::MAX`), so + // `with_capacity(entry_count)` would OOM long before the decode of a short body failed. Grow + // on demand instead — a lying count simply runs out of body and fails closed on the next decode. + let mut leaves: Vec<(Bytes32, Bytes32)> = Vec::new(); + + // Amplification bound (HIGH remote pre-auth OOM/CPU-DoS). `chunk_indices` is attacker-controlled + // and — because the producer dedups chunks — legitimately permits REPEATED/non-increasing indices + // (two identical chunks, or a chunk shared with an earlier resource). So repeats cannot be banned; + // instead the TOTAL referenced ciphertext bytes across the whole module is capped at + // `MAX_STORE_BYTES` (the ceiling a genuine store cannot exceed). Without this a ~1 MB module with + // `chunk_indices = [0; K]` over one large chunk would reference K×|chunk| bytes (e.g. 100 GB), + // aborting the allocator and crashing the node. The recompute streams each ciphertext into the + // leaf hash (below) so memory stays O(1); this cap additionally bounds the CPU of the hashing. + let mut total_referenced_bytes: u64 = 0; + + for _ in 0..entry_count { + let Ok(entry) = KeyTableEntry::decode(&mut decoder) else { + return Err("the module's KeyTable does not decode into entries"); + }; + // Skip prior-generation entries: only the CURRENT generation's resources fold into the + // committed root (see the fn docs). Every entry must still DECODE — the count/body must be + // well-formed — but a stale-generation entry contributes no leaf. + if entry.generation.0 != committed_root { + continue; + } + // Stream the resource's ciphertexts into its leaf hash rather than materializing their + // concatenation: `resource_leaf(concat_output(cts)) == sha256(ct0 ++ ct1 ++ …)` because + // `resource_leaf` is plain SHA-256 and `concat_output` is plain concatenation, so hashing + // incrementally is byte-identical AND holds no second copy of the content in RAM. + let mut hasher = Sha256::new(); + for &global_index in &entry.chunk_indices { + let Some(ciphertext) = read_chunk(chunk_pool_body, global_index) else { + return Err("a KeyTable entry references a chunk absent from the ChunkPool"); + }; + total_referenced_bytes = total_referenced_bytes.saturating_add(ciphertext.len() as u64); + if total_referenced_bytes > digstore_core::MAX_STORE_BYTES { + return Err("the module's KeyTable references more content than a store may hold"); + } + hasher.update(ciphertext); + } + let leaf = Bytes32(hasher.finalize().into()); + leaves.push((entry.static_key, leaf)); + } + + leaves.sort_by_key(|(static_key, _)| static_key.0); + Ok(leaves.into_iter().map(|(_, leaf)| leaf).collect()) +} + #[async_trait::async_trait] impl ModuleAnchorVerifier for ChainAnchoredModuleVerifier { /// dig-download 0.15 hands the staged module as a borrowed READER rather than a slice, so the @@ -310,7 +460,11 @@ fn section_id32(view: &DataView<'_>, id: SectionId) -> Option<[u8; 32]> { #[cfg(test)] mod tests { use super::*; - use digstore_core::datasection::encode_blob; + use digstore_core::datasection::{ + encode_blob, encode_chunk_pool, encode_key_table, encode_merkle_nodes, + }; + use digstore_core::merkle::resource_leaf; + use digstore_core::serving::concat_output; const STORE: [u8; 32] = [0xa1; 32]; const CHAIN_ROOT: [u8; 32] = [0xb2; 32]; @@ -321,14 +475,158 @@ mod tests { bytes.iter().map(|b| format!("{b:02x}")).collect() } - /// A `.dig`-shaped data-section blob committing `store` + `root`. + /// A `.dig`-shaped blob committing `store` + `root` with an EXPLICIT `MerkleNodes` set, but NO + /// `ChunkPool`/`KeyTable` — reaches rules 1–4 exactly, and is (correctly) rejected by rule 5 if it + /// ever gets that far. Used only by tests whose verdict is decided at rules 1–4. fn module_committing(store: [u8; 32], root: [u8; 32]) -> Vec { encode_blob(&[ (SectionId::StoreId as u16, store.to_vec()), (SectionId::CurrentRoot as u16, root.to_vec()), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(root)]), + ), ]) } + /// One resource in a synthetic capsule: its `static_key` and its ordered chunk ciphertexts. + struct Resource { + static_key: [u8; 32], + chunks: Vec>, + } + + /// Build a FAITHFUL `.dig`-shaped blob from `resources`, committing `store` + the root recomputed + /// FROM the content, with a mutually-consistent `ChunkPool`, `KeyTable`, and `MerkleNodes` — the + /// shape a genuine capsule has, so rule 5 admits it. Returns `(blob, root)`. + /// + /// Mirrors the producer recipe (`digstore-store` `store.rs`): chunks land in the pool in the order + /// resources are given (assigning global indices), each resource's leaf is + /// `resource_leaf(concat_output(its ciphertexts))`, and the leaves are sorted ASCENDING by + /// `static_key` before folding — while the `KeyTable` retains the caller's (possibly unsorted) + /// order, so a verifier that fails to sort independently computes the wrong root. + fn honest_capsule_blob(store: [u8; 32], resources: &[Resource]) -> (Vec, [u8; 32]) { + let mut pool: Vec> = Vec::new(); + let mut entries: Vec = Vec::new(); + let mut pairs: Vec<([u8; 32], Bytes32)> = Vec::new(); + + for resource in resources { + let mut chunk_indices = Vec::new(); + let mut total_size: u64 = 0; + for chunk in &resource.chunks { + chunk_indices.push(pool.len() as u32); + total_size += chunk.len() as u64; + pool.push(chunk.clone()); + } + let slices: Vec<&[u8]> = resource.chunks.iter().map(|c| c.as_slice()).collect(); + let leaf = resource_leaf(&concat_output(&slices)); + pairs.push((resource.static_key, leaf)); + entries.push(KeyTableEntry { + static_key: Bytes32(resource.static_key), + generation: Bytes32([0u8; 32]), + chunk_indices, + total_size, + }); + } + + pairs.sort_by_key(|(static_key, _)| *static_key); + let leaves: Vec = pairs.iter().map(|(_, leaf)| *leaf).collect(); + let root = MerkleTree::from_leaves(leaves.clone()).root().0; + for entry in &mut entries { + entry.generation = Bytes32(root); + } + + let pool_slices: Vec<&[u8]> = pool.iter().map(|c| c.as_slice()).collect(); + let blob = encode_blob(&[ + (SectionId::StoreId as u16, store.to_vec()), + (SectionId::CurrentRoot as u16, root.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&entries)), + (SectionId::ChunkPool as u16, encode_chunk_pool(&pool_slices)), + (SectionId::MerkleNodes as u16, encode_merkle_nodes(&leaves)), + ]); + (blob, root) + } + + /// One generation of a synthetic multi-generation capsule: the resources present in that + /// generation. A resource re-appearing (same `static_key`) across generations models an UPDATE. + struct Generation { + resources: Vec, + } + + /// Build a FAITHFUL multi-generation `.dig`-shaped blob, mirroring the producer + /// (`digstore-compiler`): the `KeyTable` carries one entry PER (generation, resource) — each + /// stamped with THAT generation's root — the `ChunkPool` holds every generation's chunks in + /// global-index order, but the committed `CurrentRoot` + `MerkleNodes` are over the CURRENT + /// (last) generation ONLY. This is the shape any published store that has been updated at least + /// once has, so it exercises the current-generation scoping of `content_leaves`. Returns + /// `(blob, current_root)`. + fn multi_generation_capsule_blob( + store: [u8; 32], + generations: &[Generation], + ) -> (Vec, [u8; 32]) { + // Per-generation leaves (ascending by static_key) → that generation's root, exactly as the + // producer folds each generation. + let gen_root = |resources: &[Resource]| -> [u8; 32] { + let mut pairs: Vec<([u8; 32], Bytes32)> = resources + .iter() + .map(|r| { + let slices: Vec<&[u8]> = r.chunks.iter().map(|c| c.as_slice()).collect(); + (r.static_key, resource_leaf(&concat_output(&slices))) + }) + .collect(); + pairs.sort_by_key(|(static_key, _)| *static_key); + let leaves: Vec = pairs.into_iter().map(|(_, leaf)| leaf).collect(); + MerkleTree::from_leaves(leaves).root().0 + }; + + let mut pool: Vec> = Vec::new(); + let mut entries: Vec = Vec::new(); + for generation in generations { + let root = gen_root(&generation.resources); + for resource in &generation.resources { + let mut chunk_indices = Vec::new(); + let mut total_size: u64 = 0; + for chunk in &resource.chunks { + chunk_indices.push(pool.len() as u32); + total_size += chunk.len() as u64; + pool.push(chunk.clone()); + } + entries.push(KeyTableEntry { + static_key: Bytes32(resource.static_key), + generation: Bytes32(root), + chunk_indices, + total_size, + }); + } + } + + // The CURRENT generation drives the committed root + MerkleNodes. + let current = &generations.last().unwrap().resources; + let mut current_pairs: Vec<([u8; 32], Bytes32)> = current + .iter() + .map(|r| { + let slices: Vec<&[u8]> = r.chunks.iter().map(|c| c.as_slice()).collect(); + (r.static_key, resource_leaf(&concat_output(&slices))) + }) + .collect(); + current_pairs.sort_by_key(|(static_key, _)| *static_key); + let current_leaves: Vec = + current_pairs.into_iter().map(|(_, leaf)| leaf).collect(); + let current_root = MerkleTree::from_leaves(current_leaves.clone()).root().0; + + let pool_slices: Vec<&[u8]> = pool.iter().map(|c| c.as_slice()).collect(); + let blob = encode_blob(&[ + (SectionId::StoreId as u16, store.to_vec()), + (SectionId::CurrentRoot as u16, current_root.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&entries)), + (SectionId::ChunkPool as u16, encode_chunk_pool(&pool_slices)), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(¤t_leaves), + ), + ]); + (blob, current_root) + } + fn verifier() -> ChainAnchoredModuleVerifier { ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(CHAIN_ROOT)) } @@ -405,12 +703,24 @@ mod tests { } } - /// **Proves:** the genuine module — the one whose committed root IS the chain's root — is admitted. + /// **Proves:** the genuine content-bearing module — the one whose ChunkPool recomputes to the + /// chain's root — is admitted. #[test] fn admits_the_module_committing_the_chain_anchored_root() { - let module = module_committing(STORE, CHAIN_ROOT); + let (module, root) = honest_capsule_blob( + STORE, + &[Resource { + static_key: [0x01; 32], + chunks: vec![b"the one and only resource".to_vec()], + }], + ); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(root)); assert_eq!( - verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)), + futures::executor::block_on(v.verify_module_anchor( + &SliceReader(module), + &hex32(STORE), + &hex32(root) + )), ModuleAnchor::Anchored ); } @@ -509,13 +819,20 @@ mod tests { /// silently decides the anchor gate. #[test] fn hex_case_does_not_change_the_verdict() { - let module = module_committing(STORE, CHAIN_ROOT); + let (module, root) = honest_capsule_blob( + STORE, + &[Resource { + static_key: [0x07; 32], + chunks: vec![b"case-insensitive-id resource".to_vec()], + }], + ); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(root)); assert_eq!( - verdict( - &module, + futures::executor::block_on(v.verify_module_anchor( + &SliceReader(module), &hex32(STORE).to_uppercase(), - &hex32(CHAIN_ROOT).to_uppercase() - ), + &hex32(root).to_uppercase() + )), ModuleAnchor::Anchored ); } @@ -556,6 +873,295 @@ mod tests { ); } + /// **Proves:** THE refuted attack — `MerkleNodes = [chain_root]` (a single-leaf tree whose root + /// IS that leaf, so it trivially equals the committed chain root) plus an EMPTY `ChunkPool` and + /// empty `KeyTable` is refused. The prior fix recomputed from `MerkleNodes` and ADMITTED this + /// contentless phantom-holder capsule; binding the decision to the served content refuses it, + /// because empty content folds to `sha256(&[])` ≠ `chain_root`. + /// **Catches:** any rule 5 that trusts `MerkleNodes` digests instead of the `ChunkPool` bytes. + #[test] + fn rejects_single_leaf_merklenodes_equal_chain_root_with_empty_chunk_pool() { + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, CHAIN_ROOT.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[])), + (SectionId::ChunkPool as u16, encode_chunk_pool(&[])), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(CHAIN_ROOT)]), + ), + ]); + assert_eq!( + verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)), + ModuleAnchor::NotAnchored + ); + } + + /// **Proves:** the same self-consistent `MerkleNodes = [chain_root]` lie, now with a GARBAGE + /// `ChunkPool` and a `KeyTable` entry pointing at it, is still refused — the content recomputes to + /// its own (garbage) leaf, not to the chain root. + /// **Catches:** a rule 5 that trusts `MerkleNodes` regardless of what the pool actually holds. + #[test] + fn rejects_single_leaf_merklenodes_equal_chain_root_with_garbage_chunk_pool() { + let garbage: &[u8] = b"not the resource these leaves claim"; + let entry = KeyTableEntry { + static_key: Bytes32([0x01; 32]), + generation: Bytes32(CHAIN_ROOT), + chunk_indices: vec![0], + total_size: garbage.len() as u64, + }; + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, CHAIN_ROOT.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[entry])), + (SectionId::ChunkPool as u16, encode_chunk_pool(&[garbage])), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(CHAIN_ROOT)]), + ), + ]); + assert_eq!( + verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)), + ModuleAnchor::NotAnchored + ); + } + + /// **Proves:** THE decisive #2246/#2240 kill — a GENUINE capsule whose `ChunkPool` has had one + /// ciphertext byte flipped, while its `MerkleNodes` (and committed root) remain the HONEST ones, + /// is refused. The old rule read `MerkleNodes` and would ADMIT it (the honest leaves still fold to + /// the committed root); the new rule recomputes from the tampered content and refuses. + /// **Catches:** exactly the "trust the digests, not the bytes" defect the refutation exposed. + #[test] + fn rejects_content_tampered_capsule_with_honest_merklenodes() { + // Build the honest capsule to learn its committed root + honest leaf, then re-encode with a + // tampered pool but the ORIGINAL MerkleNodes + CurrentRoot — the header-honest, content-forged + // shape. + let honest_chunk = b"the honest resource ciphertext bytes".to_vec(); + let honest_leaf = resource_leaf(&concat_output(&[honest_chunk.as_slice()])); + let root = MerkleTree::from_leaves(vec![honest_leaf]).root().0; + + let mut tampered_chunk = honest_chunk.clone(); + tampered_chunk[0] ^= 0xff; + let entry = KeyTableEntry { + static_key: Bytes32([0x02; 32]), + generation: Bytes32(root), + chunk_indices: vec![0], + total_size: tampered_chunk.len() as u64, + }; + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, root.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[entry])), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[tampered_chunk.as_slice()]), + ), + // The HONEST leaves — the fabricator leaves these untouched to fool a digest-trusting gate. + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[honest_leaf]), + ), + ]); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(root)); + assert_eq!( + futures::executor::block_on(v.verify_module_anchor( + &SliceReader(module), + &hex32(STORE), + &hex32(root) + )), + ModuleAnchor::NotAnchored + ); + } + + /// **Proves:** THE remote OOM/CPU-DoS is bounded — a small module whose KeyTable entry REPEATS one + /// chunk index a huge number of times (`chunk_indices = [0; N]`, an ~1 MB blob addressing gigabytes + /// of referenced content) is refused fail-closed, NOT allocated/hashed into a crash. `store_id` + + /// `chain_root` are public so rules 1–4 pass for free on the pre-auth reshare-warm path; the total + /// referenced-bytes cap (`MAX_STORE_BYTES`) is the only thing between the crafted amplifier and an + /// allocator abort. + /// **Catches:** a `content_leaves` that concatenates/hashes attacker-repeated indices unbounded. + #[test] + fn rejects_a_chunk_index_amplification_bomb() { + // ~256 KiB real chunk, referenced 1_000_000 times ⇒ ~256 GB of referenced content: a genuine + // store cannot exceed MAX_STORE_BYTES (128 MB), so the recompute must refuse rather than fold. + let big_chunk = vec![0xabu8; 256 * 1024]; + let repeats = 1_000_000u32; + let entry = KeyTableEntry { + static_key: Bytes32([0x01; 32]), + generation: Bytes32(CHAIN_ROOT), + chunk_indices: vec![0; repeats as usize], + total_size: big_chunk.len() as u64 * repeats as u64, + }; + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, CHAIN_ROOT.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[entry])), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[big_chunk.as_slice()]), + ), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(CHAIN_ROOT)]), + ), + ]); + assert_eq!( + verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)), + ModuleAnchor::NotAnchored, + "an index-repetition amplification bomb must fail closed, not OOM the node" + ); + } + + /// **Proves:** a module missing its `ChunkPool` is refused — the content the root must bind is + /// absent, so there is nothing to recompute from (fail-closed). + #[test] + fn rejects_missing_chunk_pool() { + let entry = KeyTableEntry { + static_key: Bytes32([0x01; 32]), + generation: Bytes32(CHAIN_ROOT), + chunk_indices: vec![0], + total_size: 4, + }; + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, CHAIN_ROOT.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[entry])), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(CHAIN_ROOT)]), + ), + ]); + assert_eq!( + verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)), + ModuleAnchor::NotAnchored + ); + } + + /// **Proves:** a module missing its `KeyTable` is refused — without it there is no map from + /// resources to their chunks, so the content root cannot be reconstructed (fail-closed). + #[test] + fn rejects_missing_key_table() { + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, CHAIN_ROOT.to_vec()), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[b"orphan chunk"]), + ), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(CHAIN_ROOT)]), + ), + ]); + assert_eq!( + verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)), + ModuleAnchor::NotAnchored + ); + } + + /// **Proves:** the genuine content-bearing capsule — multiple resources whose ChunkPool folds to + /// the committed chain root — is admitted, and that the recompute SORTS by `static_key`: the + /// resources are given in DESCENDING key order (so the KeyTable stores them unsorted) while the + /// producer root is over the ascending fold, so a verifier that did not sort would compute a + /// different two-leaf root and reject an honest capsule. + #[test] + fn admits_a_genuine_content_bearing_capsule() { + let (module, root) = honest_capsule_blob( + STORE, + &[ + Resource { + static_key: [0x09; 32], + chunks: vec![b"resource nine".to_vec(), b" second chunk".to_vec()], + }, + Resource { + static_key: [0x02; 32], + chunks: vec![b"resource two".to_vec()], + }, + ], + ); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(root)); + assert_eq!( + futures::executor::block_on(v.verify_module_anchor( + &SliceReader(module), + &hex32(STORE), + &hex32(root) + )), + ModuleAnchor::Anchored + ); + } + + /// **Proves:** THE #2246 current-generation-scoping fix — a genuine capsule for a store that has + /// been UPDATED (≥2 generations) is admitted. Its embedded `KeyTable` is multi-generation + /// (gen0 = {A_v1}; gen1/current = {A_v2, B}, each entry stamped with ITS generation's root), + /// while the committed root + `MerkleNodes` are over the CURRENT generation only. + /// **Catches:** a `content_leaves` that folds EVERY KeyTable entry (all 3) instead of only the + /// current generation's (A_v2, B) — it recomputes a root over too many leaves, so it + /// false-rejects the genuine current content of any published-then-updated store as + /// `NotAnchored`, breaking admit/cache/announce for the normal update lifecycle. + #[test] + fn admits_a_genuine_multi_generation_updated_capsule() { + let key_a = [0x0a; 32]; + let key_b = [0x0b; 32]; + let (module, current_root) = multi_generation_capsule_blob( + STORE, + &[ + Generation { + resources: vec![Resource { + static_key: key_a, + chunks: vec![b"resource A, version one".to_vec()], + }], + }, + Generation { + resources: vec![ + Resource { + static_key: key_a, + chunks: vec![b"resource A, version TWO".to_vec()], + }, + Resource { + static_key: key_b, + chunks: vec![b"resource B, added in gen one".to_vec()], + }, + ], + }, + ], + ); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(current_root)); + assert_eq!( + futures::executor::block_on(v.verify_module_anchor( + &SliceReader(module), + &hex32(STORE), + &hex32(current_root) + )), + ModuleAnchor::Anchored, + "a genuine capsule for an updated (multi-generation) store must be admitted" + ); + } + + /// **Proves:** the §5.1 legitimate-empty edge — a metadata-only store with an EMPTY `KeyTable` + + /// empty `ChunkPool`, whose chain root is `sha256(&[])` (the empty-tree fold), with empty + /// `MerkleNodes`, is ADMITTED, not errored. The recompute must reproduce `from_leaves(vec![])`, + /// not reject on "no entries". + #[test] + fn admits_a_genuine_empty_metadata_only_store() { + let empty_root = sha256(&[]); + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, empty_root.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[])), + (SectionId::ChunkPool as u16, encode_chunk_pool(&[])), + (SectionId::MerkleNodes as u16, encode_merkle_nodes(&[])), + ]); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(empty_root)); + assert_eq!( + futures::executor::block_on(v.verify_module_anchor( + &SliceReader(module), + &hex32(STORE), + &hex32(empty_root) + )), + ModuleAnchor::Anchored + ); + } + /// **Proves:** a non-canonical id (wrong length, non-hex) is refused rather than treated as a /// wildcard. #[test] @@ -581,14 +1187,20 @@ mod tests { /// caller's pre-announce re-check has a self-sourced reference (no peer supplies either side). #[test] fn the_admitted_digest_is_the_digest_of_the_admitted_bytes() { - let module = module_committing(STORE, CHAIN_ROOT); - let v = verifier(); + let (module, root) = honest_capsule_blob( + STORE, + &[Resource { + static_key: [0x05; 32], + chunks: vec![b"admitted-digest resource".to_vec()], + }], + ); + let v = ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(root)); assert_eq!(v.admitted_digest(), None, "nothing admitted yet"); assert_eq!( futures::executor::block_on(v.verify_module_anchor( &SliceReader(module.clone()), &hex32(STORE), - &hex32(CHAIN_ROOT) + &hex32(root) )), ModuleAnchor::Anchored ); diff --git a/crates/dig-node-core/src/seams/dig_peer/module_reshare.rs b/crates/dig-node-core/src/seams/dig_peer/module_reshare.rs index a1514da..39d8924 100644 --- a/crates/dig-node-core/src/seams/dig_peer/module_reshare.rs +++ b/crates/dig-node-core/src/seams/dig_peer/module_reshare.rs @@ -547,21 +547,58 @@ fn decode_id(hex: &str) -> Option<[u8; 32]> { #[cfg(test)] mod tests { use super::*; - use digstore_core::datasection::{encode_blob, SectionId}; + use digstore_core::datasection::{ + encode_blob, encode_chunk_pool, encode_key_table, encode_merkle_nodes, SectionId, + }; + use digstore_core::merkle::{resource_leaf, MerkleTree}; + use digstore_core::serving::concat_output; + use digstore_core::KeyTableEntry; use std::sync::atomic::{AtomicUsize, Ordering}; const STORE: [u8; 32] = [0xa1; 32]; - const CHAIN_ROOT: [u8; 32] = [0xb2; 32]; fn hex32(bytes: [u8; 32]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } - /// A `.dig`-shaped module committing `store` + `root`. + /// The single fixed resource every reshare fixture serves: one resource with a known `static_key` + /// and one content chunk. Its merkle root is DERIVED from the content (a preimage of an arbitrary + /// root cannot be chosen), so [`chain_root`] is whatever this content folds to. + fn faithful_resource() -> ([u8; 32], Vec) { + ([0x01; 32], b"reshare capsule content".to_vec()) + } + + /// The chain-anchored generation root the reshare fixtures commit — the merkle root of + /// [`faithful_resource`]'s content. Deterministic, so every fixture and every verifier/resolver in + /// this module agree on the same root. + fn chain_root() -> [u8; 32] { + let (_static_key, chunk) = faithful_resource(); + let leaf = resource_leaf(&concat_output(&[chunk.as_slice()])); + MerkleTree::from_leaves(vec![leaf]).root().0 + } + + /// A FAITHFUL `.dig`-shaped module committing `store` + `root` whose `ChunkPool`/`KeyTable`/ + /// `MerkleNodes` reproduce `root` under the hardened admit gate (rule 5, #2246). Callers pass + /// [`chain_root`] as `root`; the served content folds to exactly that. fn module_committing(store: [u8; 32], root: [u8; 32]) -> Vec { + let (static_key, chunk) = faithful_resource(); + let leaf = resource_leaf(&concat_output(&[chunk.as_slice()])); + let leaves = vec![leaf]; + let entries = vec![KeyTableEntry { + static_key: Bytes32(static_key), + generation: Bytes32(root), + chunk_indices: vec![0], + total_size: chunk.len() as u64, + }]; encode_blob(&[ (SectionId::StoreId as u16, store.to_vec()), (SectionId::CurrentRoot as u16, root.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&entries)), + ( + SectionId::ChunkPool as u16, + encode_chunk_pool(&[chunk.as_slice()]), + ), + (SectionId::MerkleNodes as u16, encode_merkle_nodes(&leaves)), ]) } @@ -617,18 +654,18 @@ mod tests { #[test] fn promotes_the_artifact_the_gate_admitted() { let dir = temp_dir("promote-ok"); - let module = module_committing(STORE, CHAIN_ROOT); + let module = module_committing(STORE, chain_root()); let staged = dir.join("staged.dig"); std::fs::write(&staged, &module).unwrap(); let verifier = - ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(CHAIN_ROOT)); + ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(chain_root())); assert_eq!( futures::executor::block_on(dig_download::ModuleAnchorVerifier::verify_module_anchor( &verifier, &SliceReader(module.clone()), &hex32(STORE), - &hex32(CHAIN_ROOT) + &hex32(chain_root()) )), dig_download::ModuleAnchor::Anchored ); @@ -648,16 +685,16 @@ mod tests { #[test] fn promoting_an_admitted_artifact_bumps_the_refetch_counter() { let dir = temp_dir("promote-refetch-counter"); - let module = module_committing(STORE, CHAIN_ROOT); + let module = module_committing(STORE, chain_root()); let staged = dir.join("staged.dig"); std::fs::write(&staged, &module).unwrap(); let verifier = - ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(CHAIN_ROOT)); + ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(chain_root())); futures::executor::block_on(dig_download::ModuleAnchorVerifier::verify_module_anchor( &verifier, &SliceReader(module.clone()), &hex32(STORE), - &hex32(CHAIN_ROOT), + &hex32(chain_root()), )); // `>=` rather than exact `==`: `CACHE_REFETCH_COUNT` is a PROCESS-GLOBAL atomic shared with @@ -684,15 +721,15 @@ mod tests { #[test] fn refuses_an_artifact_tampered_after_the_gate_admitted_it() { let dir = temp_dir("promote-tampered"); - let module = module_committing(STORE, CHAIN_ROOT); + let module = module_committing(STORE, chain_root()); let verifier = - ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(CHAIN_ROOT)); + ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(chain_root())); assert_eq!( futures::executor::block_on(dig_download::ModuleAnchorVerifier::verify_module_anchor( &verifier, &SliceReader(module.clone()), &hex32(STORE), - &hex32(CHAIN_ROOT) + &hex32(chain_root()) )), dig_download::ModuleAnchor::Anchored ); @@ -722,9 +759,9 @@ mod tests { fn refuses_to_promote_an_artifact_no_gate_admitted() { let dir = temp_dir("promote-ungated"); let staged = dir.join("staged.dig"); - std::fs::write(&staged, module_committing(STORE, CHAIN_ROOT)).unwrap(); + std::fs::write(&staged, module_committing(STORE, chain_root())).unwrap(); let verifier = - ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(CHAIN_ROOT)); + ChainAnchoredModuleVerifier::for_generation(Bytes32(STORE), Bytes32(chain_root())); let cached = dir.join("cached.module"); assert_eq!( @@ -781,7 +818,7 @@ mod tests { #[async_trait::async_trait] impl crate::shared::AnchoredRootResolver for ConfirmingResolver { async fn anchored_root(&self, _store_id: &[u8; 32]) -> Result, String> { - Ok(Some(Bytes32(CHAIN_ROOT))) + Ok(Some(Bytes32(chain_root()))) } } @@ -856,7 +893,7 @@ mod tests { let spy = Arc::new(AnnounceSpy::default()); let warmer = warmer_with(Arc::new(ConfirmingResolver), Arc::clone(&spy), &dir); - let outcome = warmer.warm(&hex32(STORE), &hex32(CHAIN_ROOT)).await; + let outcome = warmer.warm(&hex32(STORE), &hex32(chain_root())).await; assert_eq!(outcome, WarmOutcome::Refused(WarmFailure::PullFailed)); assert_eq!( @@ -882,8 +919,8 @@ mod tests { #[tokio::test] async fn a_successful_pull_is_held_cached_and_announced_once() { let dir = temp_dir("happy-path"); - let (store_hex, root_hex) = (hex32(STORE), hex32(CHAIN_ROOT)); - let module = module_committing(STORE, CHAIN_ROOT); + let (store_hex, root_hex) = (hex32(STORE), hex32(chain_root())); + let module = module_committing(STORE, chain_root()); // One real holder, served through dig-download's own mock transport — the SAME production // `ModuleDownloader`/`FileSink` path the refusal tests exercise, just with a source that @@ -989,8 +1026,8 @@ mod tests { #[tokio::test] async fn a_successful_reshare_warm_land_sweeps_the_modules_cache_once() { let dir = temp_dir("reshare-sweep-once"); - let (store_hex, root_hex) = (hex32(STORE), hex32(CHAIN_ROOT)); - let module = module_committing(STORE, CHAIN_ROOT); + let (store_hex, root_hex) = (hex32(STORE), hex32(chain_root())); + let module = module_committing(STORE, chain_root()); let content = dig_download::module_content_id(&store_hex, &root_hex) .expect("canonical ids yield a content id"); let locator = Arc::new(dig_download::testkit::MockProviderLocator::fixed( @@ -1053,7 +1090,7 @@ mod tests { Arc::clone(&evictor) as Arc, ); - let outcome = warmer.warm(&hex32(STORE), &hex32(CHAIN_ROOT)).await; + let outcome = warmer.warm(&hex32(STORE), &hex32(chain_root())).await; assert_eq!(outcome, WarmOutcome::Refused(WarmFailure::PullFailed)); assert_eq!( @@ -1074,7 +1111,7 @@ mod tests { let spy = Arc::new(AnnounceSpy::default()); let warmer = warmer_with(Arc::new(UnreachableChain), Arc::clone(&spy), &dir); - let outcome = warmer.warm(&hex32(STORE), &hex32(CHAIN_ROOT)).await; + let outcome = warmer.warm(&hex32(STORE), &hex32(chain_root())).await; assert_eq!(outcome, WarmOutcome::Refused(WarmFailure::NoChainAnchor)); assert_eq!(spy.calls.load(Ordering::SeqCst), 0); @@ -1104,7 +1141,7 @@ mod tests { let spy = Arc::new(AnnounceSpy::default()); let warmer = warmer_with(Arc::new(ConfirmingResolver), Arc::clone(&spy), &dir); assert_eq!( - warmer.warm("not-an-id", &hex32(CHAIN_ROOT)).await, + warmer.warm("not-an-id", &hex32(chain_root())).await, WarmOutcome::Refused(WarmFailure::NoChainAnchor) ); assert_eq!(spy.calls.load(Ordering::SeqCst), 0); From 40239f8f85917bd6c71bbb39588a86313e064259 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 20:54:12 +0000 Subject: [PATCH 2/2] perf(dig-node-core): pre-index ChunkPool to kill admit-gate quadratic scan The #2246 admit gate resolved each KeyTable chunk reference via read_chunk, which is O(global_index) (re-scans the ChunkPool from offset 0 per call), so N references over an M-chunk pool were Theta(N*M). Zero-length chunks add 0 bytes, slipping past the MAX_STORE_BYTES cap, so a pool of M zero-length chunks + one current-gen entry referencing index M-1 N times pinned a core for ~Theta(module^2) per unauthenticated reshare request. Pre-index the pool once into per-chunk byte ranges (O(1) lookup, byte-identical to read_chunk) so content_leaves is O(pool + refs); additionally cap cumulative references at MAX_STORE_BYTES/4 as defense-in-depth for the zero-length case. Co-Authored-By: Claude --- DEVELOPMENT_LOG.md | 13 ++ SPEC.md | 15 +- .../src/seams/dig_peer/module_anchor.rs | 181 +++++++++++++++++- 3 files changed, 203 insertions(+), 6 deletions(-) diff --git a/DEVELOPMENT_LOG.md b/DEVELOPMENT_LOG.md index dbb848f..33d4ee2 100644 --- a/DEVELOPMENT_LOG.md +++ b/DEVELOPMENT_LOG.md @@ -25,6 +25,19 @@ dig-updater's feedsign resolves dig-node by native-package file names and fails ENTIRE manifest, that froze — then expired — stable auto-update for all five components, dig-app included. A dig-node release without its `.msi`/`.pkg`/`.deb` is not a partial dig-node release, it is an ecosystem-wide auto-update outage. +## `read_chunk` is O(global_index) — per-reference lookup is a quadratic CPU-DoS (#2246) + +`digstore_core::datasection::read_chunk(pool_body, i)` re-walks the length-prefixed `ChunkPool` from +offset 0 on EVERY call (no offset table). The admit gate's `content_leaves` called it once per +`chunk_index`, so resolving N references over an M-chunk pool was Θ(N·M). The byte cap +(`total_referenced_bytes > MAX_STORE_BYTES`) keys on `ciphertext.len()`, so ZERO-LENGTH chunks add 0 and +never trip it — an attacker sends a pool of M zero-length chunks + one current-gen entry referencing +index `M-1` N times ⇒ ≈Θ(module²) iterations (a ~10 MB module ⇒ ~10^12) with the accumulator stuck at 0, +pinning a core per unauthenticated reshare request. Fix: PRE-INDEX the pool once into per-chunk byte +ranges (O(1) lookup ⇒ recompute is O(pool + refs)) AND cap cumulative references at `MAX_STORE_BYTES / 4` +(defense-in-depth for the zero-length case the byte cap can't see). Lesson: any per-item call into a +scan-from-start reader over attacker-sized input is silently quadratic; index once. + ## Admit gate must recompute from CONTENT, not from the attacker's MerkleNodes digests (#2246/#2240) `ChainAnchoredModuleVerifier` (the capsule-admit gate shared by the reshare-admit pull AND the diff --git a/SPEC.md b/SPEC.md index 1e666e6..b3027b7 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1188,13 +1188,22 @@ attacker-controlled and permits repeated indices (the producer dedups chunks), s ciphertext bytes across the module is capped at `MAX_STORE_BYTES` and each resource's leaf is hashed by STREAMING its ciphertexts into an incremental SHA-256 (O(1) memory) rather than materializing their concatenation — without which a ~1 MB module addressing one chunk N times could reference gigabytes and -abort the allocator (#2246). The attacker-supplied `MerkleNodes` digests are NEVER trusted for this +abort the allocator (#2246). The recompute resolves each reference through a `ChunkPool` PRE-INDEX built +in ONE linear pass (per-chunk byte ranges), so the whole recompute is O(pool + references), NOT the +Θ(references × pool) it would be if each reference re-walked the pool from offset 0 (the canonical +`read_chunk` is O(global_index)); an attacker could otherwise pin a CPU core for ≈Θ(module²) with a pool +of ZERO-LENGTH chunks + one entry referencing the highest index N times — and because zero-length chunks +add 0 bytes, the byte cap alone never fired. As additional defense-in-depth the CUMULATIVE reference +count across the current generation is capped at `MAX_STORE_BYTES / 4` (a genuine store cannot frame more +chunks than that 4-byte-minimum-framing ceiling permits), bounding scan+hash work even for zero-length +chunks (#2246). The attacker-supplied `MerkleNodes` digests are NEVER trusted for this decision (retained only as a defense-in-depth cross-check that the served inclusion proofs match the content): a single-leaf `from_leaves(vec![x]).root() == x` meant a `MerkleNodes = [chain_root]` plus an empty/garbage `ChunkPool` recomputed to the committed root for free, admitting a contentless phantom-holder capsule (#2246/#2240). An absent `KeyTable`/`ChunkPool`, a chunk index the pool cannot -satisfy, an undecodable section, a `MerkleNodes`↔content mismatch, or referenced content exceeding -`MAX_STORE_BYTES` fails closed; a legitimately EMPTY store folds to `from_leaves(vec![]).root() == sha256(&[])` and passes. A header naming the chain root is +satisfy, an undecodable section (including malformed `ChunkPool` framing), a `MerkleNodes`↔content +mismatch, referenced content exceeding `MAX_STORE_BYTES`, or references exceeding `MAX_STORE_BYTES / 4` +fails closed; a legitimately EMPTY store folds to `from_leaves(vec![]).root() == sha256(&[])` and passes. A header naming the chain root is not proof the bytes hash to it. This check gives INTEGRITY, never AUTHORITY — without the writer check an opened node would be an unauthenticated cache-poison + DHT-announce-amplification surface (the #179/#1576 class). A push that diff --git a/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs b/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs index aa18577..d2439d8 100644 --- a/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs +++ b/crates/dig-node-core/src/seams/dig_peer/module_anchor.rs @@ -57,7 +57,7 @@ use std::sync::{Arc, Mutex}; -use digstore_core::datasection::{decode_merkle_leaves, read_chunk, DataView, SectionId}; +use digstore_core::datasection::{decode_merkle_leaves, DataView, SectionId}; use digstore_core::merkle::MerkleTree; use digstore_core::{Bytes32, Decode, Decoder, KeyTableEntry}; use sha2::{Digest, Sha256}; @@ -300,6 +300,16 @@ impl ChainAnchoredModuleVerifier { /// /// Processed resource-by-resource (each resource's leaf computed then its ciphertext borrows dropped) /// so no second whole-module copy is held in RAM. +/// +/// **Chunk lookup is O(1), so the whole recompute is O(pool_size + total_references).** The +/// `ChunkPool` framing is parsed ONCE into per-chunk byte ranges ([`index_chunk_pool`]) and each +/// `KeyTable` reference resolves by index into that table. This closes a quadratic CPU-DoS: the +/// canonical `read_chunk` is O(global_index) — it re-walks the pool from offset 0 on every call — so +/// resolving N references against an M-chunk pool was Θ(N·M). An attacker could make that ≈Θ(module²) +/// with a pool of M ZERO-LENGTH chunks + one current-generation entry referencing index `M-1` N times; +/// worse, zero-length chunks add 0 bytes, so the `MAX_STORE_BYTES` accumulator never tripped. The +/// pre-index removes the quadratic root cause; the cumulative reference-count cap +/// ([`MAX_MODULE_CHUNK_REFERENCES`]) bounds scan+hash work even independent of it. fn content_leaves( view: &DataView<'_>, committed_root: [u8; 32], @@ -311,6 +321,13 @@ fn content_leaves( return Err("the module commits no ChunkPool to recompute its content root from"); }; + // Pre-index the ChunkPool ONCE: a single linear pass building each chunk's byte range, so every + // reference below is an O(1) index rather than a fresh O(global_index) `read_chunk` re-scan (the + // quadratic-DoS fix). Malformed framing fails closed here, exactly as `read_chunk` would per call. + let Some(chunk_ranges) = index_chunk_pool(chunk_pool_body) else { + return Err("the module's ChunkPool framing does not decode"); + }; + let mut decoder = Decoder::new(key_table_body); let Ok(entry_count) = u32::decode(&mut decoder) else { return Err("the module's KeyTable count does not decode"); @@ -331,6 +348,12 @@ fn content_leaves( // leaf hash (below) so memory stays O(1); this cap additionally bounds the CPU of the hashing. let mut total_referenced_bytes: u64 = 0; + // Defense-in-depth CPU bound: cap the cumulative number of chunk REFERENCES across the whole + // current generation, so scan+hash work stays bounded even for ZERO-LENGTH chunks (which add + // nothing to `total_referenced_bytes` and so slip past the byte cap — the quadratic scan-bomb's + // fuel). See [`MAX_MODULE_CHUNK_REFERENCES`] for the ceiling's derivation. + let mut total_referenced_chunks: u64 = 0; + for _ in 0..entry_count { let Ok(entry) = KeyTableEntry::decode(&mut decoder) else { return Err("the module's KeyTable does not decode into entries"); @@ -347,9 +370,17 @@ fn content_leaves( // incrementally is byte-identical AND holds no second copy of the content in RAM. let mut hasher = Sha256::new(); for &global_index in &entry.chunk_indices { - let Some(ciphertext) = read_chunk(chunk_pool_body, global_index) else { + total_referenced_chunks = total_referenced_chunks.saturating_add(1); + if total_referenced_chunks > MAX_MODULE_CHUNK_REFERENCES { + return Err("the module's KeyTable references more chunks than a store may hold"); + } + // O(1) lookup into the pre-indexed pool: byte-identical to `read_chunk(chunk_pool_body, + // global_index)`, without its per-call O(global_index) re-scan. Out of range fails closed + // exactly as `read_chunk` returning `None` does. + let Some(range) = chunk_ranges.get(global_index as usize) else { return Err("a KeyTable entry references a chunk absent from the ChunkPool"); }; + let ciphertext = &chunk_pool_body[range.clone()]; total_referenced_bytes = total_referenced_bytes.saturating_add(ciphertext.len() as u64); if total_referenced_bytes > digstore_core::MAX_STORE_BYTES { return Err("the module's KeyTable references more content than a store may hold"); @@ -364,6 +395,58 @@ fn content_leaves( Ok(leaves.into_iter().map(|(_, leaf)| leaf).collect()) } +/// Ceiling on the cumulative number of `KeyTable` chunk references a module's current generation may +/// make before it fails closed. +/// +/// Derived from `MAX_STORE_BYTES / MIN_CHUNK_FRAMING`: every chunk in the `ChunkPool` costs at least +/// its 4-byte length prefix (the encoding in `datasection::encode_chunk_pool`), so a store within the +/// `MAX_STORE_BYTES` budget cannot frame — and thus a genuine current generation cannot reference — +/// more than `MAX_STORE_BYTES / 4` chunks. The existing `MAX_STORE_BYTES` byte cap bounds work for +/// NON-empty chunks; this reference-count cap additionally bounds scan+hash work for ZERO-LENGTH +/// chunks, which contribute no bytes and so cannot trip the byte cap on their own. +const MAX_MODULE_CHUNK_REFERENCES: u64 = digstore_core::MAX_STORE_BYTES / 4; + +/// Parse a `ChunkPool` body's length-prefixed framing ONCE into each chunk's byte range within +/// `pool_body`, or `None` if the framing is malformed (a truncated count, a length that overruns the +/// body). +/// +/// This is the pre-index that makes [`content_leaves`] linear. It walks the exact same encoding +/// [`read_chunk`](digstore_core::datasection::read_chunk) parses — a `u32` BE count, then per chunk a +/// `u32` BE length prefix followed by that many bytes — but records every chunk's slice bounds in one +/// pass so a later reference is an O(1) `Vec` index, not another O(global_index) walk from offset 0. +/// `pool_body[range]` for the returned range is byte-identical to `read_chunk(pool_body, index)`. +/// +/// The returned `Vec` grows on demand: the claimed count is NOT pre-allocated (it is attacker-supplied +/// up to `u32::MAX`), and a count larger than the body can satisfy simply runs the body out and fails +/// closed on the next length read. +fn index_chunk_pool(pool_body: &[u8]) -> Option>> { + if pool_body.len() < 4 { + return None; + } + let count = u32::from_be_bytes([pool_body[0], pool_body[1], pool_body[2], pool_body[3]]); + let mut ranges: Vec> = Vec::new(); + let mut pos = 4usize; + for _ in 0..count { + if pos + 4 > pool_body.len() { + return None; + } + let len = u32::from_be_bytes([ + pool_body[pos], + pool_body[pos + 1], + pool_body[pos + 2], + pool_body[pos + 3], + ]) as usize; + pos += 4; + let end = pos.checked_add(len)?; + if end > pool_body.len() { + return None; + } + ranges.push(pos..end); + pos = end; + } + Some(ranges) +} + #[async_trait::async_trait] impl ModuleAnchorVerifier for ChainAnchoredModuleVerifier { /// dig-download 0.15 hands the staged module as a borrowed READER rather than a slice, so the @@ -461,7 +544,7 @@ fn section_id32(view: &DataView<'_>, id: SectionId) -> Option<[u8; 32]> { mod tests { use super::*; use digstore_core::datasection::{ - encode_blob, encode_chunk_pool, encode_key_table, encode_merkle_nodes, + encode_blob, encode_chunk_pool, encode_key_table, encode_merkle_nodes, read_chunk, }; use digstore_core::merkle::resource_leaf; use digstore_core::serving::concat_output; @@ -1012,6 +1095,98 @@ mod tests { ); } + /// **Proves:** the quadratic scan-DoS is CLOSED — a module with M ZERO-LENGTH `ChunkPool` chunks + /// and one current-generation entry referencing the HIGHEST index N times completes in + /// O(pool+refs), not Θ(N·M). The canonical `read_chunk` is O(global_index) (it re-walks the pool + /// from offset 0 per call), and zero-length chunks add 0 bytes so the `MAX_STORE_BYTES` cap never + /// trips — so before the pre-index this input pinned a CPU core for Θ(module²) iterations per + /// unauthenticated reshare request (M=N=100k ⇒ 10^10 scans, tens of seconds→minutes). With the + /// one-pass pre-index each reference is O(1), so the whole recompute is ~200k ops and returns + /// promptly, fail-closed (`NotAnchored`: empty content folds to `sha256(&[])` ≠ chain root). + /// **Catches:** any reintroduction of a per-reference `read_chunk` re-scan. + #[test] + fn bounds_a_zero_length_chunk_scan_bomb() { + use std::time::Instant; + // M zero-length chunks in the pool; N references, all at the highest index — the worst case + // for an O(global_index) per-call scan. + const M: u32 = 100_000; + const N: usize = 100_000; + + let zero_chunks: Vec<&[u8]> = vec![&[][..]; M as usize]; + let entry = KeyTableEntry { + static_key: Bytes32([0x01; 32]), + generation: Bytes32(CHAIN_ROOT), + chunk_indices: vec![M - 1; N], + total_size: 0, + }; + let module = encode_blob(&[ + (SectionId::StoreId as u16, STORE.to_vec()), + (SectionId::CurrentRoot as u16, CHAIN_ROOT.to_vec()), + (SectionId::KeyTable as u16, encode_key_table(&[entry])), + (SectionId::ChunkPool as u16, encode_chunk_pool(&zero_chunks)), + ( + SectionId::MerkleNodes as u16, + encode_merkle_nodes(&[Bytes32(CHAIN_ROOT)]), + ), + ]); + + let start = Instant::now(); + let verdict = verdict(&module, &hex32(STORE), &hex32(CHAIN_ROOT)); + let elapsed = start.elapsed(); + + assert_eq!( + verdict, + ModuleAnchor::NotAnchored, + "a zero-length scan bomb must fail closed, never be admitted" + ); + // A generous ceiling: the pre-indexed path is milliseconds; the pre-fix Θ(N·M) path would + // take tens of seconds to minutes and blow this bound. + assert!( + elapsed.as_secs() < 5, + "content_leaves must be O(pool+refs), not O(pool*refs): took {elapsed:?} for {M}×{N}" + ); + } + + /// **Proves:** the pre-indexed lookup is byte-identical to the canonical `read_chunk` for every + /// index of a normal (mixed-length, including zero-length) pool, and mirrors its out-of-range + /// `None`. This is what lets [`content_leaves`] swap `read_chunk` for the O(1) pre-index without + /// changing which bytes fold into a leaf. + #[test] + fn the_prebuilt_chunk_index_matches_read_chunk() { + let chunks: Vec> = vec![ + b"first chunk".to_vec(), + Vec::new(), // a zero-length chunk — the framing must still advance + b"a third, longer chunk of ciphertext".to_vec(), + b"x".to_vec(), + ]; + let slices: Vec<&[u8]> = chunks.iter().map(|c| c.as_slice()).collect(); + let pool = encode_chunk_pool(&slices); + + let ranges = index_chunk_pool(&pool).expect("a well-formed pool indexes"); + assert_eq!(ranges.len(), chunks.len()); + for i in 0..chunks.len() as u32 { + let via_read = read_chunk(&pool, i).expect("read_chunk in range"); + let via_index = &pool[ranges[i as usize].clone()]; + assert_eq!( + via_index, via_read, + "pre-indexed range {i} must equal read_chunk's slice" + ); + } + // Out of range mirrors read_chunk's None. + assert!(read_chunk(&pool, chunks.len() as u32).is_none()); + assert!(ranges.get(chunks.len()).is_none()); + } + + /// **Proves:** malformed `ChunkPool` framing fails closed — a body whose declared count exceeds + /// the bytes present cannot be indexed, so the recompute refuses rather than reading past the end. + #[test] + fn a_malformed_chunk_pool_fails_the_preindex_closed() { + // Claims one chunk of length 100 but supplies no body bytes → overruns. + let mut malformed = 1u32.to_be_bytes().to_vec(); + malformed.extend_from_slice(&100u32.to_be_bytes()); + assert!(index_chunk_pool(&malformed).is_none()); + } + /// **Proves:** a module missing its `ChunkPool` is refused — the content the root must bind is /// absent, so there is nothing to recompute from (fail-closed). #[test]