From 517898497fd61d913dc29b9939adeae57ae270c6 Mon Sep 17 00:00:00 2001 From: MesTTo Date: Tue, 23 Jun 2026 16:25:44 +1000 Subject: [PATCH 1/2] Skip dense meet joined-mask scans Implements the TODO in the dense meet: instead of walking both dense nodes' full masks and re-testing membership per byte, walk the intersection mask directly, so bytes absent from either side are never visited. No public API change; is_identity handling is equivalent (the intersection walk observes the same element/identity outcomes the full walk did). Motivated by the existing meet benchmarks, measured on this tree vs its base (interleaved, two reps each side, best median): superdense_meet 8000 26.1->25.0us, 16000 42.5->39.8us, 32000 100->93.9us (a consistent 4-6% where the masks are dense); sparse_meet and binary_meet at parity to -2% (their masks are thin, so there is little scan to skip). Result-identical: the full suite and the seeded algebra differential pass, including composed with the DenseByteNode operand-selection fix (both touch this file; they auto-merge and the differential passes with both applied). --- src/dense_byte_node.rs | 162 ++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 83 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index ca253c3..fea435d 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -110,6 +110,15 @@ impl> ByteNode alloc, } } + #[inline(always)] + fn slot_in_word(mask_word: u64, word_base: usize, bit_idx: u32) -> usize { + let preceding_bits = if bit_idx == 0 { + 0 + } else { + (1u64 << bit_idx) - 1 + }; + word_base + (mask_word & preceding_bits).count_ones() as usize + } #[inline] pub fn reserve_capacity(&mut self, additional: usize) { self.values.reserve(additional) @@ -2107,10 +2116,8 @@ impl, Other } fn pmeet(&self, other: &ByteNode) -> AlgebraicResult { - // TODO this technically doesn't need to calculate and iterate over jm - // iterating over mm and calculating m such that the following suffices - // c_{self,other} += popcnt(m & {self,other}) - let jm: ByteMask = self.mask | other.mask; + // Iterate the overlap mask directly. Slot indexes are recovered with + // prefix popcounts in each dense-mask word. let mut mm: ByteMask = self.mask & other.mask; let mut is_identity = self.mask == mm; @@ -2122,58 +2129,55 @@ impl, Other let mut v = ValuesVec::with_capacity_in(len, self.alloc.clone()); let new_v = v.v.spare_capacity_mut(); - let mut l = 0; - let mut r = 0; let mut c = 0; + let mut self_word_base = 0; + let mut other_word_base = 0; for i in 0..4 { - let mut lm = jm.0[i]; + let self_word = self.mask.0[i]; + let other_word = other.mask.0[i]; + let mut lm = mm.0[i]; while lm != 0 { let index = lm.trailing_zeros(); - - if ((1u64 << index) & mm.0[i]) != 0 { - //This runs for cofrees that exist in both nodes - - let lv = unsafe { self.values.get_unchecked(l) }; - let rv = unsafe { other.values.get_unchecked(r) }; - match lv.pmeet(rv) { - AlgebraicResult::None => { - is_counter_identity = false; - is_identity = false; - mm.0[i] ^= 1u64 << index; - }, - AlgebraicResult::Identity(mask) => { - debug_assert!((mask & SELF_IDENT > 0) || (mask & COUNTER_IDENT > 0)); - if mask & SELF_IDENT == 0 { - is_identity = false; - } - if mask & COUNTER_IDENT == 0 { - is_counter_identity = false; - } - if mask & SELF_IDENT > 0 { - unsafe { new_v.get_unchecked_mut(c).write(lv.clone()) }; - } else { - let new_cf = Cf::from_cf(rv.clone()); - unsafe { new_v.get_unchecked_mut(c).write(new_cf) }; - } - c += 1; - }, - AlgebraicResult::Element(jv) => { + let l = Self::slot_in_word(self_word, self_word_base, index); + let r = Self::slot_in_word(other_word, other_word_base, index); + + //This runs for cofrees that exist in both nodes + let lv = unsafe { self.values.get_unchecked(l) }; + let rv = unsafe { other.values.get_unchecked(r) }; + match lv.pmeet(rv) { + AlgebraicResult::None => { + is_counter_identity = false; + is_identity = false; + mm.0[i] ^= 1u64 << index; + }, + AlgebraicResult::Identity(mask) => { + debug_assert!((mask & SELF_IDENT > 0) || (mask & COUNTER_IDENT > 0)); + if mask & SELF_IDENT == 0 { is_identity = false; + } + if mask & COUNTER_IDENT == 0 { is_counter_identity = false; - unsafe { new_v.get_unchecked_mut(c).write(jv) }; - c += 1; - }, - } - l += 1; - r += 1; - } else if ((1u64 << index) & self.mask.0[i]) != 0 { - l += 1; - } else { - r += 1; + } + if mask & SELF_IDENT > 0 { + unsafe { new_v.get_unchecked_mut(c).write(lv.clone()) }; + } else { + let new_cf = Cf::from_cf(rv.clone()); + unsafe { new_v.get_unchecked_mut(c).write(new_cf) }; + } + c += 1; + }, + AlgebraicResult::Element(jv) => { + is_identity = false; + is_counter_identity = false; + unsafe { new_v.get_unchecked_mut(c).write(jv) }; + c += 1; + }, } lm ^= 1u64 << index; } + self_word_base += self_word.count_ones() as usize; + other_word_base += other_word.count_ones() as usize; } unsafe{ v.v.set_len(c); } @@ -2286,13 +2290,10 @@ impl> ByteNode { fn prestrict>(&self, other: &ByteNode) -> AlgebraicResult where Self: Sized { - let mut is_identity = true; - - // TODO this technically doesn't need to calculate and iterate over jm - // iterating over mm and calculating m such that the following suffices - // c_{self,other} += popcnt(m & {self,other}) - let jm: ByteMask = self.mask | other.mask; + // Iterate the overlap mask directly. Slot indexes are recovered with + // prefix popcounts in each dense-mask word. let mut mm: ByteMask = self.mask & other.mask; + let mut is_identity = self.mask == mm && other.mask == mm; let mmc = [mm.0[0].count_ones(), mm.0[1].count_ones(), mm.0[2].count_ones(), mm.0[3].count_ones()]; @@ -2300,48 +2301,43 @@ impl> ByteNode let mut v = ValuesVec::with_capacity_in(len, self.alloc.clone()); let new_v = v.v.spare_capacity_mut(); - let mut l = 0; - let mut r = 0; let mut c = 0; + let mut self_word_base = 0; + let mut other_word_base = 0; for i in 0..4 { - let mut lm = jm.0[i]; + let self_word = self.mask.0[i]; + let other_word = other.mask.0[i]; + let mut lm = mm.0[i]; while lm != 0 { let index = lm.trailing_zeros(); + let l = Self::slot_in_word(self_word, self_word_base, index); + let r = Self::slot_in_word(other_word, other_word_base, index); - if ((1u64 << index) & mm.0[i]) != 0 { - let lv = unsafe { self.values.get_unchecked(l) }; - let rv = unsafe { other.values.get_unchecked(r) }; - // println!("dense prestrict {}", index as usize + i*64); + let lv = unsafe { self.values.get_unchecked(l) }; + let rv = unsafe { other.values.get_unchecked(r) }; + // println!("dense prestrict {}", index as usize + i*64); - match lv.prestrict(rv) { - AlgebraicResult::None => { - is_identity = false; - mm.0[i] ^= 1u64 << index; - } - AlgebraicResult::Identity(mask) => { - debug_assert_eq!(mask, SELF_IDENT); //restrict is non-commutative - unsafe { new_v.get_unchecked_mut(c).write(lv.clone()) }; - c += 1; - }, - AlgebraicResult::Element(jv) => { - is_identity = false; - unsafe { new_v.get_unchecked_mut(c).write(jv) }; - c += 1; - }, - } - l += 1; - r += 1; - } else { - is_identity = false; - if ((1u64 << index) & self.mask.0[i]) != 0 { - l += 1; - } else { - r += 1; + match lv.prestrict(rv) { + AlgebraicResult::None => { + is_identity = false; + mm.0[i] ^= 1u64 << index; } + AlgebraicResult::Identity(mask) => { + debug_assert_eq!(mask, SELF_IDENT); //restrict is non-commutative + unsafe { new_v.get_unchecked_mut(c).write(lv.clone()) }; + c += 1; + }, + AlgebraicResult::Element(jv) => { + is_identity = false; + unsafe { new_v.get_unchecked_mut(c).write(jv) }; + c += 1; + }, } lm ^= 1u64 << index; } + self_word_base += self_word.count_ones() as usize; + other_word_base += other_word.count_ones() as usize; } unsafe{ v.v.set_len(c); } From eb618587463772586edfaf831ac14bd642a57279 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sat, 25 Jul 2026 20:29:41 -0600 Subject: [PATCH 2/2] Adding new superdense_meet_low_overlap benchmark to illustrate the benefits of this change --- benches/superdense_keys.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/benches/superdense_keys.rs b/benches/superdense_keys.rs index b61c609..0371789 100644 --- a/benches/superdense_keys.rs +++ b/benches/superdense_keys.rs @@ -160,6 +160,26 @@ fn superdense_meet(bencher: Bencher, n: u64) { }); } +/// This is the success case for dense meet's intersection-mask walk: both +/// nodes have many children, but only a small fraction of those children are +/// present in both nodes. +#[divan::bench(args = [1000, 2000, 4000, 8000, 16000, 32000])] +fn superdense_meet_low_overlap(bencher: Bencher, n: u64) { + let overlap = n / 16; + let offset = n - overlap; + + let mut l: PathMap = PathMap::new(); + for i in 0..n { l.set_val_at(prefix_key(&i), i); } + let mut r: PathMap = PathMap::new(); + for i in offset..(offset + n) { r.set_val_at(prefix_key(&i), i); } + + let mut intersection: PathMap = PathMap::new(); + bencher.bench_local(|| { + *black_box(&mut intersection) = l.meet(black_box(&r)); + }); + assert_eq!(intersection.val_count(), overlap as usize); +} + /// This tests the performance of the meet op when there are already some shared nodes between the maps #[divan::bench(args = [1000, 2000, 4000, 8000, 16000, 32000])] fn superdense_meet_after_join(bencher: Bencher, n: u64) {