Verifying only K3 subset of proof indices

benches/pow.rs

@@ -5,15 +5,13 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use pprof::criterion::{Output, PProfProfiler};
 use scrypt_jane::scrypt::ScryptParams;
 
-const CHALLENGE: &[u8; 32] = b"hello world, CHALLENGE me!!!!!!!";
-
 fn bench_k2_pow(c: &mut Criterion) {
     c.bench_function("k2_pow", |b| {
         b.iter(|| {
             post::pow::find_k2_pow(
-                CHALLENGE,
+                b"hello world, CHALLENGE me!!!!!!!",
                 0,
-                ScryptParams::new(12, 0, 0),
+                ScryptParams::new(6, 0, 0),
                 black_box(0x0FFFFFFF_FFFFFFFF),
             )
         })

benches/verifying.rs

@@ -27,17 +27,18 @@ fn verifying(c: &mut Criterion) {
     };
     let num_labels = metadata.num_units as u64 * metadata.labels_per_unit;
 
-    for (k2, threads) in itertools::iproduct!(
-        [200u32, 2000],
+    for (k2, k3, threads) in itertools::iproduct!(
+        [200, 300],
+        [50, 100],
         [0, 1] // 0 == automatic
     ) {
         c.bench_with_input(
             BenchmarkId::new(
                 "verify",
-                format!("k2={k2}/threads={}", threads_to_str(threads)),
+                format!("k2={k2}/k3={k3}/threads={}", threads_to_str(threads)),
             ),
-            &(k2, threads),
-            |b, &(k2, _threads)| {
+            &(k2, k3, threads),
+            |b, &(k2, k3, threads)| {
                 let proof = Proof::new(
                     0,
                     (0..k2 as u64).collect::<Vec<u64>>().as_slice(),
@@ -48,8 +49,10 @@ fn verifying(c: &mut Criterion) {
                 let params = VerifyingParams {
                     difficulty: u64::MAX,
                     k2,
+                    k3,
                     k2_pow_difficulty: u64::MAX,
                     k3_pow_difficulty: u64::MAX,
+                    pow_scrypt: ScryptParams::new(6, 0, 0),
                     scrypt: ScryptParams::new(12, 0, 0),
                 };
 

src/config.rs

@@ -5,12 +5,16 @@ pub struct Config {
     pub k1: u32,
     /// K2 is the number of labels below the required difficulty required for a proof.
     pub k2: u32,
+    /// K3 is the size of the subset of proof indices that is validated.
+    pub k3: u32,
     /// Difficulty for K2 proof of work. Lower values increase difficulty of finding
     /// `k2_pow` for [Proof][crate::prove::Proof].
     pub k2_pow_difficulty: u64,
     /// Difficulty for K3 proof of work. Lower values increase difficulty of finding
     /// `k3_pow` for [Proof][crate::prove::Proof].
     pub k3_pow_difficulty: u64,
-
+    /// Scrypt parameters for the Proofs of Work
+    pub pow_scrypt: scrypt_jane::scrypt::ScryptParams,
+    /// Scrypt paramters for initilizing labels
     pub scrypt: scrypt_jane::scrypt::ScryptParams,
 }

src/lib.rs

@@ -6,6 +6,7 @@ pub mod initialize;
 pub mod metadata;
 pub mod pow;
 pub mod prove;
+mod random_values_gen;
 mod reader;
 pub mod verification;
 

src/prove.rs

@@ -137,7 +137,7 @@ pub fn generate_proof(
     let mut end_nonce = start_nonce + nonces as u32;
 
     let params = ProvingParams {
-        scrypt: cfg.scrypt,
+        scrypt: cfg.pow_scrypt,
         difficulty,
         k2_pow_difficulty: cfg.k2_pow_difficulty,
         k3_pow_difficulty: cfg.k3_pow_difficulty,

src/random_values_gen.rs

@@ -0,0 +1,121 @@
+#[derive(Debug, Clone)]
+struct Blake3Rng(blake3::OutputReader);
+
+impl Blake3Rng {
+    fn from_seed(seed: &[&[u8]]) -> Self {
+        let mut hasher = blake3::Hasher::new();
+        for &part in seed {
+            hasher.update(part);
+        }
+        Blake3Rng(hasher.finalize_xof())
+    }
+
+    fn next_u16(&mut self) -> u16 {
+        let mut buf = [0u8; 2];
+        self.0.fill(&mut buf);
+        u16::from_le_bytes(buf)
+    }
+}
+
+/// Picks random items from the provided Vec.
+pub(crate) struct RandomValuesIterator<T> {
+    // data shuffled in-place
+    data: Vec<T>,
+    rng: Blake3Rng,
+    idx: usize,
+}
+
+impl<T> RandomValuesIterator<T> {
+    pub(crate) fn new(data: Vec<T>, seed: &[&[u8]]) -> Self {
+        Self {
+            idx: 0,
+            data,
+            rng: Blake3Rng::from_seed(seed),
+        }
+    }
+}
+
+impl<T: Copy> Iterator for RandomValuesIterator<T> {
+    type Item = T;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let remaining = self.data.len() - self.idx;
+        if remaining == 0 {
+            return None;
+        }
+        let max_allowed = u16::MAX - u16::MAX % remaining as u16;
+        loop {
+            let rand_num = self.rng.next_u16();
+            if rand_num < max_allowed {
+                self.data
+                    .swap(self.idx, (rand_num as usize % remaining) + self.idx);
+                let value = self.data[self.idx];
+                self.idx += 1;
+                return Some(value);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::RandomValuesIterator;
+    use itertools::Itertools;
+    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
+    use std::collections::HashSet;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    /// Check if returns unique items from its data set.
+    #[test]
+    fn gives_each_value_once() {
+        let k2 = 1000;
+        let iter = RandomValuesIterator::new((0..k2).collect(), &[]);
+        let mut occurences = HashSet::new();
+        for item in iter {
+            assert!(occurences.insert(item));
+        }
+        assert_eq!(k2, occurences.len());
+    }
+
+    #[test]
+    fn test_vec() {
+        let expected = [
+            39, 13, 95, 77, 36, 41, 74, 17, 59, 87, 91, 63, 40, 20, 94, 78, 48, 60, 18, 32, 67, 43,
+            23, 69, 71, 1, 51, 79, 19, 53, 86, 80, 14, 84, 97, 92, 83, 26, 2, 81, 42, 55, 50, 88,
+            75, 82, 44, 34, 58, 72, 35, 25, 10, 68, 12, 11, 70, 27, 98, 57, 96, 16, 45, 73, 0, 15,
+            62, 46, 30, 89, 33, 54, 9, 29, 7, 90, 38, 5, 49, 61, 93, 99, 22, 6, 64, 24, 76, 85, 37,
+            65, 31, 4, 52, 3, 56, 21, 8, 28, 66, 47,
+        ];
+        let input = (0..expected.len()).collect();
+
+        let iter = RandomValuesIterator::new(input, &[]);
+        assert_eq!(&expected, iter.collect_vec().as_slice());
+    }
+
+    #[test]
+    fn distribution_is_uniform() {
+        let data_set = (0..200).collect_vec();
+        let occurences = (0..data_set.len())
+            .map(|_| AtomicUsize::new(0))
+            .collect_vec();
+
+        // Take random n values many times and count each occurence
+        let n = 50;
+        let iterations = 10_000_000;
+        (0u64..iterations).into_par_iter().for_each(|seed| {
+            for value in RandomValuesIterator::new(data_set.clone(), &[&seed.to_le_bytes()]).take(n)
+            {
+                occurences[value].fetch_add(1, Ordering::Release);
+            }
+        });
+
+        // Verify distribution
+        let expected_count = (iterations * n as u64 / data_set.len() as u64) as f64;
+        let max_deviation = 0.002;
+        for (value, count) in occurences.into_iter().enumerate() {
+            let count = count.load(Ordering::Acquire);
+            let deviation = (count as f64 - expected_count) / expected_count;
+            assert!(deviation.abs() < max_deviation, "{value} occured {count} times (expected {expected_count}). deviation {deviation} > {max_deviation}");
+        }
+    }
+}

src/verification.rs

@@ -1,4 +1,43 @@
+//! # Proof verification
+//!
+//! ## Steps to verify a proof:
+//!
+//! 1. verify k2_pow
+//! 2. verify k3_pow
+//! 3. verify number of indices == K2
+//! 4. select K3 indices
+//! 5. verify each of K3 selected indices satisfy difficulty (inferred from K1)
+//!
+//! ## Selecting subset of K3 proven indices
+//!
+//! ```text
+//! seed = concat(ch, nonce, indices, k2pow, k3pow)
+//! random_bytes = blake3(seed) // infinite blake output
+//! for (index=0; index<K3; index++) {
+//!   remaining = K2 - index
+//!   max_allowed = u16::MAX - (u16::MAX % remaining)
+//!   do {
+//!     rand_num = random_bytes.read_u16_le()
+//!   } while rand_num >= max_allowed;
+//!
+//!   to_swap = (rand_num % remaining) + index
+//!   indices.swap(index, to_swap)
+//! }
+//! ```
+//! indices[0..K3] now contains randomly picked values
+//!
+//! ## Verifying K3 indexes
+//!
+//! We must check if every index satisfies the difficulty condition.
+//! To do so, we must repeat similar work as proving. Steps:
+//! 1. Initialize AES cipher for proof's nonce.
+//! 2. For each index:
+//!     - replicate the label it points to,
+//!     - encrypt it with AES,
+//!     - convert AES output to u64,
+//!     - compare it with difficulty.
 use cipher::BlockEncrypt;
+use itertools::Itertools;
 use rayon::prelude::{ParallelBridge, ParallelIterator};
 use scrypt_jane::scrypt::{self, ScryptParams};
 
@@ -10,6 +49,7 @@ use crate::{
     metadata::ProofMetadata,
     pow::{hash_k2_pow, hash_k3_pow},
     prove::Proof,
+    random_values_gen::RandomValuesIterator,
 };
 
 #[inline]
@@ -27,8 +67,10 @@ fn generate_label(commitment: &[u8; 32], params: ScryptParams, index: u64) -> [u
 pub struct VerifyingParams {
     pub difficulty: u64,
     pub k2: u32,
+    pub k3: u32,
     pub k2_pow_difficulty: u64,
     pub k3_pow_difficulty: u64,
+    pub pow_scrypt: ScryptParams,
     pub scrypt: ScryptParams,
 }
 
@@ -37,8 +79,10 @@ impl VerifyingParams {
         Ok(Self {
             difficulty: proving_difficulty(num_labels, cfg.k1)?,
             k2: cfg.k2,
+            k3: cfg.k3,
             k2_pow_difficulty: cfg.k2_pow_difficulty,
             k3_pow_difficulty: cfg.k3_pow_difficulty,
+            pow_scrypt: cfg.pow_scrypt,
             scrypt: cfg.scrypt,
         })
     }
@@ -48,14 +92,10 @@ impl VerifyingParams {
 ///
 /// Arguments:
 ///
-/// * `proof`: The proof that we're verifying.
+/// * `proof`: The proof that to verify
 /// * `metadata`: ProofMetadata
 /// * `params`: VerifyingParams
 /// * `threads`: The number of threads to use for verification.
-///
-/// Returns:
-///
-/// A boolean value.
 pub fn verify(
     proof: &Proof,
     metadata: &ProofMetadata,
@@ -66,7 +106,7 @@ pub fn verify(
 
     // Verify K2 PoW
     let nonce_group = proof.nonce / 2;
-    let k2_pow_value = hash_k2_pow(&challenge, nonce_group, params.scrypt, proof.k2_pow);
+    let k2_pow_value = hash_k2_pow(&challenge, nonce_group, params.pow_scrypt, proof.k2_pow);
     if k2_pow_value >= params.k2_pow_difficulty {
         return Err(format!(
             "k2 pow is invalid: {k2_pow_value} >= {}",
@@ -79,7 +119,7 @@ pub fn verify(
         &challenge,
         proof.nonce,
         &proof.indices,
-        params.scrypt,
+        params.pow_scrypt,
         proof.k2_pow,
         proof.k3_pow,
     );
@@ -101,20 +141,30 @@ pub fn verify(
         ));
     }
 
-    let indexes = decompress_indexes(&proof.indices, bits_per_index);
+    let indices_unpacked = decompress_indexes(&proof.indices, bits_per_index).collect_vec();
     let commitment = calc_commitment(&metadata.node_id, &metadata.commitment_atx_id);
-
     let nonce_group = proof.nonce / 2;
     let cipher = AesCipher::new_with_k2pow(&challenge, nonce_group, proof.k2_pow);
     let output_index = (proof.nonce % 2) as usize;
 
+    // Select K3 indices
+    let seed = &[
+        challenge.as_slice(),
+        &proof.nonce.to_le_bytes(),
+        proof.indices.as_slice(),
+        &proof.k2_pow.to_le_bytes(),
+        &proof.k3_pow.to_le_bytes(),
+    ];
+
+    let k3_indices = RandomValuesIterator::new(indices_unpacked, seed).take(params.k3 as usize);
+
     let pool = rayon::ThreadPoolBuilder::new()
         .num_threads(threads)
         .build()
         .unwrap();
 
     pool.install(|| {
-        indexes.par_bridge().try_for_each(|index| {
+        k3_indices.par_bridge().try_for_each(|index| {
             let mut u64s = [0u64; 2];
             let label = generate_label(&commitment, params.scrypt, index);
             cipher.aes.encrypt_block_b2b(
@@ -181,8 +231,10 @@ mod tests {
         let params = VerifyingParams {
             difficulty: u64::MAX,
             k2: 10,
+            k3: 10,
             k2_pow_difficulty: u64::MAX / 16,
             k3_pow_difficulty: u64::MAX / 16,
+            pow_scrypt: scrypt_params,
             scrypt: scrypt_params,
         };
 

tests/generate_and_verify.rs

@@ -17,9 +17,11 @@ fn test_generate_and_verify() {
     let cfg = post::config::Config {
         k1: 32,
         k2: 32,
+        k3: 10,
         k2_pow_difficulty: u64::MAX / 8,
         k3_pow_difficulty: u64::MAX / 8,
-        scrypt: ScryptParams::new(1, 0, 0),
+        pow_scrypt: ScryptParams::new(1, 0, 0),
+        scrypt: ScryptParams::new(2, 0, 0),
     };
 
     let metadata = initialize(