From 939ca727c8b1357e30bb088ca8f327885f4ea35b Mon Sep 17 00:00:00 2001 From: Kevin Lewi Date: Sat, 13 Jan 2024 20:40:53 -0800 Subject: [PATCH] Extending limited history to properly check for the existence of past values --- README.md | 2 +- akd/Cargo.toml | 4 +- akd/src/append_only_zks.rs | 4 +- akd/src/directory.rs | 104 +++++----- akd/src/errors.rs | 5 + akd/src/lib.rs | 23 ++- akd/src/tests.rs | 116 +++++++---- akd_core/Cargo.toml | 2 +- akd_core/src/proto/mod.rs | 22 +-- akd_core/src/proto/specs/types.proto | 8 +- akd_core/src/proto/tests.rs | 42 ++-- akd_core/src/types/mod.rs | 8 +- akd_core/src/utils.rs | 77 ++++++++ akd_core/src/verify/history.rs | 217 +++++++++++++-------- examples/Cargo.toml | 2 +- examples/src/mysql_demo/tests/test_util.rs | 1 + 16 files changed, 409 insertions(+), 228 deletions(-) diff --git a/README.md b/README.md index 939499f3..60dab814 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Installation Add the following line to the dependencies of your `Cargo.toml`: ``` -akd = "0.11" +akd = "0.12.0-pre.1" ``` ### Minimum Supported Rust Version diff --git a/akd/Cargo.toml b/akd/Cargo.toml index c76890ba..bbeec451 100644 --- a/akd/Cargo.toml +++ b/akd/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "akd" -version = "0.11.0" +version = "0.12.0-pre.1" authors = ["akd contributors"] description = "An implementation of an auditable key directory" license = "MIT OR Apache-2.0" @@ -51,7 +51,7 @@ default = [ [dependencies] ## Required dependencies ## -akd_core = { version = "0.11.0", path = "../akd_core", default-features = false, features = [ +akd_core = { version = "0.12.0-pre.1", path = "../akd_core", default-features = false, features = [ "vrf", ] } async-recursion = "1" diff --git a/akd/src/append_only_zks.rs b/akd/src/append_only_zks.rs index 8ae044a3..f8acd28c 100644 --- a/akd/src/append_only_zks.rs +++ b/akd/src/append_only_zks.rs @@ -1668,9 +1668,7 @@ mod tests { // Recursively traverse the tree and check that the sibling of each node is correct let root_node = TreeNode::get_from_storage(&db, &NodeKey(NodeLabel::root()), 1).await?; let mut nodes: Vec = vec![root_node]; - while !nodes.is_empty() { - let current_node = nodes.pop().unwrap(); - + while let Some(current_node) = nodes.pop() { let left_child = current_node.get_child_node(&db, Direction::Left, 1).await?; let right_child = current_node .get_child_node(&db, Direction::Right, 1) diff --git a/akd/src/directory.rs b/akd/src/directory.rs index ce54fc13..b51a5e4a 100644 --- a/akd/src/directory.rs +++ b/akd/src/directory.rs @@ -16,11 +16,13 @@ use crate::storage::types::{DbRecord, ValueState, ValueStateRetrievalFlag}; use crate::storage::Database; use crate::{ AkdLabel, AkdValue, AppendOnlyProof, AzksElement, Digest, EpochHash, HistoryProof, LookupProof, - NonMembershipProof, UpdateProof, + MembershipProof, NonMembershipProof, UpdateProof, }; use crate::VersionFreshness; use akd_core::configuration::Configuration; +use akd_core::utils::get_marker_versions; +use akd_core::verify::history::HistoryParams; use log::{error, info}; use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; @@ -462,15 +464,6 @@ where user_data = match params { HistoryParams::Complete => user_data, HistoryParams::MostRecent(n) => user_data.into_iter().take(n).collect::>(), - HistoryParams::SinceEpoch(epoch) => { - user_data = user_data - .into_iter() - .filter(|val| val.epoch >= epoch) - .collect::>(); - // Ordering should be maintained after filtering, but let's re-sort just in case - user_data.sort_by(|a, b| b.epoch.cmp(&a.epoch)); - user_data - } }; if user_data.is_empty() { @@ -496,7 +489,8 @@ where } let mut update_proofs = Vec::::new(); - let mut last_version = 0; + let mut start_version = user_data[0].version; + let mut end_version = 0; for user_state in user_data { // Ignore states in storage that are ahead of current directory epoch if user_state.epoch <= current_epoch { @@ -504,53 +498,64 @@ where .create_single_update_proof(akd_label, &user_state) .await?; update_proofs.push(proof); - last_version = if user_state.version > last_version { + start_version = if user_state.version < start_version { + user_state.version + } else { + start_version + }; + end_version = if user_state.version > end_version { user_state.version } else { - last_version + end_version }; } } - let next_marker = get_marker_version(last_version) + 1; - let final_marker = get_marker_version(current_epoch); - let mut until_marker_vrf_proofs = Vec::>::new(); - let mut non_existence_until_marker_proofs = Vec::::new(); + if start_version == 0 { + return Err(AkdError::Directory(DirectoryError::InvalidVersion( + "Computed start version for the key history should be non-zero".to_string(), + ))); + } + + let (past_marker_versions, future_marker_versions) = + get_marker_versions(start_version, end_version, current_epoch); + + let mut past_marker_vrf_proofs = Vec::>::new(); + let mut existence_of_past_marker_proofs = Vec::::new(); - for ver in last_version + 1..(1 << next_marker) { - let label_for_ver = self + for version in past_marker_versions { + let node_label = self .vrf - .get_node_label::(akd_label, VersionFreshness::Fresh, ver) + .get_node_label::(akd_label, VersionFreshness::Fresh, version) .await?; - let non_existence_of_ver = current_azks - .get_non_membership_proof::(&self.storage, label_for_ver) + let existence_vrf = self + .vrf + .get_label_proof::(akd_label, VersionFreshness::Fresh, version) .await?; - non_existence_until_marker_proofs.push(non_existence_of_ver); - until_marker_vrf_proofs.push( - self.vrf - .get_label_proof::(akd_label, VersionFreshness::Fresh, ver) - .await? - .to_bytes() - .to_vec(), + past_marker_vrf_proofs.push(existence_vrf.to_bytes().to_vec()); + existence_of_past_marker_proofs.push( + current_azks + .get_membership_proof::(&self.storage, node_label) + .await?, ); } let mut future_marker_vrf_proofs = Vec::>::new(); let mut non_existence_of_future_marker_proofs = Vec::::new(); - for marker_power in next_marker..final_marker + 1 { - let ver = 1 << marker_power; - let label_for_ver = self + for version in future_marker_versions { + let node_label = self .vrf - .get_node_label::(akd_label, VersionFreshness::Fresh, ver) + .get_node_label::(akd_label, VersionFreshness::Fresh, version) .await?; - let non_existence_of_ver = current_azks - .get_non_membership_proof::(&self.storage, label_for_ver) - .await?; - non_existence_of_future_marker_proofs.push(non_existence_of_ver); + non_existence_of_future_marker_proofs.push( + current_azks + .get_non_membership_proof::(&self.storage, node_label) + .await?, + ); future_marker_vrf_proofs.push( self.vrf - .get_label_proof::(akd_label, VersionFreshness::Fresh, ver) + .get_label_proof::(akd_label, VersionFreshness::Fresh, version) .await? .to_bytes() .to_vec(), @@ -565,8 +570,8 @@ where Ok(( HistoryProof { update_proofs, - until_marker_vrf_proofs, - non_existence_until_marker_proofs, + past_marker_vrf_proofs, + existence_of_past_marker_proofs, future_marker_vrf_proofs, non_existence_of_future_marker_proofs, }, @@ -855,25 +860,6 @@ where } } -/// The parameters that dictate how much of the history proof to return to the consumer -/// (either a complete history, or some limited form). -#[derive(Copy, Clone)] -pub enum HistoryParams { - /// Returns a complete history for a label - Complete, - /// Returns up to the most recent N updates for a label - MostRecent(usize), - /// Returns all updates since a specified epoch (inclusive) - SinceEpoch(u64), -} - -impl Default for HistoryParams { - /// By default, we return a complete history - fn default() -> Self { - Self::Complete - } -} - /// Helpers pub(crate) fn get_marker_version(version: u64) -> u64 { diff --git a/akd/src/errors.rs b/akd/src/errors.rs index 577d0564..92179051 100644 --- a/akd/src/errors.rs +++ b/akd/src/errors.rs @@ -230,6 +230,8 @@ pub enum DirectoryError { ReadOnlyDirectory(String), /// Publish Publish(String), + /// Detected an invalid version + InvalidVersion(String), } impl std::error::Error for DirectoryError {} @@ -249,6 +251,9 @@ impl fmt::Display for DirectoryError { Self::Publish(inner_message) => { write!(f, "Directory publish error: {inner_message}") } + Self::InvalidVersion(inner_message) => { + write!(f, "Invalid version error: {inner_message}") + } } } } diff --git a/akd/src/lib.rs b/akd/src/lib.rs index c6eec37a..2db131ea 100644 --- a/akd/src/lib.rs +++ b/akd/src/lib.rs @@ -198,6 +198,7 @@ //! ``` //! //! ## History Proofs +//! //! As mentioned above, the security is defined by consistent views of the value for a key at any epoch. //! To this end, a server running an AKD needs to provide a way to check the history of a key. Note that in this case, //! the server is trusted for validating that a particular client is authorized to run a history check on a particular key. @@ -245,6 +246,18 @@ //! with respect to the latest root hash and public key, as follows. This function //! returns a list of values that have been associated with the specified entry, in //! reverse chronological order. +//! +//! Note that the same argument for [`HistoryParams`] that was used to generate +//! the key history proof must also be used to verify the proof. Otherwise, verification +//! may fail. +//! +//! We also use [`HistoryVerificationParams`] as an argument to the verification function, +//! allowing the consumer to specify whether or not a "tombstoned" value should be +//! accepted in place of a valid value for the corresponding entry. This is useful +//! in scenarios where the consumer wishes to verify that a particular entry exists, +//! but does not care about the value associated with it. The default behavior is to +//! not accept tombstoned values, but [`HistoryVerificationParams::AllowMissingValues`] can +//! be specified to enable this behavior. //! ``` //! # use akd::storage::StorageManager; //! # use akd::storage::memory::AsyncInMemoryDatabase; @@ -257,7 +270,7 @@ //! # let storage_manager = StorageManager::new_no_cache(db); //! # let vrf = HardCodedAkdVRF{}; //! # use akd::EpochHash; -//! # use akd::HistoryParams; +//! # use akd::{HistoryParams, HistoryVerificationParams}; //! # use akd::{AkdLabel, AkdValue}; //! # use akd::Digest; //! # @@ -287,7 +300,8 @@ //! epoch_hash.epoch(), //! AkdLabel::from("first entry"), //! history_proof, -//! akd::HistoryVerificationParams::default(), +//! HistoryParams::default(), +//! HistoryVerificationParams::default(), //! ).expect("Could not verify history"); //! //! assert_eq!( @@ -476,7 +490,8 @@ pub mod tree_node; pub mod local_auditing; pub use akd_core::{ - configuration, configuration::*, ecvrf, hash, hash::Digest, proto, types::*, verify, ARITY, + configuration, configuration::*, ecvrf, hash, hash::Digest, proto, types::*, verify, + verify::history::HistoryParams, ARITY, }; #[macro_use] @@ -485,7 +500,7 @@ mod utils; // ========== Type re-exports which are commonly used ========== // pub use append_only_zks::Azks; pub use client::HistoryVerificationParams; -pub use directory::{Directory, HistoryParams}; +pub use directory::Directory; pub use helper_structs::EpochHash; // ========== Constants and type aliases ========== // diff --git a/akd/src/tests.rs b/akd/src/tests.rs index 008bd9c4..73bc960b 100644 --- a/akd/src/tests.rs +++ b/akd/src/tests.rs @@ -248,6 +248,7 @@ async fn test_small_key_history() -> Result<(), AkdError> { root_hash.epoch(), AkdLabel::from("hello"), key_history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), )?; @@ -337,6 +338,7 @@ async fn test_simple_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello"), key_history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), )?; @@ -357,6 +359,7 @@ async fn test_simple_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello2"), key_history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), )?; @@ -377,6 +380,7 @@ async fn test_simple_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello3"), key_history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), )?; @@ -397,6 +401,7 @@ async fn test_simple_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello4"), key_history_proof.clone(), + HistoryParams::default(), HistoryVerificationParams::default(), )?; @@ -409,9 +414,10 @@ async fn test_simple_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello4"), borked_proof, + HistoryParams::default(), HistoryVerificationParams::default(), ); - assert!(matches!(result, Err(_)), "{}", "{result:?}"); + assert!(result.is_err(), "{}", "{result:?}"); Ok(()) } @@ -484,6 +490,7 @@ async fn test_complex_verification_many_versions() -> Result< epoch_hash.epoch(), label, history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), )?; for (j, res) in history_results.iter().enumerate() { @@ -566,8 +573,9 @@ async fn test_limited_key_history() -> Result<(), AkdError> { let current_epoch = current_azks.get_latest_epoch(); // "hello" was updated in epochs 1,2,3,5. Pull the latest item from the history (i.e. a lookup proof) + let history_params_1 = HistoryParams::MostRecent(1); let (history_proof, root_hash) = akd - .key_history(&AkdLabel::from("hello"), HistoryParams::MostRecent(1)) + .key_history(&AkdLabel::from("hello"), history_params_1) .await?; assert_eq!(1, history_proof.update_proofs.len()); assert_eq!(5, history_proof.update_proofs[0].epoch); @@ -579,12 +587,14 @@ async fn test_limited_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello"), history_proof, + history_params_1, HistoryVerificationParams::default(), )?; // Take the top 3 results, and check that we're getting the right epoch updates + let history_params_3 = HistoryParams::MostRecent(3); let (history_proof, root_hash) = akd - .key_history(&AkdLabel::from("hello"), HistoryParams::MostRecent(3)) + .key_history(&AkdLabel::from("hello"), history_params_3) .await?; assert_eq!(3, history_proof.update_proofs.len()); assert_eq!(5, history_proof.update_proofs[0].epoch); @@ -598,24 +608,7 @@ async fn test_limited_key_history() -> Result<(), AkdError> { current_epoch, AkdLabel::from("hello"), history_proof, - HistoryVerificationParams::default(), - )?; - - // "hello" was updated in epochs 1,2,3,5. Pull the updates since epoch 3 (inclusive) - let (history_proof, root_hash) = akd - .key_history(&AkdLabel::from("hello"), HistoryParams::SinceEpoch(3)) - .await?; - assert_eq!(2, history_proof.update_proofs.len()); - assert_eq!(5, history_proof.update_proofs[0].epoch); - assert_eq!(3, history_proof.update_proofs[1].epoch); - - // Now check that the key history verifies - key_history_verify::( - vrf_pk.as_bytes(), - root_hash.hash(), - current_epoch, - AkdLabel::from("hello"), - history_proof, + history_params_3, HistoryVerificationParams::default(), )?; @@ -662,6 +655,7 @@ async fn test_malicious_key_history() -> Result<(), AkdError> root_hash.epoch(), AkdLabel::from("hello"), key_history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), ).expect_err("The key history proof should fail here since the previous value was not marked stale at all"); @@ -687,6 +681,7 @@ async fn test_malicious_key_history() -> Result<(), AkdError> root_hash.epoch(), AkdLabel::from("hello"), key_history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), ).expect_err("The key history proof should fail here since the previous value was marked stale one epoch too late."); @@ -855,15 +850,15 @@ async fn test_simple_audit() -> Result<(), AkdError> { // The audit should be of more than 1 epoch let invalid_audit = akd.audit(3, 3).await; - assert!(matches!(invalid_audit, Err(_))); + assert!(invalid_audit.is_err()); // The audit epochs must be increasing let invalid_audit = akd.audit(3, 2).await; - assert!(matches!(invalid_audit, Err(_))); + assert!(invalid_audit.is_err()); // The audit should throw an error when queried for an epoch which hasn't yet taken place! let invalid_audit = akd.audit(6, 7).await; - assert!(matches!(invalid_audit, Err(_))); + assert!(invalid_audit.is_err()); Ok(()) } @@ -945,6 +940,7 @@ async fn test_read_during_publish() -> Result<(), AkdError> { root_hash.epoch(), AkdLabel::from("hello"), history_proof, + HistoryParams::default(), HistoryVerificationParams::default(), ) .unwrap(); @@ -956,7 +952,7 @@ async fn test_read_during_publish() -> Result<(), AkdError> { .unwrap(); let invalid_audit = akd.audit(2, 3).await; - assert!(matches!(invalid_audit, Err(_))); + assert!(invalid_audit.is_err()); Ok(()) } @@ -972,7 +968,7 @@ async fn test_directory_read_only_mode() -> Result<(), AkdErr let vrf = HardCodedAkdVRF {}; // There is no AZKS object in the storage layer, directory construction should fail let akd = ReadOnlyDirectory::::new(storage, vrf).await; - assert!(matches!(akd, Err(_))); + assert!(akd.is_err()); Ok(()) } @@ -1083,9 +1079,10 @@ async fn test_tombstoned_key_history() -> Result<(), AkdError root_hash.epoch(), AkdLabel::from("hello"), history_proof.clone(), + HistoryParams::default(), HistoryVerificationParams::default(), ); - assert!(matches!(tombstones, Err(_))); + assert!(tombstones.is_err()); // We should be able to verify tombstones assuming the client is accepting // of tombstoned states @@ -1095,6 +1092,7 @@ async fn test_tombstoned_key_history() -> Result<(), AkdError root_hash.epoch(), AkdLabel::from("hello"), history_proof, + HistoryParams::default(), HistoryVerificationParams::AllowMissingValues, )?; assert_ne!(crate::TOMBSTONE, results[0].value.0); @@ -1318,7 +1316,7 @@ async fn test_key_history_verify_malformed() -> Result<(), Ak for _ in 0..100 { let mut updates = vec![]; updates.push(( - AkdLabel(format!("label").as_bytes().to_vec()), + AkdLabel("label".to_string().as_bytes().to_vec()), AkdValue::random(&mut rng), )); akd.publish(updates.clone()).await?; @@ -1327,7 +1325,7 @@ async fn test_key_history_verify_malformed() -> Result<(), Ak for _ in 0..100 { let mut updates = vec![]; updates.push(( - AkdLabel(format!("another label").as_bytes().to_vec()), + AkdLabel("another label".to_string().as_bytes().to_vec()), AkdValue::random(&mut rng), )); akd.publish(updates.clone()).await?; @@ -1337,11 +1335,10 @@ async fn test_key_history_verify_malformed() -> Result<(), Ak let EpochHash(current_epoch, root_hash) = akd.get_epoch_hash().await?; // Get the VRF public key let vrf_pk = akd.get_public_key().await?; - let target_label = AkdLabel(format!("label").as_bytes().to_vec()); + let target_label = AkdLabel("label".to_string().as_bytes().to_vec()); - let (key_history_proof, _) = akd - .key_history(&target_label, HistoryParams::default()) - .await?; + let history_params_5 = HistoryParams::MostRecent(5); + let (key_history_proof, _) = akd.key_history(&target_label, history_params_5).await?; // Normal verification should succeed key_history_verify::( @@ -1350,17 +1347,37 @@ async fn test_key_history_verify_malformed() -> Result<(), Ak current_epoch, target_label.clone(), key_history_proof.clone(), + history_params_5, HistoryVerificationParams::default(), )?; + // Using an inconsistent set of history parameters should fail + for bad_params in [ + HistoryParams::MostRecent(1), + HistoryParams::MostRecent(4), + HistoryParams::MostRecent(6), + HistoryParams::default(), + ] { + assert!(key_history_verify::( + vrf_pk.as_bytes(), + root_hash, + current_epoch, + target_label.clone(), + key_history_proof.clone(), + bad_params, + HistoryVerificationParams::default(), + ) + .is_err()); + } + let mut malformed_proof_1 = key_history_proof.clone(); - malformed_proof_1.until_marker_vrf_proofs = key_history_proof.until_marker_vrf_proofs - [..key_history_proof.until_marker_vrf_proofs.len() - 1] + malformed_proof_1.past_marker_vrf_proofs = key_history_proof.past_marker_vrf_proofs + [..key_history_proof.past_marker_vrf_proofs.len() - 1] .to_vec(); let mut malformed_proof_2 = key_history_proof.clone(); - malformed_proof_2.non_existence_until_marker_proofs = key_history_proof - .non_existence_until_marker_proofs - [..key_history_proof.non_existence_until_marker_proofs.len() - 1] + malformed_proof_2.existence_of_past_marker_proofs = key_history_proof + .existence_of_past_marker_proofs + [..key_history_proof.existence_of_past_marker_proofs.len() - 1] .to_vec(); let mut malformed_proof_3 = key_history_proof.clone(); malformed_proof_3.future_marker_vrf_proofs = key_history_proof.future_marker_vrf_proofs @@ -1387,6 +1404,29 @@ async fn test_key_history_verify_malformed() -> Result<(), Ak current_epoch, target_label.clone(), malformed_proof, + history_params_5, + HistoryVerificationParams::default(), + ) + .is_err()); + } + + let mut malformed_proof_start_version_is_zero = key_history_proof.clone(); + malformed_proof_start_version_is_zero.update_proofs[0].epoch = 0; + let mut malformed_proof_end_version_exceeds_epoch = key_history_proof.clone(); + malformed_proof_end_version_exceeds_epoch.update_proofs[0].epoch = current_epoch + 1; + + // Malformed proof verification should fail + for malformed_proof in [ + malformed_proof_start_version_is_zero, + malformed_proof_end_version_exceeds_epoch, + ] { + assert!(key_history_verify::( + vrf_pk.as_bytes(), + root_hash, + current_epoch, + target_label.clone(), + malformed_proof, + history_params_5, HistoryVerificationParams::default(), ) .is_err()); diff --git a/akd_core/Cargo.toml b/akd_core/Cargo.toml index 60fcbd17..bcfa9c4d 100644 --- a/akd_core/Cargo.toml +++ b/akd_core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "akd_core" -version = "0.11.0" +version = "0.12.0-pre.1" authors = ["akd contributors"] description = "Core utilities for the akd crate" license = "MIT OR Apache-2.0" diff --git a/akd_core/src/proto/mod.rs b/akd_core/src/proto/mod.rs index d313acae..f8962a4c 100644 --- a/akd_core/src/proto/mod.rs +++ b/akd_core/src/proto/mod.rs @@ -215,7 +215,7 @@ impl TryFrom<&specs::types::SiblingProof> for crate::SiblingProof { let label: crate::NodeLabel = input.label.as_ref().unwrap().try_into()?; // get the raw data & it's length, but at most crate::hash::DIGEST_BYTES bytes - let siblings = input.siblings.get(0); + let siblings = input.siblings.first(); if siblings.is_none() { return Err(ConversionError::Deserialization( "Required field siblings missing".to_string(), @@ -459,9 +459,9 @@ impl From<&crate::HistoryProof> for specs::types::HistoryProof { .iter() .map(|proof| proof.into()) .collect::>(), - until_marker_vrf_proofs: input.until_marker_vrf_proofs.to_vec(), - non_existence_until_marker_proofs: input - .non_existence_until_marker_proofs + past_marker_vrf_proofs: input.past_marker_vrf_proofs.to_vec(), + existence_of_past_marker_proofs: input + .existence_of_past_marker_proofs .iter() .map(|proof| proof.into()) .collect::>(), @@ -482,14 +482,14 @@ impl TryFrom<&specs::types::HistoryProof> for crate::HistoryProof { fn try_from(input: &specs::types::HistoryProof) -> Result { let update_proofs = convert_from_vector!(input.update_proofs, crate::UpdateProof); - let until_marker_vrf_proofs = input - .until_marker_vrf_proofs + let past_marker_vrf_proofs = input + .past_marker_vrf_proofs .iter() .map(|item| item.to_vec()) .collect::>(); - let non_existence_until_marker_proofs = convert_from_vector!( - input.non_existence_until_marker_proofs, - crate::NonMembershipProof + let existence_of_past_marker_proofs = convert_from_vector!( + input.existence_of_past_marker_proofs, + crate::MembershipProof ); let future_marker_vrf_proofs = input @@ -504,8 +504,8 @@ impl TryFrom<&specs::types::HistoryProof> for crate::HistoryProof { Ok(Self { update_proofs, - until_marker_vrf_proofs, - non_existence_until_marker_proofs, + past_marker_vrf_proofs, + existence_of_past_marker_proofs, future_marker_vrf_proofs, non_existence_of_future_marker_proofs, }) diff --git a/akd_core/src/proto/specs/types.proto b/akd_core/src/proto/specs/types.proto index 49ed1d5c..b41246d8 100644 --- a/akd_core/src/proto/specs/types.proto +++ b/akd_core/src/proto/specs/types.proto @@ -87,13 +87,13 @@ message UpdateProof { optional bytes commitment_nonce = 8; } -/* This proof consists of an array of [`UpdateProof`]s, non-membership proofs for -the versions up until the next marker, and then non-membership proofs for future markers +/* This proof consists of an array of [`UpdateProof`]s, membership proofs for +the past markers, and non-membership proofs for future markers up until the current epoch. */ message HistoryProof { repeated UpdateProof update_proofs = 1; - repeated bytes until_marker_vrf_proofs = 2; - repeated NonMembershipProof non_existence_until_marker_proofs = 3; + repeated bytes past_marker_vrf_proofs = 2; + repeated MembershipProof existence_of_past_marker_proofs = 3; repeated bytes future_marker_vrf_proofs = 4; repeated NonMembershipProof non_existence_of_future_marker_proofs = 5; } diff --git a/akd_core/src/proto/tests.rs b/akd_core/src/proto/tests.rs index 41845a5d..8be9c7fb 100644 --- a/akd_core/src/proto/tests.rs +++ b/akd_core/src/proto/tests.rs @@ -185,6 +185,18 @@ fn test_convert_update_proof() { #[test] fn test_convert_history_proof() { + fn membership_proof() -> crate::MembershipProof { + crate::MembershipProof { + label: random_label(), + hash_val: AzksValue(random_hash()), + sibling_proofs: vec![crate::SiblingProof { + label: random_label(), + siblings: [random_azks_element()], + direction: Direction::Right, + }], + } + } + fn non_membership_proof() -> crate::NonMembershipProof { crate::NonMembershipProof { label: random_label(), @@ -209,40 +221,24 @@ fn test_convert_history_proof() { value: crate::AkdValue(random_hash().to_vec()), version: rng.gen(), existence_vrf_proof: random_hash().to_vec(), - existence_proof: crate::MembershipProof { - label: random_label(), - hash_val: AzksValue(random_hash()), - sibling_proofs: vec![crate::SiblingProof { - label: random_label(), - siblings: [random_azks_element()], - direction: Direction::Right, - }], - }, + existence_proof: membership_proof(), previous_version_vrf_proof: Some(random_hash().to_vec()), - previous_version_proof: Some(crate::MembershipProof { - label: random_label(), - hash_val: AzksValue(random_hash()), - sibling_proofs: vec![crate::SiblingProof { - label: random_label(), - siblings: [random_azks_element()], - direction: Direction::Right, - }], - }), + previous_version_proof: Some(membership_proof()), commitment_nonce: random_hash().to_vec(), } } let original = crate::HistoryProof { update_proofs: vec![upd_proof(), upd_proof(), upd_proof()], - until_marker_vrf_proofs: vec![ + past_marker_vrf_proofs: vec![ random_hash().to_vec(), random_hash().to_vec(), random_hash().to_vec(), ], - non_existence_until_marker_proofs: vec![ - non_membership_proof(), - non_membership_proof(), - non_membership_proof(), + existence_of_past_marker_proofs: vec![ + membership_proof(), + membership_proof(), + membership_proof(), ], future_marker_vrf_proofs: vec![ random_hash().to_vec(), diff --git a/akd_core/src/types/mod.rs b/akd_core/src/types/mod.rs index f375ffa6..fa08db18 100644 --- a/akd_core/src/types/mod.rs +++ b/akd_core/src/types/mod.rs @@ -471,10 +471,10 @@ pub struct UpdateProof { pub struct HistoryProof { /// The update proofs in the key history pub update_proofs: Vec, - /// VRF Proofs for the labels of the values until the next marker version - pub until_marker_vrf_proofs: Vec>, - /// Proof that the values until the next marker version did not exist at this time - pub non_existence_until_marker_proofs: Vec, + /// VRF Proofs for the labels of the values for past markers + pub past_marker_vrf_proofs: Vec>, + /// Proof that the values for the past markers exist + pub existence_of_past_marker_proofs: Vec, /// VRF proofs for the labels of future marker entries pub future_marker_vrf_proofs: Vec>, /// Proof that future markers did not exist diff --git a/akd_core/src/utils.rs b/akd_core/src/utils.rs index 4f290cbe..a408c0d3 100644 --- a/akd_core/src/utils.rs +++ b/akd_core/src/utils.rs @@ -12,10 +12,53 @@ use alloc::vec::Vec; /// Retrieve log_2 of the marker version, referring to the exponent /// of the largest power of two that is at most the input version +/// +/// Note: This will panic if called on version = 0 pub fn get_marker_version_log2(version: u64) -> u64 { 64 - (version.leading_zeros() as u64) - 1 } +/// Return two (possibly empty) lists of marker versions, given +/// a start_version and end_version for the range of update proofs, along with +/// a final epoch. +/// +/// The first list contains versions which should be checked for membership, +/// and the second list contains versions which should be checked for non-membership. +/// +/// Roughly, the intervals should be organized as follows: +/// 1 --- { previous marker versions } --- [start_version, end_version] --- { future marker versions } --- epoch +/// +/// In this implementation, the set of previous marker versions consists of the largest power of 2 +/// that is at most start_version (or is empty if start_version is already a power of 2). The set of +/// future marker versions is as described in SEEMless: the consecutively increasing set of versions +/// from end_version until the next power of 2, and then all consecutive powers of 2 up until the +/// epoch. +/// +/// This will panic if start_version = 0 +pub fn get_marker_versions( + start_version: u64, + end_version: u64, + epoch: u64, +) -> (Vec, Vec) { + // Compute past marker versions + let mut past_marker_versions: Vec = Vec::new(); + let start_marker = 1 << get_marker_version_log2(start_version); + if start_marker < start_version { + past_marker_versions.push(start_marker); + } + + // Compute future marker versions + let next_marker_log2 = get_marker_version_log2(end_version) + 1; + let final_marker_log2 = get_marker_version_log2(epoch); + let mut future_marker_versions: Vec = + ((end_version + 1)..(1 << next_marker_log2)).collect(); + for i in next_marker_log2..(final_marker_log2 + 1) { + future_marker_versions.push(1 << i); + } + + (past_marker_versions, future_marker_versions) +} + /// Corresponds to the I2OSP() function from RFC8017, prepending the length of /// a byte array to the byte array (so that it is ready for serialization and hashing) /// @@ -111,3 +154,37 @@ macro_rules! test_config_sync { } }; } + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + + #[test] + fn test_get_marker_versions() { + assert_eq!( + (vec![], vec![6, 7, 8, 16, 32]), + get_marker_versions(1, 5, 33) + ); + + assert_eq!( + (vec![], vec![6, 7, 8, 16, 32]), + get_marker_versions(2, 5, 33) + ); + + assert_eq!( + (vec![2], vec![6, 7, 8, 16, 32]), + get_marker_versions(3, 5, 33) + ); + + assert_eq!( + (vec![4], vec![13, 14, 15, 16, 32, 64, 128]), + get_marker_versions(6, 12, 128) + ); + + assert_eq!( + (vec![4], vec![13, 14, 15, 16, 32, 64]), + get_marker_versions(6, 12, 127) + ); + } +} diff --git a/akd_core/src/verify/history.rs b/akd_core/src/verify/history.rs index f8deeff0..9eb7efc8 100644 --- a/akd_core/src/verify/history.rs +++ b/akd_core/src/verify/history.rs @@ -23,6 +23,23 @@ use alloc::string::ToString; #[cfg(feature = "nostd")] use alloc::vec::Vec; +/// The parameters that dictate how much of the history proof for the server to +/// return to the consumer (either a complete history, or some limited form). +#[derive(Copy, Clone)] +pub enum HistoryParams { + /// Returns a complete history for a label + Complete, + /// Returns up to the most recent N updates for a label + MostRecent(usize), +} + +impl Default for HistoryParams { + /// By default, we return a complete history + fn default() -> Self { + Self::Complete + } +} + /// Parameters for customizing how history proof verification proceeds #[derive(Copy, Clone)] pub enum HistoryVerificationParams { @@ -40,21 +57,12 @@ impl Default for HistoryVerificationParams { } } -/// Verifies a key history proof, given the corresponding sequence of hashes. -/// Returns a vector of whether the validity of a hash could be verified. -/// When false, the value <=> hash validity at the position could not be -/// verified because the value has been removed ("tombstoned") from the storage layer. -pub fn key_history_verify( - vrf_public_key: &[u8], - root_hash: Digest, +fn verify_with_history_params( current_epoch: u64, - akd_label: AkdLabel, - proof: HistoryProof, - params: HistoryVerificationParams, -) -> Result, VerificationError> { - let mut results = Vec::new(); - let mut last_version = 0; - + akd_label: &AkdLabel, + proof: &HistoryProof, + params: HistoryParams, +) -> Result<(Vec, Vec), VerificationError> { let num_proofs = proof.update_proofs.len(); // Make sure the update proofs are non-empty @@ -81,16 +89,118 @@ pub fn key_history_verify( } } + let mut start_version = proof.update_proofs[0].version; + let mut end_version = proof.update_proofs[0].version; + proof.update_proofs.iter().for_each(|update_proof| { + if update_proof.version < start_version { + start_version = update_proof.version; + } + if update_proof.version > end_version { + end_version = update_proof.version; + } + }); + + if start_version == 0 { + return Err(VerificationError::HistoryProof( + "Computed start version for the key history should be non-zero".to_string(), + )); + } + + if end_version > current_epoch { + return Err(VerificationError::HistoryProof( + "Computed end version for the key history should not exceed current epoch".to_string(), + )); + } + + match params { + HistoryParams::Complete => { + // Make sure the start version is 1 + if start_version != 1 { + return Err(VerificationError::HistoryProof(format!( + "Expected start version to be 1 given that it is a complete history, but got start_version = {}", + start_version + ))); + } + } + HistoryParams::MostRecent(recency) => + { + #[allow(clippy::comparison_chain)] + if num_proofs < recency { + if start_version != 1 { + return Err(VerificationError::HistoryProof(format!( + "Expected start version to be 1 given that the number of proofs returned was less than + the recency parameter, but got start_version = {}", + start_version + ))); + } + } else if num_proofs > recency { + return Err(VerificationError::HistoryProof(format!( + "Expected at most {} update proofs, but got {} of them", + recency, num_proofs + ))); + } + } + } + + let (past_marker_versions, future_marker_versions) = + crate::utils::get_marker_versions(start_version, end_version, current_epoch); + + // Perform checks for expected number of past marker proofs + if past_marker_versions.len() != proof.past_marker_vrf_proofs.len() { + return Err(VerificationError::HistoryProof(format!( + "Expected {} past marker proofs, but got {}", + past_marker_versions.len(), + proof.past_marker_vrf_proofs.len() + ))); + } + if proof.past_marker_vrf_proofs.len() != proof.existence_of_past_marker_proofs.len() { + return Err(VerificationError::HistoryProof(format!( + "Expected equal number of past marker proofs, but got ({}, {})", + proof.past_marker_vrf_proofs.len(), + proof.existence_of_past_marker_proofs.len() + ))); + } + + // Perform checks for expected number of future marker proofs + if future_marker_versions.len() != proof.future_marker_vrf_proofs.len() { + return Err(VerificationError::HistoryProof(format!( + "Expected {} future marker proofs, but got {}", + future_marker_versions.len(), + proof.future_marker_vrf_proofs.len() + ))); + } + if proof.future_marker_vrf_proofs.len() != proof.non_existence_of_future_marker_proofs.len() { + return Err(VerificationError::HistoryProof(format!( + "Expected equal number of future marker proofs, but got ({}, {})", + proof.future_marker_vrf_proofs.len(), + proof.non_existence_of_future_marker_proofs.len() + ))); + } + + Ok((past_marker_versions, future_marker_versions)) +} + +/// Verifies a key history proof, given the corresponding sequence of hashes. +/// Returns a vector of whether the validity of a hash could be verified. +/// When false, the value <=> hash validity at the position could not be +/// verified because the value has been removed ("tombstoned") from the storage layer. +pub fn key_history_verify( + vrf_public_key: &[u8], + root_hash: Digest, + current_epoch: u64, + akd_label: AkdLabel, + proof: HistoryProof, + params: HistoryParams, + verification_params: HistoryVerificationParams, +) -> Result, VerificationError> { + let mut results = Vec::new(); + + let (past_marker_versions, future_marker_versions) = + verify_with_history_params(current_epoch, &akd_label, &proof, params)?; + // Verify all individual update proofs let mut maybe_previous_update_epoch = None; for update_proof in proof.update_proofs.into_iter() { - // Get the highest version sent among the update proofs. - last_version = if update_proof.version > last_version { - update_proof.version - } else { - last_version - }; - if let Some(previous_update_epoch) = maybe_previous_update_epoch { // Make sure this this epoch is more than the previous epoch you checked if update_proof.epoch > previous_update_epoch { @@ -107,78 +217,31 @@ pub fn key_history_verify( vrf_public_key, update_proof, &akd_label, - params, + verification_params, )?; results.push(result); } - // Get the least and greatest marker entries for the current version - let next_marker = crate::utils::get_marker_version_log2(last_version) + 1; - let final_marker = crate::utils::get_marker_version_log2(current_epoch); - - // Perform checks for expected number of until-marker proofs - let expected_num_until_marker_proofs = (1 << next_marker) - last_version - 1; - if expected_num_until_marker_proofs != proof.until_marker_vrf_proofs.len() as u64 { - return Err(VerificationError::HistoryProof(format!( - "Expected {} until-marker proofs, but got {}", - expected_num_until_marker_proofs, - proof.until_marker_vrf_proofs.len() - ))); - } - if proof.until_marker_vrf_proofs.len() != proof.non_existence_until_marker_proofs.len() { - return Err(VerificationError::HistoryProof(format!( - "Expected equal number of until-marker proofs, but got ({}, {})", - proof.until_marker_vrf_proofs.len(), - proof.non_existence_until_marker_proofs.len() - ))); - } - - // Verify the non-existence of future entries, up to the next marker - for (i, version) in (last_version + 1..(1 << next_marker)).enumerate() { - verify_nonexistence::( + for (i, version) in past_marker_versions.iter().enumerate() { + verify_existence::( vrf_public_key, root_hash, &akd_label, VersionFreshness::Fresh, - version, - &proof.until_marker_vrf_proofs[i], - &proof.non_existence_until_marker_proofs[i], - ) - .map_err(|_| { - VerificationError::HistoryProof(format!( - "Non-existence of next few proof of label {:?} with version - {:?} at epoch {:?} does not verify", - &akd_label, version, current_epoch - )) - })?; - } - - // Perform checks for expected number of future-marker proofs - let expected_num_future_marker_proofs = final_marker + 1 - next_marker; - if expected_num_future_marker_proofs != proof.future_marker_vrf_proofs.len() as u64 { - return Err(VerificationError::HistoryProof(format!( - "Expected {} future-marker proofs, but got {}", - expected_num_future_marker_proofs, - proof.future_marker_vrf_proofs.len() - ))); - } - if proof.future_marker_vrf_proofs.len() != proof.non_existence_of_future_marker_proofs.len() { - return Err(VerificationError::HistoryProof(format!( - "Expected equal number of future-marker proofs, but got ({}, {})", - proof.future_marker_vrf_proofs.len(), - proof.non_existence_of_future_marker_proofs.len() - ))); + *version, + &proof.past_marker_vrf_proofs[i], + &proof.existence_of_past_marker_proofs[i], + )?; } // Verify the VRFs and non-membership proofs for future markers - for (i, pow) in (next_marker..final_marker + 1).enumerate() { - let version = 1 << pow; + for (i, version) in future_marker_versions.iter().enumerate() { verify_nonexistence::( vrf_public_key, root_hash, &akd_label, VersionFreshness::Fresh, - version, + *version, &proof.future_marker_vrf_proofs[i], &proof.non_existence_of_future_marker_proofs[i], ) diff --git a/examples/Cargo.toml b/examples/Cargo.toml index f290dfd8..7c9227a2 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "examples" -version = "0.11.0" +version = "0.12.0-pre.1" authors = ["akd contributors"] license = "MIT OR Apache-2.0" edition = "2021" diff --git a/examples/src/mysql_demo/tests/test_util.rs b/examples/src/mysql_demo/tests/test_util.rs index 74544607..5d6bda5d 100644 --- a/examples/src/mysql_demo/tests/test_util.rs +++ b/examples/src/mysql_demo/tests/test_util.rs @@ -333,6 +333,7 @@ pub(crate) async fn directory_test_suite< root_hash.epoch(), key, proof, + HistoryParams::default(), akd::HistoryVerificationParams::default(), ) { panic!("History proof failed to verify {:?}", error);