Extending limited history to properly check for the existence of past values

README.md

@@ -22,7 +22,7 @@ Installation
 Add the following line to the dependencies of your `Cargo.toml`:
 
 ```
-akd = "0.11"
+akd = "0.12.0-pre.1"
 ```
 
 ### Minimum Supported Rust Version

akd/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "akd"
-version = "0.11.0"
+version = "0.12.0-pre.1"
 authors = ["akd contributors"]
 description = "An implementation of an auditable key directory"
 license = "MIT OR Apache-2.0"
@@ -51,7 +51,7 @@ default = [
 
 [dependencies]
 ## Required dependencies ##
-akd_core = { version = "0.11.0", path = "../akd_core", default-features = false, features = [
+akd_core = { version = "0.12.0-pre.1", path = "../akd_core", default-features = false, features = [
     "vrf",
 ] }
 async-recursion = "1"

akd/src/append_only_zks.rs

@@ -1668,9 +1668,7 @@ mod tests {
             // Recursively traverse the tree and check that the sibling of each node is correct
             let root_node = TreeNode::get_from_storage(&db, &NodeKey(NodeLabel::root()), 1).await?;
             let mut nodes: Vec<TreeNode> = vec![root_node];
-            while !nodes.is_empty() {
-                let current_node = nodes.pop().unwrap();
-
+            while let Some(current_node) = nodes.pop() {
                 let left_child = current_node.get_child_node(&db, Direction::Left, 1).await?;
                 let right_child = current_node
                     .get_child_node(&db, Direction::Right, 1)

akd/src/directory.rs

@@ -16,11 +16,13 @@ use crate::storage::types::{DbRecord, ValueState, ValueStateRetrievalFlag};
 use crate::storage::Database;
 use crate::{
     AkdLabel, AkdValue, AppendOnlyProof, AzksElement, Digest, EpochHash, HistoryProof, LookupProof,
-    NonMembershipProof, UpdateProof,
+    MembershipProof, NonMembershipProof, UpdateProof,
 };
 
 use crate::VersionFreshness;
 use akd_core::configuration::Configuration;
+use akd_core::utils::get_marker_versions;
+use akd_core::verify::history::HistoryParams;
 use log::{error, info};
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
@@ -462,15 +464,6 @@ where
         user_data = match params {
             HistoryParams::Complete => user_data,
             HistoryParams::MostRecent(n) => user_data.into_iter().take(n).collect::<Vec<_>>(),
-            HistoryParams::SinceEpoch(epoch) => {
-                user_data = user_data
-                    .into_iter()
-                    .filter(|val| val.epoch >= epoch)
-                    .collect::<Vec<_>>();
-                // Ordering should be maintained after filtering, but let's re-sort just in case
-                user_data.sort_by(|a, b| b.epoch.cmp(&a.epoch));
-                user_data
-            }
         };
 
         if user_data.is_empty() {
@@ -496,61 +489,73 @@ where
         }
 
         let mut update_proofs = Vec::<UpdateProof>::new();
-        let mut last_version = 0;
+        let mut start_version = user_data[0].version;
+        let mut end_version = 0;
         for user_state in user_data {
             // Ignore states in storage that are ahead of current directory epoch
             if user_state.epoch <= current_epoch {
                 let proof = self
                     .create_single_update_proof(akd_label, &user_state)
                     .await?;
                 update_proofs.push(proof);
-                last_version = if user_state.version > last_version {
+                start_version = if user_state.version < start_version {
+                    user_state.version
+                } else {
+                    start_version
+                };
+                end_version = if user_state.version > end_version {
                     user_state.version
                 } else {
-                    last_version
+                    end_version
                 };
             }
         }
-        let next_marker = get_marker_version(last_version) + 1;
-        let final_marker = get_marker_version(current_epoch);
 
-        let mut until_marker_vrf_proofs = Vec::<Vec<u8>>::new();
-        let mut non_existence_until_marker_proofs = Vec::<NonMembershipProof>::new();
+        if start_version == 0 {
+            return Err(AkdError::Directory(DirectoryError::InvalidVersion(
+                "Computed start version for the key history should be non-zero".to_string(),
+            )));
+        }
+
+        let (past_marker_versions, future_marker_versions) =
+            get_marker_versions(start_version, end_version, current_epoch);
+
+        let mut past_marker_vrf_proofs = Vec::<Vec<u8>>::new();
+        let mut existence_of_past_marker_proofs = Vec::<MembershipProof>::new();
 
-        for ver in last_version + 1..(1 << next_marker) {
-            let label_for_ver = self
+        for version in past_marker_versions {
+            let node_label = self
                 .vrf
-                .get_node_label::<TC>(akd_label, VersionFreshness::Fresh, ver)
+                .get_node_label::<TC>(akd_label, VersionFreshness::Fresh, version)
                 .await?;
-            let non_existence_of_ver = current_azks
-                .get_non_membership_proof::<TC, _>(&self.storage, label_for_ver)
+            let existence_vrf = self
+                .vrf
+                .get_label_proof::<TC>(akd_label, VersionFreshness::Fresh, version)
                 .await?;
-            non_existence_until_marker_proofs.push(non_existence_of_ver);
-            until_marker_vrf_proofs.push(
-                self.vrf
-                    .get_label_proof::<TC>(akd_label, VersionFreshness::Fresh, ver)
-                    .await?
-                    .to_bytes()
-                    .to_vec(),
+            past_marker_vrf_proofs.push(existence_vrf.to_bytes().to_vec());
+            existence_of_past_marker_proofs.push(
+                current_azks
+                    .get_membership_proof::<TC, _>(&self.storage, node_label)
+                    .await?,
             );
         }
 
         let mut future_marker_vrf_proofs = Vec::<Vec<u8>>::new();
         let mut non_existence_of_future_marker_proofs = Vec::<NonMembershipProof>::new();
 
-        for marker_power in next_marker..final_marker + 1 {
-            let ver = 1 << marker_power;
-            let label_for_ver = self
+        for version in future_marker_versions {
+            let node_label = self
                 .vrf
-                .get_node_label::<TC>(akd_label, VersionFreshness::Fresh, ver)
+                .get_node_label::<TC>(akd_label, VersionFreshness::Fresh, version)
                 .await?;
-            let non_existence_of_ver = current_azks
-                .get_non_membership_proof::<TC, _>(&self.storage, label_for_ver)
-                .await?;
-            non_existence_of_future_marker_proofs.push(non_existence_of_ver);
+            non_existence_of_future_marker_proofs.push(
+                current_azks
+                    .get_non_membership_proof::<TC, _>(&self.storage, node_label)
+                    .await?,
+            );
             future_marker_vrf_proofs.push(
                 self.vrf
-                    .get_label_proof::<TC>(akd_label, VersionFreshness::Fresh, ver)
+                    .get_label_proof::<TC>(akd_label, VersionFreshness::Fresh, version)
                     .await?
                     .to_bytes()
                     .to_vec(),
@@ -565,8 +570,8 @@ where
         Ok((
             HistoryProof {
                 update_proofs,
-                until_marker_vrf_proofs,
-                non_existence_until_marker_proofs,
+                past_marker_vrf_proofs,
+                existence_of_past_marker_proofs,
                 future_marker_vrf_proofs,
                 non_existence_of_future_marker_proofs,
             },
@@ -855,25 +860,6 @@ where
     }
 }
 
-/// The parameters that dictate how much of the history proof to return to the consumer
-/// (either a complete history, or some limited form).
-#[derive(Copy, Clone)]
-pub enum HistoryParams {
-    /// Returns a complete history for a label
-    Complete,
-    /// Returns up to the most recent N updates for a label
-    MostRecent(usize),
-    /// Returns all updates since a specified epoch (inclusive)
-    SinceEpoch(u64),
-}
-
-impl Default for HistoryParams {
-    /// By default, we return a complete history
-    fn default() -> Self {
-        Self::Complete
-    }
-}
-
 /// Helpers
 
 pub(crate) fn get_marker_version(version: u64) -> u64 {

akd/src/errors.rs

@@ -230,6 +230,8 @@ pub enum DirectoryError {
     ReadOnlyDirectory(String),
     /// Publish
     Publish(String),
+    /// Detected an invalid version
+    InvalidVersion(String),
 }
 
 impl std::error::Error for DirectoryError {}
@@ -249,6 +251,9 @@ impl fmt::Display for DirectoryError {
             Self::Publish(inner_message) => {
                 write!(f, "Directory publish error: {inner_message}")
             }
+            Self::InvalidVersion(inner_message) => {
+                write!(f, "Invalid version error: {inner_message}")
+            }
         }
     }
 }

akd/src/lib.rs

@@ -198,6 +198,7 @@
 //! ```
 //!
 //! ## History Proofs
+//!
 //! As mentioned above, the security is defined by consistent views of the value for a key at any epoch.
 //! To this end, a server running an AKD needs to provide a way to check the history of a key. Note that in this case,
 //! the server is trusted for validating that a particular client is authorized to run a history check on a particular key.
@@ -245,6 +246,18 @@
 //! with respect to the latest root hash and public key, as follows. This function
 //! returns a list of values that have been associated with the specified entry, in
 //! reverse chronological order.
+//!
+//! Note that the same argument for [`HistoryParams`] that was used to generate
+//! the key history proof must also be used to verify the proof. Otherwise, verification
+//! may fail.
+//!
+//! We also use [`HistoryVerificationParams`] as an argument to the verification function,
+//! allowing the consumer to specify whether or not a "tombstoned" value should be
+//! accepted in place of a valid value for the corresponding entry. This is useful
+//! in scenarios where the consumer wishes to verify that a particular entry exists,
+//! but does not care about the value associated with it. The default behavior is to
+//! not accept tombstoned values, but [`HistoryVerificationParams::AllowMissingValues`] can
+//! be specified to enable this behavior.
 //! ```
 //! # use akd::storage::StorageManager;
 //! # use akd::storage::memory::AsyncInMemoryDatabase;
@@ -257,7 +270,7 @@
 //! # let storage_manager = StorageManager::new_no_cache(db);
 //! # let vrf = HardCodedAkdVRF{};
 //! # use akd::EpochHash;
-//! # use akd::HistoryParams;
+//! # use akd::{HistoryParams, HistoryVerificationParams};
 //! # use akd::{AkdLabel, AkdValue};
 //! # use akd::Digest;
 //! #
@@ -287,7 +300,8 @@
 //!     epoch_hash.epoch(),
 //!     AkdLabel::from("first entry"),
 //!     history_proof,
-//!     akd::HistoryVerificationParams::default(),
+//!     HistoryParams::default(),
+//!     HistoryVerificationParams::default(),
 //! ).expect("Could not verify history");
 //!
 //! assert_eq!(
@@ -476,7 +490,8 @@ pub mod tree_node;
 pub mod local_auditing;
 
 pub use akd_core::{
-    configuration, configuration::*, ecvrf, hash, hash::Digest, proto, types::*, verify, ARITY,
+    configuration, configuration::*, ecvrf, hash, hash::Digest, proto, types::*, verify,
+    verify::history::HistoryParams, ARITY,
 };
 
 #[macro_use]
@@ -485,7 +500,7 @@ mod utils;
 // ========== Type re-exports which are commonly used ========== //
 pub use append_only_zks::Azks;
 pub use client::HistoryVerificationParams;
-pub use directory::{Directory, HistoryParams};
+pub use directory::Directory;
 pub use helper_structs::EpochHash;
 
 // ========== Constants and type aliases ========== //