From 9367f0ebc7dde560dfaf47491b1b32a5bb2eff51 Mon Sep 17 00:00:00 2001 From: Sean Lawlor Date: Mon, 13 Dec 2021 15:54:09 -0500 Subject: [PATCH 1/3] Remove the epochs vector from the history tree node since the vast majority of operations only require the birth and current epochs. (Only audit & key history require the full list of epochs a node was mutated in) Resolves #112 --- akd/src/history_tree_node.rs | 94 +++++++++++++++++--------- akd/src/storage/memory.rs | 74 +++++++++++++++++++++ akd/src/storage/mod.rs | 14 +++- akd/src/storage/tests.rs | 6 +- akd/src/tests.rs | 25 +++++-- akd_mysql/src/mysql.rs | 123 ++++++++++++++++++++--------------- 6 files changed, 244 insertions(+), 92 deletions(-) diff --git a/akd/src/history_tree_node.rs b/akd/src/history_tree_node.rs index 6e0a8ad8..e587a0fa 100644 --- a/akd/src/history_tree_node.rs +++ b/akd/src/history_tree_node.rs @@ -57,8 +57,10 @@ pub struct HistoryTreeNode { pub label: NodeLabel, /// The location of this node in the storage pub location: u64, - /// The epochs this node was updated - pub epochs: Vec, + /// The last epoch this node was updated in + pub last_epoch: u64, + /// The epoch that this node was birthed in + pub birth_epoch: u64, /// The location of this node's parent pub parent: u64, // The root node is marked its own parent. /// The type of node: leaf root or interior. @@ -106,7 +108,8 @@ impl Clone for HistoryTreeNode { Self { label: self.label, location: self.location, - epochs: self.epochs.clone(), + last_epoch: self.last_epoch, + birth_epoch: self.birth_epoch, parent: self.parent, node_type: self.node_type, } @@ -114,11 +117,12 @@ impl Clone for HistoryTreeNode { } impl HistoryTreeNode { - fn new(label: NodeLabel, location: u64, parent: u64, node_type: NodeType) -> Self { + fn new(label: NodeLabel, location: u64, parent: u64, node_type: NodeType, birth_epoch: u64) -> Self { HistoryTreeNode { label, location, - epochs: vec![], + birth_epoch, + last_epoch: birth_epoch, parent, // Root node is its own parent node_type, } @@ -248,8 +252,8 @@ impl HistoryTreeNode { new_node_location, parent.location, NodeType::Interior, + epoch, ); - new_node.epochs.push(epoch); new_node.write_to_storage(storage).await?; set_state_map( storage, @@ -470,11 +474,11 @@ impl HistoryTreeNode { match self.get_latest_epoch() { Ok(latest) => { if latest != epoch { - self.epochs.push(epoch); + self.last_epoch = epoch; } } Err(_) => { - self.epochs.push(epoch); + self.last_epoch = epoch; } } self.write_to_storage(storage).await?; @@ -578,7 +582,7 @@ impl HistoryTreeNode { } pub(crate) fn get_birth_epoch(&self) -> u64 { - self.epochs[0] + self.birth_epoch } // gets the direction of node, i.e. if it's a left @@ -623,14 +627,31 @@ impl HistoryTreeNode { if self.get_birth_epoch() > epoch { Err(HistoryTreeNodeError::NoChildInTreeAtEpoch(epoch, dir)) } else { - let mut chosen_ep = self.get_birth_epoch(); - for existing_ep in &self.epochs { - if *existing_ep <= epoch && *existing_ep > chosen_ep { - chosen_ep = *existing_ep; + let chosen_ep = { + if self.last_epoch <= epoch { + // the "last" updated epoch is <= epoch, so it is + // the last valid state at this epoch + Some(self.last_epoch) + } else if self.birth_epoch == epoch { + // we're looking at the state at the birth epoch + Some(self.birth_epoch) + } else { + // Indeterminate, we are somewhere above the + // birth epoch but we're less than the "last" epoch. + // db query is necessary + None } + }; + + if let Some(ep) = chosen_ep { + self.get_child_at_existing_epoch::<_, H>(storage, ep, direction) + .await + } else { + let target_ep = storage.get_epoch_lte_epoch(self.label, epoch).await?; + // DB query for the state <= this epoch value + self.get_child_at_existing_epoch::<_, H>(storage, target_ep, direction) + .await } - self.get_child_at_existing_epoch::<_, H>(storage, chosen_ep, direction) - .await } } } @@ -658,13 +679,28 @@ impl HistoryTreeNode { if self.get_birth_epoch() > epoch { Err(HistoryTreeNodeError::NodeDidNotExistAtEp(self.label, epoch)) } else { - let mut chosen_ep = self.get_birth_epoch(); - for existing_ep in &self.epochs { - if *existing_ep <= epoch { - chosen_ep = *existing_ep; + let chosen_ep = { + if self.last_epoch <= epoch { + // the "last" updated epoch is <= epoch, so it is + // the last valid state at this epoch + Some(self.last_epoch) + } else if self.birth_epoch == epoch { + // we're looking at the state at the birth epoch + Some(self.birth_epoch) + } else { + // Indeterminate, we are somewhere above the + // birth epoch but we're less than the "last" epoch. + // db query is necessary + None } + }; + if let Some(ep) = chosen_ep { + self.get_state_at_existing_epoch(storage, ep).await + } else { + let target_ep = storage.get_epoch_lte_epoch(self.label, epoch).await?; + // DB query for the state <= this epoch value + self.get_state_at_existing_epoch(storage, target_ep).await } - self.get_state_at_existing_epoch(storage, chosen_ep).await } } @@ -681,12 +717,7 @@ impl HistoryTreeNode { /* Functions for compression-related operations */ pub(crate) fn get_latest_epoch(&self) -> Result { - match self.epochs.len() { - 0 => Err(HistoryTreeNodeError::NodeCreatedWithoutEpochs( - self.label.get_val(), - )), - n => Ok(self.epochs[n - 1]), - } + Ok(self.last_epoch) } /////// Helpers ///////// @@ -747,9 +778,10 @@ pub(crate) async fn get_empty_root( storage: &S, ep: Option, ) -> Result { - let mut node = HistoryTreeNode::new(NodeLabel::new(0u64, 0u32), 0, 0, NodeType::Root); + let mut node = HistoryTreeNode::new(NodeLabel::new(0u64, 0u32), 0, 0, NodeType::Root, 0u64); if let Some(epoch) = ep { - node.epochs.push(epoch); + node.birth_epoch = epoch; + node.last_epoch = epoch; let new_state: HistoryNodeState = HistoryNodeState::new::(NodeStateKey(node.label, epoch)); set_state_map(storage, new_state).await?; @@ -769,7 +801,8 @@ pub(crate) async fn get_leaf_node( let node = HistoryTreeNode { label, location, - epochs: vec![birth_epoch], + birth_epoch, + last_epoch: birth_epoch, parent, node_type: NodeType::Leaf, }; @@ -794,7 +827,8 @@ pub(crate) async fn get_leaf_node_without_hashing Result { + let ids = (0..=epoch_in_question) + .map(|epoch| crate::node_state::NodeStateKey(node_label, epoch)) + .collect::>(); + let data = self + .batch_get::(ids) + .await?; + let mut epochs = data + .into_iter() + .map(|item| { + if let DbRecord::HistoryNodeState(state) = item { + state.key.1 + } else { + 0u64 + } + }) + .collect::>(); + // reverse sort + epochs.sort_unstable_by(|a, b| b.cmp(a)); + + // move through the epochs from largest to smallest, first one that's <= ```epoch_in_question``` + // and Bob's your uncle + for item in epochs { + if item <= epoch_in_question { + return Ok(item); + } + } + Err(StorageError::GetError(format!( + "Node (val: {}, len: {}) did not exist <= epoch {}", + node_label.val, node_label.len, epoch_in_question + ))) + } } // ===== In-Memory database w/caching (for benchmarking) ==== // @@ -606,4 +643,41 @@ impl Storage for AsyncInMemoryDbWithCache { } Ok(map) } + + async fn get_epoch_lte_epoch( + &self, + node_label: crate::node_state::NodeLabel, + epoch_in_question: u64, + ) -> Result { + let ids = (0..epoch_in_question) + .map(|epoch| crate::node_state::NodeStateKey(node_label, epoch)) + .collect::>(); + let data = self + .batch_get::(ids) + .await?; + let mut epochs = data + .into_iter() + .map(|item| { + if let DbRecord::HistoryNodeState(state) = item { + state.key.1 + } else { + 0u64 + } + }) + .collect::>(); + // reverse sort + epochs.sort_unstable_by(|a, b| b.cmp(a)); + + // move through the epochs from largest to smallest, first one that's <= ```epoch_in_question``` + // and Bob's your uncle + for item in epochs { + if item <= epoch_in_question { + return Ok(item); + } + } + Err(StorageError::GetError(format!( + "Node (val: {}, len: {}) did not exist <= epoch {}", + node_label.val, node_label.len, epoch_in_question + ))) + } } diff --git a/akd/src/storage/mod.rs b/akd/src/storage/mod.rs index 7aa21a67..21773705 100644 --- a/akd/src/storage/mod.rs +++ b/akd/src/storage/mod.rs @@ -79,6 +79,14 @@ pub trait Storage: Clone { /// Retrieve a stored record from the data layer async fn get(&self, id: St::Key) -> Result; + /// Retrieve the last epoch <= ```epoch_in_question``` where the node with ```node_key``` + /// was edited + async fn get_epoch_lte_epoch( + &self, + node_label: crate::node_state::NodeLabel, + epoch_in_question: u64, + ) -> Result; + /// Retrieve a batch of records by id async fn batch_get( &self, @@ -138,7 +146,8 @@ pub trait Storage: Clone { label_val: u64, label_len: u32, location: u64, - epochs: Vec, + birth_epoch: u64, + last_epoch: u64, parent: u64, node_type: u8, ) -> crate::history_tree_node::HistoryTreeNode { @@ -148,7 +157,8 @@ pub trait Storage: Clone { len: label_len, }, location, - epochs, + birth_epoch, + last_epoch, parent, node_type: crate::history_tree_node::NodeType::from_u8(node_type), } diff --git a/akd/src/storage/tests.rs b/akd/src/storage/tests.rs index be77214a..12ee7ba4 100644 --- a/akd/src/storage/tests.rs +++ b/akd/src/storage/tests.rs @@ -78,7 +78,8 @@ async fn test_get_and_set_item(storage: &Ns) { let node = HistoryTreeNode { label: NodeLabel { val: 13, len: 1 }, location: 234, - epochs: vec![123u64, 234u64, 345u64], + birth_epoch: 123, + last_epoch: 234, parent: 1, node_type: NodeType::Leaf, }; @@ -100,7 +101,8 @@ async fn test_get_and_set_item(storage: &Ns) { assert_eq!(got_node.location, node.location); assert_eq!(got_node.parent, node.parent); assert_eq!(got_node.node_type, node.node_type); - assert_eq!(got_node.epochs, node.epochs); + assert_eq!(got_node.birth_epoch, node.birth_epoch); + assert_eq!(got_node.last_epoch, node.last_epoch); } else { panic!("Failed to retrieve History Tree Node"); } diff --git a/akd/src/tests.rs b/akd/src/tests.rs index 039683c7..8be3510c 100644 --- a/akd/src/tests.rs +++ b/akd/src/tests.rs @@ -53,7 +53,10 @@ async fn test_set_child_without_hash_at_root() -> Result<(), HistoryTreeNodeErro root.get_latest_epoch().unwrap_or(0) == 1, "Latest epochs don't match!" ); - assert!(root.epochs.len() == 1, "Ask yourself some pressing questions, such as: Why are there random extra epochs in the root?"); + assert!( + root.birth_epoch == root.last_epoch, + "How would the last epoch be different from the birth epoch without an update?" + ); Ok(()) } @@ -103,7 +106,10 @@ async fn test_set_children_without_hash_at_root() -> Result<(), HistoryTreeNodeE } let latest_ep = root.get_latest_epoch(); assert!(latest_ep.unwrap_or(0) == 1, "Latest epochs don't match!"); - assert!(root.epochs.len() == 1, "Ask yourself some pressing questions, such as: Why are there random extra epochs in the root?"); + assert!( + root.birth_epoch == root.last_epoch, + "How would the last epoch be different from the birth epoch without an update?" + ); Ok(()) } @@ -173,7 +179,10 @@ async fn test_set_children_without_hash_multiple_at_root() -> Result<(), History } let latest_ep = root.get_latest_epoch(); assert!(latest_ep.unwrap_or(0) == 2, "Latest epochs don't match!"); - assert!(root.epochs.len() == 2, "Ask yourself some pressing questions, such as: Why are there random extra epochs in the root?"); + assert!( + root.birth_epoch < root.last_epoch, + "How is the last epoch not higher than the birth epoch after an udpate?" + ); Ok(()) } @@ -242,7 +251,10 @@ async fn test_get_child_at_existing_epoch_multiple_at_root() -> Result<(), Histo } let latest_ep = root.get_latest_epoch(); assert!(latest_ep.unwrap_or(0) == 2, "Latest epochs don't match!"); - assert!(root.epochs.len() == 2, "Ask yourself some pressing questions, such as: Why are there random extra epochs in the root?"); + assert!( + root.birth_epoch < root.last_epoch, + "How is the last epoch not higher than the birth epoch after an udpate?" + ); Ok(()) } @@ -316,7 +328,10 @@ pub async fn test_get_child_at_epoch_at_root() -> Result<(), HistoryTreeNodeErro } let latest_ep = root.get_latest_epoch(); assert!(latest_ep.unwrap_or(0) == 4, "Latest epochs don't match!"); - assert!(root.epochs.len() == 3, "Ask yourself some pressing questions, such as: Why are there random extra epochs in the root?"); + assert!( + root.birth_epoch < root.last_epoch, + "How is the last epoch not higher than the birth epoch after an udpate?" + ); Ok(()) } diff --git a/akd_mysql/src/mysql.rs b/akd_mysql/src/mysql.rs index 8d5bd0d7..2f49023e 100644 --- a/akd_mysql/src/mysql.rs +++ b/akd_mysql/src/mysql.rs @@ -41,7 +41,7 @@ const SQL_RECONNECTION_DELAY_SECS: u64 = 5; const SELECT_AZKS_DATA: &str = "`epoch`, `num_nodes`"; const SELECT_HISTORY_TREE_NODE_DATA: &str = - "`location`, `label_len`, `label_val`, `epochs`, `parent`, `node_type`"; + "`location`, `label_len`, `label_val`, `birth_epoch`, `last_epoch`, `parent`, `node_type`"; const SELECT_HISTORY_NODE_STATE_DATA: &str = "`label_len`, `label_val`, `epoch`, `value`, `child_states`"; const SELECT_USER_DATA: &str = @@ -337,7 +337,8 @@ impl<'a> AsyncMySqlDatabase { let command = "CREATE TABLE IF NOT EXISTS `".to_owned() + TABLE_HISTORY_TREE_NODES + "` (`location` BIGINT UNSIGNED NOT NULL, `label_len` INT UNSIGNED NOT NULL," - + " `label_val` BIGINT UNSIGNED NOT NULL, `epochs` VARBINARY(2000)," + + " `label_val` BIGINT UNSIGNED NOT NULL, `birth_epoch` BIGINT UNSIGNED NOT NULL," + + " `last_epoch` BIGINT UNSIGNED NOT NULL," + " `parent` BIGINT UNSIGNED NOT NULL, `node_type` SMALLINT UNSIGNED NOT NULL," + " PRIMARY KEY (`location`))"; tx.query_drop(command).await?; @@ -1436,6 +1437,52 @@ impl Storage for AsyncMySqlDatabase { Err(code) => Err(StorageError::GetError(code.to_string())), } } + + async fn get_epoch_lte_epoch( + &self, + node_label: akd::node_state::NodeLabel, + epoch_in_question: u64, + ) -> core::result::Result { + *(self.num_reads.write().await) += 1; + + let result = async { + let tic = Instant::now(); + + let mut conn = self.get_connection().await?; + + let statement = format!("SELECT `epoch` FROM {} WHERE `label_len` = :len AND `label_val` = :val AND `epoch` <= :epoch ORDER BY `epoch` DESC", TABLE_HISTORY_NODE_STATES); + let out = conn + .exec_first( + statement, + params! { + "len" => node_label.len, + "val" => node_label.val, + "epoch" => epoch_in_question, + }, + ) + .await; + + let toc = Instant::now() - tic; + *(self.time_read.write().await) += toc; + + let result = self.check_for_infra_error(out)?; + + match result { + Some(r) => Ok::<_, MySqlError>(r), + None => Ok::<_, MySqlError>(u64::MAX), + } + }; + + debug!("END MySQL get epoch LTE epoch"); + match result.await { + Ok(u64::MAX) => Err(StorageError::GetError(format!( + "Node (val: {}, len: {}) did not exist <= epoch {}", + node_label.val, node_label.len, epoch_in_question + ))), + Ok(ep) => Ok(ep), + Err(code) => Err(StorageError::GetError(code.to_string())), + } + } } /* Generic data structure handling for MySQL */ @@ -1467,9 +1514,6 @@ trait MySqlStorable { fn from_row(row: &mut mysql_async::Row) -> core::result::Result where Self: std::marker::Sized; - - fn serialize_epochs(epochs: &[u64]) -> Vec; - fn deserialize_epochs(bin: &[u8]) -> Option>; } impl MySqlStorable for DbRecord { @@ -1477,7 +1521,7 @@ impl MySqlStorable for DbRecord { match &self { DbRecord::Azks(_) => format!("INSERT INTO `{}` (`key`, {}) VALUES (:key, :epoch, :num_nodes) ON DUPLICATE KEY UPDATE `epoch` = :epoch, `num_nodes` = :num_nodes", TABLE_AZKS, SELECT_AZKS_DATA), DbRecord::HistoryNodeState(_) => format!("INSERT INTO `{}` ({}) VALUES (:label_len, :label_val, :epoch, :value, :child_states) ON DUPLICATE KEY UPDATE `value` = :value, `child_states` = :child_states", TABLE_HISTORY_NODE_STATES, SELECT_HISTORY_NODE_STATE_DATA), - DbRecord::HistoryTreeNode(_) => format!("INSERT INTO `{}` ({}) VALUES (:location, :label_len, :label_val, :epochs, :parent, :node_type) ON DUPLICATE KEY UPDATE `label_len` = :label_len, `label_val` = :label_val, `epochs` = :epochs, `parent` = :parent, `node_type` = :node_type", TABLE_HISTORY_TREE_NODES, SELECT_HISTORY_TREE_NODE_DATA), + DbRecord::HistoryTreeNode(_) => format!("INSERT INTO `{}` ({}) VALUES (:location, :label_len, :label_val, :birth_epoch, :last_epoch, :parent, :node_type) ON DUPLICATE KEY UPDATE `label_len` = :label_len, `label_val` = :label_val, `birth_epoch` = :birth_epoch, `last_epoch` = :last_epoch, `parent` = :parent, `node_type` = :node_type", TABLE_HISTORY_TREE_NODES, SELECT_HISTORY_TREE_NODE_DATA), DbRecord::ValueState(_) => format!("INSERT INTO `{}` ({}) VALUES (:username, :epoch, :version, :node_label_val, :node_label_len, :data)", TABLE_USER, SELECT_USER_DATA), } } @@ -1493,8 +1537,7 @@ impl MySqlStorable for DbRecord { params! { "label_len" => id.0.len, "label_val" => id.0.val, "epoch" => id.1, "value" => state.value.clone(), "child_states" => bin_data } } DbRecord::HistoryTreeNode(node) => { - let bin_data = DbRecord::serialize_epochs(&node.epochs); - params! { "location" => node.location, "label_len" => node.label.len, "label_val" => node.label.val, "epochs" => bin_data, "parent" => node.parent, "node_type" => node.node_type as u8 } + params! { "location" => node.location, "label_len" => node.label.len, "label_val" => node.label.val, "birth_epoch" => node.birth_epoch, "last_epoch" => node.last_epoch, "parent" => node.parent, "node_type" => node.node_type as u8 } } DbRecord::ValueState(state) => { params! { "username" => state.get_id().0, "epoch" => state.epoch, "version" => state.version, "node_label_len" => state.label.len, "node_label_val" => state.label.val, "data" => state.plaintext_val.0.clone() } @@ -1513,7 +1556,7 @@ impl MySqlStorable for DbRecord { ); } StorageType::HistoryTreeNode => { - parts = format!("{}(:location{}, :label_len{}, :label_val{}, :epochs{}, :parent{}, :node_type{})", parts, i, i, i, i, i, i); + parts = format!("{}(:location{}, :label_len{}, :label_val{}, :birth_epoch{}, :last_epoch{}, :parent{}, :node_type{})", parts, i, i, i, i, i, i, i); } StorageType::ValueState => { parts = format!("{}(:username{}, :epoch{}, :version{}, :node_label_val{}, :node_label_len{}, :data{})", parts, i, i, i, i, i, i); @@ -1531,7 +1574,7 @@ impl MySqlStorable for DbRecord { match St::data_type() { StorageType::Azks => format!("INSERT INTO `{}` (`key`, {}) VALUES (:key, :epoch, :num_nodes) as new ON DUPLICATE KEY UPDATE `epoch` = new.epoch, `num_nodes` = new.num_nodes", TABLE_AZKS, SELECT_AZKS_DATA), StorageType::HistoryNodeState => format!("INSERT INTO `{}` ({}) VALUES {} as new ON DUPLICATE KEY UPDATE `value` = new.value, `child_states` = new.child_states", TABLE_HISTORY_NODE_STATES, SELECT_HISTORY_NODE_STATE_DATA, parts), - StorageType::HistoryTreeNode => format!("INSERT INTO `{}` ({}) VALUES {} as new ON DUPLICATE KEY UPDATE `label_len` = new.label_len, `label_val` = new.label_val, `epochs` = new.epochs, `parent` = new.parent, `node_type` = new.node_type", TABLE_HISTORY_TREE_NODES, SELECT_HISTORY_TREE_NODE_DATA, parts), + StorageType::HistoryTreeNode => format!("INSERT INTO `{}` ({}) VALUES {} as new ON DUPLICATE KEY UPDATE `label_len` = new.label_len, `label_val` = new.label_val, `birth_epoch` = new.birth_epoch, `last_epoch` = new.last_epoch, `parent` = new.parent, `node_type` = new.node_type", TABLE_HISTORY_TREE_NODES, SELECT_HISTORY_TREE_NODE_DATA, parts), StorageType::ValueState => format!("INSERT INTO `{}` ({}) VALUES {}", TABLE_USER, SELECT_USER_DATA, parts), } } @@ -1560,12 +1603,12 @@ impl MySqlStorable for DbRecord { ] } DbRecord::HistoryTreeNode(node) => { - let bin_data = DbRecord::serialize_epochs(&node.epochs); vec![ (format!("location{}", idx), Value::from(node.location)), (format!("label_len{}", idx), Value::from(node.label.len)), (format!("label_val{}", idx), Value::from(node.label.val)), - (format!("epochs{}", idx), Value::from(bin_data)), + (format!("birth_epoch{}", idx), Value::from(node.birth_epoch)), + (format!("last_epoch{}", idx), Value::from(node.last_epoch)), (format!("parent{}", idx), Value::from(node.parent)), ( format!("node_type{}", idx), @@ -1709,7 +1752,7 @@ impl MySqlStorable for DbRecord { } StorageType::HistoryTreeNode => { format!( - "SELECT a.`location`, a.`label_len`, a.`label_val`, a.`epochs`, a.`parent`, a.`node_type` FROM `{}` a INNER JOIN {} ids ON ids.`location` = a.`location`", + "SELECT a.`location`, a.`label_len`, a.`label_val`, a.`birth_epoch`, a.`last_epoch`, a.`parent`, a.`node_type` FROM `{}` a INNER JOIN {} ids ON ids.`location` = a.`location`", TABLE_HISTORY_TREE_NODES, TEMP_IDS_TABLE ) @@ -1866,12 +1909,13 @@ impl MySqlStorable for DbRecord { } } StorageType::HistoryTreeNode => { - // `location`, `label_len`, `label_val`, `epochs`, `parent`, `node_type` + // `location`, `label_len`, `label_val`, `birth_epoch`, `last_epoch`, `parent`, `node_type` if let ( Some(Ok(location)), Some(Ok(label_len)), Some(Ok(label_val)), - Some(Ok(epochs)), + Some(Ok(birth_epoch)), + Some(Ok(last_epoch)), Some(Ok(parent)), Some(Ok(node_type)), ) = ( @@ -1881,21 +1925,18 @@ impl MySqlStorable for DbRecord { row.take_opt(3), row.take_opt(4), row.take_opt(5), + row.take_opt(6), ) { - let bin_vec: Vec = epochs; - if let Some(decoded_epochs) = DbRecord::deserialize_epochs(&bin_vec) { - let node = AsyncMySqlDatabase::build_history_tree_node( - label_val, - label_len, - location, - decoded_epochs, - parent, - node_type, - ); - return Ok(DbRecord::HistoryTreeNode(node)); - } else { - return Err(MySqlError::from("Deserialization of epochs failed")); - } + let node = AsyncMySqlDatabase::build_history_tree_node( + label_val, + label_len, + location, + birth_epoch, + last_epoch, + parent, + node_type, + ); + return Ok(DbRecord::HistoryTreeNode(node)); } } StorageType::ValueState => { @@ -1931,30 +1972,6 @@ impl MySqlStorable for DbRecord { let err = MySqlError::Driver(mysql_async::DriverError::FromRow { row: row.clone() }); Err(err) } - - fn serialize_epochs(epochs: &[u64]) -> Vec { - let mut results = vec![]; - for item in epochs { - let bytes = (*item).to_be_bytes(); - results.extend_from_slice(&bytes); - } - results - } - - fn deserialize_epochs(bin: &[u8]) -> Option> { - if bin.len() % 8 == 0 { - // modulo 8 means that we have proper length byte arrays which can be decoded into u64's - let mut results = vec![]; - for chunk in bin.chunks(8) { - let mut a: [u8; 8] = Default::default(); - a.copy_from_slice(chunk); - results.push(u64::from_be_bytes(a)); - } - Some(results) - } else { - None - } - } } trait StorageErrorWrappable { From 3fc0898cd08597b18fd54feb2f9fa458d62df412 Mon Sep 17 00:00:00 2001 From: Sean Lawlor Date: Tue, 14 Dec 2021 09:12:32 -0500 Subject: [PATCH 2/3] Add mysql limit to limit select as small as possible --- akd_mysql/src/mysql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/akd_mysql/src/mysql.rs b/akd_mysql/src/mysql.rs index 853728bf..0cd3c389 100644 --- a/akd_mysql/src/mysql.rs +++ b/akd_mysql/src/mysql.rs @@ -1448,7 +1448,7 @@ impl Storage for AsyncMySqlDatabase { let mut conn = self.get_connection().await?; - let statement = format!("SELECT `epoch` FROM {} WHERE `label_len` = :len AND `label_val` = :val AND `epoch` <= :epoch ORDER BY `epoch` DESC", TABLE_HISTORY_NODE_STATES); + let statement = format!("SELECT `epoch` FROM {} WHERE `label_len` = :len AND `label_val` = :val AND `epoch` <= :epoch ORDER BY `epoch` DESC LIMIT 1", TABLE_HISTORY_NODE_STATES); let out = conn .exec_first( statement, From bf9bf15c3e9bcd3f50bc44dcc38397d3ab20efec Mon Sep 17 00:00:00 2001 From: Sean Lawlor Date: Tue, 14 Dec 2021 09:18:50 -0500 Subject: [PATCH 3/3] Version bump to 0.3.6 --- akd/Cargo.toml | 2 +- akd_mysql/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/akd/Cargo.toml b/akd/Cargo.toml index 0388e737..2d013631 100644 --- a/akd/Cargo.toml +++ b/akd/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "akd" -version = "0.3.5" +version = "0.3.6" authors = ["Harjasleen Malvai ", "Kevin Lewi ", "Sean Lawlor "] description = "An implementation of an auditable key directory" license = "MIT OR Apache-2.0" diff --git a/akd_mysql/Cargo.toml b/akd_mysql/Cargo.toml index 7aa39173..c62c7bd7 100644 --- a/akd_mysql/Cargo.toml +++ b/akd_mysql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "akd_mysql" -version = "0.3.5" +version = "0.3.6" authors = ["Harjasleen Malvai ", "Kevin Lewi ", "Sean Lawlor "] description = "A MySQL storage layer implementation for an auditable key directory (AKD)" license = "MIT OR Apache-2.0"