,
}
-impl FollowerConfigBuilder {
- /// Sets the size of the chain updates buffer used by the [`Follower`].
+impl ChainFollower {
+ /// Follow a blockchain.
///
/// # Arguments
///
- /// * `chain_update_buffer_size`: Size of the chain updates buffer.
- #[must_use]
- pub fn chain_update_buffer_size(mut self, block_buffer_size: usize) -> Self {
- self.chain_update_buffer_size = block_buffer_size;
- self
- }
-
- /// Sets the point at which the follower will start following from.
+ /// * `chain` - The blockchain network to follow.
+ /// * `start` - The point or tip to start following from (inclusive).
+ /// * `end` - The point or tip to stop following from (inclusive).
///
- /// # Arguments
+ /// # Returns
///
- /// * `from`: Sync starting point.
- #[must_use]
- pub fn follow_from(mut self, from: P) -> Self
- where P: Into {
- self.follow_from = from.into();
- self
- }
-
- /// Sets the path of the Mithril snapshot the [`Follower`] will use.
+ /// The Chain Follower that will return blocks in the requested range.
///
- /// # Arguments
+ /// # Notes
///
- /// * `path`: Mithril snapshot path.
- #[must_use]
- pub fn mithril_snapshot_path(mut self, path: PathBuf) -> Self {
- self.mithril_snapshot_path = Some(path);
- self
- }
-
- /// Builds a [`FollowerConfig`].
+ /// IF end < start, the follower will immediately yield no blocks.
+ /// IF end is TIP, then the follower will continue to follow even when TIP is reached.
+ /// Otherwise only blocks in the request range will be returned.
+ ///
+ /// Also, UNLIKE the blockchain itself, the only relevant information is the Slot#.
+ /// The Block hash is not considered.
+ /// If start is not an exact Slot#, then the NEXT Slot immediately following will be
+ /// the first block returned.
+ /// If the end is also not an exact Slot# with a block, then the last block will be
+ /// the one immediately proceeding it.
+ ///
+ /// To ONLY follow from TIP, set BOTH start and end to TIP.
#[must_use]
- pub fn build(self) -> FollowerConfig {
- FollowerConfig {
- chain_update_buffer_size: self.chain_update_buffer_size,
- follow_from: self.follow_from,
- mithril_snapshot_path: self.mithril_snapshot_path,
+ pub async fn new(chain: Network, start: Point, end: Point) -> Self {
+ let rx = get_chain_update_rx_queue(chain).await;
+
+ ChainFollower {
+ chain,
+ end,
+ previous: UNKNOWN_POINT,
+ current: start,
+ fork: 1, // This is correct, because Mithril is Fork 0.
+ snapshot: MithrilSnapshot::new(chain),
+ mithril_follower: None,
+ mithril_tip: None,
+ sync_updates: rx,
}
}
-}
-/// Configuration for the Cardano chain follower.
-#[derive(Clone)]
-pub struct FollowerConfig {
- /// Configured chain update buffer size.
- pub chain_update_buffer_size: usize,
- /// Where to start following from.
- pub follow_from: PointOrTip,
- /// Path to the Mithril snapshot the follower should use.
- pub mithril_snapshot_path: Option,
-}
+ /// If we can, get the next update from the mithril snapshot.
+ async fn next_from_mithril(&mut self) -> Option {
+ let current_mithril_tip = latest_mithril_snapshot_id(self.chain).tip();
-/// Information used to connect to a client.
-#[derive(Clone)]
-struct ClientConnectInfo {
- /// Node's address
- address: String,
- /// Network magic
- network: Network,
-}
+ if current_mithril_tip > self.current {
+ if self.mithril_follower.is_none() {
+ self.mithril_follower = self
+ .snapshot
+ .try_read_blocks_from_point(&self.current)
+ .await;
+ }
-/// Handler for receiving the read block response from the client.
-pub struct ReadBlock(tokio::task::JoinHandle>);
-
-impl Future for ReadBlock {
- type Output = Result;
-
- fn poll(
- mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>,
- ) -> std::task::Poll {
- let p = &mut self.0;
- // Using tokio pin instead of, e.g., pin-project because we use tokio as the async runtime
- // lib for this crate.
- tokio::pin!(p);
-
- match p.poll(cx) {
- std::task::Poll::Ready(res) => {
- match res {
- Ok(res) => std::task::Poll::Ready(res),
- Err(_) => std::task::Poll::Ready(Err(Error::InternalError)),
+ if let Some(follower) = self.mithril_follower.as_mut() {
+ if let Some(next) = follower.next().await {
+ // debug!("Pre Previous update 3 : {:?}", self.previous);
+ self.previous = self.current.clone();
+ // debug!("Post Previous update 3 : {:?}", self.previous);
+ self.current = next.point();
+ self.fork = 0; // Mithril Immutable data is always Fork 0.
+ let update = ChainUpdate::new(chain_update::Kind::Block, false, next);
+ return Some(update);
}
- },
- std::task::Poll::Pending => std::task::Poll::Pending,
+ }
}
- }
-}
-/// Handler for receiving the read block range response from the client.
-pub struct ReadBlockRange(tokio::task::JoinHandle>>);
-
-impl Future for ReadBlockRange {
- type Output = Result>;
-
- fn poll(
- mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>,
- ) -> std::task::Poll {
- let p = &mut self.0;
- // Using tokio pin instead of, e.g., pin-project because we use tokio as the async runtime
- // lib for this crate.
- tokio::pin!(p);
-
- match p.poll(cx) {
- std::task::Poll::Ready(res) => {
- match res {
- Ok(res) => std::task::Poll::Ready(res),
- Err(_) => std::task::Poll::Ready(Err(Error::InternalError)),
- }
- },
- std::task::Poll::Pending => std::task::Poll::Pending,
+ if (self.mithril_tip.is_none() || current_mithril_tip > self.mithril_tip)
+ && self.current < self.mithril_tip
+ {
+ let snapshot = MithrilSnapshot::new(self.chain);
+ if let Some(block) = snapshot.read_block_at(¤t_mithril_tip).await {
+ // The Mithril Tip has moved forwards.
+ self.mithril_tip = Some(current_mithril_tip);
+ // Get the mithril tip block.
+ let update =
+ ChainUpdate::new(chain_update::Kind::ImmutableBlockRollForward, false, block);
+ return Some(update);
+ }
+ error!(
+ tip = ?self.mithril_tip,
+ current = ?current_mithril_tip,
+ "Mithril Tip Block is not in snapshot. Should not happen."
+ );
}
+
+ None
}
-}
-/// Cardano chain follower.
-pub struct Follower {
- /// Client connection information.
- ///
- /// This is used to open more connections when needed.
- client_connect_info: ClientConnectInfo,
- /// Chain update receiver.
- chain_update_rx: mpsc::Receiver>,
- /// Follow task request sender.
- follow_task_request_tx: mpsc::Sender,
- /// Follow task thread join handle.
- follow_task_join_handle: JoinHandle<()>,
- /// Optional Mithril snapshot information.
- mithril_snapshot: Option,
-}
+ /// If we can, get the next update from the mithril snapshot.
+ async fn next_from_live_chain(&mut self) -> Option {
+ let mut next_block: Option = None;
+ let mut update_type = chain_update::Kind::Block;
+ let mut rollback_depth: u64 = 0;
+
+ // Special Case: point = TIP_POINT. Just return the latest block in the live chain.
+ if self.current == TIP_POINT {
+ next_block = {
+ let block = get_live_block(self.chain, &self.current, -1, false)?;
+ Some(block)
+ };
+ }
-impl Follower {
- /// Connects the follower to a producer using the node-to-node protocol.
- ///
- /// # Arguments
- ///
- /// * `address`: Address of the node to connect to.
- /// * `network`: The [Network] the client is assuming it's connecting to.
- /// * `config`: Follower's configuration (see [`FollowerConfigBuilder`]).
- ///
- /// # Errors
- ///
- /// Returns Err if the connection could not be established.
- pub async fn connect(address: &str, network: Network, config: FollowerConfig) -> Result {
- let mut client = PeerClient::connect(address, network.into())
- .await
- .map_err(Error::Client)?;
-
- let Some(follow_from) = set_client_read_pointer(&mut client, config.follow_from).await?
- else {
- return Err(Error::SetReadPointer);
- };
-
- let mithril_snapshot = if let Some(path) = config.mithril_snapshot_path {
- Some(MithrilSnapshot::from_path(path)?)
- } else {
- None
- };
-
- let connect_info = ClientConnectInfo {
- address: address.to_string(),
- network,
- };
-
- let (task_request_tx, chain_update_rx, task_join_handle) = task::FollowTask::spawn(
- client,
- connect_info,
- mithril_snapshot.clone(),
- config.chain_update_buffer_size,
- follow_from,
- );
+ // In most cases we will be able to get the next block.
+ if next_block.is_none() {
+ // If we don't know the previous block, get the block requested.
+ let advance = i64::from(!self.previous.is_unknown());
+ next_block = get_live_block(self.chain, &self.current, advance, true);
+ }
+
+ // If we can't get the next consecutive block, then
+ // Get the best previous block.
+ if next_block.is_none() {
+ debug!("No blocks left in live chain.");
+
+ // IF this is an update still, and not us having caught up, then it WILL be a rollback.
+ update_type = chain_update::Kind::Rollback;
+ next_block = if let Some((block, depth)) =
+ find_best_fork_block(self.chain, &self.current, &self.previous, self.fork)
+ {
+ debug!("Found fork block: {block}");
+ // IF the block is the same as our current previous, there has been no chain
+ // advancement, so just return None.
+ if block.point().strict_eq(&self.current) {
+ None
+ } else {
+ rollback_depth = depth;
+ Some(block)
+ }
+ } else {
+ debug!("No block to find, rewinding to latest mithril tip.");
+ let latest_mithril_point = latest_mithril_snapshot_id(self.chain).tip();
+ if let Some(block) = MithrilSnapshot::new(self.chain)
+ .read_block_at(&latest_mithril_point)
+ .await
+ {
+ rollback_depth = live_chain_length(self.chain) as u64;
+ Some(block)
+ } else {
+ return None;
+ }
+ }
+ }
- let client_connect_info = ClientConnectInfo {
- address: address.to_string(),
- network,
- };
-
- Ok(Self {
- client_connect_info,
- chain_update_rx,
- follow_task_request_tx: task_request_tx,
- follow_task_join_handle: task_join_handle,
- mithril_snapshot,
- })
+ if let Some(next_block) = next_block {
+ // Update rollback stats for the follower if one is reported.
+ if update_type == chain_update::Kind::Rollback {
+ rollback(self.chain, stats::RollbackType::Follower, rollback_depth);
+ }
+ // debug!("Pre Previous update 4 : {:?}", self.previous);
+ self.previous = self.current.clone();
+ // debug!("Post Previous update 4 : {:?}", self.previous);
+ self.current = next_block.point().clone();
+ self.fork = next_block.fork();
+
+ let tip = point_at_tip(self.chain, &self.current).await;
+ let update = ChainUpdate::new(update_type, tip, next_block);
+ return Some(update);
+ }
+
+ None
}
- /// Set the follower's chain read-pointer. Returns None if the point was
- /// not found on the chain.
- ///
- /// # Arguments
- ///
- /// * `at`: Point at which to set the read-pointer.
- ///
- /// # Errors
- ///
- /// Returns Err if something went wrong while communicating with the producer.
- pub async fn set_read_pointer(&self, at: P) -> Result>
- where P: Into {
- let (response_tx, response_rx) = oneshot::channel();
-
- let req = task::SetReadPointerRequest {
- at: at.into(),
- response_tx,
- };
-
- self.follow_task_request_tx
- .send(req)
- .await
- .map_err(|_| Error::FollowTaskNotRunning)?;
-
- response_rx.await.map_err(|_| Error::FollowTaskNotRunning)?
+ /// Update the current Point, and return `false` if this fails.
+ fn update_current(&mut self, update: &Option) -> bool {
+ if let Some(update) = update {
+ let decoded = update.block_data().decode();
+ self.current = Point::new(decoded.slot(), decoded.hash().to_vec());
+ return true;
+ }
+ false
}
- /// Requests the client to read a block.
+ /// This is an unprotected version of `next()` which can ONLY be used within this
+ /// crate. Its purpose is to allow the chain data to be inspected/validate prior
+ /// to unlocking it for general access.
///
- /// # Arguments
+ /// This function can NOT return None, but that state is used to help process data.
///
- /// * `at`: Point at which to read the block.
- #[must_use]
- pub fn read_block(&self, at: P) -> ReadBlock
- where P: Into {
- let connect_info = self.client_connect_info.clone();
- let mithril_snapshot = self.mithril_snapshot.clone();
- let at = at.into();
-
- let join_handle = tokio::spawn(async move {
- let mut client = PeerClient::connect(connect_info.address, connect_info.network.into())
- .await
- .map_err(Error::Client)?;
-
- match at {
- PointOrTip::Tip => {
- let point = resolve_tip(&mut client).await?;
- read_block_from_network(&mut client, point).await
- },
-
- PointOrTip::Point(point) => {
- let snapshot_res = mithril_snapshot
- .as_ref()
- .and_then(|snapshot| snapshot.try_read_block(point.clone()).ok())
- .flatten();
-
- match snapshot_res {
- Some(block_data) => {
- tracing::trace!("Read block from Mithril snapshot");
- Ok(block_data)
- },
- None => read_block_from_network(&mut client, point).await,
- }
- },
+ /// This function must not be exposed for general use.
+ #[allow(clippy::unused_async)]
+ pub(crate) async fn unprotected_next(&mut self) -> Option {
+ let mut update;
+
+ // We will loop here until we can successfully return a new block
+ loop {
+ // Check if Immutable TIP has advanced, and if so, send a ChainUpdate about it.
+ // Should only happen once every ~6hrs.
+ // TODO.
+
+ // Try and get the next update from the mithril chain, and return it if we are
+ // successful.
+ update = self.next_from_mithril().await;
+ if update.is_some() {
+ break;
}
- });
- ReadBlock(join_handle)
- }
+ // No update from Mithril Data, so try and get one from the live chain.
+ update = self.next_from_live_chain().await;
+ if update.is_some() {
+ break;
+ }
- /// Request the client to read a block range.
- ///
- /// # Arguments
- ///
- /// * `from`: Block range start.
- /// * `to`: Block range end.
- #[must_use]
- pub fn read_block_range(&self, from: Point, to: P) -> ReadBlockRange
- where P: Into {
- let connect_info = self.client_connect_info.clone();
- let mithril_snapshot = self.mithril_snapshot.clone();
- let to = to.into();
-
- let join_handle = tokio::spawn(async move {
- let mut client = PeerClient::connect(connect_info.address, connect_info.network.into())
- .await
- .map_err(Error::Client)?;
-
- match to {
- PointOrTip::Tip => {
- let to_point = resolve_tip(&mut client).await?;
- read_block_range_from_network(&mut client, from, to_point).await
+ // IF we can't get a new block directly from the mithril data, or the live chain, then
+ // wait for something to change which might mean we can get the next block.
+ let update = self.sync_updates.recv().await;
+ match update {
+ Ok(kind) => {
+ debug!("Update kind: {kind}");
+ },
+ Err(tokio::sync::broadcast::error::RecvError::Lagged(distance)) => {
+ debug!("Lagged by {} updates", distance);
},
- PointOrTip::Point(to) => {
- let snapshot_res = mithril_snapshot
- .as_ref()
- .and_then(|snapshot| {
- snapshot.try_read_block_range(from.clone(), to.clone()).ok()
- })
- .flatten();
-
- match snapshot_res {
- Some((last_point_read, mut block_data_vec)) => {
- // If we couldn't get all the blocks from the snapshot,
- // try fetching the remaining ones from the network.
- if last_point_read.slot_or_default() < to.slot_or_default() {
- let network_blocks =
- read_block_range_from_network(&mut client, last_point_read, to)
- .await?;
-
- // Discard 1st point as it's already been read from
- // the snapshot
- let mut network_blocks_iter = network_blocks.into_iter();
- drop(network_blocks_iter.next());
-
- block_data_vec.extend(network_blocks_iter);
- }
-
- Ok(block_data_vec)
- },
- None => read_block_range_from_network(&mut client, from, to).await,
- }
+ Err(tokio::sync::broadcast::error::RecvError::Closed) => {
+ // We are closed, so we need to wait for the next update.
+ // This is not an error.
+ return None;
},
}
- });
+ }
+
+ // Update the current block, so we know which one to get next.
+ if !self.update_current(&update) {
+ return None;
+ }
+
+ update
+ }
+
+ /// Get the next block from the follower.
+ /// Returns NONE is there is no block left to return.
+ pub async fn next(&mut self) -> Option {
+ // If we aren't syncing TIP, and Current >= End, then return None
+ if self.end != TIP_POINT && self.current >= self.end {
+ return None;
+ }
+
+ // Can't follow if SYNC is not ready.
+ block_until_sync_ready(self.chain).await;
- ReadBlockRange(join_handle)
+ // Get next block from the iteration.
+ self.unprotected_next().await
}
- /// Receive the next chain update from the producer.
+ /// Get a single block from the chain by its point.
///
- /// # Errors
+ /// If the Point does not point exactly at a block, it will return the next
+ /// consecutive block.
///
- /// Returns Err if any producer communication errors occurred.
- pub async fn next(&mut self) -> Result {
- self.chain_update_rx
- .recv()
- .await
- .ok_or(Error::FollowTaskNotRunning)?
+ /// This is a convenience function which just used `ChainFollower` to fetch a single
+ /// block.
+ pub async fn get_block(chain: Network, point: Point) -> Option {
+ // Get the block from the chain.
+ // This function suppose to run only once, so the end point
+ // can be set to `TIP_POINT`
+ let mut follower = Self::new(chain, point, TIP_POINT).await;
+ follower.next().await
}
- /// Closes the follower connection and stops its background task.
+ /// Get the current Immutable and live tips.
///
- /// # Errors
- ///
- /// Returns Err if some error occurred in the background task.
- pub async fn close(self) -> std::result::Result<(), tokio::task::JoinError> {
- // NOTE(FelipeRosa): For now just abort the task since it needs no cancellation
- self.follow_task_join_handle.abort();
+ /// Note, this will block until the chain is synced, ready to be followed.
+ pub async fn get_tips(chain: Network) -> (Point, Point) {
+ // Can't follow if SYNC is not ready.
+ block_until_sync_ready(chain).await;
+
+ let tips = Statistics::tips(chain);
- self.follow_task_join_handle.await
+ let mithril_tip = Point::fuzzy(tips.0);
+ let live_tip = Point::fuzzy(tips.1);
+
+ (mithril_tip, live_tip)
}
-}
-/// Contains functions related to the Follower's background task.
-mod task {
- use pallas::{
- ledger::traverse::MultiEraHeader,
- network::{
- facades::PeerClient,
- miniprotocols::{chainsync, Point},
- },
- };
- use tokio::sync::{mpsc, oneshot};
-
- use super::{set_client_read_pointer, ChainUpdate, ClientConnectInfo};
- use crate::{mithril_snapshot::MithrilSnapshot, Error, MultiEraBlockData, PointOrTip, Result};
-
- /// Request the task to set the read pointer to the given point or to the
- /// tip.
- pub(super) struct SetReadPointerRequest {
- /// Point at which to set the read pointer.
- pub(super) at: PointOrTip,
- /// The channel that will be used to send the request's response.
- pub(super) response_tx: oneshot::Sender>>,
+ /// Schedule a transaction to be posted to the blockchain.
+ ///
+ /// # Arguments
+ ///
+ /// * `chain` - The blockchain to post the transaction on.
+ /// * `txn` - The transaction to be posted.
+ ///
+ /// # Returns
+ ///
+ /// `TxId` - The ID of the transaction that was queued.
+ #[allow(clippy::unused_async)]
+ pub async fn post_txn(chain: Network, txn: TxBody) -> TxId {
+ #[allow(clippy::no_effect_underscore_binding)]
+ let _unused = chain;
+ #[allow(clippy::no_effect_underscore_binding)]
+ let _unused = txn;
+
+ "unimplemented".to_string()
}
- /// Holds state for a follow task.
- pub(super) struct FollowTask {
- /// Client connection info.
- connect_info: ClientConnectInfo,
- /// Optional Mithril Snapshot that will be used by the follow task when fetching
- /// chain updates.
- mithril_snapshot: Option,
- /// Request receiver.
- request_rx: mpsc::Receiver,
- /// Chain update sender.
- chain_update_tx: mpsc::Sender>,
+ /// Check if a transaction, known by its `TxId`, has been sent to the Peer Node.
+ ///
+ /// Note, the `TxId` can ONLY be checked for ~6 hrs after it was posted.
+ /// After which, it should be on the blockchain, and its the applications job to track
+ /// if a transaction made it on-chain or not.
+ #[allow(clippy::unused_async)]
+ pub async fn txn_sent(chain: Network, id: TxId) -> bool {
+ #[allow(clippy::no_effect_underscore_binding)]
+ let _unused = chain;
+ #[allow(clippy::no_effect_underscore_binding)]
+ let _unused = id;
+
+ false
}
+}
- impl FollowTask {
- /// Spawn a follow task.
- pub(super) fn spawn(
- client: PeerClient, connect_info: ClientConnectInfo,
- mithril_snapshot: Option, buffer_size: usize, follow_from: Point,
- ) -> (
- mpsc::Sender,
- mpsc::Receiver>,
- tokio::task::JoinHandle<()>,
- ) {
- let (request_tx, request_rx) = mpsc::channel(1);
- let (chain_update_tx, chain_update_rx) = mpsc::channel(buffer_size);
-
- let this = Self {
- connect_info,
- mithril_snapshot,
- request_rx,
- chain_update_tx,
- };
+// TODO(SJ) - Add a function to check if a transaction is pending, or has been sent to the
+// chain.
- (
- request_tx,
- chain_update_rx,
- tokio::spawn(this.run(client, follow_from)),
- )
- }
+#[cfg(test)]
+mod tests {
+ use super::*;
- /// Runs the follow task.
- ///
- /// It keeps asking the connected node for new chain updates. Every update and
- /// communication errors are sent through the channel to the follower.
- ///
- /// Backpressure is achieved with the chain update channel's limited size.
- async fn run(mut self, client: PeerClient, from: Point) {
- let fetch_chain_updates_fut = Self::fetch_chain_updates(
- client,
- self.mithril_snapshot.as_ref(),
- self.chain_update_tx.clone(),
- from,
- );
- tokio::pin!(fetch_chain_updates_fut);
-
- loop {
- tokio::select! {
- Some(SetReadPointerRequest { at, response_tx }) = self.request_rx.recv() => {
- let res = PeerClient::connect(&self.connect_info.address, self.connect_info.network.into())
- .await;
-
- let Ok(mut client) = res else {
- drop(response_tx.send(Err(crate::Error::SetReadPointer)));
- continue;
- };
-
- match set_client_read_pointer(&mut client, at).await {
- Ok(Some(from)) => {
- fetch_chain_updates_fut.set(Self::fetch_chain_updates(
- client,
- self.mithril_snapshot.as_ref(),
- self.chain_update_tx.clone(),
- from.clone(),
- ));
-
- drop(response_tx.send(Ok(Some(from))));
- }
- Ok(None) => {
- drop(response_tx.send(Ok(None)));
- }
- Err(_) => {
- drop(response_tx.send(Err(crate::Error::SetReadPointer)));
- continue;
- }
- }
- }
-
- () = &mut fetch_chain_updates_fut => {}
- }
- }
- }
+ fn mock_block() -> MultiEraBlock {
+ let raw_block = hex::decode(include_str!("./../test_data/shelley.block"))
+ .expect("Failed to decode hex block.");
- /// Sends the next chain update to the follower.
- /// This can be either read from the Mithril snapshot (if configured) or
- /// from the N2N remote client.
- async fn fetch_chain_updates(
- mut client: PeerClient, mithril_snapshot: Option<&MithrilSnapshot>,
- chain_update_tx: mpsc::Sender>, from: Point,
- ) {
- let mut current_point = from;
-
- let set_to_snapshot = mithril_snapshot
- .and_then(|snapshot| snapshot.try_read_blocks_from_point(current_point.clone()));
-
- if let Some(iter) = set_to_snapshot {
- let mut last_recv_from_snapshot = false;
-
- for result in iter {
- let mut fallback = false;
-
- if let Ok(raw_block_data) = result {
- let block_data = MultiEraBlockData(raw_block_data);
-
- match block_data.decode() {
- Ok(block) => {
- current_point =
- Point::Specific(block.slot(), block.hash().to_vec());
-
- if chain_update_tx
- .send(Ok(ChainUpdate::Block(block_data)))
- .await
- .is_err()
- {
- return;
- }
-
- last_recv_from_snapshot = true;
- },
- Err(_) => {
- fallback = true;
- },
- }
- } else {
- fallback = true;
- }
-
- // If we, for any reason, we failed to get the block from the
- // Mithril snapshot, fallback to the getting it from the client.
- if fallback {
- let res = set_client_read_pointer(
- &mut client,
- PointOrTip::Point(current_point.clone()),
- )
- .await;
-
- match res {
- Ok(Some(p)) => {
- current_point = p;
-
- if !Self::send_next_chain_update(
- &mut client,
- chain_update_tx.clone(),
- )
- .await
- {
- return;
- }
- },
- Ok(None) | Err(_) => {
- drop(
- chain_update_tx
- .send(Err(crate::Error::SetReadPointer))
- .await,
- );
- return;
- },
- }
- }
- }
+ let pallas_block = pallas::ledger::traverse::MultiEraBlock::decode(raw_block.as_slice())
+ .expect("cannot decode block");
- if last_recv_from_snapshot {
- let res = set_client_read_pointer(
- &mut client,
- PointOrTip::Point(current_point.clone()),
- )
- .await;
+ let previous_point = Point::new(
+ pallas_block.slot() - 1,
+ pallas_block
+ .header()
+ .previous_hash()
+ .expect("cannot get previous hash")
+ .to_vec(),
+ );
- if let Err(e) = res {
- drop(chain_update_tx.send(Err(e)).await);
- return;
- }
+ MultiEraBlock::new(Network::Preprod, raw_block.clone(), &previous_point, 1)
+ .expect("cannot create block")
+ }
- // Skip the next update from the client since we've already
- // read it the Mithril snapshot.
- drop(Self::next_from_client(&mut client).await);
- }
- }
+ #[tokio::test]
+ async fn test_chain_follower_new() {
+ let chain = Network::Mainnet;
+ let start = Point::new(100u64, vec![]);
+ let end = Point::fuzzy(999u64);
+
+ let follower = ChainFollower::new(chain, start.clone(), end.clone()).await;
+
+ assert_eq!(follower.chain, chain);
+ assert_eq!(follower.end, end);
+ assert_eq!(follower.previous, UNKNOWN_POINT);
+ assert_eq!(follower.current, start);
+ assert_eq!(follower.fork, 1);
+ assert!(follower.mithril_follower.is_none());
+ assert!(follower.mithril_tip.is_none());
+ }
- while Self::send_next_chain_update(&mut client, chain_update_tx.clone()).await {}
- }
+ #[tokio::test]
+ async fn test_chain_follower_update_current_none() {
+ let chain = Network::Mainnet;
+ let start = Point::new(100u64, vec![]);
+ let end = Point::fuzzy(999u64);
- /// Waits for the next update from the node the client is connected to.
- ///
- /// Is cancelled by closing the `chain_update_tx` receiver end (explicitly or by
- /// dropping it).
- async fn next_from_client(client: &mut PeerClient) -> crate::Result> {
- tracing::trace!("Requesting next chain update");
- let res = {
- match client.chainsync().state() {
- chainsync::State::CanAwait => client.chainsync().recv_while_can_await().await,
- chainsync::State::MustReply => client.chainsync().recv_while_must_reply().await,
- _ => client.chainsync().request_next().await,
- }
- .map_err(Error::Chainsync)?
- };
+ let mut follower = ChainFollower::new(chain, start.clone(), end.clone()).await;
- tracing::trace!("Received block data from client");
-
- match res {
- chainsync::NextResponse::RollForward(header, _tip) => {
- let decoded_header = MultiEraHeader::decode(
- header.variant,
- header.byron_prefix.map(|p| p.0),
- &header.cbor,
- )
- .map_err(Error::Codec)?;
-
- let point =
- Point::Specific(decoded_header.slot(), decoded_header.hash().to_vec());
- tracing::trace!(point = ?point, "Fetching roll forward block data");
- let block_data = client
- .blockfetch()
- .fetch_single(point)
- .await
- .map_err(Error::Blockfetch)?;
-
- Ok(Some(ChainUpdate::Block(MultiEraBlockData(block_data))))
- },
- chainsync::NextResponse::RollBackward(point, _tip) => {
- tracing::trace!(point = ?point, "Fetching roll backward block data");
- let block_data = client
- .blockfetch()
- .fetch_single(point)
- .await
- .map_err(Error::Blockfetch)?;
-
- Ok(Some(ChainUpdate::Rollback(MultiEraBlockData(block_data))))
- },
- chainsync::NextResponse::Await => Ok(None),
- }
- }
+ let result = follower.update_current(&None);
- /// Sends the next chain update through the follower's chain update channel.
- async fn send_next_chain_update(
- client: &mut PeerClient, chain_update_tx: mpsc::Sender>,
- ) -> bool {
- loop {
- let res = Self::next_from_client(client).await;
-
- match res {
- Err(err) => {
- if chain_update_tx.send(Err(err)).await.is_err() {
- return false;
- }
- },
- Ok(next_response) => {
- if let Some(chain_update) = next_response {
- if chain_update_tx.send(Ok(chain_update)).await.is_err() {
- return false;
- }
-
- return true;
- }
- },
- }
- }
- }
+ assert!(!result);
}
-}
-/// Sets the N2N remote client's read pointer.
-async fn set_client_read_pointer(client: &mut PeerClient, at: PointOrTip) -> Result> {
- match at {
- PointOrTip::Point(Point::Origin) => {
- client
- .chainsync()
- .intersect_origin()
- .await
- .map(Some)
- .map_err(Error::Chainsync)
- },
- PointOrTip::Point(p @ Point::Specific(..)) => {
- client
- .chainsync()
- .find_intersect(vec![p])
- .await
- .map(|(point, _)| point)
- .map_err(Error::Chainsync)
- },
- PointOrTip::Tip => {
- client
- .chainsync()
- .intersect_tip()
- .await
- .map(Some)
- .map_err(Error::Chainsync)
- },
- }
-}
+ #[tokio::test]
+ async fn test_chain_follower_update_current() {
+ let chain = Network::Mainnet;
+ let start = Point::new(100u64, vec![]);
+ let end = Point::fuzzy(999u64);
-/// Finds the tip point.
-///
-/// NOTE: This changes the client's read pointer position.
-#[inline]
-async fn resolve_tip(client: &mut PeerClient) -> Result {
- client
- .chainsync()
- .intersect_tip()
- .await
- .map_err(Error::Chainsync)
-}
+ let mut follower = ChainFollower::new(chain, start.clone(), end.clone()).await;
-/// Reads a block from the network using the N2N client.
-async fn read_block_from_network(
- blockfetch_client: &mut PeerClient, point: Point,
-) -> Result {
- // Used in tracing
- let slot = point.slot_or_default();
-
- let block_data = blockfetch_client
- .blockfetch()
- .fetch_single(point)
- .await
- .map_err(Error::Blockfetch)?;
-
- tracing::trace!(slot, "Block read from n2n");
- Ok(MultiEraBlockData(block_data))
-}
+ let block_data = mock_block();
+ let update = ChainUpdate::new(chain_update::Kind::Block, false, block_data);
+
+ let result = follower.update_current(&Some(update.clone()));
-/// Reads a range of blocks from the network using the N2N client.
-async fn read_block_range_from_network(
- blockfetch_client: &mut PeerClient, from: Point, to: Point,
-) -> Result> {
- // Used in tracing
- let from_slot = from.slot_or_default();
- let to_slot = to.slot_or_default();
-
- let data_vec = blockfetch_client
- .blockfetch()
- .fetch_range((from, to))
- .await
- .map_err(Error::Blockfetch)?
- .into_iter()
- .map(MultiEraBlockData)
- .collect();
-
- tracing::trace!(from_slot, to_slot, "Block range read from n2n");
-
- Ok(data_vec)
+ assert!(result);
+ assert_eq!(follower.current, update.block_data().point());
+ }
}
diff --git a/rust/cardano-chain-follower/src/lib.rs b/rust/cardano-chain-follower/src/lib.rs
index bda6e7f9015..692ac6e3493 100644
--- a/rust/cardano-chain-follower/src/lib.rs
+++ b/rust/cardano-chain-follower/src/lib.rs
@@ -1,174 +1,35 @@
//! Cardano chain follower.
+mod chain_sync;
+mod chain_sync_config;
+mod chain_sync_live_chains;
+mod chain_sync_ready;
+mod chain_update;
+mod error;
mod follow;
+pub mod metadata;
+mod mithril_query;
mod mithril_snapshot;
-
-use std::str::FromStr;
-
-pub use follow::*;
-pub use pallas::network::miniprotocols::Point;
-use pallas::{
- ledger::traverse::{wellknown::GenesisValues, MultiEraBlock},
- network::miniprotocols::{
- chainsync, MAINNET_MAGIC, PREVIEW_MAGIC, PRE_PRODUCTION_MAGIC, TESTNET_MAGIC,
- },
-};
-use thiserror::Error;
-
-/// Crate error type.
-#[derive(Debug, Error)]
-pub enum Error {
- /// Data encoding/decoding error.
- #[error("Codec error: {0:?}")]
- Codec(pallas::ledger::traverse::Error),
- /// Client connection error.
- #[error("Client error: {0:?}")]
- Client(pallas::network::facades::Error),
- /// Blockfetch protocol error.
- #[error("Blockfetch error: {0:?}")]
- Blockfetch(pallas::network::miniprotocols::blockfetch::ClientError),
- /// Chainsync protocol error.
- #[error("Chainsync error: {0:?}")]
- Chainsync(chainsync::ClientError),
- /// Follower failed to set its read pointer.
- #[error("Failed to set follower read pointer")]
- SetReadPointer,
- /// Follower background follow task has stopped.
- #[error("Follower follow task is not running")]
- FollowTaskNotRunning,
- /// Mithril snapshot error.
- #[error("Failed to read block(s) from Mithril snapshot")]
- MithrilSnapshot,
- /// Failed to parse
- #[error("Failed to parse network")]
- ParseNetwork,
- /// Internal Error
- #[error("Internal error")]
- InternalError,
-}
-
-/// Crate result type.
-pub type Result = std::result::Result;
-
-/// A point in the chain or the tip.
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub enum PointOrTip {
- /// Represents a specific point of the chain.
- Point(Point),
- /// Represents the tip of the chain.
- Tip,
-}
-
-impl From for PointOrTip {
- fn from(point: Point) -> Self {
- Self::Point(point)
- }
-}
-
-/// CBOR encoded data of a multi-era block.
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct MultiEraBlockData(Vec);
-
-impl MultiEraBlockData {
- /// Decodes the data into a multi-era block.
- ///
- /// # Errors
- ///
- /// Returns Err if the block's era couldn't be decided or if the encoded data is
- /// invalid.
- pub fn decode(&self) -> Result {
- let block = MultiEraBlock::decode(&self.0).map_err(Error::Codec)?;
-
- Ok(block)
- }
-
- /// Consumes the [`MultiEraBlockData`] returning the block data raw bytes.
- #[must_use]
- pub fn into_raw_data(self) -> Vec {
- self.0
- }
-}
-
-impl AsRef<[u8]> for MultiEraBlockData {
- fn as_ref(&self) -> &[u8] {
- &self.0
- }
-}
-
-/// Enum of possible Cardano networks.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
-pub enum Network {
- /// Cardano mainnet network.
- Mainnet,
- /// Cardano pre-production network.
- Preprod,
- /// Cardano preview network.
- Preview,
- /// Cardano testnet network.
- Testnet,
-}
-
-/// The human readable name of the Cardano mainnet network.
-const MAINNET_NAME: &str = "mainnet";
-/// The human readable name of the Cardano pre-production network.
-const PREPROD_NAME: &str = "preprod";
-/// The human readable name of the Cardano preview network.
-const PREVIEW_NAME: &str = "preview";
-/// The human readable name of a Cardano local testnet network.
-const TESTNET_NAME: &str = "testnet";
-
-impl FromStr for Network {
- type Err = Error;
-
- fn from_str(input: &str) -> std::result::Result {
- match input {
- MAINNET_NAME => Ok(Network::Mainnet),
- PREPROD_NAME => Ok(Network::Preprod),
- PREVIEW_NAME => Ok(Network::Preview),
- TESTNET_NAME => Ok(Network::Testnet),
- _ => Err(Error::ParseNetwork),
- }
- }
-}
-
-impl std::fmt::Display for Network {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- Network::Mainnet => write!(f, "{MAINNET_NAME}"),
- Network::Preprod => write!(f, "{PREPROD_NAME}"),
- Network::Preview => write!(f, "{PREVIEW_NAME}"),
- Network::Testnet => write!(f, "{TESTNET_NAME}"),
- }
- }
-}
-
-impl From for u64 {
- fn from(network: Network) -> Self {
- match network {
- Network::Mainnet => MAINNET_MAGIC,
- Network::Preprod => PRE_PRODUCTION_MAGIC,
- Network::Preview => PREVIEW_MAGIC,
- Network::Testnet => TESTNET_MAGIC,
- }
- }
-}
-
-/// Return genesis values for given network
-#[must_use]
-pub fn network_genesis_values(network: &Network) -> Option {
- match network {
- Network::Mainnet => GenesisValues::from_magic(MAINNET_MAGIC),
- Network::Preprod => GenesisValues::from_magic(PRE_PRODUCTION_MAGIC),
- Network::Preview => GenesisValues::from_magic(PREVIEW_MAGIC),
- Network::Testnet => GenesisValues::from_magic(TESTNET_MAGIC),
- }
-}
-
-/// Validate a multi-era block.
-///
-/// This does not execute Plutus scripts nor validates ledger state.
-/// It only checks that the block is correctly formatted for its era.
-#[allow(dead_code)]
-fn validate_multiera_block(_block: &MultiEraBlock) {
- // (fsgr): Not sure about hwo the validation will be done in here yet.
-}
+mod mithril_snapshot_config;
+mod mithril_snapshot_data;
+mod mithril_snapshot_iterator;
+mod mithril_snapshot_sync;
+mod mithril_turbo_downloader;
+mod multi_era_block_data;
+mod network;
+mod point;
+mod snapshot_id;
+mod stats;
+pub mod turbo_downloader;
+mod utils;
+mod witness;
+
+pub use chain_sync_config::ChainSyncConfig;
+pub use chain_update::{ChainUpdate, Kind};
+pub use error::Result;
+pub use follow::ChainFollower;
+pub use metadata as Metadata;
+pub use multi_era_block_data::MultiEraBlock;
+pub use network::Network;
+pub use point::{Point, ORIGIN_POINT, TIP_POINT};
+pub use stats::Statistics;
diff --git a/rust/cardano-chain-follower/src/metadata/cip36.rs b/rust/cardano-chain-follower/src/metadata/cip36.rs
new file mode 100644
index 00000000000..4a38f390184
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/cip36.rs
@@ -0,0 +1,1102 @@
+//! Decoder and Validator for CIP36 Metadata
+
+use std::sync::Arc;
+
+use ed25519_dalek::Verifier;
+use minicbor::Decoder;
+use pallas::ledger::traverse::MultiEraTx;
+use tracing::debug;
+
+use super::{
+ DecodedMetadata, DecodedMetadataItem, DecodedMetadataValues, RawAuxData, ValidationReport,
+};
+use crate::Network;
+
+/// CIP36 Metadata Label
+pub const LABEL: u64 = 61284;
+/// CIP36 Metadata Signature label
+pub const SIG_LABEL: u64 = 61285;
+
+/// Project Catalyst Purpose
+pub const PROJECT_CATALYST_PURPOSE: u64 = 0;
+
+/// Signdata Preamble = `{ 61284: ?? }`
+/// CBOR Decoded =
+/// A1 # map(1)
+/// 19 EF64 # unsigned(61284)
+pub const SIGNDATA_PREAMBLE: [u8; 4] = [0xA1, 0x19, 0xEF, 0x64];
+
+/// Ed25519 Public Key
+type Ed25519PubKey = ed25519_dalek::VerifyingKey;
+
+/// Voting Public Key - Also known as Delegation in the CIP36 Specification
+#[derive(Clone, Debug)]
+pub struct VotingPubKey {
+ /// Ed25519 Public Key
+ pub voting_pk: Ed25519PubKey,
+ /// Weight of the Voting Public Key
+ pub weight: u32,
+}
+
+/// CIP 36 Registration Data.
+#[derive(Clone, Debug, Default)]
+pub struct Cip36 {
+ /// Is this CIP36 or CIP15 format.
+ #[allow(clippy::struct_field_names)]
+ pub cip36: Option,
+ /// Voting Keys (Called Delegations in the CIP-36 Spec)
+ /// If No Voting Keys could be decoded, this will be an empty array.
+ pub voting_keys: Vec,
+ /// Stake Address to associate with the Voting Keys
+ pub stake_pk: Option,
+ /// Payment Address to associate with the Voting Keys
+ /// No Payment key decoded will be an empty vec.
+ pub payment_addr: Vec,
+ /// Is the address able to be paid to? (Can't be a script or Stake address)
+ pub payable: bool,
+ /// Raw Nonce (Nonce that has not had slot correction applied)
+ pub raw_nonce: u64,
+ /// Nonce (Nonce that has been slot corrected)
+ pub nonce: u64,
+ /// Registration Purpose (Always 0 for Catalyst)
+ pub purpose: u64,
+ /// Signature Validates
+ pub signed: bool,
+ /// Strict Catalyst Validated
+ pub strict_catalyst: bool,
+}
+
+impl Cip36 {
+ /// Decode and validate CIP36/15 Metadata
+ ///
+ /// CIP15 is a subset of CIP36.
+ ///
+ /// See:
+ /// *
+ /// *
+ ///
+ /// # Parameters
+ /// * `decoded_metadata` - Decoded Metadata - Will be updated only if CIP36 Metadata
+ /// is found.
+ /// * `slot` - Current Slot
+ /// * `txn` - Transaction Aux data was attached to and to be validated/decoded
+ /// against. Not used for CIP36 Metadata.
+ /// * `raw_aux_data` - Raw Auxiliary Data for the transaction.
+ /// * `catalyst_strict` - Strict Catalyst Validation - otherwise Catalyst Specific
+ /// rules/workarounds are not applied.
+ ///
+ /// # Returns
+ ///
+ /// Nothing. IF CIP36 Metadata is found it will be updated in `decoded_metadata`.
+ #[allow(clippy::too_many_lines)]
+ pub(crate) fn decode_and_validate(
+ decoded_metadata: &DecodedMetadata, slot: u64, txn: &MultiEraTx, raw_aux_data: &RawAuxData,
+ catalyst_strict: bool, chain: Network,
+ ) {
+ let k61284 = raw_aux_data.get_metadata(LABEL);
+ let k61285 = raw_aux_data.get_metadata(SIG_LABEL);
+
+ let mut cip36 = Cip36 {
+ strict_catalyst: catalyst_strict,
+ ..Default::default()
+ };
+
+ // If there is NO Cip36/Cip15 Metadata then nothing to decode or validate, so quickly
+ // exit.
+ if k61284.is_none() && k61285.is_none() {
+ return;
+ }
+
+ // if let Some(reg) = k61284.as_ref() {
+ // debug!("CIP36 Metadata Detected: {slot}, {reg:02x?}");
+ //}
+ // if let Some(sig) = k61285.as_ref() {
+ // debug!("CIP36 Signature Detected: {slot}, {sig:02x?}");
+ //}
+
+ // Any Decode/Validation errors go here.
+ let mut validation_report = ValidationReport::new();
+
+ // Check if we actually have metadata to decode for the CIP36 Registration.
+ let Some(raw_cip36) = k61284 else {
+ cip36.decoding_failed(
+ "No CIP36 Metadata found, but CIP36 Signature Metadata found.",
+ &mut validation_report,
+ decoded_metadata,
+ );
+ debug!("decoded 1: {decoded_metadata:?}");
+ return;
+ };
+
+ let cip36_slice = raw_cip36.as_slice();
+
+ let mut decoder = Decoder::new(cip36_slice);
+
+ // It should be a definite map, get the number of entries in the map.
+ let Some(cip36_map_entries) =
+ cip36.decode_map_entries(&mut decoder, &mut validation_report, decoded_metadata)
+ else {
+ debug!("decoded 2: {decoded_metadata:?}");
+ return;
+ };
+
+ let mut found_keys: Vec = Vec::new();
+
+ for _entry in 0..cip36_map_entries {
+ let Some(key) =
+ cip36.decode_map_key(&mut decoder, &mut validation_report, decoded_metadata)
+ else {
+ debug!("decoded 3: {decoded_metadata:?} : {raw_cip36:02x?}");
+ return;
+ };
+
+ if found_keys.contains(&key) {
+ validation_report.push(format!("Duplicate key found in CIP36 Metadata: {key}"));
+ } else {
+ found_keys.push(key);
+ match key {
+ 1 => {
+ if cip36
+ .decode_voting_key(
+ &mut decoder,
+ &mut validation_report,
+ decoded_metadata,
+ )
+ .is_none()
+ {
+ // debug!("decoded 4: {decoded_metadata:?} : {validation_report:?} :
+ // {raw_cip36:02x?}");
+ return;
+ }
+ },
+ 2 => {
+ if cip36
+ .decode_stake_pub(
+ &mut decoder,
+ &mut validation_report,
+ decoded_metadata,
+ )
+ .is_none()
+ {
+ // debug!("decoded 5: {decoded_metadata:?} : {validation_report:?} :
+ // {raw_cip36:02x?}");
+ return;
+ }
+ },
+ 3 => {
+ if cip36
+ .decode_payment_address(
+ &mut decoder,
+ &mut validation_report,
+ decoded_metadata,
+ txn,
+ chain,
+ )
+ .is_none()
+ {
+ debug!("decoded 6: {decoded_metadata:?} : {validation_report:?} : {raw_cip36:02x?}");
+ return;
+ }
+ },
+ 4 => {
+ if cip36
+ .decode_nonce(
+ &mut decoder,
+ &mut validation_report,
+ decoded_metadata,
+ slot,
+ )
+ .is_none()
+ {
+ debug!("decoded 7: {decoded_metadata:?} : {validation_report:?} : {raw_cip36:02x?}");
+ return;
+ }
+ },
+ 5 => {
+ if cip36
+ .decode_purpose(&mut decoder, &mut validation_report, decoded_metadata)
+ .is_none()
+ {
+ debug!("decoded 8: {decoded_metadata:?} : {validation_report:?} : {raw_cip36:02x?}");
+ return;
+ }
+ },
+ _ => {
+ validation_report
+ .push(format!("Invalid key found in CIP36 Metadata: {key}"));
+ },
+ }
+ }
+ }
+
+ // Validate that all keys required to be present in the CIP36 Metadata are present.
+ if !found_keys.contains(&1) {
+ validation_report.push(
+ "The CIP36 Metadata Voting Key/Delegation is missing from the data.".to_string(),
+ );
+ }
+ if !found_keys.contains(&2) {
+ validation_report
+ .push("The CIP36 Metadata Stake Address is missing from the data.".to_string());
+ }
+ if !found_keys.contains(&3) {
+ validation_report
+ .push("The CIP36 Metadata Payment Address is missing from the data.".to_string());
+ }
+ if !found_keys.contains(&4) {
+ validation_report
+ .push("The CIP36 Metadata Nonce is missing from the data.".to_string());
+ }
+
+ if !decoded_metadata.0.is_empty() {
+ debug!("decoded 9: {decoded_metadata:?}");
+ }
+ // If we get this far, decode the signature, and verify it.
+ cip36.validate_signature(&raw_cip36, k61285, &mut validation_report, decoded_metadata);
+ }
+
+ /// Decoding of the CIP36 metadata failed, and can not continue.
+ fn decoding_failed(
+ &self, reason: &str, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) {
+ validation_report.push(reason.into());
+ decoded_metadata.0.insert(
+ LABEL,
+ Arc::new(DecodedMetadataItem {
+ value: DecodedMetadataValues::Cip36(Arc::new(self.clone()).clone()),
+ report: validation_report.clone(),
+ }),
+ );
+ }
+
+ /// Decode number of entries in the CIP36 metadata map.
+ fn decode_map_entries(
+ &self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ let cip36_map_entries = match decoder.map() {
+ Ok(None) => {
+ self.decoding_failed(
+ "CIP36 Metadata was Indefinite Map, Invalid Encoding.",
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ Ok(Some(entries)) => entries,
+ Err(error) => {
+ self.decoding_failed(
+ format!("CIP36 Metadata was error decoding Map: {error}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ Some(cip36_map_entries)
+ }
+
+ /// Decode the Key of an entry in the CIP36 Metadata map.
+ fn decode_map_key(
+ &self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ let key = match decoder.u64() {
+ Ok(key) => key,
+ Err(err) => {
+ self.decoding_failed(
+ format!("CIP36 Metadata was error decoding Map Entry Key: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ Some(key)
+ }
+
+ /// Decode the Registration Purpose in the CIP36 Metadata map.
+ fn decode_purpose(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ let purpose = match decoder.u64() {
+ Ok(key) => key,
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error decoding Purpose Value: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ if self.strict_catalyst && purpose != PROJECT_CATALYST_PURPOSE {
+ validation_report.push(format!("Registration contains unknown purpose: {purpose}"));
+ }
+
+ self.purpose = purpose;
+
+ Some(purpose)
+ }
+
+ /// Decode the Registration Nonce in the CIP36 Metadata map.
+ fn decode_nonce(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata, slot: u64,
+ ) -> Option {
+ let raw_nonce = match decoder.u64() {
+ Ok(key) => key,
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error decoding Purpose Value: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ let nonce = if self.strict_catalyst && raw_nonce > slot {
+ slot
+ } else {
+ raw_nonce
+ };
+
+ self.raw_nonce = raw_nonce;
+ self.nonce = nonce;
+
+ Some(nonce)
+ }
+
+ /// Decode the Payment Address Metadata in the CIP36 Metadata map.
+ fn decode_payment_address(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata, _txn: &MultiEraTx, chain: Network,
+ ) -> Option {
+ let raw_address = match decoder.bytes() {
+ Ok(address) => address,
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error decoding Payment Address: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ let Some(header_byte) = raw_address.first() else {
+ self.decoding_failed(
+ "Error decoding Payment Address: Empty",
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ };
+
+ // See: https://cips.cardano.org/cip/CIP-19 for details on address decoding.
+ let network_tag = header_byte & 0x0F;
+ let header_type = header_byte >> 4;
+ match header_type {
+ 0..=3 => {
+ if raw_address.len() != 57 {
+ validation_report.push(format!("Address Length {} != 57", raw_address.len()));
+ }
+ },
+ 4 | 5 => {
+ if raw_address.len() < 29 {
+ validation_report
+ .push(format!("Pointer Address Length {} < 29", raw_address.len()));
+ }
+ },
+ 6 | 7 | 14 | 15 => {
+ if raw_address.len() != 29 {
+ validation_report.push(format!(
+ "Pointer Address Length {} != 29",
+ raw_address.len()
+ ));
+ }
+ },
+ _ => {
+ validation_report.push(format!(
+ "Address Type {header_type} is invalid and unsupported"
+ ));
+ },
+ }
+
+ // Check address is for the correct network of the transaction.
+ if header_type == 8 {
+ validation_report.push("Byron Addresses are unsupported".to_string());
+ } else {
+ let valid = match chain {
+ Network::Mainnet => network_tag == 1,
+ Network::Preprod | Network::Preview => network_tag == 0,
+ };
+ if !valid {
+ validation_report.push(format!(
+ "Network Tag {network_tag} does not match transactions Network ID"
+ ));
+ }
+ }
+
+ // Addresses are only payable if they are a normal payment address and not a script
+ // address.
+ self.payable = header_type <= 7 && (header_type & 0x1 == 0);
+ self.payment_addr = raw_address.to_vec();
+
+ Some(self.payment_addr.len())
+ }
+
+ /// Decode the Payment Address Metadata in the CIP36 Metadata map.
+ fn decode_ed25519_pub_key(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata, key_type: &str,
+ ) -> Option {
+ let pub_key = match decoder.bytes() {
+ Ok(pub_key) => pub_key,
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error decoding {key_type}: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ if pub_key.len() == ed25519_dalek::PUBLIC_KEY_LENGTH {
+ // Safe to use `unwrap()` here because the length is fixed and we know it's 32 bytes
+ // long.
+ #[allow(clippy::unwrap_used)]
+ let pub_key: [u8; ed25519_dalek::PUBLIC_KEY_LENGTH] = pub_key.try_into().unwrap();
+ match ed25519_dalek::VerifyingKey::from_bytes(&pub_key) {
+ Ok(pk) => return Some(pk),
+ Err(error) => {
+ validation_report.push(format!("{key_type} not valid Ed25519: {error}"));
+ },
+ }
+ } else {
+ validation_report.push(format!(
+ "{key_type} Length {} != {}",
+ pub_key.len(),
+ ed25519_dalek::PUBLIC_KEY_LENGTH
+ ));
+ }
+
+ None
+ }
+
+ /// Decode the Staking Public Key in the CIP36 Metadata map.
+ fn decode_stake_pub(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ let pk = self.decode_ed25519_pub_key(
+ decoder,
+ validation_report,
+ decoded_metadata,
+ "Stake Public Key",
+ )?;
+ self.stake_pk = Some(pk);
+
+ Some(self.stake_pk.as_slice().len())
+ }
+
+ /// Decode an individual delegation entry from the CIP36 Metadata map.
+ fn decode_delegation(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ match decoder.array() {
+ Ok(Some(2)) => {
+ let vk = self.decode_ed25519_pub_key(
+ decoder,
+ validation_report,
+ decoded_metadata,
+ "Delegation Public Key",
+ )?;
+ let weight = match decoder.u32() {
+ Ok(weight) => weight,
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error Decoding CIP36 Delegations Entry Weight: {err}.")
+ .as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ self.voting_keys.push(VotingPubKey {
+ voting_pk: vk,
+ weight,
+ });
+ },
+ Ok(Some(entries)) => {
+ self.decoding_failed(
+ format!("Error Decoding CIP36 Delegations Entry Array: Must have exactly 2 elements, had {entries}.").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ Ok(None) => {
+ self.decoding_failed(
+ "Error Decoding CIP36 Delegations Entry Array: Indefinite Array is invalid encoding.",
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error Decoding CIP36 Delegations Entry Array: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ }
+
+ Some(self.voting_keys.len())
+ }
+
+ /// Decode the Voting Key(s) in the CIP36 Metadata map.
+ fn decode_voting_key(
+ &mut self, decoder: &mut Decoder, validation_report: &mut ValidationReport,
+ decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ match decoder.datatype() {
+ Ok(key_type) => {
+ match key_type {
+ minicbor::data::Type::Bytes => {
+ // CIP 15 type registration (single voting key).
+ self.cip36 = Some(false);
+ let vk = self.decode_ed25519_pub_key(
+ decoder,
+ validation_report,
+ decoded_metadata,
+ "Voting Public Key",
+ )?;
+ self.voting_keys.push(VotingPubKey {
+ voting_pk: vk,
+ weight: 1,
+ });
+ },
+ minicbor::data::Type::Array => {
+ // CIP 36 type registration (multiple voting keys).
+ self.cip36 = Some(true);
+ match decoder.array() {
+ Ok(Some(entries)) => {
+ for _entry in 0..entries {
+ self.decode_delegation(
+ decoder,
+ validation_report,
+ decoded_metadata,
+ )?;
+ }
+ },
+ Ok(None) => {
+ self.decoding_failed(
+ "Error Decoding CIP36 Delegations Array: Indefinite Array is invalid encoding.",
+ validation_report,
+ decoded_metadata,
+ );
+ },
+ Err(err) => {
+ self.decoding_failed(
+ format!("Error Decoding CIP36 Delegations Array: {err}")
+ .as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ }
+ },
+ _ => {
+ self.decoding_failed(
+ format!(
+ "Error inspecting Voting Key type: Unexpected CBOR Type {key_type}"
+ )
+ .as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ },
+ }
+ },
+ Err(error) => {
+ self.decoding_failed(
+ format!("Error inspecting Voting Key type: {error}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ }
+
+ if self.strict_catalyst && self.voting_keys.len() != 1 {
+ validation_report.push(format!(
+ "Catalyst Supports only a single Voting Key per registration. Found {}",
+ self.voting_keys.len()
+ ));
+ }
+
+ Some(self.voting_keys.len())
+ }
+
+ /// Decode a signature from the Signature metadata in 61285
+ /// Also checks that the signature is valid against the public key.
+ #[allow(clippy::too_many_lines)]
+ fn validate_signature(
+ &mut self, metadata: &Arc>, sig_metadata: Option>>,
+ validation_report: &mut ValidationReport, decoded_metadata: &DecodedMetadata,
+ ) -> Option {
+ // Check if we actually have metadata to decode for the CIP36 Registration.
+ let Some(raw_cip36) = sig_metadata else {
+ self.decoding_failed(
+ "No CIP36 Signature found, but CIP36 Metadata found.",
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ };
+
+ let cip36_slice = raw_cip36.as_slice();
+
+ let mut decoder = Decoder::new(cip36_slice);
+
+ match decoder.map() {
+ Ok(Some(1)) => (), // Ok
+ Ok(Some(x)) => {
+ self.decoding_failed(
+ format!("CIP36 Signature Map decoding failed: Has {x} entries, should have 1.")
+ .as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ Ok(None) => {
+ self.decoding_failed(
+ "CIP36 Signature Map is Indefinite. Decoding failed.",
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ Err(err) => {
+ self.decoding_failed(
+ format!("CIP36 Signature Map decoding failed: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ }
+
+ match decoder.u64() {
+ Ok(1) => (), // Ok
+ Ok(x) => {
+ self.decoding_failed(
+ format!("CIP36 Signature Map decoding failed: Map entry was {x} MUST BE 1.")
+ .as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ Err(err) => {
+ self.decoding_failed(
+ format!("CIP36 Signature Map Key decoding failed: {err}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ }
+
+ let sig: ed25519_dalek::Signature = match decoder.bytes() {
+ Ok(sig) => {
+ match ed25519_dalek::Signature::from_slice(sig) {
+ Ok(sig) => sig,
+ Err(err) => {
+ self.decoding_failed(
+ format!("CIP36 Signature Decoding failed: {err}",).as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ }
+ },
+ Err(error) => {
+ self.decoding_failed(
+ format!("CIP36 Signature Decode error: {error}.",).as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ },
+ };
+
+ // Ok, if we get this far then we have a valid CIP36 Signature.
+ let Some(pk) = self.stake_pk else {
+ self.decoding_failed(
+ "CIP36 Signature Verification Failed, no Staking Public Key.",
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ };
+
+ // Now we have both the Public Key and the signature. So calculate the hash of the
+ // metadata.
+ let hash = blake2b_simd::Params::new()
+ .hash_length(32)
+ .to_state()
+ .update(&SIGNDATA_PREAMBLE)
+ .update(metadata)
+ .finalize();
+
+ // debug!(
+ // "Hash = {:02x?}, pk = {:02x?}, sig = {:02x?}",
+ // hash.as_bytes(),
+ // pk.as_ref(),
+ // sig.to_bytes()
+ //);
+
+ if let Err(error) = pk.verify(hash.as_bytes(), &sig) {
+ self.signed = false;
+ self.decoding_failed(
+ format!("CIP36 Signature Verification Failed: {error}").as_str(),
+ validation_report,
+ decoded_metadata,
+ );
+ return None;
+ };
+
+ // If we get this far then we have a valid CIP36 Signature (Doesn't mean there aren't
+ // other issues).
+ self.signed = true;
+
+ // Record the fully validated Cip36 metadata
+ decoded_metadata.0.insert(
+ LABEL,
+ Arc::new(DecodedMetadataItem {
+ value: DecodedMetadataValues::Cip36(Arc::new(self.clone()).clone()),
+ report: validation_report.clone(),
+ }),
+ );
+
+ Some(true)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use dashmap::DashMap;
+
+ use super::*;
+
+ fn create_empty_cip36(strict: bool) -> Cip36 {
+ Cip36 {
+ cip36: None,
+ voting_keys: vec![],
+ stake_pk: None,
+ payment_addr: vec![],
+ payable: false,
+ raw_nonce: 0,
+ nonce: 0,
+ purpose: 0,
+ signed: false,
+ strict_catalyst: strict,
+ }
+ }
+
+ #[test]
+ fn test_decode_purpose_1() {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(true);
+ let mut decoder = Decoder::new(&[0x00]);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_purpose(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.purpose, 0);
+ assert_eq!(rc, Some(0));
+ }
+
+ #[test]
+ fn test_decode_purpose_2() {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(true);
+ let mut decoder = Decoder::new(&[0x19, 0x30, 0x39]);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_purpose(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 1);
+ assert_eq!(cip36.purpose, 12345);
+ assert_eq!(rc, Some(12345));
+ }
+
+ #[test]
+ fn test_decode_purpose_3() {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&[0x19, 0x30, 0x39]);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_purpose(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.purpose, 12345);
+ assert_eq!(rc, Some(12345));
+ }
+
+ #[test]
+ fn test_decode_purpose_4() {
+ let bytes_cases: &[&[u8]] = &[
+ &[0x80], // array(0)
+ &[0xA0], // map(0)
+ &[0x21], // negative(1)
+ &[0xF9, 0x3C, 0x00], // primitive(15360) - 1.0
+ ];
+
+ for bytes in bytes_cases {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(bytes);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_purpose(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 1);
+ assert_eq!(cip36.purpose, 0);
+ assert_eq!(rc, None);
+ }
+ }
+
+ #[test]
+ // valid `nonce`, strict = false, raw_nonce > slot
+ fn test_decode_nonce_1() {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&[0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_nonce(&mut decoder, &mut report, &decoded_metadata, 0);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.raw_nonce, u64::MAX);
+ assert_eq!(cip36.nonce, u64::MAX);
+ assert_eq!(rc, Some(u64::MAX));
+ }
+
+ #[test]
+ // valid `nonce`, strict = false, raw_nonce < slot
+ fn test_decode_nonce_2() {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&[0x01]);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_nonce(&mut decoder, &mut report, &decoded_metadata, 99);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.raw_nonce, 1);
+ assert_eq!(cip36.nonce, 1);
+ assert_eq!(rc, Some(1));
+ }
+
+ #[test]
+ // valid `nonce`, strict = true, raw_nonce > slot
+ fn test_decode_nonce_3() {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(true);
+ let mut decoder = Decoder::new(&[0x10]);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_nonce(&mut decoder, &mut report, &decoded_metadata, 1);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.raw_nonce, 16);
+ assert_eq!(cip36.nonce, 1);
+ assert_eq!(rc, Some(1));
+ }
+
+ #[test]
+ fn test_decode_nonce_4() {
+ let bytes_cases: &[&[u8]] = &[
+ &[0x80], // array(0)
+ &[0xA0], // map(0)
+ &[0x21], // negative(1)
+ &[0xF9, 0x3C, 0x00], // primitive(15360) - 1.0
+ ];
+
+ for bytes in bytes_cases {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(bytes);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_nonce(&mut decoder, &mut report, &decoded_metadata, 0);
+
+ assert_eq!(report.len(), 1);
+ assert_eq!(cip36.raw_nonce, 0);
+ assert_eq!(cip36.nonce, 0);
+ assert_eq!(rc, None);
+ }
+ }
+
+ #[test]
+ fn test_decode_payment_address_1() {
+ let hex_data = hex::decode(
+ // 0x004777561e7d9ec112ec307572faec1aff61ff0cfed68df4cd5c847f1872b617657881e30ad17c46e4010c9cb3ebb2440653a34d32219c83e9
+ "5839004777561E7D9EC112EC307572FAEC1AFF61FF0CFED68DF4CD5C847F1872B617657881E30AD17C46E4010C9CB3EBB2440653A34D32219C83E9"
+ ).expect("cannot decode hex");
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&hex_data);
+ let mut report = ValidationReport::new();
+ let multi_era_tx: *const MultiEraTx = std::ptr::null();
+ let multi_era_tx = unsafe { &*multi_era_tx };
+
+ let rc = cip36.decode_payment_address(
+ &mut decoder,
+ &mut report,
+ &decoded_metadata,
+ multi_era_tx,
+ Network::Preprod,
+ );
+
+ assert_eq!(report.len(), 0);
+ assert!(cip36.payable);
+ assert_eq!(cip36.payment_addr.len(), 57);
+ assert_eq!(rc, Some(57));
+ }
+
+ #[test]
+ fn test_decode_stake_pub_1() {
+ let hex_data = hex::decode(
+ // 0xe3cd2404c84de65f96918f18d5b445bcb933a7cda18eeded7945dd191e432369
+ "5820E3CD2404C84DE65F96918F18D5B445BCB933A7CDA18EEDED7945DD191E432369",
+ )
+ .expect("cannot decode hex");
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&hex_data);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_stake_pub(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 0);
+ assert!(cip36.stake_pk.is_some());
+ assert_eq!(rc, Some(1));
+ }
+
+ #[test]
+ fn test_decode_stake_pub_2() {
+ let bytes_cases: &[Vec] = &[
+ vec![],
+ hex::decode(
+ // 0xe3cd2404c84de65f96918f18d5b445bcb933a7cda18eeded7945dd19 (28 bytes)
+ "581CE3CD2404C84DE65F96918F18D5B445BCB933A7CDA18EEDED7945DD19",
+ )
+ .expect("cannot decode hex"),
+ ];
+
+ for bytes in bytes_cases {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(bytes);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_stake_pub(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 1);
+ assert_eq!(rc, None);
+ }
+ }
+
+ #[test]
+ // cip-36 version
+ fn test_decode_voting_key_1() {
+ let hex_data = hex::decode(
+ // [["0x0036ef3e1f0d3f5989e2d155ea54bdb2a72c4c456ccb959af4c94868f473f5a0", 1]]
+ "818258200036EF3E1F0D3F5989E2D155EA54BDB2A72C4C456CCB959AF4C94868F473F5A001",
+ )
+ .expect("cannot decode hex");
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&hex_data);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_voting_key(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.cip36, Some(true));
+ assert_eq!(cip36.voting_keys.len(), 1);
+ assert_eq!(rc, Some(1));
+ }
+
+ #[test]
+ // cip-15 version
+ fn test_decode_voting_key_2() {
+ let hex_data = hex::decode(
+ // 0x0036ef3e1f0d3f5989e2d155ea54bdb2a72c4c456ccb959af4c94868f473f5a0
+ "58200036EF3E1F0D3F5989E2D155EA54BDB2A72C4C456CCB959AF4C94868F473F5A0",
+ )
+ .expect("cannot decode hex");
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(&hex_data);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_voting_key(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 0);
+ assert_eq!(cip36.cip36, Some(false));
+ assert_eq!(cip36.voting_keys.len(), 1);
+ assert_eq!(rc, Some(1));
+ }
+
+ #[test]
+ fn test_decode_voting_key_3() {
+ let bytes_cases: &[Vec] = &[
+ vec![],
+ hex::decode(
+ // [[]] (empty)
+ "8180",
+ )
+ .expect("cannot decode hex"),
+ hex::decode(
+ // [["0x0036ef3e1f0d3f5989e2d155ea54bdb2a72c4c456ccb959af4c94868f473f5a0"]]
+ // (without weight)
+ "818158200036EF3E1F0D3F5989E2D155EA54BDB2A72C4C456CCB959AF4C94868F473F5A0",
+ )
+ .expect("cannot decode hex"),
+ ];
+
+ for bytes in bytes_cases {
+ let decoded_metadata = DecodedMetadata(DashMap::new());
+ let mut cip36 = create_empty_cip36(false);
+ let mut decoder = Decoder::new(bytes);
+ let mut report = ValidationReport::new();
+
+ let rc = cip36.decode_voting_key(&mut decoder, &mut report, &decoded_metadata);
+
+ assert_eq!(report.len(), 1);
+ assert_eq!(rc, None);
+ }
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/cip509.rs b/rust/cardano-chain-follower/src/metadata/cip509.rs
new file mode 100644
index 00000000000..023d973faea
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/cip509.rs
@@ -0,0 +1,82 @@
+//! Cardano metadata module
+
+mod x509;
+
+use minicbor::{decode, Decode, Decoder};
+use strum::EnumDiscriminants;
+use x509::X509Metadatum;
+
+/// Enum of metadata currently supported
+#[derive(Debug, PartialEq, EnumDiscriminants)]
+#[strum_discriminants(name(MetadataListType))]
+enum MetadataList {
+ /// x509 metadatum
+ X509Metadatum(X509Metadatum),
+}
+
+impl MetadataListType {
+ /// Convert associated label unsigned integer to enum.
+ fn from_u16(value: u16) -> Option {
+ match value {
+ 509 => Some(MetadataListType::X509Metadatum),
+ _ => None,
+ }
+ }
+}
+
+/// Struct of metadata
+#[derive(Debug, PartialEq)]
+struct Metadata {
+ /// A label of the metadata
+ label: u16,
+ /// A possible list of metadata currently supported
+ tx_metadata: MetadataList,
+}
+
+impl Decode<'_, ()> for Metadata {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ // Map of label
+ d.map()?;
+ let label = d.u16()?;
+ let metadata =
+ MetadataListType::from_u16(label).ok_or(decode::Error::message("Invalid label"))?;
+ match metadata {
+ MetadataListType::X509Metadatum => {
+ let x509_metadatum = X509Metadatum::decode(d, &mut ())?;
+ Ok(Self {
+ label,
+ tx_metadata: MetadataList::X509Metadatum(x509_metadatum),
+ })
+ },
+ }
+ }
+}
+
+#[cfg(test)]
+mod test_metadata {
+ use super::*;
+
+ #[test]
+ fn test_raw_metadata() {
+ let data = "a11901fda50050ca7a1457ef9f4c7f9c747f8c4a4cfa6c015008b5fcdd1e7487bc8b8db6e6c52f2b9b0258204d3f576f26db29139981a69443c2325daa812cc353a31b5a4db794a5bcbb06c20a8d5840a50a81590238308202343082019da00302010202145afc371daf301793cf0b1835a118c2f90363d5d9300d06092a864886f70d01010b05003045310b300906035840550406130241553113301106035504080c0a536f6d652d53746174653121301f060355040a0c18496e7465726e6574205769646769747320507479204c74643058401e170d3234303731313038353733365a170d3235303731313038353733365a3045310b30090603550406130241553113301106035504080c0a536f6d652d537458406174653121301f060355040a0c18496e7465726e6574205769646769747320507479204c746430819f300d06092a864886f70d010101050003818d00308189025840818100cd28e20b157ca70c85433c1689b1d5890ec479bdd1ffdcc5647ae12be9badf4af20764cd24bd64130831a57506dfbbdd3e924c96b259c6ccedf24d6a255840618f0819643c739f145b733c3c94333e5937b499ada9a4ffc127457c7cb557f2f5623dcadea1e06f09129db9584b0aee949244b3252b52afde5d385c65e563a65840efb07f0203010001a321301f301d0603551d0e0416041492eb169818b833588321957a846077aa239cf3a0300d06092a864886f70d01010b0500038181002e5f584073333ce667e4172b252416eaa1d2e9681f59943724b4f366a8b930443ca6b69b12dd9debee9c8a6307695ee1884da4b00136195d1d8223d1c253ff408edfc8ed584003af1819244c35d3843855fb9af86e84fb7636fa3f4a0fc396f6fb6fd16d3bcebde68a8bd81be61e8ee7d77e9f7f9804e03ebc31b4581313c955a667658b14815840588b004301f50d6b52464320746573742043411a63b0cd001a6955b90047010123456789ab01582102b1216ab96e5b3b3340f5bdf02e693f16213a04525ed444584050b1019c2dfd3838ab010058406fc903015259a38c0800a3d0b2969ca21977e8ed6ec344964d4e1c6b37c8fb541274c3bb81b2f53073c5f101a5ac2a928865835840b6a2679b6e682d2a26945ed0b2181e81d9800558203b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da2918288250667e69bd56a0fb583ed2d4db363e3bb017a150fdec9a8c902152433c25668aa3883cc9186481a5000001820a000250667e69bd56a0fbd2d4db363e3bb017a103000a64546573741863584091af2d6092350d47cf0b412cec3df7810004c8261271f719f81a33f414dd363b0a865864d9cae3bca22e03ef46e0b1b9740dc09633b6e86ec37b31a0f83d9e0a";
+ let hex_data = hex::decode(data).expect("Failed to decode hex data");
+ let mut decoder = Decoder::new(&hex_data);
+ let _metadata = Metadata::decode(&mut decoder, &mut ()).expect("Failed to decode metadata");
+ }
+
+ #[test]
+ fn test_brotli_metadata() {
+ let data = "a11901fda50050ca7a1457ef9f4c7f9c747f8c4a4cfa6c015008b5fcdd1e7487bc8b8db6e6c52f2b9b0258204d3f576f26db29139981a69443c2325daa812cc353a31b5a4db794a5bcbb06c20b8c58401b3d030866084fcb259de07496d3197e913a39fd628a3db0a4ed6839261a00c51cb0a5b9c16194064132ace375ea23c75c60659400cba304d0d689c00086195d5840ff28714da02c35e7295815ba58b77f227e576fa254c464e2f9c6f9dfa900a0208250033c054a468c38e08819601d073c034a4727a524ff39995477443c1fca235840839c927599b253887f50487c1caf757c0aaf79bc3fcacd42252b8f2ae1f1a8b282929ca22bb5c2885cc23a66005c0cc1ca20142b82310c3a137d44c1943e40995840a7a7ce5c3475b5887a3765ede2ff3b7bfea90f255e2edf37fd44e27f26b8e6cf408aef4b20bebf7257b3dabc7eda65fff4ed278b50219f0a52367ff5b80e46b758403875f55a394d17a5d9a6b1a1deff5b2206e9e9734e9fbefa6a1cdfeb7a104546dfb6e46c46feaeb65a7f4648c276e29e87b27bc053bffef79359300220d0c3875840f2a05cc4880317358e19c758fd9ab9917551ce3987af2e35d73b6958a0f5732784621b0c92f68a93537f16f48445424890f955d7a597c13c2eb54a82b39f0307584097507df5fef916fabb6dafdfb516fb9184783e2cb4e89d048a6c1e5c04818bdb76ffb5cbef1fbe452658d904cd152ee72a3bfc6efe1199fb3b51f1979629cd4e5840fdb7df511723d4cead3d2b2eb9c1f18cbbfcf9f5cc8eac46dc03cd55fcac3303c391437f50400923e65c02e981af5461b6867a47fb25ebe9b0fb4d9e41ec210e58404b9011000206414523c0990f9ee20b5d8a745393d3febaf6413a448b994f1567eb7945df7a0ab44afd55561e0190b376d411026c5d7a4a49a19e0bd3f5addd6c5840492fde46eee8d75b587286291dfeb6a78fdf59c1a6bfa2717b1f41dfa878756140ce7c77504b64b094b870ade78569566eec66369133af5aa8c8eab9f95e29df58409ec10be251547101b24c495c8ff4fa55378dbb4a5c6e89b18a12ac033343d61c3b7f5fba725b51536d92a5cbfaef9be6d24a3e5b3d75a1c0e29e42f523567fac4d0f8200811c822d2210b97f570818635840711ac54a3ab2dad900ce14028b0c05da5a1aa9a50c7ef5d14b378b9e8ca4fde8e026410b0c560fcae6f11e314208d677d692be0953fab2c96aa8f303b8561104";
+ let hex_data = hex::decode(data).expect("Failed to decode hex data");
+ let mut decoder = Decoder::new(&hex_data);
+ let _metadata = Metadata::decode(&mut decoder, &mut ()).expect("Failed to decode metadata");
+ }
+
+ #[test]
+ fn test_zstd_metadata() {
+ let data = "a11901fda50050ca7a1457ef9f4c7f9c747f8c4a4cfa6c015008b5fcdd1e7487bc8b8db6e6c52f2b9b0258204d3f576f26db29139981a69443c2325daa812cc353a31b5a4db794a5bcbb06c20c8c584028b52ffd605002251700942ca50a81590238308202343082019da00302010202145afc371daf301793cf0b1835a118c2f90363d5d9300d06092a864886f70d015840010b05003045310b300906035504061302415531133011080c0a536f6d652d53746174653121301f0a0c18496e7465726e65742057696467697473205074792058404c7464301e170d3234303731313038353733365a170d3235819f01050003818d0030818902818100cd28e20b157ca70c85433c1689b1d5890ec479bdd1ffdcc55840647ae12be9badf4af20764cd24bd64130831a57506dfbbdd3e924c96b259c6ccedf24d6a25618f0819643c739f145b733c3c94333e5937b499ada9a4ffc1274558407c7cb557f2f5623dcadea1e06f09129db9584b0aee949244b3252b52afde5d385c65e563a6efb07f0203010001a321301f301d0603551d0e0416041492eb1698584018b833588321957a846077aa239cf3a00b81002e5f73333ce667e4172b252416eaa1d2e9681f59943724b4f366a8b930443ca6b69b12dd9debee9c8a6307695e5840e1884da4b00136195d1d8223d1c253ff408edfc8ed03af1819244c35d3843855fb9af86e84fb7636fa3f4a0fc396f6fb6fd16d3bcebde68a8bd81be61e8ee7d758407e9f7f9804e03ebc31b4581313c955a667658b1481588b004301f50d6b52464320746573742043411a63b0cd001a6955b90047010123456789ab01582102b12158406ab96e5b3b3340f5bdf02e693f16213a04525ed44450b1019c2dfd3838ab010058406fc903015259a38c0800a3d0b2969ca21977e8ed6ec344964d4e1c6b37c85840fb541274c3bb81b2f53073c5f101a5ac2a92886583b6a2679b6e682d2a26945ed0b2181e81d9800558203b6a27bcceb6a42d62a3a8d02a6f0d73653215771de2584043a63ac048a18b59da2918288250667e69bd56a0fbd2d4db363e3bb017a150fdec9a8c902152433c25668aa3883cc9186481a5000001820a000250667e69bd56582ea0fbd2d4db363e3bb017a103000a64546573740000080084391c0898ad681c1a1ad7a506644166c038791758a7191863584023441d5f1348cd213bb5502838c439ae8fea22b3295c73d34df29cb56887f35cb3e0ba04cc9609ec9f17d2360b1a5f876098dfa5e60ba549d3b6ea2a6026b507";
+ let hex_data = hex::decode(data).expect("Failed to decode hex data");
+ let mut decoder = Decoder::new(&hex_data);
+ let _metadata = Metadata::decode(&mut decoder, &mut ()).expect("Failed to decode metadata");
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/mod.rs b/rust/cardano-chain-follower/src/metadata/mod.rs
new file mode 100644
index 00000000000..5c191c07c09
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/mod.rs
@@ -0,0 +1,175 @@
+//! Metadata decoding and validating.
+
+use std::{fmt::Debug, sync::Arc};
+
+use cip36::Cip36;
+use dashmap::DashMap;
+use pallas::ledger::traverse::{MultiEraBlock, MultiEraTx};
+use raw_aux_data::RawAuxData;
+use tracing::error;
+
+use crate::{utils::usize_from_saturating, Network};
+
+pub mod cip36;
+mod raw_aux_data;
+
+/// List of all validation errors (as strings) Metadata is considered Valid if this list
+/// is empty.
+pub type ValidationReport = Vec;
+
+/// Possible Decoded Metadata Values.
+/// Must match the key they relate too, but the consumer needs to check this.
+#[derive(Debug)]
+pub enum DecodedMetadataValues {
+ // Json Metadata // TODO
+ // Json(serde_json::Value), // TODO
+ /// CIP-36/CIP-15 Catalyst Registration metadata.
+ Cip36(Arc),
+}
+
+/// An individual decoded metadata item.
+#[derive(Debug)]
+pub struct DecodedMetadataItem {
+ /// The decoded metadata itself.
+ pub value: DecodedMetadataValues,
+ /// Validation report for this metadata item.
+ pub report: ValidationReport,
+}
+
+/// Decoded Metadata for a single transaction.
+/// The key is the Primary Label of the Metadata.
+/// For example, CIP15/36 uses labels 61284 & 61285,
+/// 61284 is the primary label, so decoded metadata
+/// will be under that label.
+pub(crate) struct DecodedMetadata(DashMap>);
+
+impl DecodedMetadata {
+ /// Create new decoded metadata for a transaction.
+ fn new(chain: Network, slot: u64, txn: &MultiEraTx, raw_aux_data: &RawAuxData) -> Self {
+ let decoded_metadata = Self(DashMap::new());
+
+ // Process each known type of metadata here, and record the decoded result.
+ Cip36::decode_and_validate(&decoded_metadata, slot, txn, raw_aux_data, true, chain);
+
+ // if !decoded_metadata.0.is_empty() {
+ // debug!("Decoded Metadata final: {decoded_metadata:?}");
+ //}
+ decoded_metadata
+ }
+
+ /// Get the decoded metadata item at the given slot, or None if it doesn't exist.
+ pub fn get(&self, primary_label: u64) -> Option> {
+ let entry = self.0.get(&primary_label)?;
+ let value = entry.value();
+ Some(value.clone())
+ }
+}
+
+impl Debug for DecodedMetadata {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("DecodedMetadata {")?;
+ for kv in &self.0 {
+ let k = kv.key();
+ let v = kv.value().clone();
+ f.write_fmt(format_args!("{k:?}:{v:?} "))?;
+ }
+ f.write_str("}")
+ }
+}
+
+/// Decoded Metadata for a all transactions in a block.
+/// The Key for both entries is the Transaction offset in the block.
+#[derive(Debug)]
+pub struct DecodedTransaction {
+ /// The Raw Auxiliary Data for each transaction in the block.
+ raw: DashMap,
+ /// The Decoded Metadata for each transaction in the block.
+ decoded: DashMap,
+}
+
+impl DecodedTransaction {
+ /// Insert another transaction worth of data into the Decoded Aux Data
+ fn insert(
+ &mut self, chain: Network, slot: u64, txn_idx: u32, cbor_data: &[u8],
+ transactions: &[MultiEraTx],
+ ) {
+ let txn_idx = usize_from_saturating(txn_idx);
+
+ let Some(txn) = transactions.get(txn_idx) else {
+ error!("No transaction at index {txn_idx} trying to decode metadata.");
+ return;
+ };
+
+ let txn_raw_aux_data = RawAuxData::new(cbor_data);
+ let txn_metadata = DecodedMetadata::new(chain, slot, txn, &txn_raw_aux_data);
+
+ self.raw.insert(txn_idx, txn_raw_aux_data);
+ self.decoded.insert(txn_idx, txn_metadata);
+ }
+
+ /// Create a new `DecodedTransaction`.
+ pub(crate) fn new(chain: Network, block: &MultiEraBlock) -> Self {
+ let mut decoded_aux_data = DecodedTransaction {
+ raw: DashMap::new(),
+ decoded: DashMap::new(),
+ };
+
+ if block.has_aux_data() {
+ let transactions = block.txs();
+ let slot = block.slot();
+
+ if let Some(_metadata) = block.as_byron() {
+ // Nothing to do here.
+ } else if let Some(alonzo_block) = block.as_alonzo() {
+ for (txn_idx, metadata) in alonzo_block.auxiliary_data_set.iter() {
+ decoded_aux_data.insert(
+ chain,
+ slot,
+ *txn_idx,
+ metadata.raw_cbor(),
+ &transactions,
+ );
+ }
+ } else if let Some(babbage_block) = block.as_babbage() {
+ for (txn_idx, metadata) in babbage_block.auxiliary_data_set.iter() {
+ decoded_aux_data.insert(
+ chain,
+ slot,
+ *txn_idx,
+ metadata.raw_cbor(),
+ &transactions,
+ );
+ }
+ } else if let Some(conway_block) = block.as_conway() {
+ for (txn_idx, metadata) in conway_block.auxiliary_data_set.iter() {
+ decoded_aux_data.insert(
+ chain,
+ slot,
+ *txn_idx,
+ metadata.raw_cbor(),
+ &transactions,
+ );
+ }
+ } else {
+ error!("Undecodable metadata, unknown Era");
+ };
+ }
+ decoded_aux_data
+ }
+
+ /// Get metadata for a given label in a transaction if it exists.
+ #[must_use]
+ pub fn get_metadata(&self, txn_idx: usize, label: u64) -> Option> {
+ let txn_metadata = self.decoded.get(&txn_idx)?;
+ let txn_metadata = txn_metadata.value();
+ txn_metadata.get(label)
+ }
+
+ /// Get raw metadata for a given label in a transaction if it exists.
+ #[must_use]
+ pub fn get_raw_metadata(&self, txn_idx: usize, label: u64) -> Option>> {
+ let txn_metadata = self.raw.get(&txn_idx)?;
+ let txn_metadata = txn_metadata.value();
+ txn_metadata.get_metadata(label)
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/raw_aux_data.rs b/rust/cardano-chain-follower/src/metadata/raw_aux_data.rs
new file mode 100644
index 00000000000..f8d39ed571a
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/raw_aux_data.rs
@@ -0,0 +1,269 @@
+//! Raw Auxiliary Data Decoding
+
+use std::sync::Arc;
+
+use anyhow::bail;
+use dashmap::DashMap;
+use minicbor::{data::Type, Decoder};
+use tracing::{error, warn};
+
+/// What type of smart contract is this list.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, strum::Display, Hash)]
+pub enum SmartContractType {
+ /// Native smart contracts
+ Native,
+ /// Plutus smart contracts (with version number 1-x)
+ Plutus(u64),
+}
+
+// We CAN NOT use the Pallas library metadata decoding because it does not preserve raw
+// metadata values which are critical for performing operations like signature checks on
+// data. So we have a bespoke metadata decoder here.
+#[derive(Debug)]
+pub(crate) struct RawAuxData {
+ /// Metadata: key = label, value = raw metadata bytes
+ metadata: DashMap>>,
+ /// Scripts: 1 = Native, 2 = Plutus V1, 3 = Plutus V2, 4 = Plutus V3
+ scripts: DashMap>>>,
+}
+
+impl RawAuxData {
+ /// Create a new `RawDecodedMetadata`.
+ pub(crate) fn new(aux_data: &[u8]) -> Self {
+ let mut raw_decoded_data = Self {
+ metadata: DashMap::new(),
+ scripts: DashMap::new(),
+ };
+
+ let mut decoder = Decoder::new(aux_data);
+
+ match decoder.datatype() {
+ Ok(minicbor::data::Type::Map) => {
+ if let Err(error) = Self::decode_shelley_map(&mut raw_decoded_data, &mut decoder) {
+ error!("Failed to Deserialize Shelley Metadata: {error}: {aux_data:02x?}");
+ }
+ },
+ Ok(minicbor::data::Type::Array) => {
+ if let Err(error) =
+ Self::decode_shelley_ma_array(&mut raw_decoded_data, &mut decoder)
+ {
+ error!("Failed to Deserialize Shelley-MA Metadata: {error}: {aux_data:02x?}");
+ }
+ },
+ Ok(minicbor::data::Type::Tag) => {
+ if let Err(error) =
+ Self::decode_alonzo_plus_map(&mut raw_decoded_data, &mut decoder)
+ {
+ error!("Failed to Deserialize Alonzo+ Metadata: {error}: {aux_data:02x?}");
+ }
+ },
+ Ok(unexpected) => {
+ error!("Unexpected datatype for Aux data: {unexpected}: {aux_data:02x?}");
+ },
+ Err(error) => {
+ error!("Error decoding metadata: {error}: {aux_data:02x?}");
+ },
+ }
+
+ raw_decoded_data
+ }
+
+ /// Decode the Shelley map of metadata.
+ fn decode_shelley_map(
+ raw_decoded_data: &mut Self, decoder: &mut minicbor::Decoder,
+ ) -> anyhow::Result<()> {
+ let entries = match decoder.map() {
+ Ok(Some(entries)) => entries,
+ Ok(None) => {
+ // Sadly... Indefinite Maps are allowed in Cardano CBOR Encoding.
+ u64::MAX
+ },
+ Err(error) => {
+ bail!("Error decoding metadata: {error}");
+ },
+ };
+
+ // debug!("Decoding shelley metadata map with {} entries", entries);
+
+ let raw_metadata = decoder.input();
+
+ for _ in 0..entries {
+ let key = match decoder.u64() {
+ Ok(key) => key,
+ Err(error) => {
+ bail!("Error decoding metadata key: {error}");
+ },
+ };
+ let value_start = decoder.position();
+ if let Err(error) = decoder.skip() {
+ bail!("Error decoding metadata value: {error}");
+ }
+ let value_end = decoder.position();
+ let Some(value_slice) = raw_metadata.get(value_start..value_end) else {
+ bail!("Invalid metadata value found. Unable to extract raw value slice.");
+ };
+ let value = value_slice.to_vec();
+
+ // debug!("Decoded metadata key: {key}, value: {value:?}");
+
+ let _unused = raw_decoded_data.metadata.insert(key, Arc::new(value));
+
+ // Look for End Sentinel IF its an indefinite MAP (which we know because entries is
+ // u64::MAX).
+ if entries == u64::MAX {
+ match decoder.datatype() {
+ Ok(Type::Break) => {
+ // Skip over the break token.
+ let _unused = decoder.skip();
+ break;
+ },
+ Ok(_) => (), // Not break, so do next loop, should be the next key.
+ Err(error) => {
+ bail!("Error checking indefinite metadata map end sentinel: {error}");
+ },
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Decode a Shelley-MA Auxiliary Data Array
+ fn decode_shelley_ma_array(
+ raw_decoded_data: &mut Self, decoder: &mut minicbor::Decoder,
+ ) -> anyhow::Result<()> {
+ match decoder.array() {
+ Ok(Some(entries)) => {
+ if entries != 2 {
+ bail!(
+ "Invalid number of entries in Metadata Array. Expected 2, found {entries}."
+ );
+ }
+ },
+ Ok(None) => {
+ bail!("Indefinite Array found decoding Metadata. Invalid.");
+ },
+ Err(error) => {
+ bail!("Error decoding metadata: {error}");
+ },
+ };
+
+ // First entry is the metadata map, so just decode that now.
+ Self::decode_shelley_map(raw_decoded_data, decoder)?;
+ // Second entry is an array of native scripts.
+ Self::decode_script_array(raw_decoded_data, decoder, SmartContractType::Native)?;
+
+ Ok(())
+ }
+
+ /// Decode a Shelley-MA Auxiliary Data Array
+ fn decode_alonzo_plus_map(
+ raw_decoded_data: &mut Self, decoder: &mut minicbor::Decoder,
+ ) -> anyhow::Result<()> {
+ match decoder.tag() {
+ Ok(tag) => {
+ if tag.as_u64() != 259 {
+ bail!("Invalid tag for alonzo+ aux data. Expected 259, found {tag}.");
+ }
+ },
+ Err(error) => {
+ bail!("Error decoding tag for alonzo+ aux data: {error}");
+ },
+ }
+
+ let entries = match decoder.map() {
+ Ok(Some(entries)) => entries,
+ Ok(None) => bail!("Indefinite Map found decoding Alonzo+ Metadata. Invalid."),
+ Err(error) => bail!("Error decoding Alonzo+ Metadata: {error}"),
+ };
+
+ // iterate the map
+ for _ in 0..entries {
+ let aux_type_key = match decoder.u64() {
+ Ok(key) => key,
+ Err(error) => {
+ bail!("Error decoding Alonzo+ Metadata Aux Data Type Key: {error}");
+ },
+ };
+
+ let contract_type = match aux_type_key {
+ 0 => {
+ if raw_decoded_data.metadata.is_empty() {
+ Self::decode_shelley_map(raw_decoded_data, decoder)?;
+ continue;
+ }
+ bail!("Multiple Alonzo+ Metadata entries found. Invalid.");
+ },
+ 1 => SmartContractType::Native,
+ _ => {
+ if aux_type_key > 4 {
+ warn!(
+ "Auxiliary Type Key > 4 detected, assuming its a plutus script > V3."
+ );
+ }
+ SmartContractType::Plutus(aux_type_key - 1)
+ },
+ };
+
+ if raw_decoded_data.scripts.contains_key(&contract_type) {
+ bail!("Multiple Alonzo+ Scripts of type {contract_type} found. Invalid.");
+ }
+
+ Self::decode_script_array(raw_decoded_data, decoder, contract_type)?;
+ }
+ Ok(())
+ }
+
+ /// Decode an array of smart contract scripts
+ fn decode_script_array(
+ raw_decoded_data: &mut Self, decoder: &mut minicbor::Decoder,
+ contract_type: SmartContractType,
+ ) -> anyhow::Result<()> {
+ let mut scripts: Vec> = Vec::new();
+
+ let entries = match decoder.array() {
+ Ok(Some(entries)) => entries,
+ Ok(None) => {
+ bail!("Indefinite Script Array found decoding Metadata. Invalid.");
+ },
+ Err(error) => {
+ bail!("Error decoding metadata: {error}");
+ },
+ };
+
+ let raw_metadata = decoder.input();
+
+ for _entry in 0..entries {
+ if contract_type == SmartContractType::Native {
+ // Native Scripts are actually CBOR arrays, so capture their data as bytes for
+ // later processing.
+ let value_start = decoder.position();
+ if let Err(error) = decoder.skip() {
+ bail!("Error decoding native script value: {error}");
+ }
+ let value_end = decoder.position();
+ let Some(value_slice) = raw_metadata.get(value_start..value_end) else {
+ bail!("Invalid metadata value found. Unable to extract native script slice.");
+ };
+ scripts.push(value_slice.to_vec());
+ } else {
+ let script = match decoder.bytes() {
+ Ok(script) => script,
+ Err(error) => bail!("Error decoding script data from metadata: {error}"),
+ };
+ scripts.push(script.to_vec());
+ }
+ }
+
+ let _unused = raw_decoded_data
+ .scripts
+ .insert(contract_type, Arc::new(scripts));
+
+ Ok(())
+ }
+
+ /// Get Raw metadata for a given metadata label, if it exists.
+ pub(crate) fn get_metadata(&self, label: u64) -> Option>> {
+ self.metadata.get(&label).map(|v| v.value().clone())
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/x509/mod.rs b/rust/cardano-chain-follower/src/metadata/x509/mod.rs
new file mode 100644
index 00000000000..cb34eccc696
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/x509/mod.rs
@@ -0,0 +1,243 @@
+//! x509 metadata
+//! Doc Reference:
+//! CDDL Reference:
+
+mod rbac;
+
+use std::io::Read;
+
+use minicbor::{decode, Decode, Decoder};
+use rbac::X509RbacMetadata;
+use strum::FromRepr;
+
+/// Enum of compression algorithms used to compress chunks.
+#[derive(FromRepr, Debug, PartialEq)]
+#[repr(u8)]
+pub enum CompressionAlgorithm {
+ /// Raw data, no compression.
+ Raw = 10,
+ /// Brotli compression.
+ Brotli = 11,
+ /// Zstd compression.
+ Zstd = 12,
+}
+
+/// Struct of x509 chunks.
+#[derive(Debug, PartialEq)]
+struct X509Chunks {
+ /// The compression algorithm used to compress the data.
+ chunk_type: CompressionAlgorithm,
+ /// The decompressed data.
+ chunk_data: X509RbacMetadata,
+}
+
+impl X509Chunks {
+ /// Create new instance of `X509Chunks`.
+ fn new(chunk_type: CompressionAlgorithm, chunk_data: X509RbacMetadata) -> Self {
+ Self {
+ chunk_type,
+ chunk_data,
+ }
+ }
+}
+
+impl Decode<'_, ()> for X509Chunks {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ // Determine the algorithm
+ let algorithm = CompressionAlgorithm::from_repr(d.u8()?)
+ .ok_or(decode::Error::message("Invalid chunk data type"))?;
+
+ // Decompress the data
+ let decompressed = decompress(d, &algorithm)
+ .map_err(|e| decode::Error::message(format!("Failed to decompress {e}")))?;
+
+ // Decode the decompressed data.
+ let mut decoder = Decoder::new(&decompressed);
+ let chunk_data = X509RbacMetadata::decode(&mut decoder, &mut ())
+ .map_err(|e| decode::Error::message(format!("Failed to decode {e}")))?;
+
+ Ok(X509Chunks {
+ chunk_type: algorithm,
+ chunk_data,
+ })
+ }
+}
+
+/// Decompress the data using the given algorithm.
+fn decompress(d: &mut Decoder, algorithm: &CompressionAlgorithm) -> anyhow::Result> {
+ let chunk_len = d
+ .array()
+ .map_err(|e| anyhow::anyhow!(e.to_string()))?
+ .ok_or(anyhow::anyhow!("Error indefinite array in X509Chunks"))?;
+ // Vector containing the concatenated chunks
+ let mut concat_chunk = vec![];
+ for _ in 0..chunk_len {
+ let chunk_data = d.bytes().map_err(|e| anyhow::anyhow!(e.to_string()))?;
+ concat_chunk.extend_from_slice(chunk_data);
+ }
+
+ let mut buffer = vec![];
+
+ match algorithm {
+ CompressionAlgorithm::Raw => {
+ buffer.extend_from_slice(concat_chunk.as_slice());
+ },
+ CompressionAlgorithm::Zstd => {
+ zstd::stream::copy_decode(concat_chunk.as_slice(), &mut buffer)?;
+ },
+ CompressionAlgorithm::Brotli => {
+ let mut decoder = brotli::Decompressor::new(concat_chunk.as_slice(), 4096);
+ decoder
+ .read_to_end(&mut buffer)
+ .map_err(|_| anyhow::anyhow!("Failed to decompress using Brotli algorithm"))?;
+ },
+ }
+ Ok(buffer)
+}
+
+/// x509 metadatum.
+#[derive(Debug, PartialEq)]
+pub(crate) struct X509Metadatum {
+ /// `UUIDv4` Purpose .
+ purpose: [u8; 16], // (bytes .size 16)
+ /// Transaction inputs hash.
+ txn_inputs_hash: [u8; 16], // bytes .size 16
+ /// Optional previous transaction ID.
+ prv_tx_id: Option<[u8; 32]>, // bytes .size 32
+ /// x509 chunks.
+ x509_chunks: X509Chunks, // chunk_type => [ + x509_chunk ]
+ /// Validation signature.
+ validation_signature: Vec, // bytes size (1..64)
+}
+
+#[allow(clippy::module_name_repetitions)]
+impl X509Metadatum {
+ /// Create a new instance of `X509Metadatum`.
+ fn new() -> Self {
+ Self {
+ purpose: [0; 16],
+ txn_inputs_hash: [0; 16],
+ prv_tx_id: None,
+ x509_chunks: X509Chunks::new(CompressionAlgorithm::Raw, X509RbacMetadata::new()),
+ validation_signature: vec![],
+ }
+ }
+
+ /// Set the purpose.
+ fn set_purpose(&mut self, purpose: [u8; 16]) {
+ self.purpose = purpose;
+ }
+
+ /// Set the transaction inputs hash.
+ fn set_txn_inputs_hash(&mut self, txn_inputs_hash: [u8; 16]) {
+ self.txn_inputs_hash = txn_inputs_hash;
+ }
+
+ /// Set the previous transaction ID.
+ fn set_prv_tx_id(&mut self, prv_tx_id: [u8; 32]) {
+ self.prv_tx_id = Some(prv_tx_id);
+ }
+
+ /// Set the x509 chunks.
+ fn set_x509_chunks(&mut self, x509_chunks: X509Chunks) {
+ self.x509_chunks = x509_chunks;
+ }
+
+ /// Set the validation signature.
+ fn set_validation_signature(&mut self, validation_signature: Vec) {
+ self.validation_signature = validation_signature;
+ }
+}
+
+/// Enum of x509 metadatum with its associated unsigned integer value.
+#[allow(clippy::module_name_repetitions)]
+#[derive(FromRepr, Debug, PartialEq)]
+#[repr(u8)]
+pub enum X509MetadatumInt {
+ /// Purpose.
+ Purpose = 0,
+ /// Transaction inputs hash.
+ TxInputsHash = 1,
+ /// Previous transaction ID.
+ PreviousTxId = 2,
+ /// Validation signature.
+ ValidationSignature = 99,
+}
+
+impl Decode<'_, ()> for X509Metadatum {
+ fn decode(d: &mut Decoder, ctx: &mut ()) -> Result {
+ let map_len = d.map()?.ok_or(decode::Error::message(
+ "Error indefinite array in X509Metadatum",
+ ))?;
+ let mut x509_metadatum = X509Metadatum::new();
+ for _ in 0..map_len {
+ // Use probe to peak
+ let key = d.probe().u8()?;
+
+ if let Some(key) = X509MetadatumInt::from_repr(key) {
+ // Consuming the int
+ d.u8()?;
+ match key {
+ X509MetadatumInt::Purpose => {
+ x509_metadatum.set_purpose(
+ d.bytes()?.try_into().map_err(|_| {
+ decode::Error::message("Invalid data size of Purpose")
+ })?,
+ );
+ },
+ X509MetadatumInt::TxInputsHash => {
+ x509_metadatum.set_txn_inputs_hash(d.bytes()?.try_into().map_err(
+ |_| decode::Error::message("Invalid data size of TxInputsHash"),
+ )?);
+ },
+ X509MetadatumInt::PreviousTxId => {
+ x509_metadatum.set_prv_tx_id(d.bytes()?.try_into().map_err(|_| {
+ decode::Error::message("Invalid data size of PreviousTxId")
+ })?);
+ },
+ X509MetadatumInt::ValidationSignature => {
+ let validation_signature = d.bytes()?;
+ if validation_signature.is_empty() || validation_signature.len() > 64 {
+ return Err(decode::Error::message(
+ "Invalid data size of ValidationSignature",
+ ));
+ }
+ x509_metadatum.set_validation_signature(validation_signature.to_vec());
+ },
+ }
+ } else {
+ // Handle the x509 chunks 10 11 12
+ let x509_chunks = X509Chunks::decode(d, ctx)?;
+ x509_metadatum.set_x509_chunks(x509_chunks);
+ }
+ }
+ Ok(x509_metadatum)
+ }
+}
+
+/// Decode any in CDDL, only support basic datatype
+pub(crate) fn decode_any(d: &mut Decoder) -> Result, decode::Error> {
+ match d.datatype()? {
+ minicbor::data::Type::Bytes => Ok(d.bytes()?.to_vec()),
+ minicbor::data::Type::String => Ok(d.str()?.as_bytes().to_vec()),
+ minicbor::data::Type::Array => {
+ let arr_len = d.array()?.ok_or(decode::Error::message(
+ "Error indefinite length in decoding any",
+ ))?;
+ let mut buffer = vec![];
+ for _ in 0..arr_len {
+ buffer.extend_from_slice(&decode_any(d)?);
+ }
+ Ok(buffer)
+ },
+ minicbor::data::Type::U8
+ | minicbor::data::Type::U16
+ | minicbor::data::Type::U32
+ | minicbor::data::Type::U64 => Ok(d.u64()?.to_be_bytes().to_vec()),
+ minicbor::data::Type::I8
+ | minicbor::data::Type::I16
+ | minicbor::data::Type::I32
+ | minicbor::data::Type::I64 => Ok(d.i64()?.to_be_bytes().to_vec()),
+ _ => Err(decode::Error::message("Data type not supported")),
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/x509/rbac/certs.rs b/rust/cardano-chain-follower/src/metadata/x509/rbac/certs.rs
new file mode 100644
index 00000000000..ae8b954637f
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/x509/rbac/certs.rs
@@ -0,0 +1,93 @@
+//! Certificates for the RBAC metadata.
+
+use c509_certificate::c509::C509;
+use minicbor::{decode, Decode, Decoder};
+use x509_cert::{der::Decode as x509Decode, Certificate};
+
+// ------------------x509------------------------
+
+/// A struct of X509 certificate.
+#[derive(Debug, PartialEq)]
+pub(crate) struct X509DerCert(Vec);
+
+impl Decode<'_, ()> for X509DerCert {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ let data = d.bytes()?;
+ Certificate::from_der(data)
+ .map_err(|_| decode::Error::message("Invalid x509 certificate"))?;
+ Ok(Self(data.to_vec()))
+ }
+}
+
+// ------------------c509-----------------------
+
+/// Enum of possible c509 certificate.
+#[derive(Debug, PartialEq)]
+pub(crate) enum C509Cert {
+ /// A c509 certificate in metadatum reference.
+ C509CertInMetadatumReference(C509CertInMetadatumReference),
+ /// A c509 certificate.
+ C509Certificate(Box),
+}
+
+impl Decode<'_, ()> for C509Cert {
+ fn decode(d: &mut Decoder, ctx: &mut ()) -> Result {
+ if d.datatype()? == minicbor::data::Type::Array {
+ let arr_len = d
+ .array()?
+ .ok_or(decode::Error::message("Error indefinite array in C509Cert"))?;
+ // C509CertInMetadatumReference must have 3 items
+ if arr_len == 3 {
+ Ok(Self::C509CertInMetadatumReference(
+ C509CertInMetadatumReference::decode(d, ctx)?,
+ ))
+ } else {
+ Err(decode::Error::message(
+ "Invalid length C509CertInMetadatumReference, expected 3",
+ ))
+ }
+ } else {
+ // Consuming the c509 bytes
+ let c509 = d.bytes()?;
+ let mut c509_d = Decoder::new(c509);
+ Ok(Self::C509Certificate(Box::new(C509::decode(
+ &mut c509_d,
+ ctx,
+ )?)))
+ }
+ }
+}
+
+/// A struct of c509 certificate in metadatum reference.
+#[derive(Debug, PartialEq)]
+pub(crate) struct C509CertInMetadatumReference {
+ /// Transaction output field.
+ txn_output_field: u8,
+ /// Transaction output index.
+ txn_output_index: u64,
+ /// Optional certificate reference.
+ cert_ref: Option>,
+}
+
+impl Decode<'_, ()> for C509CertInMetadatumReference {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ let txn_output_field = d.u8()?;
+ let txn_output_index = d.u64()?;
+ let cert_ref = match d.datatype()? {
+ minicbor::data::Type::Array => {
+ let len = d.array()?.ok_or(decode::Error::message(
+ "Error indefinite array in C509CertInMetadatumReference",
+ ))?;
+ let arr: Result, _> = (0..len).map(|_| d.u64()).collect();
+ arr.map(Some)
+ },
+ minicbor::data::Type::Null => Ok(None),
+ _ => Ok(Some(vec![d.u64()?])),
+ }?;
+ Ok(Self {
+ txn_output_field,
+ txn_output_index,
+ cert_ref,
+ })
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/x509/rbac/mod.rs b/rust/cardano-chain-follower/src/metadata/x509/rbac/mod.rs
new file mode 100644
index 00000000000..2075abb6a21
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/x509/rbac/mod.rs
@@ -0,0 +1,170 @@
+//! Role Based Access Control (RBAC) metadata for X509 certificates.
+//! Doc Reference:
+//! CDDL Reference:
+
+mod certs;
+mod pub_key;
+mod role_data;
+
+use std::collections::HashMap;
+
+use certs::{C509Cert, X509DerCert};
+use minicbor::{decode, Decode, Decoder};
+use pub_key::SimplePublicKeyType;
+use role_data::RoleData;
+use strum::FromRepr;
+
+use super::decode_any;
+
+/// Struct of x509 RBAC metadata.
+#[derive(Debug, PartialEq)]
+pub(crate) struct X509RbacMetadata {
+ /// Optional list of x509 certificates.
+ x509_certs: Option>,
+ /// Optional list of c509 certificates.
+ /// The value can be either the c509 certificate or c509 metadatum reference.
+ c509_certs: Option>,
+ /// Optional list of Public keys.
+ pub_keys: Option>,
+ /// Optional list of revocation list.
+ revocation_list: Option>,
+ /// Optional list of role data.
+ role_set: Option>,
+ /// Optional map of purpose key data.
+ /// Empty map if no purpose key data is present.
+ purpose_key_data: HashMap>,
+}
+
+/// The first valid purpose key.
+const FIRST_PURPOSE_KEY: u16 = 200;
+/// The last valid purpose key.
+const LAST_PURPOSE_KEY: u16 = 299;
+
+/// Enum of x509 RBAC metadata with its associated unsigned integer value.
+#[derive(FromRepr, Debug, PartialEq)]
+#[repr(u16)]
+pub enum X509RbacMetadataInt {
+ /// x509 certificates.
+ X509Certs = 10,
+ /// c509 certificates.
+ C509Certs = 20,
+ /// Public keys.
+ PubKeys = 30,
+ /// Revocation list.
+ RevocationList = 40,
+ /// Role data set.
+ RoleSet = 100,
+}
+
+impl X509RbacMetadata {
+ /// Create a new instance of `X509RbacMetadata`.
+ pub(crate) fn new() -> Self {
+ Self {
+ x509_certs: None,
+ c509_certs: None,
+ pub_keys: None,
+ revocation_list: None,
+ role_set: None,
+ purpose_key_data: HashMap::new(),
+ }
+ }
+
+ /// Set the x509 certificates.
+ fn set_x509_certs(&mut self, x509_certs: Vec) {
+ self.x509_certs = Some(x509_certs);
+ }
+
+ /// Set the c509 certificates.
+ fn set_c509_certs(&mut self, c509_certs: Vec) {
+ self.c509_certs = Some(c509_certs);
+ }
+
+ /// Set the public keys.
+ fn set_pub_keys(&mut self, pub_keys: Vec) {
+ self.pub_keys = Some(pub_keys);
+ }
+
+ /// Set the revocation list.
+ fn set_revocation_list(&mut self, revocation_list: Vec<[u8; 16]>) {
+ self.revocation_list = Some(revocation_list);
+ }
+
+ /// Set the role data set.
+ fn set_role_set(&mut self, role_set: Vec) {
+ self.role_set = Some(role_set);
+ }
+}
+
+impl Decode<'_, ()> for X509RbacMetadata {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ let map_len = d.map()?.ok_or(decode::Error::message(
+ "Error indefinite map in X509RbacMetadata",
+ ))?;
+ let mut x509_rbac_metadata = X509RbacMetadata::new();
+ for _ in 0..map_len {
+ let key = d.u16()?;
+ if let Some(key) = X509RbacMetadataInt::from_repr(key) {
+ match key {
+ X509RbacMetadataInt::X509Certs => {
+ let x509_certs = decode_array(d)?;
+ x509_rbac_metadata.set_x509_certs(x509_certs);
+ },
+ X509RbacMetadataInt::C509Certs => {
+ let c509_certs = decode_array(d)?;
+ x509_rbac_metadata.set_c509_certs(c509_certs);
+ },
+ X509RbacMetadataInt::PubKeys => {
+ let pub_keys = decode_array(d)?;
+ x509_rbac_metadata.set_pub_keys(pub_keys);
+ },
+ X509RbacMetadataInt::RevocationList => {
+ let revocation_list = decode_revocation_list(d)?;
+ x509_rbac_metadata.set_revocation_list(revocation_list);
+ },
+ X509RbacMetadataInt::RoleSet => {
+ let role_set = decode_array(d)?;
+ x509_rbac_metadata.set_role_set(role_set);
+ },
+ }
+ } else {
+ if !(FIRST_PURPOSE_KEY..=LAST_PURPOSE_KEY).contains(&key) {
+ return Err(decode::Error::message(format!("Invalid purpose key set, should be with the range {FIRST_PURPOSE_KEY} - {LAST_PURPOSE_KEY}")));
+ }
+ x509_rbac_metadata
+ .purpose_key_data
+ .insert(key, decode_any(d)?);
+ }
+ }
+ Ok(x509_rbac_metadata)
+ }
+}
+
+/// Decode an array of type T.
+fn decode_array<'b, T>(d: &mut Decoder<'b>) -> Result, decode::Error>
+where T: Decode<'b, ()> {
+ let len = d.array()?.ok_or(decode::Error::message(
+ "Error indefinite array in X509RbacMetadata",
+ ))?;
+ let mut vec = Vec::with_capacity(usize::try_from(len).map_err(decode::Error::message)?);
+ for _ in 0..len {
+ vec.push(T::decode(d, &mut ())?);
+ }
+ Ok(vec)
+}
+
+/// Decode an array of revocation list.
+fn decode_revocation_list(d: &mut Decoder) -> Result, decode::Error> {
+ let len = d.array()?.ok_or(decode::Error::message(
+ "Error indefinite array in X509RbacMetadata revocation list",
+ ))?;
+ let mut revocation_list =
+ Vec::with_capacity(usize::try_from(len).map_err(decode::Error::message)?);
+ for _ in 0..len {
+ let arr: [u8; 16] = d
+ .bytes()?
+ .try_into()
+ .map_err(|_| decode::Error::message("Invalid revocation list size"))?;
+ revocation_list.push(arr);
+ }
+ Ok(revocation_list)
+}
diff --git a/rust/cardano-chain-follower/src/metadata/x509/rbac/pub_key.rs b/rust/cardano-chain-follower/src/metadata/x509/rbac/pub_key.rs
new file mode 100644
index 00000000000..f801bf8ef84
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/x509/rbac/pub_key.rs
@@ -0,0 +1,66 @@
+//! Public key type for RBAC metadata
+
+use minicbor::{data::Tag, decode, Decode, Decoder};
+
+/// Enum of possible public key type.
+#[derive(Debug, PartialEq)]
+pub(crate) enum SimplePublicKeyType {
+ /// Undefined indicates skipped element.
+ Undefined,
+ /// Deleted indicates the key is deleted.
+ Deleted,
+ /// Ed25519 key.
+ Ed25519([u8; 32]),
+}
+
+/// Enum of possible public key tag.
+enum PublicKeyTag {
+ /// Deleted Key tag 31.
+ Deleted,
+ /// Ed25519 Key tag 32773.
+ Ed25519,
+}
+
+impl PublicKeyTag {
+ /// Get the tag value.
+ fn tag(self) -> Tag {
+ match self {
+ PublicKeyTag::Deleted => Tag::new(0x31),
+ PublicKeyTag::Ed25519 => Tag::new(0x8005),
+ }
+ }
+}
+
+impl Decode<'_, ()> for SimplePublicKeyType {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ match d.datatype()? {
+ minicbor::data::Type::Tag => {
+ let tag = d.tag()?;
+ match tag {
+ t if t == PublicKeyTag::Deleted.tag() => Ok(SimplePublicKeyType::Deleted),
+ t if t == PublicKeyTag::Ed25519.tag() => {
+ let bytes = d.bytes()?;
+ let mut ed25519 = [0u8; 32];
+ if bytes.len() == 32 {
+ ed25519.copy_from_slice(bytes);
+ Ok(SimplePublicKeyType::Ed25519(ed25519))
+ } else {
+ Err(decode::Error::message("Invalid length for Ed25519 key"))
+ }
+ },
+ _ => {
+ Err(decode::Error::message(
+ "Unknown tag for SimplePublicKeyType",
+ ))
+ },
+ }
+ },
+ minicbor::data::Type::Undefined => Ok(SimplePublicKeyType::Undefined),
+ _ => {
+ Err(decode::Error::message(
+ "Invalid datatype for SimplePublicKeyType",
+ ))
+ },
+ }
+ }
+}
diff --git a/rust/cardano-chain-follower/src/metadata/x509/rbac/role_data.rs b/rust/cardano-chain-follower/src/metadata/x509/rbac/role_data.rs
new file mode 100644
index 00000000000..630909804a4
--- /dev/null
+++ b/rust/cardano-chain-follower/src/metadata/x509/rbac/role_data.rs
@@ -0,0 +1,168 @@
+//! Role data for RBAC metadata.
+
+use std::collections::HashMap;
+
+use minicbor::{decode, Decode, Decoder};
+use strum::FromRepr;
+
+use super::X509RbacMetadataInt;
+use crate::metadata::x509::decode_any;
+
+/// Struct of role data.
+#[derive(Debug, PartialEq)]
+pub(crate) struct RoleData {
+ /// Role number.
+ role_number: u8,
+ /// Optional role signing key.
+ role_signing_key: Option,
+ /// Optional role encryption key.
+ role_encryption_key: Option,
+ /// Optional payment key.
+ payment_key: Option,
+ /// Optional role extended data keys.
+ /// Empty map if no role extended data keys.
+ role_extended_data_keys: HashMap>,
+}
+
+/// The first valid role extended data key.
+const FIRST_ROLE_EXT_KEY: u8 = 10;
+/// The last valid role extended data key.
+const LAST_ROLE_EXT_KEY: u8 = 99;
+
+/// Enum of role data with its associated unsigned integer value.
+#[allow(clippy::module_name_repetitions)]
+#[derive(FromRepr, Debug, PartialEq)]
+#[repr(u8)]
+pub enum RoleDataInt {
+ /// Role number.
+ RoleNumber = 0,
+ /// Role signing key.
+ RoleSigningKey = 1,
+ /// Role encryption key.
+ RoleEncryptionKey = 2,
+ /// Payment key.
+ PaymentKey = 3,
+}
+
+#[allow(clippy::module_name_repetitions)]
+impl RoleData {
+ /// Create a new instance of `RoleData`.
+ fn new() -> Self {
+ Self {
+ role_number: 0,
+ role_signing_key: None,
+ role_encryption_key: None,
+ payment_key: None,
+ role_extended_data_keys: HashMap::new(),
+ }
+ }
+
+ /// Set the role number.
+ fn set_role_number(&mut self, role_number: u8) {
+ self.role_number = role_number;
+ }
+
+ /// Set the role signing key.
+ fn set_role_signing_key(&mut self, key: KeyReference) {
+ self.role_signing_key = Some(key);
+ }
+
+ /// Set the role encryption key.
+ fn set_role_encryption_key(&mut self, key: KeyReference) {
+ self.role_encryption_key = Some(key);
+ }
+
+ /// Set the payment key.
+ fn set_payment_key(&mut self, key: u64) {
+ self.payment_key = Some(key);
+ }
+}
+
+impl Decode<'_, ()> for RoleData {
+ fn decode(d: &mut Decoder, ctx: &mut ()) -> Result {
+ let map_len = d
+ .map()?
+ .ok_or(decode::Error::message("role set has indefinite length"))?;
+ let mut role_data = RoleData::new();
+ for _ in 0..map_len {
+ let key = d.u8()?;
+ if let Some(key) = RoleDataInt::from_repr(key) {
+ match key {
+ RoleDataInt::RoleNumber => {
+ role_data.set_role_number(d.u8()?);
+ },
+ RoleDataInt::RoleSigningKey => {
+ role_data.set_role_signing_key(KeyReference::decode(d, ctx)?);
+ },
+ RoleDataInt::RoleEncryptionKey => {
+ role_data.set_role_encryption_key(KeyReference::decode(d, ctx)?);
+ },
+ RoleDataInt::PaymentKey => {
+ role_data.set_payment_key(d.u64()?);
+ },
+ }
+ } else {
+ if !(FIRST_ROLE_EXT_KEY..=LAST_ROLE_EXT_KEY).contains(&key) {
+ return Err(decode::Error::message(format!("Invalid role extended data key, should be with the range {FIRST_ROLE_EXT_KEY} - {LAST_ROLE_EXT_KEY}")));
+ }
+ role_data
+ .role_extended_data_keys
+ .insert(key, decode_any(d)?);
+ }
+ }
+ Ok(role_data)
+ }
+}
+
+/// Enum of key reference.
+#[derive(Debug, PartialEq)]
+enum KeyReference {
+ /// Key local reference.
+ KeyLocalRef(KeyLocalRef),
+ /// Key hash.
+ KeyHash(Vec),
+}
+
+impl Decode<'_, ()> for KeyReference {
+ fn decode(d: &mut Decoder, ctx: &mut ()) -> Result {
+ match d.datatype()? {
+ minicbor::data::Type::Array => Ok(Self::KeyLocalRef(KeyLocalRef::decode(d, ctx)?)),
+ minicbor::data::Type::Bytes => Ok(Self::KeyHash(d.bytes()?.to_vec())),
+ _ => Err(decode::Error::message("Invalid data type for KeyReference")),
+ }
+ }
+}
+
+/// Struct of key local reference.
+#[derive(Debug, PartialEq)]
+struct KeyLocalRef {
+ /// Local reference.
+ local_ref: LocalRefInt,
+ /// Key offset.
+ key_offset: u64,
+}
+
+/// Enum of local reference with its associated unsigned integer value.
+#[derive(FromRepr, Debug, PartialEq)]
+#[repr(u8)]
+enum LocalRefInt {
+ /// x509 certificates.
+ X509Certs = X509RbacMetadataInt::X509Certs as u8, // 10
+ /// c509 certificates.
+ C509Certs = X509RbacMetadataInt::C509Certs as u8, // 20
+ /// Public keys.
+ PubKeys = X509RbacMetadataInt::PubKeys as u8, // 30
+}
+
+impl Decode<'_, ()> for KeyLocalRef {
+ fn decode(d: &mut Decoder, _ctx: &mut ()) -> Result {
+ d.array()?;
+ let local_ref = LocalRefInt::from_repr(d.u8()?)
+ .ok_or(decode::Error::message("Invalid local reference"))?;
+ let key_offset = d.u64()?;
+ Ok(Self {
+ local_ref,
+ key_offset,
+ })
+ }
+}
diff --git a/rust/cardano-chain-follower/src/mithril_query.rs b/rust/cardano-chain-follower/src/mithril_query.rs
new file mode 100644
index 00000000000..7c6e38a134e
--- /dev/null
+++ b/rust/cardano-chain-follower/src/mithril_query.rs
@@ -0,0 +1,47 @@
+//! Functions which query or interact with the Immutable blockchain on disk.
+
+use std::path::Path;
+
+use pallas_hardano::storage::immutable::FallibleBlock;
+use tokio::task;
+
+use crate::{
+ error::{Error, Result},
+ Point,
+};
+
+/// Synchronous Immutable block iterator.
+pub(crate) type ImmutableBlockIterator = Box + Send + Sync>;
+
+/// Get a mithril snapshot iterator.
+pub(crate) async fn make_mithril_iterator(
+ path: &Path, start: &Point,
+) -> Result {
+ let path = path.to_path_buf();
+ let start = start.clone();
+ // Initial input
+ let res = task::spawn_blocking(move || {
+ pallas_hardano::storage::immutable::read_blocks_from_point(&path, start.clone().into())
+ .map_err(|error| Error::MithrilSnapshot(Some(error)))
+ })
+ .await;
+
+ match res {
+ Ok(res) => res,
+ Err(_error) => Err(Error::MithrilSnapshot(None)),
+ }
+}
+
+/// Get latest TIP of the Mithril Immutable Chain.
+pub(crate) async fn get_mithril_tip_point(path: &Path) -> Result {
+ let path = path.to_path_buf();
+ let res =
+ task::spawn_blocking(move || pallas_hardano::storage::immutable::get_tip(&path)).await;
+
+ match res {
+ Ok(Ok(Some(res))) => Ok(res.into()),
+ Ok(Ok(None)) => Err(Error::MithrilSnapshot(None)),
+ Ok(Err(error)) => Err(Error::MithrilSnapshot(Some(error))),
+ Err(_error) => Err(Error::MithrilSnapshot(None)),
+ }
+}
diff --git a/rust/cardano-chain-follower/src/mithril_snapshot.rs b/rust/cardano-chain-follower/src/mithril_snapshot.rs
index 94c51121652..4c59cb51dd1 100644
--- a/rust/cardano-chain-follower/src/mithril_snapshot.rs
+++ b/rust/cardano-chain-follower/src/mithril_snapshot.rs
@@ -1,137 +1,47 @@
//! Internal Mithril snapshot functions.
-use std::path::PathBuf;
-
-use pallas::network::miniprotocols::Point;
-use pallas_hardano::storage::immutable::FallibleBlock;
-
-use crate::{Error, MultiEraBlockData, Result};
-
-/// Wraps the iterator type returned by Pallas.
-pub(crate) struct MithrilSnapshotIterator {
- /// Inner iterator.
- inner: Box + Send + Sync>,
-}
-
-impl Iterator for MithrilSnapshotIterator {
- type Item = FallibleBlock;
-
- fn next(&mut self) -> Option {
- self.inner.next()
- }
-}
+use logcall::logcall;
+use tracing_log::log;
+
+use crate::{
+ mithril_snapshot_data::latest_mithril_snapshot_id,
+ mithril_snapshot_iterator::MithrilSnapshotIterator, network::Network, MultiEraBlock, Point,
+};
+
+// Any single program using this crate can have EXACTLY THREE Mithril snapshots.
+// One, for each of the known networks.
+// If more mithril snapshots are configured, then the crate will error.
+// It IS possible to redundantly configure mithril snapshots, provided they are
+// identically configured. The only config option that can change, is if the snapshot is
+// auto-updated, ANY follower which sets this enables this function and it can not be
+// disabled once started without stopping the program.
/// Holds information about a Mithril snapshot.
#[derive(Clone)]
pub(crate) struct MithrilSnapshot {
- /// Path to the Mithril snapshot.
- pub path: PathBuf,
- /// Snapshot's tip.
- pub tip: Point,
+ /// Network that this snapshot is configured for
+ chain: Network,
}
impl MithrilSnapshot {
- /// Gets information about the snapshot at the given path.
- ///
- /// # Arguments
- ///
- /// * `path`: Mithril snapshot path.
- ///
- /// # Errors
- ///
- /// Returns Err if it can't read where the tip is at in the snapshot or
- /// if reading the snapshot files fails.
- pub fn from_path(path: PathBuf) -> Result {
- let tip = pallas_hardano::storage::immutable::get_tip(&path)
- .map_err(|_| Error::MithrilSnapshot)?
- .ok_or(Error::MithrilSnapshot)?;
-
- Ok(Self { path, tip })
+ /// Create a new Mithril Snapshot handler
+ pub(crate) fn new(chain: Network) -> Self {
+ Self { chain }
}
- /// Tries reading a block from the Mithril snapshot. Returns None if the point
- /// is not contained in the snapshot.
+ /// Checks if the snapshot contains a given point.
///
/// # Arguments
+ /// * `network`: The network that this function should check against.
+ /// * `point`: The point to be checked for existence within the specified Mithril
+ /// snapshot.
///
- /// * `point`: Point at which to read the block.
- ///
- /// # Errors
- ///
- /// Returns Err if anything fails while reading the block data.
- pub fn try_read_block(&self, point: Point) -> Result> {
- if !self.contains_point(&point) {
- return Ok(None);
- }
-
- let mut block_data_iter =
- pallas_hardano::storage::immutable::read_blocks_from_point(&self.path, point)
- .map_err(|_| Error::MithrilSnapshot)?;
-
- match block_data_iter.next() {
- Some(res) => {
- let block_data = res.map_err(|_| Error::MithrilSnapshot)?;
+ /// Returns true if the point exists within the Mithril snapshot for the specified
+ /// network, false otherwise.
+ pub(crate) fn contains_point(&self, point: &Point) -> bool {
+ let latest_id = latest_mithril_snapshot_id(self.chain);
- Ok(Some(MultiEraBlockData(block_data)))
- },
- None => Ok(None),
- }
- }
-
- /// Tries reading a range of blocks from the Mithril snapshot.
- /// Returns None if the range is not contained in the snapshot.
- ///
- /// This returns the last point that was read. This is useful to check
- /// if the range was partially read.
- ///
- /// # Arguments
- ///
- /// * `from`: Start point.
- /// * `to`: End point.
- ///
- /// # Errors
- ///
- /// Returns Err if anything fails while reading any block's data.
- #[allow(clippy::needless_pass_by_value)]
- pub fn try_read_block_range(
- &self, from: Point, to: Point,
- ) -> Result)>> {
- if !self.contains_point(&from) {
- return Ok(None);
- }
-
- let blocks_iter =
- pallas_hardano::storage::immutable::read_blocks_from_point(&self.path, from)
- .map_err(|_| Error::MithrilSnapshot)?;
-
- let mut block_data_vec = Vec::new();
- for result in blocks_iter {
- let block_data = MultiEraBlockData(result.map_err(|_| Error::MithrilSnapshot)?);
-
- // TODO(fsgr): Should we check the hash as well?
- // Maybe throw an error if we don't get the block we were expecting at that
- // slot?
- if block_data.decode()?.slot() > to.slot_or_default() {
- break;
- }
-
- block_data_vec.push(block_data);
- }
-
- // Get the point from last block read.
- // Pop here to get an owned value (we'll insert it back later).
- match block_data_vec.pop() {
- Some(last_block_data) => {
- let last_block = last_block_data.decode()?;
- let last_block_point = Point::new(last_block.slot(), last_block.hash().to_vec());
-
- // Push the last block data back
- block_data_vec.push(last_block_data);
-
- Ok(Some((last_block_point, block_data_vec)))
- },
- None => Ok(None),
- }
+ point.slot_or_default() <= latest_id.tip().slot_or_default()
}
/// Tries get an iterator that reads blocks from the Mithril snapshot from a given
@@ -143,26 +53,38 @@ impl MithrilSnapshot {
///
/// # Errors
///
- /// Returns Err if anything fails while trying to find the starting point in the
- /// snapshot.
- pub fn try_read_blocks_from_point(&self, point: Point) -> Option {
- if !self.contains_point(&point) {
+ /// Returns None if its not possible to iterate a mithril snapshot from the requested
+ /// point for ANY reason.
+ #[allow(clippy::indexing_slicing)]
+ #[logcall("debug")]
+ pub(crate) async fn try_read_blocks_from_point(
+ &self, point: &Point,
+ ) -> Option {
+ let snapshot_id = latest_mithril_snapshot_id(self.chain);
+ let snapshot_path = snapshot_id.immutable_path();
+
+ // Quick check if the block can be within the immutable data.
+ if !self.contains_point(point) {
return None;
}
- let iter = pallas_hardano::storage::immutable::read_blocks_from_point(&self.path, point)
- .map_err(|_| Error::MithrilSnapshot)
- .ok()?;
-
- Some(MithrilSnapshotIterator { inner: iter })
+ // We don't know the previous block, so we need to find it.
+ MithrilSnapshotIterator::new(self.chain, &snapshot_path, point, None)
+ .await
+ .ok()
}
- /// Naively checks if the snapshot contains a point.
- ///
- /// # Arguments
- ///
- /// * `point`: Point to check.
- pub fn contains_point(&self, point: &Point) -> bool {
- point.slot_or_default() <= self.tip.slot_or_default()
+ /// Read a single block from a known point.
+ #[allow(clippy::indexing_slicing)]
+ #[logcall("debug")]
+ pub(crate) async fn read_block_at(&self, point: &Point) -> Option {
+ if let Some(iterator) = self.try_read_blocks_from_point(point).await {
+ let block = iterator.next().await;
+ return block;
+ }
+ None
}
}
+
+#[cfg(test)]
+mod tests {}
diff --git a/rust/cardano-chain-follower/src/mithril_snapshot_config.rs b/rust/cardano-chain-follower/src/mithril_snapshot_config.rs
new file mode 100644
index 00000000000..b317d9a6f91
--- /dev/null
+++ b/rust/cardano-chain-follower/src/mithril_snapshot_config.rs
@@ -0,0 +1,528 @@
+//! Configuration for the Mithril Snapshot used by the follower.
+
+use std::{
+ path::{Path, PathBuf},
+ str::FromStr,
+ sync::LazyLock,
+};
+
+use anyhow::bail;
+use dashmap::DashMap;
+use futures::future::join_all;
+use strum::IntoEnumIterator;
+use tokio::{
+ fs::{self},
+ io::{self},
+ sync::{mpsc, Mutex},
+ task::JoinHandle,
+};
+use tracing::{debug, error};
+
+use crate::{
+ error::{Error, Result},
+ mithril_snapshot_data::{latest_mithril_snapshot_id, SnapshotData},
+ mithril_snapshot_sync::background_mithril_update,
+ network::Network,
+ point::ORIGIN_POINT,
+ snapshot_id::SnapshotId,
+ turbo_downloader::DlConfig,
+ Point,
+};
+
+/// Type we use to manage the Sync Task handle map.
+type SyncMap = DashMap>>>;
+/// Handle to the mithril sync thread. One for each Network ONLY.
+static SYNC_JOIN_HANDLE_MAP: LazyLock = LazyLock::new(|| {
+ let map = DashMap::new();
+ for network in Network::iter() {
+ map.insert(network, Mutex::new(None));
+ }
+ map
+});
+
+/// Subdirectory where we unpack archives temporarily.
+const TMP_SUB_DIR: &str = "tmp";
+
+/// Message we send when Mithril Snapshot updates
+#[derive(Debug)]
+pub(crate) struct MithrilUpdateMessage {
+ /// The largest block on the mithril snapshot.
+ pub tip: Point,
+ /// The block immediately before it.
+ pub previous: Point,
+}
+
+/// Configuration used for the Mithril Snapshot downloader.
+#[derive(Clone, Debug)]
+pub struct MithrilSnapshotConfig {
+ /// What Blockchain network are we configured for.
+ pub chain: Network,
+ /// Path to the Mithril snapshot the follower should use.
+ /// Note: this is a base directory. The Actual data will be stored under here.
+ /// archive downloads -> `/dl`
+ /// unpacked snapshots -> `/`
+ /// extracting snapshots -> `/tmp`
+ pub path: PathBuf,
+ /// Address of the Mithril Aggregator to use to find the latest snapshot data to
+ /// download.
+ pub aggregator_url: String,
+ /// The Genesis Key needed for a network to do Mithril snapshot validation.
+ pub genesis_key: String,
+ /// Downloader configuration.
+ pub dl_config: Option,
+}
+
+impl MithrilSnapshotConfig {
+ /// Sets the defaults for a given cardano network.
+ /// Each network has a different set of defaults, so no single "default" can apply.
+ /// This function is preferred to the `default()` standard function.
+ #[must_use]
+ pub fn default_for(chain: Network) -> Self {
+ Self {
+ chain,
+ path: chain.default_mithril_path(),
+ aggregator_url: chain.default_mithril_aggregator(),
+ genesis_key: chain.default_mithril_genesis_key(),
+ dl_config: None,
+ }
+ }
+
+ /// Set a custom downloader configuration.
+ pub fn with_dl_config(mut self, config: DlConfig) -> Self {
+ self.dl_config = Some(config);
+ self
+ }
+
+ /// Try and recover the latest snapshot id from the files on disk.
+ #[must_use]
+ pub(crate) async fn recover_latest_snapshot_id(&self) -> Option {
+ // Can we read directory entries from the base path, if not then there is no latest
+ // snapshot.
+ let path = self.path.clone();
+ debug!("Recovering latest snapshot id from {:?}", &path);
+
+ let Ok(mut entries) = fs::read_dir(&self.path).await else {
+ error!(
+ "Getting latest snapshot failed: Can't read entries from {}",
+ self.path.to_string_lossy()
+ );
+ return None;
+ };
+
+ let mut latest_immutable_file: u64 = 0; // Can't have a 0 file.
+ let mut latest_path = PathBuf::new();
+
+ loop {
+ // Get the next entry, stop on any error, or no entries left.
+ let Ok(Some(entry)) = entries.next_entry().await else {
+ break;
+ };
+
+ if let Some(immutable_file) = SnapshotId::parse_path(&entry.path()) {
+ if immutable_file > latest_immutable_file {
+ latest_immutable_file = immutable_file;
+ latest_path = entry.path();
+ }
+ }
+ }
+
+ if latest_immutable_file > 0 {
+ return SnapshotId::try_new(self.chain, &latest_path).await;
+ }
+
+ None
+ }
+
+ /// Activate the tmp mithril path to a numbered snapshot path.
+ /// And then remove any left over files in download or the tmp path, or old snapshots.
+ pub(crate) async fn activate(&self, snapshot_number: u64) -> io::Result {
+ let new_path = self.mithril_path(snapshot_number);
+ let latest_id = latest_mithril_snapshot_id(self.chain);
+
+ debug!(
+ "Activating snapshot: {} {} {:?}",
+ snapshot_number,
+ new_path.to_string_lossy(),
+ latest_id
+ );
+
+ // Can't activate anything if the tmp directory does not exist.
+ if !self.tmp_path().is_dir() {
+ error!("No tmp path found to activate.");
+ return Err(io::Error::new(io::ErrorKind::NotFound, "No tmp path found"));
+ }
+
+ // Check if we would actually be making a newer snapshot active. (Should never fail, but
+ // check anyway.)
+ if latest_id >= snapshot_number {
+ error!("Latest snapshot {latest_id:?} is >= than requested snapshot {snapshot_number}");
+ return Err(io::Error::new(
+ io::ErrorKind::NotFound,
+ "Latest snapshot is newer or equal",
+ ));
+ }
+
+ // Rename the tmp path to the new numbered path.
+ fs::rename(self.tmp_path(), &new_path).await?;
+
+ Ok(new_path)
+ }
+
+ /// Cleanup the tmp mithril path, all old mithril paths and the dl path.
+ /// Removes those directories if they exist and all the files they contain.
+ pub(crate) async fn cleanup(&self) -> io::Result<()> {
+ let mut cleanup_tasks = Vec::new();
+
+ // Cleanup up the tmp path. (Shouldn't normally exist, but clean it anyway)
+ let tmp = self.tmp_path();
+ if tmp.exists() {
+ debug!("Cleaning up TMP @ {}", tmp.display());
+ cleanup_tasks.push(fs::remove_dir_all(tmp.clone()));
+ }
+
+ // Cleanup all numbered paths which are not this latest path
+ match fs::read_dir(&self.path).await {
+ Err(err) => {
+ error!(
+ "Unexpected failure reading entries in the mithril path {} : {}",
+ self.path.to_string_lossy(),
+ err
+ );
+ },
+ Ok(mut entries) => {
+ // Get latest mithril snapshot path and number.
+ let latest_snapshot = latest_mithril_snapshot_id(self.chain);
+
+ loop {
+ // Get the next entry, stop on any error, or no entries left.
+ let Ok(Some(entry)) = entries.next_entry().await else {
+ break;
+ };
+
+ // If None, its not a snapshot path, so continue.
+ if let Some(this_snapshot) = SnapshotId::new(&entry.path(), ORIGIN_POINT) {
+ // Don't do anything with the latest snapshot.
+ // Comparison does NOT use `tip` so we construct a temporary ID without it.
+ if this_snapshot != latest_snapshot {
+ debug!(
+ "Cleaning up non-latest snapshot @ {}",
+ entry.path().display()
+ );
+ cleanup_tasks.push(fs::remove_dir_all(entry.path()));
+ }
+ };
+ }
+ },
+ }
+
+ for result in join_all(cleanup_tasks).await {
+ match result {
+ Ok(()) => (),
+ Err(err) => {
+ error!("Failed to cleanup snapshot: {err:?}");
+ },
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Deduplicate a file in the tmp directory vs its equivalent in the current snapshot.
+ ///
+ /// This does not check if they SHOULD be de-duped, only de-dupes the files specified.
+ pub(crate) fn dedup_tmp(
+ &self, tmp_file: &Path, latest_snapshot: &SnapshotData,
+ ) -> anyhow::Result<()> {
+ // Get the matching src file in the latest mithril snapshot to compare against.
+ let snapshot_path = latest_snapshot.id().as_ref();
+ let tmp_path = self.tmp_path();
+
+ let Ok(relative_file) = tmp_file.strip_prefix(&tmp_path) else {
+ error!("Failed to get relative path of file.");
+ bail!("Failed to strip prefix: {tmp_path:?}");
+ };
+
+ // IF we make it here, the files are identical, so we can de-dup them safely.
+ // Remove the tmp file momentarily.
+ if tmp_file.exists() {
+ if let Err(error) = std::fs::remove_file(tmp_file) {
+ error!(
+ "Error removing tmp file {} : {}",
+ tmp_file.to_string_lossy(),
+ error
+ );
+ bail!("Failed to remove tmp file: {tmp_file:?}");
+ }
+ }
+
+ let src_file = snapshot_path.join(relative_file);
+ let src_file = src_file.as_path();
+ // Hardlink the src file to the tmp file.
+ if let Some(parent) = tmp_file.parent() {
+ if let Err(error) = std::fs::create_dir_all(parent) {
+ error!("Error creating parent dir {parent:?} for tmp file {tmp_file:?}: {error}");
+ }
+ }
+ if let Err(error) = std::fs::hard_link(src_file, tmp_file) {
+ error!(
+ "Error linking src file {} to tmp file {} : {}",
+ src_file.to_string_lossy(),
+ tmp_file.to_string_lossy(),
+ error
+ );
+ bail!("Failed to link src file: {src_file:?}");
+ }
+
+ // And if we made it here, file was successfully de-duped. YAY.
+ debug!("DeDup OK: {tmp_file:?}");
+ Ok(())
+ }
+
+ /// Returns the path to Latest Tmp Snapshot Data.
+ /// Will use a path relative to mithril data path.
+ #[must_use]
+ pub(crate) fn tmp_path(&self) -> PathBuf {
+ let mut snapshot_path = self.path.clone();
+ snapshot_path.push(TMP_SUB_DIR);
+ snapshot_path
+ }
+
+ /// Returns the path to the Numbered Snapshot Data.
+ /// Will use a path relative to mithril data path.
+ #[must_use]
+ pub(crate) fn mithril_path(&self, snapshot_number: u64) -> PathBuf {
+ let mut snapshot_path = self.path.clone();
+ snapshot_path.push(snapshot_number.to_string());
+ snapshot_path
+ }
+
+ /// Check if the Mithril Snapshot Path is valid an usable.
+ async fn validate_path(&self) -> Result<()> {
+ let path = self.path.clone();
+ debug!(
+ path = path.to_string_lossy().to_string(),
+ "Validating Mithril Snapshot Path"
+ );
+
+ // If the path does not exist, try and make it.
+ if !path.exists() {
+ // Try and make the directory.
+ fs::create_dir_all(&path)
+ .await
+ .map_err(|e| Error::MithrilSnapshotDirectoryCreation(path.clone(), e))?;
+ }
+
+ // If the path is NOT a directory, then we can't use it.
+ if !path.is_dir() {
+ return Err(Error::MithrilSnapshotDirectoryNotFound(
+ path.display().to_string(),
+ ));
+ }
+
+ // If the directory is not writable then we can't use
+ if !check_writable(&path) {
+ return Err(Error::MithrilSnapshotDirectoryNotWritable(path.clone()));
+ }
+
+ Ok(())
+ }
+
+ /// Validate the Genesis VKEY is at least the correct kind of data.
+ fn validate_genesis_vkey(&self) -> Result<()> {
+ // First sanitize the vkey by removing all whitespace and make sure its actually valid
+ // hex.
+ let vkey = remove_whitespace(&self.genesis_key);
+ if !is_hex(&vkey) {
+ return Err(Error::MithrilGenesisVKeyNotHex(self.chain));
+ }
+
+ Ok(())
+ }
+
+ /// Validate the Aggregator is resolvable and responsive.
+ async fn validate_aggregator_url(&self) -> Result<()> {
+ let url = self.aggregator_url.clone();
+ let key = self.genesis_key.clone();
+
+ debug!(url = url, "Validating Aggregator URL");
+
+ // Not configured already, and not already in use, so make sure its valid.
+ // We do this by trying to use it to get a list of snapshots.
+ let client = mithril_client::ClientBuilder::aggregator(&url, &key)
+ .build()
+ .map_err(|e| Error::MithrilClient(self.chain, url.clone(), e))?;
+
+ let snapshots = client
+ .snapshot()
+ .list()
+ .await
+ .map_err(|e| Error::MithrilClient(self.chain, url.clone(), e))?;
+
+ // Check we have a snapshot, and its for our network.
+ match snapshots.first() {
+ Some(snapshot) => {
+ let _aggregator_network =
+ Network::from_str(&snapshot.beacon.network).map_err(|_err| {
+ Error::MithrilClientNetworkMismatch(
+ self.chain,
+ snapshot.beacon.network.clone(),
+ )
+ })?;
+ },
+ None => return Err(Error::MithrilClientNoSnapshots(self.chain, url)),
+ }
+
+ Ok(())
+ }
+
+ /// Validate the mithril sync configuration is correct.
+ pub(crate) async fn validate(&self) -> Result<()> {
+ // Validate the path exists and is a directory, and is writable.
+ self.validate_path().await?;
+ // Validate the genesis vkey is valid.
+ self.validate_genesis_vkey()?;
+ // Validate the Aggregator is valid and responsive.
+ self.validate_aggregator_url().await?;
+
+ Ok(())
+ }
+
+ /// Run a Mithril Follower for the given network and configuration.
+ pub(crate) async fn run(&self) -> Result> {
+ debug!(
+ chain = self.chain.to_string(),
+ "Mithril Auto-update : Starting"
+ );
+
+ // Start the Mithril Sync - IFF its not already running.
+ let lock_entry = match SYNC_JOIN_HANDLE_MAP.get(&self.chain) {
+ None => {
+ error!("Join Map improperly initialized: Missing {}!!", self.chain);
+ return Err(Error::Internal); // Should not get here.
+ },
+ Some(entry) => entry,
+ };
+ let mut locked_handle = lock_entry.value().lock().await;
+
+ if (*locked_handle).is_some() {
+ debug!("Mithril Already Running for {}", self.chain);
+ return Err(Error::MithrilSnapshotSyncAlreadyRunning(self.chain));
+ }
+
+ self.validate().await?;
+
+ // Create a Queue we use to signal the Live Blockchain Follower that the Mithril Snapshot
+ // TIP has changed.
+ // Given how long even the smallest blockchains take to download, a queue depth of 2 is
+ // plenty.
+ let (tx, rx) = mpsc::channel::(2);
+
+ // let handle = tokio::spawn(background_mithril_update(chain, self.clone(), tx));
+ *locked_handle = Some(tokio::spawn(background_mithril_update(self.clone(), tx)));
+
+ // sync_map.insert(chain, handle);
+ debug!(
+ chain = self.chain.to_string(),
+ "Mithril Auto-update : Started"
+ );
+
+ Ok(rx)
+ }
+}
+
+/// Check that a given mithril snapshot path and everything in it is writable.
+/// We don't care why its NOT writable, just that it is either all writable, or not.
+/// Will return false on the first detection of a read only file or directory.
+fn check_writable(path: &Path) -> bool {
+ // Check the permissions of the current path
+ if let Ok(metadata) = path.metadata() {
+ if metadata.permissions().readonly() {
+ return false;
+ }
+ }
+
+ // Can't read the directory for any reason, so can't write to the directory.
+ let path_iterator = match path.read_dir() {
+ Err(_) => return false,
+ Ok(entries) => entries,
+ };
+
+ // Recursively check the contents of the directory
+ for entry in path_iterator {
+ let Ok(entry) = entry else { return false };
+
+ // If the entry is a directory, recursively check its permissions
+ // otherwise just check we could re-write it.
+ if let Ok(metadata) = entry.metadata() {
+ if metadata.is_dir() {
+ // This can NOT be combined with the `if` above.
+ // Doing so will cause the `else` to run on non-writable directories.
+ // Which is wrong.
+ if !check_writable(&entry.path()) {
+ return false;
+ }
+ } else {
+ // If its not a directory then it must be a file.
+ if metadata.permissions().readonly() {
+ return false;
+ }
+ }
+ } else {
+ // Can't identify the file type, so we can't dedup it.
+ return false;
+ }
+ }
+ // Otherwise we could write everything we scanned.
+ true
+}
+
+/// Remove whitespace from a string and return the new string
+fn remove_whitespace(s: &str) -> String {
+ s.chars()
+ .filter(|&c| !c.is_ascii_whitespace())
+ .collect::()
+}
+
+/// Check if a string is an even number of hex digits.
+fn is_hex(s: &str) -> bool {
+ s.chars().count() % 2 == 0 && s.chars().all(|c| c.is_ascii_hexdigit())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn test_default_for() {
+ let network = Network::Preprod;
+ let config = MithrilSnapshotConfig::default_for(network);
+
+ assert_eq!(config.chain, network);
+ assert_eq!(config.path, network.default_mithril_path());
+ assert_eq!(config.aggregator_url, network.default_mithril_aggregator());
+ assert_eq!(config.genesis_key, network.default_mithril_genesis_key());
+ }
+
+ #[tokio::test]
+ async fn test_validate_genesis_vkey() {
+ let config = MithrilSnapshotConfig {
+ chain: Network::Preprod,
+ path: PathBuf::new(),
+ aggregator_url: String::new(),
+ genesis_key: "1234abcd".to_string(),
+ dl_config: None,
+ };
+
+ assert!(config.validate_genesis_vkey().is_ok());
+
+ let invalid_config = MithrilSnapshotConfig {
+ chain: Network::Preprod,
+ path: PathBuf::new(),
+ aggregator_url: String::new(),
+ genesis_key: "1234abcz".to_string(),
+ dl_config: None,
+ };
+
+ assert!(invalid_config.validate_genesis_vkey().is_err());
+ }
+}
diff --git a/rust/cardano-chain-follower/src/mithril_snapshot_data.rs b/rust/cardano-chain-follower/src/mithril_snapshot_data.rs
new file mode 100644
index 00000000000..8c3f146ac86
--- /dev/null
+++ b/rust/cardano-chain-follower/src/mithril_snapshot_data.rs
@@ -0,0 +1,62 @@
+//! Data about the current Mithril Snapshot
+use std::{default, sync::LazyLock};
+
+use dashmap::DashMap;
+
+use crate::{network::Network, snapshot_id::SnapshotId};
+
+/// Current Mithril Snapshot Data for a network.
+#[derive(Debug, Clone)]
+pub(crate) struct SnapshotData {
+ /// Snapshot ID the data represents
+ id: SnapshotId,
+}
+
+impl SnapshotData {
+ /// Create a new Snapshot Data.
+ pub(crate) fn new(id: SnapshotId) -> Self {
+ SnapshotData { id }
+ }
+
+ /// Get the snapshot ID of this Snapshot Data.
+ pub(crate) fn id(&self) -> &SnapshotId {
+ &self.id
+ }
+}
+
+impl default::Default for SnapshotData {
+ /// The default snapshot data represents there is no latest snapshot.
+ fn default() -> Self {
+ SnapshotData {
+ id: SnapshotId::default(),
+ }
+ }
+}
+
+/// Current Mithril Snapshot for a network.
+static CURRENT_MITHRIL_SNAPSHOT: LazyLock> =
+ LazyLock::new(DashMap::new);
+
+/// Get the current latest snapshot data we have recorded.
+pub(crate) fn latest_mithril_snapshot_data(chain: Network) -> SnapshotData {
+ // There should ALWAYS be a snapshot for the chain if this is called.
+
+ match CURRENT_MITHRIL_SNAPSHOT.get(&chain) {
+ Some(snapshot_data) => snapshot_data.value().clone(),
+ None => SnapshotData::default(),
+ }
+}
+
+/// Get the latest Mithril Snapshot for a network.
+pub(crate) fn latest_mithril_snapshot_id(chain: Network) -> SnapshotId {
+ // There should ALWAYS be a snapshot for the chain if this is called.
+ latest_mithril_snapshot_data(chain).id
+}
+
+/// Update the latest snapshot data.
+pub(crate) fn update_latest_mithril_snapshot(chain: Network, snapshot_id: SnapshotId) {
+ let snapshot_data = SnapshotData::new(snapshot_id);
+
+ // Save the current snapshot
+ CURRENT_MITHRIL_SNAPSHOT.insert(chain, snapshot_data);
+}
diff --git a/rust/cardano-chain-follower/src/mithril_snapshot_iterator.rs b/rust/cardano-chain-follower/src/mithril_snapshot_iterator.rs
new file mode 100644
index 00000000000..7c23210dc87
--- /dev/null
+++ b/rust/cardano-chain-follower/src/mithril_snapshot_iterator.rs
@@ -0,0 +1,262 @@
+//! Internal Mithril snapshot iterator functions.
+
+use std::{
+ fmt::Debug,
+ path::Path,
+ sync::{Arc, Mutex},
+};
+
+use logcall::logcall;
+use tokio::task;
+use tracing::{debug, error};
+use tracing_log::log;
+
+use crate::{
+ error::{Error, Result},
+ mithril_query::{make_mithril_iterator, ImmutableBlockIterator},
+ network::Network,
+ point::ORIGIN_POINT,
+ MultiEraBlock, Point,
+};
+
+/// Search backwards by 60 slots (seconds) looking for a previous block.
+/// This search window is doubled until the search succeeds.
+const BACKWARD_SEARCH_SLOT_INTERVAL: u64 = 60;
+
+/// Synchronous Inner Iterator state
+struct MithrilSnapshotIteratorInner {
+ /// The chain being iterated
+ chain: Network,
+ /// Where we really want to start iterating from
+ start: Point,
+ /// Previous iteration point.
+ previous: Point,
+ /// Inner iterator.
+ inner: ImmutableBlockIterator,
+}
+
+impl Debug for MithrilSnapshotIteratorInner {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "MithrilSnapshotIteratorInner {{ chain: {:?}, start: {:?}, previous: {:?} }}",
+ self.chain, self.start, self.previous
+ )
+ }
+}
+
+/// Wraps the iterator type returned by Pallas.
+#[derive(Debug)]
+pub(crate) struct MithrilSnapshotIterator {
+ /// Inner Mutable Synchronous Iterator State
+ inner: Arc>,
+}
+
+/// Create a probe point used in iterations to find the start when its not exactly known.
+pub(crate) fn probe_point(point: &Point, distance: u64) -> Point {
+ // Now that we have the tip, step back about 4 block intervals from tip, and do a fuzzy
+ // iteration to find the exact two blocks at the end of the immutable chain.
+ let step_back_search = point.slot_or_default().saturating_sub(distance);
+
+ // We stepped back to the origin, so just return Origin
+ if step_back_search == 0 {
+ return ORIGIN_POINT;
+ }
+
+ // Create a fuzzy search probe by making the hash zero length.
+ Point::fuzzy(step_back_search)
+}
+
+impl MithrilSnapshotIterator {
+ /// Try and probe to establish the iterator from the desired point.
+ async fn try_fuzzy_iterator(
+ chain: Network, path: &Path, from: &Point, search_interval: u64,
+ ) -> Option {
+ let point = probe_point(from, search_interval);
+ let Ok(mut iterator) = make_mithril_iterator(path, &point).await else {
+ return None;
+ };
+
+ let mut previous = None;
+ let mut this = None;
+
+ loop {
+ let next = iterator.next();
+
+ match next {
+ Some(Ok(raw_block)) => {
+ let Ok(block) = pallas::ledger::traverse::MultiEraBlock::decode(&raw_block)
+ else {
+ return None;
+ };
+
+ let point = Point::new(block.slot(), block.hash().to_vec());
+ previous = this;
+ this = Some(point.clone());
+
+ debug!("Searching for {from}. {this:?} > {previous:?}");
+
+ // Stop as soon as we find the point, or exceed it.
+ if point >= *from {
+ break;
+ }
+ },
+ Some(Err(err)) => {
+ error!("Error while iterating fuzzy mithril data: {}", err);
+ return None;
+ },
+ None => break,
+ };
+ }
+
+ debug!("Best Found for {from}. {this:?} > {previous:?}");
+
+ // Fail if we didn't find the destination block, or its immediate predecessor.
+ previous.as_ref()?;
+ let this = this?;
+
+ // Remake the iterator, based on the new known point.
+ let Ok(iterator) = make_mithril_iterator(path, &this).await else {
+ return None;
+ };
+
+ Some(MithrilSnapshotIterator {
+ inner: Arc::new(Mutex::new(MithrilSnapshotIteratorInner {
+ chain,
+ start: this,
+ previous: previous?,
+ inner: iterator,
+ })),
+ })
+ }
+
+ /// Do a fuzzy search to establish the iterator.
+ /// We use this when we don't know the previous point, and need to find it.
+ #[allow(clippy::indexing_slicing)]
+ #[logcall("debug")]
+ async fn fuzzy_iterator(chain: Network, path: &Path, from: &Point) -> MithrilSnapshotIterator {
+ let mut backwards_search = BACKWARD_SEARCH_SLOT_INTERVAL;
+ loop {
+ if let Some(iterator) =
+ Self::try_fuzzy_iterator(chain, path, from, backwards_search).await
+ {
+ return iterator;
+ }
+
+ backwards_search *= 2;
+ }
+ }
+
+ /// Create a mithril iterator, optionally where we know the previous point.
+ ///
+ /// # Arguments
+ ///
+ /// `chain`: The blockchain network to iterate.
+ /// `from`: The point to start iterating from. If the `Point` does not contain a
+ /// hash, the iteration start is fuzzy. `previous`: The previous point we are
+ /// iterating, if known. If the previous is NOT known, then the first block
+ /// yielded by the iterator is discarded and becomes the known previous.
+ #[allow(clippy::indexing_slicing)]
+ #[logcall(ok = "debug", err = "error")]
+ pub(crate) async fn new(
+ chain: Network, path: &Path, from: &Point, previous_point: Option,
+ ) -> Result {
+ if from.is_fuzzy() || (!from.is_origin() && previous_point.is_none()) {
+ return Ok(Self::fuzzy_iterator(chain, path, from).await);
+ }
+
+ let previous = if from.is_origin() {
+ ORIGIN_POINT
+ } else {
+ let Some(previous) = previous_point else {
+ return Err(Error::Internal);
+ };
+ previous
+ };
+
+ debug!("Actual Mithril Iterator Start: {}", from);
+
+ let iterator = make_mithril_iterator(path, from).await?;
+
+ Ok(MithrilSnapshotIterator {
+ inner: Arc::new(Mutex::new(MithrilSnapshotIteratorInner {
+ chain,
+ start: from.clone(),
+ previous,
+ inner: iterator,
+ })),
+ })
+ }
+
+ /// Get the next block, in a way that is Async friendly.
+ /// Returns the next block, or None if there are no more blocks.
+ pub(crate) async fn next(&self) -> Option {
+ let inner = self.inner.clone();
+
+ let res = task::spawn_blocking(move || {
+ #[allow(clippy::unwrap_used)] // Unwrap is safe here because the lock can't be poisoned.
+ let mut inner_iterator = inner.lock().unwrap();
+ inner_iterator.next()
+ })
+ .await;
+
+ match res {
+ Ok(res) => res,
+ Err(_error) => None,
+ }
+ }
+}
+
+impl Iterator for MithrilSnapshotIteratorInner {
+ type Item = MultiEraBlock;
+
+ fn next(&mut self) -> Option {
+ for maybe_block in self.inner.by_ref() {
+ if let Ok(block) = maybe_block {
+ if !self.previous.is_unknown() {
+ // We can safely fully decode this block.
+ match MultiEraBlock::new(self.chain, block, &self.previous, 0) {
+ Ok(block_data) => {
+ // Update the previous point
+ // debug!("Pre Previous update 1 : {:?}", self.previous);
+ self.previous = block_data.point();
+ // debug!("Post Previous update 1 : {:?}", self.previous);
+
+ // Make sure we got to the start, otherwise this could be a block
+ // artifact from a discover previous point
+ // search.
+ if block_data < self.start {
+ continue;
+ }
+
+ return Some(block_data);
+ },
+ Err(error) => {
+ error!(previous=%self.previous, error=%error, "Error decoding a block from the snapshot");
+ break;
+ },
+ }
+ }
+
+ // We cannot fully decode this block because we don't know its previous point,
+ // So this MUST be the first block in iteration, so use it as the previous.
+ if let Ok(raw_decoded_block) =
+ pallas::ledger::traverse::MultiEraBlock::decode(&block)
+ {
+ // debug!("Pre Previous update 2 : {:?}", self.previous);
+ self.previous =
+ Point::new(raw_decoded_block.slot(), raw_decoded_block.hash().to_vec());
+ // debug!("Post Previous update 2 : {:?}", self.previous);
+ continue;
+ }
+ error!("Error decoding block to use for previous from the snapshot.");
+ break;
+ }
+
+ error!("Error while fetching a block from the snapshot");
+ break;
+ }
+
+ None
+ }
+}
diff --git a/rust/cardano-chain-follower/src/mithril_snapshot_sync.rs b/rust/cardano-chain-follower/src/mithril_snapshot_sync.rs
new file mode 100644
index 00000000000..76dfd061a2a
--- /dev/null
+++ b/rust/cardano-chain-follower/src/mithril_snapshot_sync.rs
@@ -0,0 +1,812 @@
+//! Internal Mithril snapshot downloader task.
+//!
+//! This task is responsible for downloading Mithril snapshot files. It downloads the
+//! latest snapshot file and then sleeps until the next snapshot is available.
+use std::{
+ path::{Path, PathBuf},
+ sync::Arc,
+};
+
+use chrono::{TimeDelta, Utc};
+use dashmap::DashSet;
+use humantime::format_duration;
+use logcall::logcall;
+use mithril_client::{Client, MessageBuilder, MithrilCertificate, Snapshot, SnapshotListItem};
+use tokio::{
+ fs::remove_dir_all,
+ sync::mpsc::Sender,
+ time::{sleep, Duration},
+};
+use tracing::{debug, error};
+use tracing_log::log;
+
+use crate::{
+ error::{Error, Result},
+ mithril_query::get_mithril_tip_point,
+ mithril_snapshot_config::{MithrilSnapshotConfig, MithrilUpdateMessage},
+ mithril_snapshot_data::update_latest_mithril_snapshot,
+ mithril_snapshot_iterator::MithrilSnapshotIterator,
+ mithril_turbo_downloader::MithrilTurboDownloader,
+ network::Network,
+ snapshot_id::SnapshotId,
+ stats::{self, mithril_sync_failure, mithril_validation_state},
+ MultiEraBlock,
+};
+
+/// The minimum duration between checks for a new Mithril Snapshot. (Must be same as
+/// `MINIMUM_MITHRIL_UPDATE_CHECK_DURATION`)
+const MINIMUM_MITHRIL_UPDATE_CHECK_INTERVAL: TimeDelta = TimeDelta::minutes(10); // 10 Minutes
+/// The minimum duration between checks for a new Mithril Snapshot. (Must be same as
+/// `MINIMUM_MITHRIL_UPDATE_CHECK_INTERVAL`)
+const MINIMUM_MITHRIL_UPDATE_CHECK_DURATION: Duration = Duration::from_secs(10 * 60); // 10 Minutes
+/// Average Mithril Update is 6 Hrs, so don't wait longer than 7.
+const MAXIMUM_MITHRIL_UPDATE_CHECK_INTERVAL: TimeDelta = TimeDelta::hours(7); // 7 Hours
+/// Average Mithril Update is 6 Hrs, so don't wait longer than 7.
+const EXPECTED_MITHRIL_UPDATE_CHECK_INTERVAL: TimeDelta = TimeDelta::hours(6); // 6 Hours
+/// We shouldn't get errors that need to wait for this, but if we do wait this long.
+/// These errors should be transient if they occur.
+const DOWNLOAD_ERROR_RETRY_DURATION: Duration = Duration::from_secs(2 * 60); // 2 Minutes
+
+/// Returns the Latest and chronologically previous snapshots data from the Aggregator.
+/// Will return None if it can not get the Snapshot list, or there are no entries in it.
+/// If there is only a single entry then the latest and chronologically next will be
+/// identical.
+async fn get_latest_snapshots(
+ client: &Client, network: Network,
+) -> Option<(SnapshotListItem, SnapshotListItem)> {
+ // Get current latest snapshot from the aggregator
+ let snapshots = match client.snapshot().list().await {
+ Ok(s) => s,
+ Err(e) => {
+ error!("Unexpected Error [{}]: Unable to get Snapshot List from Aggregator for {}. Mithril Snapshots can not update. Sleeping.", network, e);
+ return None;
+ },
+ };
+
+ // Get the current latest snapshot.
+ let Some(latest_snapshot) = snapshots.first() else {
+ error!("Unexpected Error: Empty Snapshot List from Aggregator for {}. Mithril Snapshots can not update. Sleeping", network);
+ return None;
+ };
+
+ let chronologically_previous = snapshots.get(1).unwrap_or(latest_snapshot);
+
+ Some((latest_snapshot.clone(), chronologically_previous.clone()))
+}
+
+/// Given a particular snapshot ID, find the Actual Snapshot for it.
+async fn get_snapshot_by_id(
+ client: &Client, network: Network, snapshot_id: &SnapshotId,
+) -> Option {
+ let snapshots = match client.snapshot().list().await {
+ Ok(s) => s,
+ Err(e) => {
+ error!("Unexpected Error [{}]: Unable to get Snapshot List from Aggregator for {}. Mithril Snapshots can not update. Sleeping.", network, e);
+ return None;
+ },
+ };
+
+ // Try and find the current snapshot in the list of available snapshots.
+ for snapshot in snapshots {
+ // debug!("Checking Snapshot : {:?}", snapshot);
+ if *snapshot_id == snapshot.beacon.immutable_file_number {
+ return Some(snapshot);
+ }
+ }
+
+ None
+}
+
+/// Create a client, should never fail, but return None if it does, because we can't
+/// continue.
+fn create_client(cfg: &MithrilSnapshotConfig) -> Option<(Client, Arc)> {
+ let downloader = Arc::new(MithrilTurboDownloader::new(cfg.clone()));
+
+ // This can't fail, because we already tested it works. But just in case...
+ let client = match mithril_client::ClientBuilder::aggregator(
+ &cfg.aggregator_url,
+ &cfg.genesis_key,
+ )
+ //.add_feedback_receiver(receiver)
+ .with_snapshot_downloader(downloader.clone())
+ .build()
+ {
+ Ok(c) => c,
+ Err(err) => {
+ error!(chain=cfg.chain.to_string(),"Unexpected Error [{}]: Unable to create Mithril Client. Mithril Snapshots can not update.", err);
+ return None;
+ },
+ };
+
+ Some((client, downloader))
+}
+
+/// Calculate how long we should wait before we check for another Mithril snapshot.
+fn calculate_sleep_duration(
+ latest_snapshot: &SnapshotListItem, previous_snapshot: &SnapshotListItem,
+) -> Duration {
+ // All times are relative to UTC.
+ let now = Utc::now();
+ let mut next_sleep = MINIMUM_MITHRIL_UPDATE_CHECK_INTERVAL;
+
+ // How long between snapshots,
+ let mut snapshot_interval = (latest_snapshot.created_at - previous_snapshot.created_at)
+ .max(MAXIMUM_MITHRIL_UPDATE_CHECK_INTERVAL);
+
+ // We should never be negative, but we CAN be zero if there was no chronologically
+ // previous snapshot. In this case GUESS how long the interval should be based on
+ // experience.
+ if snapshot_interval <= TimeDelta::seconds(0) {
+ snapshot_interval = EXPECTED_MITHRIL_UPDATE_CHECK_INTERVAL;
+ }
+
+ let next_expected_snapshot = latest_snapshot.created_at + snapshot_interval;
+
+ if next_expected_snapshot > now {
+ // We are behind schedule. Sleep until the next expected snapshot should be published.
+ next_sleep = next_expected_snapshot - now;
+ }
+
+ next_sleep
+ .to_std()
+ .unwrap_or(MINIMUM_MITHRIL_UPDATE_CHECK_DURATION)
+}
+
+/// Get the actual snapshot from the specified `snapshot_item` from the list of snapshots.
+/// Returns None if there are any issues doing this, otherwise the Snapshot.
+/// The only issues should be transient communications errors.
+async fn get_snapshot(
+ client: &Client, snapshot_item: &SnapshotListItem, network: Network,
+) -> Option {
+ let latest_digest = snapshot_item.digest.as_ref();
+ let snapshot = match client.snapshot().get(latest_digest).await {
+ Ok(snapshot) => {
+ if let Some(snapshot) = snapshot {
+ snapshot
+ } else {
+ // Some kind of communications error has ocurred.
+ error!("No snapshot returned for {} ???", network);
+ return None;
+ }
+ },
+ Err(err) => {
+ // Some kind of communications error has ocurred.
+ error!(
+ "Failure to get the latest snapshot for {} with error: {}",
+ network, err
+ );
+ return None;
+ },
+ };
+
+ Some(snapshot)
+}
+
+/// Download and Verify the Snapshots certificate
+async fn download_and_verify_snapshot_certificate(
+ client: &Client, snapshot: &Snapshot, network: Network,
+) -> Option {
+ let certificate = match client
+ .certificate()
+ .verify_chain(&snapshot.certificate_hash)
+ .await
+ {
+ Ok(certificate) => certificate,
+ Err(err) => {
+ // The certificate is invalid.
+ error!("The certificate for {} is invalid: {}", network, err);
+ return None;
+ },
+ };
+
+ Some(certificate)
+}
+
+/// This function creates a client based on the given network and genesis vkey.
+///
+/// # Arguments
+///
+/// * `network` - The network type for the client to connect to.
+/// * `aggregator_url` - A reference to the URL of an aggregator that can be used to
+/// create the client.
+/// * `genesis_vkey` - The genesis verification key, which is needed to authenticate with
+/// the server.
+///
+/// # Returns
+///
+/// This function returns a `Client` object if it successfully connects to the specified
+/// URL and creates a client. If it fails, it waits for `DOWNLOAD_ERROR_RETRY_DURATION`
+/// before attempting again. This never times out, as we can not attempt this if the
+/// aggregator was not contactable when the parameters were defined.
+async fn connect_client(cfg: &MithrilSnapshotConfig) -> (Client, Arc) {
+ // Note: we pre-validated connection before we ran, so failure here should be transient.
+ // Just wait if we fail, and try again later.
+ loop {
+ if let Some(client) = create_client(cfg) {
+ return client;
+ }
+
+ // If we couldn't create a client, then we don' t need to do anything.
+ // Error already logged in create_client, no need to print anything here.
+ sleep(DOWNLOAD_ERROR_RETRY_DURATION).await;
+ }
+}
+
+/// Relative Directory for Immutable data within a full mithril snapshot.
+pub(crate) const MITHRIL_IMMUTABLE_SUB_DIRECTORY: &str = "immutable";
+
+/// Get the tip block from the Immutable chain, and the block immediately proceeding it.
+///
+/// # Arguments
+///
+/// * `path` - The path where the immutable chain is stored.
+///
+/// # Returns
+///
+/// This function returns the tip block point, and the block point immediately proceeding
+/// it in a tuple.
+#[allow(clippy::indexing_slicing)]
+#[logcall(ok = "debug", err = "error")]
+pub(crate) async fn get_mithril_tip(chain: Network, path: &Path) -> Result {
+ let mut path = path.to_path_buf();
+ path.push(MITHRIL_IMMUTABLE_SUB_DIRECTORY);
+
+ debug!(
+ "Calculating TIP from Immutable storage @ {}",
+ path.to_string_lossy()
+ );
+
+ // Read the Tip (fuzzy), and if we don't get one, or we error, its an error.
+ // has to be Fuzzy, because we intend to iterate and don't know the previous.
+ // Nor is there a subsequent block we can use as next.
+ let tip = get_mithril_tip_point(&path).await?.as_fuzzy();
+ debug!("Mithril Tip: {tip}");
+
+ // Decode and read the tip from the Immutable chain.
+ let tip_iterator = MithrilSnapshotIterator::new(chain, &path, &tip, None).await?;
+ let Some(tip_block) = tip_iterator.next().await else {
+ error!("Failed to fetch the TIP block from the immutable chain.");
+
+ return Err(Error::MithrilSnapshot(None));
+ };
+
+ // Yay, we got a tip, so return it.
+ Ok(tip_block)
+}
+
+/// Get the Snapshot Data itself from the Aggregator, and a validate Certificate.
+async fn get_mithril_snapshot_and_certificate(
+ chain: Network, client: &Client, item: &SnapshotListItem,
+) -> Option<(Snapshot, MithrilCertificate)> {
+ debug!("Mithril Snapshot background updater for: {chain} : Download snapshot from aggregator.");
+
+ // Download the snapshot from the aggregator.
+ let Some(snapshot) = get_snapshot(client, item, chain).await else {
+ // If we couldn't get the snapshot then we don't need to do anything else, transient
+ // error.
+ return None;
+ };
+
+ debug!("Mithril Snapshot background updater for: {chain} : Download/Verify certificate.");
+
+ // Download and Verify the certificate.
+ let certificate = download_and_verify_snapshot_certificate(client, &snapshot, chain).await?;
+
+ Some((snapshot, certificate))
+}
+
+/// Validate that a Mithril Snapshot downloaded matches its certificate.
+async fn validate_mithril_snapshot(
+ chain: Network, certificate: &MithrilCertificate, path: &Path,
+) -> bool {
+ let cert = certificate.clone();
+ let mithril_path = path.to_path_buf();
+ match tokio::spawn(async move {
+ // This can be long running and CPU Intensive.
+ // So we spawn it off to a background task.
+ MessageBuilder::new()
+ .compute_snapshot_message(&cert, &mithril_path)
+ .await
+ })
+ .await
+ {
+ Ok(Ok(result)) => {
+ if certificate.match_message(&result) {
+ true
+ } else {
+ // If we couldn't match then assume its a transient error.
+ error!("Failed to Match Certificate and Computed Snapshot Message for {chain}!");
+ false
+ }
+ },
+ Ok(Err(error)) => {
+ // If we got an error then it must be false.
+ error!("Failed to Compute Snapshot Message: {error}");
+ false
+ },
+ Err(error) => {
+ error!("Snapshot Certificate computation failed: {error}");
+ false
+ },
+ }
+}
+
+/// See if we have a latest snapshot already, and if so, validate it.
+///
+/// For a existing mithril snapshot to be valid it has to be:
+/// 1. The actual latest mithril snapshot; AND
+/// 2. It must
+async fn get_latest_validated_mithril_snapshot(
+ chain: Network, client: &Client, cfg: &MithrilSnapshotConfig,
+) -> Option {
+ /// Purge a bad mithril snapshot from disk.
+ async fn purge_bad_mithril_snapshot(chain: Network, latest_mithril: &SnapshotId) {
+ debug!("Purging Bad Mithril Snapshot: {latest_mithril}");
+ if let Err(error) = remove_dir_all(&latest_mithril).await {
+ // This should NOT happen because we already checked the Mithril path is fully writable.
+ error!("Mithril Snapshot background updater for: {chain}: Failed to remove old snapshot {latest_mithril}: {error}");
+ }
+ }
+
+ // Check if we already have a Mithril snapshot downloaded, and IF we do validate it is
+ // intact.
+ let latest_mithril = cfg.recover_latest_snapshot_id().await?;
+
+ debug!("Latest Recovered Mithril ID = {latest_mithril}");
+
+ // Get the actual latest snapshot, shouldn't fail, but say the current is invalid if it
+ // does.
+ let (actual_latest, _) = get_latest_snapshots(client, chain).await?;
+
+ // IF the mithril data we have is NOT the current latest (or the immediately previous), it
+ // may as well be invalid.
+ if latest_mithril < actual_latest.beacon.immutable_file_number - 1 {
+ return None;
+ }
+
+ let Some(snapshot) = get_snapshot_by_id(client, chain, &latest_mithril).await else {
+ // We have a latest snapshot, but the Aggregator does not know it.
+ error!("Mithril Snapshot background updater for: {chain}: Latest snapshot {latest_mithril} does not exist on the Aggregator.");
+ purge_bad_mithril_snapshot(chain, &latest_mithril).await;
+ return None;
+ };
+
+ // Download the snapshot/certificate from the aggregator.
+ let Some((_, certificate)) =
+ get_mithril_snapshot_and_certificate(chain, client, &snapshot).await
+ else {
+ error!("Mithril Snapshot : Failed to get Snapshot and certificate (Transient Error).");
+
+ // If we couldn't get the snapshot then we don't need to do anything else, transient
+ // error.
+ // purge_bad_mithril_snapshot(chain, &latest_mithril).await;
+ return None;
+ };
+
+ let path = latest_mithril.as_ref();
+ let valid = validate_mithril_snapshot(chain, &certificate, path).await;
+
+ if !valid {
+ error!("Mithril Snapshot : Snapshot fails to validate, can not be recovered.");
+ purge_bad_mithril_snapshot(chain, &latest_mithril).await;
+ return None;
+ }
+
+ Some(latest_mithril)
+}
+
+/// Get the Mithril client and recover our existing mithril snapshot data, if any.
+async fn recover_existing_snapshot(
+ cfg: &MithrilSnapshotConfig, tx: &Sender,
+) -> Option {
+ // This is a Mithril Validation, so record it.
+ mithril_validation_state(cfg.chain, stats::MithrilValidationState::Start);
+
+ // Note: we pre-validated connection before we ran, so failure here should be transient.
+ // Just wait if we fail, and try again later.
+ let (client, downloader) = connect_client(cfg).await;
+
+ debug!(
+ "Mithril Snapshot background updater for: {} : Client connected.",
+ cfg.chain
+ );
+
+ let mut current_snapshot = None;
+
+ // Check if we already have a Mithril snapshot downloaded, and IF we do validate it is
+ // intact.
+ if let Some(active_snapshot) =
+ get_latest_validated_mithril_snapshot(cfg.chain, &client, cfg).await
+ {
+ // Read the actual TIP block from the Mithril chain.
+ match get_mithril_tip(cfg.chain, &active_snapshot.path()).await {
+ Ok(tip_block) => {
+ // Validate the Snapshot ID matches the true TIP.
+ if active_snapshot.tip() == tip_block.point() {
+ current_snapshot = Some(active_snapshot.clone());
+ update_latest_mithril_snapshot(cfg.chain, active_snapshot);
+
+ // Tell the live sync service the current Mithril TIP.
+ let update = MithrilUpdateMessage {
+ tip: tip_block.point(),
+ previous: tip_block.previous(),
+ };
+ if let Err(error) = tx.send(update).await {
+ error!(
+ "Failed to send new tip to the live updater for: {}: {error}",
+ cfg.chain
+ );
+ };
+ } else {
+ error!(
+ "Actual Tip Block and Active SnapshotID Point Mismatch. {:?} != {:?}",
+ active_snapshot.tip(),
+ tip_block.point()
+ );
+ }
+ },
+ Err(error) => {
+ error!("Mithril snapshot validation failed for: {}. Could not read the TIP Block : {}.", cfg.chain, error);
+ },
+ }
+ } else {
+ debug!("No latest validated snapshot for: {}", cfg.chain);
+ }
+
+ if current_snapshot.is_none() {
+ mithril_validation_state(cfg.chain, stats::MithrilValidationState::Failed);
+ } else {
+ mithril_validation_state(cfg.chain, stats::MithrilValidationState::Finish);
+ }
+
+ // Explicitly free the resources claimed by the Mithril Client and Downloader.
+ drop(client);
+ drop(downloader);
+
+ current_snapshot
+}
+
+/// Status of checking if we have a new snapshot to get or not.
+enum SnapshotStatus {
+ /// No update, sleep for this long before checking again
+ Sleep(Duration),
+ /// Snapshot has updated, here are the details.
+ Updated((Snapshot, MithrilCertificate)),
+}
+
+/// Check if we have a new snapshot to download, and if so, return its details.
+async fn check_snapshot_to_download(
+ chain: Network, client: &Client, current_snapshot: &Option,
+) -> SnapshotStatus {
+ debug!("Mithril Snapshot background updater for: {chain} : Getting Latest Snapshot.");
+
+ // This should only fail if the Aggregator is offline.
+ // Because we check we can talk to the aggregator before we create the downloader task.
+ let Some((latest_snapshot, chronologically_previous_snapshot)) =
+ get_latest_snapshots(client, chain).await
+ else {
+ return SnapshotStatus::Sleep(DOWNLOAD_ERROR_RETRY_DURATION);
+ };
+
+ debug!("Mithril Snapshot background updater for: {chain} : Checking if we are up-to-date {current_snapshot:?}.");
+
+ // Check if the latest snapshot is different from our actual previous one.
+ if let Some(current_mithril_snapshot) = ¤t_snapshot {
+ let latest_immutable_file_number = latest_snapshot.beacon.immutable_file_number;
+ debug!("We have a current snapshot: {current_mithril_snapshot} == {latest_immutable_file_number} ??");
+ if *current_mithril_snapshot == latest_immutable_file_number {
+ debug!("Current Snapshot and latest are the same, so wait for it to likely to have changed.");
+ let next_sleep =
+ calculate_sleep_duration(&latest_snapshot, &chronologically_previous_snapshot);
+ return SnapshotStatus::Sleep(next_sleep);
+ }
+ }
+
+ // Download the snapshot/certificate from the aggregator.
+ let Some((snapshot, certificate)) =
+ get_mithril_snapshot_and_certificate(chain, client, &latest_snapshot).await
+ else {
+ // If we couldn't get the snapshot then we don't need to do anything else, transient
+ // error.
+ debug!("Failed to retrieve the snapshot and certificate from aggregator.");
+ return SnapshotStatus::Sleep(DOWNLOAD_ERROR_RETRY_DURATION);
+ };
+
+ SnapshotStatus::Updated((snapshot, certificate))
+}
+
+/// Start Mithril Validation in the background, and return a handle so we can check when
+/// it finishes.
+fn background_validate_mithril_snapshot(
+ chain: Network, certificate: MithrilCertificate, tmp_path: PathBuf,
+) -> tokio::task::JoinHandle {
+ tokio::spawn(async move {
+ debug!(
+ "Mithril Snapshot background updater for: {} : Check Certificate.",
+ chain
+ );
+
+ stats::mithril_validation_state(chain, stats::MithrilValidationState::Start);
+
+ if !validate_mithril_snapshot(chain, &certificate, &tmp_path).await {
+ stats::mithril_validation_state(chain, stats::MithrilValidationState::Failed);
+ // If we couldn't build the message then assume its a transient error.
+ error!(
+ chain = %chain,
+ "Failed to Compute Snapshot Message"
+ );
+ return false;
+ }
+ stats::mithril_validation_state(chain, stats::MithrilValidationState::Finish);
+
+ debug!(
+ "Mithril Snapshot background updater for: {} : Certificate Validated OK.",
+ chain
+ );
+
+ true
+ })
+}
+
+/// Convert a chunk filename into its numeric equivalent.
+fn chunk_filename_to_chunk_number(chunk: &Path) -> Option {
+ if let Some(stem) = chunk.file_stem().map(Path::new) {
+ if let Some(base) = stem.file_name().map(|s| s.to_string_lossy().to_string()) {
+ if let Ok(num) = base.parse::() {
+ return Some(num);
+ }
+ }
+ }
+ None
+}
+
+/// Remove any chunks from the chunk list which exceed the `max_chunk`.
+fn trim_chunk_list(chunk_list: &Arc>, max_chunks: u64) {
+ chunk_list.retain(|entry| {
+ if let Some(chunk_index) = chunk_filename_to_chunk_number(entry) {
+ if chunk_index > max_chunks {
+ debug!("Removing Non immutable Chunk: {:?}", entry);
+ false
+ } else {
+ true
+ }
+ } else {
+ // Huh, not a valid filename, so purge it.
+ error!("Found an invalid chunk name: {:?}", entry);
+ false
+ }
+ });
+}
+
+/// Downloads and validates a snapshot from the aggregator.
+async fn download_and_validate_snapshot(
+ client: &Client, downloader: Arc, cfg: &MithrilSnapshotConfig,
+ snapshot: &Snapshot, certificate: MithrilCertificate,
+) -> bool {
+ debug!(
+ "Mithril Snapshot background updater for: {} : Download and unpack the Mithril snapshot.",
+ cfg.chain
+ );
+
+ // Download and unpack the actual snapshot archive.
+ if let Err(error) = client
+ .snapshot()
+ .download_unpack(snapshot, &cfg.tmp_path())
+ .await
+ {
+ // If we couldn't download and unpack, assume its a transient error,
+ error!("Failed to Download and Unpack snapshot: {error}");
+ return false;
+ }
+
+ debug!(
+ "Mithril Snapshot background updater for: {} : Add statistics for download.",
+ cfg.chain
+ );
+
+ if let Err(error) = client.snapshot().add_statistics(snapshot).await {
+ // Just log not fatal to anything.
+ error!(
+ "Could not increment snapshot download statistics for {}: {error}",
+ cfg.chain
+ );
+ // We can process the download even after this fails.
+ }
+
+ debug!(
+ "Mithril Snapshot background updater for: {} : Index and Check Certificate.",
+ cfg.chain
+ );
+
+ let chunk_list = downloader.get_new_chunks();
+ // Remove the last chunks from the list, if they are > the max_chunk thats immutable.
+ let max_chunk = snapshot.beacon.immutable_file_number;
+ trim_chunk_list(&chunk_list, max_chunk);
+
+ let validate_handle =
+ background_validate_mithril_snapshot(cfg.chain, certificate, cfg.tmp_path());
+
+ if !validate_handle.await.unwrap_or(false) {
+ error!("Failed to validate for {}", cfg.chain);
+ return false;
+ }
+
+ true
+}
+
+/// We can accumulate junk depending on errors or when we terminate, make sure we are
+/// always clean.
+async fn cleanup(cfg: &MithrilSnapshotConfig) {
+ if let Err(error) = cfg.cleanup().await {
+ error!(
+ "Mithril Snapshot background updater for: {} : Error cleaning up: {:?}",
+ cfg.chain, error
+ );
+ }
+}
+
+/// Sleep until its likely there has been another mithril snapshot update.
+async fn sleep_until_next_probable_update(
+ cfg: &MithrilSnapshotConfig, next_sleep: &Duration,
+) -> Duration {
+ debug!(
+ "Mithril Snapshot background updater for: {} : Sleeping for {}.",
+ cfg.chain,
+ format_duration(*next_sleep)
+ );
+ // Wait until its likely we have a new snapshot ready to download.
+ sleep(*next_sleep).await;
+
+ // Default sleep if we end up back at the top of this loop because of an error.
+ DOWNLOAD_ERROR_RETRY_DURATION
+}
+
+/// Cleanup the client explicitly and do a new iteration of the loop.
+macro_rules! next_iteration {
+ ($client:ident, $downloader:ident) => {
+ drop($client);
+ drop($downloader);
+
+ continue;
+ };
+}
+
+/// Handle the background downloading of Mithril snapshots for a given network.
+/// Note: There can ONLY be at most three of these running at any one time.
+/// This is because there can ONLY be one snapshot for each of the three known Cardano
+/// networks.
+/// # Arguments
+///
+/// * `network` - The network type for the client to connect to.
+/// * `aggregator_url` - A reference to the URL of an aggregator that can be used to
+/// create the client.
+/// * `genesis_vkey` - The genesis verification key, which is needed to authenticate with
+/// the server.
+///
+/// # Returns
+///
+/// This does not return, it is a background task.
+#[allow(clippy::too_many_lines)]
+pub(crate) async fn background_mithril_update(
+ cfg: MithrilSnapshotConfig, tx: Sender,
+) {
+ debug!(
+ "Mithril Snapshot background updater for: {} from {} to {} : Starting",
+ cfg.chain,
+ cfg.aggregator_url,
+ cfg.path.to_string_lossy()
+ );
+ let mut next_sleep = Duration::from_secs(0);
+
+ let mut current_snapshot = recover_existing_snapshot(&cfg, &tx).await;
+
+ loop {
+ debug!("Background Mithril Updater - New Loop");
+
+ cleanup(&cfg).await;
+
+ next_sleep = sleep_until_next_probable_update(&cfg, &next_sleep).await;
+
+ let (client, downloader) = connect_client(&cfg).await;
+
+ let (snapshot, certificate) =
+ match check_snapshot_to_download(cfg.chain, &client, ¤t_snapshot).await {
+ SnapshotStatus::Sleep(sleep) => {
+ next_sleep = sleep;
+ next_iteration!(client, downloader);
+ },
+ SnapshotStatus::Updated(update) => update,
+ };
+
+ if !download_and_validate_snapshot(
+ &client,
+ downloader.clone(),
+ &cfg,
+ &snapshot,
+ certificate,
+ )
+ .await
+ {
+ error!("Failed to Download or Validate a snapshot.");
+ mithril_sync_failure(cfg.chain, stats::MithrilSyncFailures::DownloadOrValidation);
+
+ next_iteration!(client, downloader);
+ }
+
+ // Download was A-OK - Update the new immutable tip.
+ let tip = match get_mithril_tip(cfg.chain, &cfg.tmp_path()).await {
+ Ok(tip) => tip,
+ Err(error) => {
+ // If we couldn't get the tip then assume its a transient error.
+ error!(
+ "Failed to Get Tip from Snapshot for {}: {error}",
+ cfg.chain
+ );
+ mithril_sync_failure(cfg.chain, stats::MithrilSyncFailures::FailedToGetTip);
+
+ next_iteration!(client, downloader);
+ },
+ };
+
+ debug!("New Immutable TIP = {}", tip);
+
+ // Check that the new tip is more advanced than the OLD tip.
+ if let Some(active_snapshot) = current_snapshot.clone() {
+ if tip <= active_snapshot.tip() {
+ error!(
+ "New Tip is not more advanced than the old tip for: {}",
+ cfg.chain
+ );
+ mithril_sync_failure(cfg.chain, stats::MithrilSyncFailures::TipDidNotAdvance);
+ next_iteration!(client, downloader);
+ }
+ }
+
+ // Got a good new tip, so switch to the new mithril image.
+ match cfg.activate(snapshot.beacon.immutable_file_number).await {
+ Ok(new_path) => {
+ debug!(
+ "Mithril Snapshot background updater for: {} : Updated TIP.",
+ cfg.chain
+ );
+ current_snapshot = SnapshotId::new(&new_path, tip.point());
+
+ if let Some(latest_snapshot) = current_snapshot.clone() {
+ // Update the latest snapshot data record
+ update_latest_mithril_snapshot(cfg.chain, latest_snapshot);
+
+ // Tell the live updater that the Immutable TIP has updated.
+ if let Err(error) = tx
+ .send(MithrilUpdateMessage {
+ tip: tip.point(),
+ previous: tip.previous(),
+ })
+ .await
+ {
+ error!(
+ "Failed to send new tip to the live updater for: {}: {error}",
+ cfg.chain
+ );
+ mithril_sync_failure(
+ cfg.chain,
+ stats::MithrilSyncFailures::TipFailedToSendToUpdater,
+ );
+ next_iteration!(client, downloader);
+ };
+ }
+ },
+ Err(err) => {
+ error!(
+ chain = cfg.chain.to_string(),
+ "Failed to activate new snapshot : {err}"
+ );
+ mithril_sync_failure(
+ cfg.chain,
+ stats::MithrilSyncFailures::FailedToActivateNewSnapshot,
+ );
+ next_iteration!(client, downloader);
+ },
+ }
+ next_iteration!(client, downloader);
+ }
+}
diff --git a/rust/cardano-chain-follower/src/mithril_turbo_downloader.rs b/rust/cardano-chain-follower/src/mithril_turbo_downloader.rs
new file mode 100644
index 00000000000..de0d030e58d
--- /dev/null
+++ b/rust/cardano-chain-follower/src/mithril_turbo_downloader.rs
@@ -0,0 +1,384 @@
+//! Turbo Downloads for Mithril Snapshots.
+
+use std::{
+ cmp,
+ ffi::OsStr,
+ io::{BufReader, Read},
+ path::{Path, PathBuf},
+ // process::Stdio,
+ sync::{
+ atomic::{AtomicU64, Ordering},
+ Arc, OnceLock,
+ },
+};
+
+use anyhow::{anyhow, bail};
+use async_trait::async_trait;
+use dashmap::DashSet;
+use fmmap::MmapFileExt;
+use memx::memcmp;
+use mithril_client::{
+ common::CompressionAlgorithm, snapshot_downloader::SnapshotDownloader, MithrilResult,
+};
+use tar::{Archive, EntryType};
+use tokio::{fs::create_dir_all, task::spawn_blocking};
+use tracing::{debug, error};
+use zstd::Decoder;
+
+use crate::{
+ mithril_snapshot_config::MithrilSnapshotConfig,
+ mithril_snapshot_data::latest_mithril_snapshot_data,
+ stats::{self},
+ turbo_downloader::ParallelDownloadProcessor,
+ utils::usize_from_saturating,
+};
+
+/// A snapshot downloader that accelerates Download using `aria2`.
+pub struct Inner {
+ /// Configuration for the snapshot sync.
+ cfg: MithrilSnapshotConfig,
+ /// Last hashmap/list of changed chunks from the previous download
+ new_chunks: Arc>,
+
+ /// The number of files that were new in this download.
+ new_files: AtomicU64,
+ /// The number of files that changed in this download.
+ chg_files: AtomicU64,
+ /// The total number of files in the download.
+ tot_files: AtomicU64,
+ /// The total size of the files extracted in the download.
+ ext_size: AtomicU64,
+ /// The total size of the files we deduplicated.
+ dedup_size: AtomicU64,
+
+ /// The download processor for the current file download.
+ dl_handler: std::sync::OnceLock,
+}
+
+/// This macro is what happens every time the file is different from previous.
+macro_rules! changed_file {
+ ($self:ident, $rel_file:ident, $abs_file:ident, $new_size:ident) => {
+ $self.chg_files.fetch_add(1, Ordering::SeqCst);
+ if $abs_file.extension() == Some(OsStr::new("chunk")) {
+ $self.new_chunks.insert($abs_file);
+ }
+ };
+}
+
+/// This macro is what happens every time we decide the file can't be deduplicated.
+macro_rules! new_file {
+ ($self:ident, $rel_file:ident, $abs_file:ident, $new_size:ident) => {
+ $self.new_files.fetch_add(1, Ordering::SeqCst);
+ if $abs_file.extension() == Some(OsStr::new("chunk")) {
+ $self.new_chunks.insert($abs_file);
+ }
+ };
+}
+
+impl Inner {
+ /// Synchronous Download and Dedup archive.
+ ///
+ /// Stream Downloads and Decompresses files, and deduplicates them as they are
+ /// extracted from the embedded tar archive.
+ ///
+ /// Per Entry:
+ /// If the file is NOT to be deduplicated, OR A previous file with the same name and
+ /// size does not exist, then just extract it where its supposed to go.
+ ///
+ /// To Dedup, the original file is mam-mapped.
+ /// The new file is extracted to an in-memory buffer.
+ /// If they compare the same, the original file is `HardLinked` to the new file name.
+ /// Otherwise the new file buffer is saved to disk with the new file name.
+ fn dl_and_dedup(&self, _location: &str, _target_dir: &Path) -> MithrilResult<()> {
+ let mut archive = self.create_archive_extractor()?;
+
+ // Iterate the files in the archive.
+ let entries = match archive.entries() {
+ Ok(entries) => entries,
+ Err(error) => bail!("Failed to get entries from the archive: {error}"),
+ };
+
+ let tmp_dir = self.cfg.tmp_path();
+ let latest_snapshot = latest_mithril_snapshot_data(self.cfg.chain);
+
+ for entry in entries {
+ let mut entry = match entry {
+ Ok(entry) => entry,
+ Err(error) => bail!("Failed to get an entry from the archive: {error}"),
+ };
+ let rel_file = entry.path()?.to_path_buf();
+ let entry_size = entry.size();
+
+ // debug!(chain = %self.cfg.chain, "DeDup : Extracting {}:{} loc {location} target {}",
+ // rel_file.to_string_lossy(), entry_size, target_dir.to_string_lossy());
+
+ // Check if we need to extract this path or not.
+ if !self.check_for_extract(&rel_file, entry.header().entry_type()) {
+ continue;
+ }
+
+ // Count total files processed.
+ self.tot_files.fetch_add(1, Ordering::SeqCst);
+
+ let mut abs_file = tmp_dir.clone();
+ abs_file.push(rel_file.clone());
+
+ let mut prev_file = latest_snapshot.id().path_if_exists();
+ if let Some(prev_file) = &mut prev_file {
+ prev_file.push(rel_file.clone());
+ }
+
+ // debug!(chain = %self.cfg.chain, "DeDup : tmp_dir {} abs_file {} prev_file
+ // {prev_file:?}", tmp_dir.to_string_lossy(), abs_file.to_string_lossy() );
+
+ self.ext_size.fetch_add(entry_size, Ordering::SeqCst);
+
+ // Try and deduplicate the file if we can, otherwise just extract it.
+ if let Ok((prev_mmap, _)) = Self::can_deduplicate(&rel_file, entry_size, &prev_file) {
+ let expected_file_size = usize_from_saturating(entry_size);
+ let mut buf: Vec = Vec::with_capacity(expected_file_size);
+ if entry.read_to_end(&mut buf)? != expected_file_size {
+ bail!(
+ "Failed to read file {} of size {} got {}",
+ rel_file.display(),
+ entry_size,
+ buf.len()
+ );
+ }
+ // Got the full file and its the expected size. Is it different?
+ if memcmp(prev_mmap.as_slice(), buf.as_slice()) == cmp::Ordering::Equal {
+ // Same so lets Hardlink it, and throw away the temp buffer.
+
+ // Make sure our big mmap get dropped.
+ drop(prev_mmap);
+
+ // File is the same, so dedup it.
+ if self.cfg.dedup_tmp(&abs_file, &latest_snapshot).is_ok() {
+ self.dedup_size.fetch_add(entry_size, Ordering::SeqCst);
+ changed_file!(self, rel_file, abs_file, entry_size);
+ drop(buf);
+ continue;
+ }
+ }
+
+ if let Err(error) = std::fs::write(&abs_file, buf) {
+ error!(chain = %self.cfg.chain, "Failed to write file {} got {}", abs_file.display(), error);
+ bail!("Failed to write file {} got {}", abs_file.display(), error);
+ }
+ } else {
+ // No dedup, just extract it into the tmp directory as-is.
+ entry.unpack_in(&tmp_dir)?;
+ debug!(chain = %self.cfg.chain, "DeDup: Extracted file {rel_file:?}:{entry_size}");
+ }
+ new_file!(self, rel_file, abs_file, entry_size);
+ }
+
+ let Some(dl_handler) = self.dl_handler.get() else {
+ bail!("Failed to get the Parallel Download processor!");
+ };
+
+ debug!(chain = %self.cfg.chain, "Download {} bytes", dl_handler.dl_size());
+
+ stats::mithril_dl_finished(self.cfg.chain, Some(dl_handler.dl_size()));
+
+ Ok(())
+ }
+
+ /// Create a TAR archive extractor from the downloading file and a zstd decompressor.
+ fn create_archive_extractor(
+ &self,
+ ) -> MithrilResult>>>>
+ {
+ let Some(dl_handler) = self.dl_handler.get() else {
+ bail!("Failed to get the Parallel Download processor!");
+ };
+ let buf_reader = BufReader::new(dl_handler.clone());
+ let decoder = match zstd::Decoder::new(buf_reader) {
+ Ok(decoder) => decoder,
+ Err(error) => bail!("Failed to create ZSTD decoder: {error}"),
+ };
+ Ok(tar::Archive::new(decoder))
+ }
+
+ /// Check if we are supposed to extract this file from the archive or not.
+ fn check_for_extract(&self, path: &Path, extract_type: EntryType) -> bool {
+ if path.is_absolute() {
+ error!(chain = %self.cfg.chain, "DeDup : Cannot extract an absolute path: {:?}", path);
+ return false;
+ }
+
+ if extract_type.is_dir() {
+ // We don't do anything with just a path, so skip it.
+ return false;
+ }
+
+ if !extract_type.is_file() {
+ error!(chain = %self.cfg.chain, "DeDup : Cannot extract a non-file: {:?}:{:?}", path, extract_type);
+ return false;
+ }
+
+ true
+ }
+
+ /// Check if a given path from the archive is able to be deduplicated.
+ fn can_deduplicate(
+ rel_file: &Path, file_size: u64, prev_file: &Option,
+ ) -> MithrilResult<(fmmap::MmapFile, u64)> {
+ // Can't dedup if the current file is not de-dupable (must be immutable)
+ if rel_file.starts_with("immutable") {
+ // Can't dedup if we don't have a previous file to dedup against.
+ if let Some(prev_file) = prev_file {
+ if let Some(current_size) = get_file_size_sync(prev_file) {
+ // If the current file is not exactly the same as the previous file size, we
+ // can't dedup.
+ if file_size == current_size {
+ if let Ok(pref_file_loaded) = mmap_open_sync(prev_file) {
+ if pref_file_loaded.1 == file_size {
+ return Ok(pref_file_loaded);
+ }
+ }
+ }
+ }
+ }
+ }
+ bail!("Can not deduplicate.");
+ }
+}
+
+/// A snapshot downloader that accelerates Download using `aria2`.
+pub struct MithrilTurboDownloader {
+ /// inner arc wrapped configuration
+ inner: Arc,
+}
+
+impl MithrilTurboDownloader {
+ /// Constructs a new `HttpSnapshotDownloader`.
+ pub fn new(cfg: MithrilSnapshotConfig) -> Self {
+ // Test if the HTTP Client can properly be created.
+ let dl_config = cfg.dl_config.clone().unwrap_or_default();
+
+ let cfg = cfg.with_dl_config(dl_config);
+
+ Self {
+ inner: Arc::new(Inner {
+ cfg,
+ new_chunks: Arc::new(DashSet::new()),
+ new_files: AtomicU64::new(0),
+ chg_files: AtomicU64::new(0),
+ tot_files: AtomicU64::new(0),
+ ext_size: AtomicU64::new(0),
+ dedup_size: AtomicU64::new(0),
+ dl_handler: OnceLock::new(),
+ }),
+ }
+ }
+
+ /// Take the hashmap for the previous download.
+ pub fn get_new_chunks(&self) -> Arc> {
+ self.inner.new_chunks.clone()
+ }
+
+ /// Create directories required to exist for download to succeed.
+ async fn create_directories(&self, target_dir: &Path) -> MithrilResult<()> {
+ if let Err(error) = create_dir_all(target_dir).await {
+ let msg = format!(
+ "Target directory {} could not be created: {}",
+ target_dir.to_string_lossy(),
+ error
+ );
+ Err(anyhow!(msg.clone()).context(msg))?;
+ }
+
+ Ok(())
+ }
+
+ /// Parallel Download, Extract and Dedup the Mithril Archive.
+ async fn dl_and_dedup(&self, location: &str, target_dir: &Path) -> MithrilResult<()> {
+ // Get a copy of the inner data to use in the sync download task.
+ let inner = self.inner.clone();
+ let location = location.to_owned();
+ let target_dir = target_dir.to_owned();
+
+ // This is fully synchronous IO, so do it on a sync thread.
+ let result = spawn_blocking(move || inner.dl_and_dedup(&location, &target_dir)).await;
+
+ if let Ok(result) = result {
+ return result;
+ }
+
+ stats::mithril_dl_finished(self.inner.cfg.chain, None);
+ bail!("Download and Dedup task failed");
+ }
+}
+
+/// Get the size of a particular file. None = failed to get size (doesn't matter why).
+fn get_file_size_sync(file: &Path) -> Option {
+ let Ok(metadata) = file.metadata() else {
+ return None;
+ };
+ Some(metadata.len())
+}
+
+/// Open a file using mmap for performance.
+fn mmap_open_sync(path: &Path) -> MithrilResult<(fmmap::MmapFile, u64)> {
+ match fmmap::MmapFile::open_with_options(path, fmmap::Options::new().read(true).populate()) {
+ Ok(file) => {
+ let len = file.len() as u64;
+ Ok((file, len))
+ },
+ Err(error) => {
+ error!(error=%error, file=%path.to_string_lossy(), "Failed to open file");
+ Err(error.into())
+ },
+ }
+}
+
+#[async_trait]
+impl SnapshotDownloader for MithrilTurboDownloader {
+ async fn download_unpack(
+ &self, location: &str, target_dir: &Path, _compression_algorithm: CompressionAlgorithm,
+ _download_id: &str, _snapshot_size: u64,
+ ) -> MithrilResult<()> {
+ self.create_directories(target_dir).await?;
+
+ // DL Start stats set after DL actually started inside the probe call.
+ self.dl_and_dedup(location, target_dir).await?;
+
+ let tot_files = self.inner.tot_files.load(Ordering::SeqCst);
+ let chg_files = self.inner.chg_files.load(Ordering::SeqCst);
+ let new_files = self.inner.new_files.load(Ordering::SeqCst);
+
+ stats::mithril_extract_finished(
+ self.inner.cfg.chain,
+ Some(self.inner.ext_size.load(Ordering::SeqCst)),
+ self.inner.dedup_size.load(Ordering::SeqCst),
+ tot_files - (chg_files + new_files),
+ chg_files,
+ new_files,
+ );
+
+ debug!("Download and Unpack finished='{location}' to '{target_dir:?}'.");
+
+ Ok(())
+ }
+
+ async fn probe(&self, location: &str) -> MithrilResult<()> {
+ debug!("Probe Snapshot location='{location}'.");
+
+ let dl_config = self.inner.cfg.dl_config.clone().unwrap_or_default();
+ let dl_processor = ParallelDownloadProcessor::new(location, dl_config).await?;
+
+ // Decompress and extract and de-dupe each file in the archive.
+ stats::mithril_extract_started(self.inner.cfg.chain);
+
+ // We also immediately start downloading now.
+ stats::mithril_dl_started(self.inner.cfg.chain);
+
+ // Save the DownloadProcessor in the inner struct for use to process the downloaded data.
+ if let Err(_error) = self.inner.dl_handler.set(dl_processor) {
+ bail!("Failed to set the inner dl_handler. Must already be set?");
+ }
+
+ Ok(())
+ }
+}
diff --git a/rust/cardano-chain-follower/src/multi_era_block_data.rs b/rust/cardano-chain-follower/src/multi_era_block_data.rs
new file mode 100644
index 00000000000..c8c21ea410a
--- /dev/null
+++ b/rust/cardano-chain-follower/src/multi_era_block_data.rs
@@ -0,0 +1,693 @@
+//! Multi Era CBOR Encoded Block Data
+//!
+//! Data about how the block/transactions can be encoded is found here:
+//!
+//!
+//! DO NOT USE the documentation/cddl definitions from the head of this repo because it
+//! currently lacks most of the documentation needed to understand the format and is also
+//! incorrectly generated and contains errors that will be difficult to discern.
+
+use std::{cmp::Ordering, fmt::Display, sync::Arc};
+
+use ouroboros::self_referencing;
+use tracing::debug;
+
+use crate::{
+ error::Error, metadata, stats::stats_invalid_block, witness::TxWitness, Network, Point,
+};
+
+/// Self-referencing CBOR encoded data of a multi-era block.
+/// Note: The fields in the original struct can not be accessed directly
+/// The builder creates accessor methods which are called
+/// `borrow_raw_data()` and `borrow_block()`
+#[self_referencing]
+#[derive(Debug)]
+pub(crate) struct SelfReferencedMultiEraBlock {
+ /// The CBOR encoded data of a multi-era block.
+ raw_data: Vec,
+
+ /// The decoded multi-era block.
+ /// References the `raw_data` field.
+ #[borrows(raw_data)]
+ #[covariant]
+ block: pallas::ledger::traverse::MultiEraBlock<'this>,
+}
+
+/// Multi-era block - inner.
+#[derive(Debug)]
+pub struct MultiEraBlockInner {
+ /// What blockchain was the block produced on.
+ //#[allow(dead_code)]
+ pub chain: Network,
+ /// The Point on the blockchain this block can be found.
+ point: Point,
+ /// The previous point on the blockchain before this block.
+ /// When the current point is Genesis, so is the previous.
+ previous: Point,
+ /// The decoded multi-era block.
+ data: SelfReferencedMultiEraBlock,
+ /// Decoded Metadata in the transactions in the block.
+ metadata: metadata::DecodedTransaction,
+ /// A map of public key hashes to the public key and transaction numbers they are in.
+ #[allow(dead_code)]
+ witness_map: Option,
+}
+
+/// Multi-era block.
+#[derive(Clone, Debug)]
+pub struct MultiEraBlock {
+ /// What fork is the block on?
+ /// This is NOT part of the inner block, because it is not to be protected by the Arc.
+ /// It can change at any time due to rollbacks detected on the live-chain.
+ /// This means that any holder of a `MultiEraBlock` will have the actual fork their
+ /// block was on when they read it, the live-chain code can modify the actual fork
+ /// count at any time without that impacting consumers processing the data.
+ /// The fork count itself is used so an asynchronous follower can properly work out
+ /// how far to roll back on the live-chain in order to resynchronize, without
+ /// keeping a full state of processed blocks.
+ /// Followers, simply need to step backwards on the live chain until they find the
+ /// previous block they followed, or reach a fork that is <= the fork of the
+ /// previous block they followed. They can then safely re-follow from that earlier
+ /// point, with full integrity. fork is 0 on any immutable block.
+ /// It starts at 1 for live blocks, and is only incremented if the live-chain tip is
+ /// purged because of a detected fork based on data received from the peer node.
+ /// It does NOT count the strict number of forks reported by the peer node.
+ fork: u64,
+ /// The Immutable decoded data about the block itself.
+ inner: Arc,
+}
+
+impl MultiEraBlock {
+ /// Creates a new `MultiEraBlockData` from the given bytes.
+ ///
+ /// # Errors
+ ///
+ /// If the given bytes cannot be decoded as a multi-era block, an error is returned.
+ fn new_block(
+ chain: Network, raw_data: Vec, previous: &Point, fork: u64,
+ ) -> anyhow::Result {
+ let builder = SelfReferencedMultiEraBlockTryBuilder {
+ raw_data,
+ block_builder: |raw_data| -> Result<_, Error> {
+ pallas::ledger::traverse::MultiEraBlock::decode(raw_data)
+ .map_err(|err| Error::Codec(err.to_string()))
+ },
+ };
+ let self_ref_block = builder.try_build()?;
+ let decoded_block = self_ref_block.borrow_block();
+
+ let witness_map = TxWitness::new(&decoded_block.txs()).ok();
+
+ let slot = decoded_block.slot();
+
+ let point = Point::new(slot, decoded_block.hash().to_vec());
+
+ let byron_block = matches!(
+ decoded_block,
+ pallas::ledger::traverse::MultiEraBlock::Byron(_)
+ );
+
+ // debug!("New Block: {slot} {point} {}", *previous);
+
+ // Dump the early mainnet blocks because somethings funny in there.
+ // if slot == 0 || slot == 21600 {
+ // debug!("Block of interest {slot} {:?}", decoded_block);
+ //}
+
+ // Validate that the Block point is valid.
+ if !previous.is_origin() {
+ // Every 21600 Blocks, Byron Era has duplicated sequential slot#'s.
+ // So this filters them out from the sequential point check.
+ // The Hash chain is still checked.
+ if (!byron_block || ((slot % 21600) != 0)) && *previous >= slot {
+ return Err(Error::Codec(format!(
+ "Previous slot is not less than current slot:{slot}"
+ )));
+ }
+
+ // Special case, when the previous block is actually UNKNOWN, we can't check it.
+ if !previous.is_unknown()
+ // Otherwise, we make sure the hash chain is intact
+ && !previous.cmp_hash(&decoded_block.header().previous_hash())
+ {
+ debug!("{}, {:?}", previous, decoded_block.header().previous_hash());
+
+ return Err(Error::Codec(
+ "Previous Block Hash mismatch with block".to_string(),
+ ));
+ }
+ }
+
+ let metadata = metadata::DecodedTransaction::new(chain, decoded_block);
+
+ Ok(Self {
+ fork,
+ inner: Arc::new(MultiEraBlockInner {
+ chain,
+ point,
+ previous: previous.clone(),
+ data: self_ref_block,
+ metadata,
+ witness_map,
+ }),
+ })
+ }
+
+ /// Creates a new `MultiEraBlockData` from the given bytes.
+ ///
+ /// # Errors
+ ///
+ /// If the given bytes cannot be decoded as a multi-era block, an error is returned.
+ pub fn new(
+ chain: Network, raw_data: Vec, previous: &Point, fork: u64,
+ ) -> anyhow::Result {
+ // This lets us reliably count any bad block arising from deserialization.
+ let block = MultiEraBlock::new_block(chain, raw_data, previous, fork);
+ if block.is_err() {
+ stats_invalid_block(chain, fork == 0);
+ }
+ block
+ }
+
+ /// Remake the block on a new fork.
+ pub fn set_fork(&mut self, fork: u64) {
+ self.fork = fork;
+ }
+
+ /// Decodes the data into a multi-era block.
+ #[must_use]
+ #[allow(clippy::missing_panics_doc)]
+ pub fn decode(&self) -> &pallas::ledger::traverse::MultiEraBlock {
+ self.inner.data.borrow_block()
+ }
+
+ /// Decodes the data into a multi-era block.
+ #[must_use]
+ #[allow(clippy::missing_panics_doc)]
+ pub fn raw(&self) -> &Vec {
+ self.inner.data.borrow_raw_data()
+ }
+
+ /// Returns the block point of this block.
+ #[must_use]
+ pub fn point(&self) -> Point {
+ self.inner.point.clone()
+ }
+
+ /// Returns the block point of the previous block.
+ #[must_use]
+ pub fn previous(&self) -> Point {
+ self.inner.previous.clone()
+ }
+
+ /// Is the block data immutable on-chain.
+ #[must_use]
+ pub fn immutable(&self) -> bool {
+ self.fork == 0
+ }
+
+ /// Is the block data immutable on-chain.
+ #[must_use]
+ pub fn fork(&self) -> u64 {
+ self.fork
+ }
+
+ /// What chain was the block from
+ #[must_use]
+ pub fn chain(&self) -> Network {
+ self.inner.chain
+ }
+
+ /// Get The Decoded Metadata fora a transaction and known label from the block
+ #[must_use]
+ pub fn txn_metadata(
+ &self, txn_idx: usize, label: u64,
+ ) -> Option> {
+ self.inner.metadata.get_metadata(txn_idx, label)
+ }
+
+ /// Get The Raw Metadata fora a transaction and known label from the block
+ #[must_use]
+ pub fn txn_raw_metadata(&self, txn_idx: usize, label: u64) -> Option