Skip to content

Commit

Permalink
Change BlockScanner to return a list of Blocks instead of a list of T…
Browse files Browse the repository at this point in the history 
…ransactions
  • Loading branch information
@Alenar
Alenar committed Apr 16, 2024
1 parent 994349d commit 1193057
Show file tree
Hide file tree
Showing 3 changed files with 190 additions and 133 deletions.
92 changes: 53 additions & 39 deletions mithril-aggregator/src/services/cardano_transactions_importer.rs
View file
Expand Up @@ -79,7 +79,13 @@ impl CardanoTransactionsImporter {
return Ok(());
}

let parsed_transactions = self.block_scanner.parse(&self.dirpath, from, until).await?;
// todo: temp algorithm, should be optimized to avoid loading all blocks & transactions
// at once in memory (probably using iterators)
let scanned_blocks = self.block_scanner.scan(&self.dirpath, from, until).await?;
let parsed_transactions: Vec<CardanoTransaction> = scanned_blocks
.into_iter()
.flat_map(|b| b.into_transactions())
.collect();
debug!(
self.logger,
"TransactionsImporter retrieved '{}' Cardano transactions between immutables '{}' and '{until}'",
Expand Down Expand Up @@ -114,6 +120,8 @@ mod tests {
use mockall::mock;
use mockall::predicate::eq;

use mithril_common::cardano_block_scanner::ScannedBlock;

use crate::database::repository::CardanoTransactionRepository;
use crate::database::test_helper::cardano_tx_db_connection;

Expand All @@ -124,19 +132,23 @@ mod tests {

#[async_trait]
impl BlockScanner for BlockScannerImpl {
async fn parse(
async fn scan(
&self,
dirpath: &Path,
from_immutable: Option<ImmutableFileNumber>,
until_immutable: ImmutableFileNumber,
) -> StdResult<Vec<CardanoTransaction>>;
) -> StdResult<Vec<ScannedBlock>>;
}
}

fn build_importer(
scanner_mock_config: &dyn Fn(&mut MockBlockScannerImpl),
store_mock_config: &dyn Fn(&mut MockTransactionStore),
) -> CardanoTransactionsImporter {
fn build_importer<TParser, TStore>(
scanner_mock_config: TParser,
store_mock_config: TStore,
) -> CardanoTransactionsImporter
where
TParser: FnOnce(&mut MockBlockScannerImpl),
TStore: FnOnce(&mut MockTransactionStore),
{
let db_path = Path::new("");
let mut scanner = MockBlockScannerImpl::new();
scanner_mock_config(&mut scanner);
Expand All @@ -156,23 +168,24 @@ mod tests {

#[tokio::test]
async fn if_nothing_stored_parse_and_store_all_transactions() {
let transactions = vec![
CardanoTransaction::new("tx_hash-1", 10, 15, "block_hash-1", 11),
CardanoTransaction::new("tx_hash-2", 10, 20, "block_hash-1", 11),
CardanoTransaction::new("tx_hash-3", 20, 25, "block_hash-2", 12),
CardanoTransaction::new("tx_hash-4", 20, 30, "block_hash-2", 12),
let blocks = vec![
ScannedBlock::new("block_hash-1", 10, 15, 11, vec!["tx_hash-1", "tx_hash-2"]),
ScannedBlock::new("block_hash-2", 20, 25, 12, vec!["tx_hash-3", "tx_hash-4"]),
];
let transactions: Vec<CardanoTransaction> = blocks
.iter()
.flat_map(|b| b.clone().into_transactions())
.collect();
let up_to_beacon = 12;

let importer = build_importer(
&|scanner_mock| {
let parsed_transactions = transactions.clone();
|scanner_mock| {
scanner_mock
.expect_parse()
.expect_scan()
.withf(move |_, from, until| from.is_none() && until == &up_to_beacon)
.return_once(move |_, _, _| Ok(parsed_transactions));
.return_once(move |_, _, _| Ok(blocks));
},
&|store_mock| {
|store_mock| {
let expected_stored_transactions = transactions.clone();
store_mock
.expect_get_highest_beacon()
Expand All @@ -196,10 +209,10 @@ mod tests {
let up_to_beacon = 12;

let importer = build_importer(
&|scanner_mock| {
scanner_mock.expect_parse().never();
|scanner_mock| {
scanner_mock.expect_scan().never();
},
&|store_mock| {
|store_mock| {
store_mock
.expect_get_highest_beacon()
.returning(|| Ok(Some(12)));
Expand All @@ -215,23 +228,25 @@ mod tests {

#[tokio::test]
async fn if_all_half_are_stored_the_other_half_is_parsed_and_stored() {
let transactions = vec![
CardanoTransaction::new("tx_hash-1", 10, 15, "block_hash-10", 11),
CardanoTransaction::new("tx_hash-2", 20, 20, "block_hash-20", 12),
CardanoTransaction::new("tx_hash-3", 30, 25, "block_hash-30", 13),
CardanoTransaction::new("tx_hash-4", 40, 30, "block_hash-40", 14),
let blocks = [
ScannedBlock::new("block_hash-1", 10, 15, 11, vec!["tx_hash-1", "tx_hash-2"]),
ScannedBlock::new("block_hash-2", 20, 25, 12, vec!["tx_hash-3", "tx_hash-4"]),
];
let transactions: Vec<CardanoTransaction> = blocks
.iter()
.flat_map(|b| b.clone().into_transactions())
.collect();
let up_to_beacon = 14;

let importer = build_importer(
&|scanner_mock| {
let parsed_transactions = transactions[2..=3].to_vec();
|scanner_mock| {
let scanned_blocks = vec![blocks[1].clone()];
scanner_mock
.expect_parse()
.expect_scan()
.withf(move |_, from, until| from == &Some(13) && until == &up_to_beacon)
.return_once(move |_, _, _| Ok(parsed_transactions));
.return_once(move |_, _, _| Ok(scanned_blocks));
},
&|store_mock| {
|store_mock| {
store_mock
.expect_get_highest_beacon()
.returning(|| Ok(Some(12)));
Expand All @@ -253,19 +268,18 @@ mod tests {
#[tokio::test]
async fn importing_twice_starting_with_nothing_in_a_real_db_should_yield_the_transactions_in_same_order(
) {
let transactions = vec![
CardanoTransaction::new("tx_hash-1", 10, 15, "block_hash-1", 11),
CardanoTransaction::new("tx_hash-2", 10, 20, "block_hash-1", 11),
CardanoTransaction::new("tx_hash-3", 20, 25, "block_hash-2", 12),
CardanoTransaction::new("tx_hash-4", 20, 30, "block_hash-2", 12),
let blocks = vec![
ScannedBlock::new("block_hash-1", 10, 15, 11, vec!["tx_hash-1", "tx_hash-2"]),
ScannedBlock::new("block_hash-2", 20, 25, 12, vec!["tx_hash-3", "tx_hash-4"]),
];
let transactions: Vec<CardanoTransaction> = blocks
.iter()
.flat_map(|b| b.clone().into_transactions())
.collect();
let importer = {
let connection = cardano_tx_db_connection().unwrap();
let parsed_transactions = transactions.clone();
let mut scanner = MockBlockScannerImpl::new();
scanner
.expect_parse()
.return_once(move |_, _, _| Ok(parsed_transactions));
scanner.expect_scan().return_once(move |_, _, _| Ok(blocks));

CardanoTransactionsImporter::new(
Arc::new(scanner),
Expand Down

0 comments on commit 1193057

Please sign in to comment.